fixed bug, should work with small vocabs

kornai · Sep 20, 2016 · c4c93bb1bd1d41c16e5521a97fc36813fd82c185 · c4c93bb
1 parent 6f4a15e
commit c4c93bb1bd1d41c16e5521a97fc36813fd82c185
Unified Split

Showing with 10,019 additions and 2 deletions.

+3 −2 src/fourlang/context.py

+10,016 −0 test/input/stanford_output_test.txt
diff --git a/src/fourlang/context.py b/src/fourlang/context.py
@@ -67,7 +67,7 @@ def build_ndarray(self):
        edges, subjs, objs = self.coocc
        for i, edge in enumerate(edges):
            subj, obj = subjs[i], objs[i]
-  
-            self.ndarray[i][subj][obj] += 1
+  
+            self.ndarray[edge][subj][obj] += 1
    def get_w_index(self, word):
        if word in self.vocabulary:
@@ -166,10 +166,11 @@ def main():
    cfg = get_cfg(cfg_file)
    context = Context(cfg)
+  
+    # context.build_from_stanford_output(filter_fnc=first_only_filter)
    # context.build_from_stanford_output(filter_fnc=short_only_filter)
    context.build_from_stanford_output()
    context.freeze_vocab()
-  
-    context.build_ndarray()
+  
+    # context.build_ndarray()  # causes MemoryError
    context.save()
    context.print_to_files()