Update model

config.json

@@ -8,58 +8,8 @@
   },
   "eos_token_id": 8,
   "pad_token_id": 9,
-  "reciprocals": [
-    [
-      4,
-      3
-    ],
-    [
-      5,
-      2
-    ],
-    [
-      6,
-      1
-    ]
-  ],
-  "reducables": [
-    [
-      [
-        4
-      ],
-      [
-        3
-      ]
-    ],
-    [
-      [
-        5
-      ],
-      [
-        2
-      ]
-    ],
-    [
-      [
-        6
-      ],
-      [
-        1
-      ]
-    ],
-    [
-      [
-        4,
-        5,
-        6
-      ],
-      [
-        1,
-        2,
-        3
-      ]
-    ]
-  ],
+  "reciprocals": null,
+  "reducables": null,
   "torch_dtype": "float32",
   "transformers_version": "4.21.1",
   "vocab_size": 10

configuration_greedy.py

@@ -5,6 +5,16 @@ class GreedyConfig(PretrainedConfig):
 
     @classmethod
     def from_tokenizer(cls, freegroup_dimension, tokenizer: PreTrainedTokenizerBase, **kwargs):
+        config = cls(
+            vocab_size = len(tokenizer),
+            eos_token_id = tokenizer.eos_token_id,
+            pad_token_id = tokenizer.pad_token_id,
+            **kwargs
+        )
+        config._from_tokenizer(freegroup_dimension, tokenizer)
+        return config
+
+    def _from_tokenizer(self, freegroup_dimension, tokenizer):
 
         freegroup_generators = list(range(1, freegroup_dimension + 1))
 
@@ -17,15 +27,7 @@ class GreedyConfig(PretrainedConfig):
         for reducable, closure_generator in zip(reducables, [[x] for x in freegroup_generators] + [freegroup_generators[::]]):
             reducable.append(tokenizer.convert_tokens_to_ids(list(map(str, closure_generator))))
             reducable.append(tokenizer.convert_tokens_to_ids(list(map(str, tools.reciprocal(closure_generator)))))
-        
-        return cls(
-            reciprocals = reciprocals,
-            reducables = reducables,
-            vocab_size = len(tokenizer),
-            eos_token_id = tokenizer.eos_token_id,
-            pad_token_id = tokenizer.pad_token_id,
-            **kwargs
-        )
+
 
     def __init__(self, **kwargs):
         # reciporcals: List[List[int]]: i.e. ['x', 'X'], ...