tok class

config.json

@@ -17,6 +17,7 @@
   "num_hidden_layers": 32,
   "rotary_emb_base": 10000,
   "tie_word_embeddings": false,
+  "tokenizer_class": "GeoVTokenizer",
   "torch_dtype": "bfloat16",
   "transformers_version": "4.28.0.dev0",
   "use_cache": true,

configuration_geov.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """ GeoV model configuration"""
-import geov.tokenization_geov
 from transformers.configuration_utils import PretrainedConfig
 from transformers.utils import logging
 
@@ -74,7 +73,6 @@ class GeoVConfig(PretrainedConfig):
     >>> configuration = model.config  # doctest: +SKIP
     ```"""
     model_type = "geov"
-    tokenizer_class = geov.tokenization_geov.GeoVTokenizer
 
     def __init__(
             self,
@@ -91,10 +89,15 @@ class GeoVConfig(PretrainedConfig):
             bos_token_id=0,
             eos_token_id=2,
             tie_word_embeddings=False,
+            tokenizer_class="GeoVTokenizer",
             **kwargs,
     ):
         super().__init__(
-            bos_token_id=bos_token_id, eos_token_id=eos_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            tokenizer_class=tokenizer_class,
+            **kwargs
         )
         self.vocab_size = vocab_size
         self.max_position_embeddings = max_position_embeddings