Initial upload of FreeChunk model with custom code

config.json

@@ -27,6 +27,6 @@
   "max_power": 4,
   "auto_map": {
     "AutoConfig": "configuration_freechunker.FreeChunkerConfig",
-    "AutoModel": "modeling_freechunker.FreeChunkerModel"
+    "AutoModel": "encoder.UnifiedEncoder"
   }
 }

encoder.py

@@ -9,9 +9,10 @@ import numpy as np
 import pickle
 import os
 from typing import List, Tuple, Union
-from sentenizer import Sentenceizer
-from freechunker import FreeChunkerModel
-from aggregator import TextAggregator
+from .sentenizer import Sentenceizer
+from .freechunker import FreeChunkerModel
+from .aggregator import TextAggregator
+from . import utils
 
 class UnifiedEncoder:
     """
@@ -80,6 +81,10 @@ class UnifiedEncoder:
             
         return cls(model_name=model_name, model_name_or_path=model_name_or_path, **kwargs)
 
+    @classmethod
+    def register_for_auto_class(cls, auto_class="AutoModel"):
+        return
+
     def encode(self, text: str, show_progress: bool = True) -> Tuple[List[str], np.ndarray, List[List[str]]]:
         """
         Split text and encode, return results grouped by shift_matrix

freechunker.py

@@ -35,7 +35,7 @@ from transformers.utils import (
     get_torch_version,
     logging
 )
-from configuration import FreeChunkerConfig
+from .configuration_freechunker import FreeChunkerConfig
 
 
 logger = logging.get_logger(__name__)