Initial upload of FreeChunk model with custom code

README.md

@@ -32,28 +32,22 @@ pip install torch transformers sentence-transformers numpy
 ### Abstract Usage
 
 ```python
-from sentenizer import Sentenceizer 
-from modeling_freechunker import FreeChunkerModel 
-import torch 
- 
-# 1. Setup Sentenceizer with Backbone 
-sentenceizer = Sentenceizer(model_name="nomic-ai/nomic-embed-text-v1.5") 
- 
-# 2. Load FreeChunker Model 
-model = FreeChunkerModel.from_pretrained("XiaSheng/FreeChunk-nomic", trust_remote_code=True) 
-model.eval() 
- 
-# 3. Process Text 
+from transformers import AutoModel
+import torch
+
+# 1. Load Model (UnifiedEncoder)
+model = AutoModel.from_pretrained("XiaSheng/FreeChunk-nomic", trust_remote_code=True)
+
+# 2. Build Vector Store from Text
 text = "Your text..."
-sentences, embeddings = sentenceizer.split_and_encode(text)
-
-# 4. Forward pass through FreeChunker
-inputs_embeds = torch.tensor(embeddings).unsqueeze(0) # Batch size 1
-with torch.no_grad():
-    outputs = model(inputs_embeds=inputs_embeds)
-    
-# outputs['embedding'] contains refined embeddings
-# outputs['shift_matrix'] contains chunking information
+model.build_vector_store(text)
+
+# 3. Query with Post-Aggregation (Default)
+query = "Your query..."
+results = model.query(query, top_k=1, aggregation_mode='post')
+
+print(f"Query: {query}")
+print(f"Result: {results}")
 ```
 
 ## Files