Update services/model_service.py

services/model_service.py

@@ -26,10 +26,25 @@ class ModelService:
     def _load_models(self):
         try:
             self.tokenizer = AutoTokenizer.from_pretrained(settings.MODEL_NAME)
+
+                  # Modify the model configuration to use a valid rope_scaling format
+            config = LlamaConfig.from_pretrained(model_name)
+            if hasattr(config, "rope_scaling") and isinstance(config.rope_scaling, dict):
+                config.rope_scaling = {
+                    "type": "linear",
+                    "factor": config.rope_scaling.get("factor", 32.0)
+                }
+            
+            # Load model with updated configuration
+            #self.model = AutoModelForCausalLM.from_pretrained(model_name, config=config).to(device)
+
+        
+            
             self.model = AutoModelForCausalLM.from_pretrained(
                 settings.MODEL_NAME,
                 torch_dtype=torch.float16 if settings.DEVICE == "cuda" else torch.float32,
-                device_map="auto" if settings.DEVICE == "cuda" else None
+                device_map="auto" if settings.DEVICE == "cuda" else None,
+                config=config
             )
             self.embedder = SentenceTransformer(settings.EMBEDDER_MODEL)
         except Exception as e: