history error temp fix

app.py

@@ -152,7 +152,7 @@ def query_endpoint(req: QueryRequest):
     categories, urgency = classify_text(req.query)
 
     # RAG
-    answer = generate_answer(req.query,history)
+    answer = generate_answer(req.query,"1")
 
     # Update conversation memory
     add_message(req.user_id, "user", req.query)

service/llm_service.py

@@ -6,21 +6,30 @@ class LLMService:
         model_id = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
 
         self.tokenizer = AutoTokenizer.from_pretrained(model_id)
+
         self.model = AutoModelForCausalLM.from_pretrained(
             model_id,
-            torch_dtype=torch.float16,
-            device_map="auto"
+            torch_dtype=torch.float32,   # IMPORTANT
         )
 
-    def generate(self, prompt: str) -> str:
-        inputs = self.tokenizer(prompt, return_tensors="pt").to(self.model.device)
+        self.model.to("cpu")
+        self.model.eval()
 
-        output = self.model.generate(
-            **inputs,
-            max_new_tokens=200,
-            temperature=0.3,
-            top_p=0.9,
-            do_sample=True
+    def generate(self, prompt: str) -> str:
+        inputs = self.tokenizer(
+            prompt,
+            return_tensors="pt",
+            truncation=True,
+            max_length=2048
         )
 
+        with torch.no_grad():
+            output = self.model.generate(
+                **inputs,
+                max_new_tokens=200,
+                temperature=0.3,
+                top_p=0.9,
+                do_sample=True
+            )
+
         return self.tokenizer.decode(output[0], skip_special_tokens=True)