fix dtype kwarg + cut max_new_tokens to 64 for faster CPU inference

app/gradio_app.py

@@ -26,7 +26,7 @@ def _get_assistant():
 def respond(message: str, history, mode: str) -> str:
     assistant = _get_assistant()
 
-    result = assistant.generate(message, mode=mode, max_new_tokens=128, return_sources=True)
+    result = assistant.generate(message, mode=mode, max_new_tokens=64, return_sources=True)
     if isinstance(result, tuple):
         code, sources = result
         src_lines = "\n".join(

src/rag/generator.py

@@ -41,7 +41,7 @@ class CodeAssistant:
         self.top_k = top_k
         self.tok = AutoTokenizer.from_pretrained(gen_model)
         self.model = AutoModelForCausalLM.from_pretrained(
-            gen_model, dtype="auto", device_map=device_map
+            gen_model, torch_dtype="auto", device_map=device_map
         )
 
     # ---- prompt builders ------------------------------------------------