fix

app.py

@@ -15,16 +15,9 @@ from transformers import (
 import torch.nn.functional as F
 from collections import defaultdict
 HF_TOKEN = os.getenv("HF_token")
-CHAT_MODEL_ID = "meta-llama/Llama-3.1-8B-Instruct"
+CHAT_MODEL_ID = "NousResearch/Meta-Llama-3-8B-Instruct"
 EMB_MODEL_ID  = "mixedbread-ai/mxbai-embed-large-v1"
 MAX_PROMPT_TOKENS = 8192
-import transformers
-pipeline = transformers.pipeline(
-    "text-generation",
-    model=CHAT_MODEL_ID,
-    model_kwargs={"torch_dtype": torch.bfloat16},
-    device_map="auto",
-)
 
 # ---------- new defaults & helper ------------------
 DEFAULT_TEMP        = 0.7
@@ -125,11 +118,16 @@ def build_llm_prompt(system: str, context: list[str], user_question: str) -> str
     # 套用 LLaMA-style prompt 格式
     input_token = tokenizer.apply_chat_template(
         conversation,
-        tokenize=False,
-        add_generation_prompt=False
+        add_generation_prompt=True,
+        return_tensors="pt"
     )
 
-    return tokenizer.decode(input_token)
+    terminators = [
+        tokenizer.eos_token_id,
+        tokenizer.convert_tokens_to_ids("<|eot_id|>")
+    ]
+
+    return input_token, terminators
 
 # ---------- 4. Gradio playground (same UI as before) --------------------------
 def store_doc(doc_text: str,user_id="demo",chunk_size=DEFAULT_CHUNK_SIZE,chunk_overlap=DEFAULT_CHUNK_OVERLAP):
@@ -173,25 +171,13 @@ def answer(system: str, context: str, question: str,
             context_list += store["texts"]
 
         # 2.  Build a Qwen-chat prompt (helper defined earlier)
-        prompt = build_llm_prompt(system, context_list, question)
-
-        # 3.  Tokenise & cap
-        load_chat()
-        tokens = tokenizer(
-            prompt,
-            return_tensors="pt",
-            add_special_tokens=False,        # we built the chat template ourselves
-        )
-
-        if tokens["input_ids"].size(1) > MAX_PROMPT_TOKENS:
-            tokens = {k: v[:, -MAX_PROMPT_TOKENS:] for k, v in tokens.items()}
-
-        tokens = {k: v.to(chat_model.device) for k, v in tokens.items()}
+        input_ids, terminators = build_llm_prompt(system, context_list, question)
 
         # --- generate ------------------------------------------------------
         output = chat_model.generate(
-            **tokens,
+            input_ids,
             max_new_tokens=512,
+            eos_token_id=terminators,
             max_length=MAX_PROMPT_TOKENS + 512,
             do_sample=True,
             temperature=temperature,