Update chatbot_rag.py

chatbot_rag.py

@@ -40,29 +40,37 @@ def build_qa():
 
     # 3. Load LLM (Phi-3 mini)
     print("🔹 Loading LLM...") 
+
+    model_id = "microsoft/phi-2"
+    
+    # Load tokenizer
+    tokenizer = AutoTokenizer.from_pretrained(model_id)
+    
+    # Load model
+    model = AutoModelForCausalLM.from_pretrained(
+        model_id,
+        device_map="auto",       # put on GPU if available, else CPU
+        torch_dtype="auto",      # auto precision
+        trust_remote_code=True   # allow custom model code
+    )
     
-    model_id = "meta-llama/Llama-3.2-3B-Instruct" 
-    tokenizer = AutoTokenizer.from_pretrained(model_id) 
-    model = AutoModelForCausalLM.from_pretrained( 
-        model_id, 
-        device_map="auto", 
-        trust_remote_code=True # ensures it runs on available CPU 
-    ) 
-    pipe = pipeline( 
-        "text-generation", 
-        model=model, 
-        tokenizer=tokenizer, 
-        max_new_tokens=256, 
-        temperature=0.2, # keeps answers deterministic but less rigid than 0 
-        do_sample=True, # allow some randomness 
-        top_p=0.9, # nucleus sampling to avoid loops 
-        repetition_penalty=1.2, # 🚀 penalize repeats 
-        eos_token_id=tokenizer.eos_token_id, # stop at EOS 
-        return_full_text=False 
-    ) 
+    # Create pipeline
+    pipe = pipeline(
+        "text-generation",
+        model=model,
+        tokenizer=tokenizer,
+        max_new_tokens=256,       # control length of response
+        temperature=0.2,          # more deterministic
+        do_sample=False,          # no randomness (deterministic answers)
+        top_p=0.9,                # nucleus sampling
+        repetition_penalty=1.2,   # 🚀 reduce loops/repeats
+        eos_token_id=tokenizer.eos_token_id,
+        return_full_text=False
+    )
+    
+    # Wrap into LangChain LLM
     llm = HuggingFacePipeline(pipeline=pipe)
 
-
     # 4. Retriever
     retriever = vectorstore.as_retriever(search_kwargs={"k": 3})
 
@@ -79,7 +87,7 @@ def build_qa():
     {context}
 
     Question: {question}
-    Answer (one short sentence):
+    Answer :
     """,
     )