changing model as per huggingFace

app.py

@@ -4,11 +4,14 @@ import time
 import chromadb
 from langchain_chroma import Chroma
 import transformers
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
 from langchain_huggingface import HuggingFaceEmbeddings
 from langchain_core.prompts import PromptTemplate
 from langchain_core.output_parsers import StrOutputParser
 from langchain_ollama import ChatOllama
 from langchain_core.documents import Document
+from langchain_community.llms import HuggingFacePipeline
+
 
 # Initialize in-memory ChromaDB client
 # This client runs entirely within the app.py script.
@@ -58,7 +61,23 @@ def using_ollama_model(retriever, query, results, conversation_history):
     
     doc_texts = "\\n".join([doc.page_content for doc in results])
     
-    llm = ChatOllama(model="llama3.2", temperature=0.4, num_predict=512)
+    # llm = ChatOllama(model="llama3.2", temperature=0.4, num_predict=512)
+    tokenizer = AutoTokenizer.from_pretrained("llama3.2")
+    model = AutoModelForCausalLM.from_pretrained("llama3.2")
+    pipe = pipeline(
+        "text-generation",
+        model=model,
+        tokenizer=tokenizer,
+        max_new_tokens=256,
+        do_sample=True,
+        temperature=0.7,
+        top_p=0.95,
+        repetition_penalty=1.2
+    )
+
+     # Use the pipeline with LangChain's HuggingFacePipeline
+    llm = HuggingFacePipeline(pipeline=pipe)
+
     
     rag_chain = template | llm | StrOutputParser()