groq-llama-3.1

app.py

@@ -5,7 +5,9 @@ from dotenv import load_dotenv
 import inspect
 import pandas as pd
 from langchain_community.tools import DuckDuckGoSearchRun
+from langchain_groq import ChatGroq
 from langchain_huggingface import ChatHuggingFace, HuggingFacePipeline
+from openai import api_key
 from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
 
 
@@ -18,16 +20,12 @@ class BasicAgent:
 
 
     def __init__(self):
-        self.model_id = "Qwen/Qwen2-1.5B-Instruct"
-        self.tokenizer = AutoTokenizer.from_pretrained(self.model_id, token=os.getenv("HF_TOKEN"))
-        self.model = AutoModelForCausalLM.from_pretrained(self.model_id, token=os.getenv("HF_TOKEN"))
-        pipe = pipeline(
-            "text-generation", model=self.model, tokenizer=self.tokenizer, temperature=0.1, device_map="auto",
-            max_new_tokens=1024
+        self.model = ChatGroq(
+            model="llama-3.1-8b-instant",
+            temperature=0.0,
+            max_new_tokens=1024,
+            api_key=os.getenv("GROQ_API_KEY")
         )
-        self.llm = HuggingFacePipeline(pipeline=pipe)
-        self.model = ChatHuggingFace(llm=self.llm, verbose=True)
-        self.search = DuckDuckGoSearchRun()
         print("BasicAgent initialized.")
 
 
@@ -37,9 +35,6 @@ class BasicAgent:
         prompt = f"""
         Give concise answer to this question: {question}
         """
-        # try:
-        #     fixed_answer = self.search.invoke(question)
-        # except Exception as _:
         pattern = r'.*?<start_of_turn>model'
         fixed_answer = self.model.invoke(prompt).content
         fixed_answer = re.sub(pattern, '', fixed_answer, flags=re.DOTALL)