change model from openai to grok

api/generate_response.py

@@ -1,9 +1,9 @@
 import logging
-from openai import OpenAI
+from groq import Groq
 import tiktoken
 
 from utils.token import count_tokens
-from config import API_URL, MODEL, GH_API_TOKEN, SYSTEM_PROMPT
+from config import API_URL, MODEL, GROQ_API_KEY, SYSTEM_PROMPT
 
 logger = logging.getLogger(__name__)
 
@@ -14,7 +14,7 @@ except KeyError:
     encoder = tiktoken.get_encoding("cl100k_base")
 
 try:
-    client = OpenAI(base_url=API_URL, api_key=GH_API_TOKEN, timeout=60)
+    client = Groq(api_key=GROQ_API_KEY)
     logging.info("OpenAI client initialized.")
 except Exception as e:
     logging.critical("Failed to initialize OpenAI client as %s", e)

config.py

@@ -17,8 +17,9 @@ COOKIES_FILE = Path("utils/youtube_cookies.txt")
 API_URL = "https://models.github.ai/inference"
 HF_API_TOKEN = os.environ.get("HF_API_TOKEN")
 GH_API_TOKEN = os.environ.get("GITHUB_TOKEN")
-MODEL = "openai/gpt-4.1"
-MAX_CONTEXT_TOKENS = 7000
+GROQ_API_KEY = os.environ.get("GROQ_API_KEY")
+MODEL = "llama-3.1-8b-instant"
+MAX_CONTEXT_TOKENS = 4500
 SYSTEM_PROMPT = """
 You are speaking as Acharya Prashant.
 

pyproject.toml

@@ -7,6 +7,7 @@ requires-python = ">=3.11"
 dependencies = [
     "faiss-cpu==1.9.0",
     "fastapi==0.116.1",
+    "groq>=1.0.0",
     "openai==2.8.1",
     "pytz==2025.2",
     "sentence-transformers==3.0.1",

uv.lock

@@ -15,6 +15,7 @@ source = { virtual = "." }
 dependencies = [
     { name = "faiss-cpu" },
     { name = "fastapi" },
+    { name = "groq" },
     { name = "openai" },
     { name = "pytz" },
     { name = "sentence-transformers" },
@@ -29,6 +30,7 @@ dependencies = [
 requires-dist = [
     { name = "faiss-cpu", specifier = "==1.9.0" },
     { name = "fastapi", specifier = "==0.116.1" },
+    { name = "groq", specifier = ">=1.0.0" },
     { name = "openai", specifier = "==2.8.1" },
     { name = "pytz", specifier = "==2025.2" },
     { name = "sentence-transformers", specifier = "==3.0.1" },
@@ -225,6 +227,23 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e6/ab/fb21f4c939bb440104cc2b396d3be1d9b7a9fd3c6c2a53d98c45b3d7c954/fsspec-2026.2.0-py3-none-any.whl", hash = "sha256:98de475b5cb3bd66bedd5c4679e87b4fdfe1a3bf4d707b151b3c07e58c9a2437", size = 202505, upload-time = "2026-02-05T21:50:51.819Z" },
 ]
 
+[[package]]
+name = "groq"
+version = "1.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "anyio" },
+    { name = "distro" },
+    { name = "httpx" },
+    { name = "pydantic" },
+    { name = "sniffio" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3f/12/f4099a141677fcd2ed79dcc1fcec431e60c52e0e90c9c5d935f0ffaf8c0e/groq-1.0.0.tar.gz", hash = "sha256:66cb7bb729e6eb644daac7ce8efe945e99e4eb33657f733ee6f13059ef0c25a9", size = 146068, upload-time = "2025-12-17T23:34:23.115Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/4a/88/3175759d2ef30406ea721f4d837bfa1ba4339fde3b81ba8c5640a96ed231/groq-1.0.0-py3-none-any.whl", hash = "sha256:6e22bf92ffad988f01d2d4df7729add66b8fd5dbfb2154b5bbf3af245b72c731", size = 138292, upload-time = "2025-12-17T23:34:21.957Z" },
+]
+
 [[package]]
 name = "h11"
 version = "0.16.0"
@@ -1322,6 +1341,9 @@ wheels = [
     { url = "https://download.pytorch.org/whl/cpu/torch-2.10.0-1-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:0826ac8e409551e12b2360ac18b4161a838cbd111933e694752f351191331d09" },
     { url = "https://download.pytorch.org/whl/cpu/torch-2.10.0-1-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:7fbbf409143a4fe0812a40c0b46a436030a7e1d14fe8c5234dfbe44df47f617e" },
     { url = "https://download.pytorch.org/whl/cpu/torch-2.10.0-1-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:b39cafff7229699f9d6e172cac74d85fd71b568268e439e08d9c540e54732a3e" },
+    { url = "https://download.pytorch.org/whl/cpu/torch-2.10.0-2-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:90821a3194b8806d9fa9fdaa9308c1bc73df0c26808274b14129a97c99f35794" },
+    { url = "https://download.pytorch.org/whl/cpu/torch-2.10.0-2-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:358bd7125cbec6e692d60618a5eec7f55a51b29e3652a849fd42af021d818023" },
+    { url = "https://download.pytorch.org/whl/cpu/torch-2.10.0-2-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:470de4176007c2700735e003a830828a88d27129032a3add07291da07e2a94e8" },
     { url = "https://download.pytorch.org/whl/cpu/torch-2.10.0-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:4584ab167995c0479f6821e3dceaf199c8166c811d3adbba5d8eedbbfa6764fd" },
     { url = "https://download.pytorch.org/whl/cpu/torch-2.10.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:45a1c5057629444aeb1c452c18298fa7f30f2f7aeadd4dc41f9d340980294407" },
     { url = "https://download.pytorch.org/whl/cpu/torch-2.10.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:339e05502b6c839db40e88720cb700f5a3b50cda332284873e851772d41b2c1e" },