import os import json import asyncio import logging from fastapi import FastAPI from pydantic import BaseModel from fastapi.responses import StreamingResponse from huggingface_hub import hf_hub_download from llama_cpp import Llama from tavily import TavilyClient from groq import Groq logging.basicConfig(level=logging.INFO, format='%(message)s') groq_client = Groq(api_key=os.environ.get("GROQ_API_KEY")) tavily_client = TavilyClient(api_key=os.environ.get("TAVILY_API_KEY")) logging.info("Downloading Konkani GGUF Model from friend's repo...") model_path = hf_hub_download( repo_id="dom0804/konkani_companion_GGUF", filename="qwen2.5-3b-instruct.Q4_K_M.gguf" ) logging.info("Loading into CPU...") llm = Llama(model_path=model_path, n_ctx=2048, n_threads=2) app = FastAPI() class ChatPayload(BaseModel): messages: list stream: bool = False @app.post("/openai/v1/chat/completions") async def chat_endpoint(payload: ChatPayload): user_query = payload.messages[-1]["content"] # --- PHASE 1: SMART REWRITER --- history_text = "\n".join([f"{m['role']}: {m['content']}" for m in payload.messages[:-1]]) rewriter_instruction = f"""You are a Search Query Optimizer. RULES: 1. If the query is purely conversational (e.g., "hi", "how are you"), output EXACTLY: SKIP_SEARCH 2. If it is factual, generate a SINGLE optimized Google search string. 3. Output ONLY the raw search query or SKIP_SEARCH. History: {history_text} Latest Query: {user_query} """ try: rewriter_response = groq_client.chat.completions.create( messages=[{"role": "user", "content": rewriter_instruction}], model="llama-3.1-8b-instant", temperature=0.1 ) optimized_query = rewriter_response.choices[0].message.content.strip() except Exception: optimized_query = user_query # --- PHASE 2: SMART SEARCH --- if "SKIP_SEARCH" in optimized_query: fact = "This is a conversational query. Respond naturally." else: try: search_data = tavily_client.search(query=optimized_query, search_depth="basic") fact = "\n".join([result['content'] for result in search_data['results'][:2]]) except Exception: fact = "No live info available." # --- PHASE 3: GROQ BRAIN --- brain_instruction = f""" You are the Core Reasoning Engine for a cross-cultural Konkani conversational AI. {fact} 1. CONVERSATIONAL FLUIDITY: Respond warmly and naturally to social queries. 2. FACTUAL GROUNDING: Base answers on the . Do not hallucinate. 3. DIRECT TRANSLATION OVERRIDE: If asked to translate, output EXACTLY the target English text. 4. DOWNSTREAM SAFETY: Max 3 sentences per paragraph. NO markdown. """ groq_messages = [{"role": "system", "content": brain_instruction}] for msg in payload.messages[:-1]: groq_messages.append({"role": msg["role"], "content": msg["content"]}) groq_messages.append({"role": "user", "content": user_query}) response = groq_client.chat.completions.create( messages=groq_messages, model="llama-3.3-70b-versatile" ) english_paragraphs = [p for p in response.choices[0].message.content.strip().split('\n') if p.strip()] # --- PHASE 4: GGUF TRANSLATION STREAM --- async def stream_generator(): for para in english_paragraphs: formatted_prompt = f"<|im_start|>system\nYou are a highly accurate translation model. Translate the following English text into Konkani.<|im_end|>\n<|im_start|>user\n{para}<|im_end|>\n<|im_start|>assistant\n" stream = llm(formatted_prompt, max_tokens=1024, stop=["<|im_end|>"], stream=True, temperature=0.1) for chunk in stream: yield f"data: {json.dumps({'id': 'chatcmpl-custom', 'object': 'chat.completion.chunk', 'choices': [{'delta': {'content': chunk['choices'][0]['text']}}]})}\n\n" await asyncio.sleep(0.01) yield f"data: {json.dumps({'choices': [{'delta': {'content': '\n\n'}}]})}\n\n" yield "data: [DONE]\n\n" if payload.stream: return StreamingResponse(stream_generator(), media_type="text/event-stream") else: final_text = "" async for chunk in stream_generator(): if chunk != "data: [DONE]\n\n": data_dict = json.loads(chunk.replace("data: ", "").strip()) if "content" in data_dict["choices"][0]["delta"]: final_text += data_dict["choices"][0]["delta"]["content"] return {"choices": [{"message": {"role": "assistant", "content": final_text.strip()}}]}