xipper / main.py
kanhaiya-ML's picture
Adding state language
262d4d2
Raw
History Blame Contribute Delete
11.2 kB
from fastapi.middleware.cors import CORSMiddleware
from fastapi import FastAPI,UploadFile,File,HTTPException
from pydantic import BaseModel
from fastapi.responses import FileResponse
from translate import translate_to_target_language,translate_input_to_english
from graph_main import graph
from sarvamai import SarvamAI
import tempfile
from groq import Groq
# from elevenlabs.client import ElevenLabs
from deepgram import DeepgramClient,FileSource,PrerecordedOptions
import base64
from pydub import AudioSegment
import io
import os
import time
import string
from dotenv import load_dotenv
load_dotenv()
app = FastAPI()
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_methods=["*"],
allow_headers=["*"],
allow_credentials=True
)
print("CORS middleware added")
class ChatRequest(BaseModel):
message: str
# phoneNumberId: str
# elevenlabs_client = ElevenLabs(api_key=os.getenv("ELEVENLABS_API_KEY"))
groq_client = Groq(api_key=os.getenv("GROQ_API_KEY"))
sarvam_client = SarvamAI(api_subscription_key=os.getenv("SARVAM_API_KEY"))
deepgram_client = DeepgramClient(os.getenv("DEEPGRAM_API_KEY"))
SARVAM_LANG_MAP = {
"hindi": "hi-IN", "hi": "hi-IN",
"tamil": "ta-IN", "ta": "ta-IN",
"marathi": "mr-IN", "mr": "mr-IN",
"bengali": "bn-IN", "bn": "bn-IN",
"telugu": "te-IN", "te": "te-IN",
"gujarati": "gu-IN", "gu": "gu-IN",
"english": "en-IN", "en": "en-IN",
"kannada": "kn-IN", "kn": "kn-IN"
}
@app.post("/chat")
async def chat(request: ChatRequest):
try:
user_text = request.message
config = {"configurable": {"thread_id": "hotel_owner_1"}}
result = await graph.ainvoke({"query": user_text},config=config)
# print(f"Full result: {result}")
response = result.get("response","sorry, i couldn't process that request")
stored_lang = result.get("conversation_language")
is_mid_conversation = result.get("pending_manager")
if stored_lang and is_mid_conversation:
detected_language = stored_lang
print(f"Reusing stored language: {detected_language}")
else:
lang_prompt = f"""Detect the language of this text: "{user_text}"
Return ONLY one of these codes: hi, en, ta, mr, bn, te, gu, kn
Return ONLY the code, nothing else."""
from agents.llm import llm
lang_response = llm.invoke(lang_prompt)
detected_language = lang_response.content.strip().lower()
print(f"Detected language: {detected_language}")
await graph.aupdate_state(config, {"conversation_language": detected_language})
final_response = translate_to_target_language(response,detected_language)
print(f"Translated response: {final_response}")
sarvam_lang = SARVAM_LANG_MAP.get(detected_language,"hi-IN")
print(f"Sarvam lang: {sarvam_lang}")
for attempt in range(3):
try:
tts_start = time.time()
tts_response = sarvam_client.text_to_speech.convert(
text=final_response,
target_language_code=sarvam_lang,
speaker="abhilash",
model="bulbul:v2"
)
print(f"TTS Latency: {time.time() - tts_start:.2f}s")
# print(f"TTS response: {tts_response}")
# print(f"Audio list: {tts_response.audios}")
break
except Exception as e:
# print(f"❌ TTS attempt {attempt} failed: {e}") # ← add this
if attempt == 2:
raise HTTPException(status_code=503, detail=f"TTS unavailable: {e}")
audio_bytes = base64.b64decode(tts_response.audios[0])
audio_base64 = base64.b64encode(audio_bytes).decode("utf-8")
return {
"response": response,
"audio": audio_base64
}
except Exception as e:
import traceback
print(f"Chat endpoint error: {e}")
traceback.print_exc()
return {"response": "Sorry, I encountered an error."}
@app.get("/")
def root():
return FileResponse("index.html")
@app.post("/voice")
async def voice(audio: UploadFile = File(...)):
temp_file = None
try:
audio_bytes = await audio.read()
with open("received.webm", "wb") as f:
f.write(audio_bytes)
if len(audio_bytes) > 5*1024*1024:
return HTTPException(status_code=400,detail="Audio file is too large. Max 5 MB file is allowed")
# try:
# raw_audio = AudioSegment.from_file(io.BytesIO(audio_bytes))
# clean_audio = raw_audio.set_frame_rate(16000).set_channels(1)
# except Exception as e:
# raise HTTPException(status_code=400,detail=f"failed to process audio format: {e}")
f = tempfile.NamedTemporaryFile(delete=False,suffix=".wav")
temp_file = f.name
f.write(audio_bytes)
# clean_audio.export(temp_file, format="wav")
f.close()
total_start = time.time()
#this is one of the most wrost way never use this way to make stt better
# hotel_vocab = "Kanhaiya, Aman, Maharani, Deluxe, Standard, Suite, Presidential, EP, CP, MAP, AP, room only, breakfast, booking, check-in, check-out, advance"
with open(temp_file,"rb") as f:
for attempt in range(3):
try:
stt_start = time.time()
payload = {
"buffer": f.read(),
}
# The Magic Parameters
options = PrerecordedOptions(
model="nova-2", # The fastest, most accurate model
language="hi", # Deepgram's Hindi model flawlessly handles Hinglish
smart_format=True, # Forces dates, times, and phone numbers into digits!
punctuate=True # Adds periods and commas so LangGraph reads it easily
)
response = deepgram_client.listen.rest.v("1").transcribe_file(payload, options)
# Extract the text from the JSON response
transcript_data = response["results"]["channels"][0]["alternatives"][0]
raw_transcript = transcript_data["transcript"]
print(f"Deepgram Heard: {raw_transcript}")
# transcript = elevenlabs_client.speech_to_text.convert(
# file=f,
# model_id="scribe_v2"
# )
# transcript = groq_client.audio.transcriptions.create(
# model="whisper-large-v3",
# file=f,
# response_format="verbose_json",
# prompt=hotel_vocab,
# temperature=0.0
# )
print(f"STT Latency: {time.time() - stt_start:.2f}s")
break
except Exception as e:
if attempt == 2:
raise HTTPException(status_code=503,detail="STT service unavailable")
# raw_transcript = transcript.text.strip()
try:
trans_start = time.time()
clean_english_query = translate_input_to_english(raw_transcript)
print(f"Translation Latency: {time.time() - trans_start:.2f}s")
except Exception as e:
raise HTTPException(status_code=503,detail="Translation not possible")
detected_language = clean_english_query.get("language", "hi")
print(f"Original: {raw_transcript} | Translated: {clean_english_query} | Language: {detected_language}")
# if detected_language in ["ur", "urdu"]:
# detected_language = "hi"
english_text = clean_english_query.get("english_text","")
clean_english_query = english_text.translate(str.maketrans('', '', string.punctuation)).strip()
if not clean_english_query:
return {"transcript":"","response": "Sorry, I couldn't here you. Please try again.","audio_base64": None}
# detected_language = transcript.language
# hinglish_query = translate_to_hinglish(clean_english_query)
# print(f"Hinglish Query: {hinglish_query}")
# print(f"Detected language: {detected_language}")
# print(f"whisper transcript: {query}")
config = {"configurable": {"thread_id":"hotel_owner_1"}}
try:
graph_start = time.time()
result = await graph.ainvoke({"query": clean_english_query},config=config)
print(f"Graph Latency: {time.time() - graph_start:.2f}s")
except Exception as e:
# IF GRAPH CRASHES, CATCH IT AND PRINT THE EXACT LINE!
print("\n" + "🔥"*25)
print(f"CRASH IN LANGGRAPH: {e}")
import traceback
traceback.print_exc()
print("🔥"*25 + "\n")
result = {
"response": "I'm sorry, I encountered an internal glitch. Could you repeat that?"
}
graph_response = result.get("response", "Sorry, I couldn't process that request")
print(f"graph output: {graph_response}")
final_response = translate_to_target_language(graph_response,detected_language)
print(f"final response: {final_response}")
sarvam_lang = SARVAM_LANG_MAP.get(detected_language.lower(),"hi-IN")
for attempt in range(3):
try:
tts_start = time.time()
tts_response = sarvam_client.text_to_speech.convert(
text=final_response,
target_language_code=sarvam_lang,
speaker="abhilash",
model="bulbul:v2"
)
print(f"TTS Latency: {time.time() - tts_start:.2f}s")
break
except Exception as e:
if attempt == 2:
raise HTTPException(status_code=503,detail="TTS service unavailable")
audio_bytes = base64.b64decode(tts_response.audios[0])
audio_base64 = base64.b64encode(audio_bytes).decode("utf-8")
with open("test_output.wav", "wb") as f:
f.write(audio_bytes)
print(f"Total Latency: {time.time() - total_start:.2f}s")
booking_link = result.get("booking_link","")
return {
"transcript": clean_english_query,
"detected_language": detected_language,
"response": final_response,
"audio": audio_base64,
"booking_link": booking_link
}
finally:
if temp_file and os.path.exists(temp_file):
os.remove(temp_file)