| import os |
| import pandas as pd |
| from datetime import datetime |
| import threading |
| import uuid |
| from fastapi import FastAPI, HTTPException |
| from fastapi.middleware.cors import CORSMiddleware |
| from pydantic import BaseModel, Field |
| from openai import AsyncOpenAI |
| from dotenv import load_dotenv |
| from src.dialect_rules import ( |
| hausa_variety_instruction, |
| nigerian_variety_instruction, |
| nigerian_variety_retry_prompt, |
| nigerian_variety_retry_reason, |
| ) |
|
|
| load_dotenv() |
|
|
| app = FastAPI(title="PACYCx Hybrid Backend", version="1.0.0") |
|
|
| |
| app.add_middleware( |
| CORSMiddleware, |
| allow_origins=["*"], |
| allow_credentials=True, |
| allow_methods=["*"], |
| allow_headers=["*"], |
| ) |
|
|
| |
| QWEN_API_KEY = os.getenv("QWEN_API_KEY") |
| QWEN_BASE_URL = os.getenv("QWEN_BASE_URL", "https://dashscope-intl.aliyuncs.com/compatible-mode/v1") |
| QWEN_MODEL_NAME = os.getenv("QWEN_MODEL_NAME", "qwen3-coder-80b-instruct") |
|
|
| GROQ_API_KEY = os.getenv("GROQ_API_KEY") |
| GROQ_MODEL_NAME = os.getenv("GROQ_MODEL_NAME", "llama-3.3-70b-versatile") |
| DEEPSEEK_API_KEY = os.getenv("DEEPSEEK_API_KEY") |
| DEEPSEEK_BASE_URL = os.getenv("DEEPSEEK_BASE_URL", "https://api.deepseek.com") |
| DEEPSEEK_MODEL_NAME = os.getenv("DEEPSEEK_MODEL_NAME", "deepseek-chat") |
| GEMINI_API_KEY = os.getenv("GEMINI_API_KEY") |
| GEMINI_BASE_URL = os.getenv("GEMINI_BASE_URL", "https://generativelanguage.googleapis.com/v1beta/openai/") |
| GEMINI_MODEL_NAME = os.getenv("GEMINI_MODEL_NAME", "gemini-2.5-flash") |
| OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY") |
| OPENROUTER_BASE_URL = os.getenv("OPENROUTER_BASE_URL", "https://openrouter.ai/api/v1") |
| OPENROUTER_FREE_MODEL_NAME = os.getenv("OPENROUTER_FREE_MODEL_NAME", "openrouter/free") |
| OPENROUTER_NEMOTRON_MODEL_NAME = os.getenv("OPENROUTER_NEMOTRON_MODEL_NAME", "nvidia/nemotron-3-nano-30b-a3b:free") |
| OPENROUTER_GPT_OSS_MODEL_NAME = os.getenv("OPENROUTER_GPT_OSS_MODEL_NAME", "openai/gpt-oss-20b:free") |
| OPENROUTER_LFM_MODEL_NAME = os.getenv("OPENROUTER_LFM_MODEL_NAME", "liquid/lfm-2.5-1.2b-instruct:free") |
|
|
| AI_ROUTES = {} |
| if QWEN_API_KEY and QWEN_API_KEY != "your-api-key-here": |
| AI_ROUTES["qwen"] = (AsyncOpenAI(api_key=QWEN_API_KEY, base_url=QWEN_BASE_URL), QWEN_MODEL_NAME, "Qwen Hybrid Node") |
| if GROQ_API_KEY: |
| AI_ROUTES["llama"] = (AsyncOpenAI(api_key=GROQ_API_KEY, base_url="https://api.groq.com/openai/v1"), GROQ_MODEL_NAME, "Groq Llama Hybrid Node") |
| if DEEPSEEK_API_KEY: |
| AI_ROUTES["deepseek"] = (AsyncOpenAI(api_key=DEEPSEEK_API_KEY, base_url=DEEPSEEK_BASE_URL), DEEPSEEK_MODEL_NAME, "DeepSeek Hybrid Node") |
| if GEMINI_API_KEY: |
| AI_ROUTES["gemini"] = (AsyncOpenAI(api_key=GEMINI_API_KEY, base_url=GEMINI_BASE_URL), GEMINI_MODEL_NAME, "Gemini Hybrid Node") |
| if OPENROUTER_API_KEY: |
| openrouter_client = AsyncOpenAI(api_key=OPENROUTER_API_KEY, base_url=OPENROUTER_BASE_URL) |
| AI_ROUTES["openrouter-free"] = (openrouter_client, OPENROUTER_FREE_MODEL_NAME, "OpenRouter Free Node") |
| AI_ROUTES["nemotron"] = (openrouter_client, OPENROUTER_NEMOTRON_MODEL_NAME, "OpenRouter Nemotron Free Node") |
| AI_ROUTES["gpt-oss"] = (openrouter_client, OPENROUTER_GPT_OSS_MODEL_NAME, "OpenRouter GPT-OSS Free Node") |
| AI_ROUTES["lfm"] = (openrouter_client, OPENROUTER_LFM_MODEL_NAME, "OpenRouter LFM Free Node") |
|
|
| if "qwen" in AI_ROUTES: |
| client, MODEL_NAME, NODE_TYPE = AI_ROUTES["qwen"] |
| elif "llama" in AI_ROUTES: |
| client, MODEL_NAME, NODE_TYPE = AI_ROUTES["llama"] |
| elif "deepseek" in AI_ROUTES: |
| client, MODEL_NAME, NODE_TYPE = AI_ROUTES["deepseek"] |
| elif "gemini" in AI_ROUTES: |
| client, MODEL_NAME, NODE_TYPE = AI_ROUTES["gemini"] |
| elif "openrouter-free" in AI_ROUTES: |
| client, MODEL_NAME, NODE_TYPE = AI_ROUTES["openrouter-free"] |
| else: |
| client = None |
| MODEL_NAME = None |
| NODE_TYPE = "Offline" |
|
|
| def resolve_ai_route(ai_model: str, source_label: str, target_label: str, text: str): |
| choice = (ai_model or "auto").strip().lower() |
| if choice == "auto": |
| hint = f"{source_label} {target_label} {text}".lower() |
| if any(token in hint for token in [ |
| "korean", "hangul", "chinese", "mandarin", "cantonese", "arabic", |
| "japanese", "thai", "vietnamese", "code-switch", "multilingual" |
| ]): |
| choice = "qwen" |
| elif any(token in hint for token in [ |
| "reason", "explain", "ambiguity", "semantic", "pragmatic", "cultural", "review", "oracle" |
| ]): |
| choice = "nemotron" |
| else: |
| choice = "llama" |
|
|
| ordered = [choice, "qwen", "llama", "nemotron", "gpt-oss", "lfm", "openrouter-free", "deepseek", "gemini"] |
| for route_name in ordered: |
| if route_name in AI_ROUTES: |
| return AI_ROUTES[route_name] |
| return client, MODEL_NAME, NODE_TYPE |
|
|
| class TranslationRequest(BaseModel): |
| text: str |
| source_language: str = "Unknown" |
| source_dialect: str = "Standard" |
| target_language: str |
| target_dialect: str |
| user_key: str = "Polyglot Player" |
| ai_model: str = "auto" |
|
|
| class TranslationResponse(BaseModel): |
| original_text: str |
| translated_text: str |
| target_dialect: str |
| node: str |
|
|
| class PolyglotReviewSubmission(BaseModel): |
| interaction_id: str = Field(min_length=8, max_length=128) |
| supersedes_interaction_id: str = Field(default="", max_length=128) |
| app_source: str = Field(default="PACYCx", min_length=2, max_length=64) |
| user_key: str = Field(default="Polyglot Player", max_length=256) |
| source_text: str = Field(min_length=1, max_length=10000) |
| source_input_mode: str = Field(default="text", max_length=32) |
| machine_transcript_initial: str = Field(default="", max_length=10000) |
| user_transcript_final: str = Field(default="", max_length=10000) |
| machine_translation_initial: str = Field(min_length=1, max_length=10000) |
| user_translation_final: str = Field(min_length=1, max_length=10000) |
| source_language: str = Field(default="Unknown", max_length=128) |
| source_dialect: str = Field(default="Standard", max_length=256) |
| target_language: str = Field(default="Unknown", max_length=128) |
| target_dialect: str = Field(default="Standard", max_length=256) |
| asr_model: str = Field(default="", max_length=128) |
| audio_sanitation: bool = False |
| ai_model: str = Field(default="auto", max_length=128) |
| translation_route: str = Field(default="frontend-reviewed", max_length=128) |
| consent_confirmed: bool = False |
| consent_version: str = Field(default="polyglot-reviewed-submit-v1", max_length=128) |
|
|
|
|
| _PENDING_QUEUE_LOCK = threading.Lock() |
|
|
|
|
| def _pending_queue_path(): |
| configured = os.environ.get("PENDING_APPROVALS_FILE", "").strip() |
| if configured: |
| return configured |
| return "/app/pending_approvals.csv" if os.path.exists("/app") else "pending_approvals.csv" |
|
|
|
|
| def _translation_edit_distance(initial_text: str, final_text: str): |
| initial = str(initial_text or "").casefold().split() |
| final = str(final_text or "").casefold().split() |
| if not initial and not final: |
| return 0.0 |
|
|
| previous = list(range(len(final) + 1)) |
| for row_index, initial_token in enumerate(initial, start=1): |
| current = [row_index] |
| for column_index, final_token in enumerate(final, start=1): |
| substitution_cost = 0 if initial_token == final_token else 1 |
| current.append( |
| min( |
| current[-1] + 1, |
| previous[column_index] + 1, |
| previous[column_index - 1] + substitution_cost, |
| ) |
| ) |
| previous = current |
|
|
| return round(previous[-1] / max(len(initial), len(final), 1), 4) |
|
|
|
|
| def _sync_pending_queue_to_hub(pending_file: str, queue_id: str): |
| hf_token = os.environ.get("HF_TOKEN") |
| if not hf_token: |
| return False |
|
|
| from huggingface_hub import HfApi |
|
|
| api = HfApi(token=hf_token) |
| api.upload_file( |
| path_or_fileobj=pending_file, |
| path_in_repo="pending_approvals.csv", |
| repo_id="toecm/PureChain_Dataset", |
| repo_type="dataset", |
| commit_message=f"Reviewed Polyglot Chat submission {queue_id}", |
| ) |
| return True |
|
|
|
|
| def _append_polyglot_review(request: PolyglotReviewSubmission): |
| pending_file = _pending_queue_path() |
| submitted_at = datetime.now().strftime("%Y-%m-%d %H:%M:%S") |
| queue_id = f"polyglot-{uuid.uuid4()}" |
| final_source_text = (request.user_transcript_final or request.source_text).strip() |
| new_entry = { |
| "User": request.user_key, |
| "Data_Origin": "Game: Polyglot Chat", |
| "Utterance": final_source_text, |
| "Dialect": request.target_dialect.strip(), |
| "Clarification": request.user_translation_final.strip(), |
| "Clarification_Source": f"User-reviewed / {request.ai_model}", |
| "Tone": "Neutral / Conversational", |
| "Context": f"Translated from {request.source_language} ({request.source_dialect})", |
| "Pragmatic_Analysis": "", |
| "Audio": "", |
| "Timestamp": submitted_at, |
| "Chain_ID": "", |
| "Approvers": "", |
| "Language": request.target_language.strip(), |
| "Queue_ID": queue_id, |
| "Interaction_ID": request.interaction_id.strip(), |
| "Supersedes_Interaction_ID": request.supersedes_interaction_id.strip(), |
| "App_Source": request.app_source.strip(), |
| "Submission_Status": "Pending Review", |
| "Consent_Confirmed": "true", |
| "Consent_Version": request.consent_version.strip(), |
| "Source_Language": request.source_language.strip(), |
| "Source_Dialect": request.source_dialect.strip(), |
| "Source_Input_Mode": request.source_input_mode.strip().lower() or "text", |
| "Machine_Transcript_Initial": request.machine_transcript_initial.strip(), |
| "User_Transcript_Final": final_source_text, |
| "Transcript_Edit_Distance": _translation_edit_distance( |
| request.machine_transcript_initial, |
| final_source_text, |
| ) if request.machine_transcript_initial.strip() else 0.0, |
| "ASR_Model": request.asr_model.strip(), |
| "Audio_Sanitation": str(request.audio_sanitation).lower(), |
| "Audio_Retained": "false", |
| "Target_Language": request.target_language.strip(), |
| "Target_Dialect": request.target_dialect.strip(), |
| "Machine_Translation_Initial": request.machine_translation_initial.strip(), |
| "User_Translation_Final": request.user_translation_final.strip(), |
| "Translation_Edit_Distance": _translation_edit_distance( |
| request.machine_translation_initial, |
| request.user_translation_final, |
| ), |
| "AI_Model": request.ai_model.strip(), |
| "Translation_Route": request.translation_route.strip(), |
| "Review_Submitted_At": submitted_at, |
| } |
|
|
| with _PENDING_QUEUE_LOCK: |
| if os.path.exists(pending_file): |
| df = pd.read_csv(pending_file, dtype=str).fillna("") |
| else: |
| parent = os.path.dirname(os.path.abspath(pending_file)) |
| os.makedirs(parent, exist_ok=True) |
| df = pd.DataFrame() |
|
|
| if "Interaction_ID" in df.columns: |
| duplicate = df[df["Interaction_ID"].astype(str) == request.interaction_id.strip()] |
| if not duplicate.empty: |
| existing = duplicate.iloc[0] |
| existing_final = str( |
| existing.get("User_Translation_Final", "") |
| or existing.get("Clarification", "") |
| ).strip() |
| if existing_final != request.user_translation_final.strip(): |
| raise HTTPException( |
| status_code=409, |
| detail="This interaction ID already belongs to a different reviewed translation.", |
| ) |
| existing_queue_id = str(existing.get("Queue_ID", "")) |
| synced_to_hub = _sync_pending_queue_to_hub( |
| pending_file, |
| existing_queue_id or request.interaction_id.strip(), |
| ) |
| return { |
| "queued": True, |
| "duplicate": True, |
| "queue_id": existing_queue_id, |
| "status": str(existing.get("Submission_Status", "Pending Review")), |
| "synced_to_hub": synced_to_hub, |
| } |
|
|
| for column in new_entry: |
| if column not in df.columns: |
| df[column] = "" |
| row = {column: new_entry.get(column, "") for column in df.columns} |
| df.loc[len(df)] = row |
|
|
| temp_file = f"{pending_file}.tmp" |
| df.to_csv(temp_file, index=False) |
| os.replace(temp_file, pending_file) |
| synced_to_hub = _sync_pending_queue_to_hub(pending_file, queue_id) |
|
|
| return { |
| "queued": True, |
| "duplicate": False, |
| "queue_id": queue_id, |
| "status": "Pending Review", |
| "synced_to_hub": synced_to_hub, |
| } |
|
|
|
|
| @app.post("/api/polyglot-chat/submit") |
| def submit_polyglot_review(request: PolyglotReviewSubmission): |
| if not request.consent_confirmed: |
| raise HTTPException( |
| status_code=400, |
| detail="Explicit consent is required before a translation can enter pending review.", |
| ) |
| try: |
| return _append_polyglot_review(request) |
| except HTTPException: |
| raise |
| except Exception as exc: |
| print(f"Failed to submit reviewed Polyglot Chat entry: {exc}") |
| raise HTTPException(status_code=503, detail="Pending review submission failed.") from exc |
|
|
| @app.post("/api/translate", response_model=TranslationResponse) |
| async def translate_text(request: TranslationRequest): |
| source_label = f"{request.source_language} ({request.source_dialect})" |
| target_label = f"{request.target_language} ({request.target_dialect})" |
| route_client, route_model, route_node = resolve_ai_route(request.ai_model, source_label, target_label, request.text) |
| if not route_client: |
| raise HTTPException(status_code=500, detail="No LLM API key configured for Qwen, Llama/Groq, OpenRouter, DeepSeek, or Gemini.") |
|
|
| variety_instruction = "\n".join(filter(None, [ |
| nigerian_variety_instruction(source_label, target_label), |
| hausa_variety_instruction(source_label, target_label), |
| ])) |
|
|
| system_prompt = ( |
| f"You are an expert polyglot interpreter specializing in deep cultural and linguistic dialects.\n" |
| f"Translate the following text from {source_label} " |
| f"into {target_label}.\n" |
| f"Output ONLY the raw translated string. Do not include quotes, explanations, or thinking traces.\n" |
| f"Use the target language's native writing system. Korean, Jeju, and Satoori outputs must use Hangul only, not romanization and not Chinese or Japanese characters. " |
| f"Arabic outputs must use Arabic script. Igbo outputs must keep proper Igbo letters and tone/dot marks such as ị, ụ, ọ, ṅ, ẹ, á, and à where natural.\n" |
| f"{variety_instruction}" |
| ) |
| |
| try: |
| response = await route_client.chat.completions.create( |
| model=route_model, |
| messages=[ |
| {"role": "system", "content": system_prompt}, |
| {"role": "user", "content": request.text} |
| ], |
| temperature=0.3, |
| max_tokens=256 |
| ) |
| |
| translated_text = response.choices[0].message.content.strip() |
| boundary_reason = nigerian_variety_retry_reason(translated_text, target_label) |
| if boundary_reason: |
| retry_prompt = system_prompt + "\n" + nigerian_variety_retry_prompt( |
| request.text, source_label, target_label, translated_text, boundary_reason |
| ) |
| retry_response = await route_client.chat.completions.create( |
| model=route_model, |
| messages=[ |
| {"role": "system", "content": retry_prompt}, |
| {"role": "user", "content": request.text} |
| ], |
| temperature=0.2, |
| max_tokens=256 |
| ) |
| retry_text = retry_response.choices[0].message.content.strip() |
| if retry_text and not nigerian_variety_retry_reason(retry_text, target_label): |
| translated_text = retry_text |
| |
| return TranslationResponse( |
| original_text=request.text, |
| translated_text=translated_text, |
| target_dialect=f"{request.target_language} ({request.target_dialect})", |
| node=route_node |
| ) |
| |
| except Exception as e: |
| print(f"Error calling {route_node} API: {e}") |
| raise HTTPException(status_code=500, detail=str(e)) |
|
|
| @app.get("/api/health") |
| async def root(): |
| return {"message": f"PACYCx Hybrid Backend Online ({NODE_TYPE})"} |
|
|
| if __name__ == "__main__": |
| import uvicorn |
| uvicorn.run("api:app", host="0.0.0.0", port=8000, reload=True) |
|
|
