# utils.py # Core helper tools for character scripting, text tokenization estimates, # model complexity routing, and speech-delivery formatting overrides. import re from typing import List, Dict, Any from language_config import ( classify_arabic_script, detect_roman_urdu, DEVANAGARI, GURMUKHI, has_arabic_script, normalize_language, SUPPORTED_LANGUAGES, URDU_SHARED_PHRASES, ) def needs70B(prompt: str, detected_language: str) -> bool: """ Determine if the prompt requires the 70B model based on complexity. Routes low-tier requests to 8B models to preserve real-time call performance. """ complex_keywords = [ "analyze", "calculate", "reasoning", "explain step by step", "complex", "complain", "refund", "manager", ] requires_complex_reasoning = any(keyword in prompt.lower() for keyword in complex_keywords) return requires_complex_reasoning or normalize_language(detected_language) == "ur" def detect_language_from_content(text: str) -> str: """ Detect language from text using script rules. Returns only 'en' or 'ur'. Hindi (Devanagari) and Punjabi (Gurmukhi) are rerouted to 'ur'. """ if not text or not text.strip(): return "en" if DEVANAGARI.search(text) or GURMUKHI.search(text): return "ur" if has_arabic_script(text): return "ur" if detect_roman_urdu(text): return "ur" text_lower = text.lower() for phrase in URDU_SHARED_PHRASES: if phrase.lower() in text_lower: return "ur" return "en" def get_arabic_clarification_flag(text: str) -> bool: """Return True when Arabic script input needs a clarification prompt.""" return classify_arabic_script(text) == "needs_clarification" def estimate_tokens_from_text(text: str) -> int: """Rough token estimator based on character count splits.""" return max(1, (len(text) + 3) // 4) def estimate_payload_tokens(payload: List[Dict[str, Any]]) -> int: """Aggregate token runtime footings for structural context bounds.""" total = 0 for p in payload: try: total += estimate_tokens_from_text(str(p.get("content", ""))) except Exception: pass return total def strip_formatting(text: str) -> str: """ Remove code fences, inline blocks, markdown structures, headers, parenthetical bracket groups, and extra spacing elements to format raw strings cleanly for TTS processing. """ if not text: return text patterns = [ r"```[\s\S]*?```", r"`[^`]*`", r"!\[[^\]]*\]\([^\)]*\)", r"\[[^\]]*\]\([^\)]*\)", r"\(.*?\)", r"(^|\n)#{1,6}\s+", r"\*\*([^*]+)\*\*", r"\*([^*]+)\*", r"__([^_]+)__", r"_([^_]+)_", r"\s{2,}", ] for pat in patterns: text = re.sub(pat, " ", text, flags=re.MULTILINE) return text.strip()