| |
| |
| |
|
|
| import re |
| from typing import List, Dict, Any |
|
|
| from language_config import ( |
| classify_arabic_script, |
| detect_roman_urdu, |
| DEVANAGARI, |
| GURMUKHI, |
| has_arabic_script, |
| normalize_language, |
| SUPPORTED_LANGUAGES, |
| URDU_SHARED_PHRASES, |
| ) |
|
|
|
|
| def needs70B(prompt: str, detected_language: str) -> bool: |
| """ |
| Determine if the prompt requires the 70B model based on complexity. |
| Routes low-tier requests to 8B models to preserve real-time call performance. |
| """ |
| complex_keywords = [ |
| "analyze", "calculate", "reasoning", "explain step by step", |
| "complex", "complain", "refund", "manager", |
| ] |
| requires_complex_reasoning = any(keyword in prompt.lower() for keyword in complex_keywords) |
| return requires_complex_reasoning or normalize_language(detected_language) == "ur" |
|
|
|
|
| def detect_language_from_content(text: str) -> str: |
| """ |
| Detect language from text using script rules. Returns only 'en' or 'ur'. |
| Hindi (Devanagari) and Punjabi (Gurmukhi) are rerouted to 'ur'. |
| """ |
| if not text or not text.strip(): |
| return "en" |
|
|
| if DEVANAGARI.search(text) or GURMUKHI.search(text): |
| return "ur" |
|
|
| if has_arabic_script(text): |
| return "ur" |
|
|
| if detect_roman_urdu(text): |
| return "ur" |
|
|
| text_lower = text.lower() |
| for phrase in URDU_SHARED_PHRASES: |
| if phrase.lower() in text_lower: |
| return "ur" |
|
|
| return "en" |
|
|
|
|
| def get_arabic_clarification_flag(text: str) -> bool: |
| """Return True when Arabic script input needs a clarification prompt.""" |
| return classify_arabic_script(text) == "needs_clarification" |
|
|
|
|
| def estimate_tokens_from_text(text: str) -> int: |
| """Rough token estimator based on character count splits.""" |
| return max(1, (len(text) + 3) // 4) |
|
|
|
|
| def estimate_payload_tokens(payload: List[Dict[str, Any]]) -> int: |
| """Aggregate token runtime footings for structural context bounds.""" |
| total = 0 |
| for p in payload: |
| try: |
| total += estimate_tokens_from_text(str(p.get("content", ""))) |
| except Exception: |
| pass |
| return total |
|
|
|
|
| def strip_formatting(text: str) -> str: |
| """ |
| Remove code fences, inline blocks, markdown structures, headers, |
| parenthetical bracket groups, and extra spacing elements to format |
| raw strings cleanly for TTS processing. |
| """ |
| if not text: |
| return text |
| patterns = [ |
| r"```[\s\S]*?```", |
| r"`[^`]*`", |
| r"!\[[^\]]*\]\([^\)]*\)", |
| r"\[[^\]]*\]\([^\)]*\)", |
| r"\(.*?\)", |
| r"(^|\n)#{1,6}\s+", |
| r"\*\*([^*]+)\*\*", |
| r"\*([^*]+)\*", |
| r"__([^_]+)__", |
| r"_([^_]+)_", |
| r"\s{2,}", |
| ] |
| for pat in patterns: |
| text = re.sub(pat, " ", text, flags=re.MULTILINE) |
| return text.strip() |
|
|
