# gibberish_detector.py - V274.1 (Multi-line LaTeX Fix + SymPy Separation) # STABLE VERSION - No aggressive Hebrew removal! import re, copy, asyncio from typing import Tuple, Dict, Any, List print("[BIT-LOG: GibberishDetector V274.1 Loaded (Multi-line LaTeX Fix)]") # ==================== REPAIR MAPS ==================== # Simple replace fixes (Not regex) REPAIR_MAP = { '\\\\\\\\\\\\\\\\': '\\\\\\\\', '\\\\\\\\': '\\\\', '$$$': '$$', 'תוילקבמ': 'מקביליות', 'םיחטש': 'שטחים', 'הדוקנה': 'הנקודה', } # Broken LaTeX patterns (Regex) LATEX_FIXES = { r'\\rac\{': r'\\frac{', # Missing 'f' r'\\eta\{': r'\\beta{', # Wrong letter r'\\sqr\{': r'\\sqrt{', # Missing 't' r'(? dict: """ Advanced gibberish detection. Returns: { "has_issues": bool, "issues": [...] } """ issues = [] # 1. Reversed Hebrew for wrong, correct in REPAIR_MAP.items(): if wrong in text and len(wrong) > 3: issues.append({ "type": "REVERSED_HEBREW", "wrong": wrong, "correct": correct, "fixable": True }) # 2. Broken LaTeX for pattern, fix in LATEX_FIXES.items(): if re.search(pattern, text): issues.append({ "type": "BROKEN_LATEX", "pattern": pattern, "fix": fix, "fixable": True }) # 3. Hebrew inside $...$ hebrew_in_math = re.findall(r'\$[^$]*[\u0590-\u05FF][^$]*\$', text) for match in hebrew_in_math: issues.append({ "type": "HEBREW_IN_MATH", "line": match, "fixable": False, "needs_llm": True }) # 4. Double $$ issues if '$$$$' in text or '$$ $' in text: issues.append({ "type": "DOUBLE_DOLLARS", "fixable": True, "fix": "$$" }) return { "has_issues": len(issues) > 0, "issues": issues } # ==================== AUTO-FIX ==================== def fix_multiline_latex_blocks(text: str) -> str: """ V274.1: Fix multi-line LaTeX blocks that Flutter can't render. Converts: $$$$line1 line2 line3$$$$ To: $$line1$$ , $$line2$$ , $$line3$$ """ def split_block(match): content = match.group(1) lines = [line.strip() for line in content.split('\n') if line.strip()] if not lines: return "" # V274.1: Add logical comma separation between blocks to prevent SymPy squashing return ' , \n '.join(f'$${line}$$' for line in lines) return re.sub(r'\$\$\$\$(.*?)\$\$\$\$', split_block, text, flags=re.DOTALL) def auto_fix_gibberish(text: str) -> str: """Auto-fix simple gibberish issues with regex/replace.""" if not text: return text fixed = text # V274.0: Fix multi-line LaTeX blocks FIRST fixed = fix_multiline_latex_blocks(fixed) # Fix simple artifacts for wrong, correct in REPAIR_MAP.items(): fixed = fixed.replace(wrong, correct) # Fix broken LaTeX for pattern, replacement in LATEX_FIXES.items(): fixed = re.sub(pattern, replacement, fixed) # Fix dollar sign glitches fixed = fixed.replace('$$ $', '$$') fixed = fixed.replace('$ $$', '$$') return fixed # ==================== LLM RETRY ==================== async def fix_line_with_llm(problematic_line: str, llm_model) -> str: """Use LLM to fix complex gibberish (e.g., Hebrew in math).""" prompt = f"""FIX THIS LINE ONLY: Problematic: "{problematic_line}" Rules: 1. Hebrew text OUTSIDE $...$ 2. Math INSIDE $...$ 3. NO Hebrew inside math delimiters Output: Fixed line ONLY (one line, no explanation) """ try: response = await asyncio.wait_for( llm_model.generate_content_async(prompt), timeout=10.0 ) return response.text.strip() except Exception as e: print(f"⚠️ [GIBBERISH] LLM fix failed: {e}") return problematic_line # ==================== SMART FIX ==================== async def fix_gibberish_smart(text: str, llm_model=None) -> str: """Smart gibberish fixing.""" fixed = auto_fix_gibberish(text) result = detect_gibberish_advanced(fixed) if not result["has_issues"]: return fixed if llm_model: for issue in result["issues"]: if issue.get("needs_llm"): line = issue["line"] fixed_line = await fix_line_with_llm(line, llm_model) fixed = fixed.replace(line, fixed_line) return fixed # ==================== LEGACY INTERFACE ==================== def validate_and_fix_solution(solution: Dict[str, Any]) -> Tuple[Dict[str, Any], bool, str]: """Legacy interface - now uses auto-fix""" fixed = copy.deepcopy(solution) def process(obj, field=""): if isinstance(obj, dict): return {k: process(v, k) for k, v in obj.items()} if isinstance(obj, list): return [process(i, field) for i in obj] if isinstance(obj, str): return auto_fix_gibberish(_fix_string(obj, field)) return obj return process(fixed), False, "" def _fix_string(text: str, field: str) -> str: """Legacy string fixing""" if not text or text.lower() in ["none", "null"]: return text res = text.replace(r'\f\frac', r'\frac') # V231.2: Skip $$ wrapping for :: format if '::' in res: for bad, good in REPAIR_MAP.items(): res = res.replace(bad, good) return res # Hebrew-Safe $$ Rule — only for pure-math fields if '\\' in res and '$$' not in res and field in MATH_ALLOWED_FIELDS: if not bool(re.search(r'[\u0590-\u05FF]', res)): res = f"$${res.replace('$', '')}$$" for bad, good in REPAIR_MAP.items(): res = res.replace(bad, good) return res def format_solution(data): return data # ==================== USAGE EXAMPLE ==================== if __name__ == "__main__": test_text = "נשתמש בנוסחה $x = \\frac{1}{2}$ ונקבל \\rac{x}{y}" result = detect_gibberish_advanced(test_text) print(f"Issues found: {result}") fixed = auto_fix_gibberish(test_text) print(f"Auto-fixed: {fixed}")