gibberish_detector.py

# gibberish_detector.py - V274.1 (Multi-line LaTeX Fix + SymPy Separation)
# STABLE VERSION - No aggressive Hebrew removal!
import re, copy, asyncio
from typing import Tuple, Dict, Any, List

print("[BIT-LOG: GibberishDetector V274.1 Loaded (Multi-line LaTeX Fix)]")

# ==================== REPAIR MAPS ====================

# Simple replace fixes (Not regex)
REPAIR_MAP = {
    '\\\\\\\\\\\\\\\\': '\\\\\\\\',
    '\\\\\\\\': '\\\\',
    '$$$': '$$',
    'תוילקבמ': 'מקביליות',
    'םיחטש': 'שטחים',
    'הדוקנה': 'הנקודה',
}

# Broken LaTeX patterns (Regex)
LATEX_FIXES = {
    r'\\rac\{': r'\\frac{',  # Missing 'f'
    r'\\eta\{': r'\\beta{',  # Wrong letter
    r'\\sqr\{': r'\\sqrt{',  # Missing 't'
    r'(?<!\\)\b(sin|cos|tan|log|ln)\b': r'\\\1', # Missing backslash for funcs
    r'(?<!\\)\b(alpha|beta|gamma|cdot)\b': r'\\\1', # Missing backslash for greek/symbols
}

PROTECTED_FIELDS = {'section_title', 'title', 'teacher_tip', 'teacher_closing'}
MATH_ALLOWED_FIELDS = {'block_math', 'formulas', 'final_answer', 'function', 'derivative'}

# ==================== DETECTION ====================

def detect_gibberish_advanced(text: str) -> dict:
    """
    Advanced gibberish detection.
    
    Returns:
        {
            "has_issues": bool,
            "issues": [...]
        }
    """
    issues = []
    
    # 1. Reversed Hebrew
    for wrong, correct in REPAIR_MAP.items():
        if wrong in text and len(wrong) > 3:
            issues.append({
                "type": "REVERSED_HEBREW",
                "wrong": wrong,
                "correct": correct,
                "fixable": True
            })
    
    # 2. Broken LaTeX
    for pattern, fix in LATEX_FIXES.items():
        if re.search(pattern, text):
            issues.append({
                "type": "BROKEN_LATEX",
                "pattern": pattern,
                "fix": fix,
                "fixable": True
            })
    
    # 3. Hebrew inside $...$
    hebrew_in_math = re.findall(r'\$[^$]*[\u0590-\u05FF][^$]*\$', text)
    for match in hebrew_in_math:
        issues.append({
            "type": "HEBREW_IN_MATH",
            "line": match,
            "fixable": False,
            "needs_llm": True
        })
    
    # 4. Double $$ issues
    if '$$$$' in text or '$$ $' in text:
        issues.append({
            "type": "DOUBLE_DOLLARS",
            "fixable": True,
            "fix": "$$"
        })
    
    return {
        "has_issues": len(issues) > 0,
        "issues": issues
    }


# ==================== AUTO-FIX ====================

def fix_multiline_latex_blocks(text: str) -> str:
    """
    V274.1: Fix multi-line LaTeX blocks that Flutter can't render.
    
    Converts:
      $$$$line1
      line2
      line3$$$$
    
    To:
      $$line1$$ ,
      $$line2$$ ,
      $$line3$$
    """
    def split_block(match):
        content = match.group(1)
        lines = [line.strip() for line in content.split('\n') if line.strip()]
        if not lines: return ""
        # V274.1: Add logical comma separation between blocks to prevent SymPy squashing
        return ' , \n '.join(f'$${line}$$' for line in lines)
    
    return re.sub(r'\$\$\$\$(.*?)\$\$\$\$', split_block, text, flags=re.DOTALL)


def auto_fix_gibberish(text: str) -> str:
    """Auto-fix simple gibberish issues with regex/replace."""
    if not text:
        return text

    fixed = text
    
    # V274.0: Fix multi-line LaTeX blocks FIRST
    fixed = fix_multiline_latex_blocks(fixed)
    
    # Fix simple artifacts
    for wrong, correct in REPAIR_MAP.items():
        fixed = fixed.replace(wrong, correct)
    
    # Fix broken LaTeX
    for pattern, replacement in LATEX_FIXES.items():
        fixed = re.sub(pattern, replacement, fixed)
    
    # Fix dollar sign glitches
    fixed = fixed.replace('$$ $', '$$')
    fixed = fixed.replace('$ $$', '$$')
    
    return fixed


# ==================== LLM RETRY ====================

async def fix_line_with_llm(problematic_line: str, llm_model) -> str:
    """Use LLM to fix complex gibberish (e.g., Hebrew in math)."""
    prompt = f"""FIX THIS LINE ONLY:
Problematic: "{problematic_line}"

Rules:
1. Hebrew text OUTSIDE $...$
2. Math INSIDE $...$
3. NO Hebrew inside math delimiters

Output: Fixed line ONLY (one line, no explanation)
"""
    
    try:
        response = await asyncio.wait_for(
            llm_model.generate_content_async(prompt),
            timeout=10.0
        )
        return response.text.strip()
    except Exception as e:
        print(f"⚠️ [GIBBERISH] LLM fix failed: {e}")
        return problematic_line


# ==================== SMART FIX ====================

async def fix_gibberish_smart(text: str, llm_model=None) -> str:
    """Smart gibberish fixing."""
    fixed = auto_fix_gibberish(text)
    result = detect_gibberish_advanced(fixed)
    
    if not result["has_issues"]:
        return fixed
    
    if llm_model:
        for issue in result["issues"]:
            if issue.get("needs_llm"):
                line = issue["line"]
                fixed_line = await fix_line_with_llm(line, llm_model)
                fixed = fixed.replace(line, fixed_line)
    
    return fixed


# ==================== LEGACY INTERFACE ====================

def validate_and_fix_solution(solution: Dict[str, Any]) -> Tuple[Dict[str, Any], bool, str]:
    """Legacy interface - now uses auto-fix"""
    fixed = copy.deepcopy(solution)
    
    def process(obj, field=""):
        if isinstance(obj, dict):
            return {k: process(v, k) for k, v in obj.items()}
        if isinstance(obj, list):
            return [process(i, field) for i in obj]
        if isinstance(obj, str):
            return auto_fix_gibberish(_fix_string(obj, field))
        return obj
    
    return process(fixed), False, ""


def _fix_string(text: str, field: str) -> str:
    """Legacy string fixing"""
    if not text or text.lower() in ["none", "null"]:
        return text
    
    res = text.replace(r'\f\frac', r'\frac')
    
    # V231.2: Skip $$ wrapping for :: format
    if '::' in res:
        for bad, good in REPAIR_MAP.items():
            res = res.replace(bad, good)
        return res
    
    # Hebrew-Safe $$ Rule — only for pure-math fields
    if '\\' in res and '$$' not in res and field in MATH_ALLOWED_FIELDS:
        if not bool(re.search(r'[\u0590-\u05FF]', res)):
            res = f"$${res.replace('$', '')}$$"
    
    for bad, good in REPAIR_MAP.items():
        res = res.replace(bad, good)
    
    return res


def format_solution(data):
    return data


# ==================== USAGE EXAMPLE ====================

if __name__ == "__main__":
    test_text = "נשתמש בנוסחה $x = \\frac{1}{2}$ ונקבל \\rac{x}{y}"
    
    result = detect_gibberish_advanced(test_text)
    print(f"Issues found: {result}")
    
    fixed = auto_fix_gibberish(test_text)
    print(f"Auto-fixed: {fixed}")