Delete script_analyzer.py

script_analyzer.py

@@ -1,462 +0,0 @@
-"""
-Advanced Script Analysis Engine
-Deep Unicode-based analysis: script distribution, mixing, directionality,
-transliteration detection, noise analysis, and linguistic features.
-"""
-from __future__ import annotations
-
-import re
-import unicodedata
-from collections import Counter, defaultdict
-from dataclasses import dataclass, field
-from typing import Dict, List, Optional, Set, Tuple
-
-import numpy as np
-from loguru import logger
-
-from src.utils.languages import (
-    SCRIPT_UNICODE_BLOCKS,
-    SCRIPT_TO_DIRECTION,
-    LANGUAGE_REGISTRY,
-)
-
-
-# ─────────────────────────────────────────────
-# Unicode Block Database
-# ─────────────────────────────────────────────
-
-UNICODE_BLOCK_NAMES: List[Tuple[int, int, str]] = [
-    (0x0000, 0x007F, "Basic Latin"),
-    (0x0080, 0x00FF, "Latin-1 Supplement"),
-    (0x0100, 0x017F, "Latin Extended-A"),
-    (0x0180, 0x024F, "Latin Extended-B"),
-    (0x0250, 0x02AF, "IPA Extensions"),
-    (0x0300, 0x036F, "Combining Diacritical Marks"),
-    (0x0370, 0x03FF, "Greek"),
-    (0x0400, 0x04FF, "Cyrillic"),
-    (0x0500, 0x052F, "Cyrillic Supplement"),
-    (0x0600, 0x06FF, "Arabic"),
-    (0x0700, 0x074F, "Syriac"),
-    (0x0750, 0x077F, "Arabic Supplement"),
-    (0x0900, 0x097F, "Devanagari"),
-    (0x0980, 0x09FF, "Bengali"),
-    (0x0A00, 0x0A7F, "Gurmukhi"),
-    (0x0A80, 0x0AFF, "Gujarati"),
-    (0x0B00, 0x0B7F, "Oriya"),
-    (0x0B80, 0x0BFF, "Tamil"),
-    (0x0C00, 0x0C7F, "Telugu"),
-    (0x0C80, 0x0CFF, "Kannada"),
-    (0x0D00, 0x0D7F, "Malayalam"),
-    (0x0D80, 0x0DFF, "Sinhala"),
-    (0x0E00, 0x0E7F, "Thai"),
-    (0x0F00, 0x0FFF, "Tibetan"),
-    (0x1000, 0x109F, "Myanmar"),
-    (0x10A0, 0x10FF, "Georgian"),
-    (0x1100, 0x11FF, "Hangul Jamo"),
-    (0x1E00, 0x1EFF, "Latin Extended Additional"),
-    (0x2000, 0x206F, "General Punctuation"),
-    (0x20A0, 0x20CF, "Currency Symbols"),
-    (0x2100, 0x214F, "Letterlike Symbols"),
-    (0x2200, 0x22FF, "Mathematical Operators"),
-    (0x3000, 0x303F, "CJK Symbols and Punctuation"),
-    (0x3040, 0x309F, "Hiragana"),
-    (0x30A0, 0x30FF, "Katakana"),
-    (0x4E00, 0x9FFF, "CJK Unified Ideographs"),
-    (0xFB00, 0xFB4F, "Alphabetic Presentation Forms"),
-    (0xFB50, 0xFDFF, "Arabic Presentation Forms-A"),
-    (0xFE70, 0xFEFF, "Arabic Presentation Forms-B"),
-    (0xFF00, 0xFFEF, "Halfwidth and Fullwidth Forms"),
-]
-
-
-def get_unicode_block(cp: int) -> str:
-    for lo, hi, name in UNICODE_BLOCK_NAMES:
-        if lo <= cp <= hi:
-            return name
-    return f"U+{cp:04X} (unknown block)"
-
-
-# ─────────────────────────────────────────────
-# Data Structures
-# ─────────────────────────────────────────────
-
-@dataclass
-class CharacterAnalysis:
-    char: str
-    codepoint: int
-    unicode_name: str
-    unicode_category: str
-    unicode_block: str
-    script: Optional[str]
-    is_alpha: bool
-    is_digit: bool
-    is_punct: bool
-    is_combining: bool
-
-
-@dataclass
-class ScriptSegment:
-    script: str
-    start: int
-    end: int
-    text: str
-    char_count: int
-
-
-@dataclass
-class ScriptAnalysisResult:
-    text: str
-    total_chars: int
-    alpha_chars: int
-    digit_chars: int
-    punct_chars: int
-    space_chars: int
-    other_chars: int
-
-    # Script distribution
-    script_distribution: Dict[str, float]
-    dominant_script: str
-    dominant_script_ratio: float
-
-    # Mixing analysis
-    is_mixed_script: bool
-    mixing_score: float           # 0=pure, 1=maximally mixed
-    script_segments: List[ScriptSegment]
-    script_transitions: int
-
-    # Directionality
-    primary_direction: str
-    has_bidi: bool
-    rtl_ratio: float
-
-    # Noise indicators
-    noise_score: float
-    has_zero_width: bool
-    has_control_chars: bool
-    has_excessive_punct: bool
-    url_count: int
-    emoji_count: int
-    latin_ratio: float            # in otherwise Indic text → potential transliteration
-
-    # Unicode statistics
-    unicode_block_distribution: Dict[str, float]
-    unique_chars: int
-    char_entropy: float
-
-    # Language hints
-    script_to_lang_hints: Dict[str, List[str]]
-
-    def to_dict(self) -> Dict:
-        return {
-            "total_chars": self.total_chars,
-            "alpha_chars": self.alpha_chars,
-            "alpha_ratio": self.alpha_chars / max(self.total_chars, 1),
-            "dominant_script": self.dominant_script,
-            "dominant_script_ratio": self.dominant_script_ratio,
-            "is_mixed_script": self.is_mixed_script,
-            "mixing_score": self.mixing_score,
-            "script_transitions": self.script_transitions,
-            "primary_direction": self.primary_direction,
-            "has_bidi": self.has_bidi,
-            "noise_score": self.noise_score,
-            "char_entropy": self.char_entropy,
-            "script_distribution": self.script_distribution,
-            "script_to_lang_hints": self.script_to_lang_hints,
-        }
-
-
-# ─────────────────────────────────────────────
-# Script Analyser
-# ─────────────────────────────────────────────
-
-class ScriptAnalyzer:
-    """
-    Deep script and Unicode analysis for multilingual text.
-    Designed for Indic languages with mixed-script handling.
-    """
-
-    _URL_RE = re.compile(r"https?://\S+|www\.\S+")
-    _EMOJI_RE = re.compile(
-        "["
-        "\U0001F600-\U0001F64F"
-        "\U0001F300-\U0001F5FF"
-        "\U0001F680-\U0001F6FF"
-        "\U0001F1E0-\U0001F1FF"
-        "\U00002702-\U000027B0"
-        "\U000024C2-\U0001F251"
-        "]+",
-        re.UNICODE
-    )
-    _ZW_RE = re.compile(r"[\u200b-\u200f\u202a-\u202e\ufeff\u00ad]")
-    _CTRL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]")
-
-    # Script → language hints
-    SCRIPT_LANG_HINTS: Dict[str, List[str]] = {
-        "Devanagari": ["hin", "mar"],
-        "Bengali": ["ben"],
-        "Gurmukhi": ["pan"],
-        "Gujarati": ["guj"],
-        "Oriya": ["ori"],
-        "Tamil": ["tam"],
-        "Telugu": ["tel"],
-        "Kannada": ["kan"],
-        "Malayalam": ["mal"],
-        "Arabic": ["urd"],
-        "Latin": ["eng"],
-    }
-
-    def analyze(self, text: str) -> ScriptAnalysisResult:
-        """Comprehensive script analysis of input text."""
-        if not text:
-            return self._empty_result(text)
-
-        chars = list(text)
-        n = len(chars)
-
-        # Character category counts
-        alpha = sum(c.isalpha() for c in chars)
-        digits = sum(c.isdigit() for c in chars)
-        spaces = sum(c.isspace() for c in chars)
-        puncts = sum(unicodedata.category(c).startswith("P") for c in chars)
-        other = n - alpha - digits - spaces - puncts
-
-        # Script assignment per alpha char
-        script_of: List[Optional[str]] = [self._assign_script(c) for c in chars]
-        alpha_scripts = [s for c, s in zip(chars, script_of) if c.isalpha() and s]
-
-        # Script distribution
-        script_counter = Counter(alpha_scripts)
-        total_alpha = max(len(alpha_scripts), 1)
-        script_dist = {s: c / total_alpha for s, c in script_counter.most_common()}
-
-        dominant = script_counter.most_common(1)[0][0] if script_counter else "Unknown"
-        dom_ratio = script_dist.get(dominant, 0.0)
-
-        # Segment extraction
-        segments = self._extract_segments(chars, script_of)
-        transitions = max(0, len(segments) - 1)
-
-        # Mixing score: entropy-based
-        mixing_score = self._script_entropy(script_dist)
-
-        # Directionality
-        rtl_langs = {"Arabic", "Hebrew"}
-        rtl_chars = sum(1 for s in alpha_scripts if s in rtl_langs)
-        rtl_ratio = rtl_chars / max(total_alpha, 1)
-        primary_direction = "RTL" if rtl_ratio > 0.5 else "LTR"
-        has_bidi = 0.05 < rtl_ratio < 0.95
-
-        # Unicode block distribution
-        block_counter: Counter = Counter()
-        for c in chars:
-            block = get_unicode_block(ord(c))
-            block_counter[block] += 1
-        block_dist = {b: c / n for b, c in block_counter.most_common(10)}
-
-        # Noise analysis
-        url_count = len(self._URL_RE.findall(text))
-        emoji_count = len(self._EMOJI_RE.findall(text))
-        has_zero_width = bool(self._ZW_RE.search(text))
-        has_control = bool(self._CTRL_RE.search(text))
-        has_excessive_punct = (puncts / max(n, 1)) > 0.15
-        latin_ratio = script_dist.get("Basic Latin", 0.0) + script_dist.get("Latin", 0.0)
-
-        noise_components = [
-            0.3 * min(1.0, url_count * 0.5),
-            0.2 * int(has_zero_width),
-            0.2 * int(has_control),
-            0.15 * int(has_excessive_punct),
-            0.15 * min(1.0, emoji_count * 0.2),
-        ]
-        noise_score = sum(noise_components)
-
-        # Char entropy
-        char_freq = Counter(text)
-        total_f = sum(char_freq.values())
-        entropy = -sum(
-            (c / total_f) * np.log2(c / total_f)
-            for c in char_freq.values() if c > 0
-        )
-
-        # Lang hints based on detected scripts
-        lang_hints: Dict[str, List[str]] = {
-            s: self.SCRIPT_LANG_HINTS.get(s, [])
-            for s in script_dist
-        }
-
-        return ScriptAnalysisResult(
-            text=text,
-            total_chars=n,
-            alpha_chars=alpha,
-            digit_chars=digits,
-            punct_chars=puncts,
-            space_chars=spaces,
-            other_chars=other,
-            script_distribution=script_dist,
-            dominant_script=dominant,
-            dominant_script_ratio=dom_ratio,
-            is_mixed_script=len(script_dist) > 1 and dom_ratio < 0.95,
-            mixing_score=mixing_score,
-            script_segments=segments,
-            script_transitions=transitions,
-            primary_direction=primary_direction,
-            has_bidi=has_bidi,
-            rtl_ratio=rtl_ratio,
-            noise_score=noise_score,
-            has_zero_width=has_zero_width,
-            has_control_chars=has_control,
-            has_excessive_punct=has_excessive_punct,
-            url_count=url_count,
-            emoji_count=emoji_count,
-            latin_ratio=latin_ratio,
-            unicode_block_distribution=block_dist,
-            unique_chars=len(set(text)),
-            char_entropy=float(entropy),
-            script_to_lang_hints=lang_hints,
-        )
-
-    def analyze_batch(self, texts: List[str]) -> List[ScriptAnalysisResult]:
-        return [self.analyze(t) for t in texts]
-
-    def _assign_script(self, char: str) -> Optional[str]:
-        if not char.isalpha():
-            return None
-        cp = ord(char)
-        for block_start, block_end, block_name in UNICODE_BLOCK_NAMES:
-            if block_start <= cp <= block_end:
-                return block_name
-        return "Unknown"
-
-    def _extract_segments(
-        self, chars: List[str], scripts: List[Optional[str]]
-    ) -> List[ScriptSegment]:
-        """Extract contiguous script segments from text."""
-        segments = []
-        if not chars:
-            return segments
-
-        current_script = None
-        start = 0
-        seg_chars = []
-
-        for i, (c, s) in enumerate(zip(chars, scripts)):
-            if not c.isalpha():
-                seg_chars.append(c)
-                continue
-            if s != current_script:
-                if current_script is not None and seg_chars:
-                    segments.append(ScriptSegment(
-                        script=current_script,
-                        start=start,
-                        end=i,
-                        text="".join(seg_chars),
-                        char_count=sum(1 for ch in seg_chars if ch.isalpha()),
-                    ))
-                current_script = s
-                start = i
-                seg_chars = [c]
-            else:
-                seg_chars.append(c)
-
-        if current_script and seg_chars:
-            segments.append(ScriptSegment(
-                script=current_script,
-                start=start,
-                end=len(chars),
-                text="".join(seg_chars),
-                char_count=sum(1 for ch in seg_chars if ch.isalpha()),
-            ))
-
-        return [s for s in segments if s.char_count >= 3]
-
-    def _script_entropy(self, dist: Dict[str, float]) -> float:
-        """Shannon entropy over script distribution (normalized to [0,1])."""
-        if len(dist) <= 1:
-            return 0.0
-        probs = list(dist.values())
-        entropy = -sum(p * np.log2(p) for p in probs if p > 0)
-        max_entropy = np.log2(len(probs))
-        return float(entropy / max_entropy) if max_entropy > 0 else 0.0
-
-    def _empty_result(self, text: str) -> ScriptAnalysisResult:
-        return ScriptAnalysisResult(
-            text=text, total_chars=0, alpha_chars=0, digit_chars=0,
-            punct_chars=0, space_chars=0, other_chars=0,
-            script_distribution={}, dominant_script="Unknown",
-            dominant_script_ratio=0.0, is_mixed_script=False,
-            mixing_score=0.0, script_segments=[], script_transitions=0,
-            primary_direction="LTR", has_bidi=False, rtl_ratio=0.0,
-            noise_score=0.0, has_zero_width=False, has_control_chars=False,
-            has_excessive_punct=False, url_count=0, emoji_count=0,
-            latin_ratio=0.0, unicode_block_distribution={},
-            unique_chars=0, char_entropy=0.0, script_to_lang_hints={},
-        )
-
-    def compare_texts(self, text1: str, text2: str) -> Dict:
-        """Compare script profiles of two texts."""
-        r1 = self.analyze(text1)
-        r2 = self.analyze(text2)
-
-        scripts1 = set(r1.script_distribution.keys())
-        scripts2 = set(r2.script_distribution.keys())
-        shared = scripts1 & scripts2
-
-        jaccard = len(shared) / max(len(scripts1 | scripts2), 1)
-
-        return {
-            "script_overlap": list(shared),
-            "jaccard_similarity": jaccard,
-            "dominant_scripts_match": r1.dominant_script == r2.dominant_script,
-            "text1_dominant": r1.dominant_script,
-            "text2_dominant": r2.dominant_script,
-            "text1_mixing_score": r1.mixing_score,
-            "text2_mixing_score": r2.mixing_score,
-        }
-
-    def detect_transliteration(self, text: str) -> Dict:
-        """
-        Detect if text is a transliteration of an Indic language
-        written in Latin script (Romanized Indic).
-        """
-        result = self.analyze(text)
-
-        # If predominantly Latin but contains Indic vocabulary patterns
-        latin_dominated = result.dominant_script in ("Basic Latin", "Latin Extended-A")
-        has_indic_patterns = self._has_indic_romanization_patterns(text)
-
-        # Common Romanization markers
-        has_aspirates = bool(re.search(r"\b\w*(kh|gh|ch|jh|th|dh|ph|bh)\w*\b", text, re.I))
-        has_retroflex = bool(re.search(r"\b\w*[ṭḍṇṣ]\w*\b", text))
-        has_nasal_markers = bool(re.search(r"\b\w*(ng|ny|ñ|ṅ)\w*\b", text, re.I))
-
-        is_likely_transliteration = (
-            latin_dominated and (has_indic_patterns or has_aspirates)
-        )
-
-        return {
-            "is_transliteration": is_likely_transliteration,
-            "confidence": 0.8 if (is_likely_transliteration and has_aspirates) else 0.5,
-            "evidence": {
-                "latin_dominated": latin_dominated,
-                "has_indic_patterns": has_indic_patterns,
-                "has_aspirates": has_aspirates,
-                "has_retroflex": has_retroflex,
-                "has_nasal_markers": has_nasal_markers,
-            }
-        }
-
-    def _has_indic_romanization_patterns(self, text: str) -> bool:
-        """Common romanization patterns across Indic languages."""
-        patterns = [
-            r"\b(hai|hain|nahin|nahi|aur|yeh|woh|kya|kaun|kahan|kyun|mera|tera|apna)\b",  # Hindi
-            r"\b(amma|appa|anna|akka|vanakkam|nandri|romba|paaru|paar)\b",   # Tamil
-            r"\b(ahe|aahe|nahi|tumhi|amhi|tya|tyanche|mhanje)\b",           # Marathi
-        ]
-        for p in patterns:
-            if re.search(p, text, re.I):
-                return True
-        return False
-
-
-# Singleton for reuse
-default_analyzer = ScriptAnalyzer()