Fix spell checking issues: remove duplicates, auto-detect languages, handle hyphenated words, optimize allowlist

pdf_comparator.py

@@ -67,7 +67,10 @@ else:
 
 if HAS_SPELLCHECK:
     _SPELL_EN = SpellChecker(language="en")
-    _SPELL_FR = SpellChecker(language="fr")
+    try:
+        _SPELL_FR = SpellChecker(language="fr")
+    except Exception:
+        _SPELL_FR = None
 else:
     _SPELL_EN = None
     _SPELL_FR = None
@@ -76,10 +79,12 @@ _DOMAIN_ALLOWLIST = {
     "Furry", "Fox", "Packaging", "Digitaljoint", "ProofCheck", "PDF",
     "SKU", "SKUs", "ISO", "G7", "WebCenter", "Hybrid"
 }
+_DOMAIN_ALLOWLIST_LOWER = {w.lower() for w in _DOMAIN_ALLOWLIST}
 
-if _SPELL_EN and _SPELL_FR:
-    _SPELL_EN.word_frequency.load_words(w.lower() for w in _DOMAIN_ALLOWLIST)
-    _SPELL_FR.word_frequency.load_words(w.lower() for w in _DOMAIN_ALLOWLIST)
+if _SPELL_EN:
+    _SPELL_EN.word_frequency.load_words(_DOMAIN_ALLOWLIST_LOWER)
+if _SPELL_FR:
+    _SPELL_FR.word_frequency.load_words(_DOMAIN_ALLOWLIST_LOWER)
 
 def _normalize_text(s: str) -> str:
     s = unicodedata.normalize("NFC", s)
@@ -97,8 +102,15 @@ def _has_digits(tok: str) -> bool:
 
 def _is_known_word(tok: str) -> bool:
     t = tok.lower()
-    if t in (w.lower() for w in _DOMAIN_ALLOWLIST) or _looks_like_acronym(tok) or _has_digits(tok):
+    if t in _DOMAIN_ALLOWLIST_LOWER or _looks_like_acronym(tok) or _has_digits(tok):
         return True
+    
+    # Check hyphenated words - if any part is known, consider the whole word known
+    if '-' in tok:
+        parts = tok.split('-')
+        if all(_is_known_word(part) for part in parts):
+            return True
+    
     if _SPELL_EN and not _SPELL_EN.unknown([t]):  # known in EN
         return True
     if _SPELL_FR and not _SPELL_FR.unknown([t]):  # known in FR
@@ -281,38 +293,44 @@ def _looks_like_acronym(tok: str) -> bool:
 def _has_digits(tok: str) -> bool:
     return any(ch.isdigit() for ch in tok)
 
-def _is_known_word(tok: str) -> bool:
-    t = tok.lower()
-    if t in (w.lower() for w in _DOMAIN_ALLOWLIST) or _looks_like_acronym(tok) or _has_digits(tok):
-        return True
-    if not _SPELL_EN.unknown([t]):  # known in EN
-        return True
-    if not _SPELL_FR.unknown([t]):  # known in FR
-        return True
-    return False
-
 # (optional) keep a compatibility shim so any other code calling normalize_token() won't break
 def normalize_token(token: str) -> str:
     toks = _extract_tokens(token)
     return (toks[0].lower() if toks else "")
 
-
-def normalize_token(token: str) -> str:
-    cleaned = re.sub(r"[^A-Za-z']", "", token)
-    return cleaned.lower()
+def _get_available_tesseract_langs():
+    """Get available Tesseract languages"""
+    try:
+        langs = pytesseract.get_languages()
+        if 'eng' in langs and 'fra' in langs:
+            return "eng+fra"
+        elif 'eng' in langs:
+            return "eng"
+        elif langs:
+            return langs[0]
+        else:
+            return "eng"
+    except Exception:
+        return "eng"
 
 def find_misspell_boxes(
     img: Image.Image,
     *,
     min_conf: int = 60,
-    lang: str = "eng+fra",
+    lang: Optional[str] = None,
     extra_allow: Optional[Iterable[str]] = None
-) -> List["Box"]:
+) -> List[Box]:
     if not (HAS_OCR and HAS_SPELLCHECK):
         return []
+    
+    # Auto-detect language if not provided
+    if lang is None:
+        lang = _get_available_tesseract_langs()
+    
     try:
-        if extra_allow and _SPELL_EN and _SPELL_FR:
+        if extra_allow and _SPELL_EN:
             _SPELL_EN.word_frequency.load_words(w.lower() for w in extra_allow)
+        if extra_allow and _SPELL_FR:
             _SPELL_FR.word_frequency.load_words(w.lower() for w in extra_allow)
 
         data = pytesseract.image_to_data(
@@ -325,7 +343,7 @@ def find_misspell_boxes(
         return []
 
     n = len(data.get("text", [])) or 0
-    boxes: List["Box"] = []
+    boxes: List[Box] = []
 
     for i in range(n):
         raw = data["text"][i]