Spaces:
Sleeping
Sleeping
Pointf5ive commited on
Commit Β·
7152efa
1
Parent(s): d109898
Add persistent book page scope, safe titles, and OCR stall guards
Browse files- smoke_signal/scripts/font_library.py +423 -0
- smoke_signal/scripts/punct_corrector.py +43 -13
- smoke_signal/tessdata/README.md +22 -0
- smoke_signal_tab.py +447 -33
smoke_signal/scripts/font_library.py
ADDED
|
@@ -0,0 +1,423 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Smoke Signal β Font Detection + Font-Aware OCR Helpers
|
| 3 |
+
======================================================
|
| 4 |
+
Lightweight utilities for:
|
| 5 |
+
1) identifying page fonts via MixFont Lens API
|
| 6 |
+
2) maintaining per-font OCR preference metadata
|
| 7 |
+
3) selecting per-font custom Tesseract models when available
|
| 8 |
+
4) exposing font-specific punctuation-map paths
|
| 9 |
+
|
| 10 |
+
This module is intentionally defensive:
|
| 11 |
+
- if MixFont key is missing, it degrades gracefully
|
| 12 |
+
- if no custom .traineddata exists, it falls back to "eng"
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import json
|
| 18 |
+
import os
|
| 19 |
+
import urllib.parse
|
| 20 |
+
import urllib.request
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
from typing import Optional
|
| 23 |
+
|
| 24 |
+
SS_ROOT = Path(os.environ.get("SS_DATA_ROOT", os.environ.get("SS_ROOT", "/tmp/smoke_signal")))
|
| 25 |
+
CALIBRATION_DIR = SS_ROOT / "calibration"
|
| 26 |
+
FONTS_DIR = SS_ROOT / "fonts"
|
| 27 |
+
TESSDATA_DIR = SS_ROOT / "tessdata"
|
| 28 |
+
REPO_TESSDATA_DIR = Path(__file__).resolve().parents[1] / "tessdata"
|
| 29 |
+
FONT_REGISTRY_PATH = CALIBRATION_DIR / "font_registry.json"
|
| 30 |
+
|
| 31 |
+
for _dir in (CALIBRATION_DIR, FONTS_DIR, TESSDATA_DIR, REPO_TESSDATA_DIR):
|
| 32 |
+
_dir.mkdir(parents=True, exist_ok=True)
|
| 33 |
+
|
| 34 |
+
MIXFONT_API_KEY = os.environ.get("MIXFONT_API_KEY", "").strip()
|
| 35 |
+
# MixFont Lens API docs endpoint.
|
| 36 |
+
MIXFONT_API_URL = os.environ.get("MIXFONT_API_URL", "https://www.mixfont.com/v1/api/lens").strip()
|
| 37 |
+
try:
|
| 38 |
+
MIXFONT_TIMEOUT_SEC = max(1.0, float(os.environ.get("MIXFONT_TIMEOUT_SEC", "4")))
|
| 39 |
+
except Exception:
|
| 40 |
+
MIXFONT_TIMEOUT_SEC = 4.0
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def _slug_font_name(font_name: str) -> str:
|
| 44 |
+
cleaned = "".join(ch.lower() if ch.isalnum() else "_" for ch in (font_name or "").strip())
|
| 45 |
+
while "__" in cleaned:
|
| 46 |
+
cleaned = cleaned.replace("__", "_")
|
| 47 |
+
return cleaned.strip("_")
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _load_registry() -> dict:
|
| 51 |
+
if not FONT_REGISTRY_PATH.exists():
|
| 52 |
+
return {}
|
| 53 |
+
try:
|
| 54 |
+
return json.loads(FONT_REGISTRY_PATH.read_text(encoding="utf-8"))
|
| 55 |
+
except Exception:
|
| 56 |
+
return {}
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _save_registry(registry: dict) -> None:
|
| 60 |
+
FONT_REGISTRY_PATH.write_text(
|
| 61 |
+
json.dumps(registry, indent=2, ensure_ascii=False),
|
| 62 |
+
encoding="utf-8",
|
| 63 |
+
)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _ensure_registry_entry(font_name: str) -> dict:
|
| 67 |
+
registry = _load_registry()
|
| 68 |
+
entry = registry.get(font_name, {})
|
| 69 |
+
if not entry:
|
| 70 |
+
entry = {
|
| 71 |
+
"identified_count": 0,
|
| 72 |
+
"status": "unknown", # unknown | available | missing | specimen_only
|
| 73 |
+
"preferred_engine": "surya", # surya | tesseract
|
| 74 |
+
"tesseract_model": None,
|
| 75 |
+
"avg_confidence_surya": None,
|
| 76 |
+
"avg_confidence_tess": None,
|
| 77 |
+
"last_identified_at": None,
|
| 78 |
+
}
|
| 79 |
+
registry[font_name] = entry
|
| 80 |
+
_save_registry(registry)
|
| 81 |
+
return entry
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def _build_public_image_url(local_image_path: str) -> Optional[str]:
|
| 85 |
+
"""
|
| 86 |
+
Construct a publicly reachable URL for a local rendered page image.
|
| 87 |
+
Priority:
|
| 88 |
+
1) MIXFONT_IMAGE_URL_TEMPLATE with {path} placeholder
|
| 89 |
+
2) MIXFONT_IMAGE_BASE_URL + /gradio_api/file=<abs_path>
|
| 90 |
+
3) SPACE_HOST + /gradio_api/file=<abs_path>
|
| 91 |
+
4) HF_SPACE_URL + /gradio_api/file=<abs_path>
|
| 92 |
+
"""
|
| 93 |
+
raw_path = str(Path(local_image_path).resolve())
|
| 94 |
+
encoded_path = urllib.parse.quote(raw_path, safe="")
|
| 95 |
+
|
| 96 |
+
template = os.environ.get("MIXFONT_IMAGE_URL_TEMPLATE", "").strip()
|
| 97 |
+
if template:
|
| 98 |
+
if "{path}" in template:
|
| 99 |
+
return template.replace("{path}", encoded_path)
|
| 100 |
+
return template
|
| 101 |
+
|
| 102 |
+
base = (
|
| 103 |
+
os.environ.get("MIXFONT_IMAGE_BASE_URL", "").strip()
|
| 104 |
+
or os.environ.get("HF_SPACE_URL", "").strip()
|
| 105 |
+
or ""
|
| 106 |
+
)
|
| 107 |
+
if not base:
|
| 108 |
+
space_id = os.environ.get("SPACE_ID", "").strip()
|
| 109 |
+
if space_id:
|
| 110 |
+
base = f"https://huggingface.co/spaces/{space_id}"
|
| 111 |
+
if not base:
|
| 112 |
+
space_host = os.environ.get("SPACE_HOST", "").strip()
|
| 113 |
+
if space_host:
|
| 114 |
+
if space_host.startswith("http://") or space_host.startswith("https://"):
|
| 115 |
+
base = space_host
|
| 116 |
+
else:
|
| 117 |
+
base = f"https://{space_host}"
|
| 118 |
+
|
| 119 |
+
if not base:
|
| 120 |
+
return None
|
| 121 |
+
|
| 122 |
+
return f"{base.rstrip('/')}/gradio_api/file={encoded_path}"
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def mixfont_preflight() -> dict:
|
| 126 |
+
"""
|
| 127 |
+
Return runtime readiness checks for MixFont integration.
|
| 128 |
+
"""
|
| 129 |
+
template = os.environ.get("MIXFONT_IMAGE_URL_TEMPLATE", "").strip()
|
| 130 |
+
base = (
|
| 131 |
+
os.environ.get("MIXFONT_IMAGE_BASE_URL", "").strip()
|
| 132 |
+
or os.environ.get("HF_SPACE_URL", "").strip()
|
| 133 |
+
or ""
|
| 134 |
+
)
|
| 135 |
+
if not base:
|
| 136 |
+
space_id = os.environ.get("SPACE_ID", "").strip()
|
| 137 |
+
if space_id:
|
| 138 |
+
base = f"https://huggingface.co/spaces/{space_id}"
|
| 139 |
+
space_host = os.environ.get("SPACE_HOST", "").strip()
|
| 140 |
+
has_public_source = bool(template or base or space_host)
|
| 141 |
+
return {
|
| 142 |
+
"api_key_set": bool(MIXFONT_API_KEY),
|
| 143 |
+
"api_url": MIXFONT_API_URL,
|
| 144 |
+
"image_url_template_set": bool(template),
|
| 145 |
+
"image_base_set": bool(base),
|
| 146 |
+
"space_host_set": bool(space_host),
|
| 147 |
+
"public_image_url_source_available": has_public_source,
|
| 148 |
+
}
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def identify_page_font(image_path: str, image_url: Optional[str] = None) -> dict:
|
| 152 |
+
"""
|
| 153 |
+
Identify font via MixFont Lens API.
|
| 154 |
+
Returns:
|
| 155 |
+
{
|
| 156 |
+
"font_name": str|None,
|
| 157 |
+
"confidence": float,
|
| 158 |
+
"alternatives": list[str],
|
| 159 |
+
"image_url": str|None,
|
| 160 |
+
"error": str|None
|
| 161 |
+
}
|
| 162 |
+
"""
|
| 163 |
+
if not MIXFONT_API_KEY:
|
| 164 |
+
return {
|
| 165 |
+
"font_name": None,
|
| 166 |
+
"confidence": 0.0,
|
| 167 |
+
"alternatives": [],
|
| 168 |
+
"image_url": image_url,
|
| 169 |
+
"error": "MIXFONT_API_KEY not set",
|
| 170 |
+
}
|
| 171 |
+
|
| 172 |
+
path_obj = Path(image_path)
|
| 173 |
+
if not path_obj.exists():
|
| 174 |
+
return {
|
| 175 |
+
"font_name": None,
|
| 176 |
+
"confidence": 0.0,
|
| 177 |
+
"alternatives": [],
|
| 178 |
+
"image_url": image_url,
|
| 179 |
+
"error": f"image not found: {image_path}",
|
| 180 |
+
}
|
| 181 |
+
|
| 182 |
+
resolved_url = image_url or _build_public_image_url(str(path_obj))
|
| 183 |
+
if not resolved_url:
|
| 184 |
+
return {
|
| 185 |
+
"font_name": None,
|
| 186 |
+
"confidence": 0.0,
|
| 187 |
+
"alternatives": [],
|
| 188 |
+
"image_url": None,
|
| 189 |
+
"error": "no public image URL (set MIXFONT_IMAGE_BASE_URL or MIXFONT_IMAGE_URL_TEMPLATE)",
|
| 190 |
+
}
|
| 191 |
+
|
| 192 |
+
payload = json.dumps({"image_url": resolved_url}).encode("utf-8")
|
| 193 |
+
request = urllib.request.Request(
|
| 194 |
+
MIXFONT_API_URL,
|
| 195 |
+
data=payload,
|
| 196 |
+
method="POST",
|
| 197 |
+
headers={
|
| 198 |
+
"Content-Type": "application/json",
|
| 199 |
+
"x-api-key": MIXFONT_API_KEY,
|
| 200 |
+
},
|
| 201 |
+
)
|
| 202 |
+
|
| 203 |
+
try:
|
| 204 |
+
with urllib.request.urlopen(request, timeout=MIXFONT_TIMEOUT_SEC) as response:
|
| 205 |
+
body = response.read().decode("utf-8", errors="replace")
|
| 206 |
+
data = json.loads(body)
|
| 207 |
+
except Exception as e:
|
| 208 |
+
return {
|
| 209 |
+
"font_name": None,
|
| 210 |
+
"confidence": 0.0,
|
| 211 |
+
"alternatives": [],
|
| 212 |
+
"image_url": resolved_url,
|
| 213 |
+
"error": str(e),
|
| 214 |
+
}
|
| 215 |
+
|
| 216 |
+
matches = data.get("font_matches", [])
|
| 217 |
+
if not isinstance(matches, list):
|
| 218 |
+
matches = []
|
| 219 |
+
|
| 220 |
+
top = matches[0] if matches else {}
|
| 221 |
+
if not isinstance(top, dict):
|
| 222 |
+
top = {}
|
| 223 |
+
|
| 224 |
+
font_name = str(top.get("name") or "").strip() or None
|
| 225 |
+
try:
|
| 226 |
+
confidence = float(top.get("confidence", 0.0))
|
| 227 |
+
except Exception:
|
| 228 |
+
confidence = 0.0
|
| 229 |
+
|
| 230 |
+
alternatives = []
|
| 231 |
+
for match in matches[1:4]:
|
| 232 |
+
if isinstance(match, dict):
|
| 233 |
+
name = str(match.get("name") or "").strip()
|
| 234 |
+
if name:
|
| 235 |
+
alternatives.append(name)
|
| 236 |
+
|
| 237 |
+
if font_name:
|
| 238 |
+
registry = _load_registry()
|
| 239 |
+
entry = registry.get(font_name, _ensure_registry_entry(font_name))
|
| 240 |
+
entry["identified_count"] = int(entry.get("identified_count", 0)) + 1
|
| 241 |
+
entry["last_identified_at"] = str(data.get("datetime") or "")
|
| 242 |
+
registry[font_name] = entry
|
| 243 |
+
_save_registry(registry)
|
| 244 |
+
|
| 245 |
+
return {
|
| 246 |
+
"font_name": font_name,
|
| 247 |
+
"confidence": round(confidence, 4),
|
| 248 |
+
"alternatives": alternatives,
|
| 249 |
+
"image_url": resolved_url,
|
| 250 |
+
"error": None,
|
| 251 |
+
}
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def register_font_model(
|
| 255 |
+
font_name: str,
|
| 256 |
+
model_name: Optional[str] = None,
|
| 257 |
+
preferred_engine: str = "tesseract",
|
| 258 |
+
status: str = "available",
|
| 259 |
+
) -> dict:
|
| 260 |
+
"""
|
| 261 |
+
Manually register a font->tesseract model mapping.
|
| 262 |
+
Use this for bespoke fonts where only a specimen-based traineddata exists.
|
| 263 |
+
"""
|
| 264 |
+
if not font_name:
|
| 265 |
+
return {"ok": False, "error": "font_name required"}
|
| 266 |
+
|
| 267 |
+
model = (model_name or _slug_font_name(font_name) or "eng").strip()
|
| 268 |
+
registry = _load_registry()
|
| 269 |
+
entry = registry.get(font_name, _ensure_registry_entry(font_name))
|
| 270 |
+
entry["tesseract_model"] = model
|
| 271 |
+
entry["preferred_engine"] = preferred_engine if preferred_engine in {"surya", "tesseract"} else "tesseract"
|
| 272 |
+
entry["status"] = status
|
| 273 |
+
registry[font_name] = entry
|
| 274 |
+
_save_registry(registry)
|
| 275 |
+
return {"ok": True, "font_name": font_name, "tesseract_model": model}
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
def resolve_tesseract_lang(font_name: Optional[str]) -> str:
|
| 279 |
+
"""
|
| 280 |
+
Return best tesseract language/model to use for this font.
|
| 281 |
+
Defaults to "eng" if no custom traineddata is available.
|
| 282 |
+
"""
|
| 283 |
+
if not font_name:
|
| 284 |
+
return "eng"
|
| 285 |
+
|
| 286 |
+
registry = _load_registry()
|
| 287 |
+
entry = registry.get(font_name, {})
|
| 288 |
+
candidates = []
|
| 289 |
+
|
| 290 |
+
explicit_model = str(entry.get("tesseract_model") or "").strip()
|
| 291 |
+
if explicit_model:
|
| 292 |
+
candidates.append(explicit_model)
|
| 293 |
+
|
| 294 |
+
slug = _slug_font_name(font_name)
|
| 295 |
+
if slug:
|
| 296 |
+
candidates.append(slug)
|
| 297 |
+
candidates.append(slug.replace("_", ""))
|
| 298 |
+
|
| 299 |
+
# unique keep order
|
| 300 |
+
seen = set()
|
| 301 |
+
deduped = []
|
| 302 |
+
for c in candidates:
|
| 303 |
+
if c and c not in seen:
|
| 304 |
+
deduped.append(c)
|
| 305 |
+
seen.add(c)
|
| 306 |
+
|
| 307 |
+
tess_dirs = [TESSDATA_DIR, REPO_TESSDATA_DIR]
|
| 308 |
+
for model_name in deduped:
|
| 309 |
+
if any((tess_dir / f"{model_name}.traineddata").exists() for tess_dir in tess_dirs):
|
| 310 |
+
if entry:
|
| 311 |
+
entry["status"] = "available"
|
| 312 |
+
entry["preferred_engine"] = "tesseract"
|
| 313 |
+
entry["tesseract_model"] = model_name
|
| 314 |
+
registry[font_name] = entry
|
| 315 |
+
_save_registry(registry)
|
| 316 |
+
return model_name
|
| 317 |
+
|
| 318 |
+
# Common misplacement: .traineddata.otf (font file, not Tesseract model)
|
| 319 |
+
otf_like_candidates = []
|
| 320 |
+
for model_name in deduped:
|
| 321 |
+
otf_like_candidates.extend(
|
| 322 |
+
[
|
| 323 |
+
REPO_TESSDATA_DIR / f"{model_name}.traineddata.otf",
|
| 324 |
+
TESSDATA_DIR / f"{model_name}.traineddata.otf",
|
| 325 |
+
REPO_TESSDATA_DIR / f"{model_name}.otf",
|
| 326 |
+
TESSDATA_DIR / f"{model_name}.otf",
|
| 327 |
+
]
|
| 328 |
+
)
|
| 329 |
+
if any(path.exists() for path in otf_like_candidates):
|
| 330 |
+
if entry:
|
| 331 |
+
entry["status"] = "font_file_only"
|
| 332 |
+
entry["preferred_engine"] = "surya"
|
| 333 |
+
registry[font_name] = entry
|
| 334 |
+
_save_registry(registry)
|
| 335 |
+
|
| 336 |
+
# If a specimen exists but no model compiled yet, make that explicit in registry.
|
| 337 |
+
if (FONTS_DIR / "gruffalo_specimen.png").exists():
|
| 338 |
+
if entry:
|
| 339 |
+
entry["status"] = entry.get("status") or "specimen_only"
|
| 340 |
+
if entry["status"] == "unknown":
|
| 341 |
+
entry["status"] = "specimen_only"
|
| 342 |
+
registry[font_name] = entry
|
| 343 |
+
_save_registry(registry)
|
| 344 |
+
|
| 345 |
+
return "eng"
|
| 346 |
+
|
| 347 |
+
|
| 348 |
+
def resolve_tessdata_dir(model_name: Optional[str]) -> Optional[str]:
|
| 349 |
+
if not model_name:
|
| 350 |
+
return None
|
| 351 |
+
model = str(model_name).strip()
|
| 352 |
+
if not model:
|
| 353 |
+
return None
|
| 354 |
+
for tess_dir in (TESSDATA_DIR, REPO_TESSDATA_DIR):
|
| 355 |
+
if (tess_dir / f"{model}.traineddata").exists():
|
| 356 |
+
return str(tess_dir)
|
| 357 |
+
return None
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
def update_font_engine_stats(font_name: Optional[str], surya_conf: float, tess_conf: float) -> None:
|
| 361 |
+
if not font_name:
|
| 362 |
+
return
|
| 363 |
+
registry = _load_registry()
|
| 364 |
+
entry = registry.get(font_name, _ensure_registry_entry(font_name))
|
| 365 |
+
alpha = 0.3
|
| 366 |
+
|
| 367 |
+
prev_surya = entry.get("avg_confidence_surya")
|
| 368 |
+
prev_tess = entry.get("avg_confidence_tess")
|
| 369 |
+
try:
|
| 370 |
+
prev_surya_val = float(prev_surya) if prev_surya is not None else None
|
| 371 |
+
except Exception:
|
| 372 |
+
prev_surya_val = None
|
| 373 |
+
try:
|
| 374 |
+
prev_tess_val = float(prev_tess) if prev_tess is not None else None
|
| 375 |
+
except Exception:
|
| 376 |
+
prev_tess_val = None
|
| 377 |
+
|
| 378 |
+
surya_new = float(surya_conf or 0.0)
|
| 379 |
+
tess_new = float(tess_conf or 0.0)
|
| 380 |
+
|
| 381 |
+
surya_avg = surya_new if prev_surya_val is None else round(alpha * surya_new + (1 - alpha) * prev_surya_val, 4)
|
| 382 |
+
tess_avg = tess_new if prev_tess_val is None else round(alpha * tess_new + (1 - alpha) * prev_tess_val, 4)
|
| 383 |
+
|
| 384 |
+
entry["avg_confidence_surya"] = surya_avg
|
| 385 |
+
entry["avg_confidence_tess"] = tess_avg
|
| 386 |
+
entry["preferred_engine"] = "tesseract" if tess_avg > surya_avg else "surya"
|
| 387 |
+
if entry.get("status") == "unknown":
|
| 388 |
+
entry["status"] = "available"
|
| 389 |
+
|
| 390 |
+
registry[font_name] = entry
|
| 391 |
+
_save_registry(registry)
|
| 392 |
+
|
| 393 |
+
|
| 394 |
+
def font_correction_map_path(font_name: Optional[str]) -> Path:
|
| 395 |
+
"""
|
| 396 |
+
Font-specific punctuation learning map path.
|
| 397 |
+
"""
|
| 398 |
+
if not font_name:
|
| 399 |
+
return CALIBRATION_DIR / "punct_correction_map.json"
|
| 400 |
+
safe = _slug_font_name(font_name) or "unknown"
|
| 401 |
+
return CALIBRATION_DIR / f"punct_correction_map_{safe}.json"
|
| 402 |
+
|
| 403 |
+
|
| 404 |
+
def font_registry_summary() -> list[dict]:
|
| 405 |
+
registry = _load_registry()
|
| 406 |
+
rows = []
|
| 407 |
+
for name, row in sorted(
|
| 408 |
+
registry.items(),
|
| 409 |
+
key=lambda x: int(x[1].get("identified_count", 0)),
|
| 410 |
+
reverse=True,
|
| 411 |
+
):
|
| 412 |
+
rows.append(
|
| 413 |
+
{
|
| 414 |
+
"font_name": name,
|
| 415 |
+
"pages_seen": int(row.get("identified_count", 0)),
|
| 416 |
+
"status": str(row.get("status", "unknown")),
|
| 417 |
+
"preferred_engine": str(row.get("preferred_engine", "surya")),
|
| 418 |
+
"tesseract_model": row.get("tesseract_model"),
|
| 419 |
+
"surya_avg": row.get("avg_confidence_surya"),
|
| 420 |
+
"tess_avg": row.get("avg_confidence_tess"),
|
| 421 |
+
}
|
| 422 |
+
)
|
| 423 |
+
return rows
|
smoke_signal/scripts/punct_corrector.py
CHANGED
|
@@ -38,17 +38,36 @@ KNOWN_SUBSTITUTIONS: dict[str, str] = {
|
|
| 38 |
MIN_PUNCT_PER_10_WORDS = 0.8
|
| 39 |
|
| 40 |
|
| 41 |
-
def
|
| 42 |
-
if
|
|
|
|
|
|
|
| 43 |
try:
|
| 44 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
except Exception:
|
| 46 |
return {}
|
| 47 |
return {}
|
| 48 |
|
| 49 |
|
| 50 |
-
def _save_correction_map(
|
| 51 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
|
| 53 |
|
| 54 |
def _top_substitution(cmap: dict[str, dict[str, int]], raw_char: str) -> Optional[str]:
|
|
@@ -153,7 +172,13 @@ def _align_chars(raw: str, gold: str) -> list[tuple[str, str]]:
|
|
| 153 |
return out
|
| 154 |
|
| 155 |
|
| 156 |
-
def record_punctuation_correction(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 157 |
if not raw_text or not gold_text or raw_text == gold_text:
|
| 158 |
return 0
|
| 159 |
|
|
@@ -161,12 +186,12 @@ def record_punctuation_correction(raw_text: str, gold_text: str, book_id: str =
|
|
| 161 |
if not pairs:
|
| 162 |
return 0
|
| 163 |
|
| 164 |
-
cmap = _load_correction_map()
|
| 165 |
for raw_char, gold_char in pairs:
|
| 166 |
if raw_char not in cmap:
|
| 167 |
cmap[raw_char] = {}
|
| 168 |
cmap[raw_char][gold_char] = int(cmap[raw_char].get(gold_char, 0)) + 1
|
| 169 |
-
_save_correction_map(cmap)
|
| 170 |
|
| 171 |
entry = {
|
| 172 |
"ts": datetime.now(timezone.utc).isoformat(),
|
|
@@ -179,12 +204,17 @@ def record_punctuation_correction(raw_text: str, gold_text: str, book_id: str =
|
|
| 179 |
return len(pairs)
|
| 180 |
|
| 181 |
|
| 182 |
-
def apply_punctuation_corrections(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
_ = book_id
|
| 184 |
if not raw_text:
|
| 185 |
return raw_text, [], 1.0
|
| 186 |
|
| 187 |
-
cmap = _load_correction_map()
|
| 188 |
|
| 189 |
lines = raw_text.splitlines()
|
| 190 |
corrected_lines: list[str] = []
|
|
@@ -219,8 +249,8 @@ def apply_punctuation_corrections(raw_text: str, book_id: str = "") -> tuple[str
|
|
| 219 |
return text, flags, punct_score
|
| 220 |
|
| 221 |
|
| 222 |
-
def correction_map_summary() -> dict:
|
| 223 |
-
cmap = _load_correction_map()
|
| 224 |
pairs: list[dict] = []
|
| 225 |
for raw_char, golds in cmap.items():
|
| 226 |
for gold_char, count in golds.items():
|
|
@@ -229,5 +259,5 @@ def correction_map_summary() -> dict:
|
|
| 229 |
return {
|
| 230 |
"total_pairs": len(pairs),
|
| 231 |
"top_substitutions": pairs[:20],
|
| 232 |
-
"map_path": str(
|
| 233 |
}
|
|
|
|
| 38 |
MIN_PUNCT_PER_10_WORDS = 0.8
|
| 39 |
|
| 40 |
|
| 41 |
+
def _resolve_map_path(font_name: Optional[str] = None, map_path: Optional[Path] = None) -> Path:
|
| 42 |
+
if map_path is not None:
|
| 43 |
+
return Path(map_path)
|
| 44 |
+
if font_name:
|
| 45 |
try:
|
| 46 |
+
from smoke_signal.scripts.font_library import font_correction_map_path
|
| 47 |
+
return Path(font_correction_map_path(font_name))
|
| 48 |
+
except Exception:
|
| 49 |
+
pass
|
| 50 |
+
return CORRECTION_MAP_PATH
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _load_correction_map(font_name: Optional[str] = None, map_path: Optional[Path] = None) -> dict[str, dict[str, int]]:
|
| 54 |
+
target = _resolve_map_path(font_name=font_name, map_path=map_path)
|
| 55 |
+
if target.exists():
|
| 56 |
+
try:
|
| 57 |
+
return json.loads(target.read_text(encoding="utf-8"))
|
| 58 |
except Exception:
|
| 59 |
return {}
|
| 60 |
return {}
|
| 61 |
|
| 62 |
|
| 63 |
+
def _save_correction_map(
|
| 64 |
+
cmap: dict[str, dict[str, int]],
|
| 65 |
+
font_name: Optional[str] = None,
|
| 66 |
+
map_path: Optional[Path] = None,
|
| 67 |
+
) -> None:
|
| 68 |
+
target = _resolve_map_path(font_name=font_name, map_path=map_path)
|
| 69 |
+
target.parent.mkdir(parents=True, exist_ok=True)
|
| 70 |
+
target.write_text(json.dumps(cmap, indent=2, ensure_ascii=False), encoding="utf-8")
|
| 71 |
|
| 72 |
|
| 73 |
def _top_substitution(cmap: dict[str, dict[str, int]], raw_char: str) -> Optional[str]:
|
|
|
|
| 172 |
return out
|
| 173 |
|
| 174 |
|
| 175 |
+
def record_punctuation_correction(
|
| 176 |
+
raw_text: str,
|
| 177 |
+
gold_text: str,
|
| 178 |
+
book_id: str = "",
|
| 179 |
+
font_name: Optional[str] = None,
|
| 180 |
+
map_path: Optional[Path] = None,
|
| 181 |
+
) -> int:
|
| 182 |
if not raw_text or not gold_text or raw_text == gold_text:
|
| 183 |
return 0
|
| 184 |
|
|
|
|
| 186 |
if not pairs:
|
| 187 |
return 0
|
| 188 |
|
| 189 |
+
cmap = _load_correction_map(font_name=font_name, map_path=map_path)
|
| 190 |
for raw_char, gold_char in pairs:
|
| 191 |
if raw_char not in cmap:
|
| 192 |
cmap[raw_char] = {}
|
| 193 |
cmap[raw_char][gold_char] = int(cmap[raw_char].get(gold_char, 0)) + 1
|
| 194 |
+
_save_correction_map(cmap, font_name=font_name, map_path=map_path)
|
| 195 |
|
| 196 |
entry = {
|
| 197 |
"ts": datetime.now(timezone.utc).isoformat(),
|
|
|
|
| 204 |
return len(pairs)
|
| 205 |
|
| 206 |
|
| 207 |
+
def apply_punctuation_corrections(
|
| 208 |
+
raw_text: str,
|
| 209 |
+
book_id: str = "",
|
| 210 |
+
font_name: Optional[str] = None,
|
| 211 |
+
map_path: Optional[Path] = None,
|
| 212 |
+
) -> tuple[str, list[dict], float]:
|
| 213 |
_ = book_id
|
| 214 |
if not raw_text:
|
| 215 |
return raw_text, [], 1.0
|
| 216 |
|
| 217 |
+
cmap = _load_correction_map(font_name=font_name, map_path=map_path)
|
| 218 |
|
| 219 |
lines = raw_text.splitlines()
|
| 220 |
corrected_lines: list[str] = []
|
|
|
|
| 249 |
return text, flags, punct_score
|
| 250 |
|
| 251 |
|
| 252 |
+
def correction_map_summary(font_name: Optional[str] = None, map_path: Optional[Path] = None) -> dict:
|
| 253 |
+
cmap = _load_correction_map(font_name=font_name, map_path=map_path)
|
| 254 |
pairs: list[dict] = []
|
| 255 |
for raw_char, golds in cmap.items():
|
| 256 |
for gold_char, count in golds.items():
|
|
|
|
| 259 |
return {
|
| 260 |
"total_pairs": len(pairs),
|
| 261 |
"top_substitutions": pairs[:20],
|
| 262 |
+
"map_path": str(_resolve_map_path(font_name=font_name, map_path=map_path)),
|
| 263 |
}
|
smoke_signal/tessdata/README.md
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Smoke Signal Tessdata Drop Folder
|
| 2 |
+
|
| 3 |
+
Put custom Tesseract model files here as `.traineddata`.
|
| 4 |
+
|
| 5 |
+
Naming rule:
|
| 6 |
+
- Use lowercase + underscores based on detected font name.
|
| 7 |
+
- Example: `Gill Sans Infant` -> `gill_sans_infant.traineddata`
|
| 8 |
+
|
| 9 |
+
The OCR runtime will auto-pick models from either:
|
| 10 |
+
- `/tmp/smoke_signal/tessdata/`
|
| 11 |
+
- `smoke_signal/tessdata/` (this folder)
|
| 12 |
+
|
| 13 |
+
Important:
|
| 14 |
+
- `.otf` / `.ttf` files are not Tesseract models.
|
| 15 |
+
- A file like `gill_sans_infant.traineddata.otf` is still a font file, not usable by Tesseract OCR.
|
| 16 |
+
|
| 17 |
+
If your model name is custom and does not match slug, register it once:
|
| 18 |
+
|
| 19 |
+
```python
|
| 20 |
+
from smoke_signal.scripts.font_library import register_font_model
|
| 21 |
+
register_font_model("Gill Sans Infant", model_name="my_model_name")
|
| 22 |
+
```
|
smoke_signal_tab.py
CHANGED
|
@@ -27,6 +27,7 @@ import hashlib
|
|
| 27 |
import importlib.util
|
| 28 |
import json
|
| 29 |
import os
|
|
|
|
| 30 |
import tempfile
|
| 31 |
import threading
|
| 32 |
import time
|
|
@@ -60,6 +61,21 @@ DECISIONS_CSV = REVIEW_DIR / "review_decisions.csv"
|
|
| 60 |
QUEUE_CSV = REVIEW_DIR / "review_queue.csv"
|
| 61 |
BANNER_DATA_URI_FILE = Path(__file__).resolve().parent / "assets" / "smoke_signal_banner_data_uri.txt"
|
| 62 |
PUNCT_CORRECTOR_PATH = Path(__file__).resolve().parent / "smoke_signal" / "scripts" / "punct_corrector.py"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
|
| 64 |
|
| 65 |
def _load_banner_image_css() -> str:
|
|
@@ -83,6 +99,8 @@ SS_BANNER_IMAGE_CSS = _load_banner_image_css()
|
|
| 83 |
|
| 84 |
_PUNCT_MODULE = None
|
| 85 |
_PUNCT_MODULE_ERROR = None
|
|
|
|
|
|
|
| 86 |
|
| 87 |
|
| 88 |
def _load_punct_module():
|
|
@@ -108,12 +126,39 @@ def _load_punct_module():
|
|
| 108 |
return None
|
| 109 |
|
| 110 |
|
| 111 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 112 |
module = _load_punct_module()
|
| 113 |
if module is None:
|
| 114 |
return text, [], 1.0
|
| 115 |
try:
|
| 116 |
-
corrected, flags, score = module.apply_punctuation_corrections(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 117 |
return corrected, flags, float(score)
|
| 118 |
except Exception:
|
| 119 |
return text, [], 1.0
|
|
@@ -129,16 +174,126 @@ def _punctuation_confidence_penalty(text: str, confidence: float) -> float:
|
|
| 129 |
return float(confidence)
|
| 130 |
|
| 131 |
|
| 132 |
-
def _record_punctuation_correction(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 133 |
module = _load_punct_module()
|
| 134 |
if module is None:
|
| 135 |
return 0
|
| 136 |
try:
|
| 137 |
-
return int(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 138 |
except Exception:
|
| 139 |
return 0
|
| 140 |
|
| 141 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 142 |
def _punctuation_map_summary() -> dict:
|
| 143 |
module = _load_punct_module()
|
| 144 |
if module is None:
|
|
@@ -591,15 +746,19 @@ def sha256_file(path: Path) -> str:
|
|
| 591 |
|
| 592 |
def load_manifest_df() -> pd.DataFrame:
|
| 593 |
if not MANIFEST_CSV.exists():
|
| 594 |
-
return pd.DataFrame(columns=
|
| 595 |
-
|
| 596 |
-
|
| 597 |
-
|
| 598 |
-
|
|
|
|
| 599 |
|
| 600 |
|
| 601 |
def save_manifest_df(df: pd.DataFrame) -> None:
|
| 602 |
-
|
|
|
|
|
|
|
|
|
|
| 603 |
|
| 604 |
|
| 605 |
def next_book_id(df: pd.DataFrame) -> str:
|
|
@@ -673,6 +832,72 @@ def _parse_page_selection(spec: str, max_page: int | None = None) -> tuple[set[i
|
|
| 673 |
return out, None
|
| 674 |
|
| 675 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 676 |
# ββ Step 1: INGEST βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 677 |
def ingest_pdfs(files, rights_class: str, notes: str) -> tuple:
|
| 678 |
"""Register uploaded PDFs into the source manifest."""
|
|
@@ -734,6 +959,9 @@ def ingest_pdfs(files, rights_class: str, notes: str) -> tuple:
|
|
| 734 |
"status": "pending",
|
| 735 |
"acquisition_date": datetime.utcnow().date().isoformat(),
|
| 736 |
"notes": notes,
|
|
|
|
|
|
|
|
|
|
| 737 |
}])
|
| 738 |
df = pd.concat([df, new_row], ignore_index=True)
|
| 739 |
log.append(log_line(f"β Registered {book_id} β {path.name} ({page_count or '?'} pages)"))
|
|
@@ -960,6 +1188,14 @@ try:
|
|
| 960 |
SS_TESSERACT_BATCH_HARD_TIMEOUT_SEC = max(15, int(os.environ.get("SS_TESSERACT_BATCH_HARD_TIMEOUT_SEC", "45")))
|
| 961 |
except Exception:
|
| 962 |
SS_TESSERACT_BATCH_HARD_TIMEOUT_SEC = 45
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 963 |
|
| 964 |
|
| 965 |
def _load_surya_runtime():
|
|
@@ -1107,7 +1343,11 @@ def _regions_from_page_result(page_result):
|
|
| 1107 |
return regions, conf
|
| 1108 |
|
| 1109 |
|
| 1110 |
-
def _run_tesseract_batch(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1111 |
"""
|
| 1112 |
Tesseract fallback for OCR when Surya is unavailable/slow.
|
| 1113 |
Returns list of dicts with regions/confidence/method aligned to input order.
|
|
@@ -1115,17 +1355,28 @@ def _run_tesseract_batch(images):
|
|
| 1115 |
try:
|
| 1116 |
import pytesseract
|
| 1117 |
except Exception as e:
|
| 1118 |
-
return [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1119 |
|
| 1120 |
# Prefer parallel image-level OCR with single-threaded internal OpenMP for better CPU utilization.
|
| 1121 |
os.environ.setdefault("OMP_THREAD_LIMIT", "1")
|
| 1122 |
|
| 1123 |
-
def _ocr_single(img):
|
| 1124 |
try:
|
|
|
|
|
|
|
|
|
|
| 1125 |
data = pytesseract.image_to_data(
|
| 1126 |
img,
|
| 1127 |
-
lang="eng",
|
| 1128 |
-
config=
|
| 1129 |
output_type=pytesseract.Output.DICT,
|
| 1130 |
timeout=SS_TESSERACT_TIMEOUT_SEC,
|
| 1131 |
)
|
|
@@ -1167,7 +1418,12 @@ def _run_tesseract_batch(images):
|
|
| 1167 |
|
| 1168 |
# Hard gate for fallback quality: do not keep OCR text below minimum page confidence.
|
| 1169 |
if regions and avg_conf < SS_TESSERACT_MIN_CONF_KEEP:
|
| 1170 |
-
return {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1171 |
|
| 1172 |
# Filter obvious OCR noise from illustration texture (common in no-text pages).
|
| 1173 |
full_text = " ".join(r["text"] for r in regions).strip()
|
|
@@ -1177,24 +1433,62 @@ def _run_tesseract_batch(images):
|
|
| 1177 |
tokens = [t for t in full_text.split() if t]
|
| 1178 |
avg_token_len = (sum(len(t) for t in tokens) / len(tokens)) if tokens else 0.0
|
| 1179 |
if regions and avg_conf <= 0.42 and alpha_ratio < 0.62 and avg_token_len < 3.2:
|
| 1180 |
-
return {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1181 |
|
| 1182 |
-
return {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1183 |
except RuntimeError as e:
|
| 1184 |
-
return {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1185 |
except Exception as e:
|
| 1186 |
-
return {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1187 |
|
| 1188 |
if not images:
|
| 1189 |
return []
|
| 1190 |
|
| 1191 |
max_workers = min(SS_TESSERACT_WORKERS, len(images), max(1, os.cpu_count() or 1))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1192 |
if max_workers <= 1:
|
| 1193 |
-
return [
|
|
|
|
|
|
|
|
|
|
| 1194 |
|
| 1195 |
outputs = [None] * len(images)
|
| 1196 |
executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_workers)
|
| 1197 |
-
futures = {
|
|
|
|
|
|
|
|
|
|
| 1198 |
pending = set(futures.keys())
|
| 1199 |
batch_timeout = max(
|
| 1200 |
SS_TESSERACT_BATCH_HARD_TIMEOUT_SEC,
|
|
@@ -1225,6 +1519,7 @@ def _run_tesseract_batch(images):
|
|
| 1225 |
"regions": [],
|
| 1226 |
"confidence": 0.0,
|
| 1227 |
"method": f"error-tesseract-batch-timeout ({batch_timeout}s)",
|
|
|
|
| 1228 |
}
|
| 1229 |
future.cancel()
|
| 1230 |
finally:
|
|
@@ -1233,7 +1528,12 @@ def _run_tesseract_batch(images):
|
|
| 1233 |
|
| 1234 |
for i, out in enumerate(outputs):
|
| 1235 |
if out is None:
|
| 1236 |
-
outputs[i] = {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1237 |
|
| 1238 |
return outputs
|
| 1239 |
|
|
@@ -1269,8 +1569,10 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
|
|
| 1269 |
selection_set, selection_error = _parse_page_selection(page_selection)
|
| 1270 |
if selection_error:
|
| 1271 |
return _ocr_status_html(), f"{debug}Invalid page selection: {selection_error}"
|
|
|
|
| 1272 |
exclusion_spec = (page_exclusion or "").strip().lower()
|
| 1273 |
exclusion_request = None if exclusion_spec in ("", "none", "-") else exclusion_spec
|
|
|
|
| 1274 |
if exclusion_request is not None:
|
| 1275 |
exclusion_set, exclusion_error = _parse_page_selection(exclusion_request)
|
| 1276 |
if exclusion_error:
|
|
@@ -1310,6 +1612,23 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
|
|
| 1310 |
log.append(log_line(f"β Punctuation corrector unavailable ({_PUNCT_MODULE_ERROR or 'unknown'})"))
|
| 1311 |
else:
|
| 1312 |
log.append(log_line("β Punctuation corrector active (rule-check + confidence penalty + learning map)"))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1313 |
|
| 1314 |
for _, row in eligible.iterrows():
|
| 1315 |
book_id = row["book_id"]
|
|
@@ -1343,11 +1662,31 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
|
|
| 1343 |
if excluded_pages_for_book is None:
|
| 1344 |
excluded_pages_for_book = set(page_numbers)
|
| 1345 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1346 |
selected_pages_for_book = set(selected_pages_for_book) - set(excluded_pages_for_book)
|
| 1347 |
if not selected_pages_for_book:
|
| 1348 |
log.append(log_line(f"β {book_id}: no pages left after applying selection/exclusion"))
|
| 1349 |
continue
|
| 1350 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1351 |
selected_preview = ",".join(str(p) for p in sorted(selected_pages_for_book))
|
| 1352 |
log.append(log_line(f"βΉ {book_id}: effective selected pages {selected_preview}"))
|
| 1353 |
processed_books.append(book_id)
|
|
@@ -1369,9 +1708,14 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
|
|
| 1369 |
# First pass: render OCR/hybrid pages once and keep PIL images for batch OCR.
|
| 1370 |
ocr_targets = []
|
| 1371 |
render_dpi = SS_RENDER_DPI_SURYA if surya is not None else SS_RENDER_DPI_FALLBACK
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1372 |
for page_data in profile.get("pages", []):
|
| 1373 |
page_num = page_data["page_number"]
|
| 1374 |
route = page_data["route"]
|
|
|
|
| 1375 |
if selected_pages_for_book is not None and int(page_num) not in selected_pages_for_book:
|
| 1376 |
continue
|
| 1377 |
|
|
@@ -1415,10 +1759,41 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
|
|
| 1415 |
except Exception as e:
|
| 1416 |
log.append(log_line(f" β {book_id} p{page_num}: render failed ({e})"))
|
| 1417 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1418 |
ocr_targets.append({
|
| 1419 |
"page_num": page_num,
|
| 1420 |
"route": route,
|
| 1421 |
"render_path": page_data.get("render_path"),
|
|
|
|
| 1422 |
})
|
| 1423 |
|
| 1424 |
# Batch OCR for non-embedded pages.
|
|
@@ -1430,6 +1805,8 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
|
|
| 1430 |
batch_pages = [item["page_num"] for item in batch]
|
| 1431 |
batch_images = []
|
| 1432 |
batch_items_with_images = []
|
|
|
|
|
|
|
| 1433 |
try:
|
| 1434 |
from PIL import Image
|
| 1435 |
except Exception as e:
|
|
@@ -1460,6 +1837,9 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
|
|
| 1460 |
img = Image.open(render_abs).convert("RGB")
|
| 1461 |
batch_images.append(img)
|
| 1462 |
batch_items_with_images.append(item)
|
|
|
|
|
|
|
|
|
|
| 1463 |
except Exception as e:
|
| 1464 |
ocr_lookup[item["page_num"]] = {
|
| 1465 |
"regions": [],
|
|
@@ -1481,7 +1861,7 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
|
|
| 1481 |
"method": "surya",
|
| 1482 |
}
|
| 1483 |
else:
|
| 1484 |
-
fallback_preds = _run_tesseract_batch(batch_images)
|
| 1485 |
timeout_count = sum(1 for pred in fallback_preds if "batch-timeout" in str(pred.get("method", "")))
|
| 1486 |
for item, pred in zip(batch_items_with_images, fallback_preds):
|
| 1487 |
ocr_lookup[item["page_num"]] = pred
|
|
@@ -1496,7 +1876,7 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
|
|
| 1496 |
# If Surya batch fails, try Tesseract for this batch before giving up.
|
| 1497 |
if surya is not None:
|
| 1498 |
log.append(log_line(f" β {book_id} batch {batch_pages[0]}-{batch_pages[-1]} Surya error: {e}; retrying with Tesseract"))
|
| 1499 |
-
fallback_preds = _run_tesseract_batch(batch_images)
|
| 1500 |
timeout_count = sum(1 for pred in fallback_preds if "batch-timeout" in str(pred.get("method", "")))
|
| 1501 |
for item, pred in zip(batch_items_with_images, fallback_preds):
|
| 1502 |
ocr_lookup[item["page_num"]] = pred
|
|
@@ -1552,8 +1932,17 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
|
|
| 1552 |
method = "skipped-no-surya"
|
| 1553 |
|
| 1554 |
raw_text = " ".join(r["text"] for r in regions)[:500]
|
| 1555 |
-
corrected_text, punct_flags, punct_score = _apply_punctuation_corrections(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1556 |
conf_adjusted = _punctuation_confidence_penalty(corrected_text, conf)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1557 |
|
| 1558 |
if method in ("tesseract-noise-filtered", "tesseract-lowconf-filtered") and not regions:
|
| 1559 |
conf_class = "auto-accept"
|
|
@@ -1585,6 +1974,8 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
|
|
| 1585 |
"extraction_method": method,
|
| 1586 |
"punctuation_score": punct_score,
|
| 1587 |
"punctuation_flags": punct_flags,
|
|
|
|
|
|
|
| 1588 |
"render_path": page_data.get("render_path"),
|
| 1589 |
"ocred_at": datetime.utcnow().isoformat() + "Z",
|
| 1590 |
})
|
|
@@ -1605,6 +1996,8 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
|
|
| 1605 |
"confidence_class": conf_class,
|
| 1606 |
"punct_score": punct_score,
|
| 1607 |
"punct_flags_count": len(punct_flags),
|
|
|
|
|
|
|
| 1608 |
"status": "quarantine" if conf_class == "quarantine" else "pending",
|
| 1609 |
"reviewer": "",
|
| 1610 |
"correction": "",
|
|
@@ -1739,8 +2132,9 @@ def get_review_item(idx: int) -> tuple:
|
|
| 1739 |
except Exception:
|
| 1740 |
img_path = None
|
| 1741 |
|
|
|
|
| 1742 |
info = (f"<div style='font-family:monospace;font-size:11px;color:var(--ss-muted)'>"
|
| 1743 |
-
f"{item['book_id']} Β· page {item['page']} Β· "
|
| 1744 |
f"conf: <b style='color:{'var(--ss-red)' if float(item.get('confidence',0)) < 0.6 else 'var(--ss-gold)'}'>"
|
| 1745 |
f"{float(item.get('confidence',0)):.0%}</b></div>")
|
| 1746 |
|
|
@@ -1782,6 +2176,7 @@ def save_review_decision(idx: int, final_text: str, action: str, reviewer: str,
|
|
| 1782 |
"raw_text_original": raw_text_original,
|
| 1783 |
"reason_code": reason,
|
| 1784 |
"reviewer": reviewer or "reviewer",
|
|
|
|
| 1785 |
"was_correct": was_correct,
|
| 1786 |
"decided_at": datetime.utcnow().isoformat() + "Z",
|
| 1787 |
}
|
|
@@ -1816,7 +2211,12 @@ def save_review_decision(idx: int, final_text: str, action: str, reviewer: str,
|
|
| 1816 |
|
| 1817 |
learned_pairs = 0
|
| 1818 |
if action == "edited":
|
| 1819 |
-
learned_pairs = _record_punctuation_correction(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1820 |
|
| 1821 |
cal = load_calibration()
|
| 1822 |
default = cal.get("_default", DEFAULT_CALIBRATION["_default"])
|
|
@@ -1878,6 +2278,7 @@ def run_export() -> tuple:
|
|
| 1878 |
|
| 1879 |
records.append({
|
| 1880 |
"book_id": book_id,
|
|
|
|
| 1881 |
"source_hash": manifest_row.get("sha256","") if isinstance(manifest_row, pd.Series) else "",
|
| 1882 |
"page_number": dec["page"],
|
| 1883 |
"region_id": dec["region_id"],
|
|
@@ -1997,7 +2398,7 @@ def smoke_signal_tab():
|
|
| 1997 |
interactive=False,
|
| 1998 |
wrap=True,
|
| 1999 |
)
|
| 2000 |
-
ingest_log = gr.Textbox(label="Log", lines=6, interactive=False, elem_classes=["ss-log"])
|
| 2001 |
refresh_btn = gr.Button("β» Refresh Manifest", size="sm")
|
| 2002 |
refresh_btn.click(lambda: load_manifest_df(), outputs=[manifest_table])
|
| 2003 |
|
|
@@ -2017,6 +2418,19 @@ def smoke_signal_tab():
|
|
| 2017 |
outputs=[ingest_status, manifest_table, ingest_log],
|
| 2018 |
)
|
| 2019 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2020 |
ingest_btn.click(
|
| 2021 |
ingest_pdfs,
|
| 2022 |
inputs=[pdf_upload, rights_dd, ingest_notes],
|
|
@@ -2035,7 +2449,7 @@ def smoke_signal_tab():
|
|
| 2035 |
|
| 2036 |
profile_status = gr.HTML(_profile_status_html())
|
| 2037 |
profile_btn = gr.Button("Re-run Profiler (Optional) β", elem_classes=["ss-btn-run"])
|
| 2038 |
-
profile_log = gr.Textbox(label="Log", lines=10, interactive=False, elem_classes=["ss-log"])
|
| 2039 |
|
| 2040 |
profile_btn.click(run_profile, outputs=[profile_status, profile_log])
|
| 2041 |
|
|
@@ -2073,13 +2487,13 @@ def smoke_signal_tab():
|
|
| 2073 |
value=True,
|
| 2074 |
)
|
| 2075 |
ocr_btn = gr.Button("Run OCR β", elem_classes=["ss-btn-run"])
|
| 2076 |
-
ocr_log = gr.Textbox(label="Log", lines=10, interactive=False, elem_classes=["ss-log"])
|
| 2077 |
|
| 2078 |
ocr_run_event = ocr_btn.click(
|
| 2079 |
run_ocr,
|
| 2080 |
inputs=[ocr_page_selection, ocr_page_exclusion, ocr_replace_queue],
|
| 2081 |
outputs=[ocr_status, ocr_log],
|
| 2082 |
-
show_progress="
|
| 2083 |
)
|
| 2084 |
|
| 2085 |
# ββ STEP 4: REVIEW ββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
@@ -2178,7 +2592,7 @@ def smoke_signal_tab():
|
|
| 2178 |
ocr_run_event.then(
|
| 2179 |
load_review,
|
| 2180 |
outputs=review_load_outputs,
|
| 2181 |
-
show_progress="
|
| 2182 |
)
|
| 2183 |
accept_btn.click(do_accept, inputs=[current_idx, final_text_box, reviewer_name, reason_code], outputs=action_outputs)
|
| 2184 |
edit_btn.click(do_accept, inputs=[current_idx, final_text_box, reviewer_name, reason_code], outputs=action_outputs)
|
|
|
|
| 27 |
import importlib.util
|
| 28 |
import json
|
| 29 |
import os
|
| 30 |
+
import re
|
| 31 |
import tempfile
|
| 32 |
import threading
|
| 33 |
import time
|
|
|
|
| 61 |
QUEUE_CSV = REVIEW_DIR / "review_queue.csv"
|
| 62 |
BANNER_DATA_URI_FILE = Path(__file__).resolve().parent / "assets" / "smoke_signal_banner_data_uri.txt"
|
| 63 |
PUNCT_CORRECTOR_PATH = Path(__file__).resolve().parent / "smoke_signal" / "scripts" / "punct_corrector.py"
|
| 64 |
+
FONT_LIBRARY_PATH = Path(__file__).resolve().parent / "smoke_signal" / "scripts" / "font_library.py"
|
| 65 |
+
|
| 66 |
+
MANIFEST_COLUMNS = [
|
| 67 |
+
"book_id",
|
| 68 |
+
"filename",
|
| 69 |
+
"sha256",
|
| 70 |
+
"page_count",
|
| 71 |
+
"rights_class",
|
| 72 |
+
"status",
|
| 73 |
+
"acquisition_date",
|
| 74 |
+
"notes",
|
| 75 |
+
"story_pages_include",
|
| 76 |
+
"story_pages_exclude",
|
| 77 |
+
"safe_title",
|
| 78 |
+
]
|
| 79 |
|
| 80 |
|
| 81 |
def _load_banner_image_css() -> str:
|
|
|
|
| 99 |
|
| 100 |
_PUNCT_MODULE = None
|
| 101 |
_PUNCT_MODULE_ERROR = None
|
| 102 |
+
_FONT_MODULE = None
|
| 103 |
+
_FONT_MODULE_ERROR = None
|
| 104 |
|
| 105 |
|
| 106 |
def _load_punct_module():
|
|
|
|
| 126 |
return None
|
| 127 |
|
| 128 |
|
| 129 |
+
def _load_font_module():
|
| 130 |
+
global _FONT_MODULE, _FONT_MODULE_ERROR
|
| 131 |
+
if _FONT_MODULE is not None:
|
| 132 |
+
return _FONT_MODULE
|
| 133 |
+
if _FONT_MODULE_ERROR is not None:
|
| 134 |
+
return None
|
| 135 |
+
if not FONT_LIBRARY_PATH.exists():
|
| 136 |
+
_FONT_MODULE_ERROR = f"not found: {FONT_LIBRARY_PATH}"
|
| 137 |
+
return None
|
| 138 |
+
try:
|
| 139 |
+
spec = importlib.util.spec_from_file_location("smoke_signal_font_library", str(FONT_LIBRARY_PATH))
|
| 140 |
+
if spec is None or spec.loader is None:
|
| 141 |
+
_FONT_MODULE_ERROR = "invalid import spec"
|
| 142 |
+
return None
|
| 143 |
+
module = importlib.util.module_from_spec(spec)
|
| 144 |
+
spec.loader.exec_module(module)
|
| 145 |
+
_FONT_MODULE = module
|
| 146 |
+
return _FONT_MODULE
|
| 147 |
+
except Exception as e:
|
| 148 |
+
_FONT_MODULE_ERROR = str(e)
|
| 149 |
+
return None
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def _apply_punctuation_corrections(text: str, book_id: str, font_name: Optional[str] = None) -> tuple[str, list, float]:
|
| 153 |
module = _load_punct_module()
|
| 154 |
if module is None:
|
| 155 |
return text, [], 1.0
|
| 156 |
try:
|
| 157 |
+
corrected, flags, score = module.apply_punctuation_corrections(
|
| 158 |
+
text or "",
|
| 159 |
+
book_id=book_id,
|
| 160 |
+
font_name=font_name,
|
| 161 |
+
)
|
| 162 |
return corrected, flags, float(score)
|
| 163 |
except Exception:
|
| 164 |
return text, [], 1.0
|
|
|
|
| 174 |
return float(confidence)
|
| 175 |
|
| 176 |
|
| 177 |
+
def _record_punctuation_correction(
|
| 178 |
+
raw_text: str,
|
| 179 |
+
gold_text: str,
|
| 180 |
+
book_id: str,
|
| 181 |
+
font_name: Optional[str] = None,
|
| 182 |
+
) -> int:
|
| 183 |
module = _load_punct_module()
|
| 184 |
if module is None:
|
| 185 |
return 0
|
| 186 |
try:
|
| 187 |
+
return int(
|
| 188 |
+
module.record_punctuation_correction(
|
| 189 |
+
raw_text or "",
|
| 190 |
+
gold_text or "",
|
| 191 |
+
book_id=book_id,
|
| 192 |
+
font_name=font_name,
|
| 193 |
+
)
|
| 194 |
+
)
|
| 195 |
except Exception:
|
| 196 |
return 0
|
| 197 |
|
| 198 |
|
| 199 |
+
def _identify_page_font(render_abs_path: str) -> dict:
|
| 200 |
+
module = _load_font_module()
|
| 201 |
+
if module is None:
|
| 202 |
+
return {
|
| 203 |
+
"font_name": None,
|
| 204 |
+
"confidence": 0.0,
|
| 205 |
+
"alternatives": [],
|
| 206 |
+
"image_url": None,
|
| 207 |
+
"error": _FONT_MODULE_ERROR or "font module unavailable",
|
| 208 |
+
}
|
| 209 |
+
try:
|
| 210 |
+
result = module.identify_page_font(render_abs_path)
|
| 211 |
+
return result if isinstance(result, dict) else {
|
| 212 |
+
"font_name": None,
|
| 213 |
+
"confidence": 0.0,
|
| 214 |
+
"alternatives": [],
|
| 215 |
+
"image_url": None,
|
| 216 |
+
"error": "invalid response from font module",
|
| 217 |
+
}
|
| 218 |
+
except Exception as e:
|
| 219 |
+
return {
|
| 220 |
+
"font_name": None,
|
| 221 |
+
"confidence": 0.0,
|
| 222 |
+
"alternatives": [],
|
| 223 |
+
"image_url": None,
|
| 224 |
+
"error": str(e),
|
| 225 |
+
}
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
def _resolve_tesseract_lang(font_name: Optional[str]) -> str:
|
| 229 |
+
module = _load_font_module()
|
| 230 |
+
if module is None:
|
| 231 |
+
return "eng"
|
| 232 |
+
try:
|
| 233 |
+
lang = str(module.resolve_tesseract_lang(font_name) or "").strip()
|
| 234 |
+
return lang if lang else "eng"
|
| 235 |
+
except Exception:
|
| 236 |
+
return "eng"
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def _resolve_tessdata_dir(model_name: Optional[str]) -> Optional[str]:
|
| 240 |
+
module = _load_font_module()
|
| 241 |
+
if module is None:
|
| 242 |
+
return None
|
| 243 |
+
try:
|
| 244 |
+
value = module.resolve_tessdata_dir(model_name)
|
| 245 |
+
return str(value) if value else None
|
| 246 |
+
except Exception:
|
| 247 |
+
return None
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
def _update_font_engine_stats(font_name: Optional[str], surya_conf: float, tess_conf: float) -> None:
|
| 251 |
+
module = _load_font_module()
|
| 252 |
+
if module is None:
|
| 253 |
+
return
|
| 254 |
+
try:
|
| 255 |
+
module.update_font_engine_stats(font_name, surya_conf, tess_conf)
|
| 256 |
+
except Exception:
|
| 257 |
+
return
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def _mixfont_preflight() -> dict:
|
| 261 |
+
module = _load_font_module()
|
| 262 |
+
if module is None:
|
| 263 |
+
return {
|
| 264 |
+
"api_key_set": False,
|
| 265 |
+
"api_url": "",
|
| 266 |
+
"image_url_template_set": False,
|
| 267 |
+
"image_base_set": False,
|
| 268 |
+
"space_host_set": False,
|
| 269 |
+
"public_image_url_source_available": False,
|
| 270 |
+
"error": _FONT_MODULE_ERROR or "font module unavailable",
|
| 271 |
+
}
|
| 272 |
+
try:
|
| 273 |
+
data = module.mixfont_preflight()
|
| 274 |
+
if not isinstance(data, dict):
|
| 275 |
+
return {
|
| 276 |
+
"api_key_set": False,
|
| 277 |
+
"api_url": "",
|
| 278 |
+
"image_url_template_set": False,
|
| 279 |
+
"image_base_set": False,
|
| 280 |
+
"space_host_set": False,
|
| 281 |
+
"public_image_url_source_available": False,
|
| 282 |
+
"error": "invalid preflight response",
|
| 283 |
+
}
|
| 284 |
+
return data
|
| 285 |
+
except Exception as e:
|
| 286 |
+
return {
|
| 287 |
+
"api_key_set": False,
|
| 288 |
+
"api_url": "",
|
| 289 |
+
"image_url_template_set": False,
|
| 290 |
+
"image_base_set": False,
|
| 291 |
+
"space_host_set": False,
|
| 292 |
+
"public_image_url_source_available": False,
|
| 293 |
+
"error": str(e),
|
| 294 |
+
}
|
| 295 |
+
|
| 296 |
+
|
| 297 |
def _punctuation_map_summary() -> dict:
|
| 298 |
module = _load_punct_module()
|
| 299 |
if module is None:
|
|
|
|
| 746 |
|
| 747 |
def load_manifest_df() -> pd.DataFrame:
|
| 748 |
if not MANIFEST_CSV.exists():
|
| 749 |
+
return pd.DataFrame(columns=MANIFEST_COLUMNS)
|
| 750 |
+
df = pd.read_csv(MANIFEST_CSV)
|
| 751 |
+
for col in MANIFEST_COLUMNS:
|
| 752 |
+
if col not in df.columns:
|
| 753 |
+
df[col] = ""
|
| 754 |
+
return df[MANIFEST_COLUMNS]
|
| 755 |
|
| 756 |
|
| 757 |
def save_manifest_df(df: pd.DataFrame) -> None:
|
| 758 |
+
for col in MANIFEST_COLUMNS:
|
| 759 |
+
if col not in df.columns:
|
| 760 |
+
df[col] = ""
|
| 761 |
+
df[MANIFEST_COLUMNS].to_csv(MANIFEST_CSV, index=False)
|
| 762 |
|
| 763 |
|
| 764 |
def next_book_id(df: pd.DataFrame) -> str:
|
|
|
|
| 832 |
return out, None
|
| 833 |
|
| 834 |
|
| 835 |
+
def _safe_title_slug(title: str) -> str:
|
| 836 |
+
raw = str(title or "").strip().lower()
|
| 837 |
+
if not raw:
|
| 838 |
+
return ""
|
| 839 |
+
slug = re.sub(r"[^a-z0-9]+", "-", raw).strip("-")
|
| 840 |
+
return slug[:120]
|
| 841 |
+
|
| 842 |
+
|
| 843 |
+
def _clean_page_spec(spec: str) -> str:
|
| 844 |
+
raw = (spec or "").strip().lower()
|
| 845 |
+
if raw in ("", "all", "*", "none", "-"):
|
| 846 |
+
return ""
|
| 847 |
+
parsed, err = _parse_page_selection(raw)
|
| 848 |
+
if err:
|
| 849 |
+
raise ValueError(err)
|
| 850 |
+
if parsed is None:
|
| 851 |
+
return ""
|
| 852 |
+
return ",".join(str(p) for p in sorted(parsed))
|
| 853 |
+
|
| 854 |
+
|
| 855 |
+
def _normalize_saved_spec(value) -> str:
|
| 856 |
+
raw = str(value if value is not None else "").strip().lower()
|
| 857 |
+
if raw in ("", "nan", "none", "null", "-", "all", "*"):
|
| 858 |
+
return ""
|
| 859 |
+
return raw
|
| 860 |
+
|
| 861 |
+
|
| 862 |
+
def save_book_scope(book_id: str, include_spec: str, exclude_spec: str, safe_title: str) -> tuple:
|
| 863 |
+
df = load_manifest_df()
|
| 864 |
+
bid = (book_id or "").strip()
|
| 865 |
+
if df.empty or not bid:
|
| 866 |
+
return _ingest_status_html("idle"), df, "Book ID is required."
|
| 867 |
+
if bid not in df["book_id"].values:
|
| 868 |
+
return _ingest_status_html("idle"), df, f"Book ID {bid} not found."
|
| 869 |
+
|
| 870 |
+
try:
|
| 871 |
+
include_clean = _clean_page_spec(include_spec)
|
| 872 |
+
except ValueError as e:
|
| 873 |
+
return _ingest_status_html("idle"), df, f"Invalid include pages: {e}"
|
| 874 |
+
|
| 875 |
+
try:
|
| 876 |
+
exclude_clean = _clean_page_spec(exclude_spec)
|
| 877 |
+
except ValueError as e:
|
| 878 |
+
return _ingest_status_html("idle"), df, f"Invalid exclude pages: {e}"
|
| 879 |
+
|
| 880 |
+
title_raw = (safe_title or "").strip()
|
| 881 |
+
if title_raw:
|
| 882 |
+
safe = _safe_title_slug(title_raw)
|
| 883 |
+
else:
|
| 884 |
+
filename = str(df.loc[df["book_id"] == bid, "filename"].values[0] or "")
|
| 885 |
+
stem = Path(filename).stem if filename else bid
|
| 886 |
+
safe = _safe_title_slug(stem)
|
| 887 |
+
|
| 888 |
+
df.loc[df["book_id"] == bid, "story_pages_include"] = include_clean
|
| 889 |
+
df.loc[df["book_id"] == bid, "story_pages_exclude"] = exclude_clean
|
| 890 |
+
df.loc[df["book_id"] == bid, "safe_title"] = safe
|
| 891 |
+
save_manifest_df(df)
|
| 892 |
+
|
| 893 |
+
line1 = log_line(f"β Saved scope for {bid}")
|
| 894 |
+
line2 = log_line(
|
| 895 |
+
f"βΉ include={include_clean or 'all'} Β· exclude={exclude_clean or 'none'} Β· safe_title={safe}"
|
| 896 |
+
)
|
| 897 |
+
msg = f"{line1}\n{line2}"
|
| 898 |
+
return _ingest_status_html("done"), load_manifest_df(), msg
|
| 899 |
+
|
| 900 |
+
|
| 901 |
# ββ Step 1: INGEST βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 902 |
def ingest_pdfs(files, rights_class: str, notes: str) -> tuple:
|
| 903 |
"""Register uploaded PDFs into the source manifest."""
|
|
|
|
| 959 |
"status": "pending",
|
| 960 |
"acquisition_date": datetime.utcnow().date().isoformat(),
|
| 961 |
"notes": notes,
|
| 962 |
+
"story_pages_include": "",
|
| 963 |
+
"story_pages_exclude": "",
|
| 964 |
+
"safe_title": _safe_title_slug(Path(path.name).stem),
|
| 965 |
}])
|
| 966 |
df = pd.concat([df, new_row], ignore_index=True)
|
| 967 |
log.append(log_line(f"β Registered {book_id} β {path.name} ({page_count or '?'} pages)"))
|
|
|
|
| 1188 |
SS_TESSERACT_BATCH_HARD_TIMEOUT_SEC = max(15, int(os.environ.get("SS_TESSERACT_BATCH_HARD_TIMEOUT_SEC", "45")))
|
| 1189 |
except Exception:
|
| 1190 |
SS_TESSERACT_BATCH_HARD_TIMEOUT_SEC = 45
|
| 1191 |
+
try:
|
| 1192 |
+
SS_MIXFONT_MAX_DETECT_PAGES = max(1, int(os.environ.get("SS_MIXFONT_MAX_DETECT_PAGES", "2")))
|
| 1193 |
+
except Exception:
|
| 1194 |
+
SS_MIXFONT_MAX_DETECT_PAGES = 2
|
| 1195 |
+
try:
|
| 1196 |
+
SS_MIXFONT_MAX_ERRORS = max(1, int(os.environ.get("SS_MIXFONT_MAX_ERRORS", "1")))
|
| 1197 |
+
except Exception:
|
| 1198 |
+
SS_MIXFONT_MAX_ERRORS = 1
|
| 1199 |
|
| 1200 |
|
| 1201 |
def _load_surya_runtime():
|
|
|
|
| 1343 |
return regions, conf
|
| 1344 |
|
| 1345 |
|
| 1346 |
+
def _run_tesseract_batch(
|
| 1347 |
+
images,
|
| 1348 |
+
langs: Optional[list[str]] = None,
|
| 1349 |
+
tessdata_dirs: Optional[list[Optional[str]]] = None,
|
| 1350 |
+
):
|
| 1351 |
"""
|
| 1352 |
Tesseract fallback for OCR when Surya is unavailable/slow.
|
| 1353 |
Returns list of dicts with regions/confidence/method aligned to input order.
|
|
|
|
| 1355 |
try:
|
| 1356 |
import pytesseract
|
| 1357 |
except Exception as e:
|
| 1358 |
+
return [
|
| 1359 |
+
{
|
| 1360 |
+
"regions": [],
|
| 1361 |
+
"confidence": 0.0,
|
| 1362 |
+
"method": f"error-no-tesseract ({e})",
|
| 1363 |
+
"tesseract_lang": (langs[idx] if langs and idx < len(langs) else "eng"),
|
| 1364 |
+
}
|
| 1365 |
+
for idx, _ in enumerate(images)
|
| 1366 |
+
]
|
| 1367 |
|
| 1368 |
# Prefer parallel image-level OCR with single-threaded internal OpenMP for better CPU utilization.
|
| 1369 |
os.environ.setdefault("OMP_THREAD_LIMIT", "1")
|
| 1370 |
|
| 1371 |
+
def _ocr_single(img, lang_hint: str, tessdata_dir: Optional[str]):
|
| 1372 |
try:
|
| 1373 |
+
config = f"--oem 1 --psm {SS_TESSERACT_PSM}"
|
| 1374 |
+
if tessdata_dir:
|
| 1375 |
+
config = f"{config} --tessdata-dir \"{tessdata_dir}\""
|
| 1376 |
data = pytesseract.image_to_data(
|
| 1377 |
img,
|
| 1378 |
+
lang=lang_hint or "eng",
|
| 1379 |
+
config=config,
|
| 1380 |
output_type=pytesseract.Output.DICT,
|
| 1381 |
timeout=SS_TESSERACT_TIMEOUT_SEC,
|
| 1382 |
)
|
|
|
|
| 1418 |
|
| 1419 |
# Hard gate for fallback quality: do not keep OCR text below minimum page confidence.
|
| 1420 |
if regions and avg_conf < SS_TESSERACT_MIN_CONF_KEEP:
|
| 1421 |
+
return {
|
| 1422 |
+
"regions": [],
|
| 1423 |
+
"confidence": 0.0,
|
| 1424 |
+
"method": "tesseract-lowconf-filtered",
|
| 1425 |
+
"tesseract_lang": lang_hint or "eng",
|
| 1426 |
+
}
|
| 1427 |
|
| 1428 |
# Filter obvious OCR noise from illustration texture (common in no-text pages).
|
| 1429 |
full_text = " ".join(r["text"] for r in regions).strip()
|
|
|
|
| 1433 |
tokens = [t for t in full_text.split() if t]
|
| 1434 |
avg_token_len = (sum(len(t) for t in tokens) / len(tokens)) if tokens else 0.0
|
| 1435 |
if regions and avg_conf <= 0.42 and alpha_ratio < 0.62 and avg_token_len < 3.2:
|
| 1436 |
+
return {
|
| 1437 |
+
"regions": [],
|
| 1438 |
+
"confidence": 0.0,
|
| 1439 |
+
"method": "tesseract-noise-filtered",
|
| 1440 |
+
"tesseract_lang": lang_hint or "eng",
|
| 1441 |
+
}
|
| 1442 |
|
| 1443 |
+
return {
|
| 1444 |
+
"regions": regions,
|
| 1445 |
+
"confidence": avg_conf,
|
| 1446 |
+
"method": "tesseract",
|
| 1447 |
+
"tesseract_lang": lang_hint or "eng",
|
| 1448 |
+
}
|
| 1449 |
except RuntimeError as e:
|
| 1450 |
+
return {
|
| 1451 |
+
"regions": [],
|
| 1452 |
+
"confidence": 0.0,
|
| 1453 |
+
"method": f"error-tesseract-timeout ({e})",
|
| 1454 |
+
"tesseract_lang": lang_hint or "eng",
|
| 1455 |
+
}
|
| 1456 |
except Exception as e:
|
| 1457 |
+
return {
|
| 1458 |
+
"regions": [],
|
| 1459 |
+
"confidence": 0.0,
|
| 1460 |
+
"method": f"error-tesseract ({e})",
|
| 1461 |
+
"tesseract_lang": lang_hint or "eng",
|
| 1462 |
+
}
|
| 1463 |
|
| 1464 |
if not images:
|
| 1465 |
return []
|
| 1466 |
|
| 1467 |
max_workers = min(SS_TESSERACT_WORKERS, len(images), max(1, os.cpu_count() or 1))
|
| 1468 |
+
lang_list = list(langs) if langs else []
|
| 1469 |
+
if len(lang_list) < len(images):
|
| 1470 |
+
lang_list.extend(["eng"] * (len(images) - len(lang_list)))
|
| 1471 |
+
elif len(lang_list) > len(images):
|
| 1472 |
+
lang_list = lang_list[: len(images)]
|
| 1473 |
+
|
| 1474 |
+
tess_dir_list = list(tessdata_dirs) if tessdata_dirs else []
|
| 1475 |
+
if len(tess_dir_list) < len(images):
|
| 1476 |
+
tess_dir_list.extend([None] * (len(images) - len(tess_dir_list)))
|
| 1477 |
+
elif len(tess_dir_list) > len(images):
|
| 1478 |
+
tess_dir_list = tess_dir_list[: len(images)]
|
| 1479 |
+
|
| 1480 |
if max_workers <= 1:
|
| 1481 |
+
return [
|
| 1482 |
+
_ocr_single(img, lang_list[idx], tess_dir_list[idx])
|
| 1483 |
+
for idx, img in enumerate(images)
|
| 1484 |
+
]
|
| 1485 |
|
| 1486 |
outputs = [None] * len(images)
|
| 1487 |
executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_workers)
|
| 1488 |
+
futures = {
|
| 1489 |
+
executor.submit(_ocr_single, img, lang_list[idx], tess_dir_list[idx]): idx
|
| 1490 |
+
for idx, img in enumerate(images)
|
| 1491 |
+
}
|
| 1492 |
pending = set(futures.keys())
|
| 1493 |
batch_timeout = max(
|
| 1494 |
SS_TESSERACT_BATCH_HARD_TIMEOUT_SEC,
|
|
|
|
| 1519 |
"regions": [],
|
| 1520 |
"confidence": 0.0,
|
| 1521 |
"method": f"error-tesseract-batch-timeout ({batch_timeout}s)",
|
| 1522 |
+
"tesseract_lang": lang_list[idx] if idx < len(lang_list) else "eng",
|
| 1523 |
}
|
| 1524 |
future.cancel()
|
| 1525 |
finally:
|
|
|
|
| 1528 |
|
| 1529 |
for i, out in enumerate(outputs):
|
| 1530 |
if out is None:
|
| 1531 |
+
outputs[i] = {
|
| 1532 |
+
"regions": [],
|
| 1533 |
+
"confidence": 0.0,
|
| 1534 |
+
"method": "error-tesseract-missing-output",
|
| 1535 |
+
"tesseract_lang": lang_list[i] if i < len(lang_list) else "eng",
|
| 1536 |
+
}
|
| 1537 |
|
| 1538 |
return outputs
|
| 1539 |
|
|
|
|
| 1569 |
selection_set, selection_error = _parse_page_selection(page_selection)
|
| 1570 |
if selection_error:
|
| 1571 |
return _ocr_status_html(), f"{debug}Invalid page selection: {selection_error}"
|
| 1572 |
+
manual_selection = selection_set is not None
|
| 1573 |
exclusion_spec = (page_exclusion or "").strip().lower()
|
| 1574 |
exclusion_request = None if exclusion_spec in ("", "none", "-") else exclusion_spec
|
| 1575 |
+
manual_exclusion = exclusion_request is not None
|
| 1576 |
if exclusion_request is not None:
|
| 1577 |
exclusion_set, exclusion_error = _parse_page_selection(exclusion_request)
|
| 1578 |
if exclusion_error:
|
|
|
|
| 1612 |
log.append(log_line(f"β Punctuation corrector unavailable ({_PUNCT_MODULE_ERROR or 'unknown'})"))
|
| 1613 |
else:
|
| 1614 |
log.append(log_line("β Punctuation corrector active (rule-check + confidence penalty + learning map)"))
|
| 1615 |
+
font_mod = _load_font_module()
|
| 1616 |
+
mixfont_enabled = False
|
| 1617 |
+
if font_mod is None:
|
| 1618 |
+
log.append(log_line(f"β Font library unavailable ({_FONT_MODULE_ERROR or 'unknown'})"))
|
| 1619 |
+
else:
|
| 1620 |
+
preflight = _mixfont_preflight()
|
| 1621 |
+
if preflight.get("api_key_set") and preflight.get("public_image_url_source_available"):
|
| 1622 |
+
log.append(log_line("β MixFont detection enabled (per-page font routing)"))
|
| 1623 |
+
mixfont_enabled = True
|
| 1624 |
+
else:
|
| 1625 |
+
missing = []
|
| 1626 |
+
if not preflight.get("api_key_set"):
|
| 1627 |
+
missing.append("MIXFONT_API_KEY")
|
| 1628 |
+
if not preflight.get("public_image_url_source_available"):
|
| 1629 |
+
missing.append("MIXFONT_IMAGE_BASE_URL (or MIXFONT_IMAGE_URL_TEMPLATE/SPACE_HOST)")
|
| 1630 |
+
missing_text = ", ".join(missing) if missing else "unknown config"
|
| 1631 |
+
log.append(log_line(f"βΉ MixFont disabled this run β missing {missing_text}"))
|
| 1632 |
|
| 1633 |
for _, row in eligible.iterrows():
|
| 1634 |
book_id = row["book_id"]
|
|
|
|
| 1662 |
if excluded_pages_for_book is None:
|
| 1663 |
excluded_pages_for_book = set(page_numbers)
|
| 1664 |
|
| 1665 |
+
saved_include_spec = _normalize_saved_spec(row.get("story_pages_include", ""))
|
| 1666 |
+
if saved_include_spec and not manual_selection:
|
| 1667 |
+
saved_include_set, saved_inc_err = _parse_page_selection(saved_include_spec, max_page=max_page)
|
| 1668 |
+
if saved_inc_err:
|
| 1669 |
+
log.append(log_line(f"β {book_id}: invalid saved include pages '{saved_include_spec}' ({saved_inc_err})"))
|
| 1670 |
+
elif saved_include_set is not None:
|
| 1671 |
+
selected_pages_for_book = set(selected_pages_for_book) & set(saved_include_set)
|
| 1672 |
+
|
| 1673 |
+
saved_exclude_spec = _normalize_saved_spec(row.get("story_pages_exclude", ""))
|
| 1674 |
+
if saved_exclude_spec and not manual_exclusion:
|
| 1675 |
+
saved_exclude_set, saved_exc_err = _parse_page_selection(saved_exclude_spec, max_page=max_page)
|
| 1676 |
+
if saved_exc_err:
|
| 1677 |
+
log.append(log_line(f"β {book_id}: invalid saved exclude pages '{saved_exclude_spec}' ({saved_exc_err})"))
|
| 1678 |
+
elif saved_exclude_set is not None:
|
| 1679 |
+
excluded_pages_for_book = set(excluded_pages_for_book) | set(saved_exclude_set)
|
| 1680 |
+
|
| 1681 |
selected_pages_for_book = set(selected_pages_for_book) - set(excluded_pages_for_book)
|
| 1682 |
if not selected_pages_for_book:
|
| 1683 |
log.append(log_line(f"β {book_id}: no pages left after applying selection/exclusion"))
|
| 1684 |
continue
|
| 1685 |
|
| 1686 |
+
if saved_include_spec and not manual_selection:
|
| 1687 |
+
log.append(log_line(f"βΉ {book_id}: saved include pages {saved_include_spec}"))
|
| 1688 |
+
if saved_exclude_spec and not manual_exclusion:
|
| 1689 |
+
log.append(log_line(f"βΉ {book_id}: saved exclude pages {saved_exclude_spec}"))
|
| 1690 |
selected_preview = ",".join(str(p) for p in sorted(selected_pages_for_book))
|
| 1691 |
log.append(log_line(f"βΉ {book_id}: effective selected pages {selected_preview}"))
|
| 1692 |
processed_books.append(book_id)
|
|
|
|
| 1708 |
# First pass: render OCR/hybrid pages once and keep PIL images for batch OCR.
|
| 1709 |
ocr_targets = []
|
| 1710 |
render_dpi = SS_RENDER_DPI_SURYA if surya is not None else SS_RENDER_DPI_FALLBACK
|
| 1711 |
+
book_detected_font = None
|
| 1712 |
+
mixfont_attempts = 0
|
| 1713 |
+
mixfont_errors = 0
|
| 1714 |
+
mixfont_disabled_for_book = not mixfont_enabled
|
| 1715 |
for page_data in profile.get("pages", []):
|
| 1716 |
page_num = page_data["page_number"]
|
| 1717 |
route = page_data["route"]
|
| 1718 |
+
font_name = page_data.get("font_name")
|
| 1719 |
if selected_pages_for_book is not None and int(page_num) not in selected_pages_for_book:
|
| 1720 |
continue
|
| 1721 |
|
|
|
|
| 1759 |
except Exception as e:
|
| 1760 |
log.append(log_line(f" β {book_id} p{page_num}: render failed ({e})"))
|
| 1761 |
|
| 1762 |
+
font_name = page_data.get("font_name")
|
| 1763 |
+
if (not font_name) and book_detected_font:
|
| 1764 |
+
page_data["font_name"] = book_detected_font
|
| 1765 |
+
font_name = book_detected_font
|
| 1766 |
+
|
| 1767 |
+
if (
|
| 1768 |
+
(not font_name)
|
| 1769 |
+
and render_path is not None
|
| 1770 |
+
and (not mixfont_disabled_for_book)
|
| 1771 |
+
and mixfont_attempts < SS_MIXFONT_MAX_DETECT_PAGES
|
| 1772 |
+
):
|
| 1773 |
+
mixfont_attempts += 1
|
| 1774 |
+
font_result = _identify_page_font(str(render_path))
|
| 1775 |
+
detected_font = font_result.get("font_name")
|
| 1776 |
+
if detected_font:
|
| 1777 |
+
page_data["font_name"] = detected_font
|
| 1778 |
+
page_data["font_confidence"] = float(font_result.get("confidence", 0.0) or 0.0)
|
| 1779 |
+
page_data["font_detected_at"] = datetime.utcnow().isoformat() + "Z"
|
| 1780 |
+
book_detected_font = detected_font
|
| 1781 |
+
font_name = detected_font
|
| 1782 |
+
log.append(log_line(f"βΉ {book_id}: detected font '{detected_font}'"))
|
| 1783 |
+
else:
|
| 1784 |
+
page_data["font_name"] = None
|
| 1785 |
+
page_data["font_detect_error"] = font_result.get("error")
|
| 1786 |
+
mixfont_errors += 1
|
| 1787 |
+
if mixfont_errors >= SS_MIXFONT_MAX_ERRORS:
|
| 1788 |
+
mixfont_disabled_for_book = True
|
| 1789 |
+
err_txt = page_data.get("font_detect_error") or "unknown"
|
| 1790 |
+
log.append(log_line(f"β {book_id}: MixFont disabled for this run ({err_txt})"))
|
| 1791 |
+
|
| 1792 |
ocr_targets.append({
|
| 1793 |
"page_num": page_num,
|
| 1794 |
"route": route,
|
| 1795 |
"render_path": page_data.get("render_path"),
|
| 1796 |
+
"font_name": page_data.get("font_name"),
|
| 1797 |
})
|
| 1798 |
|
| 1799 |
# Batch OCR for non-embedded pages.
|
|
|
|
| 1805 |
batch_pages = [item["page_num"] for item in batch]
|
| 1806 |
batch_images = []
|
| 1807 |
batch_items_with_images = []
|
| 1808 |
+
batch_tess_langs = []
|
| 1809 |
+
batch_tess_dirs = []
|
| 1810 |
try:
|
| 1811 |
from PIL import Image
|
| 1812 |
except Exception as e:
|
|
|
|
| 1837 |
img = Image.open(render_abs).convert("RGB")
|
| 1838 |
batch_images.append(img)
|
| 1839 |
batch_items_with_images.append(item)
|
| 1840 |
+
lang_model = _resolve_tesseract_lang(item.get("font_name"))
|
| 1841 |
+
batch_tess_langs.append(lang_model)
|
| 1842 |
+
batch_tess_dirs.append(_resolve_tessdata_dir(lang_model))
|
| 1843 |
except Exception as e:
|
| 1844 |
ocr_lookup[item["page_num"]] = {
|
| 1845 |
"regions": [],
|
|
|
|
| 1861 |
"method": "surya",
|
| 1862 |
}
|
| 1863 |
else:
|
| 1864 |
+
fallback_preds = _run_tesseract_batch(batch_images, batch_tess_langs, batch_tess_dirs)
|
| 1865 |
timeout_count = sum(1 for pred in fallback_preds if "batch-timeout" in str(pred.get("method", "")))
|
| 1866 |
for item, pred in zip(batch_items_with_images, fallback_preds):
|
| 1867 |
ocr_lookup[item["page_num"]] = pred
|
|
|
|
| 1876 |
# If Surya batch fails, try Tesseract for this batch before giving up.
|
| 1877 |
if surya is not None:
|
| 1878 |
log.append(log_line(f" β {book_id} batch {batch_pages[0]}-{batch_pages[-1]} Surya error: {e}; retrying with Tesseract"))
|
| 1879 |
+
fallback_preds = _run_tesseract_batch(batch_images, batch_tess_langs, batch_tess_dirs)
|
| 1880 |
timeout_count = sum(1 for pred in fallback_preds if "batch-timeout" in str(pred.get("method", "")))
|
| 1881 |
for item, pred in zip(batch_items_with_images, fallback_preds):
|
| 1882 |
ocr_lookup[item["page_num"]] = pred
|
|
|
|
| 1932 |
method = "skipped-no-surya"
|
| 1933 |
|
| 1934 |
raw_text = " ".join(r["text"] for r in regions)[:500]
|
| 1935 |
+
corrected_text, punct_flags, punct_score = _apply_punctuation_corrections(
|
| 1936 |
+
raw_text,
|
| 1937 |
+
book_id,
|
| 1938 |
+
font_name=font_name,
|
| 1939 |
+
)
|
| 1940 |
conf_adjusted = _punctuation_confidence_penalty(corrected_text, conf)
|
| 1941 |
+
if font_name:
|
| 1942 |
+
if str(method).startswith("tesseract"):
|
| 1943 |
+
_update_font_engine_stats(font_name, 0.0, conf)
|
| 1944 |
+
elif str(method).startswith("surya"):
|
| 1945 |
+
_update_font_engine_stats(font_name, conf, 0.0)
|
| 1946 |
|
| 1947 |
if method in ("tesseract-noise-filtered", "tesseract-lowconf-filtered") and not regions:
|
| 1948 |
conf_class = "auto-accept"
|
|
|
|
| 1974 |
"extraction_method": method,
|
| 1975 |
"punctuation_score": punct_score,
|
| 1976 |
"punctuation_flags": punct_flags,
|
| 1977 |
+
"font_name": font_name,
|
| 1978 |
+
"tesseract_lang": ocr_lookup.get(page_num, {}).get("tesseract_lang"),
|
| 1979 |
"render_path": page_data.get("render_path"),
|
| 1980 |
"ocred_at": datetime.utcnow().isoformat() + "Z",
|
| 1981 |
})
|
|
|
|
| 1996 |
"confidence_class": conf_class,
|
| 1997 |
"punct_score": punct_score,
|
| 1998 |
"punct_flags_count": len(punct_flags),
|
| 1999 |
+
"font_name": font_name or "",
|
| 2000 |
+
"tesseract_lang": ocr_lookup.get(page_num, {}).get("tesseract_lang", ""),
|
| 2001 |
"status": "quarantine" if conf_class == "quarantine" else "pending",
|
| 2002 |
"reviewer": "",
|
| 2003 |
"correction": "",
|
|
|
|
| 2132 |
except Exception:
|
| 2133 |
img_path = None
|
| 2134 |
|
| 2135 |
+
font_suffix = f" Β· font: {item.get('font_name')}" if str(item.get("font_name", "")).strip() else ""
|
| 2136 |
info = (f"<div style='font-family:monospace;font-size:11px;color:var(--ss-muted)'>"
|
| 2137 |
+
f"{item['book_id']} Β· page {item['page']}{font_suffix} Β· "
|
| 2138 |
f"conf: <b style='color:{'var(--ss-red)' if float(item.get('confidence',0)) < 0.6 else 'var(--ss-gold)'}'>"
|
| 2139 |
f"{float(item.get('confidence',0)):.0%}</b></div>")
|
| 2140 |
|
|
|
|
| 2176 |
"raw_text_original": raw_text_original,
|
| 2177 |
"reason_code": reason,
|
| 2178 |
"reviewer": reviewer or "reviewer",
|
| 2179 |
+
"font_name": item.get("font_name", ""),
|
| 2180 |
"was_correct": was_correct,
|
| 2181 |
"decided_at": datetime.utcnow().isoformat() + "Z",
|
| 2182 |
}
|
|
|
|
| 2211 |
|
| 2212 |
learned_pairs = 0
|
| 2213 |
if action == "edited":
|
| 2214 |
+
learned_pairs = _record_punctuation_correction(
|
| 2215 |
+
raw_text_original,
|
| 2216 |
+
final_text,
|
| 2217 |
+
item.get("book_id", ""),
|
| 2218 |
+
font_name=item.get("font_name"),
|
| 2219 |
+
)
|
| 2220 |
|
| 2221 |
cal = load_calibration()
|
| 2222 |
default = cal.get("_default", DEFAULT_CALIBRATION["_default"])
|
|
|
|
| 2278 |
|
| 2279 |
records.append({
|
| 2280 |
"book_id": book_id,
|
| 2281 |
+
"safe_title": manifest_row.get("safe_title","") if isinstance(manifest_row, pd.Series) else "",
|
| 2282 |
"source_hash": manifest_row.get("sha256","") if isinstance(manifest_row, pd.Series) else "",
|
| 2283 |
"page_number": dec["page"],
|
| 2284 |
"region_id": dec["region_id"],
|
|
|
|
| 2398 |
interactive=False,
|
| 2399 |
wrap=True,
|
| 2400 |
)
|
| 2401 |
+
ingest_log = gr.Textbox(label="Log", lines=6, interactive=False, elem_classes=["ss-log"], show_copy_button=True)
|
| 2402 |
refresh_btn = gr.Button("β» Refresh Manifest", size="sm")
|
| 2403 |
refresh_btn.click(lambda: load_manifest_df(), outputs=[manifest_table])
|
| 2404 |
|
|
|
|
| 2418 |
outputs=[ingest_status, manifest_table, ingest_log],
|
| 2419 |
)
|
| 2420 |
|
| 2421 |
+
gr.Markdown("**Save Book Pages + Safe Title (persisted):**")
|
| 2422 |
+
with gr.Row():
|
| 2423 |
+
scope_book_id = gr.Textbox(label="Book ID", placeholder="SS-BOOK-0001", scale=1)
|
| 2424 |
+
scope_include = gr.Textbox(label="Story Pages Include", placeholder="all or 7-27 or 7,8,9,11-27", scale=1)
|
| 2425 |
+
scope_exclude = gr.Textbox(label="Story Pages Exclude", placeholder="e.g. 1,2,3,17,25", scale=1)
|
| 2426 |
+
scope_title = gr.Textbox(label="Safe Book Title", placeholder="e.g. the-gruffalo", scale=1)
|
| 2427 |
+
scope_save_btn = gr.Button("Save Book Pages β", size="sm")
|
| 2428 |
+
scope_save_btn.click(
|
| 2429 |
+
save_book_scope,
|
| 2430 |
+
inputs=[scope_book_id, scope_include, scope_exclude, scope_title],
|
| 2431 |
+
outputs=[ingest_status, manifest_table, ingest_log],
|
| 2432 |
+
)
|
| 2433 |
+
|
| 2434 |
ingest_btn.click(
|
| 2435 |
ingest_pdfs,
|
| 2436 |
inputs=[pdf_upload, rights_dd, ingest_notes],
|
|
|
|
| 2449 |
|
| 2450 |
profile_status = gr.HTML(_profile_status_html())
|
| 2451 |
profile_btn = gr.Button("Re-run Profiler (Optional) β", elem_classes=["ss-btn-run"])
|
| 2452 |
+
profile_log = gr.Textbox(label="Log", lines=10, interactive=False, elem_classes=["ss-log"], show_copy_button=True)
|
| 2453 |
|
| 2454 |
profile_btn.click(run_profile, outputs=[profile_status, profile_log])
|
| 2455 |
|
|
|
|
| 2487 |
value=True,
|
| 2488 |
)
|
| 2489 |
ocr_btn = gr.Button("Run OCR β", elem_classes=["ss-btn-run"])
|
| 2490 |
+
ocr_log = gr.Textbox(label="Log", lines=10, interactive=False, elem_classes=["ss-log"], show_copy_button=True)
|
| 2491 |
|
| 2492 |
ocr_run_event = ocr_btn.click(
|
| 2493 |
run_ocr,
|
| 2494 |
inputs=[ocr_page_selection, ocr_page_exclusion, ocr_replace_queue],
|
| 2495 |
outputs=[ocr_status, ocr_log],
|
| 2496 |
+
show_progress="hidden",
|
| 2497 |
)
|
| 2498 |
|
| 2499 |
# ββ STEP 4: REVIEW ββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 2592 |
ocr_run_event.then(
|
| 2593 |
load_review,
|
| 2594 |
outputs=review_load_outputs,
|
| 2595 |
+
show_progress="hidden",
|
| 2596 |
)
|
| 2597 |
accept_btn.click(do_accept, inputs=[current_idx, final_text_box, reviewer_name, reason_code], outputs=action_outputs)
|
| 2598 |
edit_btn.click(do_accept, inputs=[current_idx, final_text_box, reviewer_name, reason_code], outputs=action_outputs)
|