Pointf5ive commited on
Commit
7152efa
Β·
1 Parent(s): d109898

Add persistent book page scope, safe titles, and OCR stall guards

Browse files
smoke_signal/scripts/font_library.py ADDED
@@ -0,0 +1,423 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Smoke Signal β€” Font Detection + Font-Aware OCR Helpers
3
+ ======================================================
4
+ Lightweight utilities for:
5
+ 1) identifying page fonts via MixFont Lens API
6
+ 2) maintaining per-font OCR preference metadata
7
+ 3) selecting per-font custom Tesseract models when available
8
+ 4) exposing font-specific punctuation-map paths
9
+
10
+ This module is intentionally defensive:
11
+ - if MixFont key is missing, it degrades gracefully
12
+ - if no custom .traineddata exists, it falls back to "eng"
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import json
18
+ import os
19
+ import urllib.parse
20
+ import urllib.request
21
+ from pathlib import Path
22
+ from typing import Optional
23
+
24
+ SS_ROOT = Path(os.environ.get("SS_DATA_ROOT", os.environ.get("SS_ROOT", "/tmp/smoke_signal")))
25
+ CALIBRATION_DIR = SS_ROOT / "calibration"
26
+ FONTS_DIR = SS_ROOT / "fonts"
27
+ TESSDATA_DIR = SS_ROOT / "tessdata"
28
+ REPO_TESSDATA_DIR = Path(__file__).resolve().parents[1] / "tessdata"
29
+ FONT_REGISTRY_PATH = CALIBRATION_DIR / "font_registry.json"
30
+
31
+ for _dir in (CALIBRATION_DIR, FONTS_DIR, TESSDATA_DIR, REPO_TESSDATA_DIR):
32
+ _dir.mkdir(parents=True, exist_ok=True)
33
+
34
+ MIXFONT_API_KEY = os.environ.get("MIXFONT_API_KEY", "").strip()
35
+ # MixFont Lens API docs endpoint.
36
+ MIXFONT_API_URL = os.environ.get("MIXFONT_API_URL", "https://www.mixfont.com/v1/api/lens").strip()
37
+ try:
38
+ MIXFONT_TIMEOUT_SEC = max(1.0, float(os.environ.get("MIXFONT_TIMEOUT_SEC", "4")))
39
+ except Exception:
40
+ MIXFONT_TIMEOUT_SEC = 4.0
41
+
42
+
43
+ def _slug_font_name(font_name: str) -> str:
44
+ cleaned = "".join(ch.lower() if ch.isalnum() else "_" for ch in (font_name or "").strip())
45
+ while "__" in cleaned:
46
+ cleaned = cleaned.replace("__", "_")
47
+ return cleaned.strip("_")
48
+
49
+
50
+ def _load_registry() -> dict:
51
+ if not FONT_REGISTRY_PATH.exists():
52
+ return {}
53
+ try:
54
+ return json.loads(FONT_REGISTRY_PATH.read_text(encoding="utf-8"))
55
+ except Exception:
56
+ return {}
57
+
58
+
59
+ def _save_registry(registry: dict) -> None:
60
+ FONT_REGISTRY_PATH.write_text(
61
+ json.dumps(registry, indent=2, ensure_ascii=False),
62
+ encoding="utf-8",
63
+ )
64
+
65
+
66
+ def _ensure_registry_entry(font_name: str) -> dict:
67
+ registry = _load_registry()
68
+ entry = registry.get(font_name, {})
69
+ if not entry:
70
+ entry = {
71
+ "identified_count": 0,
72
+ "status": "unknown", # unknown | available | missing | specimen_only
73
+ "preferred_engine": "surya", # surya | tesseract
74
+ "tesseract_model": None,
75
+ "avg_confidence_surya": None,
76
+ "avg_confidence_tess": None,
77
+ "last_identified_at": None,
78
+ }
79
+ registry[font_name] = entry
80
+ _save_registry(registry)
81
+ return entry
82
+
83
+
84
+ def _build_public_image_url(local_image_path: str) -> Optional[str]:
85
+ """
86
+ Construct a publicly reachable URL for a local rendered page image.
87
+ Priority:
88
+ 1) MIXFONT_IMAGE_URL_TEMPLATE with {path} placeholder
89
+ 2) MIXFONT_IMAGE_BASE_URL + /gradio_api/file=<abs_path>
90
+ 3) SPACE_HOST + /gradio_api/file=<abs_path>
91
+ 4) HF_SPACE_URL + /gradio_api/file=<abs_path>
92
+ """
93
+ raw_path = str(Path(local_image_path).resolve())
94
+ encoded_path = urllib.parse.quote(raw_path, safe="")
95
+
96
+ template = os.environ.get("MIXFONT_IMAGE_URL_TEMPLATE", "").strip()
97
+ if template:
98
+ if "{path}" in template:
99
+ return template.replace("{path}", encoded_path)
100
+ return template
101
+
102
+ base = (
103
+ os.environ.get("MIXFONT_IMAGE_BASE_URL", "").strip()
104
+ or os.environ.get("HF_SPACE_URL", "").strip()
105
+ or ""
106
+ )
107
+ if not base:
108
+ space_id = os.environ.get("SPACE_ID", "").strip()
109
+ if space_id:
110
+ base = f"https://huggingface.co/spaces/{space_id}"
111
+ if not base:
112
+ space_host = os.environ.get("SPACE_HOST", "").strip()
113
+ if space_host:
114
+ if space_host.startswith("http://") or space_host.startswith("https://"):
115
+ base = space_host
116
+ else:
117
+ base = f"https://{space_host}"
118
+
119
+ if not base:
120
+ return None
121
+
122
+ return f"{base.rstrip('/')}/gradio_api/file={encoded_path}"
123
+
124
+
125
+ def mixfont_preflight() -> dict:
126
+ """
127
+ Return runtime readiness checks for MixFont integration.
128
+ """
129
+ template = os.environ.get("MIXFONT_IMAGE_URL_TEMPLATE", "").strip()
130
+ base = (
131
+ os.environ.get("MIXFONT_IMAGE_BASE_URL", "").strip()
132
+ or os.environ.get("HF_SPACE_URL", "").strip()
133
+ or ""
134
+ )
135
+ if not base:
136
+ space_id = os.environ.get("SPACE_ID", "").strip()
137
+ if space_id:
138
+ base = f"https://huggingface.co/spaces/{space_id}"
139
+ space_host = os.environ.get("SPACE_HOST", "").strip()
140
+ has_public_source = bool(template or base or space_host)
141
+ return {
142
+ "api_key_set": bool(MIXFONT_API_KEY),
143
+ "api_url": MIXFONT_API_URL,
144
+ "image_url_template_set": bool(template),
145
+ "image_base_set": bool(base),
146
+ "space_host_set": bool(space_host),
147
+ "public_image_url_source_available": has_public_source,
148
+ }
149
+
150
+
151
+ def identify_page_font(image_path: str, image_url: Optional[str] = None) -> dict:
152
+ """
153
+ Identify font via MixFont Lens API.
154
+ Returns:
155
+ {
156
+ "font_name": str|None,
157
+ "confidence": float,
158
+ "alternatives": list[str],
159
+ "image_url": str|None,
160
+ "error": str|None
161
+ }
162
+ """
163
+ if not MIXFONT_API_KEY:
164
+ return {
165
+ "font_name": None,
166
+ "confidence": 0.0,
167
+ "alternatives": [],
168
+ "image_url": image_url,
169
+ "error": "MIXFONT_API_KEY not set",
170
+ }
171
+
172
+ path_obj = Path(image_path)
173
+ if not path_obj.exists():
174
+ return {
175
+ "font_name": None,
176
+ "confidence": 0.0,
177
+ "alternatives": [],
178
+ "image_url": image_url,
179
+ "error": f"image not found: {image_path}",
180
+ }
181
+
182
+ resolved_url = image_url or _build_public_image_url(str(path_obj))
183
+ if not resolved_url:
184
+ return {
185
+ "font_name": None,
186
+ "confidence": 0.0,
187
+ "alternatives": [],
188
+ "image_url": None,
189
+ "error": "no public image URL (set MIXFONT_IMAGE_BASE_URL or MIXFONT_IMAGE_URL_TEMPLATE)",
190
+ }
191
+
192
+ payload = json.dumps({"image_url": resolved_url}).encode("utf-8")
193
+ request = urllib.request.Request(
194
+ MIXFONT_API_URL,
195
+ data=payload,
196
+ method="POST",
197
+ headers={
198
+ "Content-Type": "application/json",
199
+ "x-api-key": MIXFONT_API_KEY,
200
+ },
201
+ )
202
+
203
+ try:
204
+ with urllib.request.urlopen(request, timeout=MIXFONT_TIMEOUT_SEC) as response:
205
+ body = response.read().decode("utf-8", errors="replace")
206
+ data = json.loads(body)
207
+ except Exception as e:
208
+ return {
209
+ "font_name": None,
210
+ "confidence": 0.0,
211
+ "alternatives": [],
212
+ "image_url": resolved_url,
213
+ "error": str(e),
214
+ }
215
+
216
+ matches = data.get("font_matches", [])
217
+ if not isinstance(matches, list):
218
+ matches = []
219
+
220
+ top = matches[0] if matches else {}
221
+ if not isinstance(top, dict):
222
+ top = {}
223
+
224
+ font_name = str(top.get("name") or "").strip() or None
225
+ try:
226
+ confidence = float(top.get("confidence", 0.0))
227
+ except Exception:
228
+ confidence = 0.0
229
+
230
+ alternatives = []
231
+ for match in matches[1:4]:
232
+ if isinstance(match, dict):
233
+ name = str(match.get("name") or "").strip()
234
+ if name:
235
+ alternatives.append(name)
236
+
237
+ if font_name:
238
+ registry = _load_registry()
239
+ entry = registry.get(font_name, _ensure_registry_entry(font_name))
240
+ entry["identified_count"] = int(entry.get("identified_count", 0)) + 1
241
+ entry["last_identified_at"] = str(data.get("datetime") or "")
242
+ registry[font_name] = entry
243
+ _save_registry(registry)
244
+
245
+ return {
246
+ "font_name": font_name,
247
+ "confidence": round(confidence, 4),
248
+ "alternatives": alternatives,
249
+ "image_url": resolved_url,
250
+ "error": None,
251
+ }
252
+
253
+
254
+ def register_font_model(
255
+ font_name: str,
256
+ model_name: Optional[str] = None,
257
+ preferred_engine: str = "tesseract",
258
+ status: str = "available",
259
+ ) -> dict:
260
+ """
261
+ Manually register a font->tesseract model mapping.
262
+ Use this for bespoke fonts where only a specimen-based traineddata exists.
263
+ """
264
+ if not font_name:
265
+ return {"ok": False, "error": "font_name required"}
266
+
267
+ model = (model_name or _slug_font_name(font_name) or "eng").strip()
268
+ registry = _load_registry()
269
+ entry = registry.get(font_name, _ensure_registry_entry(font_name))
270
+ entry["tesseract_model"] = model
271
+ entry["preferred_engine"] = preferred_engine if preferred_engine in {"surya", "tesseract"} else "tesseract"
272
+ entry["status"] = status
273
+ registry[font_name] = entry
274
+ _save_registry(registry)
275
+ return {"ok": True, "font_name": font_name, "tesseract_model": model}
276
+
277
+
278
+ def resolve_tesseract_lang(font_name: Optional[str]) -> str:
279
+ """
280
+ Return best tesseract language/model to use for this font.
281
+ Defaults to "eng" if no custom traineddata is available.
282
+ """
283
+ if not font_name:
284
+ return "eng"
285
+
286
+ registry = _load_registry()
287
+ entry = registry.get(font_name, {})
288
+ candidates = []
289
+
290
+ explicit_model = str(entry.get("tesseract_model") or "").strip()
291
+ if explicit_model:
292
+ candidates.append(explicit_model)
293
+
294
+ slug = _slug_font_name(font_name)
295
+ if slug:
296
+ candidates.append(slug)
297
+ candidates.append(slug.replace("_", ""))
298
+
299
+ # unique keep order
300
+ seen = set()
301
+ deduped = []
302
+ for c in candidates:
303
+ if c and c not in seen:
304
+ deduped.append(c)
305
+ seen.add(c)
306
+
307
+ tess_dirs = [TESSDATA_DIR, REPO_TESSDATA_DIR]
308
+ for model_name in deduped:
309
+ if any((tess_dir / f"{model_name}.traineddata").exists() for tess_dir in tess_dirs):
310
+ if entry:
311
+ entry["status"] = "available"
312
+ entry["preferred_engine"] = "tesseract"
313
+ entry["tesseract_model"] = model_name
314
+ registry[font_name] = entry
315
+ _save_registry(registry)
316
+ return model_name
317
+
318
+ # Common misplacement: .traineddata.otf (font file, not Tesseract model)
319
+ otf_like_candidates = []
320
+ for model_name in deduped:
321
+ otf_like_candidates.extend(
322
+ [
323
+ REPO_TESSDATA_DIR / f"{model_name}.traineddata.otf",
324
+ TESSDATA_DIR / f"{model_name}.traineddata.otf",
325
+ REPO_TESSDATA_DIR / f"{model_name}.otf",
326
+ TESSDATA_DIR / f"{model_name}.otf",
327
+ ]
328
+ )
329
+ if any(path.exists() for path in otf_like_candidates):
330
+ if entry:
331
+ entry["status"] = "font_file_only"
332
+ entry["preferred_engine"] = "surya"
333
+ registry[font_name] = entry
334
+ _save_registry(registry)
335
+
336
+ # If a specimen exists but no model compiled yet, make that explicit in registry.
337
+ if (FONTS_DIR / "gruffalo_specimen.png").exists():
338
+ if entry:
339
+ entry["status"] = entry.get("status") or "specimen_only"
340
+ if entry["status"] == "unknown":
341
+ entry["status"] = "specimen_only"
342
+ registry[font_name] = entry
343
+ _save_registry(registry)
344
+
345
+ return "eng"
346
+
347
+
348
+ def resolve_tessdata_dir(model_name: Optional[str]) -> Optional[str]:
349
+ if not model_name:
350
+ return None
351
+ model = str(model_name).strip()
352
+ if not model:
353
+ return None
354
+ for tess_dir in (TESSDATA_DIR, REPO_TESSDATA_DIR):
355
+ if (tess_dir / f"{model}.traineddata").exists():
356
+ return str(tess_dir)
357
+ return None
358
+
359
+
360
+ def update_font_engine_stats(font_name: Optional[str], surya_conf: float, tess_conf: float) -> None:
361
+ if not font_name:
362
+ return
363
+ registry = _load_registry()
364
+ entry = registry.get(font_name, _ensure_registry_entry(font_name))
365
+ alpha = 0.3
366
+
367
+ prev_surya = entry.get("avg_confidence_surya")
368
+ prev_tess = entry.get("avg_confidence_tess")
369
+ try:
370
+ prev_surya_val = float(prev_surya) if prev_surya is not None else None
371
+ except Exception:
372
+ prev_surya_val = None
373
+ try:
374
+ prev_tess_val = float(prev_tess) if prev_tess is not None else None
375
+ except Exception:
376
+ prev_tess_val = None
377
+
378
+ surya_new = float(surya_conf or 0.0)
379
+ tess_new = float(tess_conf or 0.0)
380
+
381
+ surya_avg = surya_new if prev_surya_val is None else round(alpha * surya_new + (1 - alpha) * prev_surya_val, 4)
382
+ tess_avg = tess_new if prev_tess_val is None else round(alpha * tess_new + (1 - alpha) * prev_tess_val, 4)
383
+
384
+ entry["avg_confidence_surya"] = surya_avg
385
+ entry["avg_confidence_tess"] = tess_avg
386
+ entry["preferred_engine"] = "tesseract" if tess_avg > surya_avg else "surya"
387
+ if entry.get("status") == "unknown":
388
+ entry["status"] = "available"
389
+
390
+ registry[font_name] = entry
391
+ _save_registry(registry)
392
+
393
+
394
+ def font_correction_map_path(font_name: Optional[str]) -> Path:
395
+ """
396
+ Font-specific punctuation learning map path.
397
+ """
398
+ if not font_name:
399
+ return CALIBRATION_DIR / "punct_correction_map.json"
400
+ safe = _slug_font_name(font_name) or "unknown"
401
+ return CALIBRATION_DIR / f"punct_correction_map_{safe}.json"
402
+
403
+
404
+ def font_registry_summary() -> list[dict]:
405
+ registry = _load_registry()
406
+ rows = []
407
+ for name, row in sorted(
408
+ registry.items(),
409
+ key=lambda x: int(x[1].get("identified_count", 0)),
410
+ reverse=True,
411
+ ):
412
+ rows.append(
413
+ {
414
+ "font_name": name,
415
+ "pages_seen": int(row.get("identified_count", 0)),
416
+ "status": str(row.get("status", "unknown")),
417
+ "preferred_engine": str(row.get("preferred_engine", "surya")),
418
+ "tesseract_model": row.get("tesseract_model"),
419
+ "surya_avg": row.get("avg_confidence_surya"),
420
+ "tess_avg": row.get("avg_confidence_tess"),
421
+ }
422
+ )
423
+ return rows
smoke_signal/scripts/punct_corrector.py CHANGED
@@ -38,17 +38,36 @@ KNOWN_SUBSTITUTIONS: dict[str, str] = {
38
  MIN_PUNCT_PER_10_WORDS = 0.8
39
 
40
 
41
- def _load_correction_map() -> dict[str, dict[str, int]]:
42
- if CORRECTION_MAP_PATH.exists():
 
 
43
  try:
44
- return json.loads(CORRECTION_MAP_PATH.read_text(encoding="utf-8"))
 
 
 
 
 
 
 
 
 
 
 
45
  except Exception:
46
  return {}
47
  return {}
48
 
49
 
50
- def _save_correction_map(cmap: dict[str, dict[str, int]]) -> None:
51
- CORRECTION_MAP_PATH.write_text(json.dumps(cmap, indent=2, ensure_ascii=False), encoding="utf-8")
 
 
 
 
 
 
52
 
53
 
54
  def _top_substitution(cmap: dict[str, dict[str, int]], raw_char: str) -> Optional[str]:
@@ -153,7 +172,13 @@ def _align_chars(raw: str, gold: str) -> list[tuple[str, str]]:
153
  return out
154
 
155
 
156
- def record_punctuation_correction(raw_text: str, gold_text: str, book_id: str = "") -> int:
 
 
 
 
 
 
157
  if not raw_text or not gold_text or raw_text == gold_text:
158
  return 0
159
 
@@ -161,12 +186,12 @@ def record_punctuation_correction(raw_text: str, gold_text: str, book_id: str =
161
  if not pairs:
162
  return 0
163
 
164
- cmap = _load_correction_map()
165
  for raw_char, gold_char in pairs:
166
  if raw_char not in cmap:
167
  cmap[raw_char] = {}
168
  cmap[raw_char][gold_char] = int(cmap[raw_char].get(gold_char, 0)) + 1
169
- _save_correction_map(cmap)
170
 
171
  entry = {
172
  "ts": datetime.now(timezone.utc).isoformat(),
@@ -179,12 +204,17 @@ def record_punctuation_correction(raw_text: str, gold_text: str, book_id: str =
179
  return len(pairs)
180
 
181
 
182
- def apply_punctuation_corrections(raw_text: str, book_id: str = "") -> tuple[str, list[dict], float]:
 
 
 
 
 
183
  _ = book_id
184
  if not raw_text:
185
  return raw_text, [], 1.0
186
 
187
- cmap = _load_correction_map()
188
 
189
  lines = raw_text.splitlines()
190
  corrected_lines: list[str] = []
@@ -219,8 +249,8 @@ def apply_punctuation_corrections(raw_text: str, book_id: str = "") -> tuple[str
219
  return text, flags, punct_score
220
 
221
 
222
- def correction_map_summary() -> dict:
223
- cmap = _load_correction_map()
224
  pairs: list[dict] = []
225
  for raw_char, golds in cmap.items():
226
  for gold_char, count in golds.items():
@@ -229,5 +259,5 @@ def correction_map_summary() -> dict:
229
  return {
230
  "total_pairs": len(pairs),
231
  "top_substitutions": pairs[:20],
232
- "map_path": str(CORRECTION_MAP_PATH),
233
  }
 
38
  MIN_PUNCT_PER_10_WORDS = 0.8
39
 
40
 
41
+ def _resolve_map_path(font_name: Optional[str] = None, map_path: Optional[Path] = None) -> Path:
42
+ if map_path is not None:
43
+ return Path(map_path)
44
+ if font_name:
45
  try:
46
+ from smoke_signal.scripts.font_library import font_correction_map_path
47
+ return Path(font_correction_map_path(font_name))
48
+ except Exception:
49
+ pass
50
+ return CORRECTION_MAP_PATH
51
+
52
+
53
+ def _load_correction_map(font_name: Optional[str] = None, map_path: Optional[Path] = None) -> dict[str, dict[str, int]]:
54
+ target = _resolve_map_path(font_name=font_name, map_path=map_path)
55
+ if target.exists():
56
+ try:
57
+ return json.loads(target.read_text(encoding="utf-8"))
58
  except Exception:
59
  return {}
60
  return {}
61
 
62
 
63
+ def _save_correction_map(
64
+ cmap: dict[str, dict[str, int]],
65
+ font_name: Optional[str] = None,
66
+ map_path: Optional[Path] = None,
67
+ ) -> None:
68
+ target = _resolve_map_path(font_name=font_name, map_path=map_path)
69
+ target.parent.mkdir(parents=True, exist_ok=True)
70
+ target.write_text(json.dumps(cmap, indent=2, ensure_ascii=False), encoding="utf-8")
71
 
72
 
73
  def _top_substitution(cmap: dict[str, dict[str, int]], raw_char: str) -> Optional[str]:
 
172
  return out
173
 
174
 
175
+ def record_punctuation_correction(
176
+ raw_text: str,
177
+ gold_text: str,
178
+ book_id: str = "",
179
+ font_name: Optional[str] = None,
180
+ map_path: Optional[Path] = None,
181
+ ) -> int:
182
  if not raw_text or not gold_text or raw_text == gold_text:
183
  return 0
184
 
 
186
  if not pairs:
187
  return 0
188
 
189
+ cmap = _load_correction_map(font_name=font_name, map_path=map_path)
190
  for raw_char, gold_char in pairs:
191
  if raw_char not in cmap:
192
  cmap[raw_char] = {}
193
  cmap[raw_char][gold_char] = int(cmap[raw_char].get(gold_char, 0)) + 1
194
+ _save_correction_map(cmap, font_name=font_name, map_path=map_path)
195
 
196
  entry = {
197
  "ts": datetime.now(timezone.utc).isoformat(),
 
204
  return len(pairs)
205
 
206
 
207
+ def apply_punctuation_corrections(
208
+ raw_text: str,
209
+ book_id: str = "",
210
+ font_name: Optional[str] = None,
211
+ map_path: Optional[Path] = None,
212
+ ) -> tuple[str, list[dict], float]:
213
  _ = book_id
214
  if not raw_text:
215
  return raw_text, [], 1.0
216
 
217
+ cmap = _load_correction_map(font_name=font_name, map_path=map_path)
218
 
219
  lines = raw_text.splitlines()
220
  corrected_lines: list[str] = []
 
249
  return text, flags, punct_score
250
 
251
 
252
+ def correction_map_summary(font_name: Optional[str] = None, map_path: Optional[Path] = None) -> dict:
253
+ cmap = _load_correction_map(font_name=font_name, map_path=map_path)
254
  pairs: list[dict] = []
255
  for raw_char, golds in cmap.items():
256
  for gold_char, count in golds.items():
 
259
  return {
260
  "total_pairs": len(pairs),
261
  "top_substitutions": pairs[:20],
262
+ "map_path": str(_resolve_map_path(font_name=font_name, map_path=map_path)),
263
  }
smoke_signal/tessdata/README.md ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Smoke Signal Tessdata Drop Folder
2
+
3
+ Put custom Tesseract model files here as `.traineddata`.
4
+
5
+ Naming rule:
6
+ - Use lowercase + underscores based on detected font name.
7
+ - Example: `Gill Sans Infant` -> `gill_sans_infant.traineddata`
8
+
9
+ The OCR runtime will auto-pick models from either:
10
+ - `/tmp/smoke_signal/tessdata/`
11
+ - `smoke_signal/tessdata/` (this folder)
12
+
13
+ Important:
14
+ - `.otf` / `.ttf` files are not Tesseract models.
15
+ - A file like `gill_sans_infant.traineddata.otf` is still a font file, not usable by Tesseract OCR.
16
+
17
+ If your model name is custom and does not match slug, register it once:
18
+
19
+ ```python
20
+ from smoke_signal.scripts.font_library import register_font_model
21
+ register_font_model("Gill Sans Infant", model_name="my_model_name")
22
+ ```
smoke_signal_tab.py CHANGED
@@ -27,6 +27,7 @@ import hashlib
27
  import importlib.util
28
  import json
29
  import os
 
30
  import tempfile
31
  import threading
32
  import time
@@ -60,6 +61,21 @@ DECISIONS_CSV = REVIEW_DIR / "review_decisions.csv"
60
  QUEUE_CSV = REVIEW_DIR / "review_queue.csv"
61
  BANNER_DATA_URI_FILE = Path(__file__).resolve().parent / "assets" / "smoke_signal_banner_data_uri.txt"
62
  PUNCT_CORRECTOR_PATH = Path(__file__).resolve().parent / "smoke_signal" / "scripts" / "punct_corrector.py"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
63
 
64
 
65
  def _load_banner_image_css() -> str:
@@ -83,6 +99,8 @@ SS_BANNER_IMAGE_CSS = _load_banner_image_css()
83
 
84
  _PUNCT_MODULE = None
85
  _PUNCT_MODULE_ERROR = None
 
 
86
 
87
 
88
  def _load_punct_module():
@@ -108,12 +126,39 @@ def _load_punct_module():
108
  return None
109
 
110
 
111
- def _apply_punctuation_corrections(text: str, book_id: str) -> tuple[str, list, float]:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
112
  module = _load_punct_module()
113
  if module is None:
114
  return text, [], 1.0
115
  try:
116
- corrected, flags, score = module.apply_punctuation_corrections(text or "", book_id=book_id)
 
 
 
 
117
  return corrected, flags, float(score)
118
  except Exception:
119
  return text, [], 1.0
@@ -129,16 +174,126 @@ def _punctuation_confidence_penalty(text: str, confidence: float) -> float:
129
  return float(confidence)
130
 
131
 
132
- def _record_punctuation_correction(raw_text: str, gold_text: str, book_id: str) -> int:
 
 
 
 
 
133
  module = _load_punct_module()
134
  if module is None:
135
  return 0
136
  try:
137
- return int(module.record_punctuation_correction(raw_text or "", gold_text or "", book_id=book_id))
 
 
 
 
 
 
 
138
  except Exception:
139
  return 0
140
 
141
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
142
  def _punctuation_map_summary() -> dict:
143
  module = _load_punct_module()
144
  if module is None:
@@ -591,15 +746,19 @@ def sha256_file(path: Path) -> str:
591
 
592
  def load_manifest_df() -> pd.DataFrame:
593
  if not MANIFEST_CSV.exists():
594
- return pd.DataFrame(columns=[
595
- "book_id","filename","sha256","page_count","rights_class",
596
- "status","acquisition_date","notes"
597
- ])
598
- return pd.read_csv(MANIFEST_CSV)
 
599
 
600
 
601
  def save_manifest_df(df: pd.DataFrame) -> None:
602
- df.to_csv(MANIFEST_CSV, index=False)
 
 
 
603
 
604
 
605
  def next_book_id(df: pd.DataFrame) -> str:
@@ -673,6 +832,72 @@ def _parse_page_selection(spec: str, max_page: int | None = None) -> tuple[set[i
673
  return out, None
674
 
675
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
676
  # ── Step 1: INGEST ─────────────────────────────────────────────────────────────
677
  def ingest_pdfs(files, rights_class: str, notes: str) -> tuple:
678
  """Register uploaded PDFs into the source manifest."""
@@ -734,6 +959,9 @@ def ingest_pdfs(files, rights_class: str, notes: str) -> tuple:
734
  "status": "pending",
735
  "acquisition_date": datetime.utcnow().date().isoformat(),
736
  "notes": notes,
 
 
 
737
  }])
738
  df = pd.concat([df, new_row], ignore_index=True)
739
  log.append(log_line(f"βœ“ Registered {book_id} β€” {path.name} ({page_count or '?'} pages)"))
@@ -960,6 +1188,14 @@ try:
960
  SS_TESSERACT_BATCH_HARD_TIMEOUT_SEC = max(15, int(os.environ.get("SS_TESSERACT_BATCH_HARD_TIMEOUT_SEC", "45")))
961
  except Exception:
962
  SS_TESSERACT_BATCH_HARD_TIMEOUT_SEC = 45
 
 
 
 
 
 
 
 
963
 
964
 
965
  def _load_surya_runtime():
@@ -1107,7 +1343,11 @@ def _regions_from_page_result(page_result):
1107
  return regions, conf
1108
 
1109
 
1110
- def _run_tesseract_batch(images):
 
 
 
 
1111
  """
1112
  Tesseract fallback for OCR when Surya is unavailable/slow.
1113
  Returns list of dicts with regions/confidence/method aligned to input order.
@@ -1115,17 +1355,28 @@ def _run_tesseract_batch(images):
1115
  try:
1116
  import pytesseract
1117
  except Exception as e:
1118
- return [{"regions": [], "confidence": 0.0, "method": f"error-no-tesseract ({e})"} for _ in images]
 
 
 
 
 
 
 
 
1119
 
1120
  # Prefer parallel image-level OCR with single-threaded internal OpenMP for better CPU utilization.
1121
  os.environ.setdefault("OMP_THREAD_LIMIT", "1")
1122
 
1123
- def _ocr_single(img):
1124
  try:
 
 
 
1125
  data = pytesseract.image_to_data(
1126
  img,
1127
- lang="eng",
1128
- config=f"--oem 1 --psm {SS_TESSERACT_PSM}",
1129
  output_type=pytesseract.Output.DICT,
1130
  timeout=SS_TESSERACT_TIMEOUT_SEC,
1131
  )
@@ -1167,7 +1418,12 @@ def _run_tesseract_batch(images):
1167
 
1168
  # Hard gate for fallback quality: do not keep OCR text below minimum page confidence.
1169
  if regions and avg_conf < SS_TESSERACT_MIN_CONF_KEEP:
1170
- return {"regions": [], "confidence": 0.0, "method": "tesseract-lowconf-filtered"}
 
 
 
 
 
1171
 
1172
  # Filter obvious OCR noise from illustration texture (common in no-text pages).
1173
  full_text = " ".join(r["text"] for r in regions).strip()
@@ -1177,24 +1433,62 @@ def _run_tesseract_batch(images):
1177
  tokens = [t for t in full_text.split() if t]
1178
  avg_token_len = (sum(len(t) for t in tokens) / len(tokens)) if tokens else 0.0
1179
  if regions and avg_conf <= 0.42 and alpha_ratio < 0.62 and avg_token_len < 3.2:
1180
- return {"regions": [], "confidence": 0.0, "method": "tesseract-noise-filtered"}
 
 
 
 
 
1181
 
1182
- return {"regions": regions, "confidence": avg_conf, "method": "tesseract"}
 
 
 
 
 
1183
  except RuntimeError as e:
1184
- return {"regions": [], "confidence": 0.0, "method": f"error-tesseract-timeout ({e})"}
 
 
 
 
 
1185
  except Exception as e:
1186
- return {"regions": [], "confidence": 0.0, "method": f"error-tesseract ({e})"}
 
 
 
 
 
1187
 
1188
  if not images:
1189
  return []
1190
 
1191
  max_workers = min(SS_TESSERACT_WORKERS, len(images), max(1, os.cpu_count() or 1))
 
 
 
 
 
 
 
 
 
 
 
 
1192
  if max_workers <= 1:
1193
- return [_ocr_single(img) for img in images]
 
 
 
1194
 
1195
  outputs = [None] * len(images)
1196
  executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_workers)
1197
- futures = {executor.submit(_ocr_single, img): idx for idx, img in enumerate(images)}
 
 
 
1198
  pending = set(futures.keys())
1199
  batch_timeout = max(
1200
  SS_TESSERACT_BATCH_HARD_TIMEOUT_SEC,
@@ -1225,6 +1519,7 @@ def _run_tesseract_batch(images):
1225
  "regions": [],
1226
  "confidence": 0.0,
1227
  "method": f"error-tesseract-batch-timeout ({batch_timeout}s)",
 
1228
  }
1229
  future.cancel()
1230
  finally:
@@ -1233,7 +1528,12 @@ def _run_tesseract_batch(images):
1233
 
1234
  for i, out in enumerate(outputs):
1235
  if out is None:
1236
- outputs[i] = {"regions": [], "confidence": 0.0, "method": "error-tesseract-missing-output"}
 
 
 
 
 
1237
 
1238
  return outputs
1239
 
@@ -1269,8 +1569,10 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
1269
  selection_set, selection_error = _parse_page_selection(page_selection)
1270
  if selection_error:
1271
  return _ocr_status_html(), f"{debug}Invalid page selection: {selection_error}"
 
1272
  exclusion_spec = (page_exclusion or "").strip().lower()
1273
  exclusion_request = None if exclusion_spec in ("", "none", "-") else exclusion_spec
 
1274
  if exclusion_request is not None:
1275
  exclusion_set, exclusion_error = _parse_page_selection(exclusion_request)
1276
  if exclusion_error:
@@ -1310,6 +1612,23 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
1310
  log.append(log_line(f"⚠ Punctuation corrector unavailable ({_PUNCT_MODULE_ERROR or 'unknown'})"))
1311
  else:
1312
  log.append(log_line("βœ“ Punctuation corrector active (rule-check + confidence penalty + learning map)"))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1313
 
1314
  for _, row in eligible.iterrows():
1315
  book_id = row["book_id"]
@@ -1343,11 +1662,31 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
1343
  if excluded_pages_for_book is None:
1344
  excluded_pages_for_book = set(page_numbers)
1345
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1346
  selected_pages_for_book = set(selected_pages_for_book) - set(excluded_pages_for_book)
1347
  if not selected_pages_for_book:
1348
  log.append(log_line(f"⚠ {book_id}: no pages left after applying selection/exclusion"))
1349
  continue
1350
 
 
 
 
 
1351
  selected_preview = ",".join(str(p) for p in sorted(selected_pages_for_book))
1352
  log.append(log_line(f"β„Ή {book_id}: effective selected pages {selected_preview}"))
1353
  processed_books.append(book_id)
@@ -1369,9 +1708,14 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
1369
  # First pass: render OCR/hybrid pages once and keep PIL images for batch OCR.
1370
  ocr_targets = []
1371
  render_dpi = SS_RENDER_DPI_SURYA if surya is not None else SS_RENDER_DPI_FALLBACK
 
 
 
 
1372
  for page_data in profile.get("pages", []):
1373
  page_num = page_data["page_number"]
1374
  route = page_data["route"]
 
1375
  if selected_pages_for_book is not None and int(page_num) not in selected_pages_for_book:
1376
  continue
1377
 
@@ -1415,10 +1759,41 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
1415
  except Exception as e:
1416
  log.append(log_line(f" ⚠ {book_id} p{page_num}: render failed ({e})"))
1417
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1418
  ocr_targets.append({
1419
  "page_num": page_num,
1420
  "route": route,
1421
  "render_path": page_data.get("render_path"),
 
1422
  })
1423
 
1424
  # Batch OCR for non-embedded pages.
@@ -1430,6 +1805,8 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
1430
  batch_pages = [item["page_num"] for item in batch]
1431
  batch_images = []
1432
  batch_items_with_images = []
 
 
1433
  try:
1434
  from PIL import Image
1435
  except Exception as e:
@@ -1460,6 +1837,9 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
1460
  img = Image.open(render_abs).convert("RGB")
1461
  batch_images.append(img)
1462
  batch_items_with_images.append(item)
 
 
 
1463
  except Exception as e:
1464
  ocr_lookup[item["page_num"]] = {
1465
  "regions": [],
@@ -1481,7 +1861,7 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
1481
  "method": "surya",
1482
  }
1483
  else:
1484
- fallback_preds = _run_tesseract_batch(batch_images)
1485
  timeout_count = sum(1 for pred in fallback_preds if "batch-timeout" in str(pred.get("method", "")))
1486
  for item, pred in zip(batch_items_with_images, fallback_preds):
1487
  ocr_lookup[item["page_num"]] = pred
@@ -1496,7 +1876,7 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
1496
  # If Surya batch fails, try Tesseract for this batch before giving up.
1497
  if surya is not None:
1498
  log.append(log_line(f" ⚠ {book_id} batch {batch_pages[0]}-{batch_pages[-1]} Surya error: {e}; retrying with Tesseract"))
1499
- fallback_preds = _run_tesseract_batch(batch_images)
1500
  timeout_count = sum(1 for pred in fallback_preds if "batch-timeout" in str(pred.get("method", "")))
1501
  for item, pred in zip(batch_items_with_images, fallback_preds):
1502
  ocr_lookup[item["page_num"]] = pred
@@ -1552,8 +1932,17 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
1552
  method = "skipped-no-surya"
1553
 
1554
  raw_text = " ".join(r["text"] for r in regions)[:500]
1555
- corrected_text, punct_flags, punct_score = _apply_punctuation_corrections(raw_text, book_id)
 
 
 
 
1556
  conf_adjusted = _punctuation_confidence_penalty(corrected_text, conf)
 
 
 
 
 
1557
 
1558
  if method in ("tesseract-noise-filtered", "tesseract-lowconf-filtered") and not regions:
1559
  conf_class = "auto-accept"
@@ -1585,6 +1974,8 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
1585
  "extraction_method": method,
1586
  "punctuation_score": punct_score,
1587
  "punctuation_flags": punct_flags,
 
 
1588
  "render_path": page_data.get("render_path"),
1589
  "ocred_at": datetime.utcnow().isoformat() + "Z",
1590
  })
@@ -1605,6 +1996,8 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
1605
  "confidence_class": conf_class,
1606
  "punct_score": punct_score,
1607
  "punct_flags_count": len(punct_flags),
 
 
1608
  "status": "quarantine" if conf_class == "quarantine" else "pending",
1609
  "reviewer": "",
1610
  "correction": "",
@@ -1739,8 +2132,9 @@ def get_review_item(idx: int) -> tuple:
1739
  except Exception:
1740
  img_path = None
1741
 
 
1742
  info = (f"<div style='font-family:monospace;font-size:11px;color:var(--ss-muted)'>"
1743
- f"{item['book_id']} Β· page {item['page']} Β· "
1744
  f"conf: <b style='color:{'var(--ss-red)' if float(item.get('confidence',0)) < 0.6 else 'var(--ss-gold)'}'>"
1745
  f"{float(item.get('confidence',0)):.0%}</b></div>")
1746
 
@@ -1782,6 +2176,7 @@ def save_review_decision(idx: int, final_text: str, action: str, reviewer: str,
1782
  "raw_text_original": raw_text_original,
1783
  "reason_code": reason,
1784
  "reviewer": reviewer or "reviewer",
 
1785
  "was_correct": was_correct,
1786
  "decided_at": datetime.utcnow().isoformat() + "Z",
1787
  }
@@ -1816,7 +2211,12 @@ def save_review_decision(idx: int, final_text: str, action: str, reviewer: str,
1816
 
1817
  learned_pairs = 0
1818
  if action == "edited":
1819
- learned_pairs = _record_punctuation_correction(raw_text_original, final_text, item.get("book_id", ""))
 
 
 
 
 
1820
 
1821
  cal = load_calibration()
1822
  default = cal.get("_default", DEFAULT_CALIBRATION["_default"])
@@ -1878,6 +2278,7 @@ def run_export() -> tuple:
1878
 
1879
  records.append({
1880
  "book_id": book_id,
 
1881
  "source_hash": manifest_row.get("sha256","") if isinstance(manifest_row, pd.Series) else "",
1882
  "page_number": dec["page"],
1883
  "region_id": dec["region_id"],
@@ -1997,7 +2398,7 @@ def smoke_signal_tab():
1997
  interactive=False,
1998
  wrap=True,
1999
  )
2000
- ingest_log = gr.Textbox(label="Log", lines=6, interactive=False, elem_classes=["ss-log"])
2001
  refresh_btn = gr.Button("↻ Refresh Manifest", size="sm")
2002
  refresh_btn.click(lambda: load_manifest_df(), outputs=[manifest_table])
2003
 
@@ -2017,6 +2418,19 @@ def smoke_signal_tab():
2017
  outputs=[ingest_status, manifest_table, ingest_log],
2018
  )
2019
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2020
  ingest_btn.click(
2021
  ingest_pdfs,
2022
  inputs=[pdf_upload, rights_dd, ingest_notes],
@@ -2035,7 +2449,7 @@ def smoke_signal_tab():
2035
 
2036
  profile_status = gr.HTML(_profile_status_html())
2037
  profile_btn = gr.Button("Re-run Profiler (Optional) β†’", elem_classes=["ss-btn-run"])
2038
- profile_log = gr.Textbox(label="Log", lines=10, interactive=False, elem_classes=["ss-log"])
2039
 
2040
  profile_btn.click(run_profile, outputs=[profile_status, profile_log])
2041
 
@@ -2073,13 +2487,13 @@ def smoke_signal_tab():
2073
  value=True,
2074
  )
2075
  ocr_btn = gr.Button("Run OCR β†’", elem_classes=["ss-btn-run"])
2076
- ocr_log = gr.Textbox(label="Log", lines=10, interactive=False, elem_classes=["ss-log"])
2077
 
2078
  ocr_run_event = ocr_btn.click(
2079
  run_ocr,
2080
  inputs=[ocr_page_selection, ocr_page_exclusion, ocr_replace_queue],
2081
  outputs=[ocr_status, ocr_log],
2082
- show_progress="minimal",
2083
  )
2084
 
2085
  # ── STEP 4: REVIEW ────────────────────────────────────────────────
@@ -2178,7 +2592,7 @@ def smoke_signal_tab():
2178
  ocr_run_event.then(
2179
  load_review,
2180
  outputs=review_load_outputs,
2181
- show_progress="minimal",
2182
  )
2183
  accept_btn.click(do_accept, inputs=[current_idx, final_text_box, reviewer_name, reason_code], outputs=action_outputs)
2184
  edit_btn.click(do_accept, inputs=[current_idx, final_text_box, reviewer_name, reason_code], outputs=action_outputs)
 
27
  import importlib.util
28
  import json
29
  import os
30
+ import re
31
  import tempfile
32
  import threading
33
  import time
 
61
  QUEUE_CSV = REVIEW_DIR / "review_queue.csv"
62
  BANNER_DATA_URI_FILE = Path(__file__).resolve().parent / "assets" / "smoke_signal_banner_data_uri.txt"
63
  PUNCT_CORRECTOR_PATH = Path(__file__).resolve().parent / "smoke_signal" / "scripts" / "punct_corrector.py"
64
+ FONT_LIBRARY_PATH = Path(__file__).resolve().parent / "smoke_signal" / "scripts" / "font_library.py"
65
+
66
+ MANIFEST_COLUMNS = [
67
+ "book_id",
68
+ "filename",
69
+ "sha256",
70
+ "page_count",
71
+ "rights_class",
72
+ "status",
73
+ "acquisition_date",
74
+ "notes",
75
+ "story_pages_include",
76
+ "story_pages_exclude",
77
+ "safe_title",
78
+ ]
79
 
80
 
81
  def _load_banner_image_css() -> str:
 
99
 
100
  _PUNCT_MODULE = None
101
  _PUNCT_MODULE_ERROR = None
102
+ _FONT_MODULE = None
103
+ _FONT_MODULE_ERROR = None
104
 
105
 
106
  def _load_punct_module():
 
126
  return None
127
 
128
 
129
+ def _load_font_module():
130
+ global _FONT_MODULE, _FONT_MODULE_ERROR
131
+ if _FONT_MODULE is not None:
132
+ return _FONT_MODULE
133
+ if _FONT_MODULE_ERROR is not None:
134
+ return None
135
+ if not FONT_LIBRARY_PATH.exists():
136
+ _FONT_MODULE_ERROR = f"not found: {FONT_LIBRARY_PATH}"
137
+ return None
138
+ try:
139
+ spec = importlib.util.spec_from_file_location("smoke_signal_font_library", str(FONT_LIBRARY_PATH))
140
+ if spec is None or spec.loader is None:
141
+ _FONT_MODULE_ERROR = "invalid import spec"
142
+ return None
143
+ module = importlib.util.module_from_spec(spec)
144
+ spec.loader.exec_module(module)
145
+ _FONT_MODULE = module
146
+ return _FONT_MODULE
147
+ except Exception as e:
148
+ _FONT_MODULE_ERROR = str(e)
149
+ return None
150
+
151
+
152
+ def _apply_punctuation_corrections(text: str, book_id: str, font_name: Optional[str] = None) -> tuple[str, list, float]:
153
  module = _load_punct_module()
154
  if module is None:
155
  return text, [], 1.0
156
  try:
157
+ corrected, flags, score = module.apply_punctuation_corrections(
158
+ text or "",
159
+ book_id=book_id,
160
+ font_name=font_name,
161
+ )
162
  return corrected, flags, float(score)
163
  except Exception:
164
  return text, [], 1.0
 
174
  return float(confidence)
175
 
176
 
177
+ def _record_punctuation_correction(
178
+ raw_text: str,
179
+ gold_text: str,
180
+ book_id: str,
181
+ font_name: Optional[str] = None,
182
+ ) -> int:
183
  module = _load_punct_module()
184
  if module is None:
185
  return 0
186
  try:
187
+ return int(
188
+ module.record_punctuation_correction(
189
+ raw_text or "",
190
+ gold_text or "",
191
+ book_id=book_id,
192
+ font_name=font_name,
193
+ )
194
+ )
195
  except Exception:
196
  return 0
197
 
198
 
199
+ def _identify_page_font(render_abs_path: str) -> dict:
200
+ module = _load_font_module()
201
+ if module is None:
202
+ return {
203
+ "font_name": None,
204
+ "confidence": 0.0,
205
+ "alternatives": [],
206
+ "image_url": None,
207
+ "error": _FONT_MODULE_ERROR or "font module unavailable",
208
+ }
209
+ try:
210
+ result = module.identify_page_font(render_abs_path)
211
+ return result if isinstance(result, dict) else {
212
+ "font_name": None,
213
+ "confidence": 0.0,
214
+ "alternatives": [],
215
+ "image_url": None,
216
+ "error": "invalid response from font module",
217
+ }
218
+ except Exception as e:
219
+ return {
220
+ "font_name": None,
221
+ "confidence": 0.0,
222
+ "alternatives": [],
223
+ "image_url": None,
224
+ "error": str(e),
225
+ }
226
+
227
+
228
+ def _resolve_tesseract_lang(font_name: Optional[str]) -> str:
229
+ module = _load_font_module()
230
+ if module is None:
231
+ return "eng"
232
+ try:
233
+ lang = str(module.resolve_tesseract_lang(font_name) or "").strip()
234
+ return lang if lang else "eng"
235
+ except Exception:
236
+ return "eng"
237
+
238
+
239
+ def _resolve_tessdata_dir(model_name: Optional[str]) -> Optional[str]:
240
+ module = _load_font_module()
241
+ if module is None:
242
+ return None
243
+ try:
244
+ value = module.resolve_tessdata_dir(model_name)
245
+ return str(value) if value else None
246
+ except Exception:
247
+ return None
248
+
249
+
250
+ def _update_font_engine_stats(font_name: Optional[str], surya_conf: float, tess_conf: float) -> None:
251
+ module = _load_font_module()
252
+ if module is None:
253
+ return
254
+ try:
255
+ module.update_font_engine_stats(font_name, surya_conf, tess_conf)
256
+ except Exception:
257
+ return
258
+
259
+
260
+ def _mixfont_preflight() -> dict:
261
+ module = _load_font_module()
262
+ if module is None:
263
+ return {
264
+ "api_key_set": False,
265
+ "api_url": "",
266
+ "image_url_template_set": False,
267
+ "image_base_set": False,
268
+ "space_host_set": False,
269
+ "public_image_url_source_available": False,
270
+ "error": _FONT_MODULE_ERROR or "font module unavailable",
271
+ }
272
+ try:
273
+ data = module.mixfont_preflight()
274
+ if not isinstance(data, dict):
275
+ return {
276
+ "api_key_set": False,
277
+ "api_url": "",
278
+ "image_url_template_set": False,
279
+ "image_base_set": False,
280
+ "space_host_set": False,
281
+ "public_image_url_source_available": False,
282
+ "error": "invalid preflight response",
283
+ }
284
+ return data
285
+ except Exception as e:
286
+ return {
287
+ "api_key_set": False,
288
+ "api_url": "",
289
+ "image_url_template_set": False,
290
+ "image_base_set": False,
291
+ "space_host_set": False,
292
+ "public_image_url_source_available": False,
293
+ "error": str(e),
294
+ }
295
+
296
+
297
  def _punctuation_map_summary() -> dict:
298
  module = _load_punct_module()
299
  if module is None:
 
746
 
747
  def load_manifest_df() -> pd.DataFrame:
748
  if not MANIFEST_CSV.exists():
749
+ return pd.DataFrame(columns=MANIFEST_COLUMNS)
750
+ df = pd.read_csv(MANIFEST_CSV)
751
+ for col in MANIFEST_COLUMNS:
752
+ if col not in df.columns:
753
+ df[col] = ""
754
+ return df[MANIFEST_COLUMNS]
755
 
756
 
757
  def save_manifest_df(df: pd.DataFrame) -> None:
758
+ for col in MANIFEST_COLUMNS:
759
+ if col not in df.columns:
760
+ df[col] = ""
761
+ df[MANIFEST_COLUMNS].to_csv(MANIFEST_CSV, index=False)
762
 
763
 
764
  def next_book_id(df: pd.DataFrame) -> str:
 
832
  return out, None
833
 
834
 
835
+ def _safe_title_slug(title: str) -> str:
836
+ raw = str(title or "").strip().lower()
837
+ if not raw:
838
+ return ""
839
+ slug = re.sub(r"[^a-z0-9]+", "-", raw).strip("-")
840
+ return slug[:120]
841
+
842
+
843
+ def _clean_page_spec(spec: str) -> str:
844
+ raw = (spec or "").strip().lower()
845
+ if raw in ("", "all", "*", "none", "-"):
846
+ return ""
847
+ parsed, err = _parse_page_selection(raw)
848
+ if err:
849
+ raise ValueError(err)
850
+ if parsed is None:
851
+ return ""
852
+ return ",".join(str(p) for p in sorted(parsed))
853
+
854
+
855
+ def _normalize_saved_spec(value) -> str:
856
+ raw = str(value if value is not None else "").strip().lower()
857
+ if raw in ("", "nan", "none", "null", "-", "all", "*"):
858
+ return ""
859
+ return raw
860
+
861
+
862
+ def save_book_scope(book_id: str, include_spec: str, exclude_spec: str, safe_title: str) -> tuple:
863
+ df = load_manifest_df()
864
+ bid = (book_id or "").strip()
865
+ if df.empty or not bid:
866
+ return _ingest_status_html("idle"), df, "Book ID is required."
867
+ if bid not in df["book_id"].values:
868
+ return _ingest_status_html("idle"), df, f"Book ID {bid} not found."
869
+
870
+ try:
871
+ include_clean = _clean_page_spec(include_spec)
872
+ except ValueError as e:
873
+ return _ingest_status_html("idle"), df, f"Invalid include pages: {e}"
874
+
875
+ try:
876
+ exclude_clean = _clean_page_spec(exclude_spec)
877
+ except ValueError as e:
878
+ return _ingest_status_html("idle"), df, f"Invalid exclude pages: {e}"
879
+
880
+ title_raw = (safe_title or "").strip()
881
+ if title_raw:
882
+ safe = _safe_title_slug(title_raw)
883
+ else:
884
+ filename = str(df.loc[df["book_id"] == bid, "filename"].values[0] or "")
885
+ stem = Path(filename).stem if filename else bid
886
+ safe = _safe_title_slug(stem)
887
+
888
+ df.loc[df["book_id"] == bid, "story_pages_include"] = include_clean
889
+ df.loc[df["book_id"] == bid, "story_pages_exclude"] = exclude_clean
890
+ df.loc[df["book_id"] == bid, "safe_title"] = safe
891
+ save_manifest_df(df)
892
+
893
+ line1 = log_line(f"βœ“ Saved scope for {bid}")
894
+ line2 = log_line(
895
+ f"β„Ή include={include_clean or 'all'} Β· exclude={exclude_clean or 'none'} Β· safe_title={safe}"
896
+ )
897
+ msg = f"{line1}\n{line2}"
898
+ return _ingest_status_html("done"), load_manifest_df(), msg
899
+
900
+
901
  # ── Step 1: INGEST ─────────────────────────────────────────────────────────────
902
  def ingest_pdfs(files, rights_class: str, notes: str) -> tuple:
903
  """Register uploaded PDFs into the source manifest."""
 
959
  "status": "pending",
960
  "acquisition_date": datetime.utcnow().date().isoformat(),
961
  "notes": notes,
962
+ "story_pages_include": "",
963
+ "story_pages_exclude": "",
964
+ "safe_title": _safe_title_slug(Path(path.name).stem),
965
  }])
966
  df = pd.concat([df, new_row], ignore_index=True)
967
  log.append(log_line(f"βœ“ Registered {book_id} β€” {path.name} ({page_count or '?'} pages)"))
 
1188
  SS_TESSERACT_BATCH_HARD_TIMEOUT_SEC = max(15, int(os.environ.get("SS_TESSERACT_BATCH_HARD_TIMEOUT_SEC", "45")))
1189
  except Exception:
1190
  SS_TESSERACT_BATCH_HARD_TIMEOUT_SEC = 45
1191
+ try:
1192
+ SS_MIXFONT_MAX_DETECT_PAGES = max(1, int(os.environ.get("SS_MIXFONT_MAX_DETECT_PAGES", "2")))
1193
+ except Exception:
1194
+ SS_MIXFONT_MAX_DETECT_PAGES = 2
1195
+ try:
1196
+ SS_MIXFONT_MAX_ERRORS = max(1, int(os.environ.get("SS_MIXFONT_MAX_ERRORS", "1")))
1197
+ except Exception:
1198
+ SS_MIXFONT_MAX_ERRORS = 1
1199
 
1200
 
1201
  def _load_surya_runtime():
 
1343
  return regions, conf
1344
 
1345
 
1346
+ def _run_tesseract_batch(
1347
+ images,
1348
+ langs: Optional[list[str]] = None,
1349
+ tessdata_dirs: Optional[list[Optional[str]]] = None,
1350
+ ):
1351
  """
1352
  Tesseract fallback for OCR when Surya is unavailable/slow.
1353
  Returns list of dicts with regions/confidence/method aligned to input order.
 
1355
  try:
1356
  import pytesseract
1357
  except Exception as e:
1358
+ return [
1359
+ {
1360
+ "regions": [],
1361
+ "confidence": 0.0,
1362
+ "method": f"error-no-tesseract ({e})",
1363
+ "tesseract_lang": (langs[idx] if langs and idx < len(langs) else "eng"),
1364
+ }
1365
+ for idx, _ in enumerate(images)
1366
+ ]
1367
 
1368
  # Prefer parallel image-level OCR with single-threaded internal OpenMP for better CPU utilization.
1369
  os.environ.setdefault("OMP_THREAD_LIMIT", "1")
1370
 
1371
+ def _ocr_single(img, lang_hint: str, tessdata_dir: Optional[str]):
1372
  try:
1373
+ config = f"--oem 1 --psm {SS_TESSERACT_PSM}"
1374
+ if tessdata_dir:
1375
+ config = f"{config} --tessdata-dir \"{tessdata_dir}\""
1376
  data = pytesseract.image_to_data(
1377
  img,
1378
+ lang=lang_hint or "eng",
1379
+ config=config,
1380
  output_type=pytesseract.Output.DICT,
1381
  timeout=SS_TESSERACT_TIMEOUT_SEC,
1382
  )
 
1418
 
1419
  # Hard gate for fallback quality: do not keep OCR text below minimum page confidence.
1420
  if regions and avg_conf < SS_TESSERACT_MIN_CONF_KEEP:
1421
+ return {
1422
+ "regions": [],
1423
+ "confidence": 0.0,
1424
+ "method": "tesseract-lowconf-filtered",
1425
+ "tesseract_lang": lang_hint or "eng",
1426
+ }
1427
 
1428
  # Filter obvious OCR noise from illustration texture (common in no-text pages).
1429
  full_text = " ".join(r["text"] for r in regions).strip()
 
1433
  tokens = [t for t in full_text.split() if t]
1434
  avg_token_len = (sum(len(t) for t in tokens) / len(tokens)) if tokens else 0.0
1435
  if regions and avg_conf <= 0.42 and alpha_ratio < 0.62 and avg_token_len < 3.2:
1436
+ return {
1437
+ "regions": [],
1438
+ "confidence": 0.0,
1439
+ "method": "tesseract-noise-filtered",
1440
+ "tesseract_lang": lang_hint or "eng",
1441
+ }
1442
 
1443
+ return {
1444
+ "regions": regions,
1445
+ "confidence": avg_conf,
1446
+ "method": "tesseract",
1447
+ "tesseract_lang": lang_hint or "eng",
1448
+ }
1449
  except RuntimeError as e:
1450
+ return {
1451
+ "regions": [],
1452
+ "confidence": 0.0,
1453
+ "method": f"error-tesseract-timeout ({e})",
1454
+ "tesseract_lang": lang_hint or "eng",
1455
+ }
1456
  except Exception as e:
1457
+ return {
1458
+ "regions": [],
1459
+ "confidence": 0.0,
1460
+ "method": f"error-tesseract ({e})",
1461
+ "tesseract_lang": lang_hint or "eng",
1462
+ }
1463
 
1464
  if not images:
1465
  return []
1466
 
1467
  max_workers = min(SS_TESSERACT_WORKERS, len(images), max(1, os.cpu_count() or 1))
1468
+ lang_list = list(langs) if langs else []
1469
+ if len(lang_list) < len(images):
1470
+ lang_list.extend(["eng"] * (len(images) - len(lang_list)))
1471
+ elif len(lang_list) > len(images):
1472
+ lang_list = lang_list[: len(images)]
1473
+
1474
+ tess_dir_list = list(tessdata_dirs) if tessdata_dirs else []
1475
+ if len(tess_dir_list) < len(images):
1476
+ tess_dir_list.extend([None] * (len(images) - len(tess_dir_list)))
1477
+ elif len(tess_dir_list) > len(images):
1478
+ tess_dir_list = tess_dir_list[: len(images)]
1479
+
1480
  if max_workers <= 1:
1481
+ return [
1482
+ _ocr_single(img, lang_list[idx], tess_dir_list[idx])
1483
+ for idx, img in enumerate(images)
1484
+ ]
1485
 
1486
  outputs = [None] * len(images)
1487
  executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_workers)
1488
+ futures = {
1489
+ executor.submit(_ocr_single, img, lang_list[idx], tess_dir_list[idx]): idx
1490
+ for idx, img in enumerate(images)
1491
+ }
1492
  pending = set(futures.keys())
1493
  batch_timeout = max(
1494
  SS_TESSERACT_BATCH_HARD_TIMEOUT_SEC,
 
1519
  "regions": [],
1520
  "confidence": 0.0,
1521
  "method": f"error-tesseract-batch-timeout ({batch_timeout}s)",
1522
+ "tesseract_lang": lang_list[idx] if idx < len(lang_list) else "eng",
1523
  }
1524
  future.cancel()
1525
  finally:
 
1528
 
1529
  for i, out in enumerate(outputs):
1530
  if out is None:
1531
+ outputs[i] = {
1532
+ "regions": [],
1533
+ "confidence": 0.0,
1534
+ "method": "error-tesseract-missing-output",
1535
+ "tesseract_lang": lang_list[i] if i < len(lang_list) else "eng",
1536
+ }
1537
 
1538
  return outputs
1539
 
 
1569
  selection_set, selection_error = _parse_page_selection(page_selection)
1570
  if selection_error:
1571
  return _ocr_status_html(), f"{debug}Invalid page selection: {selection_error}"
1572
+ manual_selection = selection_set is not None
1573
  exclusion_spec = (page_exclusion or "").strip().lower()
1574
  exclusion_request = None if exclusion_spec in ("", "none", "-") else exclusion_spec
1575
+ manual_exclusion = exclusion_request is not None
1576
  if exclusion_request is not None:
1577
  exclusion_set, exclusion_error = _parse_page_selection(exclusion_request)
1578
  if exclusion_error:
 
1612
  log.append(log_line(f"⚠ Punctuation corrector unavailable ({_PUNCT_MODULE_ERROR or 'unknown'})"))
1613
  else:
1614
  log.append(log_line("βœ“ Punctuation corrector active (rule-check + confidence penalty + learning map)"))
1615
+ font_mod = _load_font_module()
1616
+ mixfont_enabled = False
1617
+ if font_mod is None:
1618
+ log.append(log_line(f"⚠ Font library unavailable ({_FONT_MODULE_ERROR or 'unknown'})"))
1619
+ else:
1620
+ preflight = _mixfont_preflight()
1621
+ if preflight.get("api_key_set") and preflight.get("public_image_url_source_available"):
1622
+ log.append(log_line("βœ“ MixFont detection enabled (per-page font routing)"))
1623
+ mixfont_enabled = True
1624
+ else:
1625
+ missing = []
1626
+ if not preflight.get("api_key_set"):
1627
+ missing.append("MIXFONT_API_KEY")
1628
+ if not preflight.get("public_image_url_source_available"):
1629
+ missing.append("MIXFONT_IMAGE_BASE_URL (or MIXFONT_IMAGE_URL_TEMPLATE/SPACE_HOST)")
1630
+ missing_text = ", ".join(missing) if missing else "unknown config"
1631
+ log.append(log_line(f"β„Ή MixFont disabled this run β€” missing {missing_text}"))
1632
 
1633
  for _, row in eligible.iterrows():
1634
  book_id = row["book_id"]
 
1662
  if excluded_pages_for_book is None:
1663
  excluded_pages_for_book = set(page_numbers)
1664
 
1665
+ saved_include_spec = _normalize_saved_spec(row.get("story_pages_include", ""))
1666
+ if saved_include_spec and not manual_selection:
1667
+ saved_include_set, saved_inc_err = _parse_page_selection(saved_include_spec, max_page=max_page)
1668
+ if saved_inc_err:
1669
+ log.append(log_line(f"⚠ {book_id}: invalid saved include pages '{saved_include_spec}' ({saved_inc_err})"))
1670
+ elif saved_include_set is not None:
1671
+ selected_pages_for_book = set(selected_pages_for_book) & set(saved_include_set)
1672
+
1673
+ saved_exclude_spec = _normalize_saved_spec(row.get("story_pages_exclude", ""))
1674
+ if saved_exclude_spec and not manual_exclusion:
1675
+ saved_exclude_set, saved_exc_err = _parse_page_selection(saved_exclude_spec, max_page=max_page)
1676
+ if saved_exc_err:
1677
+ log.append(log_line(f"⚠ {book_id}: invalid saved exclude pages '{saved_exclude_spec}' ({saved_exc_err})"))
1678
+ elif saved_exclude_set is not None:
1679
+ excluded_pages_for_book = set(excluded_pages_for_book) | set(saved_exclude_set)
1680
+
1681
  selected_pages_for_book = set(selected_pages_for_book) - set(excluded_pages_for_book)
1682
  if not selected_pages_for_book:
1683
  log.append(log_line(f"⚠ {book_id}: no pages left after applying selection/exclusion"))
1684
  continue
1685
 
1686
+ if saved_include_spec and not manual_selection:
1687
+ log.append(log_line(f"β„Ή {book_id}: saved include pages {saved_include_spec}"))
1688
+ if saved_exclude_spec and not manual_exclusion:
1689
+ log.append(log_line(f"β„Ή {book_id}: saved exclude pages {saved_exclude_spec}"))
1690
  selected_preview = ",".join(str(p) for p in sorted(selected_pages_for_book))
1691
  log.append(log_line(f"β„Ή {book_id}: effective selected pages {selected_preview}"))
1692
  processed_books.append(book_id)
 
1708
  # First pass: render OCR/hybrid pages once and keep PIL images for batch OCR.
1709
  ocr_targets = []
1710
  render_dpi = SS_RENDER_DPI_SURYA if surya is not None else SS_RENDER_DPI_FALLBACK
1711
+ book_detected_font = None
1712
+ mixfont_attempts = 0
1713
+ mixfont_errors = 0
1714
+ mixfont_disabled_for_book = not mixfont_enabled
1715
  for page_data in profile.get("pages", []):
1716
  page_num = page_data["page_number"]
1717
  route = page_data["route"]
1718
+ font_name = page_data.get("font_name")
1719
  if selected_pages_for_book is not None and int(page_num) not in selected_pages_for_book:
1720
  continue
1721
 
 
1759
  except Exception as e:
1760
  log.append(log_line(f" ⚠ {book_id} p{page_num}: render failed ({e})"))
1761
 
1762
+ font_name = page_data.get("font_name")
1763
+ if (not font_name) and book_detected_font:
1764
+ page_data["font_name"] = book_detected_font
1765
+ font_name = book_detected_font
1766
+
1767
+ if (
1768
+ (not font_name)
1769
+ and render_path is not None
1770
+ and (not mixfont_disabled_for_book)
1771
+ and mixfont_attempts < SS_MIXFONT_MAX_DETECT_PAGES
1772
+ ):
1773
+ mixfont_attempts += 1
1774
+ font_result = _identify_page_font(str(render_path))
1775
+ detected_font = font_result.get("font_name")
1776
+ if detected_font:
1777
+ page_data["font_name"] = detected_font
1778
+ page_data["font_confidence"] = float(font_result.get("confidence", 0.0) or 0.0)
1779
+ page_data["font_detected_at"] = datetime.utcnow().isoformat() + "Z"
1780
+ book_detected_font = detected_font
1781
+ font_name = detected_font
1782
+ log.append(log_line(f"β„Ή {book_id}: detected font '{detected_font}'"))
1783
+ else:
1784
+ page_data["font_name"] = None
1785
+ page_data["font_detect_error"] = font_result.get("error")
1786
+ mixfont_errors += 1
1787
+ if mixfont_errors >= SS_MIXFONT_MAX_ERRORS:
1788
+ mixfont_disabled_for_book = True
1789
+ err_txt = page_data.get("font_detect_error") or "unknown"
1790
+ log.append(log_line(f"⚠ {book_id}: MixFont disabled for this run ({err_txt})"))
1791
+
1792
  ocr_targets.append({
1793
  "page_num": page_num,
1794
  "route": route,
1795
  "render_path": page_data.get("render_path"),
1796
+ "font_name": page_data.get("font_name"),
1797
  })
1798
 
1799
  # Batch OCR for non-embedded pages.
 
1805
  batch_pages = [item["page_num"] for item in batch]
1806
  batch_images = []
1807
  batch_items_with_images = []
1808
+ batch_tess_langs = []
1809
+ batch_tess_dirs = []
1810
  try:
1811
  from PIL import Image
1812
  except Exception as e:
 
1837
  img = Image.open(render_abs).convert("RGB")
1838
  batch_images.append(img)
1839
  batch_items_with_images.append(item)
1840
+ lang_model = _resolve_tesseract_lang(item.get("font_name"))
1841
+ batch_tess_langs.append(lang_model)
1842
+ batch_tess_dirs.append(_resolve_tessdata_dir(lang_model))
1843
  except Exception as e:
1844
  ocr_lookup[item["page_num"]] = {
1845
  "regions": [],
 
1861
  "method": "surya",
1862
  }
1863
  else:
1864
+ fallback_preds = _run_tesseract_batch(batch_images, batch_tess_langs, batch_tess_dirs)
1865
  timeout_count = sum(1 for pred in fallback_preds if "batch-timeout" in str(pred.get("method", "")))
1866
  for item, pred in zip(batch_items_with_images, fallback_preds):
1867
  ocr_lookup[item["page_num"]] = pred
 
1876
  # If Surya batch fails, try Tesseract for this batch before giving up.
1877
  if surya is not None:
1878
  log.append(log_line(f" ⚠ {book_id} batch {batch_pages[0]}-{batch_pages[-1]} Surya error: {e}; retrying with Tesseract"))
1879
+ fallback_preds = _run_tesseract_batch(batch_images, batch_tess_langs, batch_tess_dirs)
1880
  timeout_count = sum(1 for pred in fallback_preds if "batch-timeout" in str(pred.get("method", "")))
1881
  for item, pred in zip(batch_items_with_images, fallback_preds):
1882
  ocr_lookup[item["page_num"]] = pred
 
1932
  method = "skipped-no-surya"
1933
 
1934
  raw_text = " ".join(r["text"] for r in regions)[:500]
1935
+ corrected_text, punct_flags, punct_score = _apply_punctuation_corrections(
1936
+ raw_text,
1937
+ book_id,
1938
+ font_name=font_name,
1939
+ )
1940
  conf_adjusted = _punctuation_confidence_penalty(corrected_text, conf)
1941
+ if font_name:
1942
+ if str(method).startswith("tesseract"):
1943
+ _update_font_engine_stats(font_name, 0.0, conf)
1944
+ elif str(method).startswith("surya"):
1945
+ _update_font_engine_stats(font_name, conf, 0.0)
1946
 
1947
  if method in ("tesseract-noise-filtered", "tesseract-lowconf-filtered") and not regions:
1948
  conf_class = "auto-accept"
 
1974
  "extraction_method": method,
1975
  "punctuation_score": punct_score,
1976
  "punctuation_flags": punct_flags,
1977
+ "font_name": font_name,
1978
+ "tesseract_lang": ocr_lookup.get(page_num, {}).get("tesseract_lang"),
1979
  "render_path": page_data.get("render_path"),
1980
  "ocred_at": datetime.utcnow().isoformat() + "Z",
1981
  })
 
1996
  "confidence_class": conf_class,
1997
  "punct_score": punct_score,
1998
  "punct_flags_count": len(punct_flags),
1999
+ "font_name": font_name or "",
2000
+ "tesseract_lang": ocr_lookup.get(page_num, {}).get("tesseract_lang", ""),
2001
  "status": "quarantine" if conf_class == "quarantine" else "pending",
2002
  "reviewer": "",
2003
  "correction": "",
 
2132
  except Exception:
2133
  img_path = None
2134
 
2135
+ font_suffix = f" Β· font: {item.get('font_name')}" if str(item.get("font_name", "")).strip() else ""
2136
  info = (f"<div style='font-family:monospace;font-size:11px;color:var(--ss-muted)'>"
2137
+ f"{item['book_id']} Β· page {item['page']}{font_suffix} Β· "
2138
  f"conf: <b style='color:{'var(--ss-red)' if float(item.get('confidence',0)) < 0.6 else 'var(--ss-gold)'}'>"
2139
  f"{float(item.get('confidence',0)):.0%}</b></div>")
2140
 
 
2176
  "raw_text_original": raw_text_original,
2177
  "reason_code": reason,
2178
  "reviewer": reviewer or "reviewer",
2179
+ "font_name": item.get("font_name", ""),
2180
  "was_correct": was_correct,
2181
  "decided_at": datetime.utcnow().isoformat() + "Z",
2182
  }
 
2211
 
2212
  learned_pairs = 0
2213
  if action == "edited":
2214
+ learned_pairs = _record_punctuation_correction(
2215
+ raw_text_original,
2216
+ final_text,
2217
+ item.get("book_id", ""),
2218
+ font_name=item.get("font_name"),
2219
+ )
2220
 
2221
  cal = load_calibration()
2222
  default = cal.get("_default", DEFAULT_CALIBRATION["_default"])
 
2278
 
2279
  records.append({
2280
  "book_id": book_id,
2281
+ "safe_title": manifest_row.get("safe_title","") if isinstance(manifest_row, pd.Series) else "",
2282
  "source_hash": manifest_row.get("sha256","") if isinstance(manifest_row, pd.Series) else "",
2283
  "page_number": dec["page"],
2284
  "region_id": dec["region_id"],
 
2398
  interactive=False,
2399
  wrap=True,
2400
  )
2401
+ ingest_log = gr.Textbox(label="Log", lines=6, interactive=False, elem_classes=["ss-log"], show_copy_button=True)
2402
  refresh_btn = gr.Button("↻ Refresh Manifest", size="sm")
2403
  refresh_btn.click(lambda: load_manifest_df(), outputs=[manifest_table])
2404
 
 
2418
  outputs=[ingest_status, manifest_table, ingest_log],
2419
  )
2420
 
2421
+ gr.Markdown("**Save Book Pages + Safe Title (persisted):**")
2422
+ with gr.Row():
2423
+ scope_book_id = gr.Textbox(label="Book ID", placeholder="SS-BOOK-0001", scale=1)
2424
+ scope_include = gr.Textbox(label="Story Pages Include", placeholder="all or 7-27 or 7,8,9,11-27", scale=1)
2425
+ scope_exclude = gr.Textbox(label="Story Pages Exclude", placeholder="e.g. 1,2,3,17,25", scale=1)
2426
+ scope_title = gr.Textbox(label="Safe Book Title", placeholder="e.g. the-gruffalo", scale=1)
2427
+ scope_save_btn = gr.Button("Save Book Pages β†’", size="sm")
2428
+ scope_save_btn.click(
2429
+ save_book_scope,
2430
+ inputs=[scope_book_id, scope_include, scope_exclude, scope_title],
2431
+ outputs=[ingest_status, manifest_table, ingest_log],
2432
+ )
2433
+
2434
  ingest_btn.click(
2435
  ingest_pdfs,
2436
  inputs=[pdf_upload, rights_dd, ingest_notes],
 
2449
 
2450
  profile_status = gr.HTML(_profile_status_html())
2451
  profile_btn = gr.Button("Re-run Profiler (Optional) β†’", elem_classes=["ss-btn-run"])
2452
+ profile_log = gr.Textbox(label="Log", lines=10, interactive=False, elem_classes=["ss-log"], show_copy_button=True)
2453
 
2454
  profile_btn.click(run_profile, outputs=[profile_status, profile_log])
2455
 
 
2487
  value=True,
2488
  )
2489
  ocr_btn = gr.Button("Run OCR β†’", elem_classes=["ss-btn-run"])
2490
+ ocr_log = gr.Textbox(label="Log", lines=10, interactive=False, elem_classes=["ss-log"], show_copy_button=True)
2491
 
2492
  ocr_run_event = ocr_btn.click(
2493
  run_ocr,
2494
  inputs=[ocr_page_selection, ocr_page_exclusion, ocr_replace_queue],
2495
  outputs=[ocr_status, ocr_log],
2496
+ show_progress="hidden",
2497
  )
2498
 
2499
  # ── STEP 4: REVIEW ────────────────────────────────────────────────
 
2592
  ocr_run_event.then(
2593
  load_review,
2594
  outputs=review_load_outputs,
2595
+ show_progress="hidden",
2596
  )
2597
  accept_btn.click(do_accept, inputs=[current_idx, final_text_box, reviewer_name, reason_code], outputs=action_outputs)
2598
  edit_btn.click(do_accept, inputs=[current_idx, final_text_box, reviewer_name, reason_code], outputs=action_outputs)