Pointf5ive commited on
Commit
b988c6d
Β·
1 Parent(s): bad810b

Smoke Signal v1: integrated pipeline wizard tab

Browse files
Files changed (2) hide show
  1. app.py +5 -1
  2. smoke_signal_tab.py +1325 -0
app.py CHANGED
@@ -26,6 +26,7 @@ from src.totem_workbook import (
26
  )
27
 
28
  from src.codex_extractor import process_upload, format_fingerprint_report
 
29
 
30
  ORIGINAL_WORKBOOK_PATH = "data/order69_macmillan_totem_rebuilt.xlsx"
31
  CODEX_CATALOGUE_PATH = Path("data/codex_catalogue.xlsx")
@@ -1175,7 +1176,7 @@ def single_score(active_path, sequence, stanza_id, draft_pass, clarity, rhythm,
1175
 
1176
  # ── GRADIO INTERFACE ──────────────────────────────────────────────────────────
1177
 
1178
- with gr.Blocks(title="TOTEM Studio", css=CSS, head=HEAD) as demo:
1179
  active_path = gr.State(str(DEFAULT_WORKBOOK))
1180
  log_state = gr.State(pd.DataFrame(columns=LOG_COLUMNS))
1181
 
@@ -1341,6 +1342,9 @@ with gr.Blocks(title="TOTEM Studio", css=CSS, head=HEAD) as demo:
1341
  </div>
1342
  """)
1343
 
 
 
 
1344
  # ── EVENT WIRING ─────────────────────────────────────────────────────────
1345
 
1346
  # Dashboard tab
 
26
  )
27
 
28
  from src.codex_extractor import process_upload, format_fingerprint_report
29
+ from smoke_signal_tab import smoke_signal_tab, SS_CSS
30
 
31
  ORIGINAL_WORKBOOK_PATH = "data/order69_macmillan_totem_rebuilt.xlsx"
32
  CODEX_CATALOGUE_PATH = Path("data/codex_catalogue.xlsx")
 
1176
 
1177
  # ── GRADIO INTERFACE ──────────────────────────────────────────────────────────
1178
 
1179
+ with gr.Blocks(title="TOTEM Studio", css=CSS + SS_CSS, head=HEAD) as demo:
1180
  active_path = gr.State(str(DEFAULT_WORKBOOK))
1181
  log_state = gr.State(pd.DataFrame(columns=LOG_COLUMNS))
1182
 
 
1342
  </div>
1343
  """)
1344
 
1345
+ # ── SMOKE SIGNAL TAB ─────────────────────────────────────────────────
1346
+ smoke_signal_tab()
1347
+
1348
  # ── EVENT WIRING ─────────────────────────────────────────────────────────
1349
 
1350
  # Dashboard tab
smoke_signal_tab.py ADDED
@@ -0,0 +1,1325 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Smoke Signal v1 β€” Gradio Tab Module
3
+ =====================================
4
+ Drop this file into your Codex_Extractor Space root.
5
+ Then add to app.py:
6
+
7
+ from smoke_signal_tab import smoke_signal_tab, SS_CSS
8
+
9
+ # Add SS_CSS to your existing CSS string
10
+ # Add smoke_signal_tab() call inside your gr.Blocks() tabs
11
+
12
+ Architecture:
13
+ Step 1: INGEST β€” upload PDFs, register + hash
14
+ Step 2: PROFILE β€” detect text vs image pages
15
+ Step 3: OCR β€” Surya extraction + confidence scoring
16
+ Step 4: REVIEW β€” human correction workbench (feeds training data)
17
+ Step 5: EXPORT β€” clean JSONL to Codex + downloadable gold set
18
+
19
+ Self-improvement loop:
20
+ Every correction β†’ recalibrates confidence thresholds in real time
21
+ Every correction β†’ appended to gold_corrections.jsonl for fine-tuning
22
+ """
23
+
24
+ import csv
25
+ import hashlib
26
+ import json
27
+ import os
28
+ import tempfile
29
+ import time
30
+ from datetime import datetime
31
+ from pathlib import Path
32
+ from typing import Optional
33
+
34
+ import gradio as gr
35
+ import pandas as pd
36
+
37
+ # ── Paths ──────────────────────────────────────────────────────────────────────
38
+ SS_ROOT = Path(__file__).parent / "smoke_signal"
39
+ SOURCE_DIR = SS_ROOT / "source_pdfs"
40
+ MANIFEST_CSV = SS_ROOT / "manifest" / "source_manifest.csv"
41
+ PROFILES_DIR = SS_ROOT / "manifest" / "page_profiles"
42
+ OCR_RAW_DIR = SS_ROOT / "ocr_raw"
43
+ RENDERS_DIR = SS_ROOT / "renders"
44
+ REGIONS_DIR = SS_ROOT / "regions"
45
+ REVIEW_DIR = SS_ROOT / "review"
46
+ EXPORTS_DIR = SS_ROOT / "exports"
47
+ GOLD_DIR = SS_ROOT / "gold"
48
+ LOGS_DIR = SS_ROOT / "logs"
49
+
50
+ for d in [SOURCE_DIR, MANIFEST_CSV.parent, PROFILES_DIR, OCR_RAW_DIR,
51
+ RENDERS_DIR, REGIONS_DIR, REVIEW_DIR, EXPORTS_DIR, GOLD_DIR, LOGS_DIR]:
52
+ d.mkdir(parents=True, exist_ok=True)
53
+
54
+ GOLD_FILE = GOLD_DIR / "gold_corrections.jsonl"
55
+ DECISIONS_CSV = REVIEW_DIR / "review_decisions.csv"
56
+ QUEUE_CSV = REVIEW_DIR / "review_queue.csv"
57
+
58
+ # ── Confidence calibration state (in-memory, persisted to disk) ────────────────
59
+ CALIBRATION_FILE = SS_ROOT / "manifest" / "confidence_calibration.json"
60
+
61
+ DEFAULT_CALIBRATION = {
62
+ "narration": {"auto_accept": 0.85, "review": 0.60, "quarantine": 0.35, "corrections": 0},
63
+ "dialogue-speech-bubble": {"auto_accept": 0.80, "review": 0.55, "quarantine": 0.30, "corrections": 0},
64
+ "caption": {"auto_accept": 0.82, "review": 0.58, "quarantine": 0.32, "corrections": 0},
65
+ "title": {"auto_accept": 0.88, "review": 0.65, "quarantine": 0.40, "corrections": 0},
66
+ "sign-label": {"auto_accept": 0.75, "review": 0.50, "quarantine": 0.25, "corrections": 0},
67
+ "_default": {"auto_accept": 0.85, "review": 0.60, "quarantine": 0.35, "corrections": 0},
68
+ }
69
+
70
+ def load_calibration() -> dict:
71
+ if CALIBRATION_FILE.exists():
72
+ try:
73
+ return json.load(open(CALIBRATION_FILE))
74
+ except Exception:
75
+ pass
76
+ return DEFAULT_CALIBRATION.copy()
77
+
78
+ def save_calibration(cal: dict) -> None:
79
+ with open(CALIBRATION_FILE, "w") as f:
80
+ json.dump(cal, f, indent=2)
81
+
82
+ def recalibrate(region_class: str, was_correct: bool, confidence: float) -> None:
83
+ """Tighten thresholds when corrections happen frequently for a region class."""
84
+ cal = load_calibration()
85
+ key = region_class if region_class in cal else "_default"
86
+ entry = cal[key]
87
+
88
+ if not was_correct:
89
+ entry["corrections"] = entry.get("corrections", 0) + 1
90
+ corrections = entry["corrections"]
91
+ # Every 5 corrections on same class: tighten auto-accept by 2%
92
+ if corrections % 5 == 0:
93
+ entry["auto_accept"] = min(0.98, entry["auto_accept"] + 0.02)
94
+ entry["review"] = min(0.90, entry["review"] + 0.01)
95
+
96
+ cal[key] = entry
97
+ save_calibration(cal)
98
+
99
+
100
+ # ── CSS ────────────────────────────────────────────────────────────────────────
101
+ SS_CSS = """
102
+ /* ── Smoke Signal palette ── */
103
+ :root {
104
+ --ss-bg: #0d1117;
105
+ --ss-surface: #161b22;
106
+ --ss-border: #30363d;
107
+ --ss-smoke: #8b949e;
108
+ --ss-signal: #f0883e;
109
+ --ss-glow: #58a6ff;
110
+ --ss-green: #3fb950;
111
+ --ss-red: #f85149;
112
+ --ss-gold: #e3b341;
113
+ --ss-text: #e6edf3;
114
+ --ss-muted: #7d8590;
115
+ }
116
+
117
+ /* Wizard step bar */
118
+ .ss-wizard {
119
+ display: flex;
120
+ align-items: center;
121
+ gap: 0;
122
+ padding: 20px 28px 0;
123
+ background: var(--ss-bg);
124
+ border-bottom: 1px solid var(--ss-border);
125
+ overflow-x: auto;
126
+ }
127
+
128
+ .ss-step {
129
+ display: flex;
130
+ align-items: center;
131
+ gap: 10px;
132
+ padding: 14px 20px;
133
+ cursor: pointer;
134
+ border-bottom: 3px solid transparent;
135
+ transition: all 0.2s;
136
+ white-space: nowrap;
137
+ font-family: 'Source Code Pro', 'Courier New', monospace;
138
+ font-size: 12px;
139
+ font-weight: 600;
140
+ color: var(--ss-muted);
141
+ letter-spacing: 1px;
142
+ text-transform: uppercase;
143
+ }
144
+
145
+ .ss-step.active {
146
+ color: var(--ss-signal);
147
+ border-bottom-color: var(--ss-signal);
148
+ }
149
+
150
+ .ss-step.complete {
151
+ color: var(--ss-green);
152
+ border-bottom-color: var(--ss-green);
153
+ }
154
+
155
+ .ss-step.locked {
156
+ color: var(--ss-border);
157
+ cursor: not-allowed;
158
+ }
159
+
160
+ .ss-num {
161
+ width: 26px;
162
+ height: 26px;
163
+ border-radius: 50%;
164
+ display: flex;
165
+ align-items: center;
166
+ justify-content: center;
167
+ font-size: 11px;
168
+ font-weight: 900;
169
+ background: var(--ss-surface);
170
+ border: 2px solid currentColor;
171
+ flex-shrink: 0;
172
+ }
173
+
174
+ .ss-step.complete .ss-num {
175
+ background: var(--ss-green);
176
+ color: var(--ss-bg);
177
+ border-color: var(--ss-green);
178
+ }
179
+
180
+ .ss-connector {
181
+ width: 40px;
182
+ height: 2px;
183
+ background: var(--ss-border);
184
+ flex-shrink: 0;
185
+ }
186
+
187
+ /* Main panel */
188
+ .ss-panel {
189
+ background: var(--ss-bg);
190
+ min-height: 600px;
191
+ padding: 28px;
192
+ font-family: 'Lato', sans-serif;
193
+ color: var(--ss-text);
194
+ }
195
+
196
+ .ss-panel-header {
197
+ display: flex;
198
+ align-items: center;
199
+ gap: 16px;
200
+ margin-bottom: 28px;
201
+ padding-bottom: 20px;
202
+ border-bottom: 1px solid var(--ss-border);
203
+ }
204
+
205
+ .ss-panel-icon {
206
+ font-size: 32px;
207
+ width: 56px;
208
+ height: 56px;
209
+ display: flex;
210
+ align-items: center;
211
+ justify-content: center;
212
+ background: var(--ss-surface);
213
+ border: 1px solid var(--ss-border);
214
+ border-radius: 10px;
215
+ }
216
+
217
+ .ss-panel-title {
218
+ font-family: 'Playfair Display', Georgia, serif;
219
+ font-size: 22px;
220
+ font-weight: 700;
221
+ color: var(--ss-text);
222
+ margin: 0;
223
+ }
224
+
225
+ .ss-panel-sub {
226
+ font-family: 'Source Code Pro', monospace;
227
+ font-size: 11px;
228
+ color: var(--ss-muted);
229
+ letter-spacing: 2px;
230
+ text-transform: uppercase;
231
+ margin: 4px 0 0;
232
+ }
233
+
234
+ /* Status pills */
235
+ .ss-pill {
236
+ display: inline-flex;
237
+ align-items: center;
238
+ gap: 6px;
239
+ padding: 4px 12px;
240
+ border-radius: 999px;
241
+ font-size: 11px;
242
+ font-weight: 700;
243
+ font-family: 'Source Code Pro', monospace;
244
+ letter-spacing: 1px;
245
+ text-transform: uppercase;
246
+ }
247
+
248
+ .ss-pill-waiting { background: #21262d; color: var(--ss-muted); border: 1px solid var(--ss-border); }
249
+ .ss-pill-running { background: #1c2a1e; color: var(--ss-gold); border: 1px solid var(--ss-gold); }
250
+ .ss-pill-done { background: #1a2f1a; color: var(--ss-green); border: 1px solid var(--ss-green); }
251
+ .ss-pill-error { background: #2d1a1a; color: var(--ss-red); border: 1px solid var(--ss-red); }
252
+ .ss-pill-review { background: #2a1f0e; color: var(--ss-signal); border: 1px solid var(--ss-signal); }
253
+
254
+ /* Cards */
255
+ .ss-card {
256
+ background: var(--ss-surface);
257
+ border: 1px solid var(--ss-border);
258
+ border-radius: 10px;
259
+ padding: 20px;
260
+ margin-bottom: 16px;
261
+ }
262
+
263
+ .ss-card-title {
264
+ font-size: 13px;
265
+ font-weight: 700;
266
+ color: var(--ss-smoke);
267
+ text-transform: uppercase;
268
+ letter-spacing: 2px;
269
+ margin-bottom: 12px;
270
+ font-family: 'Source Code Pro', monospace;
271
+ }
272
+
273
+ /* Metric row */
274
+ .ss-metrics {
275
+ display: grid;
276
+ grid-template-columns: repeat(4, 1fr);
277
+ gap: 12px;
278
+ margin-bottom: 20px;
279
+ }
280
+
281
+ .ss-metric {
282
+ background: var(--ss-surface);
283
+ border: 1px solid var(--ss-border);
284
+ border-radius: 8px;
285
+ padding: 16px;
286
+ text-align: center;
287
+ }
288
+
289
+ .ss-metric-val {
290
+ font-size: 28px;
291
+ font-weight: 900;
292
+ font-family: 'Source Code Pro', monospace;
293
+ color: var(--ss-text);
294
+ line-height: 1;
295
+ }
296
+
297
+ .ss-metric-label {
298
+ font-size: 10px;
299
+ color: var(--ss-muted);
300
+ text-transform: uppercase;
301
+ letter-spacing: 2px;
302
+ margin-top: 6px;
303
+ }
304
+
305
+ /* Progress bar */
306
+ .ss-progress-wrap {
307
+ background: var(--ss-border);
308
+ border-radius: 4px;
309
+ height: 6px;
310
+ margin: 8px 0;
311
+ overflow: hidden;
312
+ }
313
+
314
+ .ss-progress-bar {
315
+ height: 6px;
316
+ border-radius: 4px;
317
+ background: linear-gradient(90deg, var(--ss-signal), var(--ss-gold));
318
+ transition: width 0.4s ease;
319
+ }
320
+
321
+ /* Review workbench */
322
+ .ss-review-grid {
323
+ display: grid;
324
+ grid-template-columns: 240px 1fr 320px;
325
+ gap: 16px;
326
+ height: 580px;
327
+ }
328
+
329
+ .ss-queue-list {
330
+ background: var(--ss-surface);
331
+ border: 1px solid var(--ss-border);
332
+ border-radius: 8px;
333
+ overflow-y: auto;
334
+ padding: 8px;
335
+ }
336
+
337
+ .ss-queue-item {
338
+ padding: 10px 12px;
339
+ border-radius: 6px;
340
+ margin-bottom: 6px;
341
+ cursor: pointer;
342
+ border-left: 3px solid var(--ss-border);
343
+ font-size: 12px;
344
+ transition: all 0.15s;
345
+ }
346
+
347
+ .ss-queue-item:hover { background: #21262d; }
348
+ .ss-queue-item.active { background: #1c2028; border-left-color: var(--ss-signal); }
349
+ .ss-queue-item.done { border-left-color: var(--ss-green); opacity: 0.7; }
350
+ .ss-queue-item.quar { border-left-color: var(--ss-red); }
351
+
352
+ .ss-image-panel {
353
+ background: #010409;
354
+ border: 1px solid var(--ss-border);
355
+ border-radius: 8px;
356
+ display: flex;
357
+ align-items: center;
358
+ justify-content: center;
359
+ overflow: hidden;
360
+ }
361
+
362
+ .ss-action-panel {
363
+ background: var(--ss-surface);
364
+ border: 1px solid var(--ss-border);
365
+ border-radius: 8px;
366
+ padding: 16px;
367
+ display: flex;
368
+ flex-direction: column;
369
+ gap: 12px;
370
+ overflow-y: auto;
371
+ }
372
+
373
+ /* Buttons */
374
+ .ss-btn-accept { background: var(--ss-green) !important; color: #010409 !important; font-weight: 800 !important; border-radius: 6px !important; }
375
+ .ss-btn-edit { background: var(--ss-signal) !important; color: #010409 !important; font-weight: 800 !important; border-radius: 6px !important; }
376
+ .ss-btn-reject { background: var(--ss-red) !important; color: white !important; font-weight: 800 !important; border-radius: 6px !important; }
377
+ .ss-btn-quar { background: #21262d !important; color: var(--ss-gold) !important; font-weight: 800 !important; border-radius: 6px !important; border: 1px solid var(--ss-gold) !important; }
378
+ .ss-btn-next { background: var(--ss-glow) !important; color: #010409 !important; font-weight: 800 !important; border-radius: 6px !important; }
379
+ .ss-btn-run { background: linear-gradient(135deg, var(--ss-signal), var(--ss-gold)) !important; color: #010409 !important; font-weight: 900 !important; border-radius: 8px !important; font-size: 15px !important; min-height: 52px !important; }
380
+
381
+ /* Training signal */
382
+ .ss-training-badge {
383
+ display: inline-flex;
384
+ align-items: center;
385
+ gap: 6px;
386
+ padding: 6px 12px;
387
+ background: #1a2535;
388
+ border: 1px solid var(--ss-glow);
389
+ border-radius: 6px;
390
+ font-size: 11px;
391
+ color: var(--ss-glow);
392
+ font-family: 'Source Code Pro', monospace;
393
+ }
394
+
395
+ .ss-pulse {
396
+ width: 8px;
397
+ height: 8px;
398
+ border-radius: 50%;
399
+ background: var(--ss-glow);
400
+ animation: ss-pulse 1.5s infinite;
401
+ }
402
+
403
+ @keyframes ss-pulse {
404
+ 0%, 100% { opacity: 1; transform: scale(1); }
405
+ 50% { opacity: 0.4; transform: scale(0.8); }
406
+ }
407
+
408
+ /* Log terminal */
409
+ .ss-log {
410
+ background: #010409;
411
+ border: 1px solid var(--ss-border);
412
+ border-radius: 8px;
413
+ padding: 14px;
414
+ font-family: 'Source Code Pro', 'Courier New', monospace;
415
+ font-size: 12px;
416
+ color: #7ee787;
417
+ min-height: 120px;
418
+ max-height: 200px;
419
+ overflow-y: auto;
420
+ white-space: pre-wrap;
421
+ }
422
+
423
+ /* Gradio overrides for dark theme inside SS */
424
+ #ss-tab .gradio-container { background: var(--ss-bg) !important; }
425
+ #ss-tab textarea, #ss-tab input[type=text] {
426
+ background: var(--ss-surface) !important;
427
+ border: 1px solid var(--ss-border) !important;
428
+ color: var(--ss-text) !important;
429
+ border-radius: 6px !important;
430
+ font-family: 'Source Code Pro', monospace !important;
431
+ font-size: 13px !important;
432
+ }
433
+ #ss-tab .label-wrap span { color: var(--ss-smoke) !important; font-size: 11px !important; text-transform: uppercase !important; letter-spacing: 1px !important; }
434
+ """
435
+
436
+
437
+ # ── Utility functions ──────────────────────────────────────────────────────────
438
+ def sha256_file(path: Path) -> str:
439
+ h = hashlib.sha256()
440
+ with open(path, "rb") as f:
441
+ for block in iter(lambda: f.read(1 << 20), b""):
442
+ h.update(block)
443
+ return h.hexdigest()
444
+
445
+
446
+ def load_manifest_df() -> pd.DataFrame:
447
+ if not MANIFEST_CSV.exists():
448
+ return pd.DataFrame(columns=[
449
+ "book_id","filename","sha256","page_count","rights_class",
450
+ "status","acquisition_date","notes"
451
+ ])
452
+ return pd.read_csv(MANIFEST_CSV)
453
+
454
+
455
+ def save_manifest_df(df: pd.DataFrame) -> None:
456
+ df.to_csv(MANIFEST_CSV, index=False)
457
+
458
+
459
+ def next_book_id(df: pd.DataFrame) -> str:
460
+ existing = set(df["book_id"].tolist()) if not df.empty else set()
461
+ for i in range(1, 10000):
462
+ bid = f"SS-BOOK-{i:04d}"
463
+ if bid not in existing:
464
+ return bid
465
+ return "SS-BOOK-9999"
466
+
467
+
468
+ def load_queue_df() -> pd.DataFrame:
469
+ if not QUEUE_CSV.exists():
470
+ return pd.DataFrame()
471
+ return pd.read_csv(QUEUE_CSV)
472
+
473
+
474
+ def load_decisions_df() -> pd.DataFrame:
475
+ if not DECISIONS_CSV.exists():
476
+ return pd.DataFrame()
477
+ return pd.read_csv(DECISIONS_CSV)
478
+
479
+
480
+ def ts() -> str:
481
+ return datetime.utcnow().strftime("%Y%m%d-%H%M%S")
482
+
483
+
484
+ def log_line(msg: str) -> str:
485
+ return f"[{datetime.utcnow().strftime('%H:%M:%S')}] {msg}"
486
+
487
+
488
+ # ── Step 1: INGEST ─────────────────────────────────────────────────────────────
489
+ def ingest_pdfs(files, rights_class: str, notes: str) -> tuple:
490
+ """Register uploaded PDFs into the source manifest."""
491
+ if not files:
492
+ return _ingest_status_html("idle"), pd.DataFrame(), "No files uploaded."
493
+
494
+ df = load_manifest_df()
495
+ log = []
496
+ new_count = 0
497
+ dup_count = 0
498
+
499
+ for file in files:
500
+ path = Path(file.name) if hasattr(file, "name") else Path(file)
501
+ if not path.exists():
502
+ log.append(log_line(f"⚠ File not found: {path.name}"))
503
+ continue
504
+
505
+ file_hash = sha256_file(path)
506
+
507
+ # Check duplicate
508
+ if not df.empty and file_hash in df["sha256"].values:
509
+ log.append(log_line(f"↩ Duplicate: {path.name}"))
510
+ dup_count += 1
511
+ continue
512
+
513
+ # Copy to source_pdfs
514
+ dest = SOURCE_DIR / path.name
515
+ import shutil
516
+ shutil.copy2(path, dest)
517
+
518
+ # Page count
519
+ page_count = None
520
+ try:
521
+ import fitz
522
+ doc = fitz.open(str(dest))
523
+ page_count = doc.page_count
524
+ doc.close()
525
+ except Exception:
526
+ pass
527
+
528
+ book_id = next_book_id(df)
529
+ new_row = pd.DataFrame([{
530
+ "book_id": book_id,
531
+ "filename": path.name,
532
+ "sha256": file_hash,
533
+ "page_count": page_count or "",
534
+ "rights_class": rights_class,
535
+ "status": "pending",
536
+ "acquisition_date": datetime.utcnow().date().isoformat(),
537
+ "notes": notes,
538
+ }])
539
+ df = pd.concat([df, new_row], ignore_index=True)
540
+ log.append(log_line(f"βœ“ Registered {book_id} β€” {path.name} ({page_count or '?'} pages)"))
541
+ new_count += 1
542
+
543
+ save_manifest_df(df)
544
+
545
+ summary = f"Registered {new_count} new | {dup_count} duplicates skipped"
546
+ log.append(log_line(summary))
547
+
548
+ return _ingest_status_html("done", new_count, dup_count), df, "\n".join(log)
549
+
550
+
551
+ def _ingest_status_html(state: str, new=0, dups=0) -> str:
552
+ df = load_manifest_df()
553
+ total = len(df)
554
+ pending = len(df[df["status"] == "pending"]) if not df.empty else 0
555
+ unknown = len(df[df["rights_class"] == "unknown"]) if not df.empty else 0
556
+
557
+ alert = ""
558
+ if unknown > 0:
559
+ alert = f'<div style="background:#2a1f0e;border:1px solid var(--ss-signal);border-radius:6px;padding:10px 14px;margin-top:12px;font-size:12px;color:var(--ss-signal)">⚠ {unknown} sources have unknown rights class β€” set before extraction</div>'
560
+
561
+ return f"""
562
+ <div class="ss-metrics">
563
+ <div class="ss-metric"><div class="ss-metric-val">{total}</div><div class="ss-metric-label">Total Sources</div></div>
564
+ <div class="ss-metric"><div class="ss-metric-val" style="color:var(--ss-signal)">{pending}</div><div class="ss-metric-label">Pending</div></div>
565
+ <div class="ss-metric"><div class="ss-metric-val" style="color:var(--ss-green)">{total - pending}</div><div class="ss-metric-label">Processed</div></div>
566
+ <div class="ss-metric"><div class="ss-metric-val" style="color:{'var(--ss-red)' if unknown else 'var(--ss-green)'}">{unknown}</div><div class="ss-metric-label">Unknown Rights</div></div>
567
+ </div>{alert}"""
568
+
569
+
570
+ # ── Step 2: PROFILE ────────────────────────────────────────────────────────────
571
+ def run_profile() -> tuple:
572
+ """Profile all pending PDFs."""
573
+ df = load_manifest_df()
574
+ if df.empty:
575
+ return _profile_status_html(), "No sources registered. Complete Step 1 first."
576
+
577
+ pending = df[df["status"] == "pending"]
578
+ if pending.empty:
579
+ return _profile_status_html(), "No pending PDFs to profile."
580
+
581
+ log = []
582
+ try:
583
+ import fitz
584
+ except ImportError:
585
+ return _profile_status_html(), "PyMuPDF not installed. Run: pip install pymupdf"
586
+
587
+ for _, row in pending.iterrows():
588
+ book_id = row["book_id"]
589
+ filename = row["filename"]
590
+ pdf_path = SOURCE_DIR / filename
591
+
592
+ if not pdf_path.exists():
593
+ log.append(log_line(f"βœ— {book_id}: file not found"))
594
+ continue
595
+ if row.get("rights_class") in ("unknown", "excluded"):
596
+ log.append(log_line(f"↩ {book_id}: skipped β€” rights={row['rights_class']}"))
597
+ continue
598
+
599
+ try:
600
+ doc = fitz.open(str(pdf_path))
601
+ pages = []
602
+ routes = {"embedded_text": 0, "ocr": 0, "hybrid": 0}
603
+
604
+ for i in range(doc.page_count):
605
+ page = doc[i]
606
+ text = page.get_text("text").strip()
607
+ has_text = len(text) >= 20
608
+ has_imgs = len(page.get_images(full=True)) > 0
609
+ route = "embedded_text" if has_text and not has_imgs else \
610
+ "hybrid" if has_text and has_imgs else "ocr"
611
+ routes[route] += 1
612
+ pages.append({
613
+ "page_number": i + 1,
614
+ "route": route,
615
+ "char_count": len(text),
616
+ "has_images": has_imgs,
617
+ "width_pt": round(page.rect.width, 1),
618
+ "height_pt": round(page.rect.height, 1),
619
+ "rotation_deg": page.rotation,
620
+ "is_spread": (page.rect.width / max(page.rect.height, 1)) >= 1.6,
621
+ "warnings": [],
622
+ "render_path": None,
623
+ "render_dpi": None,
624
+ })
625
+ doc.close()
626
+
627
+ profile = {
628
+ "book_id": book_id, "filename": filename,
629
+ "source_hash": row["sha256"],
630
+ "page_count": len(pages),
631
+ "config_version": "ss_profiler_v0.1",
632
+ "profiled_at": datetime.utcnow().isoformat() + "Z",
633
+ "route_summary": routes,
634
+ "render_errors": [],
635
+ "pages": pages,
636
+ }
637
+ profile_path = PROFILES_DIR / f"{book_id}_page_profile.json"
638
+ with open(profile_path, "w") as f:
639
+ json.dump(profile, f, indent=2)
640
+
641
+ df.loc[df["book_id"] == book_id, "status"] = "profiled"
642
+ df.loc[df["book_id"] == book_id, "page_count"] = len(pages)
643
+ log.append(log_line(f"βœ“ {book_id}: {len(pages)}pp β€” embed={routes['embedded_text']} ocr={routes['ocr']} hybrid={routes['hybrid']}"))
644
+
645
+ except Exception as e:
646
+ log.append(log_line(f"βœ— {book_id}: {e}"))
647
+
648
+ save_manifest_df(df)
649
+ return _profile_status_html(), "\n".join(log)
650
+
651
+
652
+ def _profile_status_html() -> str:
653
+ df = load_manifest_df()
654
+ profiled = len(df[df["status"].isin(["profiled","rendered","ocred","reviewed","exported"])]) if not df.empty else 0
655
+ total = len(df)
656
+ pct = int(profiled / max(total, 1) * 100)
657
+
658
+ # Aggregate route stats from all profiles
659
+ embed = ocr = hybrid = 0
660
+ for p in PROFILES_DIR.glob("*_page_profile.json"):
661
+ try:
662
+ data = json.load(open(p))
663
+ rs = data.get("route_summary", {})
664
+ embed += rs.get("embedded_text", 0)
665
+ ocr += rs.get("ocr", 0)
666
+ hybrid += rs.get("hybrid", 0)
667
+ except Exception:
668
+ pass
669
+
670
+ return f"""
671
+ <div class="ss-metrics">
672
+ <div class="ss-metric"><div class="ss-metric-val">{profiled}/{total}</div><div class="ss-metric-label">Profiled</div></div>
673
+ <div class="ss-metric"><div class="ss-metric-val" style="color:var(--ss-green)">{embed}</div><div class="ss-metric-label">Embedded Text</div></div>
674
+ <div class="ss-metric"><div class="ss-metric-val" style="color:var(--ss-signal)">{ocr}</div><div class="ss-metric-label">β†’ OCR</div></div>
675
+ <div class="ss-metric"><div class="ss-metric-val" style="color:var(--ss-gold)">{hybrid}</div><div class="ss-metric-label">Hybrid</div></div>
676
+ </div>
677
+ <div class="ss-progress-wrap"><div class="ss-progress-bar" style="width:{pct}%"></div></div>
678
+ <div style="font-size:11px;color:var(--ss-muted);text-align:right;font-family:monospace">{pct}% profiled</div>"""
679
+
680
+
681
+ # ── Step 3: OCR ────────────────────────────────────────────────────────────────
682
+ def run_ocr() -> tuple:
683
+ """Run Surya OCR on all profiled PDFs."""
684
+ df = load_manifest_df()
685
+ if df.empty:
686
+ return _ocr_status_html(), "No sources. Complete Steps 1-2 first."
687
+
688
+ eligible = df[df["status"] == "profiled"]
689
+ if eligible.empty:
690
+ return _ocr_status_html(), "No profiled PDFs. Complete Step 2 first."
691
+
692
+ log = []
693
+
694
+ # Try to import Surya
695
+ surya = None
696
+ try:
697
+ from surya.ocr import run_ocr as surya_run
698
+ from surya.model.detection.model import load_model as load_det
699
+ from surya.model.detection.processor import load_processor as load_det_proc
700
+ from surya.model.recognition.model import load_model as load_rec
701
+ from surya.model.recognition.processor import load_processor as load_rec_proc
702
+ log.append(log_line("Loading Surya models (may take a moment)..."))
703
+ surya = {
704
+ "run": surya_run,
705
+ "det_model": load_det(), "det_proc": load_det_proc(),
706
+ "rec_model": load_rec(), "rec_proc": load_rec_proc(),
707
+ }
708
+ log.append(log_line("βœ“ Surya models loaded"))
709
+ except ImportError:
710
+ log.append(log_line("⚠ Surya not installed β€” falling back to text extraction only"))
711
+
712
+ queue_rows = []
713
+
714
+ for _, row in eligible.iterrows():
715
+ book_id = row["book_id"]
716
+ profile_path = PROFILES_DIR / f"{book_id}_page_profile.json"
717
+ if not profile_path.exists():
718
+ log.append(log_line(f"βœ— {book_id}: no profile"))
719
+ continue
720
+
721
+ profile = json.load(open(profile_path))
722
+ ocr_pages = []
723
+ review_pages = []
724
+ quarantine_pages = []
725
+ cal = load_calibration()
726
+
727
+ for page_data in profile["pages"]:
728
+ page_num = page_data["page_number"]
729
+ route = page_data["route"]
730
+
731
+ if route == "embedded_text":
732
+ # Extract directly via fitz
733
+ try:
734
+ import fitz
735
+ pdf_path = SOURCE_DIR / row["filename"]
736
+ doc = fitz.open(str(pdf_path))
737
+ page = doc[page_num - 1]
738
+ text = page.get_text("text").strip()
739
+ doc.close()
740
+ regions = [{"text": text, "confidence": 0.99, "bbox": [0,0,100,100], "word_count": len(text.split())}]
741
+ conf = 0.99
742
+ method = "embedded-text"
743
+ except Exception:
744
+ regions = []
745
+ conf = 0.0
746
+ method = "error"
747
+ elif surya:
748
+ try:
749
+ from PIL import Image
750
+ import fitz
751
+ pdf_path = SOURCE_DIR / row["filename"]
752
+ doc = fitz.open(str(pdf_path))
753
+ page = doc[page_num - 1]
754
+ mat = fitz.Matrix(300/72, 300/72)
755
+ pix = page.get_pixmap(matrix=mat, alpha=False)
756
+
757
+ # Save render
758
+ render_dir = RENDERS_DIR / book_id
759
+ render_dir.mkdir(exist_ok=True)
760
+ render_path = render_dir / f"{book_id}_page_{page_num:04d}_300dpi.png"
761
+ pix.save(str(render_path))
762
+ doc.close()
763
+
764
+ img = Image.open(render_path).convert("RGB")
765
+ result = surya["run"]([img], [["en"]], surya["det_model"], surya["det_proc"], surya["rec_model"], surya["rec_proc"])
766
+ page_result = result[0]
767
+
768
+ regions = []
769
+ for line in page_result.text_lines:
770
+ txt = line.text.strip()
771
+ if txt:
772
+ c = float(line.confidence) if hasattr(line, "confidence") else 1.0
773
+ regions.append({"text": txt, "confidence": round(c,4), "bbox": line.bbox, "word_count": len(txt.split())})
774
+
775
+ conf = sum(r["confidence"]*r["word_count"] for r in regions) / max(sum(r["word_count"] for r in regions), 1) if regions else 0.0
776
+ conf = round(conf, 4)
777
+ method = "surya"
778
+
779
+ # Update profile with render path
780
+ page_data["render_path"] = str(render_path.relative_to(SS_ROOT))
781
+ page_data["render_dpi"] = 300
782
+
783
+ except Exception as e:
784
+ log.append(log_line(f" ⚠ Page {page_num}: {e}"))
785
+ regions = []
786
+ conf = 0.0
787
+ method = "error"
788
+ else:
789
+ # No surya β€” skip OCR pages
790
+ regions = []
791
+ conf = 0.0
792
+ method = "skipped-no-surya"
793
+
794
+ # Classify confidence
795
+ default_cal = cal.get("_default", DEFAULT_CALIBRATION["_default"])
796
+ if conf >= default_cal["auto_accept"]:
797
+ conf_class = "auto-accept"
798
+ elif conf >= default_cal["review"]:
799
+ conf_class = "review-required"
800
+ review_pages.append(page_num)
801
+ elif conf >= default_cal["quarantine"]:
802
+ conf_class = "low-confidence"
803
+ review_pages.append(page_num)
804
+ else:
805
+ conf_class = "quarantine"
806
+ quarantine_pages.append(page_num)
807
+
808
+ ocr_pages.append({
809
+ "page_number": page_num,
810
+ "route": route,
811
+ "regions": regions,
812
+ "page_confidence": conf,
813
+ "confidence_class": conf_class,
814
+ "extraction_method": method,
815
+ "render_path": page_data.get("render_path"),
816
+ "ocred_at": datetime.utcnow().isoformat() + "Z",
817
+ })
818
+
819
+ # Add to review queue
820
+ if conf_class in ("review-required", "low-confidence", "quarantine"):
821
+ raw_text = " ".join(r["text"] for r in regions)[:500]
822
+ region_id = f"{book_id}_p{page_num:04d}"
823
+ queue_rows.append({
824
+ "book_id": book_id, "filename": row["filename"],
825
+ "page": page_num, "region_id": region_id,
826
+ "region_class": "narration",
827
+ "crop_path": page_data.get("render_path",""),
828
+ "raw_ocr": raw_text,
829
+ "confidence": conf, "confidence_class": conf_class,
830
+ "status": "quarantine" if conf_class == "quarantine" else "pending",
831
+ "reviewer": "", "correction": "", "reason_code": "",
832
+ })
833
+
834
+ # Save OCR raw
835
+ ocr_dir = OCR_RAW_DIR / book_id
836
+ ocr_dir.mkdir(exist_ok=True)
837
+ with open(ocr_dir / f"{book_id}_ocr_raw.json", "w") as f:
838
+ json.dump({
839
+ "book_id": book_id, "filename": row["filename"],
840
+ "source_hash": row["sha256"],
841
+ "page_count": len(ocr_pages),
842
+ "review_queue": review_pages,
843
+ "quarantine_list": quarantine_pages,
844
+ "pages": ocr_pages,
845
+ }, f, indent=2)
846
+
847
+ # Save updated profile (with render paths)
848
+ with open(profile_path, "w") as f:
849
+ json.dump(profile, f, indent=2)
850
+
851
+ df.loc[df["book_id"] == book_id, "status"] = "ocred"
852
+ total_review = len(review_pages) + len(quarantine_pages)
853
+ log.append(log_line(f"βœ“ {book_id}: {len(ocr_pages)}pp β€” review queue: {total_review}"))
854
+
855
+ # Write review queue
856
+ if queue_rows:
857
+ qdf = pd.DataFrame(queue_rows)
858
+ if QUEUE_CSV.exists():
859
+ existing = pd.read_csv(QUEUE_CSV)
860
+ qdf = pd.concat([existing, qdf], ignore_index=True).drop_duplicates(subset=["region_id"])
861
+ qdf.to_csv(QUEUE_CSV, index=False)
862
+
863
+ save_manifest_df(df)
864
+ return _ocr_status_html(), "\n".join(log)
865
+
866
+
867
+ def _ocr_status_html() -> str:
868
+ q_df = load_queue_df()
869
+ d_df = load_decisions_df()
870
+ total_queue = len(q_df)
871
+ decided = len(d_df)
872
+ pending = total_queue - decided
873
+ auto = len(q_df[q_df["confidence_class"] == "auto-accept"]) if not q_df.empty and "confidence_class" in q_df.columns else 0
874
+ quar = len(q_df[q_df["confidence_class"] == "quarantine"]) if not q_df.empty and "confidence_class" in q_df.columns else 0
875
+
876
+ cal = load_calibration()
877
+ default = cal.get("_default", DEFAULT_CALIBRATION["_default"])
878
+ corrections = sum(v.get("corrections",0) for v in cal.values() if isinstance(v, dict))
879
+
880
+ training_badge = f'<div class="ss-training-badge"><div class="ss-pulse"></div>{corrections} corrections logged Β· thresholds auto={default["auto_accept"]:.0%} review={default["review"]:.0%}</div>'
881
+
882
+ return f"""
883
+ <div class="ss-metrics">
884
+ <div class="ss-metric"><div class="ss-metric-val">{total_queue}</div><div class="ss-metric-label">Review Queue</div></div>
885
+ <div class="ss-metric"><div class="ss-metric-val" style="color:var(--ss-signal)">{pending}</div><div class="ss-metric-label">Pending</div></div>
886
+ <div class="ss-metric"><div class="ss-metric-val" style="color:var(--ss-red)">{quar}</div><div class="ss-metric-label">Quarantined</div></div>
887
+ <div class="ss-metric"><div class="ss-metric-val" style="color:var(--ss-green)">{decided}</div><div class="ss-metric-label">Decided</div></div>
888
+ </div>
889
+ <div style="margin-top:8px">{training_badge}</div>"""
890
+
891
+
892
+ # ── Step 4: REVIEW ─────────────────────────────────────────────────────────────
893
+ def get_review_item(idx: int) -> tuple:
894
+ q_df = load_queue_df()
895
+ d_df = load_decisions_df()
896
+ if q_df.empty:
897
+ return None, "", "", 0, 0
898
+
899
+ decided_ids = set(d_df["region_id"].tolist()) if not d_df.empty else set()
900
+ pending = q_df[~q_df["region_id"].isin(decided_ids)]
901
+ if pending.empty:
902
+ return None, "All items reviewed!", "", len(q_df), len(q_df)
903
+
904
+ idx = idx % len(pending)
905
+ item = pending.iloc[idx]
906
+
907
+ img_path = None
908
+ crop_path = item.get("crop_path","")
909
+ if crop_path:
910
+ candidate = SS_ROOT / crop_path
911
+ if candidate.exists():
912
+ img_path = str(candidate)
913
+
914
+ info = (f"<div style='font-family:monospace;font-size:11px;color:var(--ss-muted)'>"
915
+ f"{item['book_id']} Β· page {item['page']} Β· "
916
+ f"conf: <b style='color:{'var(--ss-red)' if float(item.get('confidence',0)) < 0.6 else 'var(--ss-gold)'}'>"
917
+ f"{float(item.get('confidence',0)):.0%}</b></div>")
918
+
919
+ return img_path, item.get("raw_ocr",""), info, len(q_df) - len(pending), len(q_df)
920
+
921
+
922
+ def save_review_decision(idx: int, final_text: str, action: str, reviewer: str, reason: str) -> tuple:
923
+ q_df = load_queue_df()
924
+ d_df = load_decisions_df()
925
+ if q_df.empty:
926
+ return "No queue.", *get_review_item(idx)[1:]
927
+
928
+ decided_ids = set(d_df["region_id"].tolist()) if not d_df.empty else set()
929
+ pending = q_df[~q_df["region_id"].isin(decided_ids)]
930
+ if pending.empty:
931
+ return "All done!", *get_review_item(0)[1:]
932
+
933
+ idx = idx % len(pending)
934
+ item = pending.iloc[idx]
935
+
936
+ raw_text = item.get("raw_ocr","")
937
+ was_correct = (final_text.strip() == raw_text.strip())
938
+ region_class = item.get("region_class","narration")
939
+
940
+ # Recalibrate confidence thresholds
941
+ recalibrate(region_class, was_correct, float(item.get("confidence",0)))
942
+
943
+ # Save decision
944
+ decision = {
945
+ "region_id": item["region_id"],
946
+ "book_id": item["book_id"],
947
+ "page": item["page"],
948
+ "status": action,
949
+ "final_text": final_text,
950
+ "raw_text": raw_text,
951
+ "reason_code": reason,
952
+ "reviewer": reviewer or "reviewer",
953
+ "was_correct": was_correct,
954
+ "decided_at": datetime.utcnow().isoformat() + "Z",
955
+ }
956
+
957
+ # Append to decisions CSV
958
+ new_row = pd.DataFrame([decision])
959
+ if DECISIONS_CSV.exists():
960
+ d_df = pd.concat([d_df, new_row], ignore_index=True)
961
+ else:
962
+ d_df = new_row
963
+ d_df.to_csv(DECISIONS_CSV, index=False)
964
+
965
+ # Append to gold training set
966
+ with open(GOLD_FILE, "a", encoding="utf-8") as f:
967
+ f.write(json.dumps({
968
+ **decision,
969
+ "region_class": region_class,
970
+ "confidence": float(item.get("confidence", 0)),
971
+ "conf_class": item.get("confidence_class",""),
972
+ }) + "\n")
973
+
974
+ cal = load_calibration()
975
+ default = cal.get("_default", DEFAULT_CALIBRATION["_default"])
976
+ corrections = sum(v.get("corrections",0) for v in cal.values() if isinstance(v,dict))
977
+ gold_count = sum(1 for _ in open(GOLD_FILE)) if GOLD_FILE.exists() else 0
978
+
979
+ feedback = (f"<div class='ss-training-badge'><div class='ss-pulse'></div>"
980
+ f"Gold set: {gold_count} examples Β· {corrections} corrections Β· "
981
+ f"auto-accept threshold: {default['auto_accept']:.0%}</div>")
982
+
983
+ return feedback, *get_review_item(0)
984
+
985
+
986
+ def _review_status_html() -> str:
987
+ q_df = load_queue_df()
988
+ d_df = load_decisions_df()
989
+ total = len(q_df)
990
+ decided = len(d_df)
991
+ pending = total - decided
992
+ accepted = len(d_df[d_df["status"] == "accepted"]) if not d_df.empty else 0
993
+ edited = len(d_df[d_df["status"] == "edited"]) if not d_df.empty else 0
994
+ rejected = len(d_df[d_df["status"] == "rejected"]) if not d_df.empty else 0
995
+ gold_count = sum(1 for _ in open(GOLD_FILE)) if GOLD_FILE.exists() else 0
996
+ pct = int(decided / max(total, 1) * 100)
997
+
998
+ return f"""
999
+ <div class="ss-metrics">
1000
+ <div class="ss-metric"><div class="ss-metric-val">{pending}</div><div class="ss-metric-label">Pending</div></div>
1001
+ <div class="ss-metric"><div class="ss-metric-val" style="color:var(--ss-green)">{accepted + edited}</div><div class="ss-metric-label">Approved</div></div>
1002
+ <div class="ss-metric"><div class="ss-metric-val" style="color:var(--ss-red)">{rejected}</div><div class="ss-metric-label">Rejected</div></div>
1003
+ <div class="ss-metric"><div class="ss-metric-val" style="color:var(--ss-glow)">{gold_count}</div><div class="ss-metric-label">Gold Examples</div></div>
1004
+ </div>
1005
+ <div class="ss-progress-wrap"><div class="ss-progress-bar" style="width:{pct}%"></div></div>"""
1006
+
1007
+
1008
+ # ── Step 5: EXPORT ─────────────────────────────────────────────────────────────
1009
+ def run_export() -> tuple:
1010
+ """Export approved decisions to Codex JSONL + gold set."""
1011
+ d_df = load_decisions_df()
1012
+ q_df = load_queue_df()
1013
+ df = load_manifest_df()
1014
+
1015
+ if d_df.empty:
1016
+ return _export_status_html(), "No review decisions. Complete Step 4 first.", None, None
1017
+
1018
+ approved = d_df[d_df["status"].isin(["accepted","edited"])]
1019
+ if approved.empty:
1020
+ return _export_status_html(), "No approved items to export.", None, None
1021
+
1022
+ log = []
1023
+ records = []
1024
+ batch = ts()
1025
+
1026
+ for _, dec in approved.iterrows():
1027
+ book_id = dec["book_id"]
1028
+ manifest_row = df[df["book_id"] == book_id].iloc[0] if not df[df["book_id"] == book_id].empty else {}
1029
+
1030
+ records.append({
1031
+ "book_id": book_id,
1032
+ "source_hash": manifest_row.get("sha256","") if isinstance(manifest_row, pd.Series) else "",
1033
+ "page_number": dec["page"],
1034
+ "region_id": dec["region_id"],
1035
+ "text_final": dec["final_text"],
1036
+ "text_raw": dec.get("raw_text",""),
1037
+ "review_status": dec["status"],
1038
+ "reviewer": dec.get("reviewer",""),
1039
+ "extraction_method": "ocr",
1040
+ "export_batch": batch,
1041
+ "exported_at": datetime.utcnow().isoformat() + "Z",
1042
+ })
1043
+
1044
+ # Write Codex JSONL
1045
+ jsonl_path = EXPORTS_DIR / f"codex_export_{batch}.jsonl"
1046
+ with open(jsonl_path, "w", encoding="utf-8") as f:
1047
+ for r in records:
1048
+ f.write(json.dumps(r, ensure_ascii=False) + "\n")
1049
+
1050
+ # Write gold set copy
1051
+ gold_export = EXPORTS_DIR / f"gold_set_{batch}.jsonl"
1052
+ if GOLD_FILE.exists():
1053
+ import shutil
1054
+ shutil.copy2(GOLD_FILE, gold_export)
1055
+
1056
+ log.append(log_line(f"βœ“ Exported {len(records)} approved records β†’ {jsonl_path.name}"))
1057
+ log.append(log_line(f"βœ“ Gold training set β†’ {gold_export.name}"))
1058
+ log.append(log_line(f"βœ“ Ready for Codex ingestion"))
1059
+
1060
+ return _export_status_html(len(records)), "\n".join(log), str(jsonl_path), str(gold_export)
1061
+
1062
+
1063
+ def _export_status_html(last_export=0) -> str:
1064
+ exports = list(EXPORTS_DIR.glob("codex_export_*.jsonl"))
1065
+ total_exports = len(exports)
1066
+ gold_count = sum(1 for _ in open(GOLD_FILE)) if GOLD_FILE.exists() else 0
1067
+ d_df = load_decisions_df()
1068
+ approved = len(d_df[d_df["status"].isin(["accepted","edited"])]) if not d_df.empty else 0
1069
+
1070
+ return f"""
1071
+ <div class="ss-metrics">
1072
+ <div class="ss-metric"><div class="ss-metric-val" style="color:var(--ss-green)">{approved}</div><div class="ss-metric-label">Approved</div></div>
1073
+ <div class="ss-metric"><div class="ss-metric-val" style="color:var(--ss-signal)">{last_export or 'β€”'}</div><div class="ss-metric-label">Last Export</div></div>
1074
+ <div class="ss-metric"><div class="ss-metric-val">{total_exports}</div><div class="ss-metric-label">Export Batches</div></div>
1075
+ <div class="ss-metric"><div class="ss-metric-val" style="color:var(--ss-glow)">{gold_count}</div><div class="ss-metric-label">Gold Examples</div></div>
1076
+ </div>"""
1077
+
1078
+
1079
+ # ── Main tab builder ───────────────────────────────────────────────────────────
1080
+ def smoke_signal_tab():
1081
+ """Call this inside your gr.Blocks() Tabs to add the Smoke Signal tab."""
1082
+
1083
+ with gr.TabItem("β—ˆ Smoke Signal", elem_id="ss-tab"):
1084
+
1085
+ # ── Wizard header ────────────────────────────────────────────────────
1086
+ gr.HTML("""
1087
+ <div style="background:linear-gradient(135deg,#0d1117,#161b22);padding:24px 28px 0;border-bottom:1px solid #30363d">
1088
+ <div style="display:flex;align-items:center;gap:16px;margin-bottom:20px">
1089
+ <div style="font-size:38px;line-height:1">β—ˆ</div>
1090
+ <div>
1091
+ <div style="font-family:'Playfair Display',Georgia,serif;font-size:26px;font-weight:900;color:#e6edf3;line-height:1">Smoke Signal</div>
1092
+ <div style="font-family:'Source Code Pro',monospace;font-size:10px;color:#7d8590;letter-spacing:3px;text-transform:uppercase;margin-top:4px">Picture-Book OCR Β· Extraction Pipeline Β· v1</div>
1093
+ </div>
1094
+ </div>
1095
+ <div class="ss-wizard">
1096
+ <div class="ss-step active"><span class="ss-num">1</span>INGEST</div>
1097
+ <div class="ss-connector"></div>
1098
+ <div class="ss-step"><span class="ss-num">2</span>PROFILE</div>
1099
+ <div class="ss-connector"></div>
1100
+ <div class="ss-step"><span class="ss-num">3</span>OCR</div>
1101
+ <div class="ss-connector"></div>
1102
+ <div class="ss-step"><span class="ss-num">4</span>REVIEW</div>
1103
+ <div class="ss-connector"></div>
1104
+ <div class="ss-step"><span class="ss-num">5</span>EXPORT</div>
1105
+ </div>
1106
+ </div>
1107
+ """)
1108
+
1109
+ with gr.Tabs() as wizard:
1110
+
1111
+ # ── STEP 1: INGEST ────────────────────────────────────────────────
1112
+ with gr.TabItem("β‘  Ingest", id="ss-ingest"):
1113
+ gr.HTML("""<div class="ss-panel-header" style="padding:20px 0 0 0">
1114
+ <div class="ss-panel-icon">πŸ“₯</div>
1115
+ <div><p class="ss-panel-title">Source Registry</p>
1116
+ <p class="ss-panel-sub">Upload PDFs Β· Register Β· Hash Β· Rights class</p></div>
1117
+ </div>""")
1118
+
1119
+ ingest_status = gr.HTML(_ingest_status_html("idle"))
1120
+
1121
+ with gr.Row():
1122
+ with gr.Column(scale=2):
1123
+ pdf_upload = gr.File(
1124
+ label="Drop picture-book PDFs here",
1125
+ file_types=[".pdf"],
1126
+ file_count="multiple",
1127
+ type="filepath",
1128
+ )
1129
+ with gr.Column(scale=1):
1130
+ rights_dd = gr.Dropdown(
1131
+ label="Rights Class",
1132
+ choices=["public-domain","licensed-owned","controlled-internal","unknown"],
1133
+ value="unknown",
1134
+ )
1135
+ ingest_notes = gr.Textbox(label="Notes", placeholder="Source, edition, etc.", lines=2)
1136
+ ingest_btn = gr.Button("Register Sources β†’", elem_classes=["ss-btn-run"])
1137
+
1138
+ manifest_table = gr.DataFrame(
1139
+ label="Source Manifest",
1140
+ interactive=False,
1141
+ wrap=True,
1142
+ )
1143
+ ingest_log = gr.Textbox(label="Log", lines=6, interactive=False, elem_classes=["ss-log"])
1144
+
1145
+ ingest_btn.click(
1146
+ ingest_pdfs,
1147
+ inputs=[pdf_upload, rights_dd, ingest_notes],
1148
+ outputs=[ingest_status, manifest_table, ingest_log],
1149
+ )
1150
+ gr.HTML('<div style="height:16px"></div>')
1151
+ wizard.load(lambda: (load_manifest_df(),), outputs=[manifest_table])
1152
+
1153
+ # ── STEP 2: PROFILE ───────────────────────────────────────────────
1154
+ with gr.TabItem("β‘‘ Profile", id="ss-profile"):
1155
+ gr.HTML("""<div class="ss-panel-header" style="padding:20px 0 0 0">
1156
+ <div class="ss-panel-icon">πŸ”</div>
1157
+ <div><p class="ss-panel-title">PDF Profiler</p>
1158
+ <p class="ss-panel-sub">Detect embedded text vs image pages Β· Route to extraction path</p></div>
1159
+ </div>""")
1160
+
1161
+ profile_status = gr.HTML(_profile_status_html())
1162
+ profile_btn = gr.Button("Run Profiler β†’", elem_classes=["ss-btn-run"])
1163
+ profile_log = gr.Textbox(label="Log", lines=10, interactive=False, elem_classes=["ss-log"])
1164
+
1165
+ profile_btn.click(run_profile, outputs=[profile_status, profile_log])
1166
+
1167
+ # ── STEP 3: OCR ───────────────────────────────────────────────────
1168
+ with gr.TabItem("β‘’ OCR", id="ss-ocr"):
1169
+ gr.HTML("""<div class="ss-panel-header" style="padding:20px 0 0 0">
1170
+ <div class="ss-panel-icon">πŸ‘</div>
1171
+ <div><p class="ss-panel-title">OCR Engine</p>
1172
+ <p class="ss-panel-sub">Surya layout + recognition Β· Confidence scoring Β· Review queue</p></div>
1173
+ </div>""")
1174
+
1175
+ ocr_status = gr.HTML(_ocr_status_html())
1176
+
1177
+ gr.HTML("""<div class="ss-card">
1178
+ <div class="ss-card-title">Self-Improvement Loop</div>
1179
+ <div style="font-size:13px;color:#8b949e;line-height:1.6">
1180
+ Every correction you make in Step 4 is logged to the gold training set and recalibrates
1181
+ the confidence thresholds for that region class in real time. The more you review,
1182
+ the smarter the pipeline gets β€” without retraining.
1183
+ </div>
1184
+ </div>""")
1185
+
1186
+ ocr_btn = gr.Button("Run OCR β†’", elem_classes=["ss-btn-run"])
1187
+ ocr_log = gr.Textbox(label="Log", lines=10, interactive=False, elem_classes=["ss-log"])
1188
+
1189
+ ocr_btn.click(run_ocr, outputs=[ocr_status, ocr_log])
1190
+
1191
+ # ── STEP 4: REVIEW ────────────────────────────────────────────────
1192
+ with gr.TabItem("β‘£ Review", id="ss-review"):
1193
+ gr.HTML("""<div class="ss-panel-header" style="padding:20px 0 0 0">
1194
+ <div class="ss-panel-icon">✎</div>
1195
+ <div><p class="ss-panel-title">Review Workbench</p>
1196
+ <p class="ss-panel-sub">Correct Β· Accept Β· Reject Β· Build gold training set</p></div>
1197
+ </div>""")
1198
+
1199
+ review_status = gr.HTML(_review_status_html())
1200
+ training_feedback = gr.HTML()
1201
+
1202
+ with gr.Row():
1203
+ with gr.Column(scale=2):
1204
+ review_image = gr.Image(
1205
+ label="Page Render",
1206
+ type="filepath",
1207
+ height=420,
1208
+ show_download_button=False,
1209
+ )
1210
+ item_info = gr.HTML()
1211
+
1212
+ with gr.Column(scale=2):
1213
+ raw_text_box = gr.Textbox(
1214
+ label="Raw OCR",
1215
+ lines=6,
1216
+ interactive=False,
1217
+ )
1218
+ final_text_box = gr.Textbox(
1219
+ label="Final Text (edit to correct)",
1220
+ lines=8,
1221
+ interactive=True,
1222
+ )
1223
+ reviewer_name = gr.Textbox(label="Your name", placeholder="e.g. jamal", scale=1)
1224
+ reason_code = gr.Dropdown(
1225
+ label="Reason code",
1226
+ choices=["","OCR_MISS","OCR_WRONG_WORD","DECORATIVE_FONT",
1227
+ "SPEECH_BUBBLE_ERROR","LOW_CONTRAST","SCAN_SKEW_BLUR",
1228
+ "NON_STORY_TEXT","LLM_OVER_CORRECTION","OTHER"],
1229
+ value="",
1230
+ )
1231
+
1232
+ with gr.Row():
1233
+ accept_btn = gr.Button("βœ“ Accept", elem_classes=["ss-btn-accept"])
1234
+ edit_btn = gr.Button("✎ Save Edit", elem_classes=["ss-btn-edit"])
1235
+ with gr.Row():
1236
+ reject_btn = gr.Button("βœ— Reject", elem_classes=["ss-btn-reject"])
1237
+ quar_btn = gr.Button("βš‘ Quarantine", elem_classes=["ss-btn-quar"])
1238
+ next_btn = gr.Button("Next Item β†’", elem_classes=["ss-btn-next"])
1239
+
1240
+ current_idx = gr.State(0)
1241
+
1242
+ review_outputs = [training_feedback, review_image, raw_text_box, item_info,
1243
+ gr.State(), gr.State()]
1244
+
1245
+ def load_review():
1246
+ img, raw, info, done, total = get_review_item(0)
1247
+ status = _review_status_html()
1248
+ return status, img, raw, raw, info
1249
+
1250
+ def next_item(idx):
1251
+ new_idx = idx + 1
1252
+ img, raw, info, done, total = get_review_item(new_idx)
1253
+ return new_idx, img, raw, raw, info
1254
+
1255
+ def do_accept(idx, final, reviewer, reason):
1256
+ action = "edited" if final.strip() != "" else "accepted"
1257
+ fb, img, raw, info, _, _ = save_review_decision(idx, final, action, reviewer, reason)
1258
+ status = _review_status_html()
1259
+ new_idx = idx + 1
1260
+ img2, raw2, info2, _, _ = get_review_item(new_idx)
1261
+ return fb, status, new_idx, img2, raw2, raw2, info2
1262
+
1263
+ def do_reject(idx, final, reviewer, reason):
1264
+ fb, img, raw, info, _, _ = save_review_decision(idx, final, "rejected", reviewer, reason)
1265
+ status = _review_status_html()
1266
+ new_idx = idx + 1
1267
+ img2, raw2, info2, _, _ = get_review_item(new_idx)
1268
+ return fb, status, new_idx, img2, raw2, raw2, info2
1269
+
1270
+ def do_quarantine(idx, final, reviewer, reason):
1271
+ fb, img, raw, info, _, _ = save_review_decision(idx, final, "quarantined", reviewer, reason)
1272
+ status = _review_status_html()
1273
+ new_idx = idx + 1
1274
+ img2, raw2, info2, _, _ = get_review_item(new_idx)
1275
+ return fb, status, new_idx, img2, raw2, raw2, info2
1276
+
1277
+ action_outputs = [training_feedback, review_status, current_idx,
1278
+ review_image, raw_text_box, final_text_box, item_info]
1279
+
1280
+ accept_btn.click(do_accept, inputs=[current_idx, final_text_box, reviewer_name, reason_code], outputs=action_outputs)
1281
+ edit_btn.click(do_accept, inputs=[current_idx, final_text_box, reviewer_name, reason_code], outputs=action_outputs)
1282
+ reject_btn.click(do_reject, inputs=[current_idx, final_text_box, reviewer_name, reason_code], outputs=action_outputs)
1283
+ quar_btn.click(do_quarantine, inputs=[current_idx, final_text_box, reviewer_name, reason_code], outputs=action_outputs)
1284
+
1285
+ def _next(idx):
1286
+ new_idx = idx + 1
1287
+ img, raw, info, _, _ = get_review_item(new_idx)
1288
+ return new_idx, img, raw, raw, info
1289
+
1290
+ next_btn.click(_next, inputs=[current_idx],
1291
+ outputs=[current_idx, review_image, raw_text_box, final_text_box, item_info])
1292
+
1293
+ wizard.load(load_review, outputs=[review_status, review_image, raw_text_box, final_text_box, item_info])
1294
+
1295
+ # ── STEP 5: EXPORT ────────────────────────────────────────────────
1296
+ with gr.TabItem("β‘€ Export", id="ss-export"):
1297
+ gr.HTML("""<div class="ss-panel-header" style="padding:20px 0 0 0">
1298
+ <div class="ss-panel-icon">⬇</div>
1299
+ <div><p class="ss-panel-title">Codex Export</p>
1300
+ <p class="ss-panel-sub">Clean JSONL Β· Gold training set Β· Auto-feed Codex</p></div>
1301
+ </div>""")
1302
+
1303
+ export_status = gr.HTML(_export_status_html())
1304
+
1305
+ gr.HTML("""<div class="ss-card">
1306
+ <div class="ss-card-title">What gets exported</div>
1307
+ <div style="font-size:13px;color:#8b949e;line-height:1.8">
1308
+ <b style="color:#e6edf3">codex_export_[batch].jsonl</b> β€” all accepted/edited text with full provenance,
1309
+ ready for Codex fingerprint analysis.<br>
1310
+ <b style="color:#e6edf3">gold_set_[batch].jsonl</b> β€” your labelled corrections for future Surya fine-tuning.
1311
+ The more you correct, the better your next training run will be.
1312
+ </div>
1313
+ </div>""")
1314
+
1315
+ export_btn = gr.Button("Export to Codex β†’", elem_classes=["ss-btn-run"])
1316
+ export_log = gr.Textbox(label="Export log", lines=6, interactive=False, elem_classes=["ss-log"])
1317
+
1318
+ with gr.Row():
1319
+ codex_download = gr.File(label="Codex JSONL", interactive=False)
1320
+ gold_download = gr.File(label="Gold Training Set", interactive=False)
1321
+
1322
+ export_btn.click(
1323
+ run_export,
1324
+ outputs=[export_status, export_log, codex_download, gold_download],
1325
+ )