Consolidate profile into ingest and make profile tab optional

smoke_signal_tab.py

@@ -10,8 +10,8 @@ Then add to app.py:
     # Add smoke_signal_tab() call inside your gr.Blocks() tabs
 
 Architecture:
-    Step 1: INGEST   — upload PDFs, register + hash
-    Step 2: PROFILE  — detect text vs image pages
+    Step 1: INGEST+PROFILE — upload PDFs, register, auto-profile
+    Step 2: PROFILE (optional) — manual re-run when needed
     Step 3: OCR      — Surya extraction + confidence scoring
     Step 4: REVIEW   — human correction workbench (feeds training data)
     Step 5: EXPORT   — clean JSONL to Codex + downloadable gold set
@@ -498,6 +498,7 @@ def ingest_pdfs(files, rights_class: str, notes: str) -> tuple:
     log  = []
     new_count = 0
     dup_count = 0
+    auto_profile_ids = []
 
     for file in files:
         path = Path(file.name) if hasattr(file, "name") else Path(file)
@@ -509,11 +510,15 @@ def ingest_pdfs(files, rights_class: str, notes: str) -> tuple:
 
         # Check duplicate — update rights/notes if changed
         if not df.empty and file_hash in df["sha256"].values:
+            existing_book_id = df.loc[df["sha256"] == file_hash, "book_id"].values[0]
+            existing_status = df.loc[df["sha256"] == file_hash, "status"].values[0]
             existing_rights = df.loc[df["sha256"] == file_hash, "rights_class"].values[0]
             if rights_class != "unknown" and existing_rights != rights_class:
                 df.loc[df["sha256"] == file_hash, "rights_class"] = rights_class
                 df.loc[df["sha256"] == file_hash, "notes"] = notes
                 log.append(log_line(f"↻ Updated rights for duplicate: {path.name} → {rights_class}"))
+                if existing_status == "pending":
+                    auto_profile_ids.append(existing_book_id)
             else:
                 log.append(log_line(f"↩ Duplicate: {path.name} (rights={existing_rights})"))
             dup_count += 1
@@ -548,12 +553,22 @@ def ingest_pdfs(files, rights_class: str, notes: str) -> tuple:
         df = pd.concat([df, new_row], ignore_index=True)
         log.append(log_line(f"✓ Registered {book_id} — {path.name} ({page_count or '?'} pages)"))
         new_count += 1
+        auto_profile_ids.append(book_id)
 
     save_manifest_df(df)
 
     summary = f"Registered {new_count} new | {dup_count} duplicates skipped"
     log.append(log_line(summary))
 
+    unique_profile_ids = sorted(set(auto_profile_ids))
+    if unique_profile_ids:
+        log.append(log_line(f"↻ Auto-profile queued for {len(unique_profile_ids)} source(s)"))
+        _, profile_log = run_profile(book_ids=unique_profile_ids)
+        for line in str(profile_log).splitlines():
+            if line.strip():
+                log.append(line)
+        df = load_manifest_df()
+
     return _ingest_status_html("done", new_count, dup_count), df, "\n".join(log)
 
 
@@ -582,20 +597,30 @@ def update_rights(book_id: str, new_rights: str) -> tuple:
     df = load_manifest_df()
     if df.empty or book_id not in df["book_id"].values:
         return _ingest_status_html("idle"), df, f"Book ID {book_id} not found."
+    prev_status = df.loc[df["book_id"] == book_id, "status"].values[0]
     df.loc[df["book_id"] == book_id, "rights_class"] = new_rights
     save_manifest_df(df)
-    return _ingest_status_html("done"), df, f"[{datetime.utcnow().strftime('%H:%M:%S')}] ✓ Updated {book_id} rights → {new_rights}"
+    msg = f"[{datetime.utcnow().strftime('%H:%M:%S')}] ✓ Updated {book_id} rights → {new_rights}"
+    if prev_status == "pending" and new_rights not in ("unknown", "excluded"):
+        _, profile_log = run_profile(book_ids=[book_id])
+        if profile_log:
+            msg = msg + "\n" + str(profile_log)
+    return _ingest_status_html("done"), load_manifest_df(), msg
 
 
 # ── Step 2: PROFILE ────────────────────────────────────────────────────────────
-def run_profile() -> tuple:
-    """Profile all pending PDFs."""
+def run_profile(book_ids: Optional[list[str]] = None) -> tuple:
+    """Profile pending PDFs. If book_ids are provided, profile only those sources."""
     df = load_manifest_df()
     if df.empty:
         return _profile_status_html(), "No sources registered. Complete Step 1 first."
 
     pending = df[df["status"] == "pending"]
+    if book_ids:
+        pending = pending[pending["book_id"].isin(book_ids)]
     if pending.empty:
+        if book_ids:
+            return _profile_status_html(), "No pending PDFs to profile for selected sources."
         return _profile_status_html(), "No pending PDFs to profile."
 
     log = []
@@ -672,7 +697,7 @@ def run_profile() -> tuple:
         log.append(
             log_line(
                 f"⚠ {skipped_rights} source(s) skipped due rights_class=unknown/excluded. "
-                "Set rights in Step 1 and rerun Profile."
+                "Set rights in Step 1 to auto-profile on update, or rerun Profile manually."
             )
         )
 
@@ -1257,7 +1282,9 @@ def run_ocr(progress=gr.Progress(track_tqdm=False)) -> tuple:
                 conf = 0.0
                 method = "skipped-no-surya"
 
-            if conf >= default_cal["auto_accept"]:
+            if method == "tesseract-noise-filtered" and not regions:
+                conf_class = "auto-accept"
+            elif conf >= default_cal["auto_accept"]:
                 conf_class = "auto-accept"
             elif conf >= default_cal["review"]:
                 conf_class = "review-required"
@@ -1589,9 +1616,9 @@ def smoke_signal_tab():
                 </div>
             </div>
             <div class="ss-wizard">
-                <div class="ss-step active"><span class="ss-num">1</span>INGEST</div>
+                <div class="ss-step active"><span class="ss-num">1</span>INGEST+PROFILE</div>
                 <div class="ss-connector"></div>
-                <div class="ss-step"><span class="ss-num">2</span>PROFILE</div>
+                <div class="ss-step"><span class="ss-num">2</span>PROFILE (OPTIONAL)</div>
                 <div class="ss-connector"></div>
                 <div class="ss-step"><span class="ss-num">3</span>OCR</div>
                 <div class="ss-connector"></div>
@@ -1609,7 +1636,7 @@ def smoke_signal_tab():
                 gr.HTML("""<div class="ss-panel-header" style="padding:20px 0 0 0">
                     <div class="ss-panel-icon">📥</div>
                     <div><p class="ss-panel-title">Source Registry</p>
-                    <p class="ss-panel-sub">Upload PDFs · Register · Hash · Rights class</p></div>
+                    <p class="ss-panel-sub">Upload PDFs · Register · Auto-profile · Rights class</p></div>
                 </div>""")
 
                 ingest_status = gr.HTML(_ingest_status_html("idle"))
@@ -1629,7 +1656,7 @@ def smoke_signal_tab():
                             value="unknown",
                         )
                         ingest_notes = gr.Textbox(label="Notes", placeholder="Source, edition, etc.", lines=2)
-                        ingest_btn = gr.Button("Register Sources →", elem_classes=["ss-btn-run"])
+                        ingest_btn = gr.Button("Register + Auto-Profile →", elem_classes=["ss-btn-run"])
 
                 manifest_table = gr.DataFrame(
                     label="Source Manifest",
@@ -1665,15 +1692,15 @@ def smoke_signal_tab():
                 # Auto-load removed — use Refresh button instead to avoid SSR hang
 
             # ── STEP 2: PROFILE ───────────────────────────────────────────────
-            with gr.TabItem("② Profile", id="ss-profile"):
+            with gr.TabItem("② Profile (Optional)", id="ss-profile"):
                 gr.HTML("""<div class="ss-panel-header" style="padding:20px 0 0 0">
                     <div class="ss-panel-icon">🔍</div>
                     <div><p class="ss-panel-title">PDF Profiler</p>
-                    <p class="ss-panel-sub">Detect embedded text vs image pages · Route to extraction path</p></div>
+                    <p class="ss-panel-sub">Auto-runs during ingest · use here only for manual re-profile</p></div>
                 </div>""")
 
                 profile_status = gr.HTML(_profile_status_html())
-                profile_btn    = gr.Button("Run Profiler →", elem_classes=["ss-btn-run"])
+                profile_btn    = gr.Button("Re-run Profiler (Optional) →", elem_classes=["ss-btn-run"])
                 profile_log    = gr.Textbox(label="Log", lines=10, interactive=False, elem_classes=["ss-log"])
 
                 profile_btn.click(run_profile, outputs=[profile_status, profile_log])