Update page_files/Upload_Data.py

page_files/Upload_Data.py

@@ -45,9 +45,8 @@ from categorized.Backend.PDF_DataExtraction import (
 # and category_push). Ensure image2.py / mapper5.py / category_push.py are on
 # the import path (same directory or added to sys.path above). If you keep them
 # under categorized/Backend/, change these three lines to:
-#   from categorized.Backend import Pdf_ImageExtraction, mapper5, category_push
-
-from categorized.Backend import Pdf_ImageExtraction 
+#   from categorized.Backend import image2, mapper5, category_push
+from categorized.Backend import Pdf_ImageExtraction as image2  # image2.py renamed
 from categorized.Backend import mapper5, category_push
 
 # Manual-entry path (unchanged — writes via the existing data_loader).
@@ -115,15 +114,24 @@ def _df_to_meta(df: pd.DataFrame) -> dict:
 # which is exactly what mapper5.map_plots_to_properties and the display expect.
 # ─────────────────────────────────────────────────────────────────────────────
 
-def extract_images(pdf_path: str) -> list:
+def extract_images(pdf_path: str, engines=None, check_missed: bool = True) -> list:
+    """Detect + (optionally) recover + crop-verify plots.
+    `engines` selects the verifier LLMs: ["gemini"] (default, fast) or
+    ["gemini","gpt","claude"] for a majority-vote cross-check that rejects more
+    non-plots (needs OPENAI_API_KEY / ANTHROPIC_API_KEY; missing keys just make
+    that engine abstain). `check_missed` toggles the per-page Gemini recovery scan."""
+    engines = engines or ["gemini"]
     try:
         with open(pdf_path, "rb") as f:
             pdf_bytes = f.read()
-            plot_results, coverage = Pdf_ImageExtraction.extract_and_verify_plots(
-            pdf_bytes, verify_engines=["gemini"]
+        plot_results, coverage = image2.extract_and_verify_plots(
+            pdf_bytes,
+            check_missed_plots=check_missed,
+            verify_crops=True,
+            verify_engines=engines,
         )
         # Drop captions naming photos/micrographs/logos and crops the verifier
-        # marked "discard" (keeps "keep"/"recrop"). Pure selection, non-mutating.
+        # marked "discard" (keeps "keep"/"recrop"/"unverified"). Non-mutating.
         plot_results = mapper5.select_real_plots(plot_results)
         st.session_state["plot_coverage"] = coverage
     except Exception as e:
@@ -535,7 +543,7 @@ def render_material_data_tab(pdf_path: str):
                 unsafe_allow_html=True,
             )
             timer.caption(
-                f"⏱ Elapsed: {elapsed:.0f}s"
+                f" Elapsed: {elapsed:.0f}s"
                 + (f" · Est. remaining: ~{est_remaining}s" if est_remaining > 0 else "")
             )
 
@@ -569,7 +577,7 @@ def render_material_data_tab(pdf_path: str):
             st.session_state.pdf_data_extracted = True
             st.session_state.pdf_extracted_meta = _df_to_meta(df)
             st.session_state.pdf_doi            = doi
-            st.success(f"Extracted {len(df)} properties in {elapsed_total:.0f}s")
+            st.success(f" Extracted {len(df)} properties in {elapsed_total:.0f}s")
         else:
             st.warning("No data extracted from PDF.")
             return
@@ -619,18 +627,57 @@ def _links_for_group(links_df: pd.DataFrame, gi: int) -> pd.DataFrame:
     return links_df[links_df["group_idx"] == gi]
 
 
+def _reset_plot_mapping():
+    """Structural edits (removing a crop/figure) shift the positional group_idx
+    that links_df is keyed on, so any edit invalidates the current mapping.
+    Clearing it forces a clean re-map on the pruned set."""
+    st.session_state.mapping_done = False
+    st.session_state.links_df     = pd.DataFrame()
+    st.session_state.df_aug       = pd.DataFrame()
+    st.session_state.store        = None
+
+
 def render_plots_tab(pdf_path: str, paper_id: str):
     st.subheader("Extracted Plot Images & Property Mapping")
 
-    # 1) Extract plots once (image2 detection + recovery + crop verification)
+    # ── Extraction settings: multi-LLM cross-check + recovery toggle ──────────
+    with st.expander("⚙ Extraction settings", expanded=not st.session_state.pdf_processed):
+        multi = st.checkbox(
+            "Cross-check each figure with GPT + Claude (majority vote — rejects more "
+            "non-plots, but ~3× slower and needs OPENAI_API_KEY / ANTHROPIC_API_KEY)",
+            value=st.session_state.get("verify_multi", False),
+            key="verify_multi",
+        )
+        recover = st.checkbox(
+            "Missed-plot recovery (per-page Gemini scan — catches plots OpenCV missed, slower)",
+            value=st.session_state.get("recover_missed", True),
+            key="recover_missed",
+        )
+        if multi and not (os.getenv("OPENAI_API_KEY") and os.getenv("ANTHROPIC_API_KEY")):
+            st.caption(" One or both of OPENAI_API_KEY / ANTHROPIC_API_KEY is not set — "
+                       "that engine will abstain; the vote falls back to the ones available.")
+        if st.session_state.pdf_processed and st.button(" Re-extract with these settings"):
+            st.session_state.pdf_processed = False
+            _reset_plot_mapping()
+            st.rerun()
+
+    # 1) Extract plots once (detection + recovery + crop verification)
     if not st.session_state.pdf_processed:
-        with st.spinner("Extracting plots from PDF…"):
-            st.session_state.plot_results  = extract_images(pdf_path)
+        engines = ["gemini", "gpt", "claude"] if st.session_state.get("verify_multi") else ["gemini"]
+        with st.spinner("Extracting plots from PDF…"
+                        + (" (cross-checking with GPT + Claude — this is slower)"
+                           if st.session_state.get("verify_multi") else "")):
+            st.session_state.plot_results  = extract_images(
+                pdf_path, engines=engines,
+                check_missed=st.session_state.get("recover_missed", True),
+            )
             st.session_state.pdf_processed = True
-            st.session_state.mapping_done  = False
-            st.session_state.links_df      = pd.DataFrame()
-            st.session_state.df_aug        = pd.DataFrame()
-            st.session_state.store         = None
+            _reset_plot_mapping()
+
+    plot_results = st.session_state.plot_results
+    if not plot_results:
+        st.warning("No plots found in this PDF.")
+        return
 
     plot_results = st.session_state.plot_results
     if not plot_results:
@@ -655,14 +702,14 @@ def render_plots_tab(pdf_path: str, paper_id: str):
     # Downloads (image2's native zipper)
     d1, d2 = st.columns(2)
     d1.download_button(
-        "⬇ Images + metadata (ZIP)",
-        data=Pdf_ImageExtraction.create_plot_zip(plot_results, include_json=True),
+        " Images + metadata (ZIP)",
+        data=image2.create_plot_zip(plot_results, include_json=True),
         file_name=f"{paper_id}_plots.zip", mime="application/zip",
         use_container_width=True, key="dl_plots_zip",
     )
     d2.download_button(
-        "⬇ Images only (ZIP)",
-        data=Pdf_ImageExtraction.create_plot_zip(plot_results, include_json=False),
+        " Images only (ZIP)",
+        data=image2.create_plot_zip(plot_results, include_json=False),
         file_name=f"{paper_id}_images.zip", mime="application/zip",
         use_container_width=True, key="dl_images_zip",
     )
@@ -703,14 +750,24 @@ def render_plots_tab(pdf_path: str, paper_id: str):
 
     st.divider()
 
-    # 3) Figure-centric review — each figure with the property rows linked to it
+    # 3) Figure-centric review — each figure with delete/remove controls and
+    #    (after mapping) the property rows linked to it.
+    st.caption("Review the crops below. Remove a bad crop with **✕**, or drop a whole "
+               "figure with **🗑**. Editing figures clears the current mapping — re-run "
+               "mapping afterwards.")
     for gi, group in enumerate(plot_results):
         caption = group.get("caption", f"Figure {gi+1}")
         page    = group.get("page", "?")
         imgs    = group.get("image_data", [])
 
         with st.container(border=True):
-            st.markdown(f"**Page {page}** — {caption}")
+            head, dele = st.columns([0.85, 0.15])
+            head.markdown(f"**Page {page}** — {caption}")
+            if dele.button(" Delete ", key=f"delfig_{gi}", use_container_width=True):
+                plot_results.pop(gi)
+                st.session_state.plot_results = plot_results
+                _reset_plot_mapping()
+                st.rerun()
 
             icols = st.columns(min(len(imgs), 4) or 1)
             for pos, im in enumerate(imgs):
@@ -720,10 +777,28 @@ def render_plots_tab(pdf_path: str, paper_id: str):
                         sub = im.get("subplot_label") or ""
                         st.image(arr, channels="BGR", width=200,
                                  caption=(sub or None))
+
                     v = im.get("verification") or {}
                     act = v.get("majority_action") if isinstance(v, dict) else None
                     if act and act != "keep":
-                        st.caption(f"crop verdict: {act}")
+                        st.caption(f"crop verdict: **{act}**")
+                    # per-engine detail when the multi-LLM cross-check ran
+                    by = v.get("by_engine") if isinstance(v, dict) else None
+                    if isinstance(by, dict):
+                        flagged = [e for e, vd in by.items()
+                                   if isinstance(vd, dict) and vd.get("recommended_action") == "discard"]
+                        if flagged:
+                            st.caption("flagged non-plot by: " + ", ".join(flagged))
+                    elif act in (None, "unverified"):
+                        st.caption("crop verdict: unverified")
+
+                    if st.button("✕ Remove", key=f"rmimg_{gi}_{pos}", use_container_width=True):
+                        imgs.pop(pos)
+                        if not imgs:
+                            plot_results.pop(gi)
+                        st.session_state.plot_results = plot_results
+                        _reset_plot_mapping()
+                        st.rerun()
 
             if mapping_done:
                 grp = _links_for_group(links_df, gi)
@@ -770,7 +845,7 @@ def render_plots_tab(pdf_path: str, paper_id: str):
                "attaches each crop's path. Push conforms those rows to the chosen "
                "category table's 33-column schema and uploads them.")
 
-    if st.button(" Store to database (SQLite)", use_container_width=True):
+    if st.button("Store to database (SQLite)", use_container_width=True):
         with st.spinner("Saving crops and writing SQLite…"):
             st.session_state.store = mapper5.store_properties_with_plots(
                 st.session_state.df_aug,
@@ -858,6 +933,8 @@ def main():
         "pdf_extracted_df":   pd.DataFrame(),
         "pdf_extracted_meta": {},
         "pdf_doi":            "",
+        "verify_multi":       False,
+        "recover_missed":     True,
     }
     for k, v in defaults.items():
         if k not in st.session_state:
@@ -898,7 +975,7 @@ def main():
         st.tabs(["Material Data", "Extracted Plots"])
         return
 
-    tab1, tab2 = st.tabs([" Material Data", "Extracted Plots"])
+    tab1, tab2 = st.tabs([" Material Data", " Extracted Plots"])
 
     tmp_file = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False, prefix="matdb_")
     try: