Update page_files/categorized/Backend/mapper5.py

page_files/categorized/Backend/mapper5.py

@@ -5,10 +5,7 @@ mapper.py — Plot ⇄ Property mapping + persistence
 Combines what used to be two files (Code 3 "mapping" + Code 4 "storage")
 into one module. It sits on top of:
 
-    2.py       — Code 1: extraction + tri-LLM consensus + source verification
-                 (imported via importlib, since a module name can't start
-                 with a digit)
-    image2.py  — Code 2: plot/figure extraction + crop verification
+
 
 WHAT IT DOES
 ------------
@@ -314,7 +311,7 @@ def _get_shared_embed_model() -> Optional[Any]:
     _embed_tried = True
     try:
         import importlib
-        _embed_model = importlib.import_module("2")._get_embed_model()  # Code 1 = 2.py
+        _embed_model = importlib.import_module("Pdf_DataExtraction")._get_embed_model()  # Code 1 = 2.py
         log.info("mapper: reusing Code 1's SciBERT embedding model.")
         return _embed_model
     except Exception as e:
@@ -847,7 +844,9 @@ def store_properties_with_plots(
     df = df_augmented.reset_index(drop=True).copy()
 
     mapped_gis = set(int(g) for g in links_df["group_idx"].tolist()) if not links_df.empty else set()
-    group_idxs = None if save_all_group_images else mapped_gis
+    # None = save every detected crop. If nothing mapped (empty links), fall back
+    # to saving all crops so images still persist instead of writing zero.
+    group_idxs = None if (save_all_group_images or not mapped_gis) else mapped_gis
 
     path_map, groups = save_plot_images(plot_results, out_dir, pdf_stem, group_idxs=group_idxs)
     group_by_idx = {g["group_idx"]: g for g in groups}
@@ -1116,7 +1115,7 @@ def _extract(pdf_bytes: bytes, doi_override: str = "",
     """Run Code 1 (2.py) + Code 2 (image2.py) ONCE. Returns a bundle with all
     six property tables + plot_results + coverage + meta + errors."""
     import importlib
-    run_pipeline = importlib.import_module("2").run_pipeline
+    run_pipeline = importlib.import_module("Pdf_DataExtraction").run_pipeline
     from Pdf_ImageExtraction import extract_and_verify_plots
     (dfc, dfv, dfcv, dfg, dfp, dfcl, _chunks, errors, meta) = run_pipeline(
         pdf_bytes, doi_override=doi_override)
@@ -1135,7 +1134,7 @@ def _extract(pdf_bytes: bytes, doi_override: str = "",
 def get_or_extract(pdf_bytes: bytes, doi_override: str = "", verify_engines=None,
                    use_disk_cache: bool = True, force: bool = False):
     """Extraction cached by the PDF's CONTENT HASH — never by source_table.
-    On a cache hit, Pdf_DataExtraction.py and Pdf_ImageExtraction.py are NOT re-run, so switching tables /
+    On a cache hit, 2.py and image2.py are NOT re-run, so switching tables /
     pruning / re-mapping is free. Returns (pdf_hash, bundle)."""
     import pickle
     h = _pdf_hash(pdf_bytes)
@@ -1194,7 +1193,7 @@ def _run_streamlit() -> None:
 
     st.set_page_config(page_title="mapper — Plot ⇄ Property", page_icon="🔗", layout="wide")
     st.title("🔗 mapper — Plot ⇄ Property mapping + database")
-    st.caption("Runs Pdf_DataExtraction.py + Pdf_ImageExtraction.py once (cached), auto-drops non-plots, lets you "
+    st.caption("Runs 2.py + image2.py once (cached), auto-drops non-plots, lets you "
                "prune what's left, then attaches plots to properties on Map.")
 
     ss = st.session_state
@@ -1244,13 +1243,13 @@ def _run_streamlit() -> None:
 
     # ---- sidebar ------------------------------------------------------------
     with st.sidebar:
-        st.header("⚙️ Settings")
+        st.header("Settings")
         st.subheader("Extraction")
         use_gpt = st.checkbox("Also verify crops with GPT", value=False)
         use_claude = st.checkbox("Also verify crops with Claude", value=False)
         doi_manual = st.text_input("DOI override (optional)", placeholder="10.xxxx/...")
         use_disk_cache = st.checkbox("Use on-disk extraction cache", value=True,
-                                     help="Skip Pdf_DataExtraction.py + Pdf_ImageExtraction.py entirely when this exact "
+                                     help="Skip 2.py + image2.py entirely when this exact "
                                           "PDF was extracted before.")
         st.divider()
         st.subheader("Auto figure filter")
@@ -1292,12 +1291,12 @@ def _run_streamlit() -> None:
                                 use_container_width=True)
     else:
         ec1.success("✓ This paper is loaded — switch tables / prune / map freely; "
-                    "Pdf_DataExtraction.py and Pdf_ImageExtraction.py will NOT re-run.")
+                    "2.py and image2.py will NOT re-run.")
     force = ec2.button("↻ Force re-extract", use_container_width=True)
 
     if do_extract or force:
         engines = ["gemini"] + (["gpt"] if use_gpt else []) + (["claude"] if use_claude else [])
-        with st.spinner("Running Pdf_DataExtraction.py extraction/consensus + Pdf_ImageExtraction.py plot detection…"):
+        with st.spinner("Running 2.py extraction/consensus + image2.py plot detection…"):
             _h, bundle = get_or_extract(pdf_bytes, doi_override=doi_manual,
                                         verify_engines=engines,
                                         use_disk_cache=use_disk_cache, force=force)
@@ -1314,15 +1313,15 @@ def _run_streamlit() -> None:
     bundle = ss["bundle"]
     stem = ss["stem"]
     if bundle.get("_cache") == "disk":
-        st.caption("↩ Loaded from on-disk cache — Pdf_DataExtraction.py and Pdf_ImageExtraction.py were not re-run.")
+        st.caption(" Loaded from on-disk cache — 2.py and image2.py were not re-run.")
 
     # ---- three top-level views ---------------------------------------------
     view_props, view_plots, work = st.tabs(
-        ["📄 Extraction (2.py)", "🖼️ Plots (Pdf_ImageExtraction.py)", "🔗 Map & curate"])
+        [" Extraction (2.py)", " Plots (image2.py)", " Map & curate"])
 
     with view_props:
         if bundle.get("api_errors"):
-            with st.expander(f"⚠️ {len(bundle['api_errors'])} API error(s)"):
+            with st.expander(f" {len(bundle['api_errors'])} API error(s)"):
                 for e in bundle["api_errors"]:
                     st.code(e)
         meta = bundle.get("meta", {})
@@ -1384,7 +1383,7 @@ def _run_streamlit() -> None:
                 with st.container(border=True):
                     hc1, hc2 = st.columns([6, 1])
                     hc1.markdown(f"**Page {group.get('page')}** — {group.get('caption', '')}")
-                    if hc2.button("🗑 Remove figure", key=f"rmfig_{gi}"):
+                    if hc2.button(" Remove figure", key=f"rmfig_{gi}"):
                         _remove_figure(gi)
                     images = group.get("image_data", [])
                     icols = st.columns(min(len(images), 4) or 1)
@@ -1399,7 +1398,7 @@ def _run_streamlit() -> None:
                             if st.button("x remove", key=f"rmimg_{gi}_{idx}"):
                                 _remove_image(gi, idx)
 
-        map_label = "③ Map — attach plots to properties" + (" (re-map)" if ss.get("mapped") else "")
+        map_label = " Map — attach plots to properties" + (" (re-map)" if ss.get("mapped") else "")
         if st.button(map_label, type="primary", use_container_width=True):
             source_df = bundle["table_map"][source_table]
             with st.spinner("Mapping property rows to figures…"):
@@ -1468,13 +1467,13 @@ def _run_streamlit() -> None:
                         out.append((imgs[ix].get("array"), cap))
             return out
 
-        names = ["🖼️ Figures & attached rows (remove links)",
-                 f"🔗 All links ({len(links_df)})",
-                 f"🧬 Augmented ({n_rows})",
-                 "📤 Store / Export"]
+        names = [" Figures & attached rows (remove links)",
+                 f" All links ({len(links_df)})",
+                 f" Augmented ({n_rows})",
+                 " Store / Export"]
         if store:
-            names.insert(3, f"🗃️ Stored ({store['n_properties']})")
-        names.append("🔎 Properties + image")
+            names.insert(3, f" Stored ({store['n_properties']})")
+        names.append(" Properties + image")
         wt = st.tabs(names)
         prop_img_tab = wt[-1]
 
@@ -1584,7 +1583,7 @@ def _run_streamlit() -> None:
             st.markdown("**Store the curated result to the database**")
             st.caption("Writes the pruned figures + edited links; each stored row points at "
                        "its saved crop on disk. Re-running for the same PDF overwrites cleanly.")
-            if st.button("💾 Store to database", type="primary", use_container_width=True):
+            if st.button(" Store to database", type="primary", use_container_width=True):
                 with st.spinner("Saving crops and writing SQLite…"):
                     ss["store"] = store_properties_with_plots(
                         ss["df_aug"], ss["links_df"], ss["plot_curated"],
@@ -1608,14 +1607,14 @@ def _run_streamlit() -> None:
                 embed_img = st.checkbox("Embed matched plot into the table's image column",
                                         value=True)
                 cA, cB = st.columns(2)
-                if cA.button("🔌 Test connection", use_container_width=True):
+                if cA.button(" Test connection", use_container_width=True):
                     try:
                         db_healthcheck()
                         st.success(f"Connected to {DB_HOST} ({DB_KIND}).")
                     except Exception as e:
                         st.error(f"Connection failed: {e}")
                 push_ready = bool(store) and store.get("n_properties", 0) > 0
-                if cB.button(f"⬆️ Push to {category}", type="primary",
+                if cB.button(f" Push to {category}", type="primary",
                              use_container_width=True, disabled=not push_ready):
                     try:
                         with st.spinner(f"Conforming + writing rows to '{category}'…"):
@@ -1628,24 +1627,24 @@ def _run_streamlit() -> None:
                     except Exception as e:
                         st.error(f"Push failed: {e}")
                 if not push_ready:
-                    st.caption("Click **💾 Store to database** first — the RDS push "
+                    st.caption("Click ** Store to database** first — the RDS push "
                                "uploads those stored rows.")
 
             st.divider()
             st.markdown("**Downloads**")
             d1, d2, d3 = st.columns(3)
             d1.download_button(
-                "⬇️ Links CSV",
+                " Links CSV",
                 links_df.drop(columns=["link_id"], errors="ignore").to_csv(index=False).encode()
                 if not links_df.empty else b"",
                 f"{stem}_links.csv", "text/csv", use_container_width=True, disabled=links_df.empty)
             d2.download_button(
-                "⬇️ Augmented CSV",
+                " Augmented CSV",
                 df_aug.drop(columns=["doi_url"], errors="ignore").to_csv(index=False).encode(),
                 f"{stem}_properties_mapped.csv", "text/csv", use_container_width=True)
             if store:
                 d3.download_button(
-                    "⬇️ DB + plots (ZIP)",
+                    " DB + plots (ZIP)",
                     data=bundle_zip(store["out_dir"], store["pdf_stem"], store["db_filename"]),
                     file_name=f"{stem}_db_bundle.zip", mime="application/zip",
                     use_container_width=True)
@@ -1680,4 +1679,6 @@ if __name__ == "__main__":
             "  links_df, df_aug = map_plots_to_properties(df_consensus, plot_results)\n"
             "  store_properties_with_plots(df_aug, links_df, plot_results,\n"
             "                              out_dir='./aim_efrc_db', pdf_stem='paper1')\n"
-        )
+        )
+        
+