Update page_files/categorized/Backend/mapper5.py
Browse files
page_files/categorized/Backend/mapper5.py
CHANGED
|
@@ -651,8 +651,8 @@ def run_full_pipeline(
|
|
| 651 |
"""Run Code 1 (2.py) extraction, Code 2 (image2.py) plots, then map.
|
| 652 |
`source_table` β {consensus, verified, consensus_verified, gemini, gpt, claude}."""
|
| 653 |
import importlib
|
| 654 |
-
run_pipeline = importlib.import_module("
|
| 655 |
-
from
|
| 656 |
|
| 657 |
(df_consensus, df_verified, df_consensus_verified,
|
| 658 |
df_gemini, df_gpt, df_claude,
|
|
@@ -1117,7 +1117,7 @@ def _extract(pdf_bytes: bytes, doi_override: str = "",
|
|
| 1117 |
six property tables + plot_results + coverage + meta + errors."""
|
| 1118 |
import importlib
|
| 1119 |
run_pipeline = importlib.import_module("2").run_pipeline
|
| 1120 |
-
from
|
| 1121 |
(dfc, dfv, dfcv, dfg, dfp, dfcl, _chunks, errors, meta) = run_pipeline(
|
| 1122 |
pdf_bytes, doi_override=doi_override)
|
| 1123 |
plot_results, coverage = extract_and_verify_plots(
|
|
@@ -1135,7 +1135,7 @@ def _extract(pdf_bytes: bytes, doi_override: str = "",
|
|
| 1135 |
def get_or_extract(pdf_bytes: bytes, doi_override: str = "", verify_engines=None,
|
| 1136 |
use_disk_cache: bool = True, force: bool = False):
|
| 1137 |
"""Extraction cached by the PDF's CONTENT HASH β never by source_table.
|
| 1138 |
-
On a cache hit,
|
| 1139 |
pruning / re-mapping is free. Returns (pdf_hash, bundle)."""
|
| 1140 |
import pickle
|
| 1141 |
h = _pdf_hash(pdf_bytes)
|
|
@@ -1194,7 +1194,7 @@ def _run_streamlit() -> None:
|
|
| 1194 |
|
| 1195 |
st.set_page_config(page_title="mapper β Plot β Property", page_icon="π", layout="wide")
|
| 1196 |
st.title("π mapper β Plot β Property mapping + database")
|
| 1197 |
-
st.caption("Runs
|
| 1198 |
"prune what's left, then attaches plots to properties on Map.")
|
| 1199 |
|
| 1200 |
ss = st.session_state
|
|
@@ -1250,7 +1250,7 @@ def _run_streamlit() -> None:
|
|
| 1250 |
use_claude = st.checkbox("Also verify crops with Claude", value=False)
|
| 1251 |
doi_manual = st.text_input("DOI override (optional)", placeholder="10.xxxx/...")
|
| 1252 |
use_disk_cache = st.checkbox("Use on-disk extraction cache", value=True,
|
| 1253 |
-
help="Skip
|
| 1254 |
"PDF was extracted before.")
|
| 1255 |
st.divider()
|
| 1256 |
st.subheader("Auto figure filter")
|
|
@@ -1292,12 +1292,12 @@ def _run_streamlit() -> None:
|
|
| 1292 |
use_container_width=True)
|
| 1293 |
else:
|
| 1294 |
ec1.success("β This paper is loaded β switch tables / prune / map freely; "
|
| 1295 |
-
"
|
| 1296 |
force = ec2.button("β» Force re-extract", use_container_width=True)
|
| 1297 |
|
| 1298 |
if do_extract or force:
|
| 1299 |
engines = ["gemini"] + (["gpt"] if use_gpt else []) + (["claude"] if use_claude else [])
|
| 1300 |
-
with st.spinner("Running
|
| 1301 |
_h, bundle = get_or_extract(pdf_bytes, doi_override=doi_manual,
|
| 1302 |
verify_engines=engines,
|
| 1303 |
use_disk_cache=use_disk_cache, force=force)
|
|
@@ -1314,11 +1314,11 @@ def _run_streamlit() -> None:
|
|
| 1314 |
bundle = ss["bundle"]
|
| 1315 |
stem = ss["stem"]
|
| 1316 |
if bundle.get("_cache") == "disk":
|
| 1317 |
-
st.caption("β© Loaded from on-disk cache β
|
| 1318 |
|
| 1319 |
# ---- three top-level views ---------------------------------------------
|
| 1320 |
view_props, view_plots, work = st.tabs(
|
| 1321 |
-
["π Extraction (2.py)", "πΌοΈ Plots (
|
| 1322 |
|
| 1323 |
with view_props:
|
| 1324 |
if bundle.get("api_errors"):
|
|
|
|
| 651 |
"""Run Code 1 (2.py) extraction, Code 2 (image2.py) plots, then map.
|
| 652 |
`source_table` β {consensus, verified, consensus_verified, gemini, gpt, claude}."""
|
| 653 |
import importlib
|
| 654 |
+
run_pipeline = importlib.import_module("Pdf_DataExtraction").run_pipeline # Code 1 = 2.py
|
| 655 |
+
from Pdf_ImageExtraction import extract_and_verify_plots # Code 2 = image2.py
|
| 656 |
|
| 657 |
(df_consensus, df_verified, df_consensus_verified,
|
| 658 |
df_gemini, df_gpt, df_claude,
|
|
|
|
| 1117 |
six property tables + plot_results + coverage + meta + errors."""
|
| 1118 |
import importlib
|
| 1119 |
run_pipeline = importlib.import_module("2").run_pipeline
|
| 1120 |
+
from Pdf_ImageExtraction import extract_and_verify_plots
|
| 1121 |
(dfc, dfv, dfcv, dfg, dfp, dfcl, _chunks, errors, meta) = run_pipeline(
|
| 1122 |
pdf_bytes, doi_override=doi_override)
|
| 1123 |
plot_results, coverage = extract_and_verify_plots(
|
|
|
|
| 1135 |
def get_or_extract(pdf_bytes: bytes, doi_override: str = "", verify_engines=None,
|
| 1136 |
use_disk_cache: bool = True, force: bool = False):
|
| 1137 |
"""Extraction cached by the PDF's CONTENT HASH β never by source_table.
|
| 1138 |
+
On a cache hit, Pdf_DataExtraction.py and Pdf_ImageExtraction.py are NOT re-run, so switching tables /
|
| 1139 |
pruning / re-mapping is free. Returns (pdf_hash, bundle)."""
|
| 1140 |
import pickle
|
| 1141 |
h = _pdf_hash(pdf_bytes)
|
|
|
|
| 1194 |
|
| 1195 |
st.set_page_config(page_title="mapper β Plot β Property", page_icon="π", layout="wide")
|
| 1196 |
st.title("π mapper β Plot β Property mapping + database")
|
| 1197 |
+
st.caption("Runs Pdf_DataExtraction.py + Pdf_ImageExtraction.py once (cached), auto-drops non-plots, lets you "
|
| 1198 |
"prune what's left, then attaches plots to properties on Map.")
|
| 1199 |
|
| 1200 |
ss = st.session_state
|
|
|
|
| 1250 |
use_claude = st.checkbox("Also verify crops with Claude", value=False)
|
| 1251 |
doi_manual = st.text_input("DOI override (optional)", placeholder="10.xxxx/...")
|
| 1252 |
use_disk_cache = st.checkbox("Use on-disk extraction cache", value=True,
|
| 1253 |
+
help="Skip Pdf_DataExtraction.py + Pdf_ImageExtraction.py entirely when this exact "
|
| 1254 |
"PDF was extracted before.")
|
| 1255 |
st.divider()
|
| 1256 |
st.subheader("Auto figure filter")
|
|
|
|
| 1292 |
use_container_width=True)
|
| 1293 |
else:
|
| 1294 |
ec1.success("β This paper is loaded β switch tables / prune / map freely; "
|
| 1295 |
+
"Pdf_DataExtraction.py and Pdf_ImageExtraction.py will NOT re-run.")
|
| 1296 |
force = ec2.button("β» Force re-extract", use_container_width=True)
|
| 1297 |
|
| 1298 |
if do_extract or force:
|
| 1299 |
engines = ["gemini"] + (["gpt"] if use_gpt else []) + (["claude"] if use_claude else [])
|
| 1300 |
+
with st.spinner("Running Pdf_DataExtraction.py extraction/consensus + Pdf_ImageExtraction.py plot detectionβ¦"):
|
| 1301 |
_h, bundle = get_or_extract(pdf_bytes, doi_override=doi_manual,
|
| 1302 |
verify_engines=engines,
|
| 1303 |
use_disk_cache=use_disk_cache, force=force)
|
|
|
|
| 1314 |
bundle = ss["bundle"]
|
| 1315 |
stem = ss["stem"]
|
| 1316 |
if bundle.get("_cache") == "disk":
|
| 1317 |
+
st.caption("β© Loaded from on-disk cache β Pdf_DataExtraction.py and Pdf_ImageExtraction.py were not re-run.")
|
| 1318 |
|
| 1319 |
# ---- three top-level views ---------------------------------------------
|
| 1320 |
view_props, view_plots, work = st.tabs(
|
| 1321 |
+
["π Extraction (2.py)", "πΌοΈ Plots (Pdf_ImageExtraction.py)", "π Map & curate"])
|
| 1322 |
|
| 1323 |
with view_props:
|
| 1324 |
if bundle.get("api_errors"):
|