Update page_files/categorized/Backend/PDF_DataExtraction.py

page_files/categorized/Backend/PDF_DataExtraction.py

@@ -284,10 +284,14 @@ def _find_value_matches(value_str: str, sentences: List[Dict[str, Any]]) -> List
 
 
 def verify_dataframe(df: pd.DataFrame, sentences: List[Dict[str, Any]]) -> pd.DataFrame:
+    # Always emit the three provenance columns, even on the empty/short-circuit
+    # paths, so downstream schemas can rely on them existing.
     if df.empty or not sentences:
         if not df.empty:
             df = df.copy()
             df["source_verified"] = False
+            df["source_text"] = ""
+            df["source_page"] = ""
         return df
 
     df = df.copy()
@@ -297,7 +301,10 @@ def verify_dataframe(df: pd.DataFrame, sentences: List[Dict[str, Any]]) -> pd.Da
     except Exception as e:
         st.warning(f"Embedding model unavailable, meaning-check skipped: {e}")
 
-    verified = []
+    verified: List[bool] = []
+    source_texts: List[str] = []
+    source_pages: List[Any] = []
+
     for _, row in df.iterrows():
         prop = str(row.get("property_name", "") or "").strip()
         mat  = str(row.get("material_name", "") or "").strip()
@@ -307,24 +314,41 @@ def verify_dataframe(df: pd.DataFrame, sentences: List[Dict[str, Any]]) -> pd.Da
         candidates = _find_value_matches(val, sentences)
         if not candidates:
             verified.append(False)
+            source_texts.append("")
+            source_pages.append("")
             continue
 
-        # STEP 2 — of the sentences containing that value, does any one's
-        # meaning match the claimed property + material?
+        # STEP 2 — of the sentences containing that value, pick the one whose
+        # meaning best matches the claimed property + material, and record it
+        # as this row's source_text / source_page.
         if model is None:
-            verified.append(True)  # value confirmed; meaning check unavailable, don't over-penalize
+            # No embedder: value confirmed, meaning-check unavailable. Record the
+            # first value-matched sentence rather than over-penalize.
+            best = candidates[0]
+            verified.append(True)
+            source_texts.append(best["text"])
+            source_pages.append(best["page"])
             continue
+
         query = f"{prop} {mat}".strip()
         try:
             q_vec = model.encode([query], normalize_embeddings=True)[0]
             texts = [c["text"] for c in candidates]
             c_vecs = model.encode(texts, normalize_embeddings=True)
-            best_score = max(float(np.dot(q_vec, v)) for v in c_vecs)
+            scores = [float(np.dot(q_vec, v)) for v in c_vecs]
+            best_i = int(np.argmax(scores)) if scores else 0
+            best_score = scores[best_i] if scores else 0.0
         except Exception:
-            best_score = 0.0
+            best_i, best_score = 0, 0.0
+
+        best = candidates[best_i]
         verified.append(best_score >= MEANING_THRESHOLD)
+        source_texts.append(best["text"])
+        source_pages.append(best["page"])
 
     df["source_verified"] = verified
+    df["source_text"] = source_texts
+    df["source_page"] = source_pages
     return df
 
 
@@ -355,7 +379,7 @@ def main():
         st.session_state["fv_sig"] = sig
         st.session_state["fv_result"] = None
 
-    if st.button(" Run Extraction", type="primary", use_container_width=True):
+    if st.button("Run Extraction", type="primary", use_container_width=True):
         with st.spinner("Extracting…"):
             data = call_gemini_from_bytes(pdf_bytes)
             doi = resolve_doi(pdf_bytes, data)