Update app.py

app.py

@@ -25,28 +25,12 @@ DEFAULT_CONTROLLED_VOCAB_JSON = """{
 
   "study_type_enum": ["in_vivo","in_vitro","epidemiology","in_silico","review","methodology","other"],
   "in_silico_method_enum": [
-    "qsar",
-    "read_across",
-    "molecular_docking",
-    "molecular_dynamics",
-    "pbpk_pbtK",
-    "aop_based",
-    "ml_model",
-    "other",
-    "not_reported"
+    "qsar","read_across","molecular_docking","molecular_dynamics","pbpk_pbtK","aop_based","ml_model","other","not_reported"
   ],
   "nams_method_enum": [
-    "high_throughput_screening_hts",
-    "omics_transcriptomics",
-    "omics_proteomics",
-    "omics_metabolomics",
-    "organ_on_chip",
-    "microphysiological_system_mps",
-    "3d_tissue_model",
-    "in_chemico_assay",
-    "in_silico_as_nams",
-    "other",
-    "not_reported"
+    "high_throughput_screening_hts","omics_transcriptomics","omics_proteomics","omics_metabolomics",
+    "organ_on_chip","microphysiological_system_mps","3d_tissue_model","in_chemico_assay",
+    "in_silico_as_nams","other","not_reported"
   ],
 
   "exposure_route_enum": ["oral","inhalation","dermal","parenteral","multiple","not_reported"],
@@ -70,22 +54,9 @@ DEFAULT_CONTROLLED_VOCAB_JSON = """{
     "not_reported"
   ],
 
-  "genotoxicity_result_enum": ["positive","negative","equivocal","not_reported"],
-
-  "genotoxicity_result_keywords": {
-    "positive": [
-      "genotoxic","mutagenic","clastogenic","statistically_significant_increase",
-      "significant_increase_in_mutations","induced_dna_damage","dose_dependent_increase"
-    ],
-    "negative": [
-      "non_genotoxic","not_genotoxic","not_mutagenic","no_evidence_of_genotoxicity",
-      "no_statistically_significant_increase","negative_result"
-    ],
-    "equivocal": ["equivocal","inconclusive"]
-  }
+  "genotoxicity_result_enum": ["positive","negative","equivocal","not_reported"]
 }"""
 
-
 DEFAULT_FIELD_SPEC = """# One field per line: Field Name | type | instructions
 # types: str, num, bool, list[str], list[num], enum[a,b,c], list[enum[a,b,c]]
 
@@ -117,7 +88,7 @@ Genotox_OECD_TG_in_vivo | list[enum[
   not_reported
 ]] | If genotoxicity in vivo tests are reported, select all applicable TGs. Otherwise not_reported.
 
-Genotoxicity_result | enum[positive,negative,equivocal,not_reported] | Classify based on reported results language (see genotoxicity_result_keywords in vocab).
+Genotoxicity_result | enum[positive,negative,equivocal,not_reported] | Classify based on reported results. If unclear, not_reported.
 Genotoxicity_result_notes | str | Short explanation grounded to the paper’s wording + what test context it applies to.
 
 Dose_metrics | list[str] | Include any reported NOAEL/LOAEL/BMD/BMDL/LD50/LC50 etc with units if available.
@@ -126,6 +97,39 @@ Conclusion | str | What does the paper conclude about safety/risk?
 """
 
 
+# =============================
+# Field presets (UI)
+# =============================
+PRESET_CORE = [
+    {"field": "Chemical(s)", "type": "list[str]", "enum_values": "", "instructions": "Primary chemical(s) studied; include common name + abbreviation if present."},
+    {"field": "CAS_numbers", "type": "list[str]", "enum_values": "", "instructions": "Extract any CAS numbers mentioned."},
+    {"field": "Study_type", "type": "enum", "enum_values": "in_vivo,in_vitro,epidemiology,in_silico,review,methodology,other", "instructions": "Choose the best match."},
+    {"field": "Exposure_route", "type": "enum", "enum_values": "oral,inhalation,dermal,parenteral,multiple,not_reported", "instructions": "Choose best match."},
+    {"field": "Species", "type": "enum", "enum_values": "human,rat,mouse,rabbit,dog,non_human_primate,cell_line,other,not_reported", "instructions": "Choose best match."},
+    {"field": "Dose_metrics", "type": "list[str]", "enum_values": "", "instructions": "Include any reported NOAEL/LOAEL/BMD/BMDL/LD50/LC50 etc with units if available."},
+    {"field": "Key_findings", "type": "str", "enum_values": "", "instructions": "2-4 bullet-like sentences summarizing the main findings."},
+    {"field": "Conclusion", "type": "str", "enum_values": "", "instructions": "What does the paper conclude about safety/risk?"},
+]
+
+PRESET_NAMS_INSILICO = [
+    {"field": "Approach", "type": "enum", "enum_values": "in_vivo,in_vitro,in_silico,nams,mixed,not_reported", "instructions": "Identify if results are in silico or NAMs; use 'mixed' if multiple."},
+    {"field": "In_silico_methods", "type": "list[enum]", "enum_values": "qsar,read_across,molecular_docking,molecular_dynamics,pbpk_pbtK,aop_based,ml_model,other,not_reported", "instructions": "If in_silico, list methods used (can be multiple)."},
+    {"field": "NAMs_methods", "type": "list[enum]", "enum_values": "high_throughput_screening_hts,omics_transcriptomics,omics_proteomics,omics_metabolomics,organ_on_chip,microphysiological_system_mps,3d_tissue_model,in_chemico_assay,in_silico_as_nams,other,not_reported", "instructions": "If NAMs, list methods used (can be multiple)."},
+]
+
+PRESET_GENOTOX_OECD = [
+    {"field": "Genotox_OECD_TG_in_vitro", "type": "list[enum]", "enum_values": "OECD_TG_471_Bacterial Reverse mutation test(AMES test),OECD_TG_473_In Vitro Mammalian Chromosomal Aberration Test,OECD_TG_476_In Vitro Mammalian Cell Gene Mutation Tests (Hprt & xprt),OECD_TG_487_In Vitro Mammalian Cell Micronucleus Test,OECD_TG_490_In Vitro Mammalian Cell Gene Mutation Tests (Thymidine Kinase),not_reported", "instructions": "If in vitro genotox tests are reported, select TGs. Otherwise not_reported."},
+    {"field": "Genotox_OECD_TG_in_vivo", "type": "list[enum]", "enum_values": "OECD_TG_474_In Vivo Mammalian Erythrocyte Micronucleus Test,OECD_TG_475_Mammalian Bone Marrow Chromosomal Aberration Test,OECD_TG_488_Transgenic Rodent Somatic & Germ Cell Gene Mutation Assays,OECD_TG_489_In Vivo Mammalian Alkaline Comet Assay,not_reported", "instructions": "If in vivo genotox tests are reported, select TGs. Otherwise not_reported."},
+    {"field": "Genotoxicity_result", "type": "enum", "enum_values": "positive,negative,equivocal,not_reported", "instructions": "Classify based on reported results. If unclear, not_reported."},
+    {"field": "Genotoxicity_result_notes", "type": "str", "enum_values": "", "instructions": "Short explanation grounded to the paper’s wording + test context."},
+]
+
+PRESET_MAP = {
+    "Core (recommended)": PRESET_CORE,
+    "NAMs + In Silico": PRESET_NAMS_INSILICO,
+    "Genotox (OECD TGs)": PRESET_GENOTOX_OECD,
+}
+
 
 # =============================
 # PDF extraction (text-based PDFs only)
@@ -183,8 +187,13 @@ def chunk_pages(pages: List[Tuple[int, str]], target_chars: int = 3000) -> List[
     return chunks
 
 
+def _text_based_pdf_warning(pages: List[Tuple[int, str]]) -> bool:
+    joined = " ".join([clean_text(t) for _, t in pages if clean_text(t)])
+    return len(joined.strip()) < 200
+
+
 # =============================
-# Lightweight retrieval (TF-IDF) to select relevant excerpts
+# Lightweight retrieval (TF-IDF)
 # =============================
 def select_relevant_chunks(
     chunks: List[Dict[str, Any]],
@@ -230,7 +239,7 @@ def build_context(selected_chunks: List[Dict[str, Any]], max_chars: int = 20000)
 
 
 # =============================
-# User-defined extraction spec -> JSON Schema
+# Spec -> JSON schema
 # =============================
 def slugify_field(name: str) -> str:
     name = name.strip()
@@ -242,7 +251,7 @@ def slugify_field(name: str) -> str:
 def parse_field_spec(spec: str) -> Tuple[Dict[str, Any], Dict[str, str]]:
     """
     spec lines: Field Name | type | instructions
-    Returns: properties dict, instructions map (field_key -> instruction)
+    types: str, num, bool, list[str], list[num], enum[a,b,c], list[enum[a,b,c]]
     """
     props: Dict[str, Any] = {}
     instr: Dict[str, str] = {}
@@ -292,18 +301,10 @@ def parse_field_spec(spec: str) -> Tuple[Dict[str, Any], Dict[str, str]]:
 
 
 def build_extraction_schema(field_props: Dict[str, Any], vocab: Dict[str, Any]) -> Dict[str, Any]:
-    """
-    IMPORTANT: Structured Outputs (strict=True) requires that for every object:
-      required must exist and include every key in properties.
-    """
-    risk_enum = vocab.get(
-        "risk_stance_enum",
-        ["acceptable", "acceptable_with_uncertainty", "not_acceptable", "insufficient_data"]
-    )
-
+    risk_enum = vocab.get("risk_stance_enum", ["acceptable","acceptable_with_uncertainty","not_acceptable","insufficient_data"])
     all_field_keys = list(field_props.keys())
 
-    schema = {
+    return {
         "type": "object",
         "additionalProperties": False,
         "properties": {
@@ -331,9 +332,8 @@ def build_extraction_schema(field_props: Dict[str, Any], vocab: Dict[str, Any])
                 }
             }
         },
-        "required": ["paper_title", "risk_stance", "risk_confidence", "risk_summary", "extracted", "evidence"]
+        "required": ["paper_title","risk_stance","risk_confidence","risk_summary","extracted","evidence"]
     }
-    return schema
 
 
 # =============================
@@ -354,10 +354,7 @@ def openai_structured_extract(
     field_instructions: Dict[str, str],
     context: str
 ) -> Dict[str, Any]:
-    field_instr_lines = []
-    for k, v in field_instructions.items():
-        field_instr_lines.append(f"- {k}: {v if v else '(no extra instructions)'}")
-
+    field_instr_lines = [f"- {k}: {v if v else '(no extra instructions)'}" for k, v in field_instructions.items()]
     vocab_text = json.dumps(controlled_vocab, indent=2)
 
     system_msg = (
@@ -368,10 +365,8 @@ def openai_structured_extract(
         "3) Provide evidence quotes + page ranges for extracted fields.\n"
         "4) risk_stance is regulatory: acceptable / acceptable_with_uncertainty / not_acceptable / insufficient_data.\n"
         "5) Prefer controlled vocab terms when applicable.\n"
-        "6) For Genotoxicity_result, use genotoxicity_result_keywords (positive/negative/equivocal) as guidance, but ONLY if the paper explicitly reports results.\n"
-        "7) For OECD TG fields, only populate if the TG is explicitly stated or clearly described; otherwise use not_reported.\n"
-        "8) For NAMs/in_silico fields, only populate if methods are explicitly described; otherwise not_reported.\n"
-
+        "6) For OECD TG fields, only populate if explicitly stated or clearly described; otherwise use not_reported.\n"
+        "7) For NAMs/in_silico fields, only populate if explicitly described; otherwise not_reported.\n"
     )
 
     user_msg = (
@@ -409,19 +404,12 @@ def openai_synthesize_across_papers(client: OpenAI, model: str, rows: List[Dict[
         "Base strictly on the provided extracted JSON (which is evidence-backed).\n"
     )
     user_msg = "EXTRACTED_ROWS_JSON:\n" + json.dumps(rows, indent=2)
-
-    resp = client.responses.create(
-        model=model,
-        input=[
-            {"role": "system", "content": system_msg},
-            {"role": "user", "content": user_msg}
-        ],
-    )
+    resp = client.responses.create(model=model, input=[{"role":"system","content":system_msg},{"role":"user","content":user_msg}])
     return resp.output_text
 
 
 # =============================
-# Grounding helpers (UI)
+# UI helpers: vertical view + evidence
 # =============================
 def _make_vertical(records: List[Dict[str, Any]], file_name: str) -> pd.DataFrame:
     if not records or not file_name:
@@ -451,17 +439,282 @@ def _render_evidence(details: List[Dict[str, Any]], file_name: str, max_items: i
     header = "### Evidence (grounding)\n"
     if not lines:
         lines = ["- (no evidence returned)"]
-    return header + "\n".join(lines) + "\n\n> Review note: evidence reflects the original extraction. If you change values, re-run extraction to refresh evidence."
+    return header + "\n".join(lines)
 
 
-def _text_based_pdf_warning(pages: List[Tuple[int, str]]) -> bool:
-    # If almost no text exists across pages, treat as non-text PDF.
-    joined = " ".join([clean_text(t) for _, t in pages if clean_text(t)])
-    return len(joined.strip()) < 200  # heuristic threshold
+def _overview_df_from_records(records: List[Dict[str, Any]]) -> pd.DataFrame:
+    if not records:
+        return pd.DataFrame(columns=["file","paper_title","risk_stance","risk_confidence"])
+    df = pd.DataFrame(records)
+    cols = ["file","paper_title","risk_stance","risk_confidence"]
+    # Include chemicals if present
+    for c in ["chemicals", "chemical_s", "chemical", "chemical_s_"]:
+        if c in df.columns and c not in cols:
+            cols.append(c)
+            break
+    cols = [c for c in cols if c in df.columns]
+    return df[cols].copy() if cols else df.head(50)
+
+def _filter_terms_df(df: pd.DataFrame, query: str) -> pd.DataFrame:
+    if df is None or df.empty:
+        return pd.DataFrame(columns=["term"])
+    q = (query or "").strip().lower()
+    if not q:
+        return df[["term"]].copy()
+    mask = df["term"].astype(str).str.lower().str.contains(q, na=False)
+    out = df.loc[mask, ["term"]].copy()
+    return out
+
+# =============================
+# Controlled vocab guided editor (lists only)
+# =============================
+vocab_search = gr.Textbox(label="Search terms", placeholder="Type to filter (e.g., 471, AMES, comet)", lines=1)
+
+vocab_terms_filtered = gr.Dataframe(
+    headers=["term"],
+    label="Filtered preview (read-only)",
+    interactive=False,
+    wrap=True
+)
+
+def vocab_init_state(vocab_json: str):
+    try:
+        vocab = json.loads(vocab_json or DEFAULT_CONTROLLED_VOCAB_JSON)
+    except Exception:
+        vocab = json.loads(DEFAULT_CONTROLLED_VOCAB_JSON)
+
+    list_keys = sorted([k for k, v in vocab.items() if isinstance(v, list)])
+    default_key = list_keys[0] if list_keys else None
+    terms = vocab.get(default_key, []) if default_key else []
+    terms_df = pd.DataFrame({"term": terms})
+    return vocab, list_keys, default_key, terms_df, json.dumps(vocab, indent=2), "✅ Vocab loaded."
+
+
+def vocab_load_category(vocab_state: Dict[str, Any], category: str, search: str):
+    if not category or category not in vocab_state:
+        empty = pd.DataFrame(columns=["term"])
+        return empty, empty, "Select a category."
+    terms = vocab_state.get(category, [])
+    if not isinstance(terms, list):
+        empty = pd.DataFrame(columns=["term"])
+        return empty, empty, "This category is not a list."
+    full = pd.DataFrame({"term": terms})
+    filtered = _filter_terms_df(full, search)
+    return full, filtered, f"Editing: {category}"
+
+
+def vocab_add_term(vocab_state: Dict[str, Any], category: str, term: str, search: str):
+    term = (term or "").strip()
+    if not term:
+        return gr.update(), gr.update(), "", "Enter a term to add."
+    if not category or category not in vocab_state or not isinstance(vocab_state.get(category), list):
+        return gr.update(), gr.update(), "", "Pick a list category first."
+
+    if term not in vocab_state[category]:
+        vocab_state[category].append(term)
+
+    full = pd.DataFrame({"term": vocab_state[category]})
+    filtered = _filter_terms_df(full, search)
+    return full, filtered, "", f"Added: {term}"
+
+
+def vocab_remove_term(vocab_state: Dict[str, Any], category: str, term: str, search: str):
+    term = (term or "").strip()
+    if not term:
+        return gr.update(), gr.update(), "", "Enter a term to remove."
+    if not category or category not in vocab_state or not isinstance(vocab_state.get(category), list):
+        return gr.update(), gr.update(), "", "Pick a list category first."
+
+    vocab_state[category] = [t for t in vocab_state[category] if t != term]
+    full = pd.DataFrame({"term": vocab_state[category]})
+    filtered = _filter_terms_df(full, search)
+    return full, filtered, "", f"Removed: {term}"
+
+
+def vocab_apply_df(vocab_state: Dict[str, Any], category: str, terms_df: Any, search: str):
+    if not category or category not in vocab_state or not isinstance(vocab_state.get(category), list):
+        return json.dumps(vocab_state, indent=2), pd.DataFrame(columns=["term"]), "Pick a list category first."
+
+    try:
+        if isinstance(terms_df, pd.DataFrame):
+            df = terms_df
+        else:
+            df = pd.DataFrame(terms_df, columns=["term"])
+    except Exception:
+        vjson = json.dumps(vocab_state, indent=2)
+        return vjson, pd.DataFrame(columns=["term"]), "Could not parse terms table."
+
+    terms = []
+    for t in df.get("term", []).tolist():
+        t = (str(t) if t is not None else "").strip()
+        if t and t not in terms:
+            terms.append(t)
+
+    vocab_state[category] = terms
+    vjson = json.dumps(vocab_state, indent=2)
+    filtered = _filter_terms_df(pd.DataFrame({"term": terms}), search)
+    return vjson, filtered, f"✅ Applied {len(terms)} terms to {category}."
+
+
+def vocab_reset_defaults():
+    return vocab_init_state(DEFAULT_CONTROLLED_VOCAB_JSON)
+
+def vocab_filter_preview(terms_df, search):
+    try:
+        df = terms_df if isinstance(terms_df, pd.DataFrame) else pd.DataFrame(terms_df, columns=["term"])
+    except Exception:
+        df = pd.DataFrame(columns=["term"])
+    return _filter_terms_df(df, search)
+
+vocab_search.change(
+    fn=vocab_filter_preview,
+    inputs=[vocab_terms_df, vocab_search],
+    outputs=[vocab_terms_filtered]
+)
+
+
+# =============================
+# Field builder (type dropdown + presets)
+# =============================
+TYPE_CHOICES = ["str", "num", "bool", "list[str]", "list[num]", "enum", "list[enum]"]
+
+def fields_init_state():
+    # start from DEFAULT_FIELD_SPEC by showing a friendly default builder (Core + Genotox + NAMs)
+    fields = []
+    for row in (PRESET_CORE + PRESET_NAMS_INSILICO + PRESET_GENOTOX_OECD):
+        fields.append(dict(row))
+    df = pd.DataFrame(fields, columns=["field","type","enum_values","instructions"])
+    spec = build_spec_from_field_df(df)
+    return fields, df, spec, "✅ Field builder loaded."
+
+def build_spec_from_field_df(df: pd.DataFrame) -> str:
+    lines = [
+        "# One field per line: Field Name | type | instructions",
+        "# types: str, num, bool, list[str], list[num], enum[a,b,c], list[enum[a,b,c]]",
+        ""
+    ]
+    for _, r in df.iterrows():
+        field = str(r.get("field","")).strip()
+        ftype = str(r.get("type","")).strip()
+        enums = str(r.get("enum_values","")).strip()
+        instr = str(r.get("instructions","")).strip()
+
+        if not field or not ftype:
+            continue
+
+        # normalize types
+        if ftype not in TYPE_CHOICES:
+            # keep as-is, but likely invalid; user can fix
+            pass
+
+        if ftype == "enum":
+            vals = [v.strip() for v in enums.split(",") if v.strip()]
+            type_str = f"enum[{','.join(vals)}]" if vals else "str"
+        elif ftype == "list[enum]":
+            vals = [v.strip() for v in enums.split(",") if v.strip()]
+            type_str = f"list[enum[{','.join(vals)}]]" if vals else "list[str]"
+        else:
+            type_str = ftype
+
+        lines.append(f"{field} | {type_str} | {instr}")
+
+    return "\n".join(lines).strip() + "\n"
+
+def fields_add_or_update(field_name: str, ftype: str, enum_values: str, instructions: str, field_rows: List[Dict[str, Any]]):
+    field_name = (field_name or "").strip()
+    ftype = (ftype or "").strip()
+    enum_values = (enum_values or "").strip()
+    instructions = (instructions or "").strip()
+
+    if not field_name or not ftype:
+        df = pd.DataFrame(field_rows, columns=["field","type","enum_values","instructions"])
+        return field_rows, df, build_spec_from_field_df(df), "Field name and type are required."
+
+    # update if exists
+    updated = False
+    for r in field_rows:
+        if str(r.get("field","")).strip().lower() == field_name.lower():
+            r["type"] = ftype
+            r["enum_values"] = enum_values
+            r["instructions"] = instructions
+            updated = True
+            break
+
+    if not updated:
+        field_rows.append({"field": field_name, "type": ftype, "enum_values": enum_values, "instructions": instructions})
+
+    df = pd.DataFrame(field_rows, columns=["field","type","enum_values","instructions"])
+    spec = build_spec_from_field_df(df)
+    return field_rows, df, spec, ("Updated field." if updated else "Added field.")
+
+def fields_remove(field_to_remove: str, field_rows: List[Dict[str, Any]]):
+    key = (field_to_remove or "").strip().lower()
+    if not key:
+        df = pd.DataFrame(field_rows, columns=["field","type","enum_values","instructions"])
+        return field_rows, df, build_spec_from_field_df(df), "Pick a field to remove."
+    field_rows = [r for r in field_rows if str(r.get("field","")).strip().lower() != key]
+    df = pd.DataFrame(field_rows, columns=["field","type","enum_values","instructions"])
+    spec = build_spec_from_field_df(df)
+    return field_rows, df, spec, "Removed."
+
+def fields_apply_df(field_rows: List[Dict[str, Any]], df_in: Any):
+    try:
+        if isinstance(df_in, pd.DataFrame):
+            df = df_in
+        else:
+            df = pd.DataFrame(df_in, columns=["field","type","enum_values","instructions"])
+    except Exception:
+        df = pd.DataFrame(field_rows, columns=["field","type","enum_values","instructions"])
+        return field_rows, df, build_spec_from_field_df(df), "Could not parse builder table."
+
+    # clean + rebuild list of dicts
+    cleaned = []
+    seen = set()
+    for _, r in df.iterrows():
+        field = str(r.get("field","")).strip()
+        ftype = str(r.get("type","")).strip()
+        enums = str(r.get("enum_values","")).strip()
+        instr = str(r.get("instructions","")).strip()
+        if not field or not ftype:
+            continue
+        k = field.lower()
+        if k in seen:
+            continue
+        seen.add(k)
+        cleaned.append({"field": field, "type": ftype, "enum_values": enums, "instructions": instr})
+
+    df2 = pd.DataFrame(cleaned, columns=["field","type","enum_values","instructions"])
+    spec = build_spec_from_field_df(df2)
+    return cleaned, df2, spec, f"✅ Applied builder table ({len(cleaned)} fields)."
+
+def fields_load_preset(preset_name: str, mode: str, field_rows: List[Dict[str, Any]]):
+    preset = PRESET_MAP.get(preset_name)
+    if not preset:
+        df = pd.DataFrame(field_rows, columns=["field","type","enum_values","instructions"])
+        return field_rows, df, build_spec_from_field_df(df), "Unknown preset."
+
+    if mode == "Replace":
+        new_rows = [dict(r) for r in preset]
+    else:
+        # Append (update existing fields if same name)
+        new_rows = [dict(r) for r in field_rows]
+        for p in preset:
+            found = False
+            for r in new_rows:
+                if str(r.get("field","")).strip().lower() == str(p.get("field","")).strip().lower():
+                    r.update(p)
+                    found = True
+                    break
+            if not found:
+                new_rows.append(dict(p))
+
+    df = pd.DataFrame(new_rows, columns=["field","type","enum_values","instructions"])
+    spec = build_spec_from_field_df(df)
+    return new_rows, df, spec, f"✅ Loaded preset: {preset_name} ({mode})."
 
 
 # =============================
-# Main extraction handler
+# Extraction handler
 # =============================
 def run_extraction(
     files,
@@ -474,26 +727,23 @@ def run_extraction(
     max_context_chars
 ):
     if not files:
-        return None, None, None, "Upload one or more PDFs.", gr.update(choices=[], value=None), [], [], pd.DataFrame(columns=["Field","Value"]), ""
+        return pd.DataFrame(), None, None, "Upload one or more PDFs.", gr.update(choices=[], value=None), [], [], pd.DataFrame(columns=["Field","Value"]), ""
 
-    # vocab
     try:
         vocab = json.loads(vocab_json or DEFAULT_CONTROLLED_VOCAB_JSON)
     except Exception as e:
-        return None, None, None, f"Controlled vocab JSON is invalid: {e}", gr.update(choices=[], value=None), [], [], pd.DataFrame(columns=["Field","Value"]), ""
+        return pd.DataFrame(), None, None, f"Controlled vocab JSON invalid: {e}", gr.update(choices=[], value=None), [], [], pd.DataFrame(columns=["Field","Value"]), ""
 
-    # field spec
     field_props, field_instr = parse_field_spec(field_spec or DEFAULT_FIELD_SPEC)
     if not field_props:
-        return None, None, None, "Field spec produced no fields. Add lines like: Field | str | instructions", gr.update(choices=[], value=None), [], [], pd.DataFrame(columns=["Field","Value"]), ""
+        return pd.DataFrame(), None, None, "Extraction spec produced no fields.", gr.update(choices=[], value=None), [], [], pd.DataFrame(columns=["Field","Value"]), ""
 
     schema = build_extraction_schema(field_props, vocab)
 
-    # OpenAI
     try:
         client = get_openai_client(api_key)
     except Exception as e:
-        return None, None, None, str(e), gr.update(choices=[], value=None), [], [], pd.DataFrame(columns=["Field","Value"]), ""
+        return pd.DataFrame(), None, None, str(e), gr.update(choices=[], value=None), [], [], pd.DataFrame(columns=["Field","Value"]), ""
 
     results: List[Dict[str, Any]] = []
     flat_rows: List[Dict[str, Any]] = []
@@ -506,14 +756,7 @@ def run_extraction(
 
         pages, page_count = extract_pages_from_pdf(pdf_path, max_pages=int(max_pages))
 
-        # enforce text-based PDFs note
         if _text_based_pdf_warning(pages):
-            # create an "empty" record with warning
-            row = {"file": filename, "paper_title": "", "risk_stance": "insufficient_data", "risk_confidence": 0.0, "risk_summary": "No extractable text found. This app supports text-based PDFs only."}
-            for k, sch in field_props.items():
-                row[k] = "" if sch.get("type") != "array" else ""
-            flat_rows.append(row)
-
             results.append({
                 "_file": filename,
                 "_pages_in_pdf": page_count,
@@ -524,67 +767,64 @@ def run_extraction(
                 "extracted": {k: ([] if field_props[k].get("type") == "array" else "") for k in field_props.keys()},
                 "evidence": []
             })
-            continue
-
-        chunks = chunk_pages(pages, target_chars=int(chunk_chars))
-
-        # Queries: risk stance + each field instruction (or field key)
-        queries = ["regulatory acceptability risk hazard concern conclusion noael loael bmd bmdl adverse effect uncertainty"]
-        for k, ins in field_instr.items():
-            queries.append(ins if ins else k)
-
-        selected = select_relevant_chunks(chunks, queries, top_per_query=2, max_chunks=12)
-        context = build_context(selected, max_chars=int(max_context_chars))
-
-        extracted = openai_structured_extract(
-            client=client,
-            model=model,
-            schema=schema,
-            controlled_vocab=vocab,
-            field_instructions=field_instr,
-            context=context
-        )
-
-        extracted["_file"] = filename
-        extracted["_pages_in_pdf"] = page_count
-        results.append(extracted)
+        else:
+            chunks = chunk_pages(pages, target_chars=int(chunk_chars))
+
+            queries = ["regulatory acceptability risk hazard concern conclusion adverse effect uncertainty noael loael bmd bmdl"]
+            for k, ins in field_instr.items():
+                queries.append(ins if ins else k)
+
+            selected = select_relevant_chunks(chunks, queries, top_per_query=2, max_chunks=12)
+            context = build_context(selected, max_chars=int(max_context_chars))
+
+            extracted = openai_structured_extract(
+                client=client,
+                model=model,
+                schema=schema,
+                controlled_vocab=vocab,
+                field_instructions=field_instr,
+                context=context
+            )
+            extracted["_file"] = filename
+            extracted["_pages_in_pdf"] = page_count
+            results.append(extracted)
 
-        # flatten to table (wide)
+        # flatten to internal records for vertical view + review/export
+        ex = results[-1]
         row = {
             "file": filename,
-            "paper_title": extracted.get("paper_title", ""),
-            "risk_stance": extracted.get("risk_stance", ""),
-            "risk_confidence": extracted.get("risk_confidence", ""),
-            "risk_summary": extracted.get("risk_summary", "")
+            "paper_title": ex.get("paper_title",""),
+            "risk_stance": ex.get("risk_stance",""),
+            "risk_confidence": ex.get("risk_confidence",""),
+            "risk_summary": ex.get("risk_summary","")
         }
-
-        ext = extracted.get("extracted") or {}
+        ext = ex.get("extracted") or {}
         for k in field_props.keys():
             v = ext.get(k, "" if field_props[k].get("type") != "array" else [])
             if isinstance(v, list):
                 row[k] = "; ".join([str(x) for x in v])
             else:
                 row[k] = v
-
         flat_rows.append(row)
 
     df = pd.DataFrame(flat_rows)
+    records = df.to_dict("records")
 
     csv_path = tmpdir / "extraction_table.csv"
     json_path = tmpdir / "extraction_details.json"
     df.to_csv(csv_path, index=False)
     json_path.write_text(json.dumps(results, indent=2), encoding="utf-8")
 
-    records = df.to_dict("records")
     choices = [r["file"] for r in records if "file" in r]
     default = choices[0] if choices else None
     vertical = _make_vertical(records, default)
     evidence = _render_evidence(results, default)
 
-    status = "Done. Use the vertical view to read cleanly. Enable Review Mode to edit and export a reviewed CSV."
+    overview = _overview_df_from_records(records)
 
+    status = "Done. Use the vertical view + evidence for review. Export reviewed CSV when ready."
     return (
-        df,
+        overview,
         str(csv_path),
         str(json_path),
         status,
@@ -602,34 +842,24 @@ def run_extraction(
 def on_pick(file_name: str, records: List[Dict[str, Any]], details: List[Dict[str, Any]]):
     return _make_vertical(records, file_name), _render_evidence(details, file_name)
 
-
 def toggle_review_mode(is_on: bool):
-    # make vertical table editable when review mode is on
     return gr.update(interactive=bool(is_on))
 
-
 def save_review_changes(file_name: str, vertical_df: Any, records: List[Dict[str, Any]]):
-    """
-    vertical_df comes from gr.Dataframe: typically list[list] or pandas df-like.
-    Expect two columns: Field, Value
-    """
     if not file_name or not records:
-        return None, records, "Nothing to save."
+        return pd.DataFrame(), records, "Nothing to save."
 
-    # Convert vertical_df into dict
     try:
         if isinstance(vertical_df, pd.DataFrame):
             dfv = vertical_df
         else:
-            # gradio may pass list-of-lists
             dfv = pd.DataFrame(vertical_df, columns=["Field", "Value"])
     except Exception:
-        return None, records, "Could not parse edited vertical table."
+        return _overview_df_from_records(records), records, "Could not parse edited vertical table."
 
     dfv = dfv.dropna(subset=["Field"])
     updates = {str(r["Field"]): r["Value"] for _, r in dfv.iterrows() if str(r["Field"]).strip()}
 
-    # Update matching record
     new_records = []
     updated = False
     for r in records:
@@ -642,10 +872,8 @@ def save_review_changes(file_name: str, vertical_df: Any, records: List[Dict[str
         else:
             new_records.append(r)
 
-    df_wide = pd.DataFrame(new_records) if new_records else pd.DataFrame()
-    msg = "Saved changes into session table. Export reviewed CSV to download." if updated else "Record not found."
-    return df_wide, new_records, msg
-
+    msg = "Saved changes into session data. Export reviewed CSV to download." if updated else "Record not found."
+    return _overview_df_from_records(new_records), new_records, msg
 
 def export_reviewed_csv(records: List[Dict[str, Any]]):
     if not records:
@@ -657,11 +885,11 @@ def export_reviewed_csv(records: List[Dict[str, Any]]):
 
 
 # =============================
-# Synthesis tab handler
+# Synthesis
 # =============================
 def run_synthesis(api_key, model, extraction_json_file):
     if extraction_json_file is None:
-        return "Upload the extraction_details.json produced by the Extract tab first."
+        return "Upload the extraction_details.json from the Extract tab first."
 
     try:
         client = get_openai_client(api_key)
@@ -675,46 +903,92 @@ def run_synthesis(api_key, model, extraction_json_file):
 # =============================
 # Gradio UI
 # =============================
-with gr.Blocks(title="Toxicology PDF → Grounded Table Extractor") as demo:
+with gr.Blocks(title="Toxicology PDF → Grounded Extractor") as demo:
     gr.Markdown(
-        "# Toxicology PDF → Grounded Table Extractor (GPT-4o)\n\n"
-        "**Important:** This app supports **text-based PDFs only** (not scanned/image PDFs). If a PDF has no extractable text, it will be flagged as insufficient_data.\n\n"
-        "You control *what* to extract using the **Extraction spec**. Outputs are grounded by evidence quotes + page ranges."
+        "# Toxicology PDF → Grounded Extractor (GPT-4o)\n\n"
+        "**Important:** Text-based PDFs only (not scanned/image PDFs). If no extractable text is found, the record is marked `insufficient_data`.\n\n"
+        "This UI is optimized for non-JSON users: **Controlled vocab editor** + **Field Builder**.\n"
+        "Raw JSON/spec are available under **Advanced**."
     )
 
-    # State stores for review mode
-    state_records = gr.State([])   # wide table rows: list[dict]
-    state_details = gr.State([])   # extraction details JSON: list[dict]
+    # State
+    state_records = gr.State([])    # list[dict]
+    state_details = gr.State([])    # list[dict]
+    vocab_state = gr.State({})      # dict
+    field_rows_state = gr.State([]) # list[dict]
 
-    with gr.Tab("Extract to Table"):
-        files = gr.File(label="Upload toxicology research PDFs", file_types=[".pdf"], file_count="multiple")
+    with gr.Tab("Extract"):
+        files = gr.File(label="Upload toxicology PDFs", file_types=[".pdf"], file_count="multiple")
 
         with gr.Row():
             api_key = gr.Textbox(label="OpenAI API key (optional if set as OPENAI_API_KEY secret)", type="password")
-            model = gr.Dropdown(
-                label="Model",
-                choices=["gpt-4o-2024-08-06", "gpt-4o", "gpt-4o-mini"],
-                value="gpt-4o-2024-08-06"
-            )
+            model = gr.Dropdown(label="Model", choices=["gpt-4o-2024-08-06", "gpt-4o", "gpt-4o-mini"], value="gpt-4o-2024-08-06")
 
         with gr.Row():
             max_pages = gr.Slider(0, 250, value=0, step=1, label="Max pages to read (0 = all)")
             chunk_chars = gr.Slider(1200, 9000, value=3200, step=100, label="Chunk size (chars)")
             max_context_chars = gr.Slider(5000, 45000, value=20000, step=1000, label="Max context sent to GPT (chars)")
 
-        vocab_json = gr.Textbox(label="Controlled vocabulary (JSON)", value=DEFAULT_CONTROLLED_VOCAB_JSON, lines=10)
-        field_spec = gr.Textbox(label="Extraction spec (you control the columns)", value=DEFAULT_FIELD_SPEC, lines=10)
+        gr.Markdown("## Controlled Vocabulary (guided editor)")
+        vocab_mode = gr.Radio(choices=["Guided", "Advanced (Raw JSON)"], value="Guided", label="Vocab editor mode")
+
+        with gr.Row():
+            vocab_category = gr.Dropdown(label="Category (lists only)", choices=[], value=None)
+            vocab_term_add = gr.Textbox(label="Add term", placeholder="type term and click Add")
+            vocab_add_btn = gr.Button("Add")
+        with gr.Row():
+            vocab_term_remove = gr.Textbox(label="Remove term", placeholder="type exact term and click Remove")
+            vocab_remove_btn = gr.Button("Remove")
+            vocab_apply_btn = gr.Button("Apply table changes to category")
+            vocab_reset_btn = gr.Button("Reset vocab to defaults")
+
+        vocab_terms_df = gr.Dataframe(headers=["term"], label="Terms (edit directly)", interactive=True, wrap=True)
+        vocab_status = gr.Textbox(label="Vocab status", interactive=False)
+
+        with gr.Accordion("Advanced: Raw vocab JSON (auto-generated)", open=False):
+            vocab_json = gr.Textbox(label="Controlled vocab JSON", lines=12, interactive=False)
+
+        gr.Markdown("## Extraction Spec (Field Builder)")
+        with gr.Row():
+            preset_name = gr.Dropdown(label="Preset", choices=list(PRESET_MAP.keys()), value="Core (recommended)")
+            preset_mode = gr.Radio(label="Preset mode", choices=["Replace", "Append"], value="Append")
+            preset_btn = gr.Button("Load preset")
+
+        with gr.Row():
+            field_name_in = gr.Textbox(label="Field name", placeholder="e.g., Genotoxicity_result")
+            field_type_in = gr.Dropdown(label="Type", choices=TYPE_CHOICES, value="str")
+        enum_values_in = gr.Textbox(label="Enum values (comma-separated; used for enum/list[enum])", placeholder="a,b,c", lines=2)
+        instructions_in = gr.Textbox(label="Instructions", placeholder="Tell the extractor exactly what to pull.", lines=2)
+
+        with gr.Row():
+            add_update_field_btn = gr.Button("Add/Update field")
+            remove_field_name = gr.Dropdown(label="Remove field", choices=[], value=None)
+            remove_field_btn = gr.Button("Remove")
+
+        fields_df = gr.Dataframe(
+            label="Fields (edit if needed, then click Apply)",
+            headers=["field","type","enum_values","instructions"],
+            interactive=True,
+            wrap=True
+        )
+        fields_apply_btn = gr.Button("Apply builder table")
+        fields_status = gr.Textbox(label="Field builder status", interactive=False)
+
+        with gr.Accordion("Advanced: Raw extraction spec (auto-generated)", open=False):
+            field_spec = gr.Textbox(label="Extraction spec", lines=12, interactive=False)
 
         extract_btn = gr.Button("Run Extraction (Grounded)")
         status = gr.Textbox(label="Status", interactive=False)
 
-        table = gr.Dataframe(
-            label="Wide Table (download-friendly)",
+        # Replace wide table with a compact overview (not duplicate)
+        overview_df = gr.Dataframe(
+            label="Batch Overview (compact)",
             interactive=False,
             wrap=True,
             show_row_numbers=True,
             buttons=["fullscreen", "copy"]
         )
+
         with gr.Row():
             out_csv = gr.File(label="Download: extraction_table.csv")
             out_json = gr.File(label="Download: extraction_details.json (evidence + structured data)")
@@ -724,7 +998,7 @@ with gr.Blocks(title="Toxicology PDF → Grounded Table Extractor") as demo:
 
         with gr.Row():
             review_mode = gr.Checkbox(label="Review mode (enable editing)", value=False)
-            save_btn = gr.Button("Save changes to session table")
+            save_btn = gr.Button("Save edits")
             export_btn = gr.Button("Export reviewed CSV")
 
         review_status = gr.Textbox(label="Review status", interactive=False)
@@ -737,38 +1011,121 @@ with gr.Blocks(title="Toxicology PDF → Grounded Table Extractor") as demo:
             label="Vertical record view (Field → Value)"
         )
         evidence_md = gr.Markdown()
-
         reviewed_csv = gr.File(label="Download: reviewed_extraction_table.csv")
 
-        # Run extraction
+        # -------------------------
+        # INIT vocab + fields on load (via a button-less trick: use .load)
+        # -------------------------
+        def _init_all():
+            v, keys, k0, df_terms, vjson, vmsg = vocab_init_state(DEFAULT_CONTROLLED_VOCAB_JSON)
+            frows, fdf, fspec, fmsg = fields_init_state()
+            remove_choices = [r["field"] for r in frows]
+            return (
+                v, gr.update(choices=keys, value=k0), df_terms, vjson, vmsg,
+                frows, fdf, fspec, fmsg, gr.update(choices=remove_choices, value=(remove_choices[0] if remove_choices else None))
+            )
+
+        demo.load(
+            _init_all,
+            inputs=None,
+            outputs=[vocab_state, vocab_category, vocab_terms_df, vocab_json, vocab_status,
+                     field_rows_state, fields_df, field_spec, fields_status, remove_field_name]
+        )
+
+        # Vocab events
+        vocab_category.change(
+            fn=vocab_load_category,
+            inputs=[vocab_state, vocab_category, vocab_search],
+            outputs=[vocab_terms_df, vocab_terms_filtered, vocab_status]
+        )
+        vocab_add_btn.click(
+            fn=vocab_add_term,
+            inputs=[vocab_state, vocab_category, vocab_term_add, vocab_search],
+            outputs=[vocab_terms_df, vocab_terms_filtered, vocab_term_add, vocab_status]
+        )
+
+        vocab_remove_btn.click(
+            fn=vocab_remove_term,
+            inputs=[vocab_state, vocab_category, vocab_term_remove, vocab_search],
+            outputs=[vocab_terms_df, vocab_terms_filtered, vocab_term_remove, vocab_status]
+        )
+
+        vocab_apply_btn.click(
+            fn=vocab_apply_df,
+            inputs=[vocab_state, vocab_category, vocab_terms_df, vocab_search],
+            outputs=[vocab_json, vocab_terms_filtered, vocab_status]
+        )
+
+        vocab_reset_btn.click(
+            fn=vocab_reset_defaults,
+            inputs=None,
+            outputs=[vocab_state, vocab_category, vocab_terms_df, vocab_json, vocab_status]
+        )
+
+        # Field builder events
+        preset_btn.click(
+            fn=fields_load_preset,
+            inputs=[preset_name, preset_mode, field_rows_state],
+            outputs=[field_rows_state, fields_df, field_spec, fields_status]
+        ).then(
+            fn=lambda rows: gr.update(choices=[r["field"] for r in rows], value=None),
+            inputs=[field_rows_state],
+            outputs=[remove_field_name]
+        )
+
+        add_update_field_btn.click(
+            fn=fields_add_or_update,
+            inputs=[field_name_in, field_type_in, enum_values_in, instructions_in, field_rows_state],
+            outputs=[field_rows_state, fields_df, field_spec, fields_status]
+        ).then(
+            fn=lambda rows: gr.update(choices=[r["field"] for r in rows], value=None),
+            inputs=[field_rows_state],
+            outputs=[remove_field_name]
+        )
+
+        remove_field_btn.click(
+            fn=fields_remove,
+            inputs=[remove_field_name, field_rows_state],
+            outputs=[field_rows_state, fields_df, field_spec, fields_status]
+        ).then(
+            fn=lambda rows: gr.update(choices=[r["field"] for r in rows], value=None),
+            inputs=[field_rows_state],
+            outputs=[remove_field_name]
+        )
+
+        fields_apply_btn.click(
+            fn=fields_apply_df,
+            inputs=[field_rows_state, fields_df],
+            outputs=[field_rows_state, fields_df, field_spec, fields_status]
+        ).then(
+            fn=lambda rows: gr.update(choices=[r["field"] for r in rows], value=None),
+            inputs=[field_rows_state],
+            outputs=[remove_field_name]
+        )
+
+        # Extraction
         extract_btn.click(
             fn=run_extraction,
             inputs=[files, api_key, model, field_spec, vocab_json, max_pages, chunk_chars, max_context_chars],
-            outputs=[table, out_csv, out_json, status, record_pick, state_records, state_details, vertical_view, evidence_md]
+            outputs=[overview_df, out_csv, out_json, status, record_pick, state_records, state_details, vertical_view, evidence_md]
         )
 
-        # On select record
+        # Vertical view selection
         record_pick.change(
             fn=on_pick,
             inputs=[record_pick, state_records, state_details],
             outputs=[vertical_view, evidence_md]
         )
 
-        # Toggle review mode editing
-        review_mode.change(
-            fn=toggle_review_mode,
-            inputs=[review_mode],
-            outputs=[vertical_view]
-        )
+        # Review mode
+        review_mode.change(fn=toggle_review_mode, inputs=[review_mode], outputs=[vertical_view])
 
-        # Save edits back to wide table + state
         save_btn.click(
             fn=save_review_changes,
             inputs=[record_pick, vertical_view, state_records],
-            outputs=[table, state_records, review_status]
+            outputs=[overview_df, state_records, review_status]
         )
 
-        # Export reviewed CSV
         export_btn.click(
             fn=export_reviewed_csv,
             inputs=[state_records],
@@ -776,47 +1133,29 @@ with gr.Blocks(title="Toxicology PDF → Grounded Table Extractor") as demo:
         )
 
     with gr.Tab("Cross-paper Synthesis"):
-        gr.Markdown("Upload the `extraction_details.json` from the Extract tab. Synthesis is based strictly on those grounded extractions.")
+        gr.Markdown("Upload `extraction_details.json` from Extract. Synthesis is based strictly on grounded extractions.")
         api_key2 = gr.Textbox(label="OpenAI API key (optional if set as OPENAI_API_KEY secret)", type="password")
-        model2 = gr.Dropdown(
-            label="Model",
-            choices=["gpt-4o-2024-08-06", "gpt-4o", "gpt-4o-mini"],
-            value="gpt-4o-2024-08-06"
-        )
+        model2 = gr.Dropdown(label="Model", choices=["gpt-4o-2024-08-06", "gpt-4o", "gpt-4o-mini"], value="gpt-4o-2024-08-06")
         extraction_json_file = gr.File(label="Upload extraction_details.json", file_types=[".json"], file_count="single")
         synth_btn = gr.Button("Synthesize Across Papers")
         synth_md = gr.Markdown()
-
-        synth_btn.click(
-            fn=run_synthesis,
-            inputs=[api_key2, model2, extraction_json_file],
-            outputs=[synth_md]
-        )
+        synth_btn.click(fn=run_synthesis, inputs=[api_key2, model2, extraction_json_file], outputs=[synth_md])
 
     with gr.Tab("Pending tasks"):
         gr.Markdown(
-            "## Product roadmap (pending tasks)\n\n"
-            "### 1) Granular data model (one row per chemical–endpoint pair)\n"
-            "- Change schema to return `records: [ {chemical, endpoint, ...} ]`\n"
-            "- Flatten into wide table; vertical viewer targets a single record\n\n"
-            "### 2) Stronger grounding & verification\n"
-            "- Require evidence per field (already)\n"
-            "- Add automatic evidence verification (quote must exist in excerpt)\n"
-            "- Add `UNVERIFIED` flags + force empty values when evidence fails\n\n"
-            "### 3) Controlled vocab expansion & mapping\n"
-            "- Add synonym lists and preferred terms\n"
-            "- Map extracted terms into: FDA taxonomy / OECD endpoints / MedDRA-like groupings\n"
-            "- Add a vocab editor + import/export vocab JSON\n\n"
-            "### 4) Column transforms (structured parsing)\n"
-            "- Parse dose metrics into `{metric, value, unit, route, duration}`\n"
-            "- Normalize units (e.g., mg/kg/day)\n"
-            "- Auto-split multi-chemical text into canonical list\n\n"
-            "### 5) Multi-document compare mode\n"
-            "- Compare by chemical or endpoint\n"
-            "- Create a consensus + disagreements table\n\n"
-            "### 6) PDF limitations\n"
-            "- Current: **text-based PDFs only**\n"
-            "- Optional future: OCR for scanned PDFs (adds heavy dependencies)\n"
+            "## Pending tasks\n\n"
+            "1) One row per chemical–endpoint pair\n"
+            "- Change schema to output `records[]` and flatten into multiple rows per paper\n\n"
+            "2) Evidence verification\n"
+            "- If evidence quote not found in context → blank value + flag UNVERIFIED\n\n"
+            "3) Taxonomy mapping\n"
+            "- Synonyms + preferred terms for FDA / OECD / MedDRA-like structure\n\n"
+            "4) Column transforms\n"
+            "- Parse NOAEL/LOAEL etc into structured {metric,value,unit,route,duration}\n\n"
+            "5) Compare mode\n"
+            "- Compare across papers by chemical/endpoint, output consensus + disagreements table\n\n"
+            "6) OCR (optional)\n"
+            "- Currently: text-based PDFs only; OCR adds heavy deps"
         )
 
 if __name__ == "__main__":