""" visual_explainer.py — Caption/table understanding for Paper2Lab. This is caption-grounded visual understanding. It does not use image pixels yet. For the hackathon MVP, this gives useful figure/table summaries without GPU cost. """ from __future__ import annotations import re from typing import Any, Dict, List _METRIC_WORDS = [ "accuracy", "precision", "recall", "f1", "auc", "bleu", "rouge", "loss", "perplexity", "score", "performance", "results", "comparison", "evaluation", "training cost", "p-value", ] _METHOD_WORDS = [ "architecture", "pipeline", "framework", "workflow", "model", "method", "procedure", "attention", "encoder", "decoder", "algorithm", "overview", ] _DATA_WORDS = [ "dataset", "data", "samples", "patients", "images", "sentences", "articles", "studies", "distribution", "statistics", "characteristics", ] def _clean(text: str) -> str: text = text or "" text = re.sub(r"\s+", " ", text) return text.strip(" .;:\n\t") def _label_type(label: str) -> str: low = (label or "").lower() if "table" in low or "tbl" in low: return "table" if "figure" in low or "fig" in low: return "figure" if "algorithm" in low: return "algorithm" if "scheme" in low: return "scheme" return "visual" def _purpose(caption: str, visual_type: str) -> str: low = caption.lower() if any(w in low for w in _METHOD_WORDS): return "method_or_architecture" if any(w in low for w in _METRIC_WORDS): return "results_or_evaluation" if any(w in low for w in _DATA_WORDS): return "data_or_dataset_description" if visual_type == "table": return "structured_results_or_metadata" return "illustrative_figure" def _summary_from_caption(label: str, caption: str, visual_type: str) -> str: caption = _clean(caption) if not caption: return f"{label} is a {visual_type}, but no caption text was extracted." # Keep concise, but grounded in caption text. if len(caption) <= 220: return caption first_sentence = re.split(r"(?<=[.!?])\s+", caption)[0] return _clean(first_sentence[:260]) def _summarize_table_data(table: Dict[str, Any]) -> str | None: data = table.get("data") if not isinstance(data, list) or not data: return None rows = [r for r in data if isinstance(r, list)] if not rows: return None n_rows = len(rows) n_cols = max((len(r) for r in rows), default=0) header = rows[0] if rows else [] header_text = ", ".join(str(x).strip() for x in header if str(x).strip())[:180] if header_text: return f"Extracted table with approximately {n_rows} rows and {n_cols} columns. Header fields include: {header_text}." return f"Extracted table with approximately {n_rows} rows and {n_cols} columns." def explain_figures_and_tables(extracted: Dict[str, Any]) -> List[Dict[str, Any]]: """Return concise explanations for extracted captions and tables.""" outputs: List[Dict[str, Any]] = [] for cap in extracted.get("captions", []) or []: label = cap.get("label", "") caption = _clean(cap.get("caption", "")) visual_type = _label_type(label) outputs.append({ "label": label, "type": visual_type, "purpose": _purpose(caption, visual_type), "summary": _summary_from_caption(label, caption, visual_type), "evidence": caption, "page_number": cap.get("page_number"), }) # Add tables that have data but no caption match. existing_table_pages = {(o.get("page_number"), o.get("label")) for o in outputs if o.get("type") == "table"} for table in extracted.get("tables", []) or []: page = table.get("page_number") caption = _clean(table.get("caption") or "") label = f"Table extracted on page {page}" if page is not None else "Extracted table" if (page, label) in existing_table_pages: continue data_summary = _summarize_table_data(table) outputs.append({ "label": label, "type": "table", "purpose": _purpose(caption or data_summary or "", "table"), "summary": caption or data_summary or "A table was detected, but its content could not be summarized reliably.", "evidence": caption or data_summary or "", "page_number": page, }) return outputs[:20]