from __future__ import annotations
import csv
import json
import os
import time
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
import gradio as gr
from scripts.export_ui_tree import build_ui_tree, load_jsonl
from slidebank.bank import BankBuildConfig, PatchVLMConfig, build_bank
from slidebank.config import load_config
from slidebank.data import safe_name
from slidebank.agent_inference_v5 import answer_question_v5
from slidebank.ui_v3_backend import (
anchor_choices,
build_demo_registry,
find_anchor,
find_tile,
first_anchor_id,
first_tile_id,
load_bank_json,
parse_anchor_choice,
parse_tile_choice,
render_global_overlay,
render_patch_cards,
render_signal_details,
render_tile_overlay,
render_tile_summary,
signal_overview_markdown,
tile_choices,
tile_from_thumbnail_click,
)
APP_ROOT = Path(__file__).resolve().parent
HOSTED_MODE = os.environ.get("SLIDEBANK_HOSTED", "").strip().lower() in {"1", "true", "yes"}
if HOSTED_MODE:
from hf_space import runtime as hosted_runtime
def _configured_path(variable: str, default: str) -> Path:
path = Path(os.environ.get(variable, default)).expanduser()
return path if path.is_absolute() else APP_ROOT / path
UI_TREE_DIR = _configured_path("SLIDEBANK_UI_TREE_DIR", "outputs/ui_tree")
BUILD_CONFIG_PATH = _configured_path("SLIDEBANK_BUILD_CONFIG", "configs/build.yaml")
INFER_CONFIG_PATH = _configured_path("SLIDEBANK_INFER_CONFIG", "configs/infer.yaml")
DEFAULT_HOST = os.environ.get("SLIDEBANK_HOST", "127.0.0.1")
DEFAULT_PORT = int(os.environ.get("SLIDEBANK_PORT", os.environ.get("GRADIO_SERVER_PORT", "7864")))
RUN_BANK_DIR = _configured_path("SLIDEBANK_RUN_DIR", "outputs/workbench_runs")
GRADIO_TEMP_DIR = _configured_path("GRADIO_TEMP_DIR", "outputs/gradio_tmp")
GRADIO_TEMP_DIR.mkdir(parents=True, exist_ok=True)
os.environ.setdefault("GRADIO_TEMP_DIR", str(GRADIO_TEMP_DIR))
def _workbench_defaults(config_path: Path) -> Dict[str, Any]:
defaults = load_config(str(config_path)) if config_path.exists() else {}
for section, keys in {
"dataset": ("csv_path", "svs_root", "thumbnail_root"),
"bank": ("persist_dir", "artifact_dir"),
"output": ("jsonl", "summary"),
}.items():
values = defaults.get(section, {})
if not isinstance(values, dict):
continue
for key in keys:
if values.get(key):
path = Path(values[key]).expanduser()
values[key] = str(path if path.is_absolute() else APP_ROOT / path)
return defaults
DEMO_REGISTRY = build_demo_registry(UI_TREE_DIR)
BUILD_DEFAULTS = _workbench_defaults(BUILD_CONFIG_PATH)
INFER_DEFAULTS = _workbench_defaults(INFER_CONFIG_PATH)
API_PROVIDERS = ["Google", "OpenAI", "Custom Endpoint"]
LOCAL_MODEL_CHOICES = ["Quilt-LLaVA", "Qwen2.5-VL-7B-Instruct"]
def _default_demo() -> Optional[str]:
return next(iter(DEMO_REGISTRY.keys()), None)
def _slide_prefix_from_demo(demo_slide_name: str) -> str:
entry = DEMO_REGISTRY.get(demo_slide_name, {})
slide_id = str(entry.get("slide_id", ""))
if "-01Z-" in slide_id:
return slide_id.split("-01Z-", 1)[0]
return demo_slide_name.strip()
def _runtime_config(
model_backend: str,
local_model: str,
api_provider: str,
api_model: str,
custom_api_model: str,
api_key: str,
api_base_url: str,
) -> Dict[str, Any]:
if HOSTED_MODE:
return hosted_runtime.hosted_runtime(INFER_DEFAULTS)
return {
"model_backend": model_backend,
"local_model": local_model,
"api_provider": api_provider,
"api_model": custom_api_model.strip() if api_model == "Custom" and custom_api_model.strip() else api_model,
"custom_api_model": custom_api_model,
"api_key": str(api_key or "").strip(),
"api_base_url": str(api_base_url or "").strip(),
}
def _runtime_label(runtime: Dict[str, Any]) -> str:
if runtime.get("hosted"):
return f"Hosted demo / {runtime.get('local_model', 'Qwen2.5-VL-3B-Instruct')}"
if str(runtime.get("model_backend", "")) == "API Model":
provider = str(runtime.get("api_provider", "") or "API").strip()
model = str(runtime.get("api_model", "") or "Custom").strip()
return f"API Model / {provider} / {model}"
model = str(runtime.get("local_model", "") or "Local Model").strip()
return f"Local Model / {model}"
def _validate_send_runtime(runtime: Dict[str, Any]) -> str:
if HOSTED_MODE:
return hosted_runtime.availability_error()
backend = str(runtime.get("model_backend", "Local Model"))
if backend == "API Model":
provider = str(runtime.get("api_provider", "")).strip()
model = str(runtime.get("api_model", "")).strip()
if not str(runtime.get("api_key", "")).strip():
return (
"API key is required for inference with API Model. "
"Please fill the API Key field on the left, then press Send again."
)
if model == "Custom" or not model:
return "Enter an API Model ID before sending with API Model."
if provider == "Custom Endpoint" and not str(runtime.get("api_base_url", "")).strip():
return "Custom Endpoint inference requires an API Base URL."
if provider not in API_PROVIDERS:
return "Choose a supported API provider: Google, OpenAI, or Custom Endpoint."
elif str(runtime.get("local_model", "")) not in LOCAL_MODEL_CHOICES:
return "Choose a supported local model."
return ""
def _config_section(name: str) -> Dict[str, Any]:
section = BUILD_DEFAULTS.get(name, {}) if isinstance(BUILD_DEFAULTS, dict) else {}
return section if isinstance(section, dict) else {}
def _infer_config_section(name: str) -> Dict[str, Any]:
section = INFER_DEFAULTS.get(name, {}) if isinstance(INFER_DEFAULTS, dict) else {}
return section if isinstance(section, dict) else {}
def _attach_agent_paths(bank: Dict[str, Any], persist_dir: str = "", artifact_dir: str = "") -> Dict[str, Any]:
if not bank:
return bank
result = dict(bank)
existing = result.get("_agent", {}) if isinstance(result.get("_agent"), dict) else {}
bank_cfg = _infer_config_section("bank")
thumb_path = str(((bank.get("global", {}) or {}).get("thumbnail", {}) or {}).get("thumbnail_path", ""))
inferred_root = thumb_path.split(f"{os.sep}artifacts{os.sep}", 1)[0] if f"{os.sep}artifacts{os.sep}" in thumb_path else ""
persist = str(os.environ.get("SLIDEBANK_BANK_DIR") or persist_dir or existing.get("persist_dir") or inferred_root or bank_cfg.get("persist_dir", "")).strip()
artifact = str(os.environ.get("SLIDEBANK_ARTIFACT_DIR") or artifact_dir or existing.get("artifact_dir") or (str(Path(inferred_root) / "artifacts") if inferred_root else "") or bank_cfg.get("artifact_dir", "") or (str(Path(persist) / "artifacts") if persist else "")).strip()
if persist:
path = Path(persist).expanduser()
persist = str(path if path.is_absolute() else APP_ROOT / path)
if artifact:
path = Path(artifact).expanduser()
artifact = str(path if path.is_absolute() else APP_ROOT / path)
result["_agent"] = {
"persist_dir": persist,
"artifact_dir": artifact,
}
return result
def _selected_api_model(api_model: str, custom_api_model: str) -> str:
model_name = custom_api_model.strip() if api_model == "Custom" and custom_api_model.strip() else api_model
return model_name
def _patch_vlm_config(
model_backend: str,
local_model: str,
api_provider: str,
api_model: str,
custom_api_model: str,
api_key: str,
api_base_url: str,
) -> PatchVLMConfig:
patch_cfg = _config_section("patch_vlm")
if model_backend == "API Model":
provider_map = {"Google": "gemini", "OpenAI": "openai", "Custom Endpoint": "openai_compatible"}
if api_provider not in provider_map:
raise RuntimeError("Choose Google, OpenAI, or Custom Endpoint.")
provider = provider_map.get(api_provider, "gemini")
key = str(api_key or "").strip()
if not key:
raise RuntimeError("Run Bank with API Model requires an API key.")
if provider == "openai_compatible" and not str(api_base_url or "").strip():
raise RuntimeError("Custom Endpoint requires an API Base URL.")
model_name = _selected_api_model(api_model, custom_api_model)
if not model_name or model_name == "Custom":
raise RuntimeError("Enter an API Model ID before building the bank.")
return PatchVLMConfig(
backend="api",
model_name=model_name,
model_path=model_name,
device_map=str(patch_cfg.get("device_map", "auto")),
dtype=str(patch_cfg.get("dtype", "fp16")),
batch_size=int(patch_cfg.get("batch_size", 32)),
max_new_tokens=int(patch_cfg.get("max_new_tokens", 224)),
api_provider=provider,
api_key=key,
api_base_url=str(api_base_url or "").strip(),
api_version=str(patch_cfg.get("api_version", "v1beta")),
api_timeout_sec=int(patch_cfg.get("api_timeout_sec", 300)),
gemini_api_env=str(patch_cfg.get("gemini_api_env", "GEMINI_API_KEY")),
gemini_timeout=int(patch_cfg.get("gemini_timeout", 120)),
)
if local_model not in LOCAL_MODEL_CHOICES:
raise RuntimeError("Choose a supported local model.")
# API model IDs configure the API branch only. The local preset can use an
# independent checkpoint, or the existing fields in an explicit local config.
explicit_local = str(patch_cfg.get("backend", "")) == "local_hf"
quilt_name = str(
patch_cfg.get("local_model_name")
or (patch_cfg.get("model_name") if explicit_local else "")
or "wisdomik/Quilt-Llava-v1.5-7b"
)
quilt_path = str(
patch_cfg.get("local_model_path")
or (patch_cfg.get("model_path") if explicit_local else "")
or quilt_name
)
model_aliases = {
"Quilt-LLaVA": (quilt_name, quilt_path),
"Qwen2.5-VL-7B-Instruct": ("Qwen/Qwen2.5-VL-7B-Instruct", "Qwen/Qwen2.5-VL-7B-Instruct"),
}
model_name, model_path = model_aliases.get(local_model, (local_model, local_model))
return PatchVLMConfig(
backend="local_hf",
model_name=model_name,
model_path=model_path,
quilt_llava_repo=str(patch_cfg.get("quilt_llava_repo", "")),
device_map=str(patch_cfg.get("device_map", "auto")),
dtype=str(patch_cfg.get("dtype", "fp16")),
batch_size=int(patch_cfg.get("batch_size", 32)),
max_new_tokens=int(patch_cfg.get("max_new_tokens", 224)),
gemini_api_env=str(patch_cfg.get("gemini_api_env", "GEMINI_API_KEY")),
gemini_timeout=int(patch_cfg.get("gemini_timeout", 120)),
)
def _bank_loaded(bank: Optional[Dict[str, Any]]) -> bool:
return bool(bank and bank.get("tiles"))
def _tile_outputs(
bank: Optional[Dict[str, Any]],
tile_id: str,
anchor_id: str = "",
patch_id: str = "",
) -> Tuple[Any, str, Any, str, str, str]:
tile = find_tile(bank, tile_id)
tile_id = str(tile.get("tile_id", "")) if tile else ""
if not anchor_id:
anchor_id = first_anchor_id(tile)
anchor = find_anchor(tile, anchor_id)
anchor_id = str(anchor.get("anchor_id", "")) if anchor else ""
choices = anchor_choices(tile)
anchor_value = next((choice for choice in choices if parse_anchor_choice(choice) == anchor_id), choices[0] if choices else None)
return (
render_tile_overlay(bank, tile_id, anchor_id, patch_id),
render_tile_summary(bank, tile_id),
gr.update(choices=choices, value=anchor_value),
render_patch_cards(bank, tile_id, anchor_id),
render_signal_details(bank, tile_id, anchor_id),
anchor_id,
)
def _initial_bank() -> Tuple[Dict[str, Any], str]:
demo_name = _default_demo()
if not demo_name:
return {}, ""
try:
return _attach_agent_paths(load_bank_json(DEMO_REGISTRY[demo_name]["bank_json_path"])), demo_name
except Exception:
return {}, demo_name
def first_evidence_tile_id(bank: Optional[Dict[str, Any]]) -> str:
for tile in (bank or {}).get("tiles", []) or []:
if tile.get("anchors"):
return str(tile.get("tile_id", ""))
return first_tile_id(bank)
INITIAL_BANK, INITIAL_DEMO = _initial_bank()
INITIAL_TILE_ID = first_evidence_tile_id(INITIAL_BANK)
INITIAL_TILE = find_tile(INITIAL_BANK, INITIAL_TILE_ID)
INITIAL_ANCHOR_ID = first_anchor_id(INITIAL_TILE)
INITIAL_TILE_CHOICES = tile_choices(INITIAL_BANK)
INITIAL_TILE_VALUE = next(
(choice for choice in INITIAL_TILE_CHOICES if parse_tile_choice(choice) == INITIAL_TILE_ID),
INITIAL_TILE_CHOICES[0] if INITIAL_TILE_CHOICES else None,
)
INITIAL_ANCHOR_CHOICES = anchor_choices(INITIAL_TILE)
INITIAL_ANCHOR_VALUE = next(
(choice for choice in INITIAL_ANCHOR_CHOICES if parse_anchor_choice(choice) == INITIAL_ANCHOR_ID),
INITIAL_ANCHOR_CHOICES[0] if INITIAL_ANCHOR_CHOICES else None,
)
INITIAL_STATUS = (
"Evidence tree loaded. Browse the evidence, then configure the model and memory-bank paths for QA."
if INITIAL_BANK
else "Export a bank with scripts/export_ui_tree.py, then set SLIDEBANK_UI_TREE_DIR or upload its JSON. See docs/WORKBENCH.md."
)
INITIAL_CHAT = [
{
"role": "assistant",
"content": "The evidence tree is loaded. Browse its grid cells and anchors below, or configure your model and memory-bank paths to ask a question.",
}
] if INITIAL_BANK else []
if HOSTED_MODE:
availability = hosted_runtime.availability_error()
INITIAL_STATUS = availability or "ZeroGPU QA is available. Requests share a GPU queue and usage quota."
INITIAL_CHAT = [{
"role": "assistant",
"content": (
"The prebuilt evidence bank is ready to explore. "
+ (availability or "Ask a question to retrieve evidence and run the hosted Qwen2.5-VL model.")
),
}]
def on_image_source_change(image_source: str) -> Tuple[Any, Any]:
return (
gr.update(visible=image_source == "Upload UI-tree JSON"),
gr.update(visible=image_source == "Configured evidence trees"),
)
def on_model_backend_change(model_backend: str) -> Tuple[Any, Any]:
return (
gr.update(visible=model_backend == "Local Model"),
gr.update(visible=model_backend == "API Model"),
)
def on_api_provider_change(api_provider: str) -> Any:
return gr.update(visible=api_provider == "Custom Endpoint")
def _bank_ui_response(
bank: Dict[str, Any],
status: str,
runtime: Dict[str, Any],
history: List[Dict[str, str]],
) -> Tuple[
Dict[str, Any],
str,
str,
str,
Any,
Any,
str,
Any,
str,
Any,
str,
str,
str,
Dict[str, Any],
List[Dict[str, str]],
]:
if not _bank_loaded(bank):
return (
{},
"",
"",
"",
render_global_overlay(None),
gr.update(choices=[], value=None),
signal_overview_markdown(None),
render_tile_overlay(None, ""),
"### Selected Grid Detail\nNo grid selected.",
gr.update(choices=[], value=None),
"
No anchor selected.
",
"No signal details available.",
status,
runtime,
history,
)
tile_id = first_evidence_tile_id(bank)
tile = find_tile(bank, tile_id)
anchor_id = first_anchor_id(tile)
tile_choice_list = tile_choices(bank)
tile_value = next(
(choice for choice in tile_choice_list if parse_tile_choice(choice) == tile_id),
tile_choice_list[0] if tile_choice_list else None,
)
tile_image, tile_md, anchor_update, patch_html, signal_details, anchor_id = _tile_outputs(bank, tile_id, anchor_id)
return (
bank,
tile_id,
anchor_id,
"",
render_global_overlay(bank, tile_id),
gr.update(choices=tile_choice_list, value=tile_value),
signal_overview_markdown(bank),
tile_image,
tile_md,
anchor_update,
patch_html,
signal_details,
status,
runtime,
history,
)
def _demo_csv_for_slide(slide_prefix: str, out_csv: Path) -> int:
dataset_cfg = _config_section("dataset")
source_csv = Path(str(dataset_cfg.get("csv_path", ""))).expanduser()
if not source_csv.is_file():
raise RuntimeError(f"Configured CSV does not exist: {source_csv}")
matched: List[Dict[str, str]] = []
with source_csv.open("r", encoding="utf-8-sig", newline="") as handle:
reader = csv.DictReader(handle)
fieldnames = list(reader.fieldnames or [])
if not fieldnames:
raise RuntimeError(f"CSV has no header: {source_csv}")
slide_keys = [key for key in ["Slide", "slide", "slide_id", "Slide ID"] if key in fieldnames]
if not slide_keys:
raise RuntimeError("CSV must contain a Slide or slide column.")
for row in reader:
slide_value = str(row.get(slide_keys[0], "")).strip()
if slide_value == slide_prefix or slide_value.startswith(slide_prefix) or slide_prefix.startswith(slide_value):
matched.append(row)
if not matched:
raise RuntimeError(f"No CSV rows matched demo slide prefix: {slide_prefix}")
out_csv.parent.mkdir(parents=True, exist_ok=True)
with out_csv.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(matched)
return len(matched)
def _export_ui_tree_from_readable(readable_dir: Path, slide_prefix: str, out_dir: Path) -> Tuple[Dict[str, Any], Path]:
if not readable_dir.is_dir():
raise RuntimeError(f"Readable bank directory was not created: {readable_dir}")
files = sorted(readable_dir.glob("*.jsonl"))
matching = [path for path in files if path.stem == slide_prefix or path.stem.startswith(slide_prefix)]
readable_path = matching[0] if matching else (files[0] if files else None)
if readable_path is None:
raise RuntimeError(f"No readable JSONL files found in {readable_dir}")
records = load_jsonl(readable_path)
tree = build_ui_tree(records, readable_path)
out_dir.mkdir(parents=True, exist_ok=True)
out_path = out_dir / f"{safe_name(Path(tree['slide_id']).stem)}.ui_tree.json"
out_path.write_text(json.dumps(tree, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
return tree, out_path
def load_prebuilt_bank(
image_source: str,
uploaded_file: Any,
demo_slide_name: str,
model_backend: str,
local_model: str,
api_provider: str,
api_model: str,
custom_api_model: str,
api_key: str,
api_base_url: str,
chat_history: List[Dict[str, str]],
) -> Tuple[
Dict[str, Any],
str,
str,
str,
Any,
Any,
str,
Any,
str,
Any,
str,
str,
str,
Dict[str, Any],
List[Dict[str, str]],
]:
runtime = _runtime_config(model_backend, local_model, api_provider, api_model, custom_api_model, api_key, api_base_url)
history = list(chat_history or [])
try:
if HOSTED_MODE and image_source != "Configured evidence trees":
raise RuntimeError("The hosted workbench uses the prebuilt evidence bank. Uploads are available in the local workbench.")
if image_source == "Configured evidence trees":
if not demo_slide_name or demo_slide_name not in DEMO_REGISTRY:
raise RuntimeError("Export an evidence tree and set SLIDEBANK_UI_TREE_DIR before loading a configured slide.")
bank = _attach_agent_paths(load_bank_json(DEMO_REGISTRY[demo_slide_name]["bank_json_path"]))
status = "Evidence tree loaded. Browse below, then configure the model and memory-bank paths for QA."
else:
file_path = getattr(uploaded_file, "name", uploaded_file)
file_path = str(file_path or "").strip()
if not file_path:
raise RuntimeError("Upload a UI-tree JSON exported with scripts/export_ui_tree.py.")
if Path(file_path).suffix.lower() == ".json":
bank = _attach_agent_paths(load_bank_json(file_path))
status = "Uploaded evidence tree loaded. Image paths must be accessible on this server; configure the memory bank for QA."
else:
raise RuntimeError("Upload a UI-tree JSON file. Build raw WSI inputs with scripts/build_bank.py first.")
history.append({"role": "assistant", "content": status})
return _bank_ui_response(bank, status, runtime, history)
except Exception as exc:
status = f"{type(exc).__name__}: {exc}"
history.append({"role": "assistant", "content": status})
return _bank_ui_response({}, status, runtime, history)
def run_bank_extraction(
image_source: str,
uploaded_file: Any,
demo_slide_name: str,
model_backend: str,
local_model: str,
api_provider: str,
api_model: str,
custom_api_model: str,
api_key: str,
api_base_url: str,
chat_history: List[Dict[str, str]],
current_bank: Dict[str, Any],
) -> Tuple[
Dict[str, Any],
str,
str,
str,
Any,
Any,
str,
Any,
str,
Any,
str,
str,
str,
Dict[str, Any],
List[Dict[str, str]],
]:
runtime = _runtime_config(model_backend, local_model, api_provider, api_model, custom_api_model, api_key, api_base_url)
history = list(chat_history or [])
try:
if image_source != "Configured evidence trees":
raise RuntimeError("Rebuild uses a configured slide and the local WSI paths in configs/build.yaml. Use Load Evidence to inspect uploaded JSON.")
if not demo_slide_name or demo_slide_name not in DEMO_REGISTRY:
raise RuntimeError("Choose a configured evidence tree to rebuild. For a new slide, run scripts/build_bank.py first.")
slide_prefix = _slide_prefix_from_demo(demo_slide_name)
run_tag = f"{safe_name(slide_prefix)}_{time.strftime('%Y%m%d_%H%M%S')}"
run_dir = RUN_BANK_DIR / run_tag
persist_dir = run_dir / "memory_bank"
artifact_dir = persist_dir / "artifacts"
filtered_csv = run_dir / "inputs" / "selected_slide.csv"
row_count = _demo_csv_for_slide(slide_prefix, filtered_csv)
dataset_cfg = _config_section("dataset")
bank_cfg = _config_section("bank")
patch_vlm = _patch_vlm_config(
model_backend=model_backend,
local_model=local_model,
api_provider=api_provider,
api_model=api_model,
custom_api_model=custom_api_model,
api_key=api_key,
api_base_url=api_base_url,
)
build_cfg = BankBuildConfig(
csv_path=str(filtered_csv),
svs_root=str(dataset_cfg.get("svs_root", "")),
svs_pattern=str(dataset_cfg.get("svs_pattern", "*/*.svs")),
persist_dir=str(persist_dir),
artifact_dir=str(artifact_dir),
max_slides=1,
rebuild=True,
force_ingest=True,
use_llm_patch_describer=bool(bank_cfg.get("use_llm_patch_describer", True)),
patch_description_strict=bool(bank_cfg.get("patch_description_strict", True)),
thumbnail_tile_size=int(bank_cfg.get("thumbnail_tile_size", 384)),
anchor_crop_size_level0=int(bank_cfg.get("anchor_crop_size_level0", 4096)),
patch_crop_size_level0=int(bank_cfg.get("patch_crop_size_level0", 512)),
quiet=False,
patch_vlm=patch_vlm,
)
for required_field in ["csv_path", "svs_root", "persist_dir", "artifact_dir"]:
if not getattr(build_cfg, required_field):
raise RuntimeError(f"Missing required build config: {required_field}")
summary = build_bank(build_cfg)
if summary.get("errors"):
raise RuntimeError("; ".join(str(item) for item in summary["errors"][:3]))
if int(summary.get("slides_ingested", 0)) < 1 and int(summary.get("slides_reused", 0)) < 1:
raise RuntimeError(f"No slides were ingested. Missing SVS: {summary.get('missing_svs', [])}")
bank, ui_tree_path = _export_ui_tree_from_readable(persist_dir / "readable", slide_prefix, run_dir / "ui_tree")
bank = _attach_agent_paths(bank, persist_dir=str(persist_dir), artifact_dir=str(artifact_dir))
status = (
f"Bank extraction complete for {slide_prefix}. "
f"Rows={row_count}; summary={summary.get('summary_path')}; ui_tree={ui_tree_path}"
)
history.append({"role": "assistant", "content": status})
return _bank_ui_response(bank, status, runtime, history)
except Exception as exc:
status = f"{type(exc).__name__}: {exc}"
history.append({"role": "assistant", "content": status})
if _bank_loaded(current_bank):
return _bank_ui_response(current_bank, status, runtime, history)
return (
{},
"",
"",
"",
render_global_overlay(None),
gr.update(choices=[], value=None),
signal_overview_markdown(None),
render_tile_overlay(None, ""),
"### Selected Grid Detail\nNo grid selected.",
gr.update(choices=[], value=None),
"No anchor selected.
",
"No signal details available.",
status,
runtime,
history,
)
def on_tile_select(tile_choice: str, bank: Dict[str, Any]) -> Tuple[str, str, str, Any, Any, str, Any, str, str, str]:
if not _bank_loaded(bank) or not tile_choice:
return (
gr.skip(),
gr.skip(),
gr.skip(),
gr.skip(),
gr.skip(),
gr.skip(),
gr.skip(),
gr.skip(),
gr.skip(),
"Please run or load the evidence bank first.",
)
tile_id = parse_tile_choice(tile_choice)
tile = find_tile(bank, tile_id)
anchor_id = first_anchor_id(tile)
tile_image, tile_md, anchor_update, patch_html, signal_details, anchor_id = _tile_outputs(bank, tile_id, anchor_id)
return (
tile_id,
anchor_id,
"",
render_global_overlay(bank, tile_id),
tile_image,
tile_md,
anchor_update,
patch_html,
signal_details,
f"Grid {tile_id} selected.",
)
def on_global_image_select(bank: Dict[str, Any], evt: gr.SelectData) -> Tuple[str, str, str, Any, Any, str, Any, str, str, str, Any]:
if not _bank_loaded(bank):
return (
gr.skip(),
gr.skip(),
gr.skip(),
gr.skip(),
gr.skip(),
gr.skip(),
gr.skip(),
gr.skip(),
gr.skip(),
"Please run or load the evidence bank first.",
gr.skip(),
)
try:
x, y = evt.index
except Exception:
x, y = 0, 0
tile_id = tile_from_thumbnail_click(bank, int(x), int(y))
tile = find_tile(bank, tile_id)
anchor_id = first_anchor_id(tile)
tile_choice_list = tile_choices(bank)
tile_value = next((choice for choice in tile_choice_list if parse_tile_choice(choice) == tile_id), tile_choice_list[0] if tile_choice_list else None)
tile_image, tile_md, anchor_update, patch_html, signal_details, anchor_id = _tile_outputs(bank, tile_id, anchor_id)
return (
tile_id,
anchor_id,
"",
render_global_overlay(bank, tile_id),
tile_image,
tile_md,
anchor_update,
patch_html,
signal_details,
f"Grid {tile_id} selected.",
gr.update(value=tile_value),
)
def on_anchor_select(anchor_choice: str, bank: Dict[str, Any], tile_id: str) -> Tuple[str, str, Any, str, str, str]:
if not _bank_loaded(bank) or not anchor_choice:
return (
gr.skip(),
gr.skip(),
gr.skip(),
gr.skip(),
gr.skip(),
"Please run or load the evidence bank first.",
)
anchor_id = parse_anchor_choice(anchor_choice)
return (
anchor_id,
"",
render_tile_overlay(bank, tile_id, anchor_id),
render_patch_cards(bank, tile_id, anchor_id),
render_signal_details(bank, tile_id, anchor_id),
f"Anchor {anchor_id} selected.",
)
def _hosted_error_message(error: Exception) -> str:
text = str(error).lower()
if "zerogpu" in text and "quota" in text:
return (
"Your remaining ZeroGPU allowance is insufficient for this request's reserved GPU duration. "
"Sign in to Hugging Face to use your account's allowance, or try again after the quota resets. "
"Evidence browsing is still available."
)
return (
"Live QA could not finish. ZeroGPU capacity, quota, or the model runtime may be unavailable. "
"Please retry later; evidence browsing is still available."
)
def submit_chat(
message: str,
chat_history: List[Dict[str, str]],
bank: Dict[str, Any],
selected_tile_id: str,
selected_anchor_id: str,
model_backend: str,
local_model: str,
api_provider: str,
api_model: str,
custom_api_model: str,
api_key: str,
api_base_url: str,
) -> Tuple[List[Dict[str, str]], str, Any, str, Dict[str, Any], str]:
history = list(chat_history or [])
question = str(message or "").strip()
if not question:
return history, "", gr.skip(), gr.skip(), gr.skip(), "Please enter a question."
runtime = _runtime_config(model_backend, local_model, api_provider, api_model, custom_api_model, api_key, api_base_url)
if HOSTED_MODE and len(question) > 2000:
return history, question, gr.skip(), gr.skip(), runtime, "Please limit hosted questions to 2,000 characters."
if not _bank_loaded(bank):
status = "Load an exported evidence tree before asking a question. See docs/WORKBENCH.md."
return history, "", gr.skip(), gr.skip(), runtime, status
validation_error = _validate_send_runtime(runtime)
if validation_error:
assistant_text = f"### Runtime Configuration Needed\n{validation_error}\n\nSelected backend: `{_runtime_label(runtime)}`"
history.extend(
[
{"role": "user", "content": question},
{"role": "assistant", "content": assistant_text},
]
)
return (
history,
"",
render_tile_overlay(bank, selected_tile_id, selected_anchor_id),
render_patch_cards(bank, selected_tile_id, selected_anchor_id),
runtime,
validation_error,
)
history_before_turn = list(history)
try:
answer_function = hosted_runtime.answer_on_gpu if HOSTED_MODE else answer_question_v5
answer = answer_function(
message=question,
bank=bank,
runtime=runtime,
defaults=INFER_DEFAULTS,
previous_messages=history_before_turn,
)
assistant_text = f"{answer.answer_markdown}\n\n{answer.trace_markdown}"
status = (
f"Agent inference complete in {answer.elapsed_sec:.1f}s. "
f"backend={_runtime_label(runtime)}; "
f"channel={answer.question_channel}; stage={answer.final_stage}; route={answer.evidence_route}"
)
runtime = dict(runtime)
runtime["agent_route"] = answer.route
runtime["question_channel"] = answer.question_channel
runtime["final_stage"] = answer.final_stage
runtime["evidence_route"] = answer.evidence_route
except Exception as exc:
if HOSTED_MODE:
status = _hosted_error_message(exc)
assistant_text = status
print(f"Hosted QA failed: {type(exc).__name__}: {exc}", flush=True)
else:
assistant_text = f"### Agent Error\n{type(exc).__name__}: {exc}"
status = f"Agent inference failed: {type(exc).__name__}: {exc}"
history.extend(
[
{"role": "user", "content": question},
{"role": "assistant", "content": assistant_text},
]
)
return (
history,
"",
render_tile_overlay(bank, selected_tile_id, selected_anchor_id),
render_patch_cards(bank, selected_tile_id, selected_anchor_id),
runtime,
status,
)
def clear_chat() -> List[Dict[str, str]]:
return []
CSS = """
.gradio-container { max-width: 1760px !important; background: #F8FAFC; color: #111827; }
#title h1 { margin: 0 0 4px 0; }
#top_controls, #chat_panel, #global_panel, #grid_panel, #patch_panel {
border: 1px solid #E5E7EB;
border-radius: 8px;
background: #FFFFFF;
padding: 12px;
}
#top_controls { min-height: 492px; }
#chat_panel { min-height: 492px; }
#chatbot { border: 1px solid #E5E7EB; border-radius: 8px; min-height: 460px; }
#chatbot .message,
#chatbot .message-content,
#chatbot .prose,
#chatbot .md,
#chatbot p,
#chatbot li {
max-height: none !important;
overflow: visible !important;
text-overflow: clip !important;
white-space: normal !important;
-webkit-line-clamp: unset !important;
line-clamp: unset !important;
}
#chat_input_row { align-items: end; }
#patch_cards .patch-panel { width: 100%; }
#patch_cards .anchor-line { display: flex; gap: 12px; align-items: baseline; margin-bottom: 10px; color: #111827; }
#patch_cards .anchor-line span { color: #6B7280; }
#patch_cards .patch-grid { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 12px; }
#patch_cards .patch-card { position: relative; border: 1px solid #E5E7EB; border-radius: 8px; background: #FFFFFF; overflow: visible; min-height: 180px; }
#patch_cards .patch-card img { display: block; width: 100%; aspect-ratio: 1 / 1; object-fit: cover; border-radius: 8px 8px 0 0; }
#patch_cards .patch-meta { display: flex; justify-content: space-between; gap: 8px; padding: 8px; font-size: 13px; color: #111827; }
#patch_cards .patch-meta span { color: #6B7280; white-space: nowrap; }
#patch_cards .patch-tip { display: none; position: absolute; z-index: 20; left: 8px; right: 8px; bottom: 48px; padding: 10px; border: 1px solid #CBD5E1; border-radius: 8px; background: #111827; color: #FFFFFF; font-size: 12px; line-height: 1.45; box-shadow: 0 8px 20px rgba(15, 23, 42, 0.18); }
#patch_cards .patch-card:hover .patch-tip, #patch_cards .patch-card:focus .patch-tip { display: block; }
#patch_cards .missing-patch, #patch_cards .empty-panel { display: flex; align-items: center; justify-content: center; min-height: 160px; color: #6B7280; background: #F3F4F6; border-radius: 8px; }
#patch_cards .extra-patches { margin-top: 12px; }
#patch_cards .extra-patches summary { cursor: pointer; color: #2563EB; font-weight: 600; margin-bottom: 10px; }
@media (max-width: 980px) {
#top_controls, #chat_panel { min-height: 0; }
#patch_cards .patch-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); }
}
"""
with gr.Blocks(title="SlideBank Workbench", css=CSS) as demo:
gr.Markdown("# SlideBank Workbench", elem_id="title")
gr.Markdown(
"Explore the prebuilt **slide → grid → anchor patches** evidence bank. "
"Live questions use **Qwen2.5-VL-3B-Instruct on ZeroGPU**, a demonstration model distinct from the paper's pathology backbones. "
"GPU requests may queue and are subject to Hugging Face usage quota."
if HOSTED_MODE else
"Load an exported evidence tree to explore **slide → grid → anchor patches**. Connect your own local model or API for question answering. [Setup guide](https://github.com/gexinh/SlideBank/blob/main/docs/WORKBENCH.md)."
)
bank_state = gr.State(INITIAL_BANK)
selected_tile_id_state = gr.State(INITIAL_TILE_ID)
selected_anchor_id_state = gr.State(INITIAL_ANCHOR_ID)
selected_patch_id_state = gr.State("")
runtime_config_state = gr.State({})
with gr.Row(equal_height=True):
with gr.Column(scale=1, min_width=340, elem_id="top_controls"):
gr.Markdown("### Evidence bank" if HOSTED_MODE else "### Input & Runtime Controls")
image_source = gr.Radio(label="Evidence Source", choices=["Configured evidence trees", "Upload UI-tree JSON"], value="Configured evidence trees", visible=not HOSTED_MODE)
upload_file = gr.File(
label="Upload exported UI-tree JSON",
file_types=[".json"],
visible=False,
)
demo_slide = gr.Dropdown(
label="Demo Slide" if HOSTED_MODE else "Configured Slide",
choices=list(DEMO_REGISTRY.keys()),
value=INITIAL_DEMO or None,
interactive=True,
)
model_backend = gr.Radio(label="Model Backend", choices=["Local Model", "API Model"], value="Local Model", visible=not HOSTED_MODE)
with gr.Group(visible=not HOSTED_MODE) as local_model_group:
local_model = gr.Dropdown(
label="Local Model",
choices=LOCAL_MODEL_CHOICES,
value="Quilt-LLaVA",
)
with gr.Group(visible=False) as api_model_group:
api_provider = gr.Dropdown(label="API Provider", choices=API_PROVIDERS, value="Google")
api_model = gr.State("Custom")
custom_api_model = gr.Textbox(label="API Model ID", placeholder="Enter the model ID offered by your provider", value="", lines=1)
api_key = gr.Textbox(label="API Key", value="", type="password", lines=1)
api_base_url = gr.Textbox(label="API Base URL", value="", lines=1, visible=False)
load_bank_btn = gr.Button("Load Evidence", variant="primary")
run_bank_btn = gr.Button("Rebuild Configured Slide", variant="secondary", visible=not HOSTED_MODE)
gr.Markdown("Rebuild reads local WSI inputs from `configs/build.yaml` and calls the selected model. For a new WSI, use `scripts/build_bank.py` first.", visible=not HOSTED_MODE)
status_box = gr.Textbox(label="Status", value=INITIAL_STATUS, interactive=False, lines=2)
with gr.Column(scale=2, min_width=620, elem_id="chat_panel"):
gr.Markdown("### Inference Chat")
chatbot = gr.Chatbot(
label="Conversation",
value=INITIAL_CHAT,
type="messages",
height=460,
max_height=720,
autoscroll=True,
resizable=True,
show_copy_button=True,
elem_id="chatbot",
)
with gr.Row(elem_id="chat_input_row"):
chat_input = gr.Textbox(
label="Message",
placeholder="Ask a question about the selected slide evidence.",
lines=2,
scale=8,
)
send_btn = gr.Button("Send", variant="primary", scale=1, interactive=not HOSTED_MODE or not hosted_runtime.availability_error())
clear_chat_btn = gr.Button("Clear Chat", variant="secondary", size="sm")
with gr.Column(elem_id="global_panel"):
gr.Markdown("### Global Slide Overview")
with gr.Row():
with gr.Column(scale=3):
global_image = gr.Image(
label="Thumbnail Grid",
value=render_global_overlay(INITIAL_BANK, INITIAL_TILE_ID),
interactive=False,
height=390,
)
with gr.Column(scale=2):
global_summary = gr.Markdown(signal_overview_markdown(INITIAL_BANK))
tile_selector = gr.Dropdown(
label="Grid Cell",
choices=INITIAL_TILE_CHOICES,
value=INITIAL_TILE_VALUE,
interactive=True,
)
with gr.Column(elem_id="grid_panel"):
gr.Markdown("### Selected Grid Detail")
with gr.Row():
with gr.Column(scale=3):
tile_image = gr.Image(
label="Grid Overlay",
value=render_tile_overlay(INITIAL_BANK, INITIAL_TILE_ID, INITIAL_ANCHOR_ID),
interactive=False,
height=390,
)
with gr.Column(scale=2):
tile_summary = gr.Markdown(render_tile_summary(INITIAL_BANK, INITIAL_TILE_ID))
anchor_selector = gr.Radio(
label="Anchor Summaries",
choices=INITIAL_ANCHOR_CHOICES,
value=INITIAL_ANCHOR_VALUE,
interactive=True,
)
with gr.Accordion("Signal Details", open=False):
signal_details = gr.Markdown(render_signal_details(INITIAL_BANK, INITIAL_TILE_ID, INITIAL_ANCHOR_ID))
with gr.Column(elem_id="patch_panel"):
gr.Markdown("### Anchor Patch Views")
patch_cards = gr.HTML(render_patch_cards(INITIAL_BANK, INITIAL_TILE_ID, INITIAL_ANCHOR_ID), elem_id="patch_cards")
if not HOSTED_MODE:
image_source.change(on_image_source_change, inputs=[image_source], outputs=[upload_file, demo_slide])
model_backend.change(on_model_backend_change, inputs=[model_backend], outputs=[local_model_group, api_model_group])
api_provider.change(on_api_provider_change, inputs=[api_provider], outputs=[api_base_url])
bank_inputs = [
image_source,
upload_file,
demo_slide,
model_backend,
local_model,
api_provider,
api_model,
custom_api_model,
api_key,
api_base_url,
chatbot,
]
bank_outputs = [
bank_state,
selected_tile_id_state,
selected_anchor_id_state,
selected_patch_id_state,
global_image,
tile_selector,
global_summary,
tile_image,
tile_summary,
anchor_selector,
patch_cards,
signal_details,
status_box,
runtime_config_state,
chatbot,
]
if not HOSTED_MODE:
run_bank_btn.click(run_bank_extraction, inputs=bank_inputs + [bank_state], outputs=bank_outputs)
load_bank_btn.click(load_prebuilt_bank, inputs=bank_inputs, outputs=bank_outputs)
demo_slide.change(load_prebuilt_bank, inputs=bank_inputs, outputs=bank_outputs)
tile_selector.change(
on_tile_select,
inputs=[tile_selector, bank_state],
outputs=[
selected_tile_id_state,
selected_anchor_id_state,
selected_patch_id_state,
global_image,
tile_image,
tile_summary,
anchor_selector,
patch_cards,
signal_details,
status_box,
],
)
global_image.select(
on_global_image_select,
inputs=[bank_state],
outputs=[
selected_tile_id_state,
selected_anchor_id_state,
selected_patch_id_state,
global_image,
tile_image,
tile_summary,
anchor_selector,
patch_cards,
signal_details,
status_box,
tile_selector,
],
)
anchor_selector.change(
on_anchor_select,
inputs=[anchor_selector, bank_state, selected_tile_id_state],
outputs=[
selected_anchor_id_state,
selected_patch_id_state,
tile_image,
patch_cards,
signal_details,
status_box,
],
preprocess=False,
)
chat_inputs = [
chat_input,
chatbot,
bank_state,
selected_tile_id_state,
selected_anchor_id_state,
model_backend,
local_model,
api_provider,
api_model,
custom_api_model,
api_key,
api_base_url,
]
chat_outputs = [chatbot, chat_input, tile_image, patch_cards, runtime_config_state, status_box]
send_btn.click(submit_chat, inputs=chat_inputs, outputs=chat_outputs)
chat_input.submit(submit_chat, inputs=chat_inputs, outputs=chat_outputs)
clear_chat_btn.click(clear_chat, inputs=None, outputs=[chatbot])
if __name__ == "__main__":
demo.queue(default_concurrency_limit=1).launch(server_name=DEFAULT_HOST, server_port=DEFAULT_PORT, share=False)