feat(uploads): ADR-0011 manual-upload safety gate (Now slice)

.dockerignore

@@ -1,37 +0,0 @@
-# ADR-0007 sandbox image build context trimming.
-# The sandbox Dockerfile only COPYs `src/` and `requirements.txt`; everything
-# else in the repo (multi-GB venvs, git history, local audit tooling, caches)
-# would otherwise be streamed to the Docker daemon on every build. Excluded here
-# so `docker build -f docker/sandbox.Dockerfile .` sends a small context.
-
-# Virtualenvs / local interpreters
-.venv/
-venv/
-security/.audit-venv/
-
-# VCS + tooling metadata
-.git/
-.github/
-.claude/
-
-# Local security-audit tooling (not needed in the sandbox image)
-security/
-
-# Python caches / build artifacts
-**/__pycache__/
-**/*.pyc
-**/*.pyo
-.pytest_cache/
-*.egg-info/
-
-# Runtime scratch / caches / generated data that must not ship in the image
-src/tmp/
-**/geo_cache/
-*.h5ad
-*.log
-
-# Docs, tests, notebooks — not part of the runtime image
-docs/
-tests/
-scripts/
-*.ipynb

.gitattributes

@@ -33,12 +33,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-
-# --- Multi-agent concurrency -------------------------------------------------
-# Append-style bookkeeping files. Several agents write these in parallel from
-# separate worktrees; union merge keeps BOTH sides of a conflicting hunk rather
-# than halting, so no lane can resolve a conflict by discarding another lane.
-# Cost: occasional duplicate lines to tidy. Never set this on code or lockfiles.
-memory.md         merge=union
-TODO.md           merge=union
-REGISTRY_TODO_PLANS.md merge=union

.github/dependabot.yml

@@ -1,36 +0,0 @@
-# ADR-0014 — automated dependency bump PRs. Active only when this repo is
-# mirrored to GitHub (see .github/workflows/security.yml for the deploy-model
-# note); the HF-only path relies on the weekly pip-audit run to surface stale,
-# vulnerable pins instead.
-version: 2
-updates:
-  - package-ecosystem: pip
-    directory: "/"
-    schedule:
-      interval: weekly
-    open-pull-requests-limit: 5
-    # gradio and mcp are pinned to the HF Space sdk_version and must not be
-    # bumped by an automated PR — bumping them requires a coordinated Space
-    # rebuild. Security advisories still surface via pip-audit.
-    ignore:
-      - dependency-name: gradio
-      - dependency-name: mcp
-
-  - package-ecosystem: github-actions
-    directory: "/"
-    schedule:
-      interval: weekly
-
-  - package-ecosystem: docker
-    directory: "/docker"
-    schedule:
-      interval: weekly
-    # The sandbox base image (ADR-0007) tracks a specific Python line; a major/
-    # minor jump (e.g. 3.11 -> 3.14, PR #3, closed 2026-07-08) can break the
-    # rpy2/scanpy/decoupler stack and is not validated by the Dockerfile scan.
-    # Allow patch bumps (security) but not major/minor without a manual build test.
-    ignore:
-      - dependency-name: python
-        update-types:
-          - "version-update:semver-major"
-          - "version-update:semver-minor"

.github/workflows/security.yml

@@ -1,61 +0,0 @@
-# ADR-0014 — CI security scan (dependencies, static analysis, secrets, image).
-#
-# NOTE ON THIS REPO'S DEPLOY MODEL: DecoupleRpy_Agent's `origin` is the
-# HuggingFace Space (git push builds the Space); HuggingFace does not run GitHub
-# Actions. This workflow therefore executes ONLY if the repo is also mirrored to
-# GitHub (or `biodata-registry`, which reuses this same file). In the HF-only
-# path the enforced check is the pre-push hook (`make install-hooks`) running the
-# identical scripts/security_scan.sh, plus the scheduled unattended run. Keeping
-# the workflow here means adding a GitHub mirror is zero extra work and the scan
-# definition never diverges between the two paths.
-name: security
-
-on:
-  pull_request:
-  push:
-    branches: [main]
-  schedule:
-    - cron: "0 6 * * 1"   # weekly, Monday 06:00 UTC — catches newly disclosed CVEs
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-jobs:
-  scan:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v7
-        with:
-          fetch-depth: 0   # gitleaks needs full history
-
-      - uses: actions/setup-python@v6
-        with:
-          python-version: "3.12"
-
-      - name: Install scanners
-        run: |
-          python -m pip install --upgrade pip
-          # uv drives the CycloneDX SBOM stage; without it that stage SKIPs, which
-          # is a failure under SECURITY_SCAN_STRICT=1.
-          pip install pip-audit bandit uv
-          # gitleaks + trivy via their official installers
-          curl -sSfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b "$HOME/.local/bin"
-          curl -sSfL https://github.com/gitleaks/gitleaks/releases/download/v8.30.1/gitleaks_8.30.1_linux_x64.tar.gz | tar -xz -C "$HOME/.local/bin" gitleaks
-          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
-
-      - name: Run shared security scan
-        env:
-          SECURITY_SCAN_STRICT: "1"   # in CI every scanner is present; a skip is a bug
-          # The GitHub runner has no R; rpy2 in API mode refuses to build without it,
-          # which breaks pip-audit's dependency resolve. ABI mode builds without R.
-          RPY2_CFFI_MODE: ABI
-        run: make security-scan
-
-      - name: Upload scan artifacts
-        if: always()
-        uses: actions/upload-artifact@v7
-        with:
-          name: security-scan-${{ github.run_id }}
-          path: security/
-          retention-days: 90

.gitignore

@@ -1,9 +1,5 @@
 .DS_Store
 .claude/
-# Local secrets — never commit (matches the other repos)
-.env
-.env.*
-!.env.example
 .venv/
 venv/
 __pycache__/
@@ -18,11 +14,3 @@ reports/
 upload_staging/
 upload_registered/
 run_logs/
-# ADR-0014 security scan — cached pip-audit venv (artifacts under security/ are
-# committed; the venv is not)
-security/.audit-venv/
-CLAUDE.md
-memory.md
-TODO.md
-docs/tasks/
-session_export.docx

.gitleaks.toml

@@ -1,50 +0,0 @@
-# Gitleaks config — ADR-0014 secret scan (shared across the three repos).
-#
-# Extends the upstream default ruleset (do not replace it) and adds allowlists
-# for this repo's known-safe matches: placeholder tokens in docs/tests, the
-# committed security scan artifacts, vendored virtualenvs, and two triaged false
-# positives. Real secrets (HF write tokens, the ADR-0012 service token) must
-# NEVER be committed — they live in HF Space secrets. This scan runs over the
-# working tree AND full git history (`gitleaks detect`), because tokens have
-# flowed through these repos.
-#
-# Run:  gitleaks detect --config .gitleaks.toml --redact --no-banner
-
-[extend]
-useDefault = true
-
-# Known-safe paths and placeholder strings.
-[[allowlists]]
-description = "known-safe paths and placeholder tokens"
-paths = [
-  '''\.venv/''',
-  '''(^|/)node_modules/''',
-  '''security/sbom\.json''',
-  '''security/pip-audit-.*\.(json|txt)''',
-  '''\.gitleaks\.toml''',
-]
-# Documentation and ADRs reference token *names* (research_agent_token,
-# ANTHROPIC_API_KEY) as identifiers, never their values.
-regexes = [
-  '''research_agent_token''',
-  '''ANTHROPIC_API_KEY''',
-  '''(?i)your[-_]?(hf|api|anthropic)[-_]?(token|key)[-_]?here''',
-  '''(?i)example[-_]?(token|key|secret)''',
-  '''xxx+|placeholder|dummy|fake[-_]?(token|key|secret)''',
-]
-stopwords = ["example", "placeholder", "changeme"]
-
-# --- False positives triaged 2026-07-02 (ADR-0014 first run) ---------------- #
-# The generic-api-key entropy rule fires on long snake_case keyword arguments in
-# the precompute build scripts (`gene_symbol_column=...`, `assignment_column=...`).
-# These are column names, not secrets. Matched against the finding text so a
-# genuinely new secret in the same file still surfaces.
-[[allowlists]]
-description = "column-name kwargs in precompute scripts (not secrets)"
-regexTarget = "match"
-regexes = ['''(gene_symbol|assignment)_column\s*=''']
-
-# Historical gcp-api-key in app.py (commit 155d8d7) was scrubbed from git history
-# on 2026-08-18 (value replaced with **REMOVED-KEY** in all blobs); the key was
-# triaged 2026-07-02 as non-functional. No value allowlist needed anymore.
-

.pre-commit-config.yaml

@@ -1,23 +0,0 @@
-# Pre-commit hooks — shared lint/format baseline across the PDAC-system repos.
-# Install once per clone:  pip install pre-commit && pre-commit install
-# Run on all files:        pre-commit run --all-files
-#
-# NOTE: this is separate from the ADR-0014 *security* pre-push hook
-# (scripts/hooks/pre-push, installed via `make install-hooks`). This one runs
-# ruff at commit time for style/correctness; that one runs the security scan at
-# push time.
-repos:
-  - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.15.20
-    hooks:
-      - id: ruff              # lint (uses ruff.toml / [tool.ruff]); --fix applies safe fixes
-        args: [--fix]
-      - id: ruff-format       # formatter (line-length + quote style from config)
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
-    hooks:
-      - id: end-of-file-fixer
-      - id: trailing-whitespace
-      - id: check-yaml
-      - id: check-added-large-files
-        args: [--maxkb=2048]

CLAUDE.md

@@ -0,0 +1,210 @@
+# DecoupleRpy Agent — Project Context
+
+This file provides stable architectural context for Claude Code sessions.
+**Do not encode current status here** — that belongs in `memory.md`.
+Update this file when architectural decisions change, not when code changes.
+
+---
+
+## Memory maintenance (after every commit)
+
+These status files drift unless updated at commit time. After any `git commit` in this
+repo — bound for prod (`origin`, the HF Space) or dev (`hf-dev`) — update whatever that
+commit changed, and skip what it didn't:
+
+- `memory.md` — current status / what just changed
+- `TODO.md` — move finished items to Done, add anything new
+- `/Users/annivoigt/Documents/GitHub/SHOWCASE_STATUS.md` — the cross-repo rollup; update especially on a prod/dev deploy
+- `CLAUDE.md` (this file) — only when the architecture itself changes (rare)
+
+A PostToolUse hook (`~/.claude/hooks/remind-memory-sync.py`, wired in
+`~/.claude/settings.json`) prints this checklist automatically after each commit. It only
+*reminds* — the edits are still done by hand.
+
+---
+
+## What This Project Is
+
+A **Paper2Agent conversion** of the [scverse/decoupler-py](https://github.com/scverse/decoupler-py) bioinformatics library. The goal is to expose decoupler's computational biology methods as MCP tools so non-coding scientists at the Brenden-Colson Center for Pancreatic Care (OHSU, Dr. Rosalie Sears lab) can run analyses via natural language.
+
+This repo is the **specialist agent** — it does the computation. It is called by a separate orchestrator (`research-coordinator`, deployed at `anne-voigt/research_coordinator` on HuggingFace).
+
+---
+
+## Two-Tier Architecture
+
+```
+User
+ └── Research Coordinator (HF Space: anne-voigt/research_coordinator)
+       ├── Claude API — conceptual/interpretive questions (direct)
+       └── gradio_client → DecoupleRpy Agent (HF Space: anne-voigt/Paper2Agent_decoupleRpy)
+                            └── MCP tools (this repo)
+```
+
+The coordinator classifies every user message and routes to the specialist for anything involving computation, datasets, or capability questions. The coordinator does NOT know what datasets are registered — it always asks the specialist.
+
+---
+
+## MCP Tool Layers
+
+Tools are organized in three layers:
+
+**Layer 1 — Tutorial tools** (`src/tools/`): Direct Paper2Agent output, one file per tutorial.
+- `rna.py` — bulk RNA analysis (DE, TF enrichment, pathway scoring, GEO loading)
+- `rna_sc.py` — single-cell RNA analysis
+- `rna_visium.py` — spatial transcriptomics (Visium)
+- `rna_pstime.py` — pseudotime analysis
+- `orthologs.py` — cross-species gene symbol translation
+- `dataset_tools.py` — dataset registry MCP tools (incl. `dataset_get_integration_plan`, the cross-dataset early/late/refuse planner)
+- `bulk_dataset_tools.py` — bulk-specific dataset operations
+- `integration_tools.py` — cross-dataset integration (`integration_mcp` sub-server): `decoupler_meta_analyze` (Mode B late integration) and `decoupler_normalization_concordance` (same-cohort sibling-variant sensitivity check — descriptive agreement, no combine; routed to by plan `mode="concordance"`)
+
+**Layer 2 — Generic workflows** (`src/workflows/`): Reusable logic called by Layer 1.
+- `geo.py` — GEO series matrix loading
+- `microarray.py` — probe collapse, data type detection
+- `activity_scoring.py` — TF/pathway activity inference
+- `activity_stats.py` — group comparisons on activity scores
+- `manifest_data_validation.py` — validates manifest semantics against loaded data
+- `metadata_validation.py` — metadata column checks
+- `survival.py` — survival analysis
+- `meta_analysis.py` — cross-dataset meta-analysis engine (`src/core/combine.py` result-envelope contract + field-based strategy dispatch: stouffer / inverse_variance / fisher; Cochran's Q + I²)
+- `concordance.py` — same-cohort normalization sensitivity engine: descriptive agreement across sibling-variant result envelopes (pairwise Pearson/Spearman on the shared effect, sign-concordance, effect spread, significant-call Jaccard). The non-combining counterpart to `meta_analysis.py`; backs `decoupler_normalization_concordance`.
+- `sanity_checks.py` — result-aware (post-compute) sanity checks (ADR-0002); pure functions emitting non-fatal `sanity_warnings` (effect-size plausibility, tissue-identity contamination, network membership). See "Two safety layers" below.
+
+**Layer 3 — Dataset manifests** (live in `biodata-registry` package): One YAML per dataset.
+Auto-discovered at import time by the registry from the installed `biodata_registry` package.
+The `src/datasets/manifests/` directory does **not** exist in this repo — `biodata-registry` is the
+sole manifest source. Zero code changes needed to add a dataset — add a YAML to `biodata-registry`
+and reinstall the package.
+
+---
+
+## Dataset Manifest System
+
+Each manifest is a YAML file validated against `DatasetManifest` (see `src/datasets/manifest_schema.py`).
+
+**Key controlled vocabularies:**
+- `modality`: `bulk_microarray`, `bulk_rnaseq`, `sc_rnaseq`, `spatial_rnaseq`, `proteomics`
+- `data_level`: `raw_counts`, `log_expression`, `log_ratio`, `normalized`, `tpm`, `fpkm`, `protein_abundance`
+- `feature_id_type`: `probe_id`, `gene_symbol`, `ensembl_gene_id`, `entrez_id`, `protein_id`
+- `expression_source.type`: `geo_series_matrix`, `geo_soft`, `url`, `gdc`, `cptac`, `local`, `h5ad` (hosted AnnData, single-cell/spatial — ADR-0006)
+
+**Analysis path routing** (`analysis_path`, derived **modality-first, then `data_level`**):
+- **Path P** — `sc_rnaseq` / `spatial_rnaseq` (checked **before** `data_level`, so an sc `raw_counts` h5ad is never mislabeled Path A) → load via `read_h5ad_cached` → pseudobulk → bulk DE / activity. **Never DESeq2/limma on per-cell counts.** (ADR-0006; aligns with biodata-registry 0.1.7's A/B/P.)
+- **Path A** — bulk `raw_counts` → DESeq2
+- **Path B** — bulk `log_expression` / `normalized` / etc. → limma or t-test
+
+`_build_loading_plan` (`src/tools/dataset_tools/_base.py`) emits a single-cell plan for Path P (load + per-cell scoring + a pseudobulk note) instead of the bulk contrast tail.
+
+**Probe collapse**: Required when `feature_id_type: probe_id` and `requires_collapse: true`. Uses GPL platform annotation downloaded from GEO.
+
+**Anti-hallucination grounding**: The live dataset registry is injected into the agent's Jinja2 system prompt at every call via `get_system_prompt()`. The agent may ONLY claim access to datasets returned by `dataset_list_available()` — never from training knowledge.
+
+---
+
+## Agent Implementation
+
+- **Framework**: LangGraph-based `CodeAgent` (`src/agent.py`)
+- **System prompt**: Jinja2 template rendered with `functions`, `packages`, and `datasets` at call time
+- **Prompt config**: `prompts.yaml` — coordinator system prompt, routing rules, available datasets section
+- **Routing**: Research coordinator uses a two-path routing prompt; capability/dataset questions must route to specialist
+- **MCP transport**: a single **persistent `server.py --transport http`** process is started once per
+  container by `GradioAgentUI.__init__` (`ensure_mcp_http_server()`), and tools are registered over HTTP
+  via `add_mcp_http(url)`. Every tool call reuses that resident process — it does NOT spawn a fresh
+  `python server.py` per call (the old stdio model, which re-imported rpy2/scanpy/decoupler, re-mounted
+  11 sub-servers, and re-read the h5ad on every call: ~44s/step). **stdio is kept as a health-checked
+  fallback** (`add_mcp(mcp_config.yaml)`) if the HTTP server fails to bind. A process-lifetime in-memory
+  AnnData cache (`src/cache.read_h5ad_cached`, keyed by path+mtime+size, returns copies so it's safe
+  under the shared server) means each h5ad is parsed once per process, not once per tool call. Live on
+  dev + prod since 2026-06-29.
+
+---
+
+## Key Design Decisions
+
+**Why manifests instead of hardcoding dataset logic in tools?**
+Manifests are the single source of truth for dataset semantics. They encode refusal rules, prohibited inferences, metadata column meanings, and loading path — keeping analysis tools generic and dataset-specific knowledge in one place.
+
+**Why route capability questions to the specialist?**
+`dataset_list_available()` returns runtime truth — a manifest with a YAML error won't load even if the file exists. The specialist's answer is richer (capabilities, limitations, metadata) than what code inspection would produce. The coordinator never answers dataset questions from its own knowledge.
+
+**Why DESeq2 for raw counts, limma/ttest for log-normalized?**
+DESeq2 expects integer counts and models dispersion — incorrect on pre-normalized data. Limma and t-test are appropriate for log-normalized expression values. The manifest's `data_level` field enforces the correct path automatically.
+
+**Why is the cross-dataset decision (early/late/refuse) made in the registry, not the agent?**
+Combining ≥2 datasets is gated by `get_integration_plan()` in `biodata-registry` (re-exposed as the agent tool `dataset_get_integration_plan`). It is a pure function of manifest metadata — `early` (pool with `dataset_id` as a batch covariate), `late` (meta-analyze per-dataset result envelopes), or `refuse` — keyed on `data_level`/`organism`/`feature_id_type`/declared contrasts, never on hardcoded dataset pairs. Keeping the decision in the registry makes it deterministic and the single source of truth; a contrast whose factor is absent from a cohort (e.g. a Bailey-subtype contrast against a cohort with no Bailey labels) is refused as `CONFOUNDED_DESIGN` rather than silently degrading to a single-dataset result. The agent acts on `mode` and surfaces `reason`. **Mode B (late / `decoupler_meta_analyze`) shipped 2026-06-19; Mode A (early pooling, T7–T9 — the `decoupler_integrate_datasets` tool, `src/workflows/integration.py`) shipped 2026-06-24 (`a7a8caa`) and is deployed to prod, so an `early` verdict now pools the datasets (feature intersection + a `batch` obs key = `dataset_id`) and runs ONE batch-aware DE with `dataset` modelled as a covariate (DESeq2 `~batch+factor` / limma `~batch+group`) — it no longer falls back to late. For per-sample activity scoring across pooled cohorts (no DE contrast), `decoupler_pool_cohorts` (ADR-0001 item 10, built 2026-06-25) ComBat-corrects the pooled matrix (`scanpy.pp.combat` on log-normalised values, keyed on `poolable_data_level`) before `dataset_score_bulk_samples`, so pooled scores are batch-corrected rather than cohort-confounded. ComBat is the scoring-path counterpart to the DE covariate route — applied only where there is no design matrix, and run without a biological covariate (caveat surfaced on the tool; it must never feed DE testing). (Live deploy status is tracked in `memory.md`, not here.)**
+
+**Same-cohort variants (sibling quantifications) — why they are NOT a meta-analysis.**
+The GSE205154 trio (`gse205154_sears` TPM / `gse205154_sears_counts` counts / `gse205154_sears_tmm` TMM) are the *same 289 samples* quantified three ways. Each manifest's header and refusal rules already forbid pooling them ("same samples in different units"), but until biodata-registry 0.1.6 that relationship was only prose — `get_integration_plan()` did not know about it, so asking to combine two siblings (e.g. TPM + TMM) hit the `data_level` gate (`tpm` ≠ `normalized`) and returned `late`, and the agent ran `decoupler_meta_analyze`. **That is the wrong tool**: meta-analysis assumes *independent* cohorts, so combining identical samples double-counts them — Stouffer inflates the combined score by ≈√2 and Cochran's Q/I² collapse to 0 by construction (zero heterogeneity is guaranteed, not evidence of robustness). The correct framing for cross-variant work is a **normalization concordance / sensitivity check** — descriptive agreement metrics (Pearson/Spearman on scores, sign-concordance, overlap of significant calls), never an inferential combine. **Encoded in `biodata-registry` 0.1.6:** the three sibling manifests carry a structured `cohort_id: gse205154` (+ `variant`), and `get_integration_plan` has a same-cohort gate (runs before the `data_level` gate) that returns `mode="concordance"` when the whole request is one cohort's variants, or refuses with `DUPLICATE_COHORT` when siblings are mixed with independent datasets — keeping the decision in the registry per ADR-0001. The agent-side **concordance routine** the new mode routes to is **built**: `decoupler_normalization_concordance` (`integration_mcp`), backed by `src/workflows/concordance.py` — it reports descriptive agreement (pairwise Pearson/Spearman, sign-concordance, effect spread, significant-call Jaccard) instead of calling `decoupler_meta_analyze`. The agent pins 0.1.6, so `get_integration_plan` returns `mode="concordance"` for same-cohort variants and the agent routes to `decoupler_normalization_concordance` rather than `late`/`decoupler_meta_analyze`. (Live deploy status is tracked in `memory.md`, not here.)
+
+**Two safety layers — why a result-aware sanity layer (ADR-0002) on top of the registry refusal engine?**
+The refusal engine above is **Layer 1**: a pure function of manifest *metadata*, run *before* any computation — it refuses what the data inventory can't support (survival without endpoints, DESeq2-on-TPM, a confounded design). It is blind to a whole class of failure that is only visible *in the results*: normal-tissue contamination of a biopsy (e.g. hepatocyte master-regulator TFs topping a liver-met contrast), an artefactual log2FC from duplicate-collapse, a requested "TF" with no regulon. `src/workflows/sanity_checks.py` is **Layer 2**: pure post-compute checks that emit non-fatal `sanity_warnings` the agent surfaces. Layer 1 *refuses* (you lack the data); Layer 2 *cautions* (the analysis ran, but the result is confounded/partly artefactual) — it never blocks or re-runs a result, because the contrast is often legitimate-but-confounded and refusal would over-block. Kept out of `biodata-registry` deliberately: contamination is data-dependent, not a metadata fact. (NB this Layer-1/Layer-2 *safety* axis is distinct from the Layer-1/2/3 *MCP-tool-organization* axis above.) Phase 1 wires the effect-size check into the DE tool and the membership check into the CollecTRI tool; contamination auto-firing in the enrichment tools and PROGENy/Hallmark parity are the ADR-0002 S2/S3 follow-ups.
+
+**Why keep GitHub and HF in sync?**
+The DecoupleRpy Agent HF Space auto-deploys from GitHub. The research-coordinator does NOT auto-sync — push directly to the HF Space repo using a write token when changes are needed.
+
+---
+
+## Testing Philosophy
+
+- Every new workflow should have a corresponding test in `tests/`
+- Integration tests use small synthetic data (20-60 samples, 50-100 genes) — fast, no network calls
+- Fixture tests (e.g., `test_moffitt_integration.py`) lock in known sample counts as regression anchors
+- Manifests are tested automatically by `test_dataset_registry.py` and `test_dataset_manifest_contract.py` on every run
+
+---
+
+## HuggingFace Deployment
+
+- **DecoupleRpy Agent Space**: `anne-voigt/Paper2Agent_decoupleRpy` — `origin` remote points directly to HF; deploy with `git push origin main`
+- **Research Coordinator Space**: `anne-voigt/research_coordinator` — does NOT auto-deploy; push directly using write token (`research_agent_token`)
+- **Dev Space**: `anne-voigt/Paper2Agent_decoupleRpy_dev` — tracked as `hf-dev` remote; push with `git push hf-dev main`
+- **Token name**: `research_agent_token` (rotate after any session where it appears in chat logs)
+
+---
+
+## Dataset Roadmap
+
+**Currently registered**: 19 manifests, all in `biodata-registry` (the agent pins
+the `0.1.5` wheel). This list is a *stable index* — the authoritative per-dataset
+validated table (platform, feature_id_type, survival, known issues, validation
+dates) lives in `biodata-registry/memory.md`. Do not re-encode validation status
+here; that is what drifted this list down to 9.
+
+*Bulk array — tumor vs normal / paired:* `gse71989_chen`, `gse62165_jiang`,
+`gse16515_mayo`, `gse28735_pdac`, `gse15471_badea`.
+
+*Bulk array — subtype / survival cohorts:* `gse71729_moffitt` (classical/basal),
+`gse17891_collisson` (Collisson subtypes), `paca_au_array` (Bailey 4-subtype),
+`puleo_2018` (Puleo 5-subtype + survival), `gse21501_stratford` (log-ratio),
+`gse57495` (survival), `gse50827_nones` (survival).
+
+*Bulk RNA-seq counts / RSEM-TPM:* `tcga_paad` (raw_counts, Path A/DESeq2),
+`paca_au_rnaseq` (Bailey RSEM, Path A), `paca_ca_rnaseq` (ICGC Canadian, ensembl),
+`cptac_pda` (RSEM TPM).
+
+*Sears GSE205154 sibling trio (same 289-sample FFPE cohort, three quantifications
+— Primary 218 / Met 71):* `gse205154_sears` (TPM, Path B), `gse205154_sears_counts`
+(est. counts, Path A/DESeq2), `gse205154_sears_tmm` (edgeR TMM, Path B). These three
+are **the same samples in different units** — each manifest's refusal rules forbid
+pooling them. The only valid cross-variant operation is a **normalization
+concordance / sensitivity check**, never an integration or meta-analysis (which
+would double-count the cohort). See "Same-cohort variants" below.
+
+**h5ad files** are hosted at `anne-voigt/pdac-research-data` on HuggingFace
+(migrated from `anni-voigt` 2026-06-12; the GSE205154 trio uploaded 2026-06-22).
+
+**Priority next datasets**:
+1. TCGA-PAAD Moffitt subtypes — classical/basal classification not in GDC/Xena clinical matrix; requires inference step
+2. Bailey et al. 2016 WGS data — somatic mutation landscape (separate from expression)
+3. Structured same-cohort-variant relationship (`cohort_id` field + integration mode) — see "Same-cohort variants" below
+
+**Before adding any new dataset**: run `dataset_validate_manifest_against_data` to confirm manifest semantics match the actual file.
+
+---
+
+## Consultant Context
+
+- **Client**: Dr. Rosalie Sears, Brenden-Colson Center for Pancreatic Care, Knight Cancer Institute, OHSU
+- **Goal**: Make computational biology methods accessible to non-coding scientists via natural language
+- **Scope**: Project 1 of 3 in the consulting engagement (Paper2Agent methodology replication & deployment)

Makefile

@@ -1,28 +0,0 @@
-# DecoupleRpy_Agent — developer entry points. The security targets are the
-# shared surface ADR-0014 standardizes across the three repos.
-.PHONY: security-scan security-baseline install-hooks help
-
-help:
-	@echo "make security-scan      Run the ADR-0014 scan (pip-audit, bandit, gitleaks, trivy)."
-	@echo "make security-baseline  Regenerate the accepted-findings bandit baseline."
-	@echo "make install-hooks      Install the pre-push security hook into .git/hooks."
-
-# ADR-0014: the one shared scan definition. CI, the pre-push hook, and a manual
-# run all call this so the check never drifts between entry points.
-security-scan:
-	@bash scripts/security_scan.sh
-
-# Regenerate the accepted-findings baseline after intentionally adding a new
-# suppressed finding. Review the diff before committing — a new entry is a new
-# accepted suppression and must be justified in security/ACCEPTED-FINDINGS.md.
-security-baseline:
-	@if command -v bandit >/dev/null 2>&1; then B="bandit"; else B="uvx bandit"; fi; \
-	$$B -r src/ -ll -c bandit.yaml -f json -o security/bandit-baseline.json -q; \
-	echo "Wrote security/bandit-baseline.json — review the diff and document any new suppression."
-
-# Wire the pre-push hook. Git hooks are not committed into .git/, so this copies
-# the tracked script into place (idempotent).
-install-hooks:
-	@cp scripts/hooks/pre-push .git/hooks/pre-push
-	@chmod +x .git/hooks/pre-push
-	@echo "Installed .git/hooks/pre-push — security scan now runs before every push."

README.md

@@ -7,45 +7,6 @@ sdk: gradio
 sdk_version: 6.18.0
 app_file: app.py
 pinned: false
-hf_oauth: true
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
-
-# DecoupleRpy Agent
-
-The computation specialist for the PDAC system — a LangGraph CodeAgent exposing
-~52 decoupler-py bioinformatics tools over MCP. Architecture is in `CLAUDE.md`;
-status in `memory.md`.
-
-## Security-review orientation
-
-The security-relevant entry points, in priority order:
-
-- **User-upload gate (`src/uploads/`)** — the ADR-0011 pipeline for
-  session-scoped file uploads, wired into the UI by `run_upload_gate()` in
-  `gradio_ui.py`. Three **mandatory** gates (de-identification attestation →
-  `stage_upload`: type-allowlist / size-cap / quarantine dir / SHA-256 →
-  `scan_upload`: magic-byte structural check, plus an AV pass only where the
-  deployment declares one) plus an **advisory** `validate_upload`. What a given
-  deployment gets is declared in `deploy/scan_posture.yaml` (read by
-  `src/uploads/posture.py`), not inferred from the host PATH: the HF Spaces run
-  `structural_only` and are therefore **not** malware-scanned, while
-  `av_required` fails closed and must only be set where a working AV exists.
-  Files are
-  only ever opened with vetted loaders (`scanpy.read_h5ad`, `pandas.read_csv`) —
-  never pickle/eval/exec — and quarantined uploads never enter a served path.
-- **MCP tool dispatch (`src/managers/tools/mcp_manager.py`)** — tools are
-  registered over HTTP (resident server) or stdio; `_parse_mcp_content` is the
-  untrusted-output boundary (raises on `isError` rather than passing error text
-  downstream). `_resolve_env_vars` does read-only literal `${VAR}` substitution
-  (no shell). The tool surface itself is mounted in `server.py`.
-- **Secrets + redaction** — `ANTHROPIC_API_KEY` / `HF_TOKEN` /
-  `decouplerpy_results_token` are read once at lazy init and never logged; AWS
-  creds come from the instance role, not env. `src/core/trace_redaction.py`
-  scrubs secret-shaped strings (and whole env dumps) from every trace before it
-  reaches any sink (file / HF / S3).
-- **Access control (`src/core/access_control.py`)** — `ADMIN_IDS` / `ALLOWED_IDS`
-  gate admin-only actions (e.g. registering an upload into the registry);
-  fail-closed on unknown identity; the resolved principal is recorded in the
-  audit trace (ADR-0012).

TODO.md

@@ -0,0 +1,486 @@
+# DecoupleRpy_Agent — TODO
+
+Canonical open-items list for this repo. Session history + detailed write-ups
+stay in `memory.md`; this file is the scannable backlog. Convention: each item
+is `High` / `Med` / `Low`. Move finished items to `## Done` with a date.
+
+Routing rule for this system: manifest/dataset-semantics → biodata-registry ·
+computation/tools/loaders → here · routing/coordinator/eval →
+pdac-analysis-orchestrator.
+
+---
+
+## Open
+
+### Deploy / infra
+
+- **Done (2026-06-29) — Salvage branch `claude/magical-banach-a2d929` ported + dropped.** The Jun-22
+  `wip(salvage)` commit (`8efd1be`) was reviewed file-by-file: **all 10 files already superseded** on
+  current `main` + biodata-registry (writable-dir `_resolve_dir`, hf_storage structured save,
+  `prohibited_inferences`, Moffitt prompt examples + 89/36 golden test, txt/docx/pdf export, stroma
+  contrasts). Nothing to port — porting would only re-add stale code on deleted paths. Branch deleted
+  (local + `origin` + defunct worktree); `8efd1be` recoverable via reflog. Detail in memory.md 2026-06-29.
+
+- **Done (2026-07-01) — Always-on audit log sink (ADR-0008) MERGED + deployed to prod.** `src/logging_sink.py`
+  (configurable `LOG_SINK=local|hf|s3`, default `local`) + always-on `persist_trace_safe` wiring in
+  `agent.py`/`gradio_ui.py` merged to `main` and pushed to the prod Space. Prod posture: `LOG_SINK=hf`.
+
+- **Med — Finish the S3 log-sink writer (AWS migration, ADR-0009).** `src/logging_sink.py` ships a
+  configurable trace sink (`LOG_SINK=local|hf|s3`, default `local`). ADR-0009 Phase 1 implemented the
+  lazy-boto3 `S3LogSink.persist_trace` (`put_object` to `${LOG_SINK_S3_PREFIX}/<run_id>/trace.json`);
+  `scripts/verify_s3_sink.py` is the live-verify script. **Remaining for the OHSU-managed AWS cutover:**
+  provision the bucket/IAM per ADR-0009 Appendix A, wire `LOG_SINK_S3_BUCKET`/`_REGION`/`_PREFIX` +
+  role creds as Space config, add `boto3` to requirements (optional/lazy), then set `LOG_SINK=s3`.
+
+- **Med — Quantify the MCP HTTP perf win (DEPLOYED to dev + prod `58b961f`, 2026-06-29).** The
+  resident `server.py --transport http` is live on both Spaces — tool calls reuse one process instead
+  of spawning a subprocess per call (was the ~44s/step tax); confirmed via run logs (`Added 52 remote
+  MCP tools`, no Pre-warm failed). Adds a process-lifetime in-memory adata cache + `[perf]`
+  instrumentation; stdio kept as fallback. **Remaining:** run a real multi-step DE question (needs API
+  credits) to capture the actual wall-clock % cut (expected 30–60% on a ~29-min run) from the `[perf]`
+  lines now in the logs.
+- **High — Bring PROD specialist live on gradio 6.18.** Migration is **pushed to
+  prod** (`origin/main` = `f098e67`, 2026-06-22); dev is RUNNING on the same code.
+  Prod Space is still PAUSED on the cpu-basic quota (3/3: dev specialist + dev
+  orchestrator + `bcc-lit-agent`). **Remaining: free one slot (e.g. pause
+  `bcc-lit-agent`) + unpause/restart the prod Space.** (User opted not to pause
+  anything yet — do this when ready.) If the prod build hits the same HF rollout
+  wedge dev did, a pause→unpause clears it.
+- **High — Top up the dev Space `ANTHROPIC_API_KEY` credits.** "Credit balance too low"
+  blocks the agent from finishing any multi-step run on dev — the concordance end-to-end
+  output couldn't be captured. Top up, then re-run a sibling-variant concordance query
+  for the final numbers.
+- **High — Promote the loader-auth + concordance stack to PROD.** On `main`/dev only
+  (`ce5168d`, `04b26ba`; the 0.1.6 concordance gate). Prod promotion = push `origin` →
+  **factory-rebuild prod** (a *normal* rebuild kept a stale pre-0.1.6 install on dev — a
+  factory rebuild was required) → set the prod `HF_TOKEN` secret (read
+  `pdac-research-data`) → ensure prod's Anthropic key has credits → re-validate. **2026-06-26
+  update: mostly DONE** — loader-auth live on prod (`5c5ca74`/`ce889e5`), prod `HF_TOKEN` CONFIRMED
+  set (user), Anthropic credits topped, XDI suite re-validated **4/4 end-to-end on prod**
+  (`20260626_131117`). Remaining: verify the 0.1.6 concordance factory-rebuild actually took on prod.
+- **Done (2026-06-26) — Metadata tools loader-auth gap fixed → dev** (`ce889e5`).
+  `dataset_count_metadata_values` / `dataset_crosstab_metadata_values` /
+  `dataset_validate_manifest_against_data` were MISSED in the ce5168d/04b26ba
+  centralization — they did `Path(adata_path).exists()` on the raw arg, so a private
+  h5ad URL returned "File not found" and never authenticated. Surfaced live: the XDI-002
+  late eval stalled here (agent fell back to unauthenticated urllib that hung on the
+  private repo). All three now route through `resolve_to_local_path`. Verified locally
+  (loads private paca_au_rnaseq.h5ad, real Bailey counts). **Dev only**; fold into the
+  prod loader-auth promotion below. **Follow-up: reconcile stale early-branch prompt text**
+  ("no batch-aware activity-scoring tool") — `81f5db6` added `decoupler_pool_cohorts`.
+- **Done (2026-06-24) — Private-data 401 fixed; loader auth centralized.** One
+  authenticated resolver `src/core/data_io.resolve_to_local_path`; `decoupler_differential_expression`
+  + Mode A `_resolve_to_local` + `bulk_dataset_tools._resolve_to_local_path` delegate
+  (`ce5168d` → `04b26ba`). Validated on dev (no 401/TaskGroup; resolver downloads the real
+  85.4 MB private h5ad). **`HF_TOKEN` Space secret CONFIRMED set on prod (user, 2026-06-26)** —
+  prod loads private data end-to-end (XDI suite 4/4, `20260626_131117`).
+
+### Cross-dataset integration (ADR-0001)
+
+Combine ≥2 datasets — early (pool + batch covariate) / late (meta-analysis) /
+refuse, driven by registry metadata. ADR + plan + per-task spin-off prompts in
+`docs/adr/ADR-0001-*.md`. All work on branch `adr-0001-phase-0` (off `main`).
+
+- **Done (2026-06-25, later) — Step-count efficiency prompt edits → dev** (`cdfee6f`,
+  `prompts.yaml`; no code). Cuts wasted generate→execute round-trips in long /
+  cross-dataset runs (diagnosis: ~90% of ~35-40 min is the serialized step loop).
+  (1) Reporting-Results rule 2 ("Re-read before you report") → a SINGLE read of an
+  ALREADY-SAVED CSV; forbids recompute/re-derive/re-save and a repeated re-read
+  (killed XDI-001's ~3 redundant end steps). (2) Cross-dataset step 1 → pass
+  design_factor ONLY for a named two-group contrast; pooled/per-sample "across all
+  samples" with no contrast calls `dataset_get_integration_plan([...])` with NO
+  design_factor (stops XDI-001's refuse-then-retry). With-contrast refusal guidance
+  unchanged. YAML verified; **dev + PROD** (`hf-dev` + `origin`; on prod 2026-06-26 via `81f5db6`).
+  **VALIDATED on PROD 2026-06-26** (XDI suite re-run `20260626_131117`, 4/4 PASS): XDI-001 early
+  pooled batch-aware DE (249 samples), no redundant recompute/re-read end-steps observed. Clean
+  step-count A/B not possible (eval bank revised since the pre-fix run — XDI-001 is now Moffitt+Puleo).
+- **Done (2026-06-25) — Late-integration prompt hardened → dev + PROD** (`0776267`,
+  `prompts.yaml`). Late requests must run the shared contrast per cohort then combine with
+  `decoupler_meta_analyze` and ALWAYS report Cochran's Q / I^2 — no hand-rolled
+  sign-concordance/intersection (fixes the 2026-06-25 eval miss). Early branch refreshed
+  to route contrasts to `decoupler_integrate_datasets`; clarifies no batch-aware
+  activity-scoring tool (flag pooled activity as batch-confounded). YAML+Jinja verified.
+  Deployed dev (`hf-dev`, `0776267`) then **PROD** (`origin`, `791bb29`→`211cc90`, clean
+  ff; only runtime delta is `prompts.yaml`, no re-pin so no factory-rebuild needed).
+  **VALIDATED 2026-06-26 (XDI-002 re-run, dev `f01f56b`, PASS):** agent calls
+  `decoupler_meta_analyze` (Stouffer, 91 TFs) and reports per-feature Cochran's Q / I^2,
+  flagging CDX2 (I^2=72.9%) + HOXD3 (51.3%) as heterogeneous — no hand-rolled concordance.
+  Needed the `ce889e5` metadata loader-auth fix to get there. **Promoted to PROD 2026-06-26**
+  (`origin` `5c5ca74`): metadata loader fix `ce889e5` + prompt point-3 reconcile `5c5ca74`
+  (pooled scoring via `decoupler_pool_cohorts` = ComBat-corrected, not confounded). Late
+  meta-analyze prompt fix + loader fix now both live on prod. **Re-validated end-to-end on PROD
+  2026-06-26** (XDI-002 in suite `20260626_131117`, PASS): real `decoupler_meta_analyze` (Stouffer)
+  + per-TF Q/I² computed on prod, no hand-rolled concordance.
+- **Done — Same-cohort concordance routine** (committed `b36d5ce`; on `main` via
+  merge `0ac610f`; deployed to **dev** `hf-dev`). `decoupler_normalization_concordance`
+  (`integration_mcp`) + `src/workflows/concordance.py` — the agent-side routine for
+  the registry's `mode="concordance"` (sibling variants). Descriptive agreement
+  (Pearson/Spearman, sign-concordance, effect spread, significant-call Jaccard); no
+  combine. Tests: `test_concordance.py` 9 pass; `test_concordance_tool.py` smoke-validated.
+  **Now live in prod:** biodata-registry 0.1.6 wheel + re-pin (`fb2091e`) **deployed**
+  (`origin/main` = `fb2091e`); the plan returns `concordance` live. Dev confirmed RUNNING
+  on 0.1.6; prod rebuilt on the push.
+- **Done — Phase 0.** `src/core/combine.py` (envelope contract + strategy
+  registry) + decision-matrix spec, committed (`6fa0b12`, `3fd9d0a`).
+- **Done — T1: `combine` descriptors on the 4 rna tools** (`src/tools/rna/analysis.py`),
+  committed (`42d31b1`). All four tools declare a Mode-B `combine` descriptor +
+  matching `@combinable` marker; `tests/test_combine_conformance.py` added (24 tests).
+- **Done — T3: `src/workflows/meta_analysis.py`** (Phase 1, step 4), committed
+  (`bb944d5`). Four strategies register into `combine.py` (inverse_variance >
+  stouffer > fisher; rank_aggregation opt-in), `compare_activity_by_group`→envelope
+  adapter with the D4 SE-of-Cohen's-d backfill, Cochran's Q / I², and
+  `combine_envelopes()` (align → field-dispatch → BH). `tests/test_meta_analysis.py`
+  (34 tests). Also completed the `src/core/__init__` re-export surface (additive).
+  Engine references no tool by name. **Branch-local; not deployed.**
+- **Done — T2: registry `get_integration_plan`** (biodata-registry) — decision
+  engine + 5th MCP tool. Merged + released as **biodata-registry 0.1.2**
+  (2026-06-19). Pin ready for T4:
+  `.../resolve/cbc083a5cd9dbe79e6740a6b64c4dc8c0639f113/biodata_registry-0.1.2-py3-none-any.whl`
+  (sha256 `607a14b0…`).
+- **Done — T4: agent `dataset_get_integration_plan` wrapper + 0.1.2 re-pin**
+  (commit `a85e426`, branch-local). Thin `get_integration_plan` passthrough in
+  `src/datasets/registry.py` + `@dataset_mcp.tool dataset_get_integration_plan`
+  in `catalog.py` (early/late/refuse; `{error}` on unknown ids; forwards
+  contrast args to the confound gate). `requirements.txt`/`.in` → 0.1.2
+  (`cbc083a`). Server 50→51 tools. `tests/test_integration_plan_tool.py` (7).
+- **Done — T5: `decoupler_meta_analyze` tool + A→B→refuse wiring** (commit
+  `c45a617`, branch-local). New `integration_mcp` sub-server +
+  `decoupler_meta_analyze` (envelope read → `combine_envelopes` → combined CSV +
+  Q/I²; refuses mismatched result_type/contrast). prompts.yaml wires
+  early→late-fallback / late / refuse. Server 51→52 tools.
+  `tests/test_meta_analyze_tool.py` (9). **Mode B (Phase 1) complete end to end.**
+- **Done — T6: release + dev deploy + e2e validation** (2026-06-19). Shipped Mode B
+  to **prod** (`origin` @ `c54b550`, 50→52 tools). Dev validation found Case 3
+  returned plan `early` not `refuse`; hardened the registry confound gate
+  (**biodata-registry 0.1.3**, `5654e86`: a cohort that can supply neither arm of a
+  specified contrast → CONFOUNDED_DESIGN) + a prompt nudge to pass contrast args.
+  4-case gradio_client eval on dev: refuse ✅ / early→late-fallback no-fabrication ✅
+  / late no-pooling ✅ / `decoupler_meta_analyze` positive path ✅ (paca_au_rnaseq +
+  paca_au_array). Q/I² reporting polish (`c54b550`). registry re-pinned 0.1.3.
+- **Done — T7: `src/workflows/integration.py`** combined-AnnData builder (gene-symbol
+  intersect + `batch` obs key = dataset_id); pure `combine_anndatas` + loader
+  `build_combined_anndata`. Branch `adr-0001-phase-2-mode-a` (`a7a8caa`); **deployed
+  to dev + prod 2026-06-24** (`5cb2737`, code-only rebuild).
+- **Done — T8: `batch_column` on the DE tool** — DESeq2 `~batch + factor`; new
+  `run_limma_covariate` (`~batch + group`) in microarray.py; ttest+batch refused (no
+  silent covariate drop). Back-compat default off. (`a7a8caa`.)
+- **Done — T9: `decoupler_integrate_datasets`** (3rd integration_mcp tool, server
+  52→53) — pools + one batch-aware DE only on plan mode=="early"; refuses/reroutes for
+  late/concordance/refuse; auto-picks deseq2 (raw counts) / limma. (`a7a8caa`.) v1
+  limits: gene-symbol axis only (no probe collapse/ortholog).
+- **Done — item 10: `decoupler_pool_cohorts`** (4th integration_mcp tool, server
+  54→55) — ComBat for the per-sample **scoring** path (the no-design-matrix counterpart
+  to T9's covariate route). Pools + `batch_correct_for_scoring` (`scanpy.pp.combat` on
+  the log-normalised matrix, keyed on `poolable_data_level`; standard ComBat, NOT
+  ComBat-seq → **no new dependency**) → one batch-corrected matrix for
+  `dataset_score_bulk_samples`. Same early-only plan gate. `src/workflows/integration.py`
+  + `tests/test_pool_cohorts_tool.py` (13 tests). Prompt early-branch rewired to call it
+  (was "no batch-aware scoring tool / flag confounded"). 2026-06-25 dev; **promoted to PROD 2026-06-26** (`origin` `81f5db6`).
+- **Low — T10: orchestrator routing + capability + reporting rules** (pdac-analysis-orchestrator).
+- **Low — T11: cross-dataset evals** (pool/fallback/refuse; needs T5/T9) — now unblocked (T9 done).
+
+### ADR housekeeping (reconciled 2026-06-24)
+
+ADR-0003/0004/0005 moved **Proposed → Accepted**; ADR-0001 checkboxes ticked to shipped
+reality (Mode B / concordance). Remaining open action items:
+
+- **Low — ADR-0003:** add a "Dev Mode inner loop" note to
+  `pdac-analysis-orchestrator/DEPLOYMENT.md` (sibling repo); set up VS Code/SSH config.
+  *(Dev Mode itself is enabled on both dev Spaces.)*
+- **Low — ADR-0005:** add a "promote to public on publication" step to the
+  dataset-onboarding checklist; record the storage posture in `biodata-registry/memory.md`.
+  *(`pdac-research-data` is already private.)*
+
+### Result-aware sanity layer (ADR-0002)
+
+Post-compute Layer-2 cautions (never refuse/block) catching what the metadata-only
+refusal engine can't: contamination, artefactual log2FC, non-TF "TFs". ADR +
+spin-off prompts in `docs/adr/ADR-0002-*.md`. `sanity_warnings` is the shared
+additive return contract S2/S3 extend.
+
+- **Done — Phase 1: `src/workflows/sanity_checks.py` + tests + DE/TF wiring**
+  (branch `feat/adr-0002-sanity-layer`, not pushed). 3 checks + `run_sanity_checks()` aggregator;
+  `tests/test_sanity_checks.py` (16); additive guarded `sanity_warnings` key on
+  `decoupler_differential_expression` (effect-size) +
+  `decoupler_tf_enrichment_collectri` (membership).
+- **Done — S1: CI verification of the additive key + memory sync** (this session).
+  Additive key breaks nothing (`tests/` 877 passed / 55 skipped); added DE + TF
+  `sanity_warnings` shape assertions to `tests/test_tool_response_schemas.py`
+  (9→11). Closes the ADR's two schema/registry caveats. **Committed on
+  `feat/adr-0002-sanity-layer`, not pushed.**
+- **Done — S2: Phase 2 — thread cohort tissues into the enrichment tools + auto-fire
+  the contamination check** (2026-06-22; committed on `feat/adr-0002-sanity-layer`,
+  not pushed). Optional `cohort_tissues` / `home_tissue` params on all three
+  enrichment tools (default `None` → check skipped, back-compat).
+  `check_tissue_identity_contamination` runs on the ranked output and is **merged
+  into the same `run_sanity_checks` report** as S3's membership warnings (CollecTRI
+  ranks TFs by |activity|, `feature_kind="tf"`; PROGENy/Hallmark run on the input DE
+  genes by |stat|, `feature_kind="gene"`, since pathway/gene-set names aren't in the
+  marker registry). Added a `prompts.yaml` rule to foreground `critical` warnings.
+  New tests `tests/test_enrichment_contamination_wiring.py` (9). `test_tool_registry.py`
+  needed no change (tests the ToolRegistry abstraction, not real tool signatures).
+- **Done — S3: parity — `sanity_warnings` (membership check) on PROGENy + Hallmark**
+  (2026-06-22; committed on `feat/adr-0002-sanity-layer` together with S2 as the
+  ADR-0002 sanity layer). Validates requested pathway/gene-set names against the
+  resource's source names (`known_non_tf=set()`); `run_sanity_checks` gained a
+  `known_non_tf` passthrough; parity tests in `tests/test_tool_response_schemas.py`
+  (`TestEnrichmentSanityParity`). The TF tool keeps membership **and** contamination
+  warnings merged (reconciled with S2, not overwritten).
+
+### PROGENy per-sample scoring (from GSE205154_sears eval, 2026-06-22)
+
+Found while reviewing an agent run for "score PROGENy for every sample in
+gse205154_sears and show the cohort-wide landscape." Analysis/routing were
+correct (single-dataset, Path B, ULM, 98.3% coverage); these are the gaps.
+
+- **High — Per-sample scoring path returns no retrievable figure.**
+  `dataset_score_bulk_samples` writes only the two CSVs to `OUTPUT_DIR`
+  (`tmp/outputs/`). Any "landscape" plot is agent-authored matplotlib that lands
+  wherever the agent chooses — observed at `/tmp/..._landscape.png`, *outside*
+  `OUTPUT_DIR` — so the end-of-run `tmp/outputs` inline-plot sweep never embeds
+  it and "show me the landscape" silently returns no image. The run's only link
+  was mislabeled "Saved run log" but pointed at the raw `.h5ad`. Fix: either (a)
+  have the tool emit a standard landscape figure (sample heatmap + mean±SD bar)
+  to `OUTPUT_DIR` as a declared artifact, or (b) add a prompts.yaml rule that
+  agent-authored plots must be written to `OUTPUT_DIR`.
+- **High — Confirm/enforce log-scale before ULM on the per-sample path.**
+  PROGENy/ULM assume roughly symmetric, log-scale input (tool docstring says
+  "log2-TPM"). The run loaded "TPM" and never stated a log2 transform was
+  applied; if *linear* TPM reaches ULM, a few high-expression genes dominate and
+  scores distort. Fix: add an input-scale heuristic in
+  `score_bulk_samples_with_decoupler` (warn or auto-log when values look linear:
+  large max / right-skew / non-log range) and/or assert the manifest `data_level`
+  is a log level; surface the applied transform in the return dict either way.
+- **Med — `DECOUPLER_DISCLAIMER` misdescribes the per-sample path.**
+  `src/core/constants.py` says values are "derived from differential-expression
+  statistics via the ULM model" — false for `dataset_score_bulk_samples`, which
+  scores the expression matrix directly (its own docstring: scores "without first
+  computing DE statistics"). Split the disclaimer into per-sample vs DE-based
+  wording (or parameterize) so the scoring tool emits the correct caveat.
+- **Med — Pin and record the PROGENy footprint.**
+  `dc.op.progeny()` is called with defaults; the run reported
+  `n_network_genes=17,610` (full model), and decoupler's default `top` has
+  shifted across versions — so scores aren't reproducible across upgrades. Pin
+  `top` explicitly in `score_bulk_samples_with_decoupler` and echo it in the
+  return dict.
+- **Low — `-F` (fibroblast) samples can't be excluded.** The 7 `-F` samples are
+  annotated `Primary` in GEO; per-sample scoring includes them and can inflate
+  stromal pathways (TGFb/NFkB). Consider a documented sample-filter hook
+  (agent-side) or a manifest note so they can be optionally excluded/segmented.
+  (Manifest-note half coordinates with biodata-registry.)
+
+### UI / output (Gradio)
+
+- **Low — Port the UI fixes to pdac-analysis-orchestrator.** The download
+  truncation, inline-plot, and full-run-PDF fixes landed here 2026-06-17 (see
+  Done). The same class of bug likely exists in the orchestrator's UI (untested
+  as of 2026-06-17) — reuse the same approach there.
+
+### GUI / observability
+
+- **Med — Tool/dataset call tracking panel** (was Known Gaps #9). Collapsible
+  "what happened" panel beneath each response showing which MCP tools ran and
+  which datasets were accessed. LangGraph already returns intermediate steps;
+  capture tool names + dataset IDs from step metadata and render an expandable
+  panel. Helps non-coding scientists see what the agent did.
+
+### Data / manifests
+
+- **Med — `roadmap` key in manifests** (was Known Gaps #10). Add a per-manifest
+  `roadmap` list in biodata-registry + a `scripts/collect_roadmap.py` that
+  prints a consolidated cross-dataset list of open items. (Coordinate with
+  biodata-registry/TODO.md.)
+- **Med — Daily URL health check** (was Known Gaps #11). Scheduled script that
+  pings each manifest's `expression_source` / `metadata_source` URLs, logs HTTP
+  status to `url_health.json`, and a startup banner in `gradio_ui.py` warns on
+  any non-2xx. Motivated by the TCGA-PAAD clinical URL 403.
+
+### Tool development & validation (from PI research to-do list, 2026-06-26)
+
+- **Med — Add the PURIST subtype operation to the toolset.** Wire PURIST
+  (single-sample basal/classical PDAC classifier) as an agent tool, then support
+  the basal-vs-classical comparison that contrasts **PURIST vs single-cell**
+  methodology. (Pairs with the Loveless single-cell ingestion in
+  biodata-registry/TODO.md.)
+- **Med — Hallmark Shiny app (Carl Pelz).** Build the Shiny / Hallmark-genes
+  tool. Review/scoping meeting with Carl Pelz is scheduled — capture his
+  requirements (which Hallmark gene sets, inputs, expected outputs) before building.
+- **Med — Single-cell RNA-seq tool: cell annotation + UMAP + thresholds.**
+  Build on `rna_sc.py` to expose the cell-annotation feature, render a UMAP, and
+  let the user specify thresholds. Carry the existing caveats: analysis works only
+  within cluster groups, and pseudo-bulk single-cell is untrustworthy for certain
+  cell types.
+- **High — Loveless single-cell serving & runtime design (agent side).**
+  *Started 2026-06-29 — design accepted as [`ADR-0006`](docs/adr/ADR-0006-loveless-single-cell-serving.md);
+  agent-side scaffold landed ahead of the biodata-registry ingestion.*
+  - **Done (scaffold, synthetic-tested):** custom-signature bulk fast path
+    `dataset_score_signature` (Role 2) + `src/workflows/signatures.py` +
+    `score_bulk_samples_with_decoupler` network seam (custom resource label no
+    longer gated when a signature net is supplied); `src/tools/rna_sc.py` now
+    loads via `read_h5ad_cached` (Role 1 cache contract). **ADR-0006 #6 done:**
+    `h5ad` `expression_source.type` + Path **P** (`analysis_path` derived
+    modality-first; sc/spatial → P, never mislabeled Path A) wired into
+    `_build_loading_plan` (single-cell plan, no bulk DESeq2/limma tail) +
+    `manifest_schema` (matches biodata-registry 0.1.7 A/B/P). Tests:
+    `tests/test_signatures.py`, `tests/test_loading_plan_h5ad.py` (all green;
+    `test_activity_scoring.py` updated for the relaxed resource gate).
+  - **Also done:** sc loaders (`rna_sc.py::_load_adata`) resolve a hosted/private
+    h5ad URL→local path via the shared authenticated resolver
+    (`resolve_to_local_path`, `HF_TOKEN`) — a `pdac-research-data` h5ad now loads
+    end-to-end (`tests/test_rna_sc_loader.py`).
+  - **Artifacts LANDED (biodata-registry 0.1.8, 2026-07-01) + integrated:** two
+    Loveless-atlas sc subsets — `gse155698_steele` (GSE155698) + `gse205013_werba`
+    (GSE205013), `modality: sc_rnaseq`/`raw_counts`, `expression_source.type: url`
+    → hosted `.h5ad`) + the `CROSS_RESOLUTION` gate. Agent re-pinned 0.1.8 on this
+    branch; both route `analysis_path=P` and produce a correct sc loading plan
+    (load via `decoupler_load_and_visualize_data`, NOT the bulk url loader).
+    Fixed `_build_loading_plan` to key Path P on **modality** (their type is `url`,
+    not `h5ad`); regression test in `tests/test_loading_plan_h5ad.py`.
+  - **Role-2 signature artifact PUBLISHED (2026-07-01):** 14 per-cell-type marker
+    signatures derived from the Steele subset (`rank_genes_groups` on `Clusters`)
+    → `loveless/signatures/gse155698_steele_celltype_signatures.csv` on
+    `pdac-research-data`. Script: `biodata-registry/scripts/ingest/loveless/derive_signatures.py`.
+    `dataset_score_signature` scores it end-to-end (66.6% coverage on a synthetic
+    cohort); `load_signature_net` + `resolve_to_local_path` now fetch a private
+    signature URL with auth (env token OR cached `huggingface-cli login`).
+  - **Done + DEPLOYED to PROD (2026-07-01) — `decoupler_load_and_visualize_data`
+    hardened for subsets lacking a precomputed UMAP/leiden** (was spun off →
+    `task_8b2a1bdc`). Merged via HF PR #1 into prod `main` (`6cda24b`→`abc16da`) +
+    **prod factory-rebooted** (folds in the pending 0.1.8 re-pin rebuild). Remaining:
+    confirm prod RUNNING post-reboot + e2e-verify the two Loveless sc datasets load +
+    produce a UMAP on prod. `_ensure_umap` reuses an existing embedding (pbmc3k unchanged) or
+    computes a bounded `normalize+log1p→pca→neighbors→leiden(igraph)→umap` pipeline
+    on the loaded copy, and NEVER hard-fails — on any failure the plot is skipped
+    and the loaded AnnData + metadata are still returned with a note. Grouping
+    detection covers R `make.names` atlas cols (`Clusters`, …), not just `leiden`;
+    leiden uses `flavor="igraph"` (no `leidenalg` dep on the Space). Sits on top of
+    the `_load_adata` seam, so the Loveless subsets now load AND visualize
+    end-to-end. Tests: `tests/test_rna_sc_load.py` (4 green).
+  - **Still open:** PROD factory rebuild on the 0.1.8 re-pin was TRIGGERED 2026-07-01
+    (with the UMAP-hardening merge) — confirm it settled to RUNNING and the 0.1.8
+    wheel actually took (both sc datasets list); `hf-dev` factory rebuild + e2e verify
+    still not done; confirm prod RAM holds the subset live; pseudobulk-aggregation tool
+    for the sample-level DE contrast; derive Werba signatures when needed. ADR-0006
+    items 7/8/9.
+  Companion to the biodata-registry Loveless ingestion (provenance/scope/gate plan
+  there). Design goal: keep user-facing runtime bulk-like.
+  - Two roles. The **Steele-subset h5ad** is an analyzable sc dataset — loads once
+    via the persistent MCP server + `read_h5ad_cached`, then pseudobulk → DE /
+    activity scoring; first load slower than a bulk series matrix, cached after,
+    then normal. The **full integrated atlas** is NOT live-computed — heavy sc
+    work (annotate / QC / derive signatures) is precomputed offline at ingestion,
+    and query-time ops score *bulk* cohorts against the derived Loveless
+    signatures / deconvolution reference on the existing fast path.
+  - Cache caveat: `read_h5ad_cached` returns **copies** (safe under the shared
+    server) — fine at MB scale, a RAM multiplier at GB scale. A large atlas on the
+    live path would need views/no-copy, or (preferred) stay off the live path via
+    the offline-signature approach.
+  - Space sizing: confirm the prod Space RAM tier holds whatever sc artifact is
+    served live — the subset should fit; the raw atlas likely won't.
+  - Expected user flows: "score the Loveless basal / CXCL10+ CAF signature in
+    TCGA-PAAD" (bulk fast path), "pseudobulk DE tumor vs normal in the Loveless
+    subset" (sc load + pseudobulk), "annotate cell types in Loveless" (heaviest —
+    prefer precomputed). Pairs with the `CROSS_RESOLUTION` gate + the PURIST
+    vs single-cell methodology comparison above.
+- **Med — Batch-adjustment validation workflow (P53 quality check).** End-to-end
+  check on the batch-adjustment path (mouse → expression change → sequencing):
+  confirm the changes hit pathways known to associate with **P53** (or other known
+  processes), and quality-check that the **same pathways appear before vs. after**
+  batch adjustment. Tooling exists (Mode A covariate DE + `decoupler_pool_cohorts`
+  ComBat scoring); this is the validation/QC analysis on top.
+- **Note — decoupleR "explain each step" + preprocessing = DONE.** The plain-
+  language Approach section + decoupleR glossary in every solution (`d8abb63`) and
+  the per-dataset `preprocessing` field (biodata 0.1.5, surfaced in prompts) cover
+  the PI's "explain how decoupleR works / add preprocessing info" items.
+
+### Blocked / long-term
+
+- **Blocked — COMPASS / Chan-Seng-Yue 2020** (was Known Gaps #13). Controlled
+  access (EGA EGAS00001002543); needs a signed DAA + DAC approval. Not pursuable
+  for an open-access deployment. Would be Path A (RNA-seq raw counts → DESeq2)
+  if access is ever obtained.
+- **Low / long-term — Additional specialist agents beyond decoupler** (was
+  Known Gaps #12). Architecture already supports it; add entries to `agents.yaml`
+  + new Space deployments when ready.
+
+### Minor / residual
+
+- **Low — GPL annotation fallback residuals** (was Known Gaps #15). `GPL_GENE_SYMBOL_CANDIDATES`
+  is a fixed 9-variant list (other naming needs explicit `gene_symbol_column`);
+  in-memory cache hits cosmetically report a possibly-different `sym_col_used`
+  (cached mapping itself is correct).
+
+---
+
+## Done (recent)
+
+- 2026-07-02 — **ADR-0011 upload safety gate ("Now" slice)** — branch
+  `feat/adr-0011-upload-gate` (off `origin/main`), **NOT pushed, no deploy**. New non-agent-facing
+  `src/uploads/` (`stage_upload` → `validate_upload` → `register_upload`) + shared
+  `src/core/integrity.py` (`compute_sha256`/`verify_sha256`, reused later by ADR-0010). Quarantine
+  + type/size allow-list + manifest-required + de-id attestation + validate-via-vetted-loader
+  (never-exec) + SHA-256 provenance into the ADR-0008 audit sink + admin-only registration
+  (`UPLOAD_ADMIN_IDS`, fail-closed). `tests/test_upload_gate.py` (24) green; reused suites (133)
+  green. ADR-0011 flipped Proposed→Accepted for this slice. **Follow-ups (open):** tabular
+  auto-validation (h5ad-only today); AWS staging bucket + pre-validation malware scan (ADR-0011
+  "At AWS"); wire `src/core/integrity.py` into `resolve_to_local_path` for ADR-0010 on-load verify.
+- 2026-06-22 — **TF semantic-annotation coverage + igraph network plot**
+  (commit `ed98c5b` on `main`, **NOT pushed**). Fixed 70 valid HGNC TF symbols
+  being mis-bucketed as `unresolved_label`: bundled `resources/semantic/hgnc_cache.tsv`
+  (1,183 CollecTRI source TFs) loaded by `normalize_gene_symbol()`, + regen
+  script. Restored `igraph` to `requirements.in`/`.txt` so `dc.pl.network` (TF
+  network plot) stops silently failing. Semantic suite 157 passed.
+  **Deploy when ready** (dev `hf-dev` first, then prod).
+- 2026-06-22 — **Item 3: per-dataset `preprocessing` field surfaced + re-pin
+  0.1.5 → merged to `main` → DEV** (`982f65a`; `hf-dev` `64a443c..982f65a`,
+  rebuilding; prod NOT pushed). Re-pinned biodata-registry 0.1.4 → 0.1.5
+  (`b46392c`, sha256 `958f498b…`, folds in the pending Sears 0.1.4 bump); added
+  `preprocessing` to `list_available_datasets()` + both agent.py dataset-dict
+  builders; prompts.yaml renders a Preprocessing bullet + an Approach rule
+  (item 4). 99 tests pass. Registry side released as 0.1.5 (schema field + all
+  19 manifests populated).
+- 2026-06-22 — **Hide microarray from advertising + plain-language Approach in
+  solutions** (commit `d8abb63`, dev `hf-dev`). Item 2 agent-side:
+  `HIDDEN_MODALITIES`/`is_advertised()`, `advertised` field, MCP tool returns
+  advertised-only + `unadvertised_dataset_ids`, prompts flag microarray NOT
+  ADVERTISED + heuristics steer to RNA-seq. Item 1: required Approach section +
+  decoupleR glossary in every `<solution>`, no missteps. 99 dataset tests green.
+- 2026-06-22 — **REALLY fixed HF Space stuck on "Starting": gradio 5.49 → 6.18
+  migration** (commits `b4e7ac0` + `5ba3350`, branch `fix/gradio-6-migration` →
+  merged to `main`). The 2026-06-21 SSR fix was a red herring (the working
+  orchestrator runs 6.18 *with SSR on*); real cause is gradio 5.49 no longer
+  completing HF's readiness handshake. Cascade: `mcp` 1.10.1→1.28.0,
+  `langchain-mcp-adapters` 0.2.2→0.3.0, `fastmcp` pinned `==3.2.3`; `gradio_ui.py`
+  ported to the 6.0 API; `ssr_mode=False` removed. Verified: 851 tests, 49 tools,
+  HTTP 200. **DEV RUNNING** on `5ba3350` (a pause→unpause cleared an HF-side
+  rollout wedge). Prod migration still pending (see Open → Deploy/infra).
+- 2026-06-21 — [SUPERSEDED by the gradio-6 migration above] Attempted SSR fix for
+  the "Starting" hang: `ssr_mode=False` default in `GradioAgentUI.launch()`
+  (`d38496a`). Did not actually fix it; removed during the migration.
+- 2026-06-17 — UI download buttons fixed: exports no longer truncate the
+  solution (the old path dropped everything after the first blank line); TXT
+  downloads again; added a **Full log (.pdf)** export of the full generated
+  logic. All four files are generated at run completion and armed as one-click
+  downloads (`_assessment_blocks` / `_log_blocks` / `_write_*` in
+  `ui_formatting.py`; `_arm_downloads` in `gradio_ui.py`).
+- 2026-06-17 — Plots render inline: end-of-run sweep of `tmp/outputs` embeds
+  figures — including direct `dc.pl`/matplotlib plots that emit no artifact
+  JSON — deduped by basename against mid-run artifacts.
+- 2026-06-17 — Full-run PDF (`full_run.pdf`) saved to the results repo
+  alongside `conversation.md` / `metadata.json`, with a direct link in the
+  saved-run notice (`HFResultsStorage.upload_run_file`).
+- 2026-06-17 — Fixed spurious "Step limit reached" notice after every run: the
+  UI streams via `graph.stream()` so `trace_logs` was always empty; completion
+  is now keyed off the real solution-shown signal (`last_solution_shown`).
+- 2026-06-17 — Rotated `research_agent_token` (was Known Gaps #5 / ACTION REQUIRED).
+- 2026-06-14 — Bailey 2016 (`paca_au_rnaseq`/`paca_au_array`) + Puleo 2018
+  (`puleo_2018`) stress-tested against real hosted data; found + fixed the
+  `subset_query` `.query()` syntax bug and a `Sample.type` allowed_values gap.
+- 2026-06-14 — TCGA-PAAD sample curation re-verified end-to-end (Known Gaps #7).
+- 2026-06-12/14 — Live GPL-annotation fallback Tier 1 + Tier 2; 4 residual
+  limitations fixed (Known Gaps #14, #15).
+- 2026-06-06 — GDC STAR-Counts loader + tcga_paad pivot to Path A.
+- 2026-06-05 — biodata-registry wired in; manifest rules in system prompt; GEO
+  datasets loaded/validated (Known Gaps #0–4).
+
+_Full detail for any item: see `memory.md`._

app.py

@@ -16,28 +16,16 @@ if os.environ.get("APT_PROBE"):
     import subprocess
 
     def _apt_probe():
-        pkgs = [
-            "r-base",
-            "r-base-dev",
-            "r-bioc-limma",
-            "libcurl4-openssl-dev",
-            "libssl-dev",
-            "libxml2-dev",
-            "libreadline-dev",
-        ]
+        pkgs = ["r-base", "r-base-dev", "r-bioc-limma", "libcurl4-openssl-dev",
+                "libssl-dev", "libxml2-dev", "libreadline-dev"]
         print("===APT_PROBE_BEGIN===", flush=True)
         for p in pkgs:
             try:
-                inst = (
-                    subprocess.run(
-                        ["dpkg-query", "-W", "-f=${Version}", p], capture_output=True, text=True
-                    ).stdout.strip()
-                    or "MISSING"
-                )
-                pol = subprocess.run(
-                    ["apt-cache", "policy", p], capture_output=True, text=True
-                ).stdout
-                cand = next((ln.split()[1] for ln in pol.splitlines() if "Candidate:" in ln), "?")
+                inst = subprocess.run(["dpkg-query", "-W", "-f=${Version}", p],
+                                      capture_output=True, text=True).stdout.strip() or "MISSING"
+                pol = subprocess.run(["apt-cache", "policy", p],
+                                     capture_output=True, text=True).stdout
+                cand = next((l.split()[1] for l in pol.splitlines() if "Candidate:" in l), "?")
                 print(f"{p} installed={inst} candidate={cand}", flush=True)
             except Exception as exc:  # never let the probe break app startup
                 print(f"{p} probe-error={type(exc).__name__}", flush=True)
@@ -50,7 +38,6 @@ if os.environ.get("APT_PROBE"):
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "src"))
 
 from dotenv import load_dotenv
-
 load_dotenv("./.env")
 
 from gradio_ui import GradioAgentUI

bandit.yaml

@@ -1,21 +0,0 @@
-# Bandit config — ADR-0014 static-analysis scan (shared across the three repos).
-#
-# Scope: src/ only (application code). Tests use synthetic data and assert on
-# error paths, so scanning them produces noise without signal.
-#
-# The intended sandboxed `exec` (ADR-0007) and the localhost-only urlopen calls
-# are annotated inline with `# nosec Bxxx` and a rationale comment; the remaining
-# accepted infra findings (0.0.0.0 bind in the dev launcher, /tmp working dirs on
-# ephemeral HF Spaces, GEO-download urlopen) are captured in
-# security/bandit-baseline.json so CI fails only on NEW findings, never on the
-# already-triaged set. Every accepted class is documented in
-# security/ACCEPTED-FINDINGS.md.
-#
-# Run:  bandit -r src/ -ll -c bandit.yaml -b security/bandit-baseline.json
-# (-ll = report medium severity and above.)
-
-exclude_dirs:
-  - tests
-  - .venv
-  - scripts
-  - docker

deploy/scan_posture.yaml

@@ -1,43 +0,0 @@
-# Declared malware-scan posture for the ADR-0011 upload safety gate.
-#
-# This file is the SINGLE SOURCE OF TRUTH for what an upload actually gets on a
-# given deployment. It exists because the posture used to be implicit: the code
-# auto-detected ClamAV, so a dev laptop with `brew install clamav` recorded
-# `scan_status: clean` while the HuggingFace Space — which has no AV binary —
-# recorded `skipped`. Local testing therefore masked production behaviour.
-#
-# Read by src/uploads/posture.py; consumed by src/uploads/scanning.py.
-#
-# posture:
-#   structural_only  The deployment has NO anti-virus binary. Uploads get the
-#                    always-on structural magic-byte check (a renamed
-#                    ELF/Mach-O/PE/ZIP/pickle is a hard stop) and NOTHING else.
-#                    An upload is never described as malware-scanned. If an AV
-#                    happens to be present (a dev host), it still runs and a
-#                    `clean` is recorded, but a caveat notes the result is a
-#                    local-host bonus that the deployment does not guarantee.
-#   av_required      The deployment DOES ship a working, signature-updated AV.
-#                    Equivalent to UPLOAD_SCAN_REQUIRED=1 — an upload with no
-#                    clean AV result fails closed. Do NOT select this without a
-#                    verified AV binary AND a current virus database: with no
-#                    signature DB, clamscan exits 2 and EVERY upload hard-fails.
-#
-# Env override (for local experiments / CI): UPLOAD_SCAN_POSTURE, and the
-# pre-existing UPLOAD_SCAN_REQUIRED, both still win over this file.
-
-# --------------------------------------------------------------------------- #
-# Current deployed posture — anne-voigt/Paper2Agent_decoupleRpy (prod) and
-# Paper2Agent_decoupleRpy_dev. Verified 2026-07-29 against the running prod
-# Space at 7ef9d42: it is an `sdk: gradio` Space, so its only apt channel is
-# packages.txt, and packages.txt has no clamav entry -> no clamdscan/clamscan on
-# PATH -> every upload there records scan_status="skipped".
-#
-# Why not install ClamAV here: ADR-0011's remaining-at-AWS item already moves
-# the managed AV pass onto the encrypted staged S3 object, which supersedes a
-# local-AV build. On a cpu-basic Gradio Space `apt install clamav` ships no
-# signature database, so it would need a freshclam download (~1 GB, several
-# minutes) on every container start — and a stale or failed freshclam turns
-# av_required into a total upload outage. Structural-only, stated honestly, is
-# the correct interim posture until the AWS staging bucket exists.
-# --------------------------------------------------------------------------- #
-posture: structural_only

docker/sandbox.Dockerfile

@@ -1,73 +0,0 @@
-# ADR-0007 Phase 1 — sandbox exec-kernel image.
-#
-# This image runs ONLY the untrusted-code exec-kernel
-# (src/managers/execution/sandbox/kernel.py) inside an isolated per-session
-# container. The vetted decoupleR/scanpy/rpy2 tool implementations live OUTSIDE
-# the sandbox in the persistent MCP HTTP server — the kernel reaches them via the
-# MCP bridge (SANDBOX_MCP_URL). Even so, the image carries the SAME scientific
-# stack as the Space (scanpy/decoupler/pydeseq2/rpy2 + R) so generated code that
-# imports those libraries locally still runs.
-#
-# The HF Space itself is a gradio SDK Space (no custom image), so there is no
-# pre-built base image to inherit; this mirrors the Space's environment from
-# python_version (3.11), packages.txt (apt), and requirements.txt (pip).
-#
-# ---------------------------------------------------------------------------
-# BUILD / RUN (needs local Docker — NOT built in the dep-light dev env):
-#
-#   docker build -f docker/sandbox.Dockerfile -t decouplerpy-sandbox:latest .
-#
-#   # ad-hoc smoke test (kernel only, no MCP):
-#   docker run --rm -p 127.0.0.1:8790:8790 decouplerpy-sandbox:latest \
-#       python /app/src/managers/execution/sandbox/kernel.py --port 8790 --host 0.0.0.0
-#   curl -s http://127.0.0.1:8790/health
-#
-# In prod the ContainerLauncher issues an equivalent `docker run` per session,
-# publishing the kernel port to LOCALHOST only. Phase-2 hardening flags
-# (--read-only, --cap-drop=ALL, --tmpfs, --pids-limit, --network) are applied by
-# the launcher / task definition, not baked here.
-# ---------------------------------------------------------------------------
-
-FROM python:3.11-slim
-
-# System packages mirror the Space's packages.txt (R + limma + build libs that
-# rpy2 / scientific wheels need). Pins are dropped here (slim/bookworm apt has
-# different candidate versions than the Space base); pin if a build needs it.
-RUN apt-get update && apt-get install -y --no-install-recommends \
-        r-base \
-        r-base-dev \
-        r-bioc-limma \
-        libcurl4-openssl-dev \
-        libssl-dev \
-        libxml2-dev \
-        libreadline-dev \
-    && rm -rf /var/lib/apt/lists/*
-
-WORKDIR /app
-
-# Install the Python deps first (layer-cached independently of source changes).
-COPY requirements.txt /app/requirements.txt
-RUN pip install --no-cache-dir -r /app/requirements.txt
-
-# Copy the agent source. The kernel is launched BY FILE PATH
-# (/app/src/managers/execution/sandbox/kernel.py), NOT `-m managers...`, so the
-# full `managers` package (agent stack) is never imported into the sandbox — only
-# the minimal kernel + its file-path-loaded siblings run here. PYTHONPATH is kept
-# as a harmless fallback.
-COPY src /app/src
-ENV PYTHONPATH=/app/src
-
-# --- Non-root user (Phase 1 minimum; Phase 2 adds fuller hardening). ----------
-RUN useradd --create-home --uid 10001 sandbox \
-    && chown -R sandbox:sandbox /app
-USER sandbox
-
-# Kernel port (per-session the launcher may override with --port).
-ENV SANDBOX_KERNEL_PORT=8790
-EXPOSE 8790
-
-# Bind to 0.0.0.0 INSIDE the container; the launcher publishes only the mapped
-# port to the host's 127.0.0.1, so the kernel is never on a routable interface.
-# File-path launch (not `-m`) keeps the agent-stack package out of the sandbox.
-ENTRYPOINT ["python", "/app/src/managers/execution/sandbox/kernel.py"]
-CMD ["--port", "8790", "--host", "0.0.0.0"]

docs/adr/ADR-0003-spaces-dev-mode.md

@@ -1,6 +1,6 @@
 # ADR-0003: Spaces Dev Mode for Iterative Development
 
-**Status:** Accepted — **CLOSED 2026-07-01.** Dev Mode enabled on both dev Spaces (`Paper2Agent_decoupleRpy_dev`, `pdac-analysis-orchestrator-dev`, confirmed 2026-06-24). The remaining housekeeping is now done: the "Dev Mode inner loop" note, the SSH / VS Code Remote setup step, and the commit-before-promote convention are all documented in `pdac-analysis-orchestrator/DEPLOYMENT.md` (tier "C. Dev Mode inner loop" + the "Golden rule"); `hf-dev/main` verified current with `main` (ahead by unpromoted dev work, `main` 0 ahead — not stale). The one inherently per-developer step (attaching VS Code Remote-SSH via the command in each Space's Dev Mode panel) is documented, not a repo deliverable. Reconciled + closed 2026-07-01.
+**Status:** Accepted — decision adopted. **Dev Mode enabled on both dev Spaces** (`Paper2Agent_decoupleRpy_dev`, `pdac-analysis-orchestrator-dev`), confirmed 2026-06-24. Remaining action items (DEPLOYMENT.md inner-loop note; VS Code/SSH config) still open. Reconciled 2026-06-24.
 **Date:** 2026-06-24
 **Deciders:** Annie Voigt (project lead)
 **Scope:** `DecoupleRpy_Agent` (specialist Space) and `pdac-analysis-orchestrator` (coordinator Space). Enabled by the HF PRO subscription (2026-06). No code change in either repo — this is a Space-settings + workflow change. The orchestrator is named here because it shares the same deploy-and-wait pain; the ADR lives in `DecoupleRpy_Agent/docs/adr/` per the existing ADR-home convention (cf. ADR-0001's cross-repo placement note).
@@ -101,10 +101,10 @@ and does not change how secrets/tokens are managed.
 ## Action Items
 
 1. [x] Enable Dev Mode on `Paper2Agent_decoupleRpy_dev` and `pdac-analysis-orchestrator-dev`. *(done — both enabled, confirmed 2026-06-24)*
-2. [x] Confirm `hf-dev` actually tracks current `main` before relying on it. *(done 2026-07-01 — `git rev-list --left-right --count main...hf-dev/main` = `0  44`: `hf-dev/main` contains everything on `main` plus unpromoted dev work; `main` is 0 ahead. Not stale. Caveat: compared against local remote-tracking refs; re-run after a `git fetch hf-dev` if in doubt.)*
-3. [x] Add a short "Dev Mode inner loop" note to `pdac-analysis-orchestrator/DEPLOYMENT.md` so the workflow is documented next to the deploy rules. *(done — see DEPLOYMENT.md tier "C. Dev Mode inner loop", which links back to this ADR.)*
-4. [x] Set up VS Code Remote / SSH config for both dev Spaces. *(procedure documented in DEPLOYMENT.md: one-time copy of the SSH command from each Space's Settings → Dev Mode panel, then attach with VS Code Remote-SSH. The attach itself is a per-developer/per-machine step, not a repo artifact.)*
-5. [x] Convention: anything proven in a Dev Mode session must land as a commit before prod promotion. *(documented as the "Golden rule (Dev Mode)" in DEPLOYMENT.md.)*
+2. [ ] Confirm `hf-dev` actually tracks current `main` before relying on it (SHOWCASE_STATUS flags it as possibly stale).
+3. [ ] Add a short "Dev Mode inner loop" note to `pdac-analysis-orchestrator/DEPLOYMENT.md` so the workflow is documented next to the deploy rules.
+4. [ ] Set up VS Code Remote / SSH config for both dev Spaces.
+5. [ ] Convention: anything proven in a Dev Mode session must land as a commit before prod promotion.
 
 ## References
 

docs/adr/ADR-0005-private-storage-persistence.md

@@ -1,6 +1,6 @@
 # ADR-0005: Private Storage as Durable Data Persistence
 
-**Status:** Accepted — **CLOSED 2026-07-01.** Durable, private-by-default data tier adopted; `pdac-research-data` confirmed private (2026-06-24). Remaining housekeeping now done: the "promote to public on publication" step and the private-Data-Studio validation workflow are documented in `biodata-registry/CLAUDE.md` ("Adding a New Dataset" step 5 + "Manifest validation workflow"), with a dated entry in `biodata-registry/memory.md`. One out-of-repo note (SHOWCASE_STATUS.md h5ad hosting) is tracked below. Reconciled + closed 2026-07-01.
+**Status:** Accepted — durable, private-by-default data tier adopted. **`pdac-research-data` confirmed private** 2026-06-24. Remaining action items (onboarding "promote to public on publication" checklist step; private Data Studio verification workflow; `biodata-registry/memory.md` note) still open. Reconciled 2026-06-24.
 **Date:** 2026-06-24
 **Deciders:** Annie Voigt (project lead)
 **Scope:** Cross-cutting data layer — primarily the `anne-voigt/pdac-research-data` HF Dataset repo (h5ad hosting consumed by `DecoupleRpy_Agent`), with knock-on benefit to `lit-agent`'s corpus snapshots. Enabled by HF PRO (2026-06). Lives in `DecoupleRpy_Agent/docs/adr/` as the primary data consumer; the `lit-agent` corpus-persistence specifics are governed by its own ADRs.
@@ -92,10 +92,10 @@ registry — only where/how the underlying files are stored and inspected.
 ## Action Items
 
 1. [x] Confirm `pdac-research-data` visibility is private and audit which files are public. *(confirmed private 2026-06-24)*
-2. [x] Stop any pruning driven solely by storage limits; retain corpus history in lit-agent's durable dataset. *(codified — `lit-agent/CLAUDE.md` "Retention (PRO storage): keep corpus snapshots across runs rather than minimizing them".)*
-3. [x] Add a "promote to public on publication" step to the dataset-onboarding checklist. *(done 2026-07-01 — `biodata-registry/CLAUDE.md` "Adding a New Dataset" step 5: private-by-default; publish = explicit promote-to-public.)*
-4. [x] Use private Data Studio as the first-pass check in the `dataset_validate_manifest_against_data` workflow. *(done 2026-07-01 — `biodata-registry/CLAUDE.md` "Manifest validation workflow".)*
-5. [x] Record the new storage posture in `SHOWCASE_STATUS.md` (h5ad hosting note) and `biodata-registry/memory.md` (validation workflow). *(done 2026-07-01 — `biodata-registry/memory.md` dated entry + a "h5ad hosting posture — private-by-default (ADR-0005)" bullet in `SHOWCASE_STATUS.md`'s biodata-registry section.)*
+2. [ ] Stop any pruning driven solely by storage limits; retain corpus history in lit-agent's durable dataset.
+3. [ ] Add a "promote to public on publication" step to the dataset-onboarding checklist.
+4. [ ] Use private Data Studio as the first-pass check in the `dataset_validate_manifest_against_data` workflow.
+5. [ ] Record the new storage posture in `SHOWCASE_STATUS.md` (h5ad hosting note) and `biodata-registry/memory.md` (validation workflow).
 
 ## References
 

docs/adr/ADR-0006-loveless-single-cell-serving.md

@@ -158,9 +158,9 @@ artifacts for.
 4. [x] Add `src/workflows/signatures.py` (load/validate a `source/target/weight` signature net) + tests with synthetic data.
 5. [x] **biodata-registry (0.1.8, 2026-07-01):** two Loveless-atlas single-cell subsets landed — `gse155698_steele` and `gse205013_werba` (`modality: sc_rnaseq`, `raw_counts`, `expression_source.type: url` → hosted `.h5ad` on `pdac-research-data`) — and the `CROSS_RESOLUTION` gate. **Role 2 signature artifact published:** `scripts/ingest/loveless/derive_signatures.py` derives 14 per-cell-type marker signatures from the Steele subset (`rank_genes_groups` on `Clusters`, top-50/cluster, log2FC≥1, padj≤0.05) → `loveless/signatures/gse155698_steele_celltype_signatures.csv`. Markers are biologically sane (ACINAR→PRSS1/CLPS, FIBROBLASTS→LUM/COL11A1, B CELLS→PAX5/IGHD, …). `dataset_score_signature` / `load_signature_net` load it via the authenticated resolver and score end-to-end. Werba signatures can be derived the same way when needed.
 6. [x] Add an `h5ad` / single-cell `expression_source.type` to `_build_loading_plan` so a Steele-subset dataset gets a registry loading plan. Done — plus the schema alignment it required: `manifest_schema` now adds `"h5ad"` to `VALID_EXPRESSION_SOURCE_TYPES` and derives `analysis_path == "P"` for `sc_rnaseq` / `spatial_rnaseq` (modality checked **before** data_level, so an sc raw_counts h5ad is never mislabeled Path A — matches biodata-registry 0.1.7's A/B/P). The plan loads via `decoupler_load_and_visualize_data` (`read_h5ad_cached`) and Path P emits per-cell scoring + a pseudobulk note instead of the bulk DESeq2/limma contrast tail. Tests: `tests/test_loading_plan_h5ad.py`. The sc loaders (`rna_sc.py::_load_adata`) now resolve a hosted/private h5ad URL→local path through the shared authenticated resolver (`resolve_to_local_path`, `HF_TOKEN`) — the same path the bulk tools use — so a private `pdac-research-data` h5ad loads end-to-end; a stable HF-cache file is parsed once per process via `read_h5ad_cached`, a one-shot temp download is read directly and deleted (`tests/test_rna_sc_loader.py`). **Reconciled with the landed 0.1.8 manifests:** they declare `expression_source.type: url` (not `h5ad`), so Path P routing in `_build_loading_plan` now keys on **modality** (checked first) — a single-cell/spatial dataset loads via `decoupler_load_and_visualize_data`, never the bulk `decoupler_load_url_counts`, regardless of the declared source-type string. Verified against the live `gse155698_steele` / `gse205013_werba` manifests.
-7. [x] Confirm the prod Space RAM tier holds the Steele-subset h5ad live (it should; the raw atlas must not be served live). **Verified 2026-08-13 by two live agent runs on prod** (runs `20260813_191535` Steele, `20260813_192108` Werba in `anne-voigt/decoupleRpy_results`): both slimmed subsets load via `dataset_load` and render a UMAP end-to-end on the cpu-basic Space (Werba, the larger at 167,366 cells, in 640 s using the atlas's precomputed `X_umap`). The raw atlas remains off the live path.
-8. [x] Re-pin biodata-registry to the release carrying the Loveless artifacts — done, and the factory-rebuild worry is closed: prod was **factory-rebuilt on 0.1.16** (2026-08-13) with both sc manifests registered; the item-7 runs verify it end-to-end on prod (superseding the planned `hf-dev` check).
-9. [ ] Pair with the PURIST-vs-single-cell methodology comparison (TODO) once the subset is loadable. *Unblocked as of 2026-08-13 (both subsets load on prod); waits on the Med TODO "Add the PURIST subtype operation".*
+7. [ ] Confirm the prod Space RAM tier holds the Steele-subset h5ad live (it should; the raw atlas must not be served live).
+8. [~] Re-pin biodata-registry to the release carrying the Loveless artifacts (0.1.8) — **done** (`requirements.in`/`.txt`; already on `origin/main` + prod, now on this branch too). **Still open:** the 0.1.8 re-pin needs a FACTORY rebuild of the consuming Space (pip/build-cache lesson from 0.1.6), then end-to-end verify on `hf-dev`.
+9. [ ] Pair with the PURIST-vs-single-cell methodology comparison (TODO) once the subset is loadable.
 
 ## References
 

docs/adr/ADR-0007-phase1-local-validation.md

@@ -1,147 +0,0 @@
-# ADR-0007 Phase 1 — Local Validation Checklist
-
-Validates the sandboxed executor on a machine with Docker + the heavy deps
-(rpy2/scanpy/decoupler) — the parts that couldn't run in the build environment.
-Branch under test: `feat/sandbox-executor` (stacked on `feat/executor-seam`).
-
-**Validation run: 2026-07-01** (macOS, Docker 28.4.0, `.venv` Python 3.12 with
-scanpy 1.12.1 / decoupler 2.1.6; image built on python:3.11-slim). Result:
-**Sections 0–8 validated** (with the noted caveats); Section 9 is the merge step,
-not run here. Two real container-launch bugs were found and fixed, and the MCP
-bridge's real streamable-http handshake was implemented (was a mock-only stub).
-
-## 0. Setup
-- [x] `git checkout feat/sandbox-executor`
-- [x] Full deps available in `.venv` (scanpy/decoupler present; **rpy2 not in the
-      local `.venv`** — R paths validated inside the container instead). Docker running.
-
-## 1. Suite still green with real deps
-- [x] Full suite green **per-file**: all 46 `tests/test_*.py` files pass or skip
-      (5 are network/data integration tests that `skip`), **0 failures**.
-- [x] `pytest tests/test_sandbox_executor.py tests/test_executor_seam.py` →
-      **27/27** (was 22; +5 regression tests added for the fixes below).
-- ⚠ A single-invocation `pytest -q` **cannot collect** the whole repo: (a)
-  `scripts/check_limma_runtime.py` is a standalone script that `sys.exit(1)`s at
-  import when rpy2 is absent, and (b) the `tests/` files insert `sys.path`
-  differently, so collecting them together pollutes `managers` import resolution.
-  **Both are pre-existing and unrelated to this branch** (confirmed: not in the
-  branch diff). Run per-file, or with `PYTHONPATH=src` and excluding `scripts/`.
-
-## 2. Default path unchanged (regression guard)
-- [x] With `EXECUTOR` unset, `get_executor()` returns an in-process
-      `PythonExecutor`; a multi-step session runs and state persists — no behavior
-      change.
-
-## 3. Subprocess launcher, end-to-end (no Docker)
-- [x] `EXECUTOR=sandbox SANDBOX_LAUNCHER=subprocess` → multi-step session.
-- [x] State persists across steps (a var from step 1 is visible in step 2/3),
-      stdout returns correctly, and the `kernel.py` subprocess is reaped on
-      `close()` (`pgrep -f sandbox/kernel.py`: 1 while open → 0 after).
-
-## 4. Build the sandbox image
-- [x] `docker build -f docker/sandbox.Dockerfile -t decouplerpy-sandbox:latest .`
-      → built (2.91 GB). **Added `.dockerignore`** so the build context is `src/`
-      + `requirements.txt`, not the full 3.2 GB repo (`.venv`, `.git`, etc.).
-- [x] Kernel is the entrypoint; container starts and `/health` returns
-      `{"status":"ok"}`.
-- [x] Runs as **non-root**: `uid=10001(sandbox)`.
-      ⚠ The `docker run --rm <image> whoami` form checks nothing — the image
-      ENTRYPOINT swallows `whoami` as a kernel arg. Use
-      `docker run --rm --entrypoint whoami <image>`.
-
-## 5. Container launcher, end-to-end
-- [x] `EXECUTOR=sandbox SANDBOX_LAUNCHER=container SANDBOX_IMAGE=decouplerpy-sandbox:latest`
-      → session runs inside the container.
-- [x] scanpy + decoupler work inside the container and **state persists** across
-      execute calls (built an AnnData in step 1, normalized it in step 2).
-- [x] Container is torn down on session end (`docker ps -a` — 0 leftover).
-- ⚠ `import rpy2.robjects` (the R/limma DE path) **fails inside the image**:
-  `tzlocal==5.4.2` in `requirements.txt` resolves to a **metadata-only wheel**
-  (its `RECORD` lists only `*.dist-info`; a force-reinstall still writes no
-  `tzlocal/` package). This is a **requirements/lockfile packaging bug, not an
-  ADR-0007 issue** — it blocks only the rpy2/limma method (ttest/DESeq2 paths and
-  all of scanpy/decoupler are unaffected). Fix separately (re-pin/repair tzlocal).
-- **Two container-launch bugs found & fixed** (`launchers.py`):
-  1. `ContainerLauncher._kernel_command()` prepended `python <kernel-path>`, but
-     the image ENTRYPOINT is *already* `python .../kernel.py` — so the container
-     ran `python kernel.py python kernel.py …`, argparse rejected it, and the
-     kernel exited before `/health`. Now returns **args only** (matching the
-     Dockerfile `CMD`).
-  2. `start()` only caught `ImportError` for the docker SDK, so an `import docker`
-     that resolves to the repo's shadowing `docker/` namespace dir (no `from_env`)
-     raised `AttributeError` instead of falling back to the CLI. Now any
-     unusable-SDK case (`ImportError`, missing `from_env`, daemon down) falls back
-     to the `docker` CLI.
-
-## 6. MCP bridge — real handshake (the marked TODO)
-- [x] Pointed `SANDBOX_MCP_URL` at a live `server.py --transport streamable-http`
-      (mounted at `/mcp/`).
-- [x] **Implemented the real handshake** in `sandbox/mcp_bridge.py`: a
-      `streamable_http` transport that drives the same `mcp` client path as
-      `mcp_manager.add_mcp_http` (`streamablehttp_client` → `initialize` →
-      `tools/call`), plus transport auto-selection (`/mcp` → streamable-http,
-      else → the existing plain-JSON mock/adapter transport;
-      `SANDBOX_MCP_TRANSPORT` overrides). Previously `call_mcp_tool` only spoke
-      the mock JSON protocol.
-- [x] Confirmed an in-kernel tool stub invokes a **real** MCP tool
-      (`dataset_list_available`) over streamable-http and returns real manifest
-      data — not the mock.
-
-## 7. Trace/logging intact through the sandbox
-- [x] Verified in-scope: tracing is **executor-agnostic**. The workflow engine
-      records `code_execution` (generated code) and `observation` (captured
-      stdout — tool results, dataset-load prints) in the *agent* process around
-      `self.python_executor(code)` (`agent.py:261`); the executor only returns a
-      stdout string. Same code → **byte-identical stdout** from `PythonExecutor`
-      and the sandbox (both reuse `NamespaceKernel.exec_capture`), so the trace
-      captures identically.
-- ⚠ Full log-**sink persistence** (`from logging_sink import …` in `agent.py`)
-  lives on `feat/configurable-log-sink`, which is **not on this branch** and, per
-  Section 9, merges *before* `feat/sandbox-executor`. End-to-end sink validation
-  belongs to that combined-branch stage; nothing in the sandbox path affects it.
-
-## 8. Light security spot-checks (full hardening is Phase 2)
-- [x] Kernel binds localhost only: subprocess launcher `host=127.0.0.1`; container
-      publishes `127.0.0.1:<port>:<port>` (`docker port` → `… -> 127.0.0.1:<port>`,
-      never `0.0.0.0`).
-- [x] Generated code in the **container** cannot read host secrets: with
-      `ANTHROPIC_API_KEY`/`MY_HOST_SECRET` set on the host, in-sandbox
-      `os.environ.get(...)` returns `None` for both (only `SANDBOX_MCP_URL` is
-      passed in).
-- ⚠ The **subprocess** launcher inherits `os.environ` (`env = dict(os.environ)`),
-  so in dev/subprocess mode generated code *can* read host env. Only the
-  **container** launcher is a real isolation boundary — the subprocess launcher is
-  a dev/test convenience, not a security boundary.
-- Phase 2 status: the container-side flags are now **done** — read-only rootfs,
-  `--cap-drop=ALL`, `--security-opt=no-new-privileges`, memory/CPU/pids limits,
-  writable tmpfs scratch, and `--network` (env-overridable), with the subprocess
-  launcher explicitly marked "not a security boundary" in code. Egress
-  deny-by-default + MCP allow-list and the read-only source mount remain
-  follow-ups (deferred to Phase 3 / a local proxy spike). Full checklist +
-  live-verification results: **`docs/adr/ADR-0007-phase2-local-hardening.md`**.
-
-## 9. Merge order (once green)
-- [ ] `feat/executor-seam` → main (Phase 0, safe no-op default).
-- [ ] `feat/configurable-log-sink` → main.
-- [ ] `feat/sandbox-executor` → main (keep `EXECUTOR=in_process` default in prod
-      until the AWS/container path is validated in a real deploy — Phase 3).
-
----
-
-## Changes made during this validation
-- `src/managers/execution/sandbox/mcp_bridge.py` — implemented the real
-  streamable-http tool-dispatch transport + `select_transport()` auto-detection
-  (Section 6); kept the plain-JSON transport for the mock/adapter.
-- `src/managers/execution/sandbox/launchers.py` — fixed the two container-launch
-  bugs in Section 5 (args-only kernel command; robust docker-SDK→CLI fallback).
-- `.dockerignore` — new; trims the build context (Section 4).
-- `tests/test_sandbox_executor.py` — +5 tests: container command is args-only,
-  localhost-only publish, SDK→CLI fallback, and MCP transport selection.
-
-## Follow-ups (out of ADR-0007 scope)
-- **`tzlocal==5.4.2` is a metadata-only wheel** → `import rpy2.robjects` fails in
-  the image (and the R/limma path in the Space if it uses the same pin). Re-pin or
-  repair. Tracked separately.
-- Repo-wide `pytest -q` can't collect in one shot (standalone `scripts/test_*`
-  that `sys.exit`s + cross-file `sys.path` pollution). A `conftest.py`/`pyproject`
-  `testpaths`+`pythonpath` config would let CI run the suite in one invocation.

docs/adr/ADR-0007-phase2-local-hardening.md

@@ -1,122 +0,0 @@
-# ADR-0007 Phase 2 — Local Hardening Checklist
-
-Locks down the `ContainerLauncher` sandbox with `docker run` least-privilege
-flags, still on **local Docker** (AWS is Phase 3). Follows Phase 1
-(`feat/sandbox-executor`, commit `ffba834` / validation `d864312`), where the
-container ran with default Docker privileges.
-
-**Scope:** `src/managers/execution/sandbox/launchers.py` (`ContainerLauncher`) +
-`tests/test_sandbox_executor.py`. `SubprocessLauncher` is deliberately **not**
-touched — it is a dev/test convenience and **not a security boundary** (its
-docstring now says so explicitly). The prod default stays `EXECUTOR=in_process`;
-these changes only affect `SANDBOX_LAUNCHER=container`.
-
-**Validated: 2026-07-01** (macOS, Docker 28.4.0, image `decouplerpy-sandbox:latest`
-already built from Phase 1). All 22 tests in `tests/test_sandbox_executor.py`
-pass, **including the live hardened-container integration test** (see §3).
-
----
-
-## 1. Container flags applied (the `_hardening()` set)
-
-All derive from ONE `_hardening()` source read at launch time, formatted for both
-the CLI (`_docker_run_cmd` → `_hardening_flags`) and the docker-SDK
-(`start` → `_hardening_kwargs`) paths so the two **cannot drift**.
-
-| Control | CLI flag | SDK kwarg | Why |
-|---|---|---|---|
-| Immutable root FS | `--read-only` | `read_only=True` | code can't tamper with the image / persist between runs |
-| Drop all caps | `--cap-drop=ALL` | `cap_drop=["ALL"]` | kernel needs none (binds an unprivileged high port) |
-| No priv-escalation | `--security-opt=no-new-privileges` | `security_opt=["no-new-privileges"]` | block setuid/setgid escalation |
-| RAM cap | `--memory` (`SANDBOX_MEMORY`, def `4g`) | `mem_limit` | OOM-kill runaway allocs |
-| CPU cap | `--cpus` (`SANDBOX_CPUS`, def `2`) | `nano_cpus=int(cpus*1e9)` | bound CPU spin |
-| PID cap | `--pids-limit` (`SANDBOX_PIDS_LIMIT`, def `512`) | `pids_limit` | fork/thread-bomb guard |
-| Network mode | `--network` (`SANDBOX_NETWORK`, def `bridge`) | `network_mode` | see §2 |
-| Writable scratch | `--tmpfs /tmp`, `--tmpfs /home/sandbox` (`SANDBOX_TMPFS_SIZE`, def `1g`) | `tmpfs={...}` | see below |
-| MCP host resolution | `--add-host=host.docker.internal:host-gateway` | `extra_hosts={...}` | MCP reachable on Linux too (dropped when `network=none`) |
-
-**Read-only rootfs + writable tmpfs.** `--read-only` freezes the whole FS, so
-anything that writes at runtime needs an explicit escape hatch:
-- **`/tmp`** (`mode=1777`) — generic scratch for generated analysis code (scanpy
-  figure exports, pydeseq2 intermediates, temp files).
-- **HOME `/home/sandbox`** (owned by uid `10001`) — matplotlib / numba /
-  fontconfig write caches under `~/.config` and `~/.cache` **at import time**;
-  with a read-only HOME, `import scanpy` (which imports matplotlib) crashes
-  before any user code runs. The live test §3(a) confirms the exec-kernel itself
-  boots and runs code fine under the read-only rootfs + these two tmpfs mounts.
-
-tmpfs is RAM-backed, size-capped (so a runaway write can't exhaust host memory),
-and vanishes on teardown — matching the ADR's "nothing persists past the
-session" intent.
-
-**Env overrides** (mirroring the `get_executor` / `get_log_sink` env-var style):
-`SANDBOX_MEMORY`, `SANDBOX_CPUS`, `SANDBOX_PIDS_LIMIT`, `SANDBOX_NETWORK`,
-`SANDBOX_TMPFS_SIZE`. All optional with sane defaults; retune per deploy without
-a code change.
-
-## 2. Egress — what's done, what remains (the honest bit)
-
-The ADR calls for **deny-by-default egress with an allow-list for only the MCP
-endpoint** ("Local egress control"). The nuance: the in-kernel MCP tool stubs
-must still reach the MCP HTTP server (`SANDBOX_MCP_URL`), so a blanket
-`--network none` is wrong — it severs the MCP bridge.
-
-**Done in Phase 2 (local):**
-- `SANDBOX_NETWORK` is env-overridable, default `bridge` — MCP reachable via
-  `host.docker.internal`, with `--add-host=host.docker.internal:host-gateway`
-  added so the name resolves on native Linux too (it's automatic on Docker
-  Desktop/Mac/Win). Live-verified reachable in §3(c).
-- `SANDBOX_NETWORK=none` is available as a **full-egress-denial** escape hatch
-  today, for operators who pre-stage all data and don't route MCP over the
-  container network. When set, the (now-meaningless) host-gateway mapping is
-  dropped.
-
-**Deferred — deny-by-default + single-host allow-list (FOLLOW-UP, not faked):**
-True "reach ONLY the MCP host, deny everything else" is **not expressible with
-plain `docker run` flags**. It needs one of:
-- an egress-filtering **proxy sidecar** (e.g. squid/envoy) the sandbox is forced
-  through, allow-listing only the MCP host:port; or
-- **iptables/nftables on a custom Docker network** — but we drop `CAP_NET_ADMIN`,
-  so the container cannot firewall *itself*; rules must live on the host/daemon.
-
-This is the right shape for **Phase 3 (AWS Fargate)**, where a VPC with egress
-off + security groups scoped to the MCP server give exactly this for free (ADR
-"Phase 3" and Open Decision #1: pre-stage into OHSU S3 → zero egress). Until
-then, `bridge` (default) permits general egress and `none` denies all; the
-in-between single-host allow-list is intentionally left for Phase 3 rather than
-half-built locally.
-
-## 3. Live behavioural verification (`test_hardened_container_confines_and_still_reaches_mcp`)
-
-Guarded/skipped when Docker or the image is absent; ran for real here. A hardened
-container launched via `ContainerLauncher` was proven to:
-- **(a)** reject a write to the root FS (`open('/nope.txt','w')` → denied) while
-  `/tmp` remains writable — the read-only rootfs + tmpfs escape hatch both work;
-- **(b)** NOT see a host env secret (`MY_HOST_SECRET` set on the host →
-  `os.environ.get(...)` is `None` inside; only `SANDBOX_MCP_URL` crosses);
-- **(c)** still reach the MCP server — a tool stub round-tripped a call to a mock
-  MCP endpoint over `host.docker.internal`.
-
-## 4. Unit coverage (no Docker needed — pure arg logic)
-
-- `test_container_launcher_has_all_hardening_flags` — every flag present at its
-  default, all preceding the image.
-- `test_container_launcher_resource_limits_env_overridable` — `SANDBOX_*`
-  overrides honored on both CLI and SDK paths.
-- `test_container_launcher_network_none_denies_egress` — `none` drops the
-  host-gateway mapping on both paths.
-- `test_container_launcher_sdk_kwargs_match_cli_flags` — the SDK kwargs and CLI
-  flags encode the identical lockdown (anti-drift guard).
-
-## 5. What remains / follow-ups
-
-- [ ] **Egress allow-list** (deny-by-default + MCP-only) — deferred to Phase 3
-  (Fargate/VPC) or a local proxy-sidecar spike; see §2. Not started; not faked.
-- [ ] Read-only **source-data mount** — Phase 2 covers the container's own FS +
-  network; a read-only bind of the source-data location is only meaningful once
-  data is mounted (local: the h5ad cache dir; AWS: the S3 bucket). Wire when the
-  mount path is decided (ties into Phase 3 / Open Decision #1).
-- [ ] `tzlocal==5.4.2` metadata-only wheel still blocks the rpy2/limma path in
-  the image — unchanged from Phase 1, tracked separately (not an ADR-0007 issue).
-- [ ] Prod default stays `EXECUTOR=in_process`; flipping to `sandbox` +
-  `SANDBOX_LAUNCHER=container` waits on the Phase 3 real-deploy validation.

docs/adr/ADR-0007-sandboxed-code-execution.md

@@ -1,157 +0,0 @@
-# ADR-0007 — Per-Session Sandboxed Code Execution
-
-**Status:** Accepted, in progress — **Phases 0–2 complete** (local Docker,
-AWS-independent; validated 2026-07-01), **Phases 3–4 blocked on AWS**
-(landing zone + Open Decisions 1–3). Two Phase-2 line items are intentionally
-deferred into Phase 3 because they only become real with an AWS mount/VPC:
-egress deny-by-default + MCP-only allow-list, and the read-only *source-data*
-mount (Phase 2 hardened the container's own FS + network, not a data mount).
-**Prod default stays `EXECUTOR=in_process`** until the container/AWS path is
-validated in a real deploy (Phase 3).
-**Date:** 2026-07-01
-**Driver:** OHSU security review of the future-state AWS deployment. The agent
-generates and executes LLM-authored Python; isolating that execution is the
-single highest-priority control the review asks for.
-
-**Phase status:** 0 ✅ (`feat/executor-seam`) · 1 ✅ (`ffba834`, local-validated) ·
-2 ✅ container hardening (`52e7143`; egress + source-mount → Phase 3) ·
-3 ⬜ AWS Fargate · 4 ⬜ data pre-stage + OHSU sign-off. Phase-2 detail:
-`ADR-0007-phase2-local-hardening.md`. To close this ADR: Phases 3–4 (per-session
-Fargate task + least-privilege IAM + VPC egress control + the OHSU write-up
-naming this as the implemented top-priority control).
-
----
-
-## Context
-
-The specialist agent is a CodeAct loop: the model emits `<execute>…</execute>`
-blocks that are run via `exec(code, self.namespace)` in
-`src/managers/execution/python_executor.py`. `PythonExecutor` is a persistent,
-Jupyter-kernel-style namespace that holds session state (loaded AnnData,
-intermediate variables) **across steps** — which is why the in-memory AnnData
-cache and persistent MCP HTTP server exist (re-loading per step is the expensive
-path).
-
-Today that `exec` runs **in-process, in the same container as the orchestration
-logic**, with whatever privileges and network access the container has. The code
-even documents the accepted risk: *"bounded by running in an isolated HF Space
-with no secrets beyond the model API key."* That bound is fine for a prototype;
-it is not sufficient for OHSU-managed AWS handling restricted research data.
-
-The review asks for: **network-isolated, least-privilege, ephemeral execution,
-with read-only access to source data.**
-
-## Decision
-
-Run generated code in a **per-session ephemeral sandbox container** — one
-isolated container per user session, holding namespace state across steps, torn
-down at session end. "Ephemeral" at session granularity (not per-step), so the
-caching model is preserved and data is not re-loaded every step.
-
-Crucially, this is implemented by **swapping the executor implementation behind
-the existing `PythonExecutor` interface**, not by rewriting the agent loop. The
-agent already depends only on `send_functions` / `send_variables` /
-`__call__(code) -> str`. A new `SandboxedExecutor` satisfies the same contract
-but proxies each call to a Python kernel running inside the sandbox container.
-
-### Architecture
-
-- **The seam.** Define an `Executor` protocol matching the current interface.
-  `PythonExecutor` (in-process) remains the default for local/dev; the sandboxed
-  implementation is selected by config — mirroring the `LOG_SINK` pattern just
-  added.
-- **What crosses the boundary is small.** `__call__` returns captured stdout (a
-  string) — trivially serializable over a socket/HTTP. No live Python objects
-  need to move.
-- **Tools stay vetted and outside the sandbox.** Injected tool "functions"
-  become thin MCP-client stubs; the actual computation runs in the existing
-  persistent MCP HTTP server. So the sandbox holds only the *untrusted*
-  free-form generated code + session namespace; the *vetted* decoupleR/scanpy
-  tool implementations run in the MCP server, reachable over a restricted
-  internal network. This is a natural extension of the current HTTP-MCP model.
-- **Data model.** Source data mounted/accessed **read-only**; results written to
-  a separate scoped location. External dataset fetch (GEO/GDC) is resolved by
-  either pre-staging data into an OHSU bucket (zero egress — preferred) or an
-  egress allow-list to approved data domains (see Open Decisions).
-
-## Phased plan
-
-Phases 0–2 are **fully AWS-independent** — buildable and testable locally with
-Docker. That is the "start now" portion. Phases 3–4 need the AWS account.
-
-### Phase 0 — Extract the executor seam *(now, ~2–3 days)*
-- Define an `Executor` Protocol/ABC from the current `PythonExecutor` surface.
-- Make the agent depend on the protocol; keep `PythonExecutor` as the default.
-- Add an `EXECUTOR` config switch (`in_process` | `sandbox`), default
-  `in_process`. No behavior change yet.
-- Test: existing suite passes unchanged with the in-process executor.
-
-### Phase 1 — Local Docker sandbox executor *(now, ~1–2 weeks)*
-- Build a sandbox image (reuse the existing Space image — rpy2/scanpy/decoupler
-  already present) running a minimal "exec kernel": accept code over a local
-  socket/HTTP, `exec` into a persistent per-session namespace, return stdout.
-- Implement `SandboxedExecutor`: starts one container per session, proxies
-  `send_functions`/`__call__`, tears down on session end.
-- Wire tool calls from inside the sandbox to the MCP HTTP server (client stubs).
-- Test: a full analysis session runs end-to-end through the sandbox with state
-  persisting across steps; trace/logging (ADR log-sink) still captures prompts,
-  code, tool calls, dataset loads.
-
-### Phase 2 — Local hardening *(now, ~3–5 days)*
-- Non-root user, dropped capabilities, read-only root FS, `--read-only` source
-  mount, tmpfs workdir, CPU/memory/pids limits, no host network.
-- Local egress control (deny-by-default; allow-list only what a session needs).
-- Verify the sandbox cannot read secrets or write to source data.
-
-### Phase 3 — AWS Fargate deployment *(needs AWS, ~2–3 weeks)*
-- Sandbox container → per-session Fargate task (or ECS-on-gVisor).
-- Least-privilege task IAM role: read-only on the source-data bucket, write to a
-  scoped results prefix, nothing else. No long-lived credentials.
-- VPC with egress off (data pre-staged) or allow-listed; security groups scoped
-  to the MCP server only.
-- Session lifecycle: task launched per session, torn down at end; results
-  plumbed back through the agent.
-
-### Phase 4 — Data pre-stage + review sign-off *(needs AWS decision, ~1 week)*
-- If pre-staging: a sync job that mirrors approved datasets into the read-only
-  OHSU source bucket (with checksums — also closes the Q6 tamper-detection gap).
-- Security validation, threat-model doc, and the OHSU-review write-up naming
-  this as the implemented top-priority control.
-
-### LOE summary
-- **Startable now (Phases 0–2):** ~2.5–3.5 engineer-weeks, no AWS.
-- **AWS-dependent (Phases 3–4):** ~3–4 engineer-weeks once foundations exist.
-- **Total:** ~4–7 engineer-weeks (consistent with the review estimate). The
-  swing factor is Phase 3 and whether OHSU provides a ready landing zone.
-
-## Open decisions (resolve before Phase 3; do NOT block Phases 0–2)
-1. **External data: pre-stage into OHSU S3 (zero egress, preferred) vs. egress
-   allow-list.** Pre-staging gives the cleanest review story and closes tamper
-   detection; allow-listing is less upfront work.
-2. **AWS landing zone** — does OHSU provide account/VPC/baseline IAM, or is that
-   part of this scope?
-3. **MCP server placement** — same isolation domain as the sandbox, or a
-   separate hardened service the sandbox reaches over a restricted network.
-
-## Consequences
-- **Positive:** the #1 review control is implemented cleanly via an existing
-  seam; no agent-loop rewrite; caching/perf model preserved; the sandbox
-  confines only the untrusted code while vetted tools stay put; work starts
-  immediately without waiting on AWS.
-- **Cost:** a new container image + session lifecycle to operate; per-session
-  container startup latency (mitigated by session-granular reuse); Phase 3 ties
-  to AWS specifics.
-- **Interim posture:** until Phase 3, the in-process executor remains — so for
-  the current prototype, keep documenting execution as "isolated Space,
-  API-key-only secrets," and present the sandbox as the funded, in-progress
-  target rather than a shipped control.
-
-## Start-now checklist
-- [x] Phase 0: extract `Executor` protocol + `EXECUTOR` config switch (`feat/executor-seam`)
-- [x] Phase 1: sandbox image + `SandboxedExecutor` + MCP client stubs (`ffba834`, local-validated)
-- [~] Phase 2: container hardening **done** (read-only rootfs, cap-drop=ALL,
-      no-new-privileges, mem/cpu/pids limits, tmpfs scratch, `--network` env-
-      overridable; SDK+CLI paths share one source; live-verified) — see
-      `ADR-0007-phase2-local-hardening.md`. Egress deny-by-default + read-only
-      source mount remain follow-ups (deferred to Phase 3 / local proxy spike).
-- [ ] Decisions 1–3 raised with OHSU cloud/security contacts (parallel track)

docs/adr/ADR-0010-dataset-integrity-verification.md

@@ -1,7 +1,6 @@
 # ADR-0010 — Dataset Integrity / Tamper Verification on Load
 
-**Status:** Accepted — step 1 (AWS-independent) fully implemented 2026-07-02 (22/22 datasets
-baselined; GEO-series-matrix load path now covered)
+**Status:** Proposed
 **Date:** 2026-07-01
 **Deciders:** Annie Voigt (project lead)
 **Driver:** OHSU security review Q7 — "detect unavailable/modified/compromised datasets." Today
@@ -49,42 +48,14 @@ Add **content-hash verification on load**, keyed off the manifest.
    SHA-256 verify in `resolve_to_local_path`; refusal path + test (good hash loads, altered file
    refused, absent hash = load with a "no integrity baseline" note); backfill hashes for the
    current registered datasets.
-   *(The shared hashing helper `src/core/integrity.py` — `compute_sha256` / `verify_sha256`,
-   streamed — landed with ADR-0011 and is the exact low-level code this step's on-load layer
-   builds on.)*
+   *(Available now: the shared hashing helper `src/core/integrity.py`
+   — `compute_sha256` / `verify_sha256`, streamed — landed with ADR-0011 and is the exact code
+   this step wires into `resolve_to_local_path`.)*
 2. **At AWS (ADR-0007 Phase 4):** the pre-stage sync job writes checksums into the read-only OHSU
    source bucket, so integrity is anchored to an OHSU-controlled copy rather than trust-on-first-use
    against the public source. This ADR's on-load check is the same code; only the hash's provenance
    improves.
 
-### Step 1 — as implemented (2026-07-02)
-
-- **Manifest field** (`biodata-registry` 0.1.9): optional `integrity:` block on `DatasetManifest`
-  — `sha256` (primary/expression file) + optional `files:` map (per-file, for separate metadata)
-  + `recorded`/`recorded_from` provenance. Validated (64-hex) in `manifest.validate()`; absent =
-  valid + a "no integrity baseline" warning. Reverse-lookup helper `expected_sha256_for_url()`.
-- **Recorder** `scripts/record_integrity.py` (in `biodata-registry`) streams SHA-256 of each source
-  and writes the block into the YAML (comment-preserving text edit). **Backfilled 22/22** registered
-  datasets. `cptac_pda_counts` was initially skipped at 0.1.9 (its hosted `cptac_pda_counts.h5ad`
-  was 404 on HF); the file was then assembled + hosted and its baseline recorded in
-  `biodata-registry` **0.1.10** (`6a6a884`; sha256 `7fd71517…`, verified against the HF LFS hash of
-  the uploaded 21.7 MB file). `DecoupleRpy_Agent` re-pins the registry to **0.1.10** so the agent
-  enforces all 22/22 at runtime.
-- **On-load verify** (`DecoupleRpy_Agent` `src/core/integrity.py`): `verify_file()` is called
-  centrally in `resolve_to_local_path` after materialize / before parse+cache. Keyed on the source
-  URL via the registry reverse-lookup, so **no tool signature changes**. Mismatch → `IntegrityError`
-  refusal naming the dataset; a tampered temp download is unlinked before raising; missing/absent
-  baseline degrades to "load unverified". Verified end-to-end (real load passes, tampered copy
-  refused).
-- **GEO-series-matrix loads now covered:** `src/workflows/geo.py`'s `load_geo_series_matrix_lines`
-  bypasses `resolve_to_local_path` (it streams the matrix straight into memory), so it now runs the
-  same check inline — `verify_bytes()` on the raw response *as served* (pre-decompression, matching
-  how the baseline is recorded) for URL fetches, and `verify_file()` for local paths. `verify_bytes`
-  is the in-memory counterpart of `verify_file` added to `src/core/integrity.py`. Both are no-ops
-  unless a manifest baselines that exact URL/path, so untracked GEO fetches are unaffected; a
-  baselined series matrix whose bytes were altered is refused before parsing. Trust-on-first-use
-  caveat below still applies until step 2.
-
 ## Consequences
 
 - **Positive:** closes the Q7 tamper gap with a deterministic control, independent of AWS; dovetails

docs/adr/ADR-0011-upload-safety-gate.md

@@ -1,9 +1,6 @@
 # ADR-0011 — Safety Gate for Manual Dataset Uploads
 
-**Status:** Accepted — "Now" (AWS-independent) slice implemented 2026-07-02, extended
-2026-07-02 with a local interim content scan + manifest-drafting helper; **wired into the Gradio
-UI 2026-07-06** (all three data-input paths — upload / URL / HF-dataset — now clear the gate before
-the agent sees a file); only the AWS staging-bucket/encryption pieces remain deferred (see Plan)
+**Status:** Accepted — "Now" (AWS-independent) slice implemented 2026-07-02; AWS pieces deferred (see Plan)
 **Date:** 2026-07-01
 **Deciders:** Annie Voigt (project lead)
 **Driver:** OHSU security review Q4 — researchers should be able to supply their own datasets.
@@ -64,91 +61,18 @@ the existing manifest/validation machinery.
   through the vetted `scanpy.read_h5ad` loader only — no `exec`/`eval`/`pickle`), and `register_upload`
   (admin-only promotion, `UPLOAD_ADMIN_IDS`, into the live registry). Every state transition persists an
   `UploadRecord` (with the de-id attestation + hash) through the always-on ADR-0008 audit sink.
-  End-to-end tests: `tests/test_upload_gate.py` (27). Auto-validation covers **h5ad** (full
-  five-check `validate_manifest_against_data`) and **flat matrices** (`.csv`/`.tsv`/`.txt` + `.gz`,
-  samples-as-rows/genes-as-columns via the vetted `pandas.read_csv`): a bare matrix has no `obs`,
-  so only the file-content checks that don't need metadata run (`data_level` + `feature_id_type`)
-  and the obs-dependent checks are recorded as explicit **caveats** on the record, not silently
-  passed — re-upload as an h5ad to validate grouping/contrasts.
-- **Interim content scan (local, AWS-independent). ✅ Added 2026-07-02** as `src/uploads/scanning.py`,
-  wired as **gate 0** of `validate_upload` (runs on the staged object *before* the loaders). Two
-  layers: (1) an always-on, zero-dependency **structural magic-byte check** — an `.h5ad` must be an
-  HDF5 container, a `.gz` must be gzip, and a flat-text matrix must carry no executable/archive/pickle
-  leader, shebang, or NUL bytes; this catches a renamed ELF/Mach-O/PE/ZIP/pickle that clears the
-  suffix-only door gate, and is *always* a hard stop. (2) An **optional ClamAV pass** (`clamdscan`/
-  `clamscan`, or any scanner named by `UPLOAD_SCAN_CMD`) — external AV *reading* the file, still no
-  `exec`/`eval`/`pickle`. Auto-detection tries the daemon client (`clamdscan`, fast when `clamd` is
-  warm) and **degrades to standalone `clamscan`** if the daemon is absent/misconfigured, rather than
-  reporting the file unscanned. When no AV is present the scan is recorded honestly as
-  `scan_status="skipped"` with a caveat (never reported as malware-scanned); set `UPLOAD_SCAN_REQUIRED=1`
-  to fail closed instead. Outcome persists on the `UploadRecord` (`scan_status`/`scanned_at`/
-  `scan_detail`) through the ADR-0008 audit sink. Tests: `tests/test_upload_scan.py`. **Verified with a
-  real local ClamAV** (`brew install clamav` + `freshclam`, 3.6M sigs): a clean matrix passes, the
-  EICAR test file is flagged `infected` and blocked before the loaders run.
-- **Manifest-drafting helper (local, AWS-independent). ✅ Added 2026-07-02** as `src/uploads/drafting.py`
-  (`draft_manifest`) — the friction mitigation from Consequences below. It opens the file with the same
-  vetted loaders and pre-fills a manifest skeleton (inferred `data_level`, `feature_id_type`, sample/
-  feature counts, candidate `group_columns`) plus an explicit `todo` list of the fields a human must
-  still supply. It does **not** weaken the gate — a drafted manifest still has to clear `validate_upload`
-  and admin registration; it only removes the blank-page problem. A local Gradio panel can wrap it as
-  a thin editable-form shell. Tests: `tests/test_upload_drafting.py`.
-- **UI wiring (local, AWS-independent). ✅ Added 2026-07-06** in `gradio_ui.py`
-  (`run_upload_gate`), called by the Upload-File / URL / HF-Dataset handlers. It runs
-  `stage_upload → scan_upload → draft_manifest → validate_upload` behind a required
-  de-identification checkbox, and only exposes the quarantined `staged_path` to the agent after the
-  security gates clear.
-  **Ordering correction (2026-07-29, from live verification of the prod Space).** The wiring
-  originally ran `draft_manifest` *first*, which meant the drafter parsed the file at its original
-  path before the type allow-list, the size cap, quarantine, or the structural magic-byte scan had
-  run — violating this ADR's own rule that nothing parses an upload until it is staged and scanned.
-  The visible symptom on the live Space: an ELF renamed `.csv` was refused, but by an incidental
-  numpy crash inside the drafter (`zero-size array to reduction operation maximum`) rather than by
-  the content scan, and a `.pdf` was refused as "could not read" rather than as a disallowed type.
-  `run_upload_gate` now stages with a `placeholder_manifest` (filename-derived, no file read), scans,
-  and only then drafts from the **quarantined** copy and attaches it via `attach_manifest`. A file
-  rejected after quarantine has its staged bytes deleted (`discard_upload`), leaving only the audit
-  record. The local test asserted merely that the ELF was "blocked", so it passed for the wrong
-  reason throughout; it now asserts the refusal comes from the content scan. **Session-use vs registration split (decision):** on this session-scoped,
-  single-file, read-only path the *security* envelope (attestation / type / size / structural + AV
-  scan / SHA-256 / never-exec) is **mandatory and blocks on failure**, but `validate_upload`
-  (manifest-vs-data consistency) is **advisory** — an ad-hoc uploader has no hand-authored manifest,
-  so a bare matrix (which fails the `group_columns`-non-empty schema rule) is still usable with a
-  visible "manifest not fully validated" warning rather than being blocked. Admin **registration**
-  into the shared registry (item 7) is unchanged and still required to make an upload a persistent,
-  listed dataset; a session upload is read by the vetted loaders for that session only and is never
-  added to the registry. Tests: `tests/test_upload_ui_gate.py`.
-- **Deployed scan posture made explicit (`deploy/scan_posture.yaml`). ✅ Added 2026-07-29.** The AV
-  layer above is auto-detected from PATH, which left the *deployed* posture implicit — and wrong in
-  the optimistic direction during testing: a dev Mac with `brew install clamav` recorded
-  `scan_status="clean"`, while the prod Space records `skipped`. Verified 2026-07-29 against the
-  RUNNING prod Space at `7ef9d42`: `anne-voigt/Paper2Agent_decoupleRpy` is an `sdk: gradio` Space,
-  so `packages.txt` is its only apt channel and it contains no `clamav` — **no AV binary exists there,
-  and every upload records `skipped`.** `UPLOAD_SCAN_REQUIRED` was never set in any deploy config.
-  **Decision: keep structural-only and say so, rather than install ClamAV in the Space image.**
-  Rationale: (a) the AWS item below already supersedes a local-AV build by moving the managed pass
-  onto the encrypted staged object; (b) `apt install clamav` ships **no signature database**, so a
-  Space would need a ~1 GB `freshclam` download on every cold start, and a stale/failed refresh turns
-  `av_required` into a total upload outage on a cpu-basic box. `deploy/scan_posture.yaml` now declares
-  `posture: structural_only`, read by `src/uploads/posture.py`; `av_required` is the alternative value
-  and is equivalent to `UPLOAD_SCAN_REQUIRED=1` (both env knobs still override the file). Every record
-  is stamped with `scan_posture`, the skipped caveat states plainly that the upload is **NOT
-  malware-scanned**, a `clean` produced under a `structural_only` posture carries an extra caveat that
-  it is a host-local result the deployment does not guarantee, and the Gradio panel says
-  "structure-checked … this is not a malware scan" instead of the ambiguous "content-scanned".
-- **At AWS (remaining):** staging bucket is a separate scoped prefix with its own encryption; the
-  ClamAV pass moves to run on the staged S3 object — at which point the posture file flips to
-  `av_required`. The basic type/size/quarantine gate **and** the local content scan above do **not**
-  require AWS — only the scoped/encrypted bucket does.
+  End-to-end tests: `tests/test_upload_gate.py` (24). Auto-validation currently covers `.h5ad`;
+  tabular uploads stage but stay quarantined pending a tabular loader.
+- **At AWS:** staging bucket is a separate scoped prefix with its own encryption; the optional
+  malware scan runs on the staged object *before* validation. The basic type/size/quarantine gate
+  here does **not** require AWS; the malware scan does.
 
 ## Consequences
 
 - **Positive:** delivers the requested capability without opening an ingress hole — uploads inherit
   the same grounding gate as curated datasets, stay de-identified by attestation, are tamper-checked,
   and are never executable. Clean story for the review.
-- **Cost / caveat:** a manifest requirement adds friction for uploaders — now mitigated by
-  `draft_manifest` (`src/uploads/drafting.py`), which drafts the manifest from the file + a few
-  prompts. Content scanning is now local: an always-on structural magic-byte check plus an optional
-  ClamAV pass. Full managed anti-malware (signature-updated, on the encrypted staging bucket) is still
-  AWS-dependent; until then, an upload that ran with no AV available carries an explicit `skipped`
-  scan caveat and must be described honestly as structurally-checked-but-not-malware-scanned rather
-  than malware-scanned — unless the deployment sets `UPLOAD_SCAN_REQUIRED` to fail closed.
+- **Cost / caveat:** a manifest requirement adds friction for uploaders (mitigate with a UI that
+  drafts the manifest from the file + a few prompts). Full anti-malware coverage is AWS-dependent;
+  until then the gate is type/size/quarantine/validation/attestation, which should be stated
+  honestly rather than described as malware-scanned.

docs/adr/ADR-0012-authentication-access-control.md

@@ -1,164 +0,0 @@
-# ADR-0012 — Authentication & Access Control on the Spaces
-
-**Status:** **Accepted — IMPLEMENTED, DEPLOYED to prod (both Spaces), and validated
-end-to-end (2026-07-02).** Front-door HF OAuth + allow-list gate live on the
-orchestrator (public); specialist flipped **private** + reached only via the
-orchestrator's service token (anonymous access verified refused); authenticated
-identity flows into the ADR-0008 trace. The **only** remaining piece is the OHSU
-SSO/IdP provider cutover, which is **deferred by design** (an OHSU decision) and does
-not block this "now" slice. *(Proposed → Accepted → Done 2026-07-02.)*
-**Date:** 2026-07-02
-**Deciders:** Annie Voigt (project lead)
-**Driver:** OHSU security review — the "Authentication & Authorization / Users"
-section describes ~5 users, 1–2 admins, service accounts, and *no shared
-accounts*. That is the intended model; today it is **not enforced in code**.
-Both Spaces call Gradio `launch()` with no `auth=` and no login gate, so access
-control is only whatever the Space's HuggingFace visibility setting provides.
-**Related:** ADR-0008 (audit trace — records *what* happened but attaches no
-authenticated identity), ADR-0011 (admin-only upload registration — assumes an
-identity this ADR supplies), future OHSU SSO/IdP integration (the AWS-era target).
-
----
-
-## Context
-
-The system is two Gradio Spaces:
-
-- **`pdac-analysis-orchestrator`** — the user-facing front door. `app.py` →
-  `ui.build().launch(server_name="0.0.0.0", server_port=7860)`. No `auth`.
-- **`Paper2Agent_decoupleRpy`** (specialist) — called by the orchestrator via
-  `gradio_client`. `GradioAgentUI.launch(share=False, **kwargs)` →
-  `app.queue(...).launch(...)`. No `auth`. It is also **directly reachable** as
-  its own Space, so it is an authentication bypass around the orchestrator.
-
-Neither app authenticates a user, and there is no per-user identity attached to
-a request or to the ADR-0008 audit trace. The review's answers about admin
-accounts, service accounts, and no-shared-accounts therefore have no technical
-enforcement behind them. This is the largest AWS-independent exposure remaining.
-
-Two distinct problems:
-1. **Front-door auth** — who may use the orchestrator at all.
-2. **Specialist bypass** — the specialist must not be usable except *through* an
-   authenticated orchestrator (or under the same gate).
-
-## Decision
-
-Add an authentication gate to both Spaces now, using HuggingFace-native
-identity (AWS-independent), structured so the later cutover to OHSU SSO/IdP is a
-provider swap, not a redesign.
-
-1. **Front door — HF OAuth, allow-listed.** Put the orchestrator behind
-   HuggingFace OAuth (`hf_oauth: true` in the Space README metadata +
-   `gr.LoginButton` / the `gr.OAuthProfile` dependency), and authorize only an
-   explicit allow-list of HF usernames (config, not code — mirrors
-   `recipients.yaml` / `UPLOAD_ADMIN_IDS`). A logged-in user not on the list is
-   denied. This ties every session to a named identity that ADR-0008 can record.
-   - *Interim fallback if OAuth is deemed too heavy for the pilot:* Gradio native
-     `auth=` with per-user credentials threaded through the existing
-     `launch(**kwargs)` seam. This is weaker (credential-based, not identity-
-     federated, easy to share) and is explicitly a stopgap, not the target.
-
-2. **Close the specialist bypass — set the specialist Space to private + a
-   service token.** The specialist becomes a **private** Space; the orchestrator
-   authenticates to it as a **service account** (HF token, the "inter-service
-   token" already named in the review) via `gradio_client(..., hf_token=...)`.
-   Direct public access is removed; only the orchestrator (holding the token)
-   can reach it. No shared human accounts (consistent with Auth Q6).
-
-3. **Identity into the audit trace.** Once a request carries an authenticated
-   principal, thread it into `get_trace()` (`config` block) so ADR-0008 traces
-   record *who* ran each analysis — the missing link between the app-layer trace
-   and identity events. (App trace still ≠ full IdP log; see ADR-0008's two-layer
-   note.)
-
-4. **Roles.** Two roles only: **user** (run analyses) and **admin** (the 1–2
-   accounts that register uploads per ADR-0011 and deploy). Role is an
-   allow-list attribute in config, not a separate auth system.
-
-## Plan
-
-- **Now (AWS-independent, ~2–4 days):**
-  1. Orchestrator: add HF OAuth + allow-list gate; deny non-listed identities.
-  2. Specialist: flip Space visibility to private; orchestrator authenticates
-     with a service token; verify direct anonymous access is refused.
-  3. Thread the authenticated username into the ADR-0008 trace `config`.
-  4. Add an `ADMIN_IDS` / allow-list config block (both Spaces) + a test that a
-     non-listed identity is rejected and an admin-only action refuses a user.
-- **At OHSU SSO/IdP (deferred — OHSU decision):** replace the HF-OAuth provider
-  with OHSU SSO (OIDC/SAML) behind the same allow-list/role seam; the app change
-  is the provider, not the gate. This is the only piece that waits on OHSU.
-
-## Implementation status (2026-07-02) — COMPLETE
-
-**Specialist (`DecoupleRpy_Agent`) — deployed to prod:**
-- **`src/core/access_control.py`** — the pure, provider-independent identity seam
-  (plan step 4). `Principal`, `role_for` / `is_authorized` / `is_admin`,
-  `resolve_principal`, and `principal_trace_fields`. Config via `ADMIN_IDS`
-  (admins) + `ALLOWED_IDS` (users), with `UPLOAD_ADMIN_IDS` (ADR-0011) honored as
-  admins for back-compat so there is one coherent admin set. **Fail-closed:**
-  empty config denies everyone; an unknown/blank identity has no role. This is
-  the seam OHSU SSO later reuses unchanged — only where the identity string comes
-  from changes.
-- **Identity into the trace** (plan step 3) — `CodeAgent.get_trace()` `config`
-  block carries `principal` + `role` via `principal_trace_fields`, sourced from
-  `self.principal`. A hidden `principal` input on `/interact_with_agent` receives
-  the identity the orchestrator forwards; no forwarded identity records
-  `principal="anonymous"`, `role=None` (honest, never misattributed).
-- **Tests** — `tests/test_access_control.py` (17).
-
-**Orchestrator (`pdac-analysis-orchestrator`) — deployed to prod:**
-- **`access.py`** — front-door allow-list gate mirroring the specialist's env
-  contract, with an `ACCESS_CONTROL` enforcement toggle (fail-closed when on) so
-  the code could ship to the public Space without locking anyone out before cutover.
-- **`README.md`** `hf_oauth: true`; **`gradio_ui.py`** `gr.LoginButton` +
-  `gr.OAuthProfile` gate in `_respond`; **`router.py`** forwards the authenticated
-  username as `principal` to the specialist (only when non-empty). Service-token
-  client side (`gradio_client(token=HF_TOKEN)`) was already wired.
-- **Tests** — `tests/test_access.py` (20); full suite 38 green.
-
-**Architecture note (asymmetric on purpose):** the **orchestrator stays public** and
-is gated at the *app layer* by HF OAuth + the allow-list (a private Space would push
-access back to the HF-visibility layer the ADR is moving away from, and make OAuth
-redundant). The **specialist is private** because it has no login of its own and must
-not be directly reachable — it is an internal service reached only by the
-orchestrator's service token.
-
-**Validated end-to-end on prod (2026-07-02):** a listed user signs in and gets a real
-analysis (proving gate-allow + orchestrator→private-specialist via token + identity
-forwarding in one shot); an anonymous request to the specialist Space is refused
-(Hub API 401 / app 404, calibrated against a known-public Space returning 200); the
-allow-list refuses a non-listed / anonymous user at the front door.
-
-**Deferred (the only open item):** the OHSU SSO/IdP provider cutover — swap HF OAuth
-for OHSU OIDC/SAML behind this same `access_control` / `access` seam. Provider swap,
-not a redesign; waits on OHSU.
-
-## Consequences
-
-- **Positive:** turns the review's stated user/admin/service-account model into
-  an enforced control; every session gains a named identity that flows into the
-  audit trace; the specialist stops being an open bypass; the SSO cutover is
-  scoped to a provider swap.
-- **Cost / caveat:** HF OAuth requires each user to have (or make) a HuggingFace
-  account — acceptable for ~5 pilot users, but call it out; it is HF-account
-  identity, **not** OHSU-managed identity, until the SSO cutover. The Gradio
-  `auth=` fallback is credential-based and must not be presented as identity
-  federation. Setting the specialist private means the orchestrator's service
-  token becomes a secret to manage (rotate; never in the trace — see ADR-0013).
-- **Honesty note for the review:** until this ships, state plainly that access
-  control is currently at the HF-Space-visibility layer only, and that app-level
-  authentication is the funded next step — do not imply the user/admin model is
-  already enforced.
-
-## Start-now checklist — ALL DONE (2026-07-02)
-- [x] Orchestrator: HF OAuth + allow-list gate (deny non-listed). *(`access.py`,
-      `README.md` `hf_oauth`, `gradio_ui.py` gate — prod)*
-- [x] Specialist: private visibility + orchestrator service-token auth; anonymous
-      direct access verified refused. *(HF console + `router.py` `token=HF_TOKEN`)*
-- [x] Authenticated username threaded into ADR-0008 trace. *(`get_trace()` `config`)*
-- [x] `ADMIN_IDS`/allow-list config + rejection tests (user vs admin).
-      *(`src/core/access_control.py`+`tests/test_access_control.py`;
-      `access.py`+`tests/test_access.py`)*
-
-**Deferred (not part of the now-slice):** OHSU SSO/IdP provider cutover — tracked for
-the AWS/OHSU-managed-identity milestone.

docs/adr/ADR-0013-audit-trace-redaction.md

@@ -1,125 +0,0 @@
-# ADR-0013 — Audit-Trace Redaction (Secret / Credential / PII Scrubbing)
-
-**Status:** Accepted — implemented 2026-07-02 (AWS-independent; live once deployed)
-**Date:** 2026-07-02
-**Deciders:** Annie Voigt (project lead)
-**Driver:** OHSU security review — the always-on audit trace (ADR-0008) captures
-the full prompt and the code the agent generated and executed. Anything a user
-pastes, or that generated code prints, is persisted **verbatim** to the trace
-store. Since that store *is* the audit control, a secret or identifier landing in
-it is a leak into the very artifact meant to be trusted.
-**Related:** ADR-0008 (always-on logging — defines the persist path this hooks),
-ADR-0009 (S3 sink — same redacted payload lands there at cutover), ADR-0011
-(upload de-identification attestation — dataset content; this ADR is about
-secrets/PII in prompts+code, a different surface), ADR-0012 (identity in trace —
-must itself not over-collect).
-
----
-
-## Context
-
-`CodeAgent.get_trace()` returns
-`{execution_time, config, messages, trace_logs}`, where `messages` is the full
-message history (user prompts + model turns) and `trace_logs` includes the
-generated code and captured stdout. `agent.run()` persists this on every live
-run via `persist_trace_safe(get_log_sink(), run_id, self.get_trace())`. There is
-**no redaction** anywhere in `src/logging_sink.py` or in `get_trace()`.
-
-Realistic leak paths into the trace:
-- A user pastes an API key, token, or a credentialed URL into the chat prompt.
-- Generated code echoes the environment (`print(os.environ)`), a connection
-  string, or a bearer token.
-- The ADR-0012 orchestrator→specialist **service token** appears in an error
-  string or a debug print.
-- Personal identifiers (email, name) in a prompt — the data is de-identified per
-  ADR-0011, but free-text prompts are not.
-
-The fail-open wrapper (`persist_trace_safe`) means a bad trace is written
-silently, so there is no natural backstop.
-
-## Decision
-
-Add a deterministic, **no-LLM** redaction pass applied to the trace immediately
-before it is persisted, in both the always-on sink path and the opt-in
-`save_trace` file dump.
-
-- **A pure function `redact_trace(trace: dict) -> dict`** (new
-  `src/core/trace_redaction.py`, mirroring the shared-helper pattern of
-  `src/core/integrity.py`). It deep-copies and walks all string values in
-  `messages` + `trace_logs` and replaces matches with a typed placeholder
-  (`«REDACTED:anthropic_key»`, `«REDACTED:hf_token»`, etc.).
-- **Pattern set (deterministic regex, high-precision):**
-  - Anthropic keys (`sk-ant-…`), HuggingFace tokens (`hf_…`), AWS access keys
-    (`AKIA…`) + secret-key-shaped high-entropy strings, generic `Bearer <token>`,
-    OpenAI-style `sk-…`, and credentialed URLs (`https://user:pass@…`).
-  - Whole-value drop for obvious environment dumps (a dict/text blob containing
-    multiple `KEY=VALUE` env lines) → `«REDACTED:env_dump»`.
-  - Email addresses → `«REDACTED:email»` (PII; conservative, on by default).
-- **Applied at one seam.** Hook `redact_trace` into `agent.run()` right before
-  `persist_trace_safe(...)` and before the file dump — so every sink (`local` /
-  `hf` / `s3`) and every path receives the redacted payload. The sinks stay
-  dumb; redaction is not per-sink.
-- **Fail-closed on redaction, fail-open on logging.** If `redact_trace` itself
-  raises, persist a **minimal** trace (run_id + timestamp + "redaction_error")
-  rather than the raw payload — never write an unredacted trace, but still never
-  crash the run.
-- **Size cap.** Truncate any single value over a configurable limit
-  (`TRACE_MAX_FIELD_CHARS`) so a pathological paste can't bloat the store.
-- **Config, allow tuning:** `TRACE_REDACTION` (`on` default | `off` for local
-  debug only) + an extensible extra-patterns list; **off is never the prod
-  posture** and that is documented.
-
-Redaction runs on a copy; the in-memory trace the UI/eval harness reads is
-unchanged, so no user-facing behavior changes — only what is *persisted*.
-
-## Plan
-
-- **Now (AWS-independent, ~2–3 days):**
-  1. `src/core/trace_redaction.py` with the pattern set + `redact_trace`.
-  2. Wire it into `agent.run()` before both persist paths.
-  3. Tests (`tests/test_trace_redaction.py`): each pattern is scrubbed; a
-     planted `sk-ant-…` / `hf_…` / env-dump never reaches a stub sink; redaction
-     failure yields the minimal trace, not the raw one; clean traces pass through
-     unchanged.
-  4. Document `TRACE_REDACTION` + the "off ≠ prod" note.
-- **No AWS dependency at all** — this hardens the payload *before* it reaches any
-  sink, so it is complete independent of the S3 cutover, and the S3 sink
-  (ADR-0009) inherits it for free.
-
-## Consequences
-
-- **Positive:** the audit store can no longer silently capture credentials/PII;
-  closes the leak into the trust artifact itself; a single seam covers all
-  sinks; deterministic + testable, no model call, no latency of note.
-- **Cost / caveat:** regex redaction is high-precision but not exhaustive — a
-  novel secret format can slip through, so this is defense-in-depth layered with
-  ADR-0012 (don't put the service token where it can be printed) and secret
-  hygiene (ADR-0014), not a guarantee. Over-eager patterns could redact
-  legitimate content (e.g. a gene identifier that looks token-shaped); keep
-  patterns anchored/high-entropy and cover with tests. State to the review that
-  redaction is best-effort scrubbing, not a proof of secret-free logs.
-- **Honesty note:** this reduces *accidental* capture; it is not a substitute for
-  not exposing secrets to the agent in the first place.
-
-## Start-now checklist
-- [x] `src/core/trace_redaction.py` (`redact_trace`, pattern set, size cap).
-- [x] Hook before `persist_trace_safe` (`agent.run()`) + the `save_trace` file
-      dump (`WorkflowEngine.save_trace_to_file`, which also covers `agent.save_trace()`).
-- [x] Fail-closed minimal-trace path on redaction error (`redact_trace_safe`).
-- [x] `tests/test_trace_redaction.py` (20 tests, network-free) + `TRACE_REDACTION`
-      config doc (module docstring + Decision above).
-
-## Implementation notes (2026-07-02)
-- Patterns landed: anthropic (`sk-ant-…`), hf (`hf_…`), AWS access key (`AKIA…`),
-  generic OpenAI `sk-…`, `Bearer …`, GitHub `gh[pousr]_…`, credentialed URL
-  (host preserved, `user:pass@` dropped), email, whole-value env-dump drop
-  (≥3 `UPPER_SNAKE=value` lines), and an entropy-gated 40+char base64 run for the
-  AWS *secret* key shape (`TRACE_REDACTION_EXTRA` adds operator regexes →
-  `«REDACTED:custom»`).
-- Entropy gate (≥4.0 bits/char) on the 40+char matcher keeps repetitive/low-entropy
-  identifiers (e.g. a long gene-id run) from tripping the secret pattern.
-- Size cap is `TRACE_MAX_FIELD_CHARS` (default 20 000); scrubbing runs *before*
-  truncation so a secret straddling the boundary is removed, not half-exposed.
-- Both seams call `redact_trace_safe`, which fails **closed** to a minimal trace
-  (`run_id` + timestamp + `redaction_error`, no payload) and never raises, so the
-  fail-open sink wrapper still governs crash-safety.

docs/adr/ADR-0014-ci-security-scanning.md

@@ -1,150 +0,0 @@
-# ADR-0014 — CI Security Scanning (Dependency, Static, Secret)
-
-**Status:** Accepted — in progress (specialist repo implemented 2026-07-02;
-biodata-registry + orchestrator pending)
-**Date:** 2026-07-02
-**Deciders:** Annie Voigt (project lead)
-**Driver:** OHSU security review — no automated scanning exists today. There is
-no `.github/workflows/` in the specialist, and no pip-audit / bandit / safety /
-Dependabot / gitleaks anywhere in the three repos. The `tzlocal==5.4.2`
-metadata-only-wheel break (found by hand during ADR-0007 validation, fixed
-reactively in `05fca82`) is exactly the supply-chain class automated scanning
-catches before it ships.
-**Related:** ADR-0010 (dataset integrity — data supply chain; this is the *code*
-supply chain), ADR-0012/0013 (secret hygiene — secret scanning here backstops
-them), ADR-0007 (pinned sandbox image — scan the image too).
-
----
-
-## Context
-
-Three repos, three different deploy models, which changes where CI can run:
-
-- **`DecoupleRpy_Agent`** (specialist) — **has a GitHub repo** that auto-deploys
-  to the HF Space. GitHub Actions runs natively here.
-- **`biodata-registry`** — **GitHub-only** pip package. GitHub Actions runs
-  natively.
-- **`pdac-analysis-orchestrator`** — **no GitHub repo**; `origin` *is* the prod
-  HF Space (git push builds the Space). GitHub Actions has nowhere to run, so it
-  needs a different hook (pre-push + manual/scheduled scan).
-
-None of them run any dependency-vulnerability scan, static-analysis pass, or
-secret scan. Dependencies are pinned (`requirements.txt`, `tests/
-requirements-test.txt`) but never checked against advisory databases, and
-tokens flow through the code (HF write tokens, the ADR-0012 service token) with
-no automated secret-scanning of commits or history.
-
-## Decision
-
-Add a standard three-part security scan — **dependencies, static analysis,
-secrets** — to every repo, using the hook appropriate to that repo's deploy
-model. All tooling is open-source and runs locally / in GitHub-hosted CI; **no
-AWS and no OHSU decision required.**
-
-**Scan set (same three everywhere):**
-1. **Dependencies — `pip-audit`** against `requirements*.txt` (PyPI advisory /
-   OSV). Fails on a known-vuln dependency; this is what would have surfaced a
-   bad `tzlocal` pin. Add **Dependabot** (GitHub repos) for automated bump PRs.
-2. **Static analysis — `bandit`** over `src/` (common Python security
-   anti-patterns: `exec`/`eval`, `subprocess shell=True`, insecure temp files,
-   hardcoded secrets). Scoped/tuned so the *intended* sandboxed `exec`
-   (ADR-0007) is an acknowledged, annotated finding, not noise.
-3. **Secrets — `gitleaks`** over the working tree **and full git history**
-   (tokens have flowed through these repos; history matters). Backstops
-   ADR-0012/0013.
-
-**Per-repo hook (deploy-model-aware):**
-- **`DecoupleRpy_Agent` + `biodata-registry` (GitHub):** a
-  `.github/workflows/security.yml` running the three on every PR + a weekly
-  schedule; Dependabot config committed.
-- **`pdac-analysis-orchestrator` (HF-only, no GitHub):** a `make security-scan`
-  target + a **pre-push git hook** running the same three locally before a push
-  builds the Space, plus optionally a **scheduled HF Job** (reusing the
-  `scripts/hf_job.sh` pattern already in `lit-agent`) so it also runs
-  unattended. The scan definition is shared so all repos run an identical check.
-- **Shared config** so thresholds/allow-lists (e.g. the annotated ADR-0007
-  `exec`) don't drift between repos.
-
-**Also scan the sandbox image (ADR-0007):** add a container-image vuln scan
-(`trivy`, open-source) of `docker/sandbox.Dockerfile` to the specialist workflow
-so the pinned image is checked, not just the Python deps.
-
-## Plan
-
-- **Now (AWS-independent, ~2–3 days):**
-  1. Author the shared three-scan definition + a documented allow-list for known
-     accepted findings (the ADR-0007 `exec`).
-  2. GitHub repos: commit `security.yml` (PR + weekly) + Dependabot config.
-  3. Orchestrator: `make security-scan` + pre-push hook (+ optional scheduled HF
-     Job).
-  4. Run once across all three, triage findings, fix or explicitly accept each,
-     and record the accepted set (so CI is green and every suppression is
-     justified — a clean artifact for the review).
-  5. Add `trivy` on the sandbox image to the specialist workflow.
-- **No AWS dependency.** Everything here is local tooling or GitHub-hosted
-  runners; nothing waits on the AWS account or an OHSU decision.
-
-## Consequences
-
-- **Positive:** turns "we pin dependencies" into "we pin *and* continuously
-  scan"; catches the next `tzlocal`-class break before deploy; secret scanning
-  backstops ADR-0012/0013; gives the review a concrete, always-on supply-chain
-  control across all three repos and the sandbox image.
-- **Cost / caveat:** the orchestrator's HF-only model means its scan is a
-  pre-push hook / scheduled Job rather than blocking CI — a developer *can*
-  bypass a local hook, so pair it with the scheduled unattended run and say so
-  honestly (it is not an enforced merge gate the way the GitHub repos are).
-  Initial triage will surface a backlog to accept or fix; budget for that first
-  pass. Scanners produce false positives — the annotated allow-list keeps CI
-  meaningful rather than ignored.
-- **Honesty note:** scanning reduces known-vulnerability and leaked-secret risk;
-  it does not prove the absence of either. It is one layer with ADR-0010 (data
-  supply chain), ADR-0012/0013 (secret handling), and ADR-0007 (execution
-  isolation).
-
-## Start-now checklist
-- [x] Shared three-scan definition (pip-audit + bandit + gitleaks) + accepted-
-      findings allow-list. — `scripts/security_scan.sh` (4 stages incl. trivy),
-      `bandit.yaml` + `security/bandit-baseline.json`, `.gitleaks.toml`,
-      `security/ACCEPTED-FINDINGS.md`.
-- [x] `security.yml` + Dependabot on `DecoupleRpy_Agent`. *(biodata-registry
-      still pending — separate repo; it reuses the same scan definition.)*
-- [x] `make security-scan` + pre-push hook on the specialist. *(See deploy-model
-      correction below — the specialist is HF-only, so this local/pre-push path
-      is its enforced check, same as the orchestrator; the orchestrator itself
-      still pending.)*
-- [x] `trivy` image scan on the sandbox Dockerfile. — clean (0 HIGH/CRITICAL).
-- [~] First full run triaged; every suppression justified. — **done; CI is NOT
-      green because the first run surfaced two genuine open items** (below),
-      which is the intended outcome of a first pass, not a failure of it.
-
-## Implementation notes (2026-07-02, specialist repo)
-
-**Deploy-model correction.** The Context above states the specialist "has a
-GitHub repo" where GitHub Actions runs natively. That is **not** true of the
-current repo: `origin` *is* the HuggingFace Space (no GitHub remote), so
-`.github/workflows/security.yml` will not execute on push — HF does not run
-Actions. The specialist therefore has the **same** deploy model as the
-orchestrator, and its enforced check is the **pre-push hook** (`make
-install-hooks`) running the identical `scripts/security_scan.sh`, plus the
-scheduled unattended run. The workflow + `dependabot.yml` are still committed so
-that adding a GitHub mirror (or reusing them in `biodata-registry`, a real
-GitHub repo) is zero extra work and the scan definition never diverges.
-
-**First-run findings (see `security/ACCEPTED-FINDINGS.md` for full triage):**
-1. **Exposed Google API key in git history** — `AIzaSy…` hardcoded as a Gradio
-   textbox default in the initial commit (`155d8d7` `app.py:488`), removed from
-   HEAD in `acf6553` but still in published history. **Requires rotation**
-   (revoke in GCP console); left intentionally un-allowlisted so the scan keeps
-   failing until handled. This is exactly the history-scan value the ADR argued
-   for.
-2. **Three fixable dependency CVEs** — `pillow 11.3.0` (→12.2.0), `langsmith
-   0.8.16` (→0.8.18), `pydantic-settings 2.14.1` (→2.14.2). None are HF
-   `sdk_version`-locked, so all three are bump candidates (a separate,
-   test-gated dependency change + redeploy).
-
-Two gitleaks false positives (column-name kwargs in a precompute script) were
-triaged and allowlisted. Bandit's 16 medium infra findings (0.0.0.0 dev bind,
-`/tmp` working dirs, GEO-download urlopen) are captured in the committed baseline
-so only *new* findings fail; the ADR-0007 sandboxed `exec` is inline-`# nosec`
-annotated.

gradio_ui.py

@@ -14,20 +14,19 @@ import sys
 import threading
 import time
 import traceback
-from collections.abc import Generator
 from datetime import datetime
+from typing import Generator
 
 import gradio as gr
 from gradio.themes.utils import fonts
+from langchain_core.messages import HumanMessage, AIMessage
 from langchain_anthropic import ChatAnthropic
-from langchain_core.messages import AIMessage, HumanMessage
 
 from agent import CodeAgent
-from core.access_control import check_access
 from core.constants import DECOUPLER_DISCLAIMER
 from core.types import AgentConfig
-from logging_sink import get_log_sink, persist_trace_safe
 from managers.hf_storage import HFResultsStorage
+from logging_sink import get_log_sink, persist_trace_safe
 from ui_formatting import _UIFormattingMixin
 
 # ---------------------------------------------------------------------------
@@ -37,9 +36,6 @@ from ui_formatting import _UIFormattingMixin
 _MCP_HTTP_PORT = 8765
 _mcp_server_proc: subprocess.Popen | None = None
 
-# Steps granted per press of the Continue button.
-STEP_LIMIT_INCREMENT = 15
-
 
 def _port_open(port: int, host: str = "127.0.0.1") -> bool:
     try:
@@ -104,494 +100,18 @@ def ensure_mcp_http_server() -> str:
     # port before committing to HTTP-vs-stdio. Catch a fast crash early.
     for i in range(15):
         if _mcp_server_proc.poll() is not None:
-            print(
-                f"[MCP] Server process exited early (code {_mcp_server_proc.returncode}) — "
-                "see logs above; prewarm will fall back to stdio"
-            )
+            print(f"[MCP] Server process exited early (code {_mcp_server_proc.returncode}) — "
+                  "see logs above; prewarm will fall back to stdio")
             return url
         if _port_open(_MCP_HTTP_PORT):
-            print(f"[MCP] HTTP server ready at {url} (took {i + 1}s)")
+            print(f"[MCP] HTTP server ready at {url} (took {i+1}s)")
             return url
         time.sleep(1)
 
-    print("[MCP] HTTP server not up in 15s — prewarm will keep waiting in the background")
+    print(f"[MCP] HTTP server not up in 15s — prewarm will keep waiting in the background")
     return url
 
 
-# ---------------------------------------------------------------------------
-# Upload safety gate (ADR-0011) — the UI wiring in front of src/uploads/.
-#
-# An uploaded/downloaded file is untrusted input. Before it is ever handed to
-# the agent it must clear the *security* envelope of ADR-0011:
-#   1. de-identification attestation (no PHI/PSI),
-#   2. type allow-list + size cap + quarantine + SHA-256 (stage_upload),
-#   3. structural magic-byte content scan (scan_upload) + an AV pass only where
-#      the deployment declares one (deploy/scan_posture.yaml; the Space does not).
-# Those three are MANDATORY — a failure blocks the file entirely.
-#
-# validate_upload (manifest-vs-data consistency) is run too, but treated as
-# ADVISORY for this session-scoped, single-file, read-only use: an ad-hoc
-# uploader has no hand-authored manifest, so we auto-draft one and surface any
-# validation gaps as warnings rather than blocking exploratory analysis. The
-# admin-only registration step (promotion into the shared biodata-registry) is
-# a separate governance action and is intentionally NOT part of this path —
-# a session upload is read by the agent's vetted loaders for that session only,
-# never added to the registry or exposed to other users.
-# ---------------------------------------------------------------------------
-def run_upload_gate(src_path, filename, session_state, deidentified):
-    """Route an upload/download through the ADR-0011 safety gate.
-
-    Returns ``(session_state, status_markdown)``. On any *security* failure the
-    file is NOT made available (``uploaded_file`` stays unset). On a
-    manifest-validation-only gap the file is allowed for session use with a
-    visible warning.
-    """
-    from src.uploads import (
-        SCAN_INFECTED,
-        UploadRejected,
-        attach_manifest,
-        discard_upload,
-        draft_manifest,
-        placeholder_manifest,
-        scan_upload,
-        stage_upload,
-        validate_upload,
-    )
-    from src.uploads.drafting import _slug
-    from src.uploads.records import STATUS_VALIDATED
-
-    def _block(msg):
-        session_state.pop("uploaded_file", None)
-        session_state.pop("uploaded_filename", None)
-        session_state.pop("upload_record_id", None)
-        return session_state, msg
-
-    # ── Gate 1: de-identification attestation (ADR-0011 item 6) ──────────────
-    if not deidentified:
-        return _block(
-            "⚠️ **Upload blocked.** Tick *“I confirm this data is de-identified"
-            " (no PHI/PSI)”* above before uploading — an upload without that"
-            " attestation is refused at the door (ADR-0011)."
-        )
-
-    uploader = session_state.get("principal") or "ui-upload"
-    dataset_id = f"upload_{_slug(filename)}"
-
-    # A metadata workbook is not an analysable dataset on its own — it only has
-    # meaning joined to a counts matrix. Say so here rather than letting it
-    # quarantine successfully and then fail in the manifest drafter.
-    if filename.lower().endswith(".xlsx"):
-        return _block(
-            "⚠️ **An Excel workbook is metadata, not a dataset.** Use the"
-            " *Assemble from TSVs* tab, where the sheet is joined to a counts"
-            " matrix to build the analysis file."
-        )
-
-    # ── Gate 2: quarantine + type/size + SHA-256 (stage_upload) ──────────────
-    # Staged with a placeholder manifest, because drafting the real one means
-    # *parsing* the file, and ADR-0011 requires that no parser touch an
-    # unstaged, unscanned, un-type-checked file. The draft is attached below,
-    # after the content scan clears.
-    try:
-        record = stage_upload(
-            src_path,
-            uploader=uploader,
-            dataset_id=dataset_id,
-            manifest=placeholder_manifest(dataset_id),
-            deidentified=True,
-        )
-    except UploadRejected as rej:
-        return _block("⚠️ **Upload rejected:** " + "; ".join(rej.record.errors))
-
-    # ── Gate 3: content scan (structural magic-byte + optional ClamAV) ───────
-    # Runs on the quarantined copy *before* anything reads its contents, so a
-    # disguised binary (an ELF wearing a .csv suffix) is stopped here rather
-    # than surfacing later as an incidental parser crash.
-    record, _scan_report = scan_upload(record)
-    if record.scan_status == SCAN_INFECTED:
-        discard_upload(record, "blocked by content scan")
-        return _block("🛑 **Upload blocked by content scan:** " + record.scan_detail)
-
-    # ── Auto-draft a manifest so the uploader faces no blank form ────────────
-    # Reads the *quarantined* copy through the vetted loaders, never the original.
-    try:
-        draft = draft_manifest(record.staged_path, dataset_id=dataset_id)
-    except Exception as exc:  # noqa: BLE001 — unreadable/unsupported file
-        discard_upload(record, f"could not draft a manifest: {exc}")
-        return _block(f"⚠️ **Upload blocked:** could not read `{filename}` — {exc}")
-    record = attach_manifest(record, draft.manifest)
-
-    # ── Advisory: manifest-vs-data validation ───────────────────────────────
-    # validate_upload re-checks scan status internally (idempotent) then runs the
-    # manifest/against-data gate. A validation gap does not block session use.
-    record, _val_report = validate_upload(record)
-
-    session_state["uploaded_file"] = record.staged_path
-    session_state["uploaded_filename"] = filename
-    session_state["upload_record_id"] = record.upload_id
-
-    lines = [f"✅ **Ready:** `{filename}`"]
-    if record.scan_status == "skipped":
-        lines.append(
-            "_Structurally checked, but **not malware-scanned** — this deployment"
-            " ships no anti-virus binary (declared posture `structural_only`, see"
-            " `deploy/scan_posture.yaml`). Treat this file as structurally-verified"
-            " only._"
-        )
-    elif record.scan_status == "clean" and record.scan_posture == "structural_only":
-        lines.append(
-            "_Structurally checked; an anti-virus on this host also reported clean,"
-            " but the deployment does not guarantee an AV pass (declared posture"
-            " `structural_only`) — production uploads are structure-checked only._"
-        )
-    if record.status != STATUS_VALIDATED and record.errors:
-        lines.append(
-            "_Manifest not fully validated (auto-drafted): "
-            + "; ".join(record.errors)
-            + ". The agent will still load the file; confirm the analysis matches"
-            " your data._"
-        )
-    elif record.caveats:
-        lines.append("_Caveats:_ " + "; ".join(record.caveats))
-    return session_state, "\n\n".join(lines)
-
-
-# ---------------------------------------------------------------------------
-# Data-Input handlers (module level so they are unit-testable — they close over
-# no UI state). All three funnel into run_upload_gate above.
-#
-# The URL and HF-Dataset paths must first materialise remote bytes on local disk
-# before the gate can hash and quarantine them. That scratch copy is *ungated*
-# untrusted input, so it goes to a private temporary directory that is deleted
-# unconditionally once the gate has taken its own quarantined copy — it is never
-# left behind in a persistent tmp/inputs/ dir for the agent to stumble onto.
-# ---------------------------------------------------------------------------
-def _incomplete_run_notice(reason: str | None, config) -> str:
-    """Explain why a run ended without an answer, and what actually helps.
-
-    A run can end incomplete four ways, and only ONE of them is fixed by
-    granting more steps. The UI used to call all of them "Step limit reached",
-    which sent users to press Continue against a wall clock or a failing tool —
-    Continue would grant steps a timed-out run has no use for.
-    """
-    step_budget = getattr(config, "max_steps", 15)
-    timeout_min = round(getattr(config, "timeout_seconds", 1200) / 60)
-
-    if reason == "step_limit":
-        body = (
-            f"⏸ <strong>Step limit reached</strong> ({step_budget} steps). "
-            f"Click <strong>Continue</strong> to give the agent {STEP_LIMIT_INCREMENT} more steps."
-        )
-    elif reason == "timeout":
-        body = (
-            f"⏱ <strong>Time limit reached</strong> (~{timeout_min} min). The agent was still "
-            "working, so <strong>Continue will not help</strong> — it grants more steps, not more "
-            "time. Re-run with a narrower question (one dataset, one contrast), or raise the "
-            "timeout if this analysis is genuinely long-running."
-        )
-    elif reason == "error_limit":
-        body = (
-            "⚠️ <strong>Stopped after repeated errors.</strong> The agent hit "
-            f"{getattr(config, 'retry_attempts', 3)} consecutive failures and gave up rather than "
-            "guess. The errors are shown above — Continue is unlikely to help until the underlying "
-            "failure is addressed."
-        )
-    else:
-        body = (
-            "⚠️ <strong>The run ended without an answer.</strong> The agent stopped without "
-            "producing a solution or any further code to run. Try rephrasing the question."
-        )
-
-    return (
-        '<div style="background-color:#dbeafe;border-left:4px solid #2563eb;'
-        'padding:10px 14px;margin:10px 0;border-radius:4px;font-size:0.9em;color:#1e3a5f;">'
-        f"{body}</div>"
-    )
-
-
-def _clear_upload(session_state, msg):
-    session_state.pop("uploaded_file", None)
-    session_state.pop("uploaded_filename", None)
-    session_state.pop("upload_record_id", None)
-    return session_state, msg
-
-
-def _gate_component_file(src_path, filename, uploader, dataset_id):
-    """Quarantine + content-scan ONE input file of a multi-file assembly.
-
-    Returns the staged path. The component files (counts TSV, TPM TSV, metadata
-    workbook) are not datasets in their own right — no manifest is drafted and
-    no validation runs for them; the assembled h5ad goes through the full gate
-    afterwards. What matters here is that nothing reads a byte of them until
-    :func:`stage_upload` has type/size-checked and hashed the file and
-    :func:`scan_upload` has cleared its structure.
-
-    Raises ``ValueError`` with a user-facing message on any gate failure.
-    """
-    from src.uploads import (
-        SCAN_INFECTED,
-        UploadRejected,
-        discard_upload,
-        placeholder_manifest,
-        scan_upload,
-        stage_upload,
-    )
-
-    try:
-        record = stage_upload(
-            src_path,
-            uploader=uploader,
-            dataset_id=dataset_id,
-            manifest=placeholder_manifest(dataset_id),
-            deidentified=True,
-        )
-    except UploadRejected as rej:
-        raise ValueError(f"`{filename}` rejected: " + "; ".join(rej.record.errors)) from rej
-
-    record, _report = scan_upload(record)
-    if record.scan_status == SCAN_INFECTED:
-        discard_upload(record, "blocked by content scan")
-        raise ValueError(f"`{filename}` blocked by content scan: {record.scan_detail}")
-    return record.staged_path
-
-
-def run_assembly_gate(
-    counts_path,
-    metadata_path,
-    tpm_path,
-    session_state,
-    deidentified,
-    *,
-    sample_column=None,
-    skip_rows=None,
-    column_map=None,
-    value_maps=None,
-    group_column=None,
-    control_label=None,
-    treatment_label="shMyc",
-):
-    """Build the analysis h5ad from raw delivery files, all inside the gate.
-
-    Replaces the hand-run `scripts/assemble_myc_kd_kmc_mouse.py` pre-step: the
-    user drops the counts TSV (+ optional TPM TSV) and the metadata sheet, and
-    the assembly happens here. Each input is quarantined and content-scanned
-    *before* it is parsed; the assembled h5ad is then put through the ordinary
-    upload gate, so the session-visible file is a normal staged upload with its
-    own record, SHA-256, drafted manifest and advisory validation.
-    """
-    import shutil as _shutil
-    import tempfile
-
-    from src.uploads.assembly import AssemblyError, assemble_h5ad
-    from src.uploads.drafting import _slug
-
-    def _block(msg):
-        session_state.pop("uploaded_file", None)
-        session_state.pop("uploaded_filename", None)
-        session_state.pop("upload_record_id", None)
-        return session_state, msg
-
-    if not deidentified:
-        return _block(
-            "⚠️ **Assembly blocked.** Tick *“I confirm this data is de-identified"
-            " (no PHI/PSI)”* above first — an upload without that attestation is"
-            " refused at the door (ADR-0011)."
-        )
-    if not counts_path or not metadata_path:
-        return _block("Provide at least a counts matrix **and** a metadata sheet.")
-
-    uploader = session_state.get("principal") or "ui-upload"
-    base_id = f"upload_{_slug(os.path.basename(counts_path))}"
-
-    try:
-        staged_counts = _gate_component_file(
-            counts_path, os.path.basename(counts_path), uploader, f"{base_id}_counts"
-        )
-        staged_meta = _gate_component_file(
-            metadata_path, os.path.basename(metadata_path), uploader, f"{base_id}_metadata"
-        )
-        staged_tpm = (
-            _gate_component_file(tpm_path, os.path.basename(tpm_path), uploader, f"{base_id}_tpm")
-            if tpm_path
-            else None
-        )
-    except ValueError as exc:
-        return _block(f"🛑 **Assembly blocked:** {exc}")
-
-    workdir = tempfile.mkdtemp(prefix="assembled_")
-    try:
-        out_path = os.path.join(workdir, "assembled.h5ad")
-        try:
-            _adata, report = assemble_h5ad(
-                staged_counts,
-                staged_meta,
-                tpm_path=staged_tpm,
-                out_path=out_path,
-                sample_column=sample_column or None,
-                skip_rows=int(skip_rows) if str(skip_rows or "").strip() else None,
-                column_map=column_map or None,
-                value_maps=value_maps or None,
-                group_column=group_column or None,
-                control_label=control_label or None,
-                treatment_label=treatment_label or "shMyc",
-                staging_script="gradio_ui.run_assembly_gate",
-            )
-        except AssemblyError as exc:
-            return _block(f"⚠️ **Could not assemble the dataset:** {exc}")
-        except Exception as exc:  # noqa: BLE001 — unreadable/unsupported input
-            return _block(f"⚠️ **Could not read the supplied files:** {exc}")
-
-        session_state, status = run_upload_gate(
-            out_path, "assembled.h5ad", session_state, deidentified
-        )
-    finally:
-        _shutil.rmtree(workdir, ignore_errors=True)
-
-    if "uploaded_file" not in session_state:
-        return session_state, status
-
-    detail = [
-        status,
-        f"_Assembled **{report['n_samples']} samples × {report['n_genes']} genes**"
-        f" from `{os.path.basename(counts_path)}`"
-        + (f" + `{os.path.basename(tpm_path)}` (TPM layer)" if tpm_path else "")
-        + f" + `{os.path.basename(metadata_path)}`._",
-        "_Groups:_ "
-        + "; ".join(f"**{col}** {counts}" for col, counts in report["obs_counts"].items()),
-    ]
-    if report["unmatched_samples"]:
-        detail.append(
-            f"_⚠️ {len(report['unmatched_samples'])} matrix sample(s) had no metadata row and"
-            f" were dropped: {', '.join(report['unmatched_samples'])}._"
-        )
-    return session_state, "\n\n".join(detail)
-
-
-def handle_assembly(
-    counts_file,
-    metadata_file,
-    tpm_file,
-    sample_column,
-    column_map,
-    value_maps,
-    group_column,
-    control_label,
-    treatment_label,
-    skip_rows,
-    deidentified,
-    session_state,
-):
-    """Gradio binding for the *Assemble from TSVs* tab."""
-    return run_assembly_gate(
-        counts_file,
-        metadata_file,
-        tpm_file,
-        session_state,
-        deidentified,
-        sample_column=sample_column,
-        skip_rows=skip_rows,
-        column_map=column_map,
-        value_maps=value_maps,
-        group_column=group_column,
-        control_label=control_label,
-        treatment_label=treatment_label,
-    )
-
-
-def handle_file_upload(file_path, deidentified, session_state):
-    """Gradio's own upload widget already wrote the file to a temp path."""
-    if file_path is None:
-        return _clear_upload(session_state, "")
-    return run_upload_gate(file_path, os.path.basename(file_path), session_state, deidentified)
-
-
-def handle_url_download(url, deidentified, session_state):
-    import gzip
-    import shutil as _shutil
-    import tempfile
-
-    import requests
-
-    from src.uploads.staging import _max_upload_bytes
-
-    if not url or not url.strip():
-        return session_state, "No URL provided"
-    url = url.strip()
-    if not url.lower().startswith(("http://", "https://")):
-        return _clear_upload(session_state, "Only http:// and https:// URLs are supported.")
-    filename = url.rstrip("/").split("/")[-1].split("?")[0]
-    if not filename or "." not in filename:
-        filename = "downloaded_data.bin"
-
-    scratch = tempfile.mkdtemp(prefix="ungated_url_")
-    try:
-        dest = os.path.join(scratch, os.path.basename(filename))
-        limit = _max_upload_bytes()
-        r = requests.get(url, stream=True, timeout=300)
-        r.raise_for_status()
-        if "text/html" in r.headers.get("Content-Type", ""):
-            return _clear_upload(
-                session_state,
-                "URL returned an HTML page, not a file. Make sure the URL points"
-                " directly to a file, not a directory.",
-            )
-        # Enforce the size cap *while* streaming, so an oversized (or endless)
-        # response is abandoned rather than fully written and rejected after.
-        written = 0
-        with open(dest, "wb") as f:
-            for chunk in r.iter_content(chunk_size=8192):
-                if not chunk:
-                    continue
-                written += len(chunk)
-                if written > limit:
-                    return _clear_upload(
-                        session_state,
-                        f"⚠️ **Download aborted:** the file exceeds the"
-                        f" {limit}-byte upload limit (UPLOAD_MAX_BYTES).",
-                    )
-                f.write(chunk)
-        if filename.endswith(".gz") and not filename.endswith(".tar.gz"):
-            decompressed = filename[:-3]
-            decompressed_dest = os.path.join(scratch, decompressed)
-            with gzip.open(dest, "rb") as f_in, open(decompressed_dest, "wb") as f_out:
-                _shutil.copyfileobj(f_in, f_out)
-            os.remove(dest)
-            dest, filename = decompressed_dest, decompressed
-        return run_upload_gate(dest, filename, session_state, deidentified)
-    except Exception as e:  # noqa: BLE001 — surfaced to the uploader
-        return _clear_upload(session_state, f"Download failed: {e}")
-    finally:
-        _shutil.rmtree(scratch, ignore_errors=True)
-
-
-def handle_hf_dataset(repo_id, filepath, deidentified, session_state):
-    import shutil as _shutil
-    import tempfile
-
-    if not repo_id or not repo_id.strip():
-        return session_state, "No repo ID provided"
-    if not filepath or not filepath.strip():
-        return session_state, "No file path provided"
-
-    scratch = tempfile.mkdtemp(prefix="ungated_hf_")
-    try:
-        from huggingface_hub import hf_hub_download
-
-        local_path = hf_hub_download(
-            repo_id=repo_id.strip(),
-            filename=filepath.strip(),
-            repo_type="dataset",
-            local_dir=scratch,
-        )
-        return run_upload_gate(
-            local_path, os.path.basename(local_path), session_state, deidentified
-        )
-    except Exception as e:  # noqa: BLE001 — surfaced to the uploader
-        return _clear_upload(session_state, f"Failed to load from HF Dataset: {e}")
-    finally:
-        _shutil.rmtree(scratch, ignore_errors=True)
-
-
 class GradioAgentUI(_UIFormattingMixin):
     """
     Gradio interface for interacting with the LangGraph ReAct Agent.
@@ -603,8 +123,9 @@ class GradioAgentUI(_UIFormattingMixin):
         if model is None:
             # Prefer the standard ANTHROPIC_API_KEY; fall back to the legacy
             # mixed-case name some older Spaces still use.
-            api_key_anthropic = os.environ.get("ANTHROPIC_API_KEY") or os.environ.get(
-                "Anthropic_API_KEY"
+            api_key_anthropic = (
+                os.environ.get("ANTHROPIC_API_KEY")
+                or os.environ.get("Anthropic_API_KEY")
             )
             if not api_key_anthropic:
                 raise RuntimeError(
@@ -612,11 +133,18 @@ class GradioAgentUI(_UIFormattingMixin):
                     "or Anthropic_API_KEY in the Space secrets."
                 )
             model = ChatAnthropic(
-                model="claude-sonnet-4-6", temperature=0, api_key=api_key_anthropic
+                model='claude-sonnet-4-6',
+                temperature=0,
+                api_key=api_key_anthropic
             )
 
         if config is None:
-            config = AgentConfig(max_steps=15, retry_attempts=3, timeout_seconds=2700, verbose=True)
+            config = AgentConfig(
+                max_steps=15,
+                retry_attempts=3,
+                timeout_seconds=1200,
+                verbose=True
+            )
 
         self.model = model
         self.config = config
@@ -632,7 +160,6 @@ class GradioAgentUI(_UIFormattingMixin):
         except Exception as e:
             print(f"[log_sink] init failed ({e}); falling back to local sink")
             from logging_sink import LocalLogSink
-
             self.log_sink = LocalLogSink()
         print(f"[log_sink] Using sink: {self.log_sink.name}")
 
@@ -640,7 +167,7 @@ class GradioAgentUI(_UIFormattingMixin):
         # doesn't pay the 2-minute subprocess startup cost.
         # The seed agent's mcp_functions (stateless closures) are cached and
         # copied into each new session agent without re-spawning anything.
-        self._mcp_cache: dict = {}  # tool_name → mcp_functions entry
+        self._mcp_cache: dict = {}           # tool_name → mcp_functions entry
         self._mcp_discovery_errors: dict = {}  # server_name -> error string, if discovery failed
         self._mcp_ready = threading.Event()  # set when pre-warm finishes
 
@@ -727,10 +254,8 @@ class GradioAgentUI(_UIFormattingMixin):
                 if not seed.tool_manager.mcp_manager.mcp_functions:
                     # HTTP discovery returned nothing — fall back to stdio so the
                     # Space is still usable (just slower) rather than tool-less.
-                    print(
-                        "[mcp-prewarm] ⚠️  HTTP discovery returned 0 tools — "
-                        "falling back to stdio add_mcp()"
-                    )
+                    print("[mcp-prewarm] ⚠️  HTTP discovery returned 0 tools — "
+                          "falling back to stdio add_mcp()")
                     self._mcp_http_url = None
                     self._probe_stdio_server()
                     seed.add_mcp(mcp_config_path)
@@ -751,17 +276,13 @@ class GradioAgentUI(_UIFormattingMixin):
                 # add_mcp() didn't raise, but discovered zero tools — this is
                 # just as broken as an exception (agent will have no analysis
                 # tools), so log it just as loudly.
-                print(
-                    "[mcp-prewarm] ⚠️  add_mcp() completed but discovered 0 MCP "
-                    "tools — analysis requests will fail. Check server.py / "
-                    "mcp_config.yaml on this Space. "
-                    f"Discovery errors: {self._mcp_discovery_errors}"
-                )
+                print("[mcp-prewarm] ⚠️  add_mcp() completed but discovered 0 MCP "
+                      "tools — analysis requests will fail. Check server.py / "
+                      "mcp_config.yaml on this Space. "
+                      f"Discovery errors: {self._mcp_discovery_errors}")
         except Exception as e:
-            print(
-                "[mcp-prewarm] ⚠️  Pre-warm failed — will retry discovery on "
-                "first query. Full traceback:"
-            )
+            print("[mcp-prewarm] ⚠️  Pre-warm failed — will retry discovery on "
+                  "first query. Full traceback:")
             traceback.print_exc()
             self._mcp_discovery_errors["_prewarm"] = f"{type(e).__name__}: {e}"
         finally:
@@ -828,8 +349,8 @@ class GradioAgentUI(_UIFormattingMixin):
         previous_plan = None
         step_timings = {}
         last_state = None
-        last_error = None  # buffer transient step errors; only the last is
-        solution_shown = False  # surfaced, and only if the run yields no solution
+        last_error = None        # buffer transient step errors; only the last is
+        solution_shown = False   # surfaced, and only if the run yields no solution
 
         # Monotonic step numbering for the UI, decoupled from the raw graph
         # step_count. The raw count is emitted twice per step (once for the
@@ -839,10 +360,11 @@ class GradioAgentUI(_UIFormattingMixin):
         # the prior run on /handle_continue so the continuation keeps counting
         # up; reset to 0 for a fresh question.
         display_step = (
-            getattr(agent.workflow_engine, "last_display_step", 0) if resume_messages else 0
+            getattr(agent.workflow_engine, "last_display_step", 0)
+            if resume_messages else 0
         )
-        last_internal_step = None  # raw step_count we last opened a header for
-        header_pending = False  # a step started; flush its header before content
+        last_internal_step = None   # raw step_count we last opened a header for
+        header_pending = False      # a step started; flush its header before content
         header_drawn = bool(resume_messages)  # leading <hr> before the first step?
 
         # Snapshot existing figures so we can detect ones this run produces and
@@ -876,29 +398,24 @@ class GradioAgentUI(_UIFormattingMixin):
                     # header is gated on it, so a deduped/empty yield no longer
                     # leaves an orphan "Step N" with no body beneath it.
                     thinking_text = ""
-                    if parts["thinking"] and len(parts["thinking"]) > 20:
-                        _t = re.sub(r"(Thinking:|Plan:)\s*", "", parts["thinking"]).strip()
-                        _t = re.sub(r"\n\s*\n\s*\n+", "\n\n", _t).strip()
-                        _t = "\n".join(ln.strip() for ln in _t.split("\n") if ln.strip())
+                    if parts['thinking'] and len(parts['thinking']) > 20:
+                        _t = re.sub(r'(Thinking:|Plan:)\s*', '', parts['thinking']).strip()
+                        _t = re.sub(r'\n\s*\n\s*\n+', '\n\n', _t).strip()
+                        _t = '\n'.join(ln.strip() for ln in _t.split('\n') if ln.strip())
                         if _t and hash(_t) not in displayed_reasoning:
                             thinking_text = _t
 
                     # Route an errored observation to the buffered-error path before
                     # it can count as renderable content (see the long note below).
-                    if parts["observation"]:
-                        _obs = re.sub(r"^\s*Code Output:\s*", "", parts["observation"]).strip()
-                        if _obs.startswith("Error:"):
+                    if parts['observation']:
+                        _obs = re.sub(r'^\s*Code Output:\s*', '', parts['observation']).strip()
+                        if _obs.startswith('Error:'):
                             last_error = _obs
-                            parts["observation"] = None
+                            parts['observation'] = None
 
                     plan_changed = bool(current_plan and current_plan != previous_plan)
-                    has_content = bool(
-                        thinking_text
-                        or plan_changed
-                        or parts["code"]
-                        or parts["observation"]
-                        or parts["solution"]
-                    )
+                    has_content = bool(thinking_text or plan_changed or parts['code']
+                                       or parts['observation'] or parts['solution'])
 
                     # One header per logical step. graph.stream emits a state after
                     # both the generate node (step N) and the execute node (still
@@ -916,15 +433,13 @@ class GradioAgentUI(_UIFormattingMixin):
                         display_step += 1
                         separator = (
                             '<hr style="border: none; border-top: 1px solid #e2e8f0; '
-                            'margin: 20px 0;">'
-                            if header_drawn
-                            else ""
+                            'margin: 20px 0;">' if header_drawn else ''
                         )
                         header_drawn = True
                         yield gr.ChatMessage(
                             role="assistant",
                             content=f"{separator}## Step {display_step}\n",
-                            metadata={"status": "done"},
+                            metadata={"status": "done"}
                         )
 
                     if thinking_text:
@@ -933,35 +448,37 @@ class GradioAgentUI(_UIFormattingMixin):
 <div style="margin: 0; white-space: pre-line; font-family: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; color: #000000; background-color: #ffffff; padding: 10px; border-radius: 4px; font-size: 14px; line-height: 1.6;">{thinking_text}</div>
 </div>"""
                         yield gr.ChatMessage(
-                            role="assistant", content=thinking_block, metadata={"status": "done"}
+                            role="assistant",
+                            content=thinking_block,
+                            metadata={"status": "done"}
                         )
                         displayed_reasoning.add(hash(thinking_text))
 
                     if current_plan and current_plan != previous_plan:
-                        formatted_plan = (
-                            current_plan.replace("[ ]", "☐")
-                            .replace("[✓]", "✅")
-                            .replace("[✗]", "❌")
-                        )
+                        formatted_plan = current_plan.replace('[ ]', '☐').replace('[✓]', '✅').replace('[✗]', '❌')
                         plan_block = f"""<div style="background-color: #f8f9fa; border-left: 4px solid #000000; padding: 12px; margin: 10px 0; border-radius: 4px;">
 <div style="font-weight: bold; color: #000000; margin-bottom: 8px;">📋 Current Plan</div>
 <pre style="margin: 0; white-space: pre-wrap; font-family: 'Consolas', 'Monaco', 'Courier New', monospace; color: #000000; background-color: #ffffff; padding: 10px; border-radius: 4px; font-size: 14px; line-height: 1.6;">{formatted_plan}</pre>
 </div>"""
                         yield gr.ChatMessage(
-                            role="assistant", content=plan_block, metadata={"status": "done"}
+                            role="assistant",
+                            content=plan_block,
+                            metadata={"status": "done"}
                         )
                         previous_plan = current_plan
 
-                    if parts["code"]:
+                    if parts['code']:
                         code_block = f"""<div style="background-color: #f7fafc; border-left: 4px solid #48bb78; padding: 12px; margin: 10px 0; border-radius: 4px;">
 <div style="font-weight: bold; color: #22543d; margin-bottom: 8px;">⚡ Executing Code</div>
 </div>
 
 ```python
-{parts["code"]}
+{parts['code']}
 ```"""
                         yield gr.ChatMessage(
-                            role="assistant", content=code_block, metadata={"status": "done"}
+                            role="assistant",
+                            content=code_block,
+                            metadata={"status": "done"}
                         )
 
                     # An execution that raised is returned by the executor as
@@ -969,8 +486,8 @@ class GradioAgentUI(_UIFormattingMixin):
                     # buffered-error path above (parts['observation'] nulled) so it
                     # never renders an alarming "Code Output: Error" block for a
                     # hiccup the agent usually self-corrects on the next step.
-                    if parts["observation"]:
-                        truncated = self.truncate_output(parts["observation"])
+                    if parts['observation']:
+                        truncated = self.truncate_output(parts['observation'])
                         result_block = f"""<div style="background-color: #fef5e7; border-left: 4px solid #f6ad55; padding: 12px; margin: 10px 0; border-radius: 4px;">
 <div style="font-weight: bold; color: #744210; margin-bottom: 8px;">📊 Execution Result</div>
 </div>
@@ -979,30 +496,32 @@ class GradioAgentUI(_UIFormattingMixin):
 {truncated}
 ```"""
                         yield gr.ChatMessage(
-                            role="assistant", content=result_block, metadata={"status": "done"}
+                            role="assistant",
+                            content=result_block,
+                            metadata={"status": "done"}
                         )
 
-                        all_artifacts = self._extract_artifacts(parts["observation"])
+                        all_artifacts = self._extract_artifacts(parts['observation'])
                         for desc, path in all_artifacts:
-                            if path.endswith(".png") and os.path.exists(path):
-                                with open(path, "rb") as f:
+                            if path.endswith('.png') and os.path.exists(path):
+                                with open(path, 'rb') as f:
                                     img_b64 = base64.b64encode(f.read()).decode()
                                 yield gr.ChatMessage(
                                     role="assistant",
                                     content=self._figure_html(desc, img_b64),
-                                    metadata={"status": "done"},
+                                    metadata={"status": "done"}
                                 )
                                 shown_fig_names.add(os.path.basename(path))
                         self.hf_storage.upload_artifacts(all_artifacts, run_id)
 
-                    if parts["solution"]:
+                    if parts['solution']:
                         solution_shown = True
                         # The standing decoupleR method limitations are appended
                         # deterministically here, after the model's text — never
                         # generated by the model — so they are identical on every
                         # run and can never be softened or dropped. The model
                         # writes only run-specific caveats.
-                        solution_text = parts["solution"] + "\n\n" + DECOUPLER_DISCLAIMER
+                        solution_text = parts['solution'] + "\n\n" + DECOUPLER_DISCLAIMER
                         solution_block = f"""<div style="background-color: #d1fae5; border-left: 4px solid #10b981; padding: 16px; margin: 10px 0; border-radius: 6px;">
 <div style="font-weight: bold; color: #065f46; margin-bottom: 12px; font-size: 16px;">✅ Final Solution</div>
 <div class="solution-content" style="color: #1f2937 !important; background-color: #ffffff; padding: 16px; border-radius: 4px; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif; font-size: 15px; line-height: 1.7; border: 1px solid #e5e7eb;">
@@ -1048,22 +567,27 @@ class GradioAgentUI(_UIFormattingMixin):
 </style>
 </div>"""
                         yield gr.ChatMessage(
-                            role="assistant", content=solution_block, metadata={"status": "done"}
+                            role="assistant",
+                            content=solution_block,
+                            metadata={"status": "done"}
                         )
 
-                    if parts["error"]:
+                    if parts['error']:
                         # Buffer, don't render. These are mostly transient
                         # NameError/AttributeError that the CodeAgent hits and then
                         # self-corrects on a later step — showing each one floods the
                         # user with red boxes for problems that were already resolved.
                         # The last error is surfaced after the loop only if the run
                         # never produced a solution (i.e. it genuinely failed).
-                        last_error = parts["error"]
+                        last_error = parts['error']
 
-                    if parts["observation"] and step_count in step_timings:
-                        footnote = f'<div style="color: #718096; font-size: 0.875em; margin-top: 8px;">Step {display_step}</div>'
+                    if parts['observation'] and step_count in step_timings:
+                        duration = time.time() - step_timings[step_count]
+                        footnote = f'<div style="color: #718096; font-size: 0.875em; margin-top: 8px;">Step {display_step} | Duration: {duration:.2f}s</div>'
                         yield gr.ChatMessage(
-                            role="assistant", content=footnote, metadata={"status": "done"}
+                            role="assistant",
+                            content=footnote,
+                            metadata={"status": "done"}
                         )
 
                     # (The inter-step <hr> divider is emitted as part of the next
@@ -1096,7 +620,7 @@ class GradioAgentUI(_UIFormattingMixin):
                 yield gr.ChatMessage(
                     role="assistant",
                     content=self._figure_html(desc, img_b64),
-                    metadata={"status": "done"},
+                    metadata={"status": "done"}
                 )
                 shown_fig_names.add(os.path.basename(path))
             if new_figures:
@@ -1111,7 +635,9 @@ class GradioAgentUI(_UIFormattingMixin):
 <div style="color: #742a2a;">{last_error}</div>
 </div>"""
                 yield gr.ChatMessage(
-                    role="assistant", content=error_block, metadata={"status": "done"}
+                    role="assistant",
+                    content=error_block,
+                    metadata={"status": "done"}
                 )
 
         except Exception as e:
@@ -1119,57 +645,23 @@ class GradioAgentUI(_UIFormattingMixin):
 <div style="font-weight: bold; color: #742a2a; margin-bottom: 8px;">💥 Critical Error</div>
 <div style="color: #742a2a;">Error during agent execution: {str(e)}</div>
 </div>"""
-            yield gr.ChatMessage(role="assistant", content=error_block, metadata={"status": "done"})
+            yield gr.ChatMessage(
+                role="assistant",
+                content=error_block,
+                metadata={"status": "done"}
+            )
 
-    def interact_with_agent(
-        self,
-        query: str,
-        chatbot_history: list,
-        session_state: dict,
-        principal: str = "",
-        profile: gr.OAuthProfile | None = None,
-    ) -> Generator:
-        """Handle interaction with the agent.
-
-        ``principal`` is the authenticated end-user identity forwarded by the
-        orchestrator (ADR-0012), which reaches this endpoint via its service
-        token. ``profile`` is injected by Gradio from HF OAuth for a **direct**
-        human on this Space's own UI (it is not part of the API inputs, so the
-        orchestrator's ``/interact_with_agent`` call is unaffected).
-
-        The effective caller is the direct human's OAuth username if present,
-        else the orchestrator-forwarded principal. When ``ACCESS_CONTROL`` is
-        enforced, a non-allow-listed or unauthenticated effective caller is
-        refused before any work — this closes the direct-UI side door around the
-        orchestrator's gate. Enforcement is OFF by default, so the existing
-        orchestrator route is byte-identical until an operator turns it on (and
-        then must list the forwarded identities here too). The effective identity
-        is recorded in the ADR-0008 audit trace via ``agent.set_principal``.
-        """
+    def interact_with_agent(self, query: str, chatbot_history: list, session_state: dict) -> Generator:
+        """Handle interaction with the agent."""
         original_query = query
 
-        # ADR-0012 app-layer gate. No-op when ACCESS_CONTROL enforcement is off.
-        username = getattr(profile, "username", None) if profile else None
-        effective_principal = username or principal
-        allowed, denial = check_access(effective_principal)
-        if not allowed:
-            chatbot_history = chatbot_history + [
-                gr.ChatMessage(role="user", content=original_query, metadata={"status": "done"}),
-                gr.ChatMessage(role="assistant", content=denial, metadata={"status": "done"}),
-            ]
-            yield chatbot_history
-            return
-        # A signed-in human's identity supersedes an empty forwarded principal so
-        # the audit trace attributes the run to whoever actually ran it.
-        principal = effective_principal
-
         uploaded_file = session_state.get("uploaded_file")
         if uploaded_file and os.path.exists(uploaded_file):
             filename = session_state.get("uploaded_filename", os.path.basename(uploaded_file))
             is_geo = False
             if filename.endswith((".txt", ".tsv")):
                 try:
-                    with open(uploaded_file, encoding="utf-8", errors="replace") as _f:
+                    with open(uploaded_file, "r", encoding="utf-8", errors="replace") as _f:
                         is_geo = _f.readline().startswith("!")
                 except Exception:
                     pass
@@ -1212,7 +704,6 @@ class GradioAgentUI(_UIFormattingMixin):
                 # Re-register into the tool catalog so the agent can call them.
                 for tool_name, tool_data in self._mcp_cache.items():
                     from managers.tools.tool_manager import ToolInfo, ToolSource
-
                     tool_info = ToolInfo(
                         name=tool_name,
                         description=tool_data.get("description", "MCP tool"),
@@ -1225,9 +716,7 @@ class GradioAgentUI(_UIFormattingMixin):
                         schema=None,
                     )
                     mgr._tool_catalog[tool_name] = tool_info
-                print(
-                    f"✅ Loaded {len(self._mcp_cache)} MCP tools from cache (no subprocess spawn)"
-                )
+                print(f"✅ Loaded {len(self._mcp_cache)} MCP tools from cache (no subprocess spawn)")
             else:
                 # Cache empty (pre-warm failed) — fall back to live discovery.
                 # Prefer the persistent HTTP server if it's reachable; only drop
@@ -1255,56 +744,37 @@ class GradioAgentUI(_UIFormattingMixin):
             mcp_tool_count = session_state["agent"].get_tool_statistics()["by_source"].get("mcp", 0)
             session_state["mcp_tool_count"] = mcp_tool_count
             if mcp_tool_count == 0:
-                print(
-                    "⚠️ Session agent has 0 MCP tools — decoupleR analysis "
-                    "tools are unavailable for this session."
-                )
+                print("⚠️ Session agent has 0 MCP tools — decoupleR analysis "
+                      "tools are unavailable for this session.")
 
         agent = session_state["agent"]
-        # ADR-0012: attach the authenticated caller (forwarded by the orchestrator)
-        # so it flows into the audit trace. Empty string -> None -> "anonymous".
-        agent.set_principal(principal or None)
         run_id = datetime.now().strftime("%Y%m%d_%H%M%S")
 
         try:
-            chatbot_history.append(
-                gr.ChatMessage(role="user", content=original_query, metadata={"status": "done"})
-            )
+            chatbot_history.append(gr.ChatMessage(role="user", content=original_query, metadata={"status": "done"}))
             yield chatbot_history
 
             if session_state.get("mcp_tool_count", 0) == 0:
                 error_notice = (
                     '<div style="background-color:#fee2e2;border-left:4px solid #dc2626;'
                     'padding:12px 16px;margin:10px 0;border-radius:4px;color:#7f1d1d;">'
-                    "⚠️ <strong>Analysis tools unavailable.</strong> The decoupleR MCP "
-                    "tools failed to load for this session, so I cannot run any real "
-                    "analysis (TF activity, pathway/hallmark scoring, differential "
-                    "expression, etc.) right now. Rather than guess at results, I'm "
-                    "stopping here — please try again in a moment, or contact the "
-                    "maintainer if this persists.</div>"
-                )
-                chatbot_history.append(
-                    gr.ChatMessage(
-                        role="assistant", content=error_notice, metadata={"status": "done"}
-                    )
+                    '⚠️ <strong>Analysis tools unavailable.</strong> The decoupleR MCP '
+                    'tools failed to load for this session, so I cannot run any real '
+                    'analysis (TF activity, pathway/hallmark scoring, differential '
+                    'expression, etc.) right now. Rather than guess at results, I\'m '
+                    'stopping here — please try again in a moment, or contact the '
+                    'maintainer if this persists.</div>'
                 )
+                chatbot_history.append(gr.ChatMessage(role="assistant", content=error_notice, metadata={"status": "done"}))
                 yield chatbot_history
                 return
 
             if uploaded_file and os.path.exists(uploaded_file):
                 file_notice = f'<div style="background:#f0fdf4;border-left:3px solid #22c55e;padding:8px 12px;margin:6px 0;border-radius:4px;font-size:0.85em;color:#166534;">Using uploaded file: <strong>{filename}</strong></div>'
-                chatbot_history.append(
-                    gr.ChatMessage(
-                        role="assistant", content=file_notice, metadata={"status": "done"}
-                    )
-                )
+                chatbot_history.append(gr.ChatMessage(role="assistant", content=file_notice, metadata={"status": "done"}))
                 yield chatbot_history
 
-            chatbot_history.append(
-                gr.ChatMessage(
-                    role="assistant", content="🤔 Processing...", metadata={"status": "pending"}
-                )
-            )
+            chatbot_history.append(gr.ChatMessage(role="assistant", content="🤔 Processing...", metadata={"status": "pending"}))
             yield chatbot_history
             chatbot_history.pop()
 
@@ -1313,10 +783,7 @@ class GradioAgentUI(_UIFormattingMixin):
                 yield chatbot_history
 
             completed = getattr(agent.workflow_engine, "last_solution_shown", False)
-            reason = getattr(agent.workflow_engine, "last_end_reason", None)
-            # Only a genuine step exhaustion is fixed by granting more steps.
-            session_state["step_limit_hit"] = (not completed) and reason == "step_limit"
-            session_state["end_reason"] = reason
+            session_state["step_limit_hit"] = not completed
             session_state["last_run_id"] = run_id
 
             # ALWAYS-ON audit trace persistence via the configured sink.
@@ -1332,18 +799,14 @@ class GradioAgentUI(_UIFormattingMixin):
                 print("⚠️ Could not persist execution trace:")
                 traceback.print_exc()
 
-            if not completed:
-                notice = _incomplete_run_notice(reason, agent.config)
-                chatbot_history.append(
-                    gr.ChatMessage(role="assistant", content=notice, metadata={"status": "done"})
+            if session_state["step_limit_hit"]:
+                notice = (
+                    '<div style="background-color:#dbeafe;border-left:4px solid #2563eb;'
+                    'padding:10px 14px;margin:10px 0;border-radius:4px;font-size:0.9em;color:#1e3a5f;">'
+                    '⏸ <strong>Step limit reached.</strong> Click <strong>Continue</strong> '
+                    'to give the agent 15 more steps.</div>'
                 )
-                yield chatbot_history
-
-            # Collapsible provenance panel: which analysis tools ran, which
-            # registered datasets were touched (GUI/observability TODO).
-            panel = self._what_happened_message(agent)
-            if panel is not None:
-                chatbot_history.append(panel)
+                chatbot_history.append(gr.ChatMessage(role="assistant", content=notice, metadata={"status": "done"}))
                 yield chatbot_history
 
             saved = self.hf_storage.save_conversation(chatbot_history, original_query, run_id)
@@ -1360,15 +823,10 @@ class GradioAgentUI(_UIFormattingMixin):
                 pdf_remote = None
                 try:
                     raw_messages = agent.workflow_engine.last_state_messages or []
-                    current_run = self._current_question_messages(chatbot_history)
-                    log_blocks = self._log_blocks(raw_messages, current_run)
+                    log_blocks = self._log_blocks(raw_messages, chatbot_history)
                     if log_blocks:
-                        # Scoped to the current question: figures from earlier
-                        # questions in the session must not reach this export.
-                        images = self._extract_images(current_run)
-                        pdf_path = self._write_pdf(
-                            log_blocks, f"DecoupleRpy Full Run — {run_id}", images
-                        )
+                        images = self._extract_images(chatbot_history)
+                        pdf_path = self._write_pdf(log_blocks, f"DecoupleRpy Full Run — {run_id}", images)
                         remote_name = f"full_run{os.path.splitext(pdf_path)[1] or '.pdf'}"
                         if self.hf_storage.upload_run_file(pdf_path, run_id, remote_name):
                             pdf_remote = remote_name
@@ -1380,19 +838,13 @@ class GradioAgentUI(_UIFormattingMixin):
                 pdf_line = ""
                 if pdf_remote:
                     pdf_url = f"https://huggingface.co/datasets/{self.hf_storage.repo_id}/resolve/main/runs/{run_id}/{pdf_remote}"
-                    pdf_line = (
-                        f'<br>📄 Full run PDF: <a href="{pdf_url}" target="_blank">{pdf_remote}</a>'
-                    )
+                    pdf_line = f'<br>📄 Full run PDF: <a href="{pdf_url}" target="_blank">{pdf_remote}</a>'
                 log_notice = (
                     f'<div style="background-color:#f0f9ff;border-left:3px solid #0ea5e9;'
                     f'padding:6px 12px;margin:6px 0;border-radius:4px;font-size:0.82em;color:#0c4a6e;">'
                     f'📁 Full run log saved: <a href="{hf_url}" target="_blank">{hf_url}</a>{pdf_line}</div>'
                 )
-                chatbot_history.append(
-                    gr.ChatMessage(
-                        role="assistant", content=log_notice, metadata={"status": "done"}
-                    )
-                )
+                chatbot_history.append(gr.ChatMessage(role="assistant", content=log_notice, metadata={"status": "done"}))
                 yield chatbot_history
 
         except Exception as e:
@@ -1400,34 +852,11 @@ class GradioAgentUI(_UIFormattingMixin):
                 gr.ChatMessage(
                     role="assistant",
                     content=f"Error: {str(e)}",
-                    metadata={"title": "💥 Error", "status": "done"},
+                    metadata={"title": "💥 Error", "status": "done"}
                 )
             )
             yield chatbot_history
 
-    def _what_happened_message(self, agent) -> "gr.ChatMessage | None":
-        """Build the collapsed 'what happened' provenance panel for a run.
-
-        Never raises — display plumbing must not take the chat down. Returns
-        None when there is nothing to show (no steps / no analysis tools).
-        """
-        try:
-            raw_messages = agent.workflow_engine.last_state_messages or []
-            try:
-                from src.datasets.registry import list_available_datasets
-
-                dataset_ids = [d["dataset_id"] for d in list_available_datasets()]
-            except Exception:
-                dataset_ids = []
-            activity = self._run_activity(raw_messages, dataset_ids)
-            html = self._what_happened_html(activity)
-            if html is None:
-                return None
-            return gr.ChatMessage(role="assistant", content=html, metadata={"status": "done"})
-        except Exception:
-            traceback.print_exc()
-            return None
-
     def _arm_downloads(self, chatbot_history: list, session_state: dict):
         """Build export files at run completion and arm the download buttons.
 
@@ -1436,12 +865,7 @@ class GradioAgentUI(_UIFormattingMixin):
         trace), then returns DownloadButton updates so each is a single
         reliable click. A button stays disabled if its file can't be built.
         """
-        # Scope every export to the most recent question's run: a session
-        # accumulates questions, and sweeping the whole history embedded stale
-        # figures from earlier questions that could contradict the current
-        # run's tables (TODO 2026-08-11 #13).
-        current_run = self._current_question_messages(chatbot_history)
-        assessment = self._assessment_blocks(current_run)
+        assessment = self._assessment_blocks(chatbot_history)
 
         raw_messages = []
         agent = session_state.get("agent")
@@ -1450,7 +874,7 @@ class GradioAgentUI(_UIFormattingMixin):
                 raw_messages = agent.workflow_engine.last_state_messages or []
             except Exception:
                 raw_messages = []
-        log_blocks = self._log_blocks(raw_messages, current_run)
+        log_blocks = self._log_blocks(raw_messages, chatbot_history)
 
         def _safe(label, fn, *args):
             try:
@@ -1459,16 +883,13 @@ class GradioAgentUI(_UIFormattingMixin):
                 print(f"[export] {label} build failed: {exc}")
                 return None
 
-        images = self._extract_images(current_run)
+        images = self._extract_images(chatbot_history)
         a_title = "DecoupleRpy Analysis"
         txt = _safe("txt", self._write_txt, assessment, a_title, images) if assessment else None
         docx = _safe("docx", self._write_docx, assessment, a_title, images) if assessment else None
         pdf = _safe("pdf", self._write_pdf, assessment, a_title, images) if assessment else None
-        log = (
-            _safe("log", self._write_pdf, log_blocks, "DecoupleRpy — Full Generated Logic", images)
-            if log_blocks
-            else None
-        )
+        log = _safe("log", self._write_pdf, log_blocks,
+                    "DecoupleRpy — Full Generated Logic", images) if log_blocks else None
 
         return (
             gr.DownloadButton(value=txt, interactive=txt is not None),
@@ -1493,14 +914,14 @@ class GradioAgentUI(_UIFormattingMixin):
 
     def _gradio_theme(self):
         return gr.themes.Monochrome(
-            font=fonts.GoogleFont("Inter"), font_mono=fonts.GoogleFont("JetBrains Mono")
+            font=fonts.GoogleFont("Inter"),
+            font_mono=fonts.GoogleFont("JetBrains Mono")
         )
 
     def create_app(self):
         """Create the Gradio app with sidebar layout."""
         with gr.Blocks(fill_height=True, title=self.name) as demo:
-            demo.load(
-                js="""
+            demo.load(js="""
             () => {
                 document.body.classList.remove('dark');
                 document.querySelector('gradio-app').classList.remove('dark');
@@ -1514,21 +935,9 @@ class GradioAgentUI(_UIFormattingMixin):
                     brandingElements.forEach(el => el.style.display = 'none');
                 }, 100);
             }
-            """
-            )
+            """)
             session_state = gr.State({})
             stored_messages = gr.State([])
-            # ADR-0012: hidden channel for the authenticated caller. The UI leaves
-            # it blank (direct use is anonymous); the orchestrator sets it over
-            # gradio_client so the identity reaches the audit trace.
-            principal_input = gr.Textbox(value="", visible=False, label="principal")
-
-            # ADR-0012: HuggingFace OAuth sign-in for direct use of this Space's
-            # UI. Requires `hf_oauth: true` in the README metadata. The injected
-            # gr.OAuthProfile is read in interact_with_agent, where the allow-list
-            # gate is enforced. Inert until OAuth is on + ACCESS_CONTROL=enforce,
-            # so it is safe to ship ahead of cutover.
-            gr.LoginButton()
 
             with gr.Row():
                 with gr.Column(scale=1):
@@ -1540,87 +949,13 @@ class GradioAgentUI(_UIFormattingMixin):
 
                     with gr.Group():
                         gr.Markdown("**Data Input (optional)**")
-                        gr.Markdown(
-                            "<span style='font-size:0.8em;color:#475569;'>Uploads are "
-                            "quarantined, structure-checked (file type verified against "
-                            "its contents), and integrity-hashed before the agent sees "
-                            "them (ADR-0011). This is not a malware scan — do not upload "
-                            "files from an untrusted source. Data must be "
-                            "de-identified.</span>"
-                        )
-                        deid_checkbox = gr.Checkbox(
-                            label="I confirm this data is de-identified (no PHI/PSI)",
-                            value=False,
-                        )
                         with gr.Tabs():
                             with gr.Tab("Upload File"):
                                 file_input = gr.File(
-                                    label="Upload .h5ad / .csv / .tsv / .txt (.gz ok)",
-                                    file_types=[
-                                        ".h5ad",
-                                        ".csv",
-                                        ".tsv",
-                                        ".txt",
-                                        ".gz",
-                                    ],
-                                    type="filepath",
-                                )
-                            with gr.Tab("Assemble from TSVs"):
-                                gr.Markdown(
-                                    "<span style='font-size:0.8em;color:#475569;'>"
-                                    "Drop a sequencing delivery as-is — a gene×sample"
-                                    " counts TSV, an optional TPM TSV, and the sample"
-                                    " metadata sheet — and the analysis file is built"
-                                    " here. Metadata must yield <code>clone</code>,"
-                                    " <code>arm</code>, <code>site</code> and"
-                                    " <code>mouse_id</code>; use the mapping fields"
-                                    " below if the sheet names them differently."
-                                    "</span>"
-                                )
-                                asm_counts = gr.File(
-                                    label="Counts matrix (.tsv / .csv, genes × samples)",
-                                    file_types=[".tsv", ".csv", ".txt", ".gz"],
-                                    type="filepath",
-                                )
-                                asm_tpm = gr.File(
-                                    label="TPM / abundance matrix (optional)",
-                                    file_types=[".tsv", ".csv", ".txt", ".gz"],
-                                    type="filepath",
-                                )
-                                asm_meta = gr.File(
-                                    label="Sample metadata (.xlsx / .csv / .tsv)",
-                                    file_types=[".xlsx", ".csv", ".tsv", ".txt"],
+                                    label="Upload .h5ad or .csv",
+                                    file_types=[".h5ad", ".csv"],
                                     type="filepath",
                                 )
-                                with gr.Accordion("Metadata column mapping", open=False):
-                                    asm_sample_col = gr.Textbox(
-                                        label="Sample-id column",
-                                        placeholder="SampleName (default: first column)",
-                                    )
-                                    asm_column_map = gr.Textbox(
-                                        label="Rename columns → obs",
-                                        placeholder="mouse_id=Mouse,site=Type",
-                                    )
-                                    asm_value_maps = gr.Textbox(
-                                        label="Recode values (one per line)",
-                                        placeholder="site=Tumor:tumor\nsite=Met:liver_met",
-                                        lines=2,
-                                    )
-                                    asm_group_col = gr.Textbox(
-                                        label="Group column (derives arm + clone)",
-                                        placeholder="Group",
-                                    )
-                                    asm_control = gr.Textbox(
-                                        label="Control label", placeholder="shCntrl"
-                                    )
-                                    asm_treatment = gr.Textbox(
-                                        label="Treatment label", value="shMyc"
-                                    )
-                                    asm_skip_rows = gr.Textbox(
-                                        label="Header row offset",
-                                        placeholder="blank = auto-detect",
-                                    )
-                                asm_btn = gr.Button("Assemble dataset", size="sm")
                             with gr.Tab("URL"):
                                 url_input = gr.Textbox(
                                     label="Public URL",
@@ -1645,7 +980,7 @@ class GradioAgentUI(_UIFormattingMixin):
                             lines=4,
                             label="Query",
                             placeholder="Enter your query here and press Enter or click Submit",
-                            value="""Use decoupleRpy MCP to load a built-in RNA-seq dataset, perform preprocessing and differential expression analysis, then run transcription factor enrichment using CollecTRI. Summarize the key regulators.""",
+                            value="""Use decoupleRpy MCP to load a built-in RNA-seq dataset, perform preprocessing and differential expression analysis, then run transcription factor enrichment using CollecTRI. Summarize the key regulators."""
                         )
                         submit_btn = gr.Button("Submit", variant="primary", size="lg")
 
@@ -1655,25 +990,23 @@ class GradioAgentUI(_UIFormattingMixin):
                             minimum=5,
                             maximum=50,
                             value=self.config.max_steps,
-                            step=1,
+                            step=1
                         )
                         temperature_input = gr.Slider(
-                            label="Temperature", minimum=0.0, maximum=1.0, value=0, step=0.1
+                            label="Temperature",
+                            minimum=0.0,
+                            maximum=1.0,
+                            value=0,
+                            step=0.1
                         )
                         apply_config_btn = gr.Button("Apply Configuration", size="sm")
 
                     with gr.Accordion("Example Queries", open=False):
                         gr.Examples(
                             examples=[
-                                [
-                                    "Use decoupleRpy MCP to load a built-in RNA-seq dataset, perform preprocessing and differential expression analysis, then run transcription factor enrichment using CollecTRI. Summarize the key regulators."
-                                ],
-                                [
-                                    "Load the uploaded dataset, run differential expression analysis between the two conditions, then perform TF enrichment with CollecTRI and pathway enrichment with PROGENy."
-                                ],
-                                [
-                                    "Run hallmark gene set enrichment on the uploaded data and identify the most significantly activated and repressed pathways."
-                                ],
+                                ["Use decoupleRpy MCP to load a built-in RNA-seq dataset, perform preprocessing and differential expression analysis, then run transcription factor enrichment using CollecTRI. Summarize the key regulators."],
+                                ["Load the uploaded dataset, run differential expression analysis between the two conditions, then perform TF enrichment with CollecTRI and pathway enrichment with PROGENy."],
+                                ["Run hallmark gene set enrichment on the uploaded data and identify the most significantly activated and repressed pathways."],
                             ],
                             inputs=text_input,
                         )
@@ -1700,18 +1033,10 @@ class GradioAgentUI(_UIFormattingMixin):
 
                     with gr.Row():
                         copy_btn = gr.Button("📋 Copy text", size="sm")
-                        export_txt_btn = gr.DownloadButton(
-                            "⬇️ .txt", size="sm", value=None, interactive=False
-                        )
-                        export_docx_btn = gr.DownloadButton(
-                            "⬇️ .docx", size="sm", value=None, interactive=False
-                        )
-                        export_pdf_btn = gr.DownloadButton(
-                            "⬇️ .pdf", size="sm", value=None, interactive=False
-                        )
-                        export_log_btn = gr.DownloadButton(
-                            "⬇️ Full log (.pdf)", size="sm", value=None, interactive=False
-                        )
+                        export_txt_btn = gr.DownloadButton("⬇️ .txt", size="sm", value=None, interactive=False)
+                        export_docx_btn = gr.DownloadButton("⬇️ .docx", size="sm", value=None, interactive=False)
+                        export_pdf_btn = gr.DownloadButton("⬇️ .pdf", size="sm", value=None, interactive=False)
+                        export_log_btn = gr.DownloadButton("⬇️ Full log (.pdf)", size="sm", value=None, interactive=False)
 
                     copy_box = gr.Textbox(
                         label="Copy-friendly text (select all → Ctrl+C / use copy button)",
@@ -1720,11 +1045,84 @@ class GradioAgentUI(_UIFormattingMixin):
                         interactive=False,
                     )
 
+            def _inputs_dir():
+                from pathlib import Path
+                d = Path(__file__).parent / "tmp" / "inputs"
+                d.mkdir(parents=True, exist_ok=True)
+                return d
+
+            def handle_file_upload(file_path, session_state):
+                import shutil
+                if file_path is None:
+                    session_state.pop("uploaded_file", None)
+                    session_state.pop("uploaded_filename", None)
+                    return session_state, ""
+                filename = os.path.basename(file_path)
+                dest = str(_inputs_dir() / filename)
+                shutil.copy2(file_path, dest)
+                session_state["uploaded_file"] = dest
+                session_state["uploaded_filename"] = filename
+                return session_state, f"Ready: {filename}"
+
+            def handle_url_download(url, session_state):
+                import gzip
+                import requests
+                import shutil as _shutil
+                if not url or not url.strip():
+                    return session_state, "No URL provided"
+                url = url.strip()
+                filename = url.rstrip("/").split("/")[-1].split("?")[0]
+                if not filename or "." not in filename:
+                    filename = "downloaded_data.bin"
+                dest = str(_inputs_dir() / filename)
+                try:
+                    r = requests.get(url, stream=True, timeout=300)
+                    r.raise_for_status()
+                    ct = r.headers.get("Content-Type", "")
+                    if "text/html" in ct:
+                        return session_state, "URL returned an HTML page, not a file. Make sure the URL points directly to a file, not a directory."
+                    with open(dest, "wb") as f:
+                        for chunk in r.iter_content(chunk_size=8192):
+                            if chunk:
+                                f.write(chunk)
+                    if filename.endswith(".gz") and not filename.endswith(".tar.gz"):
+                        decompressed = filename[:-3]
+                        decompressed_dest = str(_inputs_dir() / decompressed)
+                        with gzip.open(dest, "rb") as f_in, open(decompressed_dest, "wb") as f_out:
+                            _shutil.copyfileobj(f_in, f_out)
+                        os.remove(dest)
+                        dest, filename = decompressed_dest, decompressed
+                    session_state["uploaded_file"] = dest
+                    session_state["uploaded_filename"] = filename
+                    return session_state, f"Ready: {filename}"
+                except Exception as e:
+                    return session_state, f"Download failed: {e}"
+
+            def handle_hf_dataset(repo_id, filepath, session_state):
+                if not repo_id or not repo_id.strip():
+                    return session_state, "No repo ID provided"
+                if not filepath or not filepath.strip():
+                    return session_state, "No file path provided"
+                try:
+                    from huggingface_hub import hf_hub_download
+                    local_path = hf_hub_download(
+                        repo_id=repo_id.strip(),
+                        filename=filepath.strip(),
+                        repo_type="dataset",
+                        local_dir=str(_inputs_dir()),
+                    )
+                    filename = os.path.basename(local_path)
+                    session_state["uploaded_file"] = local_path
+                    session_state["uploaded_filename"] = filename
+                    return session_state, f"Ready: {filename}"
+                except Exception as e:
+                    return session_state, f"Failed to load from HF Dataset: {e}"
+
             def update_config(max_steps, temperature, session_state):
                 if "agent" in session_state:
                     agent = session_state["agent"]
                     agent.config.max_steps = max_steps
-                    if hasattr(agent.model, "temperature"):
+                    if hasattr(agent.model, 'temperature'):
                         agent.model.temperature = temperature
                 return "Configuration updated!"
 
@@ -1739,7 +1137,7 @@ class GradioAgentUI(_UIFormattingMixin):
                     return chatbot
 
                 agent = session_state["agent"]
-                agent.config.max_steps += STEP_LIMIT_INCREMENT
+                agent.config.max_steps += 15
                 run_id = session_state.get("last_run_id", datetime.now().strftime("%Y%m%d_%H%M%S"))
                 resume_messages = agent.workflow_engine.last_state_messages
 
@@ -1750,104 +1148,52 @@ class GradioAgentUI(_UIFormattingMixin):
                     yield chatbot
 
                 completed = getattr(agent.workflow_engine, "last_solution_shown", False)
-                reason = getattr(agent.workflow_engine, "last_end_reason", None)
-                session_state["step_limit_hit"] = (not completed) and reason == "step_limit"
-                session_state["end_reason"] = reason
-
-                if not completed:
-                    notice = _incomplete_run_notice(reason, agent.config)
-                    chatbot.append(
-                        gr.ChatMessage(
-                            role="assistant", content=notice, metadata={"status": "done"}
-                        )
+                session_state["step_limit_hit"] = not completed
+
+                if session_state["step_limit_hit"]:
+                    notice = (
+                        '<div style="background-color:#dbeafe;border-left:4px solid #2563eb;'
+                        'padding:10px 14px;margin:10px 0;border-radius:4px;font-size:0.9em;color:#1e3a5f;">'
+                        '⏸ <strong>Step limit reached again.</strong> Click <strong>Continue</strong> '
+                        'to add 15 more steps.</div>'
                     )
+                    chatbot.append(gr.ChatMessage(role="assistant", content=notice, metadata={"status": "done"}))
                     yield chatbot
 
-                panel = self._what_happened_message(agent)
-                if panel is not None:
-                    chatbot.append(panel)
-                    yield chatbot
-
-            file_input.change(
-                handle_file_upload,
-                [file_input, deid_checkbox, session_state],
-                [session_state, file_status],
-            )
-            # Re-run the gate when the attestation is toggled, so ticking the box
-            # after picking a file re-validates without re-selecting it.
-            deid_checkbox.change(
-                handle_file_upload,
-                [file_input, deid_checkbox, session_state],
-                [session_state, file_status],
-            )
-            asm_btn.click(
-                handle_assembly,
-                [
-                    asm_counts,
-                    asm_meta,
-                    asm_tpm,
-                    asm_sample_col,
-                    asm_column_map,
-                    asm_value_maps,
-                    asm_group_col,
-                    asm_control,
-                    asm_treatment,
-                    asm_skip_rows,
-                    deid_checkbox,
-                    session_state,
-                ],
-                [session_state, file_status],
-            )
-            url_btn.click(
-                handle_url_download,
-                [url_input, deid_checkbox, session_state],
-                [session_state, file_status],
-            )
-            hf_btn.click(
-                handle_hf_dataset,
-                [hf_repo_input, hf_file_input, deid_checkbox, session_state],
-                [session_state, file_status],
-            )
+            file_input.change(handle_file_upload, [file_input, session_state], [session_state, file_status])
+            url_btn.click(handle_url_download, [url_input, session_state], [session_state, file_status])
+            hf_btn.click(handle_hf_dataset, [hf_repo_input, hf_file_input, session_state], [session_state, file_status])
 
             download_btns = [export_txt_btn, export_docx_btn, export_pdf_btn, export_log_btn]
 
             type_submit = text_input.submit(
                 lambda x: (x, "", gr.Button(interactive=False)),
-                [text_input],
-                [stored_messages, text_input, submit_btn],
+                [text_input], [stored_messages, text_input, submit_btn]
             )
             type_interact = type_submit.then(
                 self.interact_with_agent,
-                [stored_messages, chatbot, session_state, principal_input],
-                [chatbot],
+                [stored_messages, chatbot, session_state], [chatbot]
             )
             type_interact.then(self._arm_downloads, [chatbot, session_state], download_btns)
             type_interact.then(lambda: gr.Button(interactive=True), None, [submit_btn])
 
             btn_submit = submit_btn.click(
                 lambda x: (x, "", gr.Button(interactive=False)),
-                [text_input],
-                [stored_messages, text_input, submit_btn],
+                [text_input], [stored_messages, text_input, submit_btn]
             )
             btn_interact = btn_submit.then(
                 self.interact_with_agent,
-                [stored_messages, chatbot, session_state, principal_input],
-                [chatbot],
+                [stored_messages, chatbot, session_state], [chatbot]
             )
             btn_interact.then(self._arm_downloads, [chatbot, session_state], download_btns)
             btn_interact.then(lambda: gr.Button(interactive=True), None, [submit_btn])
 
-            apply_config_btn.click(
-                update_config, [max_steps_input, temperature_input, session_state], None
-            )
+            apply_config_btn.click(update_config, [max_steps_input, temperature_input, session_state], None)
             clear_btn.click(
                 clear_chat, [session_state], [chatbot, stored_messages, session_state]
             ).then(
-                lambda: tuple(
-                    gr.DownloadButton(value=None, interactive=False) for _ in download_btns
-                ),
-                None,
-                download_btns,
+                lambda: tuple(gr.DownloadButton(value=None, interactive=False) for _ in download_btns),
+                None, download_btns
             )
 
             # Export / copy buttons
@@ -1863,14 +1209,12 @@ class GradioAgentUI(_UIFormattingMixin):
             # armed at run completion by _arm_downloads — a single reliable click,
             # with no recompute-on-click and no truncation.
 
-            continue_interact = continue_btn.click(
-                handle_continue, [chatbot, session_state], [chatbot]
-            )
+            continue_interact = continue_btn.click(handle_continue, [chatbot, session_state], [chatbot])
             continue_interact.then(self._arm_downloads, [chatbot, session_state], download_btns)
             stop_btn.click(
                 lambda: gr.Button(interactive=True),
                 outputs=[submit_btn],
-                cancels=[type_submit, type_interact, btn_submit, btn_interact, continue_interact],
+                cancels=[type_submit, type_interact, btn_submit, btn_interact, continue_interact]
             )
 
         return demo
@@ -1878,11 +1222,11 @@ class GradioAgentUI(_UIFormattingMixin):
     def launch(self, share: bool = False, **kwargs):
         """Launch the Gradio app."""
         app = self.create_app()
-        kwargs.setdefault("server_name", "0.0.0.0")
-        kwargs.setdefault("server_port", 7860)
-        kwargs.setdefault("show_error", True)
+        kwargs.setdefault('server_name', '0.0.0.0')
+        kwargs.setdefault('server_port', 7860)
+        kwargs.setdefault('show_error', True)
         # Gradio 6.0 moved theme + css off the Blocks() constructor onto launch().
-        kwargs.setdefault("theme", self._gradio_theme())
-        kwargs.setdefault("css", self._GRADIO_CSS)
+        kwargs.setdefault('theme', self._gradio_theme())
+        kwargs.setdefault('css', self._GRADIO_CSS)
         # gradio 6.0 removed the show_api argument from launch().
         app.queue(max_size=10).launch(share=share, favicon_path=None, **kwargs)

pdac-analysis-orchestrator-dev

@@ -0,0 +1 @@
+Subproject commit dd6588af335b3a7ee4c99b09f2dba1dd6e9ff023

prompts.yaml

@@ -15,7 +15,7 @@ system_prompt: "You are an expert assistant who can solve any task using Python
   \ generated\n  - An **Approach** section — the analysis plan, each computation step, which tools/methods ran, and (for a non-coding scientist) what each decoupleR method actually does (see ## Explaining Your Approach). Describe ONLY the clean final path — never the missteps, retries, or transient errors you self-corrected\n  - Any files created or modified\n  - Next steps or\
   \ recommendations (if applicable)\n\n## Code Execution Best Practices\n\n### Printing Results\n**YOU MUST print outputs**\
   \ - the system cannot see unpublished results:\n```python\n# ✓ CORRECT\nresult = some_function(param1, param2)\nprint(result)\n\
-  \n# ✗ WRONG - output is invisible\nsome_function(param1, param2)\n```\n\n### Saving Plots\n**Every figure you create MUST be written into the tool output directory** — the same directory the MCP tools write their CSVs and PNGs to (`tmp/outputs/`, or `$RNA_OUTPUT_DIR` when set). Only files in that directory are collected and shown inline to the user at the end of the run; a figure saved to `/tmp/`, the working directory, or any other path is invisible — the user asks to see the landscape and silently gets no image.\n\n```python\n# CORRECT - lands where the UI can find it\nimport os\nout_dir = os.environ.get('RNA_OUTPUT_DIR', 'tmp/outputs')\nos.makedirs(out_dir, exist_ok=True)\nfig_path = os.path.join(out_dir, 'progeny_landscape.png')\nplt.savefig(fig_path, dpi=200, bbox_inches='tight')\nprint(f'Saved figure: {fig_path}')\n\n# WRONG - never surfaced to the user\nplt.savefig('/tmp/progeny_landscape.png')\n```\n\n**Prefer the tool's own figure over hand-rolled matplotlib.** Several tools already emit a standard figure and return its path (e.g. `dataset_score_bulk_samples` returns `landscape_path` — a per-sample heatmap plus mean±SD bar). When a tool returns a figure path, reference that file instead of re-plotting.\n\n### Working with Data\nWhen working with data structures,\
+  \n# ✗ WRONG - output is invisible\nsome_function(param1, param2)\n```\n\n### Working with Data\nWhen working with data structures,\
   \ always inspect them **but limit output to avoid context overflow**:\n```python\n# For DataFrames - ALWAYS use head() for\
   \ large tables\nprint(f\"Shape: {df.shape}\")\nprint(f\"Columns: {list(df.columns)}\")\nprint(df.head(10))  # Show first\
   \ 10 rows max\n\n# For large DataFrames, also show basic info\nif len(df) > 20:\n    print(f\"DataFrame has {len(df)} rows.\
@@ -187,7 +187,7 @@ The ONLY datasets registered in this system are listed below with everything\
   \ features, or survival_columns for an external survival package), say so and offer to compute\
   \ those inputs instead.\n4. Name the appropriate external tool or package for the requested\
   \ analysis.\n5. End in <solution> explaining this. Then STOP — do not proceed further in this\
-  \ turn.\n\n## Explaining Your Approach (write this in every analytical solution)\n\nEvery <solution> that runs an analysis MUST open with a short **Approach** section written for a non-coding scientist, placed BEFORE the numeric results. In plain language, cover:\n\n1. **Plan** — what you set out to do and which dataset/subset you used (name the cohort and the sample groups, e.g. 'PACA-AU RNA-seq, squamous n=X vs progenitor n=Y').\n2. **Steps** — each computation step in order and why, e.g. 'loaded counts → filtered low-expression genes → differential expression → transcription-factor activity inference'.\n3. **Tools/code** — name the actual tool or method that ran at each step (e.g. decoupler_differential_expression with method=deseq2; decoupler_tf_enrichment_collectri). The user is NOT familiar with decoupleR internals, so for each decoupleR method add ONE plain-language sentence on what it actually does (see glossary below).\n4. **Preprocessing** — add ONE plain-language line on how that dataset was prepared (what the raw data was, the normalization/transform and units, and any feature-ID handling), taken from the dataset's **Preprocessing** field in ## Available Datasets. This grounds the reader in what the numbers are computed from; state only what that field says — do not invent provenance.\n\n**Hard rules:**\n- Describe ONLY the clean final path. NEVER narrate missteps, retries, NameErrors, or anything you self-corrected — the user wants the method, not the debugging.\n- Do not invent steps you did not run; the Approach must match the tools you actually called this turn.\n- Keep it tight: a few sentences or a short numbered list, not a transcript.\n\n### decoupleR method glossary (explain in plain language; never paste jargon verbatim)\n- **Differential expression (DESeq2 / limma / t-test):** finds genes whose expression differs between two groups; DESeq2 is for raw integer counts (RNA-seq), limma/t-test for already-normalized data.\n- **CollecTRI + ULM (decoupler_tf_enrichment_collectri):** CollecTRI is a curated map of which transcription factors (TFs) switch which genes on or off; ULM (univariate linear model) scores, for each TF, how consistently its target genes move — an INFERRED estimate of TF activity, not a direct measurement of TF protein or expression.\n- **PROGENy (decoupler_pathway_enrichment_progeny):** estimates the activity of ~14 cancer-relevant signalling pathways from the expression of genes those pathways are known to drive — inferred, not measured.\n- **Activity-scoring methods (ulm / mlm / zscore):** alternative statistics for turning a regulon/gene-set plus an expression profile into one activity score per TF or pathway; ulm is the default.\n- **Hallmark / gene-set enrichment:** tests whether a predefined gene set (e.g. MSigDB Hallmark) is collectively shifted up or down between groups.\n- **Meta-analysis (decoupler_meta_analyze):** combines per-cohort results into one pooled estimate and reports heterogeneity (Cochran's Q / I^2) — how consistent the effect is across cohorts.\n\nThis Approach section is IN ADDITION to the standing method-limitation boilerplate appended automatically after your <solution>; do not duplicate that boilerplate here.\n\n## Reporting Results: Provenance and Anti-Fabrication\n\nThese rules are MANDATORY for every <solution> that reports any number (activity scores, effect sizes, p-values, sample counts, gene-coverage percentages, counts of significant features, etc.).\n\n1. **Every number in <solution> MUST come from a tool/code <observation> in THIS turn.** Never write a result value from memory, from general knowledge, or by analogy to the examples in this prompt. If you did not observe a value from a tool call this turn, do not state it.\n\n2. **Re-read before you report — UNLESS the tool already handed you the table.** If the final analysis tool's observation is still in view and returned a structured result table — any `top_table` field, which every analysis-terminating tool now returns (decoupler_differential_expression, decoupler_tf_enrichment_collectri, decoupler_pathway_enrichment_progeny, decoupler_hallmark_enrichment, dataset_compare_activity_by_group). It is already joined with padj, already filtered to significant rows, and already sorted in both directions, so it needs no merge, no re-sort and no second look, quote it directly and emit <solution> next. Those numbers ARE an observation from this turn, so rule 1 is already satisfied and a re-read buys nothing. Otherwise — on a long run, earlier observations scroll out of your working context. Immediately before writing <solution>, run ONE final <execute> that ONLY re-reads the ALREADY-SAVED results file(s) and prints the exact rows you will cite — the final table must already have been saved by the step that computed it. Do NOT recompute, re-derive, or re-save the table in this step, and do NOT repeat this re-read more than once. Example:\n<execute>\nimport pandas as pd\ndf = pd.read_csv(\"<out_prefix>_results.csv\", index_col=0)\nprint(df.head(20).round(3))\n</execute>\nThen quote those printed values verbatim. Do NOT reconstruct numbers you 'remember' from earlier steps — read them back from disk. **This re-read happens AT MOST ONCE per turn, and the very next thing you emit after its output is `<solution>`.** If you have already re-read a results file this turn, you have the numbers — do not read it again, do not re-print it in a different shape, and do not 'verify' it a second time. A repeated re-read is the single most common way a run runs out of steps with the answer already in hand. When you author your OWN gene table in <execute> (anything not taken verbatim from a `top_table`), rank it by the test statistic (`stat` / Wald), NOT by log2FoldChange — low-count genes carry huge noisy fold-changes, so a log2FC-ranked table headlines pseudogene/Rik junk; keep log2FC as a column, never as the sort key. And when a legitimate re-read IS needed (a custom figure, more rows than top_table carries, a cross-contrast join), do it in ONE step: the tool result already tells you the CSV's exact schema (`de_results_columns` on the DE tool, `csv_orientation` on the enrichment tools), so never spend a separate step printing `df.columns` or `df.head()` before the real work.\n\n3. **Cite provenance in <solution>.** State the exact tools/methods you ran and the output artifact path(s) they produced (e.g. the activities/results CSV paths). This lets the user verify the results.\n\n4. **If a required tool call failed or you could not actually compute a result, say so plainly and do not present substitute numbers.** A truthful 'I could not run X' always beats a fabricated table.\n\n5. **Foreground any CRITICAL sanity warning — never bury it.** Several tools return a 'sanity_warnings' report (ADR-0002 Layer-2 result-aware checks: effect-size plausibility, tissue-identity contamination, network membership). If a tool you ran THIS turn returns sanity_warnings whose max_severity is 'critical', you MUST foreground that caution at the TOP of your <solution>, before the numbers — not in a footnote — and quote the warning's message. For a 'tissue_identity_contamination' warning specifically, state plainly that the contrast may reflect tissue COMPOSITION (normal-tissue admixture carried along in the biopsy) rather than tumour biology, name the flagged identity markers, and recommend controlling for tumour purity or comparing within a single tissue of origin before drawing biological conclusions. Never present contaminated top hits as validated tumour findings. A 'warn'/'info' sanity_warning need not headline the solution but MUST be stated in your run-specific caveats.\n\n## Non-Human (Mouse) Ad-Hoc Data\n\nAlmost everything in ## Available Datasets is human PDAC. Occasionally you are pointed at an **ad-hoc mouse h5ad** that is NOT in the registry (it carries `uns['organism'] = 'mouse'` and `uns['analysis_space'] = 'mouse'`). When the data is mouse, these rules override the human defaults:\n\n1. **Check `uns` before you analyse an unregistered h5ad.** If `uns['organism']` is `'mouse'`, say so in your Approach section and flag — as you would for any non-registered cohort — that this dataset is outside the curated PDAC registry.\n\n2. **Pass `organism='mouse'` to EVERY enrichment tool.** `decoupler_tf_enrichment_collectri`, `decoupler_pathway_enrichment_progeny` and `decoupler_hallmark_enrichment` all take `organism`, and it defaults to `'human'`. Leaving the default on mouse data silently matches MGI mouse symbols (e.g. `Myc`, `Kras`) against HGNC human symbols (`MYC`, `KRAS`), so the network overlap collapses and the scores are meaningless rather than merely noisy. The var index of a mouse h5ad is MGI symbols; there is no human mapping step and none is wanted.\n\n3. **Mouse-space results are never comparable to human-space results.** Do not pool, meta-analyse, or numerically compare a mouse result with any registered human cohort's result, and do not translate scores between them. Describe them side by side at most, and label every mouse table as mouse-space.\n\n4. **Counts vs TPM routing is the same rule as everywhere else.** A mouse counts matrix (integer `X`) is Path A — `decoupler_differential_expression(method='deseq2')`. A TPM matrix (often carried as `layers['tpm']`) is Path B — limma or t-test, never DESeq2. Do not run DESeq2 on the TPM layer just because it is in the same file.\n\n5. **Two design recipes for a paired-tumour/met, knockdown-vs-control mouse experiment** (obs columns `arm`, `clone`, `site`, `mouse_id`):\n   - **Tumour vs paired metastasis:** `design_factor='site'`, `batch_column='mouse_id'`. This is the standard paired design (`~mouse_id + site`) — each mouse contributes both a tumour and a met, so mouse is a legitimate blocking factor. Restrict to mice that actually have both samples, or the unpaired mice contribute nothing but degrees of freedom.\n   - **Knockdown vs control:** `design_factor='arm'`, and **`batch_column` MUST NOT be `mouse_id`** — each mouse belongs to exactly one arm, so mouse is nested inside (confounded with) arm and the design matrix is singular. Use `batch_column='clone'` only when clone actually crosses the arms; if every clone sits in one arm it is nested too and must be left out. Also pass `subset_query` to analyse one site at a time (e.g. tumours only) — mixing a mouse's tumour and met into one arm-level contrast is pseudo-replication. The DE tool has a nested-batch pre-flight guard that refuses a fully nested `batch_column`; if it fires, drop the covariate rather than trying to work around it.\n\n## Efficiency Rules\n\nThese rules reduce unnecessary tool calls. Follow them to avoid latency\
+  \ turn.\n\n## Explaining Your Approach (write this in every analytical solution)\n\nEvery <solution> that runs an analysis MUST open with a short **Approach** section written for a non-coding scientist, placed BEFORE the numeric results. In plain language, cover:\n\n1. **Plan** — what you set out to do and which dataset/subset you used (name the cohort and the sample groups, e.g. 'PACA-AU RNA-seq, squamous n=X vs progenitor n=Y').\n2. **Steps** — each computation step in order and why, e.g. 'loaded counts → filtered low-expression genes → differential expression → transcription-factor activity inference'.\n3. **Tools/code** — name the actual tool or method that ran at each step (e.g. decoupler_differential_expression with method=deseq2; decoupler_tf_enrichment_collectri). The user is NOT familiar with decoupleR internals, so for each decoupleR method add ONE plain-language sentence on what it actually does (see glossary below).\n4. **Preprocessing** — add ONE plain-language line on how that dataset was prepared (what the raw data was, the normalization/transform and units, and any feature-ID handling), taken from the dataset's **Preprocessing** field in ## Available Datasets. This grounds the reader in what the numbers are computed from; state only what that field says — do not invent provenance.\n\n**Hard rules:**\n- Describe ONLY the clean final path. NEVER narrate missteps, retries, NameErrors, or anything you self-corrected — the user wants the method, not the debugging.\n- Do not invent steps you did not run; the Approach must match the tools you actually called this turn.\n- Keep it tight: a few sentences or a short numbered list, not a transcript.\n\n### decoupleR method glossary (explain in plain language; never paste jargon verbatim)\n- **Differential expression (DESeq2 / limma / t-test):** finds genes whose expression differs between two groups; DESeq2 is for raw integer counts (RNA-seq), limma/t-test for already-normalized data.\n- **CollecTRI + ULM (decoupler_tf_enrichment_collectri):** CollecTRI is a curated map of which transcription factors (TFs) switch which genes on or off; ULM (univariate linear model) scores, for each TF, how consistently its target genes move — an INFERRED estimate of TF activity, not a direct measurement of TF protein or expression.\n- **PROGENy (decoupler_pathway_enrichment_progeny):** estimates the activity of ~14 cancer-relevant signalling pathways from the expression of genes those pathways are known to drive — inferred, not measured.\n- **Activity-scoring methods (ulm / mlm / zscore):** alternative statistics for turning a regulon/gene-set plus an expression profile into one activity score per TF or pathway; ulm is the default.\n- **Hallmark / gene-set enrichment:** tests whether a predefined gene set (e.g. MSigDB Hallmark) is collectively shifted up or down between groups.\n- **Meta-analysis (decoupler_meta_analyze):** combines per-cohort results into one pooled estimate and reports heterogeneity (Cochran's Q / I^2) — how consistent the effect is across cohorts.\n\nThis Approach section is IN ADDITION to the standing method-limitation boilerplate appended automatically after your <solution>; do not duplicate that boilerplate here.\n\n## Reporting Results: Provenance and Anti-Fabrication\n\nThese rules are MANDATORY for every <solution> that reports any number (activity scores, effect sizes, p-values, sample counts, gene-coverage percentages, counts of significant features, etc.).\n\n1. **Every number in <solution> MUST come from a tool/code <observation> in THIS turn.** Never write a result value from memory, from general knowledge, or by analogy to the examples in this prompt. If you did not observe a value from a tool call this turn, do not state it.\n\n2. **Re-read before you report.** On a long run, earlier observations scroll out of your working context. Immediately before writing <solution>, run ONE final <execute> that ONLY re-reads the ALREADY-SAVED results file(s) and prints the exact rows you will cite — the final table must already have been saved by the step that computed it. Do NOT recompute, re-derive, or re-save the table in this step, and do NOT repeat this re-read more than once. Example:\n<execute>\nimport pandas as pd\ndf = pd.read_csv(\"<out_prefix>_results.csv\", index_col=0)\nprint(df.head(20).round(3))\n</execute>\nThen quote those printed values verbatim. Do NOT reconstruct numbers you 'remember' from earlier steps — read them back from disk.\n\n3. **Cite provenance in <solution>.** State the exact tools/methods you ran and the output artifact path(s) they produced (e.g. the activities/results CSV paths). This lets the user verify the results.\n\n4. **If a required tool call failed or you could not actually compute a result, say so plainly and do not present substitute numbers.** A truthful 'I could not run X' always beats a fabricated table.\n\n5. **Foreground any CRITICAL sanity warning — never bury it.** Several tools return a 'sanity_warnings' report (ADR-0002 Layer-2 result-aware checks: effect-size plausibility, tissue-identity contamination, network membership). If a tool you ran THIS turn returns sanity_warnings whose max_severity is 'critical', you MUST foreground that caution at the TOP of your <solution>, before the numbers — not in a footnote — and quote the warning's message. For a 'tissue_identity_contamination' warning specifically, state plainly that the contrast may reflect tissue COMPOSITION (normal-tissue admixture carried along in the biopsy) rather than tumour biology, name the flagged identity markers, and recommend controlling for tumour purity or comparing within a single tissue of origin before drawing biological conclusions. Never present contaminated top hits as validated tumour findings. A 'warn'/'info' sanity_warning need not headline the solution but MUST be stated in your run-specific caveats.\n\n## Efficiency Rules\n\nThese rules reduce unnecessary tool calls. Follow them to avoid latency\
   \ on every request.\n\n1. **Skip `dataset_list_available` when the dataset is already identified.** If the user's message\
   \ names a specific dataset (e.g. \"Moffitt\", \"GSE71729\", \"gse71729_moffitt\") or any GSE accession number, do not call\
   \ `dataset_list_available`. The dataset is already known — proceed directly to `dataset_describe` or the analysis tool sequence.\
@@ -203,7 +203,7 @@ The ONLY datasets registered in this system are listed below with everything\
   \ ## Available Datasets. Skip straight to DE or the relevant analysis tool.\n\n5. **Always pass `subset_query` when the default contrast specifies one.** The ## Available\
   \ Datasets section lists each dataset's default contrasts, including any `subset_query`. When running DE,\
   \ read that value and pass it directly — never pre-filter the AnnData manually. Example: if the contrast\
-  \ entry shows `subset_query=\"tumor_subtype != ''\"`, pass that exact string to `decoupler_differential_expression`.\n\n6. **Never import from `tools`, `src.tools`, `server`, or any `_mcp` module, and never inspect `FunctionTool`/`.fn`/`.run()` internals.** All functions listed below are already pre-loaded as plain callables in your namespace and are re-verified before every step. If a call to one of them raises `NameError` or `'FunctionTool' object is not callable`, do NOT start importing or introspecting — simply retry the exact same call with the parameters documented below. If it still fails after one retry, move on to a different approach rather than reverse-engineering the tool wrapper.\n\n7. **Never guess how to load a registered dataset — ask for its loading plan.** FIRST check whether the tool you need already takes a `dataset_id` (e.g. `dataset_score_bulk_samples`, `dataset_compare_activity_by_group`, `dataset_score_signature`, `dataset_get_integration_plan`). **Those tools load the dataset themselves — call them directly. Do NOT call `dataset_plan_analysis` and do NOT load the data first; that wastes two steps.** Only when you need a tool that takes an `adata_path` (e.g. `decoupler_differential_expression`, `decoupler_pseudobulk_aggregate`, the metadata tools) do you need the data on disk — and then your first call is `dataset_load(dataset_id=...)`. It EXECUTES the whole loading plan in one step (the correct loader for the source type, the precomputed collapsed URL, the clinical join, the curated-sample filter) and returns the analysis-ready `adata_path` plus the analysis path, data_level and default contrast — pass that path straight to the analysis tool. Do NOT hand-pick loaders, do NOT call `dataset_plan_analysis`/`dataset_describe` first just to load (use those only when the user asks how a dataset WOULD be analyzed), and do NOT re-load a dataset already loaded this turn. Do NOT open a dataset by pattern-matching its URL out of ## Available Datasets and picking a loader that looks right: `decoupler_load_geo_series_matrix` does not read h5ad, a bare `urllib`/`requests` download cannot authenticate to the private data repo, and `ad.read_h5ad` on a URL is not a local path. Each wrong guess costs a step, and `dataset_load` already makes the right calls — including using a dataset's **collapsed** file variant when one exists (loading the uncollapsed one instead forces two extra steps, `decoupler_annotate_probes_with_gpl` + `decoupler_collapse_probes_to_genes`).\n\n8. **Load each dataset exactly once per turn.** The loaded AnnData is written to `output_path` and cached. If you already loaded a dataset this turn, reuse that path — never re-download or re-load it to 'check' something.\n\n## Available Functions\n\nYou have access to the following functions. These functions are already available\
+  \ entry shows `subset_query=\"tumor_subtype != ''\"`, pass that exact string to `decoupler_differential_expression`.\n\n6. **Never import from `tools`, `src.tools`, `server`, or any `_mcp` module, and never inspect `FunctionTool`/`.fn`/`.run()` internals.** All functions listed below are already pre-loaded as plain callables in your namespace and are re-verified before every step. If a call to one of them raises `NameError` or `'FunctionTool' object is not callable`, do NOT start importing or introspecting — simply retry the exact same call with the parameters documented below. If it still fails after one retry, move on to a different approach rather than reverse-engineering the tool wrapper.\n\n## Available Functions\n\nYou have access to the following functions. These functions are already available\
   \ in your Python environment and can be called directly:\n\n{% for func_name, schema in functions.items() %}\n**{{ schema.function.name\
   \ }}({% for param_name in schema.function.parameters.properties.keys() %}{{ param_name }}{{ \", \" if not loop.last }}{%\
   \ endfor %})**\n- Description: {{ schema.function.description }}\n- Parameters:\n  {% for param_name, param_info in schema.function.parameters.properties.items()\

pytest.ini

@@ -1,11 +0,0 @@
-[pytest]
-# Belt-and-braces. The actual fix was renaming the offender:
-# scripts/test_limma_runtime.py -> scripts/check_limma_runtime.py. It is a
-# hand-run rpy2/limma diagnostic that calls sys.exit(1) at import when rpy2 is
-# absent, and pytest collected it on the test_* filename alone — which aborted
-# the whole run with an INTERNALERROR before a single test executed.
-#
-# testpaths keeps that from recurring if another test_*.py is ever dropped into
-# scripts/ or elsewhere outside tests/. It applies only when no path argument is
-# given, so `pytest` and `pytest tests/` agree, and an explicit path still works.
-testpaths = tests

requirements.in

@@ -25,7 +25,7 @@
 # so git+https://huggingface.co/... fails the Space build. To release new
 # manifests: bump version in biodata-registry/pyproject.toml, run its
 # scripts/release.sh, then update this URL + commit.
-biodata-registry @ https://huggingface.co/anne-voigt/biodata-registry/resolve/7cc464295182f17bc95b8b5bee78a588a4e66440/biodata_registry-0.1.17-py3-none-any.whl
+biodata-registry @ https://huggingface.co/anne-voigt/biodata-registry/resolve/0e29c5efa60a66b417f79e067dc2b0d927fcdc83/biodata_registry-0.1.8-py3-none-any.whl
 
 # --- GEO platform (GPL) SOFT parsing (scripts/assemble_*.py precompute) ---
 GEOparse==2.0.4
@@ -103,22 +103,3 @@ plotnine
 # src/tools/rna/analysis.py). Not imported by first-party code, so it must be
 # pinned here explicitly or the plot is skipped at runtime.
 igraph
-
-# --- Security floors (ADR-0014 pip-audit remediation) ---
-# These are all transitive deps (not first-party imports); we constrain the
-# floor only to pull them past known CVEs. None are HF sdk_version-locked (only
-# gradio + mcp are), so they float freely to the fixed release.
-#   pillow            — CVE-2026-25990, CVE-2026-40192, CVE-2026-42309/42310/42311,
-#                       PYSEC-2026-165 (fixed in 12.2.0). via fpdf2/gradio/matplotlib.
-#   langsmith         — GHSA-f4xh-w4cj-qxq8 (fixed in 0.8.18). via langchain-core.
-#   pydantic-settings — GHSA-4xgf-cpjx-pc3j (fixed in 2.14.2). via mcp.
-#   cryptography      — CVE-2026-69247 (fixed in 50.0.0). via authlib (which backs
-#                       the ADR-0012 OAuth sign-in) and cffi→rpy2-rinterface. This
-#                       one is NOT a free float: authlib is on the auth path, so the
-#                       bump was verified on its own before shipping — see memory.md
-#                       2026-08-04. authlib 1.7.2 declares `cryptography>=3.2` and
-#                       imports only the stable hazmat primitives, so 50.x is in range.
-pillow>=12.2.0
-langsmith>=0.8.18
-pydantic-settings>=2.14.2
-cryptography>=50.0.0

requirements.txt

@@ -23,7 +23,6 @@ anyio==4.14.0
     #   anthropic
     #   gradio
     #   httpx
-    #   langsmith
     #   mcp
     #   openai
     #   py-key-value-aio
@@ -45,7 +44,7 @@ backports-tarfile==1.2.0
     # via jaraco-context
 beartype==0.22.9
     # via py-key-value-aio
-biodata-registry @ https://huggingface.co/anne-voigt/biodata-registry/resolve/7cc464295182f17bc95b8b5bee78a588a4e66440/biodata_registry-0.1.17-py3-none-any.whl
+biodata-registry @ https://huggingface.co/anne-voigt/biodata-registry/resolve/0e29c5efa60a66b417f79e067dc2b0d927fcdc83/biodata_registry-0.1.8-py3-none-any.whl
     # via -r requirements.in
 brotli==1.2.0
     # via gradio
@@ -71,9 +70,8 @@ click==8.4.1
     #   uvicorn
 contourpy==1.3.3
     # via matplotlib
-cryptography==50.0.0
+cryptography==49.0.0
     # via
-    #   -r requirements.in
     #   authlib
     #   joserfc
     #   pyjwt
@@ -89,7 +87,6 @@ defusedxml==0.7.1
 distro==1.9.0
     # via
     #   anthropic
-    #   langsmith
     #   openai
 dnspython==2.8.0
     # via email-validator
@@ -258,10 +255,8 @@ langgraph-prebuilt==1.1.0
     # via langgraph
 langgraph-sdk==0.4.2
     # via langgraph
-langsmith==0.9.7
-    # via
-    #   -r requirements.in
-    #   langchain-core
+langsmith==0.8.16
+    # via langchain-core
 legacy-api-wrap==1.5
     # via
     #   anndata
@@ -292,7 +287,7 @@ matplotlib==3.11.0
     #   pydeseq2
     #   scanpy
     #   seaborn
-mcp==1.28.1
+mcp==1.28.0
     # via
     #   -r requirements.in
     #   fastmcp
@@ -399,9 +394,8 @@ patsy==1.0.2
     # via
     #   scanpy
     #   statsmodels
-pillow==12.3.0
+pillow==11.3.0
     # via
-    #   -r requirements.in
     #   fpdf2
     #   gradio
     #   matplotlib
@@ -434,10 +428,8 @@ pydantic==2.11.10
     #   pydantic-settings
 pydantic-core==2.33.2
     # via pydantic
-pydantic-settings==2.14.2
-    # via
-    #   -r requirements.in
-    #   mcp
+pydantic-settings==2.14.1
+    # via mcp
 pydeseq2==0.5.4
     # via -r requirements.in
 pydub==0.25.1
@@ -573,7 +565,6 @@ six==1.17.0
 sniffio==1.3.1
     # via
     #   anthropic
-    #   langsmith
     #   openai
 sse-starlette==3.4.4
     # via mcp
@@ -626,7 +617,6 @@ typing-extensions==4.15.0
     #   langchain-core
     #   langchain-mcp-adapters
     #   langchain-protocol
-    #   langsmith
     #   mcp
     #   numcodecs
     #   openai

resources/mouse_ensembl_symbol_map.tsv.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1bfa6ee6b405a13f19e5d334211663fc74b7235528e0c440941741cb7d1c8dc9
-size 470035

ruff.toml

@@ -1,31 +0,0 @@
-# Ruff config — shared lint/format baseline across the PDAC-system repos.
-# Conservative rule set aimed at readability + standards, not pedantry:
-#   E/W pycodestyle, F pyflakes (dead code, undefined names, unused imports),
-#   I import sorting, UP pyupgrade, B bugbear (likely bugs).
-line-length = 100
-target-version = "py311"
-
-extend-exclude = [
-  ".claude", ".git", "security/.audit-venv", "build", "dist",
-  "__pycache__", ".venv", "node_modules", "data", "logs", "out",
-]
-
-[lint]
-select = ["E", "W", "F", "I", "UP", "B"]
-# E501 (line too long) is reported by the formatter, not worth failing lint on
-# during active dev; B008 (function call in default arg) is a FastAPI/Gradio idiom.
-ignore = ["E501", "B008"]
-
-[format]
-quote-style = "double"
-
-[lint.per-file-ignores]
-# E402: these files deliberately run setup before importing — a
-# sys.path.insert() so first-party modules resolve, or an env/R bootstrap
-# in the server/app entrypoints. The imports cannot precede that setup, so
-# E402 is a false positive here (kept active everywhere else).
-"app.py" = ["E402"]
-"server.py" = ["E402"]
-"scripts/*" = ["E402"]
-"tests/*" = ["E402"]
-"src/tools/*" = ["E402"]

scripts/_gse205154_sears_common.py

@@ -19,7 +19,6 @@ to the 289 GEO samples by name.
 
 Usage (per variant): .venv/bin/python scripts/assemble_gse205154_sears*.py [--dry-run]
 """
-
 import gzip
 import io
 import sys
@@ -150,7 +149,9 @@ def build_gse205154_anndata(matrix_url: str, *, data_level: str) -> ad.AnnData:
     if "hgnc_symbol" in var.columns:
         var["SYMBOL"] = var["hgnc_symbol"].astype(str)
     if TMM_FILTER_COLUMN in var.columns:
-        var[TMM_FILTER_COLUMN] = var[TMM_FILTER_COLUMN].astype(str).str.upper().eq("TRUE")
+        var[TMM_FILTER_COLUMN] = (
+            var[TMM_FILTER_COLUMN].astype(str).str.upper().eq("TRUE")
+        )
 
     # expression — genes x samples, transpose to samples x genes (index = ST-id).
     expr = df[sample_cols].copy()
@@ -159,11 +160,12 @@ def build_gse205154_anndata(matrix_url: str, *, data_level: str) -> ad.AnnData:
 
     # obs — join GEO metadata to the TSV sample columns by sample_title (ST-id).
     geo_obs = load_geo_obs()
-    title_to_gsm = dict(zip(geo_obs["sample_title"], geo_obs.index, strict=True))
+    title_to_gsm = dict(zip(geo_obs["sample_title"], geo_obs.index))
     missing = [st for st in x_df.index if st not in title_to_gsm]
     if missing:
         raise RuntimeError(
-            f"{len(missing)} TSV sample columns have no GEO sample_title match, e.g. {missing[:5]}"
+            f"{len(missing)} TSV sample columns have no GEO sample_title match, "
+            f"e.g. {missing[:5]}"
         )
     obs = geo_obs.set_index("sample_title").loc[x_df.index].copy()
     obs["sample_title"] = x_df.index.values
@@ -182,16 +184,11 @@ def build_gse205154_anndata(matrix_url: str, *, data_level: str) -> ad.AnnData:
 
     adata = ad.AnnData(X=X, obs=obs, var=var)
     print(f"  AnnData: {adata.shape[0]} samples x {adata.shape[1]} genes")
-    print(
-        "  tumor_type:\n"
-        + adata.obs.get("tumor_type", pd.Series(dtype=str)).value_counts().to_string()
-    )
+    print("  tumor_type:\n" + adata.obs.get("tumor_type", pd.Series(dtype=str)).value_counts().to_string())
     return adata
 
 
-def assemble(
-    dataset_id: str, matrix_url: str, data_level: str, script_name: str, dry_run: bool
-) -> None:
+def assemble(dataset_id: str, matrix_url: str, data_level: str, script_name: str, dry_run: bool) -> None:
     """End-to-end: build, stamp provenance, write + (optionally) upload to HF."""
     print(f"=== Assembling {dataset_id} (data_level={data_level}) ===")
     adata = build_gse205154_anndata(matrix_url, data_level=data_level)

scripts/_investigate_gpl6244.py

@@ -7,7 +7,6 @@ Checks whether the GSE28735-embedded GPL6244 platform table has a usable
 
 Usage: .venv/bin/python scripts/_investigate_gpl6244.py
 """
-
 import sys
 from pathlib import Path
 
@@ -16,6 +15,7 @@ sys.path.insert(0, str(ROOT))
 sys.path.insert(0, str(ROOT / "scripts"))
 
 import GEOparse
+
 from _precompute_common import _ensure_gse_family_soft
 
 GSE_ACCESSION = "GSE28735"

scripts/_precompute_common.py

@@ -8,7 +8,7 @@ shape/obs/var as a live load -- just without the per-query reasoning steps.
 """
 
 import sys
-from datetime import UTC, datetime
+from datetime import datetime, timezone
 from pathlib import Path
 
 import anndata as ad
@@ -60,7 +60,7 @@ def load_series_matrix_to_anndata(url_or_path: str) -> ad.AnnData:
 
 def _gse_range_subdir(gse_accession: str) -> str:
     """e.g. 'GSE62165' -> 'GSE62nnn', 'GSE71989' -> 'GSE71nnn'."""
-    digits = gse_accession[len("GSE") :]
+    digits = gse_accession[len("GSE"):]
     return f"GSE{digits[:-3]}nnn"
 
 
@@ -135,29 +135,21 @@ def annotate_with_gpl(adata, gse_accession, gpl_accession, gene_symbol_column="G
     source_used = f"GEOparse:{gse_accession}:{gpl_accession}"
     gpl = gse.gpls.get(gpl_accession)
     if gpl is None or gpl.table is None or gpl.table.empty:
-        return (
-            adata,
-            None,
-            {
-                "source_used": source_used,
-                "gpl_columns_available": [],
-                "requested_column_found": False,
-                "error": f"{gpl_accession} table not found/empty in {gse_accession} "
-                f"(available GPLs: {list(gse.gpls.keys())})",
-            },
-        )
+        return adata, None, {
+            "source_used": source_used,
+            "gpl_columns_available": [],
+            "requested_column_found": False,
+            "error": f"{gpl_accession} table not found/empty in {gse_accession} "
+                     f"(available GPLs: {list(gse.gpls.keys())})",
+        }
 
     gpl_columns_available = list(gpl.table.columns)
     if gene_symbol_column not in gpl_columns_available:
-        return (
-            adata,
-            None,
-            {
-                "source_used": source_used,
-                "gpl_columns_available": gpl_columns_available,
-                "requested_column_found": False,
-            },
-        )
+        return adata, None, {
+            "source_used": source_used,
+            "gpl_columns_available": gpl_columns_available,
+            "requested_column_found": False,
+        }
 
     probe_col = "ID" if "ID" in gpl.table.columns else gpl.table.columns[0]
     gpl_map = gpl.table.set_index(probe_col)[gene_symbol_column]
@@ -169,24 +161,18 @@ def annotate_with_gpl(adata, gse_accession, gpl_accession, gene_symbol_column="G
 
     n_total = len(probe_ids)
     n_annotated = int((mapped != "").sum())
-    return (
-        adata,
-        gene_symbol_column,
-        {
-            "source_used": source_used,
-            "gpl_columns_available": gpl_columns_available,
-            "n_total": n_total,
-            "n_annotated": n_annotated,
-            "n_unannotated": n_total - n_annotated,
-            "coverage_pct": round(100 * n_annotated / n_total, 1) if n_total else 0.0,
-            "requested_column_found": True,
-        },
-    )
-
-
-def annotate_with_gpl_gene_assignment(
-    adata, gse_accession, gpl_accession, assignment_column="gene_assignment"
-):
+    return adata, gene_symbol_column, {
+        "source_used": source_used,
+        "gpl_columns_available": gpl_columns_available,
+        "n_total": n_total,
+        "n_annotated": n_annotated,
+        "n_unannotated": n_total - n_annotated,
+        "coverage_pct": round(100 * n_annotated / n_total, 1) if n_total else 0.0,
+        "requested_column_found": True,
+    }
+
+
+def annotate_with_gpl_gene_assignment(adata, gse_accession, gpl_accession, assignment_column="gene_assignment"):
     """
     Annotate adata.var with a "gene_symbol" column parsed from a GPL
     'gene_assignment'-style column (Affymetrix Gene/Exon ST arrays, e.g.
@@ -212,29 +198,21 @@ def annotate_with_gpl_gene_assignment(
     source_used = f"GEOparse:{gse_accession}:{gpl_accession}:{assignment_column}"
     gpl = gse.gpls.get(gpl_accession)
     if gpl is None or gpl.table is None or gpl.table.empty:
-        return (
-            adata,
-            None,
-            {
-                "source_used": source_used,
-                "gpl_columns_available": [],
-                "requested_column_found": False,
-                "error": f"{gpl_accession} table not found/empty in {gse_accession} "
-                f"(available GPLs: {list(gse.gpls.keys())})",
-            },
-        )
+        return adata, None, {
+            "source_used": source_used,
+            "gpl_columns_available": [],
+            "requested_column_found": False,
+            "error": f"{gpl_accession} table not found/empty in {gse_accession} "
+                     f"(available GPLs: {list(gse.gpls.keys())})",
+        }
 
     gpl_columns_available = list(gpl.table.columns)
     if assignment_column not in gpl_columns_available:
-        return (
-            adata,
-            None,
-            {
-                "source_used": source_used,
-                "gpl_columns_available": gpl_columns_available,
-                "requested_column_found": False,
-            },
-        )
+        return adata, None, {
+            "source_used": source_used,
+            "gpl_columns_available": gpl_columns_available,
+            "requested_column_found": False,
+        }
 
     probe_col = "ID" if "ID" in gpl.table.columns else gpl.table.columns[0]
 
@@ -259,27 +237,21 @@ def annotate_with_gpl_gene_assignment(
 
     n_total = len(probe_ids)
     n_annotated = int((mapped != "").sum())
-    return (
-        adata,
-        "gene_symbol",
-        {
-            "source_used": source_used,
-            "gpl_columns_available": gpl_columns_available,
-            "n_total": n_total,
-            "n_annotated": n_annotated,
-            "n_unannotated": n_total - n_annotated,
-            "coverage_pct": round(100 * n_annotated / n_total, 1) if n_total else 0.0,
-            "requested_column_found": True,
-        },
-    )
-
-
-def stamp_provenance(
-    adata, *, source_url, dataset_id, biodata_registry_commit, script_name, extra=None
-):
+    return adata, "gene_symbol", {
+        "source_used": source_used,
+        "gpl_columns_available": gpl_columns_available,
+        "n_total": n_total,
+        "n_annotated": n_annotated,
+        "n_unannotated": n_total - n_annotated,
+        "coverage_pct": round(100 * n_annotated / n_total, 1) if n_total else 0.0,
+        "requested_column_found": True,
+    }
+
+
+def stamp_provenance(adata, *, source_url, dataset_id, biodata_registry_commit, script_name, extra=None):
     """Write plain-string .uns provenance keys (h5ad-safe)."""
     adata.uns["precompute_source_url"] = source_url
-    adata.uns["precompute_built_at"] = datetime.now(UTC).isoformat()
+    adata.uns["precompute_built_at"] = datetime.now(timezone.utc).isoformat()
     adata.uns["precompute_biodata_registry_commit"] = biodata_registry_commit
     adata.uns["precompute_dataset_id"] = dataset_id
     adata.uns["precompute_script"] = script_name

scripts/_verify_precompute.py

@@ -10,7 +10,6 @@ precompute-cache verification checklist:
 
 Usage: .venv/bin/python scripts/_verify_precompute.py <dataset_id>
 """
-
 import sys
 from pathlib import Path
 
@@ -57,15 +56,7 @@ def main():
     result = decoupler_load_url_counts(
         url_or_path=url, feature_id_type=feature_id_type, out_prefix=f"verify_{dataset_id}"
     )
-    for k in (
-        "message",
-        "n_obs",
-        "n_vars",
-        "shape",
-        "obs_columns",
-        "var_index_sample",
-        "output_path",
-    ):
+    for k in ("message", "n_obs", "n_vars", "shape", "obs_columns", "var_index_sample", "output_path"):
         print(f"  {k}: {result.get(k)}")
 
     # 4. dataset_validate_manifest_against_data

scripts/_vst_transform.R

@@ -1,32 +0,0 @@
-#!/usr/bin/env Rscript
-# DESeq2 variance-stabilizing transform for the GSE205154 (Sears) VST sibling.
-#
-# Reads a gzipped integer counts matrix (genes x samples; first column = gene id,
-# header row = sample ids), applies DESeq2::vst (blind, no design — this is a QC/
-# scoring transform, not a DE fit), and writes the gzipped VST matrix (same
-# orientation). Called as an Rscript subprocess from
-# assemble_gse205154_sears_vst.py (the ADR-0002 Rscript-subprocess pattern; no
-# rpy2). VST output is log2-like and homoscedastic → biodata-registry maps it to
-# data_level=log_expression (already-log; Path B). See ADR-0015.
-#
-# Usage: Rscript _vst_transform.R <counts_in.tsv.gz> <vst_out.tsv.gz>
-suppressMessages(library(DESeq2))
-
-args <- commandArgs(trailingOnly = TRUE)
-if (length(args) != 2L) stop("usage: _vst_transform.R <counts_in.tsv.gz> <vst_out.tsv.gz>")
-inp <- args[[1]]
-outp <- args[[2]]
-
-cts <- as.matrix(read.delim(gzfile(inp), row.names = 1, check.names = FALSE))
-storage.mode(cts) <- "integer"
-cat(sprintf("  R: counts matrix %d genes x %d samples\n", nrow(cts), ncol(cts)))
-
-# vst() is DESeq2's fast VST (fits the dispersion trend on nsub genes, then
-# applies the closed-form transform to all genes). blind=TRUE => design-agnostic.
-vsd <- vst(cts, blind = TRUE)
-cat(sprintf("  R: vst done; range %.3f .. %.3f\n", min(vsd), max(vsd)))
-
-con <- gzfile(outp, "wt")
-write.table(vsd, con, sep = "\t", quote = FALSE, col.names = NA)
-close(con)
-cat(sprintf("  R: wrote %s\n", outp))

scripts/analyze_post_analysis_reads.py

@@ -1,251 +0,0 @@
-#!/usr/bin/env python3
-"""Retrospective before/after analysis of post-analysis CSV re-reads.
-
-Answers the question `1ec068b` shipped without answering: **did returning
-`top_table` from `dataset_compare_activity_by_group` actually stop the agent
-re-opening `comparison_path` before writing its solution?**
-
-The intent was a retrospective control arm: historical run records carry their
-own `messages`, so `n_post_analysis_reads` could be recomputed with no re-running
-and no reverting (both Spaces run the new code, so there is no old-code
-deployment left to A/B against).
-
-**That control arm does not exist in the data.** As of 2026-08-06 all 20 run
-traces in `anne-voigt/decoupleRpy_results` persist `"messages": []`, because the
-Gradio path drives `graph.stream()` directly and never fills
-`WorkflowEngine.message_history` — the field `get_trace()` reads. Every
-production trace was therefore written empty. `src/agent.py` now falls back to
-the final graph state, so records written from here on carry the conversation;
-the pre-fix baseline is unrecoverable and has to be rebuilt forward. This script
-reports that explicitly (`no_messages`) rather than scoring an empty record as
-zero re-reads.
-
-The counting logic is imported from ``src/core/run_metrics.py`` — the SAME code
-the live instrumentation uses. Do not re-implement it here: a divergence between
-the two would silently invalidate the comparison.
-
-Arm assignment is by the record's own ``execution_time`` versus ``--cutoff``
-(the `1ec068b` prod deploy). It deliberately does NOT use which sink the run
-landed in: dev runs currently write to the PROD sink despite
-``LOG_SINK_HF_DATASET`` pointing at ``…_dev`` (open TODO). Splitting on time
-sidesteps that bug rather than depending on it — harmless today because both
-Spaces run identical code, but it is why the sink is not the discriminator.
-
-NOTE: this file must NOT be named ``test_*.py`` — pytest collects on filename
-alone and a script that exits at import kills the whole run.
-
-Usage::
-
-    # local run_logs
-    python scripts/analyze_post_analysis_reads.py --source local --dir ./run_logs
-
-    # the prod HF dataset (needs decouplerpy_results_token)
-    python scripts/analyze_post_analysis_reads.py --source hf
-
-    # dump the per-run table too
-    python scripts/analyze_post_analysis_reads.py --source hf --per-run
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import os
-import sys
-from datetime import datetime
-from pathlib import Path
-
-REPO_ROOT = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(REPO_ROOT / "src"))
-
-from core.run_metrics import compute_run_metrics  # noqa: E402
-
-# 1ec068b reached prod on 2026-08-05. Runs at or after this instant are "after".
-DEFAULT_CUTOFF = "2026-08-05T00:00:00"
-
-# The one tool the fix actually touched. Every other terminal tool still returns
-# only paths, so the re-read tail is EXPECTED to persist there — which is why
-# results are grouped by terminal tool and never averaged into one number.
-FIXED_TOOL = "dataset_compare_activity_by_group"
-
-
-# --------------------------------------------------------------------------- #
-# Record loading
-# --------------------------------------------------------------------------- #
-def load_local(directory: str) -> list[tuple[str, dict]]:
-    out = []
-    for path in sorted(Path(directory).glob("**/*.json")):
-        try:
-            with open(path, encoding="utf-8") as f:
-                record = json.load(f)
-            if is_run_record(record):
-                out.append((path.stem, record))
-        except Exception as exc:  # noqa: BLE001
-            print(f"  ! skipping {path}: {exc}", file=sys.stderr)
-    return out
-
-
-def is_run_record(record: dict) -> bool:
-    """Filter out the upload/audit records that share the same sink."""
-    return isinstance(record, dict) and "upload_id" not in record and "messages" in record
-
-
-def load_hf(repo_id: str) -> list[tuple[str, dict]]:
-    from huggingface_hub import HfApi, hf_hub_download
-
-    token = os.environ.get("decouplerpy_results_token")
-    api = HfApi(token=token)
-    files = [
-        f
-        for f in api.list_repo_files(repo_id=repo_id, repo_type="dataset")
-        if f.endswith("/trace.json")
-    ]
-    print(f"Found {len(files)} trace records in {repo_id}")
-
-    out = []
-    for remote in files:
-        try:
-            local = hf_hub_download(
-                repo_id=repo_id, filename=remote, repo_type="dataset", token=token
-            )
-            with open(local, encoding="utf-8") as f:
-                record = json.load(f)
-            if is_run_record(record):
-                out.append((remote.split("/")[-2], record))
-        except Exception as exc:  # noqa: BLE001
-            print(f"  ! skipping {remote}: {exc}", file=sys.stderr)
-    return out
-
-
-# --------------------------------------------------------------------------- #
-# Arm assignment
-# --------------------------------------------------------------------------- #
-def _parse_time(record: dict, run_id: str) -> datetime | None:
-    raw = record.get("execution_time")
-    for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S.%f", "%Y-%m-%dT%H:%M:%S"):
-        try:
-            return datetime.strptime(str(raw), fmt)
-        except (TypeError, ValueError):
-            continue
-    # run_ids are stamped YYYYmmdd_HHMMSS by the Gradio path.
-    try:
-        return datetime.strptime(run_id[:15], "%Y%m%d_%H%M%S")
-    except ValueError:
-        return None
-
-
-def summarize(records: list[tuple[str, dict]], cutoff: datetime) -> list[dict]:
-    rows = []
-    for run_id, record in records:
-        metrics = compute_run_metrics(record.get("messages") or [])
-        when = _parse_time(record, run_id)
-        rows.append(
-            {
-                "run_id": run_id,
-                "time": when,
-                "arm": None if when is None else ("after" if when >= cutoff else "before"),
-                **metrics,
-            }
-        )
-    return rows
-
-
-def fisher(before_hits, before_n, after_hits, after_n):
-    """Two-sided Fisher exact p for (≥1 re-read) before vs after."""
-    try:
-        from scipy.stats import fisher_exact
-    except ImportError:
-        return None
-    table = [
-        [before_hits, before_n - before_hits],
-        [after_hits, after_n - after_hits],
-    ]
-    return float(fisher_exact(table)[1])
-
-
-def report(rows: list[dict], per_run: bool) -> None:
-    measurable = [r for r in rows if r["n_post_analysis_reads"] is not None]
-    unmeasurable = [r for r in rows if r["n_post_analysis_reads"] is None]
-
-    print(f"\nRuns loaded:      {len(rows)}")
-    print(f"  measurable:     {len(measurable)}  (a terminal analysis tool wrote a results CSV)")
-    print(f"  not measurable: {len(unmeasurable)}  (excluded — NOT counted as zero re-reads)")
-    reasons: dict[str, int] = {}
-    for r in unmeasurable:
-        key = r.get("unmeasurable_reason") or "unknown"
-        reasons[key] = reasons.get(key, 0) + 1
-    for reason, n in sorted(reasons.items(), key=lambda kv: -kv[1]):
-        note = ""
-        if reason == "no_messages":
-            note = "  ← the record persisted an empty conversation; nothing to count"
-        print(f"      {reason:<28} {n}{note}")
-    no_arm = [r for r in measurable if r["arm"] is None]
-    if no_arm:
-        print(f"  undated:        {len(no_arm)}  (excluded from the arms)")
-
-    tools = sorted({r["terminal_tool"] or "(unknown)" for r in measurable})
-    print("\nBy terminal tool — fraction of runs with ≥1 post-analysis re-read")
-    print(f"{'terminal tool':<45} {'before':>14} {'after':>14} {'fisher p':>10}")
-    print("-" * 87)
-
-    for tool in tools:
-        arm_counts = {}
-        for arm in ("before", "after"):
-            sel = [
-                r
-                for r in measurable
-                if (r["terminal_tool"] or "(unknown)") == tool and r["arm"] == arm
-            ]
-            hits = sum(1 for r in sel if r["n_post_analysis_reads"] >= 1)
-            arm_counts[arm] = (hits, len(sel))
-        (bh, bn), (ah, an) = arm_counts["before"], arm_counts["after"]
-
-        def fmt(h, n):
-            return f"{h}/{n}" + (f" ({h / n:.0%})" if n else "  (n=0)")
-
-        p = fisher(bh, bn, ah, an) if bn and an else None
-        p_str = f"{p:.4f}" if p is not None else "—"
-        mark = "  ← the fixed tool" if tool == FIXED_TOOL else ""
-        print(f"{tool:<45} {fmt(bh, bn):>14} {fmt(ah, an):>14} {p_str:>10}{mark}")
-
-    print(
-        "\nOnly " + FIXED_TOOL + " was changed by 1ec068b; a persisting tail on the other\n"
-        "tools is expected, not a refutation. Do not average across rows."
-    )
-
-    if per_run:
-        print("\nPer-run detail")
-        for r in sorted(measurable, key=lambda r: r["time"] or datetime.min):
-            print(
-                f"  {r['run_id']:<24} {str(r['arm']):<7} "
-                f"reads={r['n_post_analysis_reads']} "
-                f"after_steps={r['n_steps_after_analysis']} "
-                f"tool={r['terminal_tool']}"
-            )
-
-
-def main() -> int:
-    ap = argparse.ArgumentParser(description=__doc__)
-    ap.add_argument("--source", choices=("local", "hf"), default="local")
-    ap.add_argument("--dir", default="./run_logs", help="local source directory")
-    ap.add_argument(
-        "--repo",
-        default=os.environ.get("LOG_SINK_HF_DATASET", "anne-voigt/decoupleRpy_results"),
-        help="HF dataset repo id (hf source)",
-    )
-    ap.add_argument("--cutoff", default=DEFAULT_CUTOFF, help="1ec068b deploy instant (ISO)")
-    ap.add_argument("--per-run", action="store_true")
-    args = ap.parse_args()
-
-    cutoff = datetime.fromisoformat(args.cutoff)
-    records = load_local(args.dir) if args.source == "local" else load_hf(args.repo)
-    if not records:
-        print("No run records found — nothing to report.", file=sys.stderr)
-        return 1
-
-    report(summarize(records, cutoff), args.per_run)
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

scripts/assemble_cptac_pda.py

@@ -5,19 +5,18 @@ Source: broad transcriptomics (RSEM TPM, gene-level collapsed).
 
 Usage: .venv/bin/python scripts/assemble_cptac_pda.py
 """
-
-import os
 import sys
+import os
 from pathlib import Path
 
 ROOT = Path(__file__).parent.parent
 sys.path.insert(0, str(ROOT))
 os.makedirs(ROOT / "tmp" / "datasets", exist_ok=True)
 
-import anndata as ad
 import cptac
-import numpy as np
 import pandas as pd
+import anndata as ad
+import numpy as np
 from huggingface_hub import HfApi
 
 OUT_H5AD = ROOT / "tmp" / "datasets" / "cptac_pda.h5ad"
@@ -31,7 +30,7 @@ def main():
 
     # Get broad transcriptomics (TPM, transcript level)
     print("Getting transcriptomics (broad/TPM)...")
-    tx = pdac.get_transcriptomics(source="broad")
+    tx = pdac.get_transcriptomics(source='broad')
     print(f"  Transcript-level shape: {tx.shape}")
     print(f"  MultiIndex levels: {tx.columns.names}")
 
@@ -52,7 +51,7 @@ def main():
     # Get clinical data
     print("\nGetting clinical data...")
     try:
-        clinical = pdac.get_clinical(source="mssm")
+        clinical = pdac.get_clinical(source='mssm')
         print(f"  Clinical shape: {clinical.shape}")
         print(f"  Clinical columns: {clinical.columns.tolist()[:20]}")
     except Exception as e:
@@ -61,7 +60,7 @@ def main():
 
     # Get follow-up (survival)
     try:
-        followup = pdac.get_followup(source="mssm")
+        followup = pdac.get_followup(source='mssm')
         print(f"  Follow-up shape: {followup.shape}")
         print(f"  Follow-up columns: {followup.columns.tolist()}")
     except Exception as e:
@@ -72,11 +71,7 @@ def main():
     common_samples = tx_gene.index.intersection(clinical.index)
     print(f"\nCommon clinical+expression samples: {len(common_samples)}")
 
-    obs_df = (
-        clinical.loc[common_samples].copy()
-        if len(common_samples) > 0
-        else pd.DataFrame(index=tx_gene.index)
-    )
+    obs_df = clinical.loc[common_samples].copy() if len(common_samples) > 0 else pd.DataFrame(index=tx_gene.index)
     tx_final = tx_gene.loc[obs_df.index]
 
     # Join follow-up (drop overlapping columns first)
@@ -88,22 +83,15 @@ def main():
         overlap = obs_df.columns.intersection(fu_sub.columns)
         fu_sub = fu_sub.drop(columns=overlap)
         if len(fu_sub.columns) > 0:
-            obs_df = obs_df.join(fu_sub, how="left")
+            obs_df = obs_df.join(fu_sub, how='left')
 
     print(f"\nFinal obs shape: {obs_df.shape}")
     print(f"Obs columns: {obs_df.columns.tolist()}")
 
     # Print key metadata
-    key_cols = [
-        "histologic_grade",
-        "tumor_site",
-        "tumor_stage_pathological",
-        "margin_status",
-        "Overall survival, days",
-        "Survival status (1, dead; 0, alive)",
-        "sex",
-        "age",
-    ]
+    key_cols = ['histologic_grade', 'tumor_site', 'tumor_stage_pathological',
+                'margin_status', 'Overall survival, days', 'Survival status (1, dead; 0, alive)',
+                'sex', 'age']
     for col in key_cols:
         if col in obs_df.columns:
             vc = obs_df[col].value_counts().head(8)
@@ -114,7 +102,7 @@ def main():
     print(f"After dedup: {obs_df.shape[1]} columns")
 
     # Cast object columns to str
-    for col in obs_df.select_dtypes(include="object").columns:
+    for col in obs_df.select_dtypes(include='object').columns:
         obs_df[col] = obs_df[col].astype(str)
 
     # Build var

scripts/assemble_cptac_pda_counts.py

@@ -106,22 +106,10 @@ def query_gdc_pancreas_files(max_size: int = 2000) -> list[dict]:
         "filters": {
             "op": "and",
             "content": [
-                {
-                    "op": "in",
-                    "content": {"field": "cases.project.project_id", "value": [GDC_PROJECT]},
-                },
-                {
-                    "op": "in",
-                    "content": {"field": "cases.primary_site", "value": [GDC_PRIMARY_SITE]},
-                },
-                {
-                    "op": "in",
-                    "content": {"field": "data_type", "value": ["Gene Expression Quantification"]},
-                },
-                {
-                    "op": "in",
-                    "content": {"field": "analysis.workflow_type", "value": ["STAR - Counts"]},
-                },
+                {"op": "in", "content": {"field": "cases.project.project_id", "value": [GDC_PROJECT]}},
+                {"op": "in", "content": {"field": "cases.primary_site", "value": [GDC_PRIMARY_SITE]}},
+                {"op": "in", "content": {"field": "data_type", "value": ["Gene Expression Quantification"]}},
+                {"op": "in", "content": {"field": "analysis.workflow_type", "value": ["STAR - Counts"]}},
             ],
         },
         "fields": (
@@ -146,9 +134,7 @@ def query_gdc_pancreas_files(max_size: int = 2000) -> list[dict]:
     total = result["data"]["pagination"]["total"]
     print(f"[gdc] CPTAC-3 / Pancreas STAR-Counts files: {len(hits)} (total reported {total})")
     if total > len(hits):
-        print(
-            f"[gdc] WARNING: {total} files exist but only {len(hits)} fetched — raise --max-size."
-        )
+        print(f"[gdc] WARNING: {total} files exist but only {len(hits)} fetched — raise --max-size.")
     return hits
 
 
@@ -160,9 +146,7 @@ def _participant_from_hit(hit: dict) -> str:
         return ""
 
 
-def build_count_matrix(
-    hits: list[dict], count_column: str
-) -> tuple[pd.DataFrame, pd.DataFrame, dict]:
+def build_count_matrix(hits: list[dict], count_column: str) -> tuple[pd.DataFrame, pd.DataFrame, dict]:
     """Download + parse STAR-Counts. Returns (counts_df samples×genes, var_df, barcode→meta)."""
     id_to_barcode = {h["file_id"]: _barcode_from_hit(h) for h in hits}
     filename_to_barcode = {h["file_name"]: _barcode_from_hit(h) for h in hits}
@@ -253,10 +237,8 @@ def load_clinical() -> pd.DataFrame:
     try:
         import cptac
     except ImportError:
-        print(
-            "[clinical] cptac package not installed — skipping clinical join "
-            "(os_*/grade/site will be empty; install `cptac` to populate)."
-        )
+        print("[clinical] cptac package not installed — skipping clinical join "
+              "(os_*/grade/site will be empty; install `cptac` to populate).")
         return pd.DataFrame()
 
     pdac = cptac.Pdac()
@@ -271,16 +253,12 @@ def load_clinical() -> pd.DataFrame:
     out.index.name = "participant_id"
     os_days = "Overall survival, days"
     os_stat = "Survival status (1, dead; 0, alive)"
-    out["os_days"] = (
-        pd.to_numeric(clin.get(os_days), errors="coerce") if os_days in clin else np.nan
-    )
+    out["os_days"] = pd.to_numeric(clin.get(os_days), errors="coerce") if os_days in clin else np.nan
     if os_stat in clin:
         out["os_event"] = pd.to_numeric(clin[os_stat], errors="coerce")
     else:
         out["os_event"] = np.nan
-    out["histologic_grade"] = (
-        clin.get("histologic_grade", "").astype(str) if "histologic_grade" in clin else ""
-    )
+    out["histologic_grade"] = clin.get("histologic_grade", "").astype(str) if "histologic_grade" in clin else ""
     out["tumor_site"] = clin.get("tumor_site", "").astype(str) if "tumor_site" in clin else ""
     return out
 
@@ -291,16 +269,8 @@ def load_clinical() -> pd.DataFrame:
 def main() -> int:
     ap = argparse.ArgumentParser()
     ap.add_argument("--subtype-table", type=Path, help="Cao 2021 Table S1 (.xlsx/.xls/.csv)")
-    ap.add_argument(
-        "--counts-only",
-        action="store_true",
-        help="Skip subtype/clinical join — verify GDC PDA filter only",
-    )
-    ap.add_argument(
-        "--count-column",
-        default="unstranded",
-        choices=["unstranded", "stranded_first", "stranded_second"],
-    )
+    ap.add_argument("--counts-only", action="store_true", help="Skip subtype/clinical join — verify GDC PDA filter only")
+    ap.add_argument("--count-column", default="unstranded", choices=["unstranded", "stranded_first", "stranded_second"])
     ap.add_argument("--max-size", type=int, default=2000)
     ap.add_argument("--dry-run", action="store_true", help="Build + report, skip HF upload")
     args = ap.parse_args()
@@ -359,15 +329,12 @@ def main() -> int:
 
     if args.dry_run:
         print("[dry-run] skipping HF upload.")
-        print(
-            "\nNEXT: paste the subtype counts above into "
-            "biodata_registry/manifests/cptac_pda_counts.yaml, then run "
-            "dataset_validate_manifest_against_data(dataset_id='cptac_pda_counts')."
-        )
+        print("\nNEXT: paste the subtype counts above into "
+              "biodata_registry/manifests/cptac_pda_counts.yaml, then run "
+              "dataset_validate_manifest_against_data(dataset_id='cptac_pda_counts').")
         return 0
 
     from huggingface_hub import HfApi
-
     HfApi().upload_file(
         path_or_fileobj=str(OUT_H5AD),
         path_in_repo=HF_FILENAME,

scripts/assemble_gse15471.py

@@ -5,19 +5,18 @@ Platform: Affymetrix Human Genome U133 Plus 2.0 (GPL570)
 
 Usage: .venv/bin/python scripts/assemble_gse15471.py
 """
-
-import os
 import sys
+import os
 from pathlib import Path
 
 ROOT = Path(__file__).parent.parent
 sys.path.insert(0, str(ROOT))
 os.makedirs(ROOT / "tmp" / "datasets", exist_ok=True)
 
-import anndata as ad
 import GEOparse
-import numpy as np
 import pandas as pd
+import anndata as ad
+import numpy as np
 from huggingface_hub import HfApi
 
 GEO_ID = "GSE15471"
@@ -92,9 +91,7 @@ def main():
                     var_df["gene_symbol"] = gpl_map.loc[
                         var_df.index.intersection(gpl_map.index), "Gene Symbol"
                     ].reindex(var_df.index)
-                    print(
-                        f"  Gene symbols mapped: {var_df['gene_symbol'].notna().sum()} / {len(var_df)}"
-                    )
+                    print(f"  Gene symbols mapped: {var_df['gene_symbol'].notna().sum()} / {len(var_df)}")
     except Exception as e:
         print(f"  Warning: Could not map GPL570: {e}")
 

scripts/assemble_gse16515_mayo.py

@@ -13,23 +13,15 @@ same linear scale as the live GEO series matrix load.
 
 Usage: .venv/bin/python scripts/assemble_gse16515_mayo.py [--dry-run]
 """
-
 import sys
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
-from _precompute_common import (
-    annotate_with_gpl,
-    load_series_matrix_to_anndata,
-    stamp_provenance,
-    write_and_upload,
-)
+from _precompute_common import annotate_with_gpl, load_series_matrix_to_anndata, stamp_provenance, write_and_upload
 
 DATASET_ID = "gse16515_mayo"
 GSE_ACCESSION = "GSE16515"
-SOURCE_URL = (
-    "https://ftp.ncbi.nlm.nih.gov/geo/series/GSE16nnn/GSE16515/matrix/GSE16515_series_matrix.txt.gz"
-)
+SOURCE_URL = "https://ftp.ncbi.nlm.nih.gov/geo/series/GSE16nnn/GSE16515/matrix/GSE16515_series_matrix.txt.gz"
 GPL_ACCESSION = "GPL570"
 GENE_SYMBOL_COLUMN = "Gene Symbol"
 BIODATA_REGISTRY_COMMIT = "b9ae20c6ebe4809e8bfea2f8e6dfef6b98170097"
@@ -42,24 +34,18 @@ def main():
     adata = load_series_matrix_to_anndata(SOURCE_URL)
     print(f"Shape: {adata.shape[0]} samples x {adata.shape[1]} probes")
     print(f"Obs columns: {adata.obs.columns.tolist()}")
-    print(
-        f"X range: {adata.X.min():.3f} - {adata.X.max():.3f} "
-        "(expect linear scale ~2-65000, NOT log2 - manifest declares data_level: normalized)"
-    )
+    print(f"X range: {adata.X.min():.3f} - {adata.X.max():.3f} "
+          "(expect linear scale ~2-65000, NOT log2 - manifest declares data_level: normalized)")
 
     if "tissue" in adata.obs.columns:
         print("\ntissue value counts:")
         print(adata.obs["tissue"].value_counts(dropna=False).to_string())
 
     print(f"\nAnnotating probes with {GPL_ACCESSION} ...")
-    adata, sym_col, stats = annotate_with_gpl(
-        adata, GSE_ACCESSION, GPL_ACCESSION, gene_symbol_column=GENE_SYMBOL_COLUMN
-    )
+    adata, sym_col, stats = annotate_with_gpl(adata, GSE_ACCESSION, GPL_ACCESSION, gene_symbol_column=GENE_SYMBOL_COLUMN)
     if sym_col is None:
         raise RuntimeError(f"GPL annotation failed: {stats}")
-    print(
-        f"  sym_col_used={sym_col!r}, n_annotated={stats['n_annotated']}/{stats['n_total']} ({stats['coverage_pct']}%)"
-    )
+    print(f"  sym_col_used={sym_col!r}, n_annotated={stats['n_annotated']}/{stats['n_total']} ({stats['coverage_pct']}%)")
 
     stamp_provenance(
         adata,

scripts/assemble_gse17891.py

@@ -5,19 +5,18 @@ GPL8321 (mouse) present — filter to human only.
 
 Usage: .venv/bin/python scripts/assemble_gse17891.py
 """
-
-import os
 import sys
+import os
 from pathlib import Path
 
 ROOT = Path(__file__).parent.parent
 sys.path.insert(0, str(ROOT))
 os.makedirs(ROOT / "tmp" / "datasets", exist_ok=True)
 
-import anndata as ad
 import GEOparse
-import numpy as np
 import pandas as pd
+import anndata as ad
+import numpy as np
 from huggingface_hub import HfApi
 
 GEO_ID = "GSE17891"
@@ -31,9 +30,8 @@ def main():
     gse = GEOparse.get_GEO(geo=GEO_ID, destdir=str(ROOT / "tmp" / "datasets"), silent=True)
 
     # Filter to human samples (GPL570 only)
-    human_gsms = {
-        k: v for k, v in gse.gsms.items() if v.metadata.get("platform_id", [""])[0] == "GPL570"
-    }
+    human_gsms = {k: v for k, v in gse.gsms.items()
+                  if v.metadata.get("platform_id", [""])[0] == "GPL570"}
     print(f"  Total GSMs: {len(gse.gsms)}, Human (GPL570): {len(human_gsms)}")
 
     # Inspect first human sample

scripts/assemble_gse205154_sears.py

@@ -8,7 +8,6 @@ Ensembl ID; var['SYMBOL'] = HGNC symbol. data_level: tpm (float32) -> limma/ttes
 
 Usage: .venv/bin/python scripts/assemble_gse205154_sears.py [--dry-run]
 """
-
 import sys
 from pathlib import Path
 

scripts/assemble_gse205154_sears_counts.py

@@ -13,7 +13,6 @@ limitations). For length-aware DE, work from the raw GEO file directly.
 
 Usage: .venv/bin/python scripts/assemble_gse205154_sears_counts.py [--dry-run]
 """
-
 import sys
 from pathlib import Path
 

scripts/assemble_gse205154_sears_filtered_tmm.py

@@ -1,75 +0,0 @@
-"""
-Assemble precomputed h5ad for gse205154_sears_filtered_tmm — Sears lab (OHSU)
-PDAC bulk RNA-seq, edgeR TMM restricted to expression-filter-passing genes
-(Path B). Subseries of GSE281129; PMID 39789181.
-
-Same 289 FFPE samples and the SAME edgeR TMM values as gse205154_sears_tmm, but
-the gene axis is SUBSET to the genes the authors' expression filter passed
-(TMM_pass_filter == TRUE in GSE205154_Gene_Level_TMM_Estimates.txt.gz). That flag
-is edgeR's own filterByExpr-style low-count filter, so this variant is "edgeR TMM
-after a standard low-count filter" — a ready-to-score matrix with low-count/noise
-genes already dropped, so the agent needn't subset at runtime.
-
-Values are edgeR TMM and NOT log-scaled (data_level: normalized) → log-transform
-before limma/ttest; never DESeq2. var.index = versioned Ensembl ID; var['SYMBOL']
-= HGNC symbol; obs is the identical 289-sample metadata as the other siblings.
-
-Usage: .venv/bin/python scripts/assemble_gse205154_sears_filtered_tmm.py [--dry-run]
-"""
-import sys
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).resolve().parent))
-from _gse205154_sears_common import (  # noqa: E402
-    BIODATA_REGISTRY_COMMIT,
-    SERIES_MATRIX_URL,
-    TMM_FILTER_COLUMN,
-    build_gse205154_anndata,
-)
-from _precompute_common import stamp_provenance, write_and_upload  # noqa: E402
-
-DATASET_ID = "gse205154_sears_filtered_tmm"
-MATRIX_URL = (
-    "https://ftp.ncbi.nlm.nih.gov/geo/series/GSE205nnn/GSE205154/suppl/"
-    "GSE205154_Gene_Level_TMM_Estimates.txt.gz"
-)
-DATA_LEVEL = "normalized"
-
-
-def main():
-    dry_run = "--dry-run" in sys.argv
-    print(f"=== Assembling {DATASET_ID} (data_level={DATA_LEVEL}) ===")
-
-    adata = build_gse205154_anndata(MATRIX_URL, data_level=DATA_LEVEL)
-    if TMM_FILTER_COLUMN not in adata.var.columns:
-        raise RuntimeError(f"{TMM_FILTER_COLUMN} not present in var — cannot filter genes")
-
-    n_before = adata.n_vars
-    keep = adata.var[TMM_FILTER_COLUMN].astype(bool).to_numpy()
-    adata = adata[:, keep].copy()
-    n_after = adata.n_vars
-    print(f"  Filtered to {TMM_FILTER_COLUMN}==TRUE: {n_before} -> {n_after} genes "
-          f"({n_before - n_after} low-count genes dropped)")
-    if n_after == 0 or n_after == n_before:
-        raise RuntimeError(f"Unexpected filter result: {n_before} -> {n_after}")
-
-    stamp_provenance(
-        adata,
-        source_url=MATRIX_URL,
-        dataset_id=DATASET_ID,
-        biodata_registry_commit=BIODATA_REGISTRY_COMMIT,
-        script_name="scripts/assemble_gse205154_sears_filtered_tmm.py",
-        extra={
-            "precompute_geo_accession": "GSE205154",
-            "precompute_superseries": "GSE281129",
-            "precompute_pmid": "39789181",
-            "precompute_data_level": DATA_LEVEL,
-            "precompute_gene_filter": f"{TMM_FILTER_COLUMN}==TRUE ({n_before}->{n_after} genes)",
-            "precompute_metadata_source": SERIES_MATRIX_URL,
-        },
-    )
-    write_and_upload(adata, DATASET_ID, dry_run=dry_run)
-
-
-if __name__ == "__main__":
-    main()

scripts/assemble_gse205154_sears_tmm.py

@@ -10,7 +10,6 @@ limma/ttest.
 
 Usage: .venv/bin/python scripts/assemble_gse205154_sears_tmm.py [--dry-run]
 """
-
 import sys
 from pathlib import Path