diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 4ad215910b37f86e653205eacf7b5fbc942cd72a..0000000000000000000000000000000000000000 --- a/.dockerignore +++ /dev/null @@ -1,37 +0,0 @@ -# ADR-0007 sandbox image build context trimming. -# The sandbox Dockerfile only COPYs `src/` and `requirements.txt`; everything -# else in the repo (multi-GB venvs, git history, local audit tooling, caches) -# would otherwise be streamed to the Docker daemon on every build. Excluded here -# so `docker build -f docker/sandbox.Dockerfile .` sends a small context. - -# Virtualenvs / local interpreters -.venv/ -venv/ -security/.audit-venv/ - -# VCS + tooling metadata -.git/ -.github/ -.claude/ - -# Local security-audit tooling (not needed in the sandbox image) -security/ - -# Python caches / build artifacts -**/__pycache__/ -**/*.pyc -**/*.pyo -.pytest_cache/ -*.egg-info/ - -# Runtime scratch / caches / generated data that must not ship in the image -src/tmp/ -**/geo_cache/ -*.h5ad -*.log - -# Docs, tests, notebooks — not part of the runtime image -docs/ -tests/ -scripts/ -*.ipynb diff --git a/.gitattributes b/.gitattributes index b7eacbb4700108048949ba0711ca12d4fbd91084..a6344aac8c09253b3b630fb776ae94478aa0275b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,12 +33,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text - -# --- Multi-agent concurrency ------------------------------------------------- -# Append-style bookkeeping files. Several agents write these in parallel from -# separate worktrees; union merge keeps BOTH sides of a conflicting hunk rather -# than halting, so no lane can resolve a conflict by discarding another lane. -# Cost: occasional duplicate lines to tidy. Never set this on code or lockfiles. -memory.md merge=union -TODO.md merge=union -REGISTRY_TODO_PLANS.md merge=union diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 2f1a7817572866e3fc111e871a54f3bd1d3556d9..0000000000000000000000000000000000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,36 +0,0 @@ -# ADR-0014 — automated dependency bump PRs. Active only when this repo is -# mirrored to GitHub (see .github/workflows/security.yml for the deploy-model -# note); the HF-only path relies on the weekly pip-audit run to surface stale, -# vulnerable pins instead. -version: 2 -updates: - - package-ecosystem: pip - directory: "/" - schedule: - interval: weekly - open-pull-requests-limit: 5 - # gradio and mcp are pinned to the HF Space sdk_version and must not be - # bumped by an automated PR — bumping them requires a coordinated Space - # rebuild. Security advisories still surface via pip-audit. - ignore: - - dependency-name: gradio - - dependency-name: mcp - - - package-ecosystem: github-actions - directory: "/" - schedule: - interval: weekly - - - package-ecosystem: docker - directory: "/docker" - schedule: - interval: weekly - # The sandbox base image (ADR-0007) tracks a specific Python line; a major/ - # minor jump (e.g. 3.11 -> 3.14, PR #3, closed 2026-07-08) can break the - # rpy2/scanpy/decoupler stack and is not validated by the Dockerfile scan. - # Allow patch bumps (security) but not major/minor without a manual build test. - ignore: - - dependency-name: python - update-types: - - "version-update:semver-major" - - "version-update:semver-minor" diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml deleted file mode 100644 index cb7c71a9e180e1637602b8895788c77e9de46520..0000000000000000000000000000000000000000 --- a/.github/workflows/security.yml +++ /dev/null @@ -1,61 +0,0 @@ -# ADR-0014 — CI security scan (dependencies, static analysis, secrets, image). -# -# NOTE ON THIS REPO'S DEPLOY MODEL: DecoupleRpy_Agent's `origin` is the -# HuggingFace Space (git push builds the Space); HuggingFace does not run GitHub -# Actions. This workflow therefore executes ONLY if the repo is also mirrored to -# GitHub (or `biodata-registry`, which reuses this same file). In the HF-only -# path the enforced check is the pre-push hook (`make install-hooks`) running the -# identical scripts/security_scan.sh, plus the scheduled unattended run. Keeping -# the workflow here means adding a GitHub mirror is zero extra work and the scan -# definition never diverges between the two paths. -name: security - -on: - pull_request: - push: - branches: [main] - schedule: - - cron: "0 6 * * 1" # weekly, Monday 06:00 UTC — catches newly disclosed CVEs - workflow_dispatch: - -permissions: - contents: read - -jobs: - scan: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 # gitleaks needs full history - - - uses: actions/setup-python@v6 - with: - python-version: "3.12" - - - name: Install scanners - run: | - python -m pip install --upgrade pip - # uv drives the CycloneDX SBOM stage; without it that stage SKIPs, which - # is a failure under SECURITY_SCAN_STRICT=1. - pip install pip-audit bandit uv - # gitleaks + trivy via their official installers - curl -sSfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b "$HOME/.local/bin" - curl -sSfL https://github.com/gitleaks/gitleaks/releases/download/v8.30.1/gitleaks_8.30.1_linux_x64.tar.gz | tar -xz -C "$HOME/.local/bin" gitleaks - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - - - name: Run shared security scan - env: - SECURITY_SCAN_STRICT: "1" # in CI every scanner is present; a skip is a bug - # The GitHub runner has no R; rpy2 in API mode refuses to build without it, - # which breaks pip-audit's dependency resolve. ABI mode builds without R. - RPY2_CFFI_MODE: ABI - run: make security-scan - - - name: Upload scan artifacts - if: always() - uses: actions/upload-artifact@v7 - with: - name: security-scan-${{ github.run_id }} - path: security/ - retention-days: 90 diff --git a/.gitignore b/.gitignore index 7ac4f2bbaf33debad62fc96b59bffd3d55793e43..a450705e183922b9dd14306e47e5cb9283b6c957 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,5 @@ .DS_Store .claude/ -# Local secrets — never commit (matches the other repos) -.env -.env.* -!.env.example .venv/ venv/ __pycache__/ @@ -18,11 +14,3 @@ reports/ upload_staging/ upload_registered/ run_logs/ -# ADR-0014 security scan — cached pip-audit venv (artifacts under security/ are -# committed; the venv is not) -security/.audit-venv/ -CLAUDE.md -memory.md -TODO.md -docs/tasks/ -session_export.docx diff --git a/.gitleaks.toml b/.gitleaks.toml deleted file mode 100644 index e0ad0e5120144685072b5fe39605234d53ec6387..0000000000000000000000000000000000000000 --- a/.gitleaks.toml +++ /dev/null @@ -1,50 +0,0 @@ -# Gitleaks config — ADR-0014 secret scan (shared across the three repos). -# -# Extends the upstream default ruleset (do not replace it) and adds allowlists -# for this repo's known-safe matches: placeholder tokens in docs/tests, the -# committed security scan artifacts, vendored virtualenvs, and two triaged false -# positives. Real secrets (HF write tokens, the ADR-0012 service token) must -# NEVER be committed — they live in HF Space secrets. This scan runs over the -# working tree AND full git history (`gitleaks detect`), because tokens have -# flowed through these repos. -# -# Run: gitleaks detect --config .gitleaks.toml --redact --no-banner - -[extend] -useDefault = true - -# Known-safe paths and placeholder strings. -[[allowlists]] -description = "known-safe paths and placeholder tokens" -paths = [ - '''\.venv/''', - '''(^|/)node_modules/''', - '''security/sbom\.json''', - '''security/pip-audit-.*\.(json|txt)''', - '''\.gitleaks\.toml''', -] -# Documentation and ADRs reference token *names* (research_agent_token, -# ANTHROPIC_API_KEY) as identifiers, never their values. -regexes = [ - '''research_agent_token''', - '''ANTHROPIC_API_KEY''', - '''(?i)your[-_]?(hf|api|anthropic)[-_]?(token|key)[-_]?here''', - '''(?i)example[-_]?(token|key|secret)''', - '''xxx+|placeholder|dummy|fake[-_]?(token|key|secret)''', -] -stopwords = ["example", "placeholder", "changeme"] - -# --- False positives triaged 2026-07-02 (ADR-0014 first run) ---------------- # -# The generic-api-key entropy rule fires on long snake_case keyword arguments in -# the precompute build scripts (`gene_symbol_column=...`, `assignment_column=...`). -# These are column names, not secrets. Matched against the finding text so a -# genuinely new secret in the same file still surfaces. -[[allowlists]] -description = "column-name kwargs in precompute scripts (not secrets)" -regexTarget = "match" -regexes = ['''(gene_symbol|assignment)_column\s*='''] - -# Historical gcp-api-key in app.py (commit 155d8d7) was scrubbed from git history -# on 2026-08-18 (value replaced with **REMOVED-KEY** in all blobs); the key was -# triaged 2026-07-02 as non-functional. No value allowlist needed anymore. - diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 41f457ec834582d4bef3321a3841beee84119898..0000000000000000000000000000000000000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Pre-commit hooks — shared lint/format baseline across the PDAC-system repos. -# Install once per clone: pip install pre-commit && pre-commit install -# Run on all files: pre-commit run --all-files -# -# NOTE: this is separate from the ADR-0014 *security* pre-push hook -# (scripts/hooks/pre-push, installed via `make install-hooks`). This one runs -# ruff at commit time for style/correctness; that one runs the security scan at -# push time. -repos: - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.20 - hooks: - - id: ruff # lint (uses ruff.toml / [tool.ruff]); --fix applies safe fixes - args: [--fix] - - id: ruff-format # formatter (line-length + quote style from config) - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 - hooks: - - id: end-of-file-fixer - - id: trailing-whitespace - - id: check-yaml - - id: check-added-large-files - args: [--maxkb=2048] diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000000000000000000000000000000000000..97639ba1cb7d0f6f1d8961c918701b9c2c7b54e4 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,210 @@ +# DecoupleRpy Agent — Project Context + +This file provides stable architectural context for Claude Code sessions. +**Do not encode current status here** — that belongs in `memory.md`. +Update this file when architectural decisions change, not when code changes. + +--- + +## Memory maintenance (after every commit) + +These status files drift unless updated at commit time. After any `git commit` in this +repo — bound for prod (`origin`, the HF Space) or dev (`hf-dev`) — update whatever that +commit changed, and skip what it didn't: + +- `memory.md` — current status / what just changed +- `TODO.md` — move finished items to Done, add anything new +- `/Users/annivoigt/Documents/GitHub/SHOWCASE_STATUS.md` — the cross-repo rollup; update especially on a prod/dev deploy +- `CLAUDE.md` (this file) — only when the architecture itself changes (rare) + +A PostToolUse hook (`~/.claude/hooks/remind-memory-sync.py`, wired in +`~/.claude/settings.json`) prints this checklist automatically after each commit. It only +*reminds* — the edits are still done by hand. + +--- + +## What This Project Is + +A **Paper2Agent conversion** of the [scverse/decoupler-py](https://github.com/scverse/decoupler-py) bioinformatics library. The goal is to expose decoupler's computational biology methods as MCP tools so non-coding scientists at the Brenden-Colson Center for Pancreatic Care (OHSU, Dr. Rosalie Sears lab) can run analyses via natural language. + +This repo is the **specialist agent** — it does the computation. It is called by a separate orchestrator (`research-coordinator`, deployed at `anne-voigt/research_coordinator` on HuggingFace). + +--- + +## Two-Tier Architecture + +``` +User + └── Research Coordinator (HF Space: anne-voigt/research_coordinator) + ├── Claude API — conceptual/interpretive questions (direct) + └── gradio_client → DecoupleRpy Agent (HF Space: anne-voigt/Paper2Agent_decoupleRpy) + └── MCP tools (this repo) +``` + +The coordinator classifies every user message and routes to the specialist for anything involving computation, datasets, or capability questions. The coordinator does NOT know what datasets are registered — it always asks the specialist. + +--- + +## MCP Tool Layers + +Tools are organized in three layers: + +**Layer 1 — Tutorial tools** (`src/tools/`): Direct Paper2Agent output, one file per tutorial. +- `rna.py` — bulk RNA analysis (DE, TF enrichment, pathway scoring, GEO loading) +- `rna_sc.py` — single-cell RNA analysis +- `rna_visium.py` — spatial transcriptomics (Visium) +- `rna_pstime.py` — pseudotime analysis +- `orthologs.py` — cross-species gene symbol translation +- `dataset_tools.py` — dataset registry MCP tools (incl. `dataset_get_integration_plan`, the cross-dataset early/late/refuse planner) +- `bulk_dataset_tools.py` — bulk-specific dataset operations +- `integration_tools.py` — cross-dataset integration (`integration_mcp` sub-server): `decoupler_meta_analyze` (Mode B late integration) and `decoupler_normalization_concordance` (same-cohort sibling-variant sensitivity check — descriptive agreement, no combine; routed to by plan `mode="concordance"`) + +**Layer 2 — Generic workflows** (`src/workflows/`): Reusable logic called by Layer 1. +- `geo.py` — GEO series matrix loading +- `microarray.py` — probe collapse, data type detection +- `activity_scoring.py` — TF/pathway activity inference +- `activity_stats.py` — group comparisons on activity scores +- `manifest_data_validation.py` — validates manifest semantics against loaded data +- `metadata_validation.py` — metadata column checks +- `survival.py` — survival analysis +- `meta_analysis.py` — cross-dataset meta-analysis engine (`src/core/combine.py` result-envelope contract + field-based strategy dispatch: stouffer / inverse_variance / fisher; Cochran's Q + I²) +- `concordance.py` — same-cohort normalization sensitivity engine: descriptive agreement across sibling-variant result envelopes (pairwise Pearson/Spearman on the shared effect, sign-concordance, effect spread, significant-call Jaccard). The non-combining counterpart to `meta_analysis.py`; backs `decoupler_normalization_concordance`. +- `sanity_checks.py` — result-aware (post-compute) sanity checks (ADR-0002); pure functions emitting non-fatal `sanity_warnings` (effect-size plausibility, tissue-identity contamination, network membership). See "Two safety layers" below. + +**Layer 3 — Dataset manifests** (live in `biodata-registry` package): One YAML per dataset. +Auto-discovered at import time by the registry from the installed `biodata_registry` package. +The `src/datasets/manifests/` directory does **not** exist in this repo — `biodata-registry` is the +sole manifest source. Zero code changes needed to add a dataset — add a YAML to `biodata-registry` +and reinstall the package. + +--- + +## Dataset Manifest System + +Each manifest is a YAML file validated against `DatasetManifest` (see `src/datasets/manifest_schema.py`). + +**Key controlled vocabularies:** +- `modality`: `bulk_microarray`, `bulk_rnaseq`, `sc_rnaseq`, `spatial_rnaseq`, `proteomics` +- `data_level`: `raw_counts`, `log_expression`, `log_ratio`, `normalized`, `tpm`, `fpkm`, `protein_abundance` +- `feature_id_type`: `probe_id`, `gene_symbol`, `ensembl_gene_id`, `entrez_id`, `protein_id` +- `expression_source.type`: `geo_series_matrix`, `geo_soft`, `url`, `gdc`, `cptac`, `local`, `h5ad` (hosted AnnData, single-cell/spatial — ADR-0006) + +**Analysis path routing** (`analysis_path`, derived **modality-first, then `data_level`**): +- **Path P** — `sc_rnaseq` / `spatial_rnaseq` (checked **before** `data_level`, so an sc `raw_counts` h5ad is never mislabeled Path A) → load via `read_h5ad_cached` → pseudobulk → bulk DE / activity. **Never DESeq2/limma on per-cell counts.** (ADR-0006; aligns with biodata-registry 0.1.7's A/B/P.) +- **Path A** — bulk `raw_counts` → DESeq2 +- **Path B** — bulk `log_expression` / `normalized` / etc. → limma or t-test + +`_build_loading_plan` (`src/tools/dataset_tools/_base.py`) emits a single-cell plan for Path P (load + per-cell scoring + a pseudobulk note) instead of the bulk contrast tail. + +**Probe collapse**: Required when `feature_id_type: probe_id` and `requires_collapse: true`. Uses GPL platform annotation downloaded from GEO. + +**Anti-hallucination grounding**: The live dataset registry is injected into the agent's Jinja2 system prompt at every call via `get_system_prompt()`. The agent may ONLY claim access to datasets returned by `dataset_list_available()` — never from training knowledge. + +--- + +## Agent Implementation + +- **Framework**: LangGraph-based `CodeAgent` (`src/agent.py`) +- **System prompt**: Jinja2 template rendered with `functions`, `packages`, and `datasets` at call time +- **Prompt config**: `prompts.yaml` — coordinator system prompt, routing rules, available datasets section +- **Routing**: Research coordinator uses a two-path routing prompt; capability/dataset questions must route to specialist +- **MCP transport**: a single **persistent `server.py --transport http`** process is started once per + container by `GradioAgentUI.__init__` (`ensure_mcp_http_server()`), and tools are registered over HTTP + via `add_mcp_http(url)`. Every tool call reuses that resident process — it does NOT spawn a fresh + `python server.py` per call (the old stdio model, which re-imported rpy2/scanpy/decoupler, re-mounted + 11 sub-servers, and re-read the h5ad on every call: ~44s/step). **stdio is kept as a health-checked + fallback** (`add_mcp(mcp_config.yaml)`) if the HTTP server fails to bind. A process-lifetime in-memory + AnnData cache (`src/cache.read_h5ad_cached`, keyed by path+mtime+size, returns copies so it's safe + under the shared server) means each h5ad is parsed once per process, not once per tool call. Live on + dev + prod since 2026-06-29. + +--- + +## Key Design Decisions + +**Why manifests instead of hardcoding dataset logic in tools?** +Manifests are the single source of truth for dataset semantics. They encode refusal rules, prohibited inferences, metadata column meanings, and loading path — keeping analysis tools generic and dataset-specific knowledge in one place. + +**Why route capability questions to the specialist?** +`dataset_list_available()` returns runtime truth — a manifest with a YAML error won't load even if the file exists. The specialist's answer is richer (capabilities, limitations, metadata) than what code inspection would produce. The coordinator never answers dataset questions from its own knowledge. + +**Why DESeq2 for raw counts, limma/ttest for log-normalized?** +DESeq2 expects integer counts and models dispersion — incorrect on pre-normalized data. Limma and t-test are appropriate for log-normalized expression values. The manifest's `data_level` field enforces the correct path automatically. + +**Why is the cross-dataset decision (early/late/refuse) made in the registry, not the agent?** +Combining ≥2 datasets is gated by `get_integration_plan()` in `biodata-registry` (re-exposed as the agent tool `dataset_get_integration_plan`). It is a pure function of manifest metadata — `early` (pool with `dataset_id` as a batch covariate), `late` (meta-analyze per-dataset result envelopes), or `refuse` — keyed on `data_level`/`organism`/`feature_id_type`/declared contrasts, never on hardcoded dataset pairs. Keeping the decision in the registry makes it deterministic and the single source of truth; a contrast whose factor is absent from a cohort (e.g. a Bailey-subtype contrast against a cohort with no Bailey labels) is refused as `CONFOUNDED_DESIGN` rather than silently degrading to a single-dataset result. The agent acts on `mode` and surfaces `reason`. **Mode B (late / `decoupler_meta_analyze`) shipped 2026-06-19; Mode A (early pooling, T7–T9 — the `decoupler_integrate_datasets` tool, `src/workflows/integration.py`) shipped 2026-06-24 (`a7a8caa`) and is deployed to prod, so an `early` verdict now pools the datasets (feature intersection + a `batch` obs key = `dataset_id`) and runs ONE batch-aware DE with `dataset` modelled as a covariate (DESeq2 `~batch+factor` / limma `~batch+group`) — it no longer falls back to late. For per-sample activity scoring across pooled cohorts (no DE contrast), `decoupler_pool_cohorts` (ADR-0001 item 10, built 2026-06-25) ComBat-corrects the pooled matrix (`scanpy.pp.combat` on log-normalised values, keyed on `poolable_data_level`) before `dataset_score_bulk_samples`, so pooled scores are batch-corrected rather than cohort-confounded. ComBat is the scoring-path counterpart to the DE covariate route — applied only where there is no design matrix, and run without a biological covariate (caveat surfaced on the tool; it must never feed DE testing). (Live deploy status is tracked in `memory.md`, not here.)** + +**Same-cohort variants (sibling quantifications) — why they are NOT a meta-analysis.** +The GSE205154 trio (`gse205154_sears` TPM / `gse205154_sears_counts` counts / `gse205154_sears_tmm` TMM) are the *same 289 samples* quantified three ways. Each manifest's header and refusal rules already forbid pooling them ("same samples in different units"), but until biodata-registry 0.1.6 that relationship was only prose — `get_integration_plan()` did not know about it, so asking to combine two siblings (e.g. TPM + TMM) hit the `data_level` gate (`tpm` ≠ `normalized`) and returned `late`, and the agent ran `decoupler_meta_analyze`. **That is the wrong tool**: meta-analysis assumes *independent* cohorts, so combining identical samples double-counts them — Stouffer inflates the combined score by ≈√2 and Cochran's Q/I² collapse to 0 by construction (zero heterogeneity is guaranteed, not evidence of robustness). The correct framing for cross-variant work is a **normalization concordance / sensitivity check** — descriptive agreement metrics (Pearson/Spearman on scores, sign-concordance, overlap of significant calls), never an inferential combine. **Encoded in `biodata-registry` 0.1.6:** the three sibling manifests carry a structured `cohort_id: gse205154` (+ `variant`), and `get_integration_plan` has a same-cohort gate (runs before the `data_level` gate) that returns `mode="concordance"` when the whole request is one cohort's variants, or refuses with `DUPLICATE_COHORT` when siblings are mixed with independent datasets — keeping the decision in the registry per ADR-0001. The agent-side **concordance routine** the new mode routes to is **built**: `decoupler_normalization_concordance` (`integration_mcp`), backed by `src/workflows/concordance.py` — it reports descriptive agreement (pairwise Pearson/Spearman, sign-concordance, effect spread, significant-call Jaccard) instead of calling `decoupler_meta_analyze`. The agent pins 0.1.6, so `get_integration_plan` returns `mode="concordance"` for same-cohort variants and the agent routes to `decoupler_normalization_concordance` rather than `late`/`decoupler_meta_analyze`. (Live deploy status is tracked in `memory.md`, not here.) + +**Two safety layers — why a result-aware sanity layer (ADR-0002) on top of the registry refusal engine?** +The refusal engine above is **Layer 1**: a pure function of manifest *metadata*, run *before* any computation — it refuses what the data inventory can't support (survival without endpoints, DESeq2-on-TPM, a confounded design). It is blind to a whole class of failure that is only visible *in the results*: normal-tissue contamination of a biopsy (e.g. hepatocyte master-regulator TFs topping a liver-met contrast), an artefactual log2FC from duplicate-collapse, a requested "TF" with no regulon. `src/workflows/sanity_checks.py` is **Layer 2**: pure post-compute checks that emit non-fatal `sanity_warnings` the agent surfaces. Layer 1 *refuses* (you lack the data); Layer 2 *cautions* (the analysis ran, but the result is confounded/partly artefactual) — it never blocks or re-runs a result, because the contrast is often legitimate-but-confounded and refusal would over-block. Kept out of `biodata-registry` deliberately: contamination is data-dependent, not a metadata fact. (NB this Layer-1/Layer-2 *safety* axis is distinct from the Layer-1/2/3 *MCP-tool-organization* axis above.) Phase 1 wires the effect-size check into the DE tool and the membership check into the CollecTRI tool; contamination auto-firing in the enrichment tools and PROGENy/Hallmark parity are the ADR-0002 S2/S3 follow-ups. + +**Why keep GitHub and HF in sync?** +The DecoupleRpy Agent HF Space auto-deploys from GitHub. The research-coordinator does NOT auto-sync — push directly to the HF Space repo using a write token when changes are needed. + +--- + +## Testing Philosophy + +- Every new workflow should have a corresponding test in `tests/` +- Integration tests use small synthetic data (20-60 samples, 50-100 genes) — fast, no network calls +- Fixture tests (e.g., `test_moffitt_integration.py`) lock in known sample counts as regression anchors +- Manifests are tested automatically by `test_dataset_registry.py` and `test_dataset_manifest_contract.py` on every run + +--- + +## HuggingFace Deployment + +- **DecoupleRpy Agent Space**: `anne-voigt/Paper2Agent_decoupleRpy` — `origin` remote points directly to HF; deploy with `git push origin main` +- **Research Coordinator Space**: `anne-voigt/research_coordinator` — does NOT auto-deploy; push directly using write token (`research_agent_token`) +- **Dev Space**: `anne-voigt/Paper2Agent_decoupleRpy_dev` — tracked as `hf-dev` remote; push with `git push hf-dev main` +- **Token name**: `research_agent_token` (rotate after any session where it appears in chat logs) + +--- + +## Dataset Roadmap + +**Currently registered**: 19 manifests, all in `biodata-registry` (the agent pins +the `0.1.5` wheel). This list is a *stable index* — the authoritative per-dataset +validated table (platform, feature_id_type, survival, known issues, validation +dates) lives in `biodata-registry/memory.md`. Do not re-encode validation status +here; that is what drifted this list down to 9. + +*Bulk array — tumor vs normal / paired:* `gse71989_chen`, `gse62165_jiang`, +`gse16515_mayo`, `gse28735_pdac`, `gse15471_badea`. + +*Bulk array — subtype / survival cohorts:* `gse71729_moffitt` (classical/basal), +`gse17891_collisson` (Collisson subtypes), `paca_au_array` (Bailey 4-subtype), +`puleo_2018` (Puleo 5-subtype + survival), `gse21501_stratford` (log-ratio), +`gse57495` (survival), `gse50827_nones` (survival). + +*Bulk RNA-seq counts / RSEM-TPM:* `tcga_paad` (raw_counts, Path A/DESeq2), +`paca_au_rnaseq` (Bailey RSEM, Path A), `paca_ca_rnaseq` (ICGC Canadian, ensembl), +`cptac_pda` (RSEM TPM). + +*Sears GSE205154 sibling trio (same 289-sample FFPE cohort, three quantifications +— Primary 218 / Met 71):* `gse205154_sears` (TPM, Path B), `gse205154_sears_counts` +(est. counts, Path A/DESeq2), `gse205154_sears_tmm` (edgeR TMM, Path B). These three +are **the same samples in different units** — each manifest's refusal rules forbid +pooling them. The only valid cross-variant operation is a **normalization +concordance / sensitivity check**, never an integration or meta-analysis (which +would double-count the cohort). See "Same-cohort variants" below. + +**h5ad files** are hosted at `anne-voigt/pdac-research-data` on HuggingFace +(migrated from `anni-voigt` 2026-06-12; the GSE205154 trio uploaded 2026-06-22). + +**Priority next datasets**: +1. TCGA-PAAD Moffitt subtypes — classical/basal classification not in GDC/Xena clinical matrix; requires inference step +2. Bailey et al. 2016 WGS data — somatic mutation landscape (separate from expression) +3. Structured same-cohort-variant relationship (`cohort_id` field + integration mode) — see "Same-cohort variants" below + +**Before adding any new dataset**: run `dataset_validate_manifest_against_data` to confirm manifest semantics match the actual file. + +--- + +## Consultant Context + +- **Client**: Dr. Rosalie Sears, Brenden-Colson Center for Pancreatic Care, Knight Cancer Institute, OHSU +- **Goal**: Make computational biology methods accessible to non-coding scientists via natural language +- **Scope**: Project 1 of 3 in the consulting engagement (Paper2Agent methodology replication & deployment) diff --git a/Makefile b/Makefile deleted file mode 100644 index e1510c442b028c2d93944da21fbab0f29b1863e4..0000000000000000000000000000000000000000 --- a/Makefile +++ /dev/null @@ -1,28 +0,0 @@ -# DecoupleRpy_Agent — developer entry points. The security targets are the -# shared surface ADR-0014 standardizes across the three repos. -.PHONY: security-scan security-baseline install-hooks help - -help: - @echo "make security-scan Run the ADR-0014 scan (pip-audit, bandit, gitleaks, trivy)." - @echo "make security-baseline Regenerate the accepted-findings bandit baseline." - @echo "make install-hooks Install the pre-push security hook into .git/hooks." - -# ADR-0014: the one shared scan definition. CI, the pre-push hook, and a manual -# run all call this so the check never drifts between entry points. -security-scan: - @bash scripts/security_scan.sh - -# Regenerate the accepted-findings baseline after intentionally adding a new -# suppressed finding. Review the diff before committing — a new entry is a new -# accepted suppression and must be justified in security/ACCEPTED-FINDINGS.md. -security-baseline: - @if command -v bandit >/dev/null 2>&1; then B="bandit"; else B="uvx bandit"; fi; \ - $$B -r src/ -ll -c bandit.yaml -f json -o security/bandit-baseline.json -q; \ - echo "Wrote security/bandit-baseline.json — review the diff and document any new suppression." - -# Wire the pre-push hook. Git hooks are not committed into .git/, so this copies -# the tracked script into place (idempotent). -install-hooks: - @cp scripts/hooks/pre-push .git/hooks/pre-push - @chmod +x .git/hooks/pre-push - @echo "Installed .git/hooks/pre-push — security scan now runs before every push." diff --git a/README.md b/README.md index db6787e35ba45a600573888b4641369b0b08b7ad..dbb7e76b7b5dff2a80059f4ea2f5757c660fd92b 100644 --- a/README.md +++ b/README.md @@ -7,45 +7,6 @@ sdk: gradio sdk_version: 6.18.0 app_file: app.py pinned: false -hf_oauth: true --- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference - -# DecoupleRpy Agent - -The computation specialist for the PDAC system — a LangGraph CodeAgent exposing -~52 decoupler-py bioinformatics tools over MCP. Architecture is in `CLAUDE.md`; -status in `memory.md`. - -## Security-review orientation - -The security-relevant entry points, in priority order: - -- **User-upload gate (`src/uploads/`)** — the ADR-0011 pipeline for - session-scoped file uploads, wired into the UI by `run_upload_gate()` in - `gradio_ui.py`. Three **mandatory** gates (de-identification attestation → - `stage_upload`: type-allowlist / size-cap / quarantine dir / SHA-256 → - `scan_upload`: magic-byte structural check, plus an AV pass only where the - deployment declares one) plus an **advisory** `validate_upload`. What a given - deployment gets is declared in `deploy/scan_posture.yaml` (read by - `src/uploads/posture.py`), not inferred from the host PATH: the HF Spaces run - `structural_only` and are therefore **not** malware-scanned, while - `av_required` fails closed and must only be set where a working AV exists. - Files are - only ever opened with vetted loaders (`scanpy.read_h5ad`, `pandas.read_csv`) — - never pickle/eval/exec — and quarantined uploads never enter a served path. -- **MCP tool dispatch (`src/managers/tools/mcp_manager.py`)** — tools are - registered over HTTP (resident server) or stdio; `_parse_mcp_content` is the - untrusted-output boundary (raises on `isError` rather than passing error text - downstream). `_resolve_env_vars` does read-only literal `${VAR}` substitution - (no shell). The tool surface itself is mounted in `server.py`. -- **Secrets + redaction** — `ANTHROPIC_API_KEY` / `HF_TOKEN` / - `decouplerpy_results_token` are read once at lazy init and never logged; AWS - creds come from the instance role, not env. `src/core/trace_redaction.py` - scrubs secret-shaped strings (and whole env dumps) from every trace before it - reaches any sink (file / HF / S3). -- **Access control (`src/core/access_control.py`)** — `ADMIN_IDS` / `ALLOWED_IDS` - gate admin-only actions (e.g. registering an upload into the registry); - fail-closed on unknown identity; the resolved principal is recorded in the - audit trace (ADR-0012). diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000000000000000000000000000000000000..d46a6bf0ce9e72590d4bdb26a99026df79f9f7de --- /dev/null +++ b/TODO.md @@ -0,0 +1,486 @@ +# DecoupleRpy_Agent — TODO + +Canonical open-items list for this repo. Session history + detailed write-ups +stay in `memory.md`; this file is the scannable backlog. Convention: each item +is `High` / `Med` / `Low`. Move finished items to `## Done` with a date. + +Routing rule for this system: manifest/dataset-semantics → biodata-registry · +computation/tools/loaders → here · routing/coordinator/eval → +pdac-analysis-orchestrator. + +--- + +## Open + +### Deploy / infra + +- **Done (2026-06-29) — Salvage branch `claude/magical-banach-a2d929` ported + dropped.** The Jun-22 + `wip(salvage)` commit (`8efd1be`) was reviewed file-by-file: **all 10 files already superseded** on + current `main` + biodata-registry (writable-dir `_resolve_dir`, hf_storage structured save, + `prohibited_inferences`, Moffitt prompt examples + 89/36 golden test, txt/docx/pdf export, stroma + contrasts). Nothing to port — porting would only re-add stale code on deleted paths. Branch deleted + (local + `origin` + defunct worktree); `8efd1be` recoverable via reflog. Detail in memory.md 2026-06-29. + +- **Done (2026-07-01) — Always-on audit log sink (ADR-0008) MERGED + deployed to prod.** `src/logging_sink.py` + (configurable `LOG_SINK=local|hf|s3`, default `local`) + always-on `persist_trace_safe` wiring in + `agent.py`/`gradio_ui.py` merged to `main` and pushed to the prod Space. Prod posture: `LOG_SINK=hf`. + +- **Med — Finish the S3 log-sink writer (AWS migration, ADR-0009).** `src/logging_sink.py` ships a + configurable trace sink (`LOG_SINK=local|hf|s3`, default `local`). ADR-0009 Phase 1 implemented the + lazy-boto3 `S3LogSink.persist_trace` (`put_object` to `${LOG_SINK_S3_PREFIX}//trace.json`); + `scripts/verify_s3_sink.py` is the live-verify script. **Remaining for the OHSU-managed AWS cutover:** + provision the bucket/IAM per ADR-0009 Appendix A, wire `LOG_SINK_S3_BUCKET`/`_REGION`/`_PREFIX` + + role creds as Space config, add `boto3` to requirements (optional/lazy), then set `LOG_SINK=s3`. + +- **Med — Quantify the MCP HTTP perf win (DEPLOYED to dev + prod `58b961f`, 2026-06-29).** The + resident `server.py --transport http` is live on both Spaces — tool calls reuse one process instead + of spawning a subprocess per call (was the ~44s/step tax); confirmed via run logs (`Added 52 remote + MCP tools`, no Pre-warm failed). Adds a process-lifetime in-memory adata cache + `[perf]` + instrumentation; stdio kept as fallback. **Remaining:** run a real multi-step DE question (needs API + credits) to capture the actual wall-clock % cut (expected 30–60% on a ~29-min run) from the `[perf]` + lines now in the logs. +- **High — Bring PROD specialist live on gradio 6.18.** Migration is **pushed to + prod** (`origin/main` = `f098e67`, 2026-06-22); dev is RUNNING on the same code. + Prod Space is still PAUSED on the cpu-basic quota (3/3: dev specialist + dev + orchestrator + `bcc-lit-agent`). **Remaining: free one slot (e.g. pause + `bcc-lit-agent`) + unpause/restart the prod Space.** (User opted not to pause + anything yet — do this when ready.) If the prod build hits the same HF rollout + wedge dev did, a pause→unpause clears it. +- **High — Top up the dev Space `ANTHROPIC_API_KEY` credits.** "Credit balance too low" + blocks the agent from finishing any multi-step run on dev — the concordance end-to-end + output couldn't be captured. Top up, then re-run a sibling-variant concordance query + for the final numbers. +- **High — Promote the loader-auth + concordance stack to PROD.** On `main`/dev only + (`ce5168d`, `04b26ba`; the 0.1.6 concordance gate). Prod promotion = push `origin` → + **factory-rebuild prod** (a *normal* rebuild kept a stale pre-0.1.6 install on dev — a + factory rebuild was required) → set the prod `HF_TOKEN` secret (read + `pdac-research-data`) → ensure prod's Anthropic key has credits → re-validate. **2026-06-26 + update: mostly DONE** — loader-auth live on prod (`5c5ca74`/`ce889e5`), prod `HF_TOKEN` CONFIRMED + set (user), Anthropic credits topped, XDI suite re-validated **4/4 end-to-end on prod** + (`20260626_131117`). Remaining: verify the 0.1.6 concordance factory-rebuild actually took on prod. +- **Done (2026-06-26) — Metadata tools loader-auth gap fixed → dev** (`ce889e5`). + `dataset_count_metadata_values` / `dataset_crosstab_metadata_values` / + `dataset_validate_manifest_against_data` were MISSED in the ce5168d/04b26ba + centralization — they did `Path(adata_path).exists()` on the raw arg, so a private + h5ad URL returned "File not found" and never authenticated. Surfaced live: the XDI-002 + late eval stalled here (agent fell back to unauthenticated urllib that hung on the + private repo). All three now route through `resolve_to_local_path`. Verified locally + (loads private paca_au_rnaseq.h5ad, real Bailey counts). **Dev only**; fold into the + prod loader-auth promotion below. **Follow-up: reconcile stale early-branch prompt text** + ("no batch-aware activity-scoring tool") — `81f5db6` added `decoupler_pool_cohorts`. +- **Done (2026-06-24) — Private-data 401 fixed; loader auth centralized.** One + authenticated resolver `src/core/data_io.resolve_to_local_path`; `decoupler_differential_expression` + + Mode A `_resolve_to_local` + `bulk_dataset_tools._resolve_to_local_path` delegate + (`ce5168d` → `04b26ba`). Validated on dev (no 401/TaskGroup; resolver downloads the real + 85.4 MB private h5ad). **`HF_TOKEN` Space secret CONFIRMED set on prod (user, 2026-06-26)** — + prod loads private data end-to-end (XDI suite 4/4, `20260626_131117`). + +### Cross-dataset integration (ADR-0001) + +Combine ≥2 datasets — early (pool + batch covariate) / late (meta-analysis) / +refuse, driven by registry metadata. ADR + plan + per-task spin-off prompts in +`docs/adr/ADR-0001-*.md`. All work on branch `adr-0001-phase-0` (off `main`). + +- **Done (2026-06-25, later) — Step-count efficiency prompt edits → dev** (`cdfee6f`, + `prompts.yaml`; no code). Cuts wasted generate→execute round-trips in long / + cross-dataset runs (diagnosis: ~90% of ~35-40 min is the serialized step loop). + (1) Reporting-Results rule 2 ("Re-read before you report") → a SINGLE read of an + ALREADY-SAVED CSV; forbids recompute/re-derive/re-save and a repeated re-read + (killed XDI-001's ~3 redundant end steps). (2) Cross-dataset step 1 → pass + design_factor ONLY for a named two-group contrast; pooled/per-sample "across all + samples" with no contrast calls `dataset_get_integration_plan([...])` with NO + design_factor (stops XDI-001's refuse-then-retry). With-contrast refusal guidance + unchanged. YAML verified; **dev + PROD** (`hf-dev` + `origin`; on prod 2026-06-26 via `81f5db6`). + **VALIDATED on PROD 2026-06-26** (XDI suite re-run `20260626_131117`, 4/4 PASS): XDI-001 early + pooled batch-aware DE (249 samples), no redundant recompute/re-read end-steps observed. Clean + step-count A/B not possible (eval bank revised since the pre-fix run — XDI-001 is now Moffitt+Puleo). +- **Done (2026-06-25) — Late-integration prompt hardened → dev + PROD** (`0776267`, + `prompts.yaml`). Late requests must run the shared contrast per cohort then combine with + `decoupler_meta_analyze` and ALWAYS report Cochran's Q / I^2 — no hand-rolled + sign-concordance/intersection (fixes the 2026-06-25 eval miss). Early branch refreshed + to route contrasts to `decoupler_integrate_datasets`; clarifies no batch-aware + activity-scoring tool (flag pooled activity as batch-confounded). YAML+Jinja verified. + Deployed dev (`hf-dev`, `0776267`) then **PROD** (`origin`, `791bb29`→`211cc90`, clean + ff; only runtime delta is `prompts.yaml`, no re-pin so no factory-rebuild needed). + **VALIDATED 2026-06-26 (XDI-002 re-run, dev `f01f56b`, PASS):** agent calls + `decoupler_meta_analyze` (Stouffer, 91 TFs) and reports per-feature Cochran's Q / I^2, + flagging CDX2 (I^2=72.9%) + HOXD3 (51.3%) as heterogeneous — no hand-rolled concordance. + Needed the `ce889e5` metadata loader-auth fix to get there. **Promoted to PROD 2026-06-26** + (`origin` `5c5ca74`): metadata loader fix `ce889e5` + prompt point-3 reconcile `5c5ca74` + (pooled scoring via `decoupler_pool_cohorts` = ComBat-corrected, not confounded). Late + meta-analyze prompt fix + loader fix now both live on prod. **Re-validated end-to-end on PROD + 2026-06-26** (XDI-002 in suite `20260626_131117`, PASS): real `decoupler_meta_analyze` (Stouffer) + + per-TF Q/I² computed on prod, no hand-rolled concordance. +- **Done — Same-cohort concordance routine** (committed `b36d5ce`; on `main` via + merge `0ac610f`; deployed to **dev** `hf-dev`). `decoupler_normalization_concordance` + (`integration_mcp`) + `src/workflows/concordance.py` — the agent-side routine for + the registry's `mode="concordance"` (sibling variants). Descriptive agreement + (Pearson/Spearman, sign-concordance, effect spread, significant-call Jaccard); no + combine. Tests: `test_concordance.py` 9 pass; `test_concordance_tool.py` smoke-validated. + **Now live in prod:** biodata-registry 0.1.6 wheel + re-pin (`fb2091e`) **deployed** + (`origin/main` = `fb2091e`); the plan returns `concordance` live. Dev confirmed RUNNING + on 0.1.6; prod rebuilt on the push. +- **Done — Phase 0.** `src/core/combine.py` (envelope contract + strategy + registry) + decision-matrix spec, committed (`6fa0b12`, `3fd9d0a`). +- **Done — T1: `combine` descriptors on the 4 rna tools** (`src/tools/rna/analysis.py`), + committed (`42d31b1`). All four tools declare a Mode-B `combine` descriptor + + matching `@combinable` marker; `tests/test_combine_conformance.py` added (24 tests). +- **Done — T3: `src/workflows/meta_analysis.py`** (Phase 1, step 4), committed + (`bb944d5`). Four strategies register into `combine.py` (inverse_variance > + stouffer > fisher; rank_aggregation opt-in), `compare_activity_by_group`→envelope + adapter with the D4 SE-of-Cohen's-d backfill, Cochran's Q / I², and + `combine_envelopes()` (align → field-dispatch → BH). `tests/test_meta_analysis.py` + (34 tests). Also completed the `src/core/__init__` re-export surface (additive). + Engine references no tool by name. **Branch-local; not deployed.** +- **Done — T2: registry `get_integration_plan`** (biodata-registry) — decision + engine + 5th MCP tool. Merged + released as **biodata-registry 0.1.2** + (2026-06-19). Pin ready for T4: + `.../resolve/cbc083a5cd9dbe79e6740a6b64c4dc8c0639f113/biodata_registry-0.1.2-py3-none-any.whl` + (sha256 `607a14b0…`). +- **Done — T4: agent `dataset_get_integration_plan` wrapper + 0.1.2 re-pin** + (commit `a85e426`, branch-local). Thin `get_integration_plan` passthrough in + `src/datasets/registry.py` + `@dataset_mcp.tool dataset_get_integration_plan` + in `catalog.py` (early/late/refuse; `{error}` on unknown ids; forwards + contrast args to the confound gate). `requirements.txt`/`.in` → 0.1.2 + (`cbc083a`). Server 50→51 tools. `tests/test_integration_plan_tool.py` (7). +- **Done — T5: `decoupler_meta_analyze` tool + A→B→refuse wiring** (commit + `c45a617`, branch-local). New `integration_mcp` sub-server + + `decoupler_meta_analyze` (envelope read → `combine_envelopes` → combined CSV + + Q/I²; refuses mismatched result_type/contrast). prompts.yaml wires + early→late-fallback / late / refuse. Server 51→52 tools. + `tests/test_meta_analyze_tool.py` (9). **Mode B (Phase 1) complete end to end.** +- **Done — T6: release + dev deploy + e2e validation** (2026-06-19). Shipped Mode B + to **prod** (`origin` @ `c54b550`, 50→52 tools). Dev validation found Case 3 + returned plan `early` not `refuse`; hardened the registry confound gate + (**biodata-registry 0.1.3**, `5654e86`: a cohort that can supply neither arm of a + specified contrast → CONFOUNDED_DESIGN) + a prompt nudge to pass contrast args. + 4-case gradio_client eval on dev: refuse ✅ / early→late-fallback no-fabrication ✅ + / late no-pooling ✅ / `decoupler_meta_analyze` positive path ✅ (paca_au_rnaseq + + paca_au_array). Q/I² reporting polish (`c54b550`). registry re-pinned 0.1.3. +- **Done — T7: `src/workflows/integration.py`** combined-AnnData builder (gene-symbol + intersect + `batch` obs key = dataset_id); pure `combine_anndatas` + loader + `build_combined_anndata`. Branch `adr-0001-phase-2-mode-a` (`a7a8caa`); **deployed + to dev + prod 2026-06-24** (`5cb2737`, code-only rebuild). +- **Done — T8: `batch_column` on the DE tool** — DESeq2 `~batch + factor`; new + `run_limma_covariate` (`~batch + group`) in microarray.py; ttest+batch refused (no + silent covariate drop). Back-compat default off. (`a7a8caa`.) +- **Done — T9: `decoupler_integrate_datasets`** (3rd integration_mcp tool, server + 52→53) — pools + one batch-aware DE only on plan mode=="early"; refuses/reroutes for + late/concordance/refuse; auto-picks deseq2 (raw counts) / limma. (`a7a8caa`.) v1 + limits: gene-symbol axis only (no probe collapse/ortholog). +- **Done — item 10: `decoupler_pool_cohorts`** (4th integration_mcp tool, server + 54→55) — ComBat for the per-sample **scoring** path (the no-design-matrix counterpart + to T9's covariate route). Pools + `batch_correct_for_scoring` (`scanpy.pp.combat` on + the log-normalised matrix, keyed on `poolable_data_level`; standard ComBat, NOT + ComBat-seq → **no new dependency**) → one batch-corrected matrix for + `dataset_score_bulk_samples`. Same early-only plan gate. `src/workflows/integration.py` + + `tests/test_pool_cohorts_tool.py` (13 tests). Prompt early-branch rewired to call it + (was "no batch-aware scoring tool / flag confounded"). 2026-06-25 dev; **promoted to PROD 2026-06-26** (`origin` `81f5db6`). +- **Low — T10: orchestrator routing + capability + reporting rules** (pdac-analysis-orchestrator). +- **Low — T11: cross-dataset evals** (pool/fallback/refuse; needs T5/T9) — now unblocked (T9 done). + +### ADR housekeeping (reconciled 2026-06-24) + +ADR-0003/0004/0005 moved **Proposed → Accepted**; ADR-0001 checkboxes ticked to shipped +reality (Mode B / concordance). Remaining open action items: + +- **Low — ADR-0003:** add a "Dev Mode inner loop" note to + `pdac-analysis-orchestrator/DEPLOYMENT.md` (sibling repo); set up VS Code/SSH config. + *(Dev Mode itself is enabled on both dev Spaces.)* +- **Low — ADR-0005:** add a "promote to public on publication" step to the + dataset-onboarding checklist; record the storage posture in `biodata-registry/memory.md`. + *(`pdac-research-data` is already private.)* + +### Result-aware sanity layer (ADR-0002) + +Post-compute Layer-2 cautions (never refuse/block) catching what the metadata-only +refusal engine can't: contamination, artefactual log2FC, non-TF "TFs". ADR + +spin-off prompts in `docs/adr/ADR-0002-*.md`. `sanity_warnings` is the shared +additive return contract S2/S3 extend. + +- **Done — Phase 1: `src/workflows/sanity_checks.py` + tests + DE/TF wiring** + (branch `feat/adr-0002-sanity-layer`, not pushed). 3 checks + `run_sanity_checks()` aggregator; + `tests/test_sanity_checks.py` (16); additive guarded `sanity_warnings` key on + `decoupler_differential_expression` (effect-size) + + `decoupler_tf_enrichment_collectri` (membership). +- **Done — S1: CI verification of the additive key + memory sync** (this session). + Additive key breaks nothing (`tests/` 877 passed / 55 skipped); added DE + TF + `sanity_warnings` shape assertions to `tests/test_tool_response_schemas.py` + (9→11). Closes the ADR's two schema/registry caveats. **Committed on + `feat/adr-0002-sanity-layer`, not pushed.** +- **Done — S2: Phase 2 — thread cohort tissues into the enrichment tools + auto-fire + the contamination check** (2026-06-22; committed on `feat/adr-0002-sanity-layer`, + not pushed). Optional `cohort_tissues` / `home_tissue` params on all three + enrichment tools (default `None` → check skipped, back-compat). + `check_tissue_identity_contamination` runs on the ranked output and is **merged + into the same `run_sanity_checks` report** as S3's membership warnings (CollecTRI + ranks TFs by |activity|, `feature_kind="tf"`; PROGENy/Hallmark run on the input DE + genes by |stat|, `feature_kind="gene"`, since pathway/gene-set names aren't in the + marker registry). Added a `prompts.yaml` rule to foreground `critical` warnings. + New tests `tests/test_enrichment_contamination_wiring.py` (9). `test_tool_registry.py` + needed no change (tests the ToolRegistry abstraction, not real tool signatures). +- **Done — S3: parity — `sanity_warnings` (membership check) on PROGENy + Hallmark** + (2026-06-22; committed on `feat/adr-0002-sanity-layer` together with S2 as the + ADR-0002 sanity layer). Validates requested pathway/gene-set names against the + resource's source names (`known_non_tf=set()`); `run_sanity_checks` gained a + `known_non_tf` passthrough; parity tests in `tests/test_tool_response_schemas.py` + (`TestEnrichmentSanityParity`). The TF tool keeps membership **and** contamination + warnings merged (reconciled with S2, not overwritten). + +### PROGENy per-sample scoring (from GSE205154_sears eval, 2026-06-22) + +Found while reviewing an agent run for "score PROGENy for every sample in +gse205154_sears and show the cohort-wide landscape." Analysis/routing were +correct (single-dataset, Path B, ULM, 98.3% coverage); these are the gaps. + +- **High — Per-sample scoring path returns no retrievable figure.** + `dataset_score_bulk_samples` writes only the two CSVs to `OUTPUT_DIR` + (`tmp/outputs/`). Any "landscape" plot is agent-authored matplotlib that lands + wherever the agent chooses — observed at `/tmp/..._landscape.png`, *outside* + `OUTPUT_DIR` — so the end-of-run `tmp/outputs` inline-plot sweep never embeds + it and "show me the landscape" silently returns no image. The run's only link + was mislabeled "Saved run log" but pointed at the raw `.h5ad`. Fix: either (a) + have the tool emit a standard landscape figure (sample heatmap + mean±SD bar) + to `OUTPUT_DIR` as a declared artifact, or (b) add a prompts.yaml rule that + agent-authored plots must be written to `OUTPUT_DIR`. +- **High — Confirm/enforce log-scale before ULM on the per-sample path.** + PROGENy/ULM assume roughly symmetric, log-scale input (tool docstring says + "log2-TPM"). The run loaded "TPM" and never stated a log2 transform was + applied; if *linear* TPM reaches ULM, a few high-expression genes dominate and + scores distort. Fix: add an input-scale heuristic in + `score_bulk_samples_with_decoupler` (warn or auto-log when values look linear: + large max / right-skew / non-log range) and/or assert the manifest `data_level` + is a log level; surface the applied transform in the return dict either way. +- **Med — `DECOUPLER_DISCLAIMER` misdescribes the per-sample path.** + `src/core/constants.py` says values are "derived from differential-expression + statistics via the ULM model" — false for `dataset_score_bulk_samples`, which + scores the expression matrix directly (its own docstring: scores "without first + computing DE statistics"). Split the disclaimer into per-sample vs DE-based + wording (or parameterize) so the scoring tool emits the correct caveat. +- **Med — Pin and record the PROGENy footprint.** + `dc.op.progeny()` is called with defaults; the run reported + `n_network_genes=17,610` (full model), and decoupler's default `top` has + shifted across versions — so scores aren't reproducible across upgrades. Pin + `top` explicitly in `score_bulk_samples_with_decoupler` and echo it in the + return dict. +- **Low — `-F` (fibroblast) samples can't be excluded.** The 7 `-F` samples are + annotated `Primary` in GEO; per-sample scoring includes them and can inflate + stromal pathways (TGFb/NFkB). Consider a documented sample-filter hook + (agent-side) or a manifest note so they can be optionally excluded/segmented. + (Manifest-note half coordinates with biodata-registry.) + +### UI / output (Gradio) + +- **Low — Port the UI fixes to pdac-analysis-orchestrator.** The download + truncation, inline-plot, and full-run-PDF fixes landed here 2026-06-17 (see + Done). The same class of bug likely exists in the orchestrator's UI (untested + as of 2026-06-17) — reuse the same approach there. + +### GUI / observability + +- **Med — Tool/dataset call tracking panel** (was Known Gaps #9). Collapsible + "what happened" panel beneath each response showing which MCP tools ran and + which datasets were accessed. LangGraph already returns intermediate steps; + capture tool names + dataset IDs from step metadata and render an expandable + panel. Helps non-coding scientists see what the agent did. + +### Data / manifests + +- **Med — `roadmap` key in manifests** (was Known Gaps #10). Add a per-manifest + `roadmap` list in biodata-registry + a `scripts/collect_roadmap.py` that + prints a consolidated cross-dataset list of open items. (Coordinate with + biodata-registry/TODO.md.) +- **Med — Daily URL health check** (was Known Gaps #11). Scheduled script that + pings each manifest's `expression_source` / `metadata_source` URLs, logs HTTP + status to `url_health.json`, and a startup banner in `gradio_ui.py` warns on + any non-2xx. Motivated by the TCGA-PAAD clinical URL 403. + +### Tool development & validation (from PI research to-do list, 2026-06-26) + +- **Med — Add the PURIST subtype operation to the toolset.** Wire PURIST + (single-sample basal/classical PDAC classifier) as an agent tool, then support + the basal-vs-classical comparison that contrasts **PURIST vs single-cell** + methodology. (Pairs with the Loveless single-cell ingestion in + biodata-registry/TODO.md.) +- **Med — Hallmark Shiny app (Carl Pelz).** Build the Shiny / Hallmark-genes + tool. Review/scoping meeting with Carl Pelz is scheduled — capture his + requirements (which Hallmark gene sets, inputs, expected outputs) before building. +- **Med — Single-cell RNA-seq tool: cell annotation + UMAP + thresholds.** + Build on `rna_sc.py` to expose the cell-annotation feature, render a UMAP, and + let the user specify thresholds. Carry the existing caveats: analysis works only + within cluster groups, and pseudo-bulk single-cell is untrustworthy for certain + cell types. +- **High — Loveless single-cell serving & runtime design (agent side).** + *Started 2026-06-29 — design accepted as [`ADR-0006`](docs/adr/ADR-0006-loveless-single-cell-serving.md); + agent-side scaffold landed ahead of the biodata-registry ingestion.* + - **Done (scaffold, synthetic-tested):** custom-signature bulk fast path + `dataset_score_signature` (Role 2) + `src/workflows/signatures.py` + + `score_bulk_samples_with_decoupler` network seam (custom resource label no + longer gated when a signature net is supplied); `src/tools/rna_sc.py` now + loads via `read_h5ad_cached` (Role 1 cache contract). **ADR-0006 #6 done:** + `h5ad` `expression_source.type` + Path **P** (`analysis_path` derived + modality-first; sc/spatial → P, never mislabeled Path A) wired into + `_build_loading_plan` (single-cell plan, no bulk DESeq2/limma tail) + + `manifest_schema` (matches biodata-registry 0.1.7 A/B/P). Tests: + `tests/test_signatures.py`, `tests/test_loading_plan_h5ad.py` (all green; + `test_activity_scoring.py` updated for the relaxed resource gate). + - **Also done:** sc loaders (`rna_sc.py::_load_adata`) resolve a hosted/private + h5ad URL→local path via the shared authenticated resolver + (`resolve_to_local_path`, `HF_TOKEN`) — a `pdac-research-data` h5ad now loads + end-to-end (`tests/test_rna_sc_loader.py`). + - **Artifacts LANDED (biodata-registry 0.1.8, 2026-07-01) + integrated:** two + Loveless-atlas sc subsets — `gse155698_steele` (GSE155698) + `gse205013_werba` + (GSE205013), `modality: sc_rnaseq`/`raw_counts`, `expression_source.type: url` + → hosted `.h5ad`) + the `CROSS_RESOLUTION` gate. Agent re-pinned 0.1.8 on this + branch; both route `analysis_path=P` and produce a correct sc loading plan + (load via `decoupler_load_and_visualize_data`, NOT the bulk url loader). + Fixed `_build_loading_plan` to key Path P on **modality** (their type is `url`, + not `h5ad`); regression test in `tests/test_loading_plan_h5ad.py`. + - **Role-2 signature artifact PUBLISHED (2026-07-01):** 14 per-cell-type marker + signatures derived from the Steele subset (`rank_genes_groups` on `Clusters`) + → `loveless/signatures/gse155698_steele_celltype_signatures.csv` on + `pdac-research-data`. Script: `biodata-registry/scripts/ingest/loveless/derive_signatures.py`. + `dataset_score_signature` scores it end-to-end (66.6% coverage on a synthetic + cohort); `load_signature_net` + `resolve_to_local_path` now fetch a private + signature URL with auth (env token OR cached `huggingface-cli login`). + - **Done + DEPLOYED to PROD (2026-07-01) — `decoupler_load_and_visualize_data` + hardened for subsets lacking a precomputed UMAP/leiden** (was spun off → + `task_8b2a1bdc`). Merged via HF PR #1 into prod `main` (`6cda24b`→`abc16da`) + + **prod factory-rebooted** (folds in the pending 0.1.8 re-pin rebuild). Remaining: + confirm prod RUNNING post-reboot + e2e-verify the two Loveless sc datasets load + + produce a UMAP on prod. `_ensure_umap` reuses an existing embedding (pbmc3k unchanged) or + computes a bounded `normalize+log1p→pca→neighbors→leiden(igraph)→umap` pipeline + on the loaded copy, and NEVER hard-fails — on any failure the plot is skipped + and the loaded AnnData + metadata are still returned with a note. Grouping + detection covers R `make.names` atlas cols (`Clusters`, …), not just `leiden`; + leiden uses `flavor="igraph"` (no `leidenalg` dep on the Space). Sits on top of + the `_load_adata` seam, so the Loveless subsets now load AND visualize + end-to-end. Tests: `tests/test_rna_sc_load.py` (4 green). + - **Still open:** PROD factory rebuild on the 0.1.8 re-pin was TRIGGERED 2026-07-01 + (with the UMAP-hardening merge) — confirm it settled to RUNNING and the 0.1.8 + wheel actually took (both sc datasets list); `hf-dev` factory rebuild + e2e verify + still not done; confirm prod RAM holds the subset live; pseudobulk-aggregation tool + for the sample-level DE contrast; derive Werba signatures when needed. ADR-0006 + items 7/8/9. + Companion to the biodata-registry Loveless ingestion (provenance/scope/gate plan + there). Design goal: keep user-facing runtime bulk-like. + - Two roles. The **Steele-subset h5ad** is an analyzable sc dataset — loads once + via the persistent MCP server + `read_h5ad_cached`, then pseudobulk → DE / + activity scoring; first load slower than a bulk series matrix, cached after, + then normal. The **full integrated atlas** is NOT live-computed — heavy sc + work (annotate / QC / derive signatures) is precomputed offline at ingestion, + and query-time ops score *bulk* cohorts against the derived Loveless + signatures / deconvolution reference on the existing fast path. + - Cache caveat: `read_h5ad_cached` returns **copies** (safe under the shared + server) — fine at MB scale, a RAM multiplier at GB scale. A large atlas on the + live path would need views/no-copy, or (preferred) stay off the live path via + the offline-signature approach. + - Space sizing: confirm the prod Space RAM tier holds whatever sc artifact is + served live — the subset should fit; the raw atlas likely won't. + - Expected user flows: "score the Loveless basal / CXCL10+ CAF signature in + TCGA-PAAD" (bulk fast path), "pseudobulk DE tumor vs normal in the Loveless + subset" (sc load + pseudobulk), "annotate cell types in Loveless" (heaviest — + prefer precomputed). Pairs with the `CROSS_RESOLUTION` gate + the PURIST + vs single-cell methodology comparison above. +- **Med — Batch-adjustment validation workflow (P53 quality check).** End-to-end + check on the batch-adjustment path (mouse → expression change → sequencing): + confirm the changes hit pathways known to associate with **P53** (or other known + processes), and quality-check that the **same pathways appear before vs. after** + batch adjustment. Tooling exists (Mode A covariate DE + `decoupler_pool_cohorts` + ComBat scoring); this is the validation/QC analysis on top. +- **Note — decoupleR "explain each step" + preprocessing = DONE.** The plain- + language Approach section + decoupleR glossary in every solution (`d8abb63`) and + the per-dataset `preprocessing` field (biodata 0.1.5, surfaced in prompts) cover + the PI's "explain how decoupleR works / add preprocessing info" items. + +### Blocked / long-term + +- **Blocked — COMPASS / Chan-Seng-Yue 2020** (was Known Gaps #13). Controlled + access (EGA EGAS00001002543); needs a signed DAA + DAC approval. Not pursuable + for an open-access deployment. Would be Path A (RNA-seq raw counts → DESeq2) + if access is ever obtained. +- **Low / long-term — Additional specialist agents beyond decoupler** (was + Known Gaps #12). Architecture already supports it; add entries to `agents.yaml` + + new Space deployments when ready. + +### Minor / residual + +- **Low — GPL annotation fallback residuals** (was Known Gaps #15). `GPL_GENE_SYMBOL_CANDIDATES` + is a fixed 9-variant list (other naming needs explicit `gene_symbol_column`); + in-memory cache hits cosmetically report a possibly-different `sym_col_used` + (cached mapping itself is correct). + +--- + +## Done (recent) + +- 2026-07-02 — **ADR-0011 upload safety gate ("Now" slice)** — branch + `feat/adr-0011-upload-gate` (off `origin/main`), **NOT pushed, no deploy**. New non-agent-facing + `src/uploads/` (`stage_upload` → `validate_upload` → `register_upload`) + shared + `src/core/integrity.py` (`compute_sha256`/`verify_sha256`, reused later by ADR-0010). Quarantine + + type/size allow-list + manifest-required + de-id attestation + validate-via-vetted-loader + (never-exec) + SHA-256 provenance into the ADR-0008 audit sink + admin-only registration + (`UPLOAD_ADMIN_IDS`, fail-closed). `tests/test_upload_gate.py` (24) green; reused suites (133) + green. ADR-0011 flipped Proposed→Accepted for this slice. **Follow-ups (open):** tabular + auto-validation (h5ad-only today); AWS staging bucket + pre-validation malware scan (ADR-0011 + "At AWS"); wire `src/core/integrity.py` into `resolve_to_local_path` for ADR-0010 on-load verify. +- 2026-06-22 — **TF semantic-annotation coverage + igraph network plot** + (commit `ed98c5b` on `main`, **NOT pushed**). Fixed 70 valid HGNC TF symbols + being mis-bucketed as `unresolved_label`: bundled `resources/semantic/hgnc_cache.tsv` + (1,183 CollecTRI source TFs) loaded by `normalize_gene_symbol()`, + regen + script. Restored `igraph` to `requirements.in`/`.txt` so `dc.pl.network` (TF + network plot) stops silently failing. Semantic suite 157 passed. + **Deploy when ready** (dev `hf-dev` first, then prod). +- 2026-06-22 — **Item 3: per-dataset `preprocessing` field surfaced + re-pin + 0.1.5 → merged to `main` → DEV** (`982f65a`; `hf-dev` `64a443c..982f65a`, + rebuilding; prod NOT pushed). Re-pinned biodata-registry 0.1.4 → 0.1.5 + (`b46392c`, sha256 `958f498b…`, folds in the pending Sears 0.1.4 bump); added + `preprocessing` to `list_available_datasets()` + both agent.py dataset-dict + builders; prompts.yaml renders a Preprocessing bullet + an Approach rule + (item 4). 99 tests pass. Registry side released as 0.1.5 (schema field + all + 19 manifests populated). +- 2026-06-22 — **Hide microarray from advertising + plain-language Approach in + solutions** (commit `d8abb63`, dev `hf-dev`). Item 2 agent-side: + `HIDDEN_MODALITIES`/`is_advertised()`, `advertised` field, MCP tool returns + advertised-only + `unadvertised_dataset_ids`, prompts flag microarray NOT + ADVERTISED + heuristics steer to RNA-seq. Item 1: required Approach section + + decoupleR glossary in every ``, no missteps. 99 dataset tests green. +- 2026-06-22 — **REALLY fixed HF Space stuck on "Starting": gradio 5.49 → 6.18 + migration** (commits `b4e7ac0` + `5ba3350`, branch `fix/gradio-6-migration` → + merged to `main`). The 2026-06-21 SSR fix was a red herring (the working + orchestrator runs 6.18 *with SSR on*); real cause is gradio 5.49 no longer + completing HF's readiness handshake. Cascade: `mcp` 1.10.1→1.28.0, + `langchain-mcp-adapters` 0.2.2→0.3.0, `fastmcp` pinned `==3.2.3`; `gradio_ui.py` + ported to the 6.0 API; `ssr_mode=False` removed. Verified: 851 tests, 49 tools, + HTTP 200. **DEV RUNNING** on `5ba3350` (a pause→unpause cleared an HF-side + rollout wedge). Prod migration still pending (see Open → Deploy/infra). +- 2026-06-21 — [SUPERSEDED by the gradio-6 migration above] Attempted SSR fix for + the "Starting" hang: `ssr_mode=False` default in `GradioAgentUI.launch()` + (`d38496a`). Did not actually fix it; removed during the migration. +- 2026-06-17 — UI download buttons fixed: exports no longer truncate the + solution (the old path dropped everything after the first blank line); TXT + downloads again; added a **Full log (.pdf)** export of the full generated + logic. All four files are generated at run completion and armed as one-click + downloads (`_assessment_blocks` / `_log_blocks` / `_write_*` in + `ui_formatting.py`; `_arm_downloads` in `gradio_ui.py`). +- 2026-06-17 — Plots render inline: end-of-run sweep of `tmp/outputs` embeds + figures — including direct `dc.pl`/matplotlib plots that emit no artifact + JSON — deduped by basename against mid-run artifacts. +- 2026-06-17 — Full-run PDF (`full_run.pdf`) saved to the results repo + alongside `conversation.md` / `metadata.json`, with a direct link in the + saved-run notice (`HFResultsStorage.upload_run_file`). +- 2026-06-17 — Fixed spurious "Step limit reached" notice after every run: the + UI streams via `graph.stream()` so `trace_logs` was always empty; completion + is now keyed off the real solution-shown signal (`last_solution_shown`). +- 2026-06-17 — Rotated `research_agent_token` (was Known Gaps #5 / ACTION REQUIRED). +- 2026-06-14 — Bailey 2016 (`paca_au_rnaseq`/`paca_au_array`) + Puleo 2018 + (`puleo_2018`) stress-tested against real hosted data; found + fixed the + `subset_query` `.query()` syntax bug and a `Sample.type` allowed_values gap. +- 2026-06-14 — TCGA-PAAD sample curation re-verified end-to-end (Known Gaps #7). +- 2026-06-12/14 — Live GPL-annotation fallback Tier 1 + Tier 2; 4 residual + limitations fixed (Known Gaps #14, #15). +- 2026-06-06 — GDC STAR-Counts loader + tcga_paad pivot to Path A. +- 2026-06-05 — biodata-registry wired in; manifest rules in system prompt; GEO + datasets loaded/validated (Known Gaps #0–4). + +_Full detail for any item: see `memory.md`._ diff --git a/app.py b/app.py index abee00c5a94dbafeaf429e15a3d7e5e96cafacab..250c9c40172b30c7e9a264f971fca10e5c7d6983 100644 --- a/app.py +++ b/app.py @@ -16,28 +16,16 @@ if os.environ.get("APT_PROBE"): import subprocess def _apt_probe(): - pkgs = [ - "r-base", - "r-base-dev", - "r-bioc-limma", - "libcurl4-openssl-dev", - "libssl-dev", - "libxml2-dev", - "libreadline-dev", - ] + pkgs = ["r-base", "r-base-dev", "r-bioc-limma", "libcurl4-openssl-dev", + "libssl-dev", "libxml2-dev", "libreadline-dev"] print("===APT_PROBE_BEGIN===", flush=True) for p in pkgs: try: - inst = ( - subprocess.run( - ["dpkg-query", "-W", "-f=${Version}", p], capture_output=True, text=True - ).stdout.strip() - or "MISSING" - ) - pol = subprocess.run( - ["apt-cache", "policy", p], capture_output=True, text=True - ).stdout - cand = next((ln.split()[1] for ln in pol.splitlines() if "Candidate:" in ln), "?") + inst = subprocess.run(["dpkg-query", "-W", "-f=${Version}", p], + capture_output=True, text=True).stdout.strip() or "MISSING" + pol = subprocess.run(["apt-cache", "policy", p], + capture_output=True, text=True).stdout + cand = next((l.split()[1] for l in pol.splitlines() if "Candidate:" in l), "?") print(f"{p} installed={inst} candidate={cand}", flush=True) except Exception as exc: # never let the probe break app startup print(f"{p} probe-error={type(exc).__name__}", flush=True) @@ -50,7 +38,6 @@ if os.environ.get("APT_PROBE"): sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "src")) from dotenv import load_dotenv - load_dotenv("./.env") from gradio_ui import GradioAgentUI diff --git a/bandit.yaml b/bandit.yaml deleted file mode 100644 index 41ff56159a2fe71aa74a3dd1b3408c390f12498e..0000000000000000000000000000000000000000 --- a/bandit.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Bandit config — ADR-0014 static-analysis scan (shared across the three repos). -# -# Scope: src/ only (application code). Tests use synthetic data and assert on -# error paths, so scanning them produces noise without signal. -# -# The intended sandboxed `exec` (ADR-0007) and the localhost-only urlopen calls -# are annotated inline with `# nosec Bxxx` and a rationale comment; the remaining -# accepted infra findings (0.0.0.0 bind in the dev launcher, /tmp working dirs on -# ephemeral HF Spaces, GEO-download urlopen) are captured in -# security/bandit-baseline.json so CI fails only on NEW findings, never on the -# already-triaged set. Every accepted class is documented in -# security/ACCEPTED-FINDINGS.md. -# -# Run: bandit -r src/ -ll -c bandit.yaml -b security/bandit-baseline.json -# (-ll = report medium severity and above.) - -exclude_dirs: - - tests - - .venv - - scripts - - docker diff --git a/deploy/scan_posture.yaml b/deploy/scan_posture.yaml deleted file mode 100644 index 89fa1e6fcaf072f372002fcf77ce65852c81eaeb..0000000000000000000000000000000000000000 --- a/deploy/scan_posture.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# Declared malware-scan posture for the ADR-0011 upload safety gate. -# -# This file is the SINGLE SOURCE OF TRUTH for what an upload actually gets on a -# given deployment. It exists because the posture used to be implicit: the code -# auto-detected ClamAV, so a dev laptop with `brew install clamav` recorded -# `scan_status: clean` while the HuggingFace Space — which has no AV binary — -# recorded `skipped`. Local testing therefore masked production behaviour. -# -# Read by src/uploads/posture.py; consumed by src/uploads/scanning.py. -# -# posture: -# structural_only The deployment has NO anti-virus binary. Uploads get the -# always-on structural magic-byte check (a renamed -# ELF/Mach-O/PE/ZIP/pickle is a hard stop) and NOTHING else. -# An upload is never described as malware-scanned. If an AV -# happens to be present (a dev host), it still runs and a -# `clean` is recorded, but a caveat notes the result is a -# local-host bonus that the deployment does not guarantee. -# av_required The deployment DOES ship a working, signature-updated AV. -# Equivalent to UPLOAD_SCAN_REQUIRED=1 — an upload with no -# clean AV result fails closed. Do NOT select this without a -# verified AV binary AND a current virus database: with no -# signature DB, clamscan exits 2 and EVERY upload hard-fails. -# -# Env override (for local experiments / CI): UPLOAD_SCAN_POSTURE, and the -# pre-existing UPLOAD_SCAN_REQUIRED, both still win over this file. - -# --------------------------------------------------------------------------- # -# Current deployed posture — anne-voigt/Paper2Agent_decoupleRpy (prod) and -# Paper2Agent_decoupleRpy_dev. Verified 2026-07-29 against the running prod -# Space at 7ef9d42: it is an `sdk: gradio` Space, so its only apt channel is -# packages.txt, and packages.txt has no clamav entry -> no clamdscan/clamscan on -# PATH -> every upload there records scan_status="skipped". -# -# Why not install ClamAV here: ADR-0011's remaining-at-AWS item already moves -# the managed AV pass onto the encrypted staged S3 object, which supersedes a -# local-AV build. On a cpu-basic Gradio Space `apt install clamav` ships no -# signature database, so it would need a freshclam download (~1 GB, several -# minutes) on every container start — and a stale or failed freshclam turns -# av_required into a total upload outage. Structural-only, stated honestly, is -# the correct interim posture until the AWS staging bucket exists. -# --------------------------------------------------------------------------- # -posture: structural_only diff --git a/docker/sandbox.Dockerfile b/docker/sandbox.Dockerfile deleted file mode 100644 index c414ef29bb62d16e9db53be9ec41661a8c310a5b..0000000000000000000000000000000000000000 --- a/docker/sandbox.Dockerfile +++ /dev/null @@ -1,73 +0,0 @@ -# ADR-0007 Phase 1 — sandbox exec-kernel image. -# -# This image runs ONLY the untrusted-code exec-kernel -# (src/managers/execution/sandbox/kernel.py) inside an isolated per-session -# container. The vetted decoupleR/scanpy/rpy2 tool implementations live OUTSIDE -# the sandbox in the persistent MCP HTTP server — the kernel reaches them via the -# MCP bridge (SANDBOX_MCP_URL). Even so, the image carries the SAME scientific -# stack as the Space (scanpy/decoupler/pydeseq2/rpy2 + R) so generated code that -# imports those libraries locally still runs. -# -# The HF Space itself is a gradio SDK Space (no custom image), so there is no -# pre-built base image to inherit; this mirrors the Space's environment from -# python_version (3.11), packages.txt (apt), and requirements.txt (pip). -# -# --------------------------------------------------------------------------- -# BUILD / RUN (needs local Docker — NOT built in the dep-light dev env): -# -# docker build -f docker/sandbox.Dockerfile -t decouplerpy-sandbox:latest . -# -# # ad-hoc smoke test (kernel only, no MCP): -# docker run --rm -p 127.0.0.1:8790:8790 decouplerpy-sandbox:latest \ -# python /app/src/managers/execution/sandbox/kernel.py --port 8790 --host 0.0.0.0 -# curl -s http://127.0.0.1:8790/health -# -# In prod the ContainerLauncher issues an equivalent `docker run` per session, -# publishing the kernel port to LOCALHOST only. Phase-2 hardening flags -# (--read-only, --cap-drop=ALL, --tmpfs, --pids-limit, --network) are applied by -# the launcher / task definition, not baked here. -# --------------------------------------------------------------------------- - -FROM python:3.11-slim - -# System packages mirror the Space's packages.txt (R + limma + build libs that -# rpy2 / scientific wheels need). Pins are dropped here (slim/bookworm apt has -# different candidate versions than the Space base); pin if a build needs it. -RUN apt-get update && apt-get install -y --no-install-recommends \ - r-base \ - r-base-dev \ - r-bioc-limma \ - libcurl4-openssl-dev \ - libssl-dev \ - libxml2-dev \ - libreadline-dev \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /app - -# Install the Python deps first (layer-cached independently of source changes). -COPY requirements.txt /app/requirements.txt -RUN pip install --no-cache-dir -r /app/requirements.txt - -# Copy the agent source. The kernel is launched BY FILE PATH -# (/app/src/managers/execution/sandbox/kernel.py), NOT `-m managers...`, so the -# full `managers` package (agent stack) is never imported into the sandbox — only -# the minimal kernel + its file-path-loaded siblings run here. PYTHONPATH is kept -# as a harmless fallback. -COPY src /app/src -ENV PYTHONPATH=/app/src - -# --- Non-root user (Phase 1 minimum; Phase 2 adds fuller hardening). ---------- -RUN useradd --create-home --uid 10001 sandbox \ - && chown -R sandbox:sandbox /app -USER sandbox - -# Kernel port (per-session the launcher may override with --port). -ENV SANDBOX_KERNEL_PORT=8790 -EXPOSE 8790 - -# Bind to 0.0.0.0 INSIDE the container; the launcher publishes only the mapped -# port to the host's 127.0.0.1, so the kernel is never on a routable interface. -# File-path launch (not `-m`) keeps the agent-stack package out of the sandbox. -ENTRYPOINT ["python", "/app/src/managers/execution/sandbox/kernel.py"] -CMD ["--port", "8790", "--host", "0.0.0.0"] diff --git a/docs/adr/ADR-0003-spaces-dev-mode.md b/docs/adr/ADR-0003-spaces-dev-mode.md index 278a211a5e97de0436a61768a6949902d5ac247d..0db685b7ee05ae082403db4aad27094b9e8f54b3 100644 --- a/docs/adr/ADR-0003-spaces-dev-mode.md +++ b/docs/adr/ADR-0003-spaces-dev-mode.md @@ -1,6 +1,6 @@ # ADR-0003: Spaces Dev Mode for Iterative Development -**Status:** Accepted — **CLOSED 2026-07-01.** Dev Mode enabled on both dev Spaces (`Paper2Agent_decoupleRpy_dev`, `pdac-analysis-orchestrator-dev`, confirmed 2026-06-24). The remaining housekeeping is now done: the "Dev Mode inner loop" note, the SSH / VS Code Remote setup step, and the commit-before-promote convention are all documented in `pdac-analysis-orchestrator/DEPLOYMENT.md` (tier "C. Dev Mode inner loop" + the "Golden rule"); `hf-dev/main` verified current with `main` (ahead by unpromoted dev work, `main` 0 ahead — not stale). The one inherently per-developer step (attaching VS Code Remote-SSH via the command in each Space's Dev Mode panel) is documented, not a repo deliverable. Reconciled + closed 2026-07-01. +**Status:** Accepted — decision adopted. **Dev Mode enabled on both dev Spaces** (`Paper2Agent_decoupleRpy_dev`, `pdac-analysis-orchestrator-dev`), confirmed 2026-06-24. Remaining action items (DEPLOYMENT.md inner-loop note; VS Code/SSH config) still open. Reconciled 2026-06-24. **Date:** 2026-06-24 **Deciders:** Annie Voigt (project lead) **Scope:** `DecoupleRpy_Agent` (specialist Space) and `pdac-analysis-orchestrator` (coordinator Space). Enabled by the HF PRO subscription (2026-06). No code change in either repo — this is a Space-settings + workflow change. The orchestrator is named here because it shares the same deploy-and-wait pain; the ADR lives in `DecoupleRpy_Agent/docs/adr/` per the existing ADR-home convention (cf. ADR-0001's cross-repo placement note). @@ -101,10 +101,10 @@ and does not change how secrets/tokens are managed. ## Action Items 1. [x] Enable Dev Mode on `Paper2Agent_decoupleRpy_dev` and `pdac-analysis-orchestrator-dev`. *(done — both enabled, confirmed 2026-06-24)* -2. [x] Confirm `hf-dev` actually tracks current `main` before relying on it. *(done 2026-07-01 — `git rev-list --left-right --count main...hf-dev/main` = `0 44`: `hf-dev/main` contains everything on `main` plus unpromoted dev work; `main` is 0 ahead. Not stale. Caveat: compared against local remote-tracking refs; re-run after a `git fetch hf-dev` if in doubt.)* -3. [x] Add a short "Dev Mode inner loop" note to `pdac-analysis-orchestrator/DEPLOYMENT.md` so the workflow is documented next to the deploy rules. *(done — see DEPLOYMENT.md tier "C. Dev Mode inner loop", which links back to this ADR.)* -4. [x] Set up VS Code Remote / SSH config for both dev Spaces. *(procedure documented in DEPLOYMENT.md: one-time copy of the SSH command from each Space's Settings → Dev Mode panel, then attach with VS Code Remote-SSH. The attach itself is a per-developer/per-machine step, not a repo artifact.)* -5. [x] Convention: anything proven in a Dev Mode session must land as a commit before prod promotion. *(documented as the "Golden rule (Dev Mode)" in DEPLOYMENT.md.)* +2. [ ] Confirm `hf-dev` actually tracks current `main` before relying on it (SHOWCASE_STATUS flags it as possibly stale). +3. [ ] Add a short "Dev Mode inner loop" note to `pdac-analysis-orchestrator/DEPLOYMENT.md` so the workflow is documented next to the deploy rules. +4. [ ] Set up VS Code Remote / SSH config for both dev Spaces. +5. [ ] Convention: anything proven in a Dev Mode session must land as a commit before prod promotion. ## References diff --git a/docs/adr/ADR-0005-private-storage-persistence.md b/docs/adr/ADR-0005-private-storage-persistence.md index 23227b04f253cbadf3dce0740f87ac4c9c63fadc..2e92fd1181a6e2bbb1a5386a4a6e2a079c32a068 100644 --- a/docs/adr/ADR-0005-private-storage-persistence.md +++ b/docs/adr/ADR-0005-private-storage-persistence.md @@ -1,6 +1,6 @@ # ADR-0005: Private Storage as Durable Data Persistence -**Status:** Accepted — **CLOSED 2026-07-01.** Durable, private-by-default data tier adopted; `pdac-research-data` confirmed private (2026-06-24). Remaining housekeeping now done: the "promote to public on publication" step and the private-Data-Studio validation workflow are documented in `biodata-registry/CLAUDE.md` ("Adding a New Dataset" step 5 + "Manifest validation workflow"), with a dated entry in `biodata-registry/memory.md`. One out-of-repo note (SHOWCASE_STATUS.md h5ad hosting) is tracked below. Reconciled + closed 2026-07-01. +**Status:** Accepted — durable, private-by-default data tier adopted. **`pdac-research-data` confirmed private** 2026-06-24. Remaining action items (onboarding "promote to public on publication" checklist step; private Data Studio verification workflow; `biodata-registry/memory.md` note) still open. Reconciled 2026-06-24. **Date:** 2026-06-24 **Deciders:** Annie Voigt (project lead) **Scope:** Cross-cutting data layer — primarily the `anne-voigt/pdac-research-data` HF Dataset repo (h5ad hosting consumed by `DecoupleRpy_Agent`), with knock-on benefit to `lit-agent`'s corpus snapshots. Enabled by HF PRO (2026-06). Lives in `DecoupleRpy_Agent/docs/adr/` as the primary data consumer; the `lit-agent` corpus-persistence specifics are governed by its own ADRs. @@ -92,10 +92,10 @@ registry — only where/how the underlying files are stored and inspected. ## Action Items 1. [x] Confirm `pdac-research-data` visibility is private and audit which files are public. *(confirmed private 2026-06-24)* -2. [x] Stop any pruning driven solely by storage limits; retain corpus history in lit-agent's durable dataset. *(codified — `lit-agent/CLAUDE.md` "Retention (PRO storage): keep corpus snapshots across runs rather than minimizing them".)* -3. [x] Add a "promote to public on publication" step to the dataset-onboarding checklist. *(done 2026-07-01 — `biodata-registry/CLAUDE.md` "Adding a New Dataset" step 5: private-by-default; publish = explicit promote-to-public.)* -4. [x] Use private Data Studio as the first-pass check in the `dataset_validate_manifest_against_data` workflow. *(done 2026-07-01 — `biodata-registry/CLAUDE.md` "Manifest validation workflow".)* -5. [x] Record the new storage posture in `SHOWCASE_STATUS.md` (h5ad hosting note) and `biodata-registry/memory.md` (validation workflow). *(done 2026-07-01 — `biodata-registry/memory.md` dated entry + a "h5ad hosting posture — private-by-default (ADR-0005)" bullet in `SHOWCASE_STATUS.md`'s biodata-registry section.)* +2. [ ] Stop any pruning driven solely by storage limits; retain corpus history in lit-agent's durable dataset. +3. [ ] Add a "promote to public on publication" step to the dataset-onboarding checklist. +4. [ ] Use private Data Studio as the first-pass check in the `dataset_validate_manifest_against_data` workflow. +5. [ ] Record the new storage posture in `SHOWCASE_STATUS.md` (h5ad hosting note) and `biodata-registry/memory.md` (validation workflow). ## References diff --git a/docs/adr/ADR-0006-loveless-single-cell-serving.md b/docs/adr/ADR-0006-loveless-single-cell-serving.md index 93c1596b386218829a02e1bab7f9a28b96f2bd93..c72330b8213809539309a6eb61aac02d2a5ee8a9 100644 --- a/docs/adr/ADR-0006-loveless-single-cell-serving.md +++ b/docs/adr/ADR-0006-loveless-single-cell-serving.md @@ -158,9 +158,9 @@ artifacts for. 4. [x] Add `src/workflows/signatures.py` (load/validate a `source/target/weight` signature net) + tests with synthetic data. 5. [x] **biodata-registry (0.1.8, 2026-07-01):** two Loveless-atlas single-cell subsets landed — `gse155698_steele` and `gse205013_werba` (`modality: sc_rnaseq`, `raw_counts`, `expression_source.type: url` → hosted `.h5ad` on `pdac-research-data`) — and the `CROSS_RESOLUTION` gate. **Role 2 signature artifact published:** `scripts/ingest/loveless/derive_signatures.py` derives 14 per-cell-type marker signatures from the Steele subset (`rank_genes_groups` on `Clusters`, top-50/cluster, log2FC≥1, padj≤0.05) → `loveless/signatures/gse155698_steele_celltype_signatures.csv`. Markers are biologically sane (ACINAR→PRSS1/CLPS, FIBROBLASTS→LUM/COL11A1, B CELLS→PAX5/IGHD, …). `dataset_score_signature` / `load_signature_net` load it via the authenticated resolver and score end-to-end. Werba signatures can be derived the same way when needed. 6. [x] Add an `h5ad` / single-cell `expression_source.type` to `_build_loading_plan` so a Steele-subset dataset gets a registry loading plan. Done — plus the schema alignment it required: `manifest_schema` now adds `"h5ad"` to `VALID_EXPRESSION_SOURCE_TYPES` and derives `analysis_path == "P"` for `sc_rnaseq` / `spatial_rnaseq` (modality checked **before** data_level, so an sc raw_counts h5ad is never mislabeled Path A — matches biodata-registry 0.1.7's A/B/P). The plan loads via `decoupler_load_and_visualize_data` (`read_h5ad_cached`) and Path P emits per-cell scoring + a pseudobulk note instead of the bulk DESeq2/limma contrast tail. Tests: `tests/test_loading_plan_h5ad.py`. The sc loaders (`rna_sc.py::_load_adata`) now resolve a hosted/private h5ad URL→local path through the shared authenticated resolver (`resolve_to_local_path`, `HF_TOKEN`) — the same path the bulk tools use — so a private `pdac-research-data` h5ad loads end-to-end; a stable HF-cache file is parsed once per process via `read_h5ad_cached`, a one-shot temp download is read directly and deleted (`tests/test_rna_sc_loader.py`). **Reconciled with the landed 0.1.8 manifests:** they declare `expression_source.type: url` (not `h5ad`), so Path P routing in `_build_loading_plan` now keys on **modality** (checked first) — a single-cell/spatial dataset loads via `decoupler_load_and_visualize_data`, never the bulk `decoupler_load_url_counts`, regardless of the declared source-type string. Verified against the live `gse155698_steele` / `gse205013_werba` manifests. -7. [x] Confirm the prod Space RAM tier holds the Steele-subset h5ad live (it should; the raw atlas must not be served live). **Verified 2026-08-13 by two live agent runs on prod** (runs `20260813_191535` Steele, `20260813_192108` Werba in `anne-voigt/decoupleRpy_results`): both slimmed subsets load via `dataset_load` and render a UMAP end-to-end on the cpu-basic Space (Werba, the larger at 167,366 cells, in 640 s using the atlas's precomputed `X_umap`). The raw atlas remains off the live path. -8. [x] Re-pin biodata-registry to the release carrying the Loveless artifacts — done, and the factory-rebuild worry is closed: prod was **factory-rebuilt on 0.1.16** (2026-08-13) with both sc manifests registered; the item-7 runs verify it end-to-end on prod (superseding the planned `hf-dev` check). -9. [ ] Pair with the PURIST-vs-single-cell methodology comparison (TODO) once the subset is loadable. *Unblocked as of 2026-08-13 (both subsets load on prod); waits on the Med TODO "Add the PURIST subtype operation".* +7. [ ] Confirm the prod Space RAM tier holds the Steele-subset h5ad live (it should; the raw atlas must not be served live). +8. [~] Re-pin biodata-registry to the release carrying the Loveless artifacts (0.1.8) — **done** (`requirements.in`/`.txt`; already on `origin/main` + prod, now on this branch too). **Still open:** the 0.1.8 re-pin needs a FACTORY rebuild of the consuming Space (pip/build-cache lesson from 0.1.6), then end-to-end verify on `hf-dev`. +9. [ ] Pair with the PURIST-vs-single-cell methodology comparison (TODO) once the subset is loadable. ## References diff --git a/docs/adr/ADR-0007-phase1-local-validation.md b/docs/adr/ADR-0007-phase1-local-validation.md deleted file mode 100644 index 7ae1f949d20f586b045b834f99f86bc144359bd4..0000000000000000000000000000000000000000 --- a/docs/adr/ADR-0007-phase1-local-validation.md +++ /dev/null @@ -1,147 +0,0 @@ -# ADR-0007 Phase 1 — Local Validation Checklist - -Validates the sandboxed executor on a machine with Docker + the heavy deps -(rpy2/scanpy/decoupler) — the parts that couldn't run in the build environment. -Branch under test: `feat/sandbox-executor` (stacked on `feat/executor-seam`). - -**Validation run: 2026-07-01** (macOS, Docker 28.4.0, `.venv` Python 3.12 with -scanpy 1.12.1 / decoupler 2.1.6; image built on python:3.11-slim). Result: -**Sections 0–8 validated** (with the noted caveats); Section 9 is the merge step, -not run here. Two real container-launch bugs were found and fixed, and the MCP -bridge's real streamable-http handshake was implemented (was a mock-only stub). - -## 0. Setup -- [x] `git checkout feat/sandbox-executor` -- [x] Full deps available in `.venv` (scanpy/decoupler present; **rpy2 not in the - local `.venv`** — R paths validated inside the container instead). Docker running. - -## 1. Suite still green with real deps -- [x] Full suite green **per-file**: all 46 `tests/test_*.py` files pass or skip - (5 are network/data integration tests that `skip`), **0 failures**. -- [x] `pytest tests/test_sandbox_executor.py tests/test_executor_seam.py` → - **27/27** (was 22; +5 regression tests added for the fixes below). -- ⚠ A single-invocation `pytest -q` **cannot collect** the whole repo: (a) - `scripts/check_limma_runtime.py` is a standalone script that `sys.exit(1)`s at - import when rpy2 is absent, and (b) the `tests/` files insert `sys.path` - differently, so collecting them together pollutes `managers` import resolution. - **Both are pre-existing and unrelated to this branch** (confirmed: not in the - branch diff). Run per-file, or with `PYTHONPATH=src` and excluding `scripts/`. - -## 2. Default path unchanged (regression guard) -- [x] With `EXECUTOR` unset, `get_executor()` returns an in-process - `PythonExecutor`; a multi-step session runs and state persists — no behavior - change. - -## 3. Subprocess launcher, end-to-end (no Docker) -- [x] `EXECUTOR=sandbox SANDBOX_LAUNCHER=subprocess` → multi-step session. -- [x] State persists across steps (a var from step 1 is visible in step 2/3), - stdout returns correctly, and the `kernel.py` subprocess is reaped on - `close()` (`pgrep -f sandbox/kernel.py`: 1 while open → 0 after). - -## 4. Build the sandbox image -- [x] `docker build -f docker/sandbox.Dockerfile -t decouplerpy-sandbox:latest .` - → built (2.91 GB). **Added `.dockerignore`** so the build context is `src/` - + `requirements.txt`, not the full 3.2 GB repo (`.venv`, `.git`, etc.). -- [x] Kernel is the entrypoint; container starts and `/health` returns - `{"status":"ok"}`. -- [x] Runs as **non-root**: `uid=10001(sandbox)`. - ⚠ The `docker run --rm whoami` form checks nothing — the image - ENTRYPOINT swallows `whoami` as a kernel arg. Use - `docker run --rm --entrypoint whoami `. - -## 5. Container launcher, end-to-end -- [x] `EXECUTOR=sandbox SANDBOX_LAUNCHER=container SANDBOX_IMAGE=decouplerpy-sandbox:latest` - → session runs inside the container. -- [x] scanpy + decoupler work inside the container and **state persists** across - execute calls (built an AnnData in step 1, normalized it in step 2). -- [x] Container is torn down on session end (`docker ps -a` — 0 leftover). -- ⚠ `import rpy2.robjects` (the R/limma DE path) **fails inside the image**: - `tzlocal==5.4.2` in `requirements.txt` resolves to a **metadata-only wheel** - (its `RECORD` lists only `*.dist-info`; a force-reinstall still writes no - `tzlocal/` package). This is a **requirements/lockfile packaging bug, not an - ADR-0007 issue** — it blocks only the rpy2/limma method (ttest/DESeq2 paths and - all of scanpy/decoupler are unaffected). Fix separately (re-pin/repair tzlocal). -- **Two container-launch bugs found & fixed** (`launchers.py`): - 1. `ContainerLauncher._kernel_command()` prepended `python `, but - the image ENTRYPOINT is *already* `python .../kernel.py` — so the container - ran `python kernel.py python kernel.py …`, argparse rejected it, and the - kernel exited before `/health`. Now returns **args only** (matching the - Dockerfile `CMD`). - 2. `start()` only caught `ImportError` for the docker SDK, so an `import docker` - that resolves to the repo's shadowing `docker/` namespace dir (no `from_env`) - raised `AttributeError` instead of falling back to the CLI. Now any - unusable-SDK case (`ImportError`, missing `from_env`, daemon down) falls back - to the `docker` CLI. - -## 6. MCP bridge — real handshake (the marked TODO) -- [x] Pointed `SANDBOX_MCP_URL` at a live `server.py --transport streamable-http` - (mounted at `/mcp/`). -- [x] **Implemented the real handshake** in `sandbox/mcp_bridge.py`: a - `streamable_http` transport that drives the same `mcp` client path as - `mcp_manager.add_mcp_http` (`streamablehttp_client` → `initialize` → - `tools/call`), plus transport auto-selection (`/mcp` → streamable-http, - else → the existing plain-JSON mock/adapter transport; - `SANDBOX_MCP_TRANSPORT` overrides). Previously `call_mcp_tool` only spoke - the mock JSON protocol. -- [x] Confirmed an in-kernel tool stub invokes a **real** MCP tool - (`dataset_list_available`) over streamable-http and returns real manifest - data — not the mock. - -## 7. Trace/logging intact through the sandbox -- [x] Verified in-scope: tracing is **executor-agnostic**. The workflow engine - records `code_execution` (generated code) and `observation` (captured - stdout — tool results, dataset-load prints) in the *agent* process around - `self.python_executor(code)` (`agent.py:261`); the executor only returns a - stdout string. Same code → **byte-identical stdout** from `PythonExecutor` - and the sandbox (both reuse `NamespaceKernel.exec_capture`), so the trace - captures identically. -- ⚠ Full log-**sink persistence** (`from logging_sink import …` in `agent.py`) - lives on `feat/configurable-log-sink`, which is **not on this branch** and, per - Section 9, merges *before* `feat/sandbox-executor`. End-to-end sink validation - belongs to that combined-branch stage; nothing in the sandbox path affects it. - -## 8. Light security spot-checks (full hardening is Phase 2) -- [x] Kernel binds localhost only: subprocess launcher `host=127.0.0.1`; container - publishes `127.0.0.1::` (`docker port` → `… -> 127.0.0.1:`, - never `0.0.0.0`). -- [x] Generated code in the **container** cannot read host secrets: with - `ANTHROPIC_API_KEY`/`MY_HOST_SECRET` set on the host, in-sandbox - `os.environ.get(...)` returns `None` for both (only `SANDBOX_MCP_URL` is - passed in). -- ⚠ The **subprocess** launcher inherits `os.environ` (`env = dict(os.environ)`), - so in dev/subprocess mode generated code *can* read host env. Only the - **container** launcher is a real isolation boundary — the subprocess launcher is - a dev/test convenience, not a security boundary. -- Phase 2 status: the container-side flags are now **done** — read-only rootfs, - `--cap-drop=ALL`, `--security-opt=no-new-privileges`, memory/CPU/pids limits, - writable tmpfs scratch, and `--network` (env-overridable), with the subprocess - launcher explicitly marked "not a security boundary" in code. Egress - deny-by-default + MCP allow-list and the read-only source mount remain - follow-ups (deferred to Phase 3 / a local proxy spike). Full checklist + - live-verification results: **`docs/adr/ADR-0007-phase2-local-hardening.md`**. - -## 9. Merge order (once green) -- [ ] `feat/executor-seam` → main (Phase 0, safe no-op default). -- [ ] `feat/configurable-log-sink` → main. -- [ ] `feat/sandbox-executor` → main (keep `EXECUTOR=in_process` default in prod - until the AWS/container path is validated in a real deploy — Phase 3). - ---- - -## Changes made during this validation -- `src/managers/execution/sandbox/mcp_bridge.py` — implemented the real - streamable-http tool-dispatch transport + `select_transport()` auto-detection - (Section 6); kept the plain-JSON transport for the mock/adapter. -- `src/managers/execution/sandbox/launchers.py` — fixed the two container-launch - bugs in Section 5 (args-only kernel command; robust docker-SDK→CLI fallback). -- `.dockerignore` — new; trims the build context (Section 4). -- `tests/test_sandbox_executor.py` — +5 tests: container command is args-only, - localhost-only publish, SDK→CLI fallback, and MCP transport selection. - -## Follow-ups (out of ADR-0007 scope) -- **`tzlocal==5.4.2` is a metadata-only wheel** → `import rpy2.robjects` fails in - the image (and the R/limma path in the Space if it uses the same pin). Re-pin or - repair. Tracked separately. -- Repo-wide `pytest -q` can't collect in one shot (standalone `scripts/test_*` - that `sys.exit`s + cross-file `sys.path` pollution). A `conftest.py`/`pyproject` - `testpaths`+`pythonpath` config would let CI run the suite in one invocation. diff --git a/docs/adr/ADR-0007-phase2-local-hardening.md b/docs/adr/ADR-0007-phase2-local-hardening.md deleted file mode 100644 index d4a71cc6fb095dd3e93c24b4eabed30c63ee86c3..0000000000000000000000000000000000000000 --- a/docs/adr/ADR-0007-phase2-local-hardening.md +++ /dev/null @@ -1,122 +0,0 @@ -# ADR-0007 Phase 2 — Local Hardening Checklist - -Locks down the `ContainerLauncher` sandbox with `docker run` least-privilege -flags, still on **local Docker** (AWS is Phase 3). Follows Phase 1 -(`feat/sandbox-executor`, commit `ffba834` / validation `d864312`), where the -container ran with default Docker privileges. - -**Scope:** `src/managers/execution/sandbox/launchers.py` (`ContainerLauncher`) + -`tests/test_sandbox_executor.py`. `SubprocessLauncher` is deliberately **not** -touched — it is a dev/test convenience and **not a security boundary** (its -docstring now says so explicitly). The prod default stays `EXECUTOR=in_process`; -these changes only affect `SANDBOX_LAUNCHER=container`. - -**Validated: 2026-07-01** (macOS, Docker 28.4.0, image `decouplerpy-sandbox:latest` -already built from Phase 1). All 22 tests in `tests/test_sandbox_executor.py` -pass, **including the live hardened-container integration test** (see §3). - ---- - -## 1. Container flags applied (the `_hardening()` set) - -All derive from ONE `_hardening()` source read at launch time, formatted for both -the CLI (`_docker_run_cmd` → `_hardening_flags`) and the docker-SDK -(`start` → `_hardening_kwargs`) paths so the two **cannot drift**. - -| Control | CLI flag | SDK kwarg | Why | -|---|---|---|---| -| Immutable root FS | `--read-only` | `read_only=True` | code can't tamper with the image / persist between runs | -| Drop all caps | `--cap-drop=ALL` | `cap_drop=["ALL"]` | kernel needs none (binds an unprivileged high port) | -| No priv-escalation | `--security-opt=no-new-privileges` | `security_opt=["no-new-privileges"]` | block setuid/setgid escalation | -| RAM cap | `--memory` (`SANDBOX_MEMORY`, def `4g`) | `mem_limit` | OOM-kill runaway allocs | -| CPU cap | `--cpus` (`SANDBOX_CPUS`, def `2`) | `nano_cpus=int(cpus*1e9)` | bound CPU spin | -| PID cap | `--pids-limit` (`SANDBOX_PIDS_LIMIT`, def `512`) | `pids_limit` | fork/thread-bomb guard | -| Network mode | `--network` (`SANDBOX_NETWORK`, def `bridge`) | `network_mode` | see §2 | -| Writable scratch | `--tmpfs /tmp`, `--tmpfs /home/sandbox` (`SANDBOX_TMPFS_SIZE`, def `1g`) | `tmpfs={...}` | see below | -| MCP host resolution | `--add-host=host.docker.internal:host-gateway` | `extra_hosts={...}` | MCP reachable on Linux too (dropped when `network=none`) | - -**Read-only rootfs + writable tmpfs.** `--read-only` freezes the whole FS, so -anything that writes at runtime needs an explicit escape hatch: -- **`/tmp`** (`mode=1777`) — generic scratch for generated analysis code (scanpy - figure exports, pydeseq2 intermediates, temp files). -- **HOME `/home/sandbox`** (owned by uid `10001`) — matplotlib / numba / - fontconfig write caches under `~/.config` and `~/.cache` **at import time**; - with a read-only HOME, `import scanpy` (which imports matplotlib) crashes - before any user code runs. The live test §3(a) confirms the exec-kernel itself - boots and runs code fine under the read-only rootfs + these two tmpfs mounts. - -tmpfs is RAM-backed, size-capped (so a runaway write can't exhaust host memory), -and vanishes on teardown — matching the ADR's "nothing persists past the -session" intent. - -**Env overrides** (mirroring the `get_executor` / `get_log_sink` env-var style): -`SANDBOX_MEMORY`, `SANDBOX_CPUS`, `SANDBOX_PIDS_LIMIT`, `SANDBOX_NETWORK`, -`SANDBOX_TMPFS_SIZE`. All optional with sane defaults; retune per deploy without -a code change. - -## 2. Egress — what's done, what remains (the honest bit) - -The ADR calls for **deny-by-default egress with an allow-list for only the MCP -endpoint** ("Local egress control"). The nuance: the in-kernel MCP tool stubs -must still reach the MCP HTTP server (`SANDBOX_MCP_URL`), so a blanket -`--network none` is wrong — it severs the MCP bridge. - -**Done in Phase 2 (local):** -- `SANDBOX_NETWORK` is env-overridable, default `bridge` — MCP reachable via - `host.docker.internal`, with `--add-host=host.docker.internal:host-gateway` - added so the name resolves on native Linux too (it's automatic on Docker - Desktop/Mac/Win). Live-verified reachable in §3(c). -- `SANDBOX_NETWORK=none` is available as a **full-egress-denial** escape hatch - today, for operators who pre-stage all data and don't route MCP over the - container network. When set, the (now-meaningless) host-gateway mapping is - dropped. - -**Deferred — deny-by-default + single-host allow-list (FOLLOW-UP, not faked):** -True "reach ONLY the MCP host, deny everything else" is **not expressible with -plain `docker run` flags**. It needs one of: -- an egress-filtering **proxy sidecar** (e.g. squid/envoy) the sandbox is forced - through, allow-listing only the MCP host:port; or -- **iptables/nftables on a custom Docker network** — but we drop `CAP_NET_ADMIN`, - so the container cannot firewall *itself*; rules must live on the host/daemon. - -This is the right shape for **Phase 3 (AWS Fargate)**, where a VPC with egress -off + security groups scoped to the MCP server give exactly this for free (ADR -"Phase 3" and Open Decision #1: pre-stage into OHSU S3 → zero egress). Until -then, `bridge` (default) permits general egress and `none` denies all; the -in-between single-host allow-list is intentionally left for Phase 3 rather than -half-built locally. - -## 3. Live behavioural verification (`test_hardened_container_confines_and_still_reaches_mcp`) - -Guarded/skipped when Docker or the image is absent; ran for real here. A hardened -container launched via `ContainerLauncher` was proven to: -- **(a)** reject a write to the root FS (`open('/nope.txt','w')` → denied) while - `/tmp` remains writable — the read-only rootfs + tmpfs escape hatch both work; -- **(b)** NOT see a host env secret (`MY_HOST_SECRET` set on the host → - `os.environ.get(...)` is `None` inside; only `SANDBOX_MCP_URL` crosses); -- **(c)** still reach the MCP server — a tool stub round-tripped a call to a mock - MCP endpoint over `host.docker.internal`. - -## 4. Unit coverage (no Docker needed — pure arg logic) - -- `test_container_launcher_has_all_hardening_flags` — every flag present at its - default, all preceding the image. -- `test_container_launcher_resource_limits_env_overridable` — `SANDBOX_*` - overrides honored on both CLI and SDK paths. -- `test_container_launcher_network_none_denies_egress` — `none` drops the - host-gateway mapping on both paths. -- `test_container_launcher_sdk_kwargs_match_cli_flags` — the SDK kwargs and CLI - flags encode the identical lockdown (anti-drift guard). - -## 5. What remains / follow-ups - -- [ ] **Egress allow-list** (deny-by-default + MCP-only) — deferred to Phase 3 - (Fargate/VPC) or a local proxy-sidecar spike; see §2. Not started; not faked. -- [ ] Read-only **source-data mount** — Phase 2 covers the container's own FS + - network; a read-only bind of the source-data location is only meaningful once - data is mounted (local: the h5ad cache dir; AWS: the S3 bucket). Wire when the - mount path is decided (ties into Phase 3 / Open Decision #1). -- [ ] `tzlocal==5.4.2` metadata-only wheel still blocks the rpy2/limma path in - the image — unchanged from Phase 1, tracked separately (not an ADR-0007 issue). -- [ ] Prod default stays `EXECUTOR=in_process`; flipping to `sandbox` + - `SANDBOX_LAUNCHER=container` waits on the Phase 3 real-deploy validation. diff --git a/docs/adr/ADR-0007-sandboxed-code-execution.md b/docs/adr/ADR-0007-sandboxed-code-execution.md deleted file mode 100644 index 28307a25f4e7305a195320e21e9149b79c3662f1..0000000000000000000000000000000000000000 --- a/docs/adr/ADR-0007-sandboxed-code-execution.md +++ /dev/null @@ -1,157 +0,0 @@ -# ADR-0007 — Per-Session Sandboxed Code Execution - -**Status:** Accepted, in progress — **Phases 0–2 complete** (local Docker, -AWS-independent; validated 2026-07-01), **Phases 3–4 blocked on AWS** -(landing zone + Open Decisions 1–3). Two Phase-2 line items are intentionally -deferred into Phase 3 because they only become real with an AWS mount/VPC: -egress deny-by-default + MCP-only allow-list, and the read-only *source-data* -mount (Phase 2 hardened the container's own FS + network, not a data mount). -**Prod default stays `EXECUTOR=in_process`** until the container/AWS path is -validated in a real deploy (Phase 3). -**Date:** 2026-07-01 -**Driver:** OHSU security review of the future-state AWS deployment. The agent -generates and executes LLM-authored Python; isolating that execution is the -single highest-priority control the review asks for. - -**Phase status:** 0 ✅ (`feat/executor-seam`) · 1 ✅ (`ffba834`, local-validated) · -2 ✅ container hardening (`52e7143`; egress + source-mount → Phase 3) · -3 ⬜ AWS Fargate · 4 ⬜ data pre-stage + OHSU sign-off. Phase-2 detail: -`ADR-0007-phase2-local-hardening.md`. To close this ADR: Phases 3–4 (per-session -Fargate task + least-privilege IAM + VPC egress control + the OHSU write-up -naming this as the implemented top-priority control). - ---- - -## Context - -The specialist agent is a CodeAct loop: the model emits `…` -blocks that are run via `exec(code, self.namespace)` in -`src/managers/execution/python_executor.py`. `PythonExecutor` is a persistent, -Jupyter-kernel-style namespace that holds session state (loaded AnnData, -intermediate variables) **across steps** — which is why the in-memory AnnData -cache and persistent MCP HTTP server exist (re-loading per step is the expensive -path). - -Today that `exec` runs **in-process, in the same container as the orchestration -logic**, with whatever privileges and network access the container has. The code -even documents the accepted risk: *"bounded by running in an isolated HF Space -with no secrets beyond the model API key."* That bound is fine for a prototype; -it is not sufficient for OHSU-managed AWS handling restricted research data. - -The review asks for: **network-isolated, least-privilege, ephemeral execution, -with read-only access to source data.** - -## Decision - -Run generated code in a **per-session ephemeral sandbox container** — one -isolated container per user session, holding namespace state across steps, torn -down at session end. "Ephemeral" at session granularity (not per-step), so the -caching model is preserved and data is not re-loaded every step. - -Crucially, this is implemented by **swapping the executor implementation behind -the existing `PythonExecutor` interface**, not by rewriting the agent loop. The -agent already depends only on `send_functions` / `send_variables` / -`__call__(code) -> str`. A new `SandboxedExecutor` satisfies the same contract -but proxies each call to a Python kernel running inside the sandbox container. - -### Architecture - -- **The seam.** Define an `Executor` protocol matching the current interface. - `PythonExecutor` (in-process) remains the default for local/dev; the sandboxed - implementation is selected by config — mirroring the `LOG_SINK` pattern just - added. -- **What crosses the boundary is small.** `__call__` returns captured stdout (a - string) — trivially serializable over a socket/HTTP. No live Python objects - need to move. -- **Tools stay vetted and outside the sandbox.** Injected tool "functions" - become thin MCP-client stubs; the actual computation runs in the existing - persistent MCP HTTP server. So the sandbox holds only the *untrusted* - free-form generated code + session namespace; the *vetted* decoupleR/scanpy - tool implementations run in the MCP server, reachable over a restricted - internal network. This is a natural extension of the current HTTP-MCP model. -- **Data model.** Source data mounted/accessed **read-only**; results written to - a separate scoped location. External dataset fetch (GEO/GDC) is resolved by - either pre-staging data into an OHSU bucket (zero egress — preferred) or an - egress allow-list to approved data domains (see Open Decisions). - -## Phased plan - -Phases 0–2 are **fully AWS-independent** — buildable and testable locally with -Docker. That is the "start now" portion. Phases 3–4 need the AWS account. - -### Phase 0 — Extract the executor seam *(now, ~2–3 days)* -- Define an `Executor` Protocol/ABC from the current `PythonExecutor` surface. -- Make the agent depend on the protocol; keep `PythonExecutor` as the default. -- Add an `EXECUTOR` config switch (`in_process` | `sandbox`), default - `in_process`. No behavior change yet. -- Test: existing suite passes unchanged with the in-process executor. - -### Phase 1 — Local Docker sandbox executor *(now, ~1–2 weeks)* -- Build a sandbox image (reuse the existing Space image — rpy2/scanpy/decoupler - already present) running a minimal "exec kernel": accept code over a local - socket/HTTP, `exec` into a persistent per-session namespace, return stdout. -- Implement `SandboxedExecutor`: starts one container per session, proxies - `send_functions`/`__call__`, tears down on session end. -- Wire tool calls from inside the sandbox to the MCP HTTP server (client stubs). -- Test: a full analysis session runs end-to-end through the sandbox with state - persisting across steps; trace/logging (ADR log-sink) still captures prompts, - code, tool calls, dataset loads. - -### Phase 2 — Local hardening *(now, ~3–5 days)* -- Non-root user, dropped capabilities, read-only root FS, `--read-only` source - mount, tmpfs workdir, CPU/memory/pids limits, no host network. -- Local egress control (deny-by-default; allow-list only what a session needs). -- Verify the sandbox cannot read secrets or write to source data. - -### Phase 3 — AWS Fargate deployment *(needs AWS, ~2–3 weeks)* -- Sandbox container → per-session Fargate task (or ECS-on-gVisor). -- Least-privilege task IAM role: read-only on the source-data bucket, write to a - scoped results prefix, nothing else. No long-lived credentials. -- VPC with egress off (data pre-staged) or allow-listed; security groups scoped - to the MCP server only. -- Session lifecycle: task launched per session, torn down at end; results - plumbed back through the agent. - -### Phase 4 — Data pre-stage + review sign-off *(needs AWS decision, ~1 week)* -- If pre-staging: a sync job that mirrors approved datasets into the read-only - OHSU source bucket (with checksums — also closes the Q6 tamper-detection gap). -- Security validation, threat-model doc, and the OHSU-review write-up naming - this as the implemented top-priority control. - -### LOE summary -- **Startable now (Phases 0–2):** ~2.5–3.5 engineer-weeks, no AWS. -- **AWS-dependent (Phases 3–4):** ~3–4 engineer-weeks once foundations exist. -- **Total:** ~4–7 engineer-weeks (consistent with the review estimate). The - swing factor is Phase 3 and whether OHSU provides a ready landing zone. - -## Open decisions (resolve before Phase 3; do NOT block Phases 0–2) -1. **External data: pre-stage into OHSU S3 (zero egress, preferred) vs. egress - allow-list.** Pre-staging gives the cleanest review story and closes tamper - detection; allow-listing is less upfront work. -2. **AWS landing zone** — does OHSU provide account/VPC/baseline IAM, or is that - part of this scope? -3. **MCP server placement** — same isolation domain as the sandbox, or a - separate hardened service the sandbox reaches over a restricted network. - -## Consequences -- **Positive:** the #1 review control is implemented cleanly via an existing - seam; no agent-loop rewrite; caching/perf model preserved; the sandbox - confines only the untrusted code while vetted tools stay put; work starts - immediately without waiting on AWS. -- **Cost:** a new container image + session lifecycle to operate; per-session - container startup latency (mitigated by session-granular reuse); Phase 3 ties - to AWS specifics. -- **Interim posture:** until Phase 3, the in-process executor remains — so for - the current prototype, keep documenting execution as "isolated Space, - API-key-only secrets," and present the sandbox as the funded, in-progress - target rather than a shipped control. - -## Start-now checklist -- [x] Phase 0: extract `Executor` protocol + `EXECUTOR` config switch (`feat/executor-seam`) -- [x] Phase 1: sandbox image + `SandboxedExecutor` + MCP client stubs (`ffba834`, local-validated) -- [~] Phase 2: container hardening **done** (read-only rootfs, cap-drop=ALL, - no-new-privileges, mem/cpu/pids limits, tmpfs scratch, `--network` env- - overridable; SDK+CLI paths share one source; live-verified) — see - `ADR-0007-phase2-local-hardening.md`. Egress deny-by-default + read-only - source mount remain follow-ups (deferred to Phase 3 / local proxy spike). -- [ ] Decisions 1–3 raised with OHSU cloud/security contacts (parallel track) diff --git a/docs/adr/ADR-0010-dataset-integrity-verification.md b/docs/adr/ADR-0010-dataset-integrity-verification.md index 165b652edfd15666512b8660cb80feb57e1b0a64..bb345b1059bbc037576696ce35fba9d10caff583 100644 --- a/docs/adr/ADR-0010-dataset-integrity-verification.md +++ b/docs/adr/ADR-0010-dataset-integrity-verification.md @@ -1,7 +1,6 @@ # ADR-0010 — Dataset Integrity / Tamper Verification on Load -**Status:** Accepted — step 1 (AWS-independent) fully implemented 2026-07-02 (22/22 datasets -baselined; GEO-series-matrix load path now covered) +**Status:** Proposed **Date:** 2026-07-01 **Deciders:** Annie Voigt (project lead) **Driver:** OHSU security review Q7 — "detect unavailable/modified/compromised datasets." Today @@ -49,42 +48,14 @@ Add **content-hash verification on load**, keyed off the manifest. SHA-256 verify in `resolve_to_local_path`; refusal path + test (good hash loads, altered file refused, absent hash = load with a "no integrity baseline" note); backfill hashes for the current registered datasets. - *(The shared hashing helper `src/core/integrity.py` — `compute_sha256` / `verify_sha256`, - streamed — landed with ADR-0011 and is the exact low-level code this step's on-load layer - builds on.)* + *(Available now: the shared hashing helper `src/core/integrity.py` + — `compute_sha256` / `verify_sha256`, streamed — landed with ADR-0011 and is the exact code + this step wires into `resolve_to_local_path`.)* 2. **At AWS (ADR-0007 Phase 4):** the pre-stage sync job writes checksums into the read-only OHSU source bucket, so integrity is anchored to an OHSU-controlled copy rather than trust-on-first-use against the public source. This ADR's on-load check is the same code; only the hash's provenance improves. -### Step 1 — as implemented (2026-07-02) - -- **Manifest field** (`biodata-registry` 0.1.9): optional `integrity:` block on `DatasetManifest` - — `sha256` (primary/expression file) + optional `files:` map (per-file, for separate metadata) - + `recorded`/`recorded_from` provenance. Validated (64-hex) in `manifest.validate()`; absent = - valid + a "no integrity baseline" warning. Reverse-lookup helper `expected_sha256_for_url()`. -- **Recorder** `scripts/record_integrity.py` (in `biodata-registry`) streams SHA-256 of each source - and writes the block into the YAML (comment-preserving text edit). **Backfilled 22/22** registered - datasets. `cptac_pda_counts` was initially skipped at 0.1.9 (its hosted `cptac_pda_counts.h5ad` - was 404 on HF); the file was then assembled + hosted and its baseline recorded in - `biodata-registry` **0.1.10** (`6a6a884`; sha256 `7fd71517…`, verified against the HF LFS hash of - the uploaded 21.7 MB file). `DecoupleRpy_Agent` re-pins the registry to **0.1.10** so the agent - enforces all 22/22 at runtime. -- **On-load verify** (`DecoupleRpy_Agent` `src/core/integrity.py`): `verify_file()` is called - centrally in `resolve_to_local_path` after materialize / before parse+cache. Keyed on the source - URL via the registry reverse-lookup, so **no tool signature changes**. Mismatch → `IntegrityError` - refusal naming the dataset; a tampered temp download is unlinked before raising; missing/absent - baseline degrades to "load unverified". Verified end-to-end (real load passes, tampered copy - refused). -- **GEO-series-matrix loads now covered:** `src/workflows/geo.py`'s `load_geo_series_matrix_lines` - bypasses `resolve_to_local_path` (it streams the matrix straight into memory), so it now runs the - same check inline — `verify_bytes()` on the raw response *as served* (pre-decompression, matching - how the baseline is recorded) for URL fetches, and `verify_file()` for local paths. `verify_bytes` - is the in-memory counterpart of `verify_file` added to `src/core/integrity.py`. Both are no-ops - unless a manifest baselines that exact URL/path, so untracked GEO fetches are unaffected; a - baselined series matrix whose bytes were altered is refused before parsing. Trust-on-first-use - caveat below still applies until step 2. - ## Consequences - **Positive:** closes the Q7 tamper gap with a deterministic control, independent of AWS; dovetails diff --git a/docs/adr/ADR-0011-upload-safety-gate.md b/docs/adr/ADR-0011-upload-safety-gate.md index 2d4f74aac38ec4506c909d57ebe619212b0b9594..76fe250d649954afd3ca714c9a34d1a032489322 100644 --- a/docs/adr/ADR-0011-upload-safety-gate.md +++ b/docs/adr/ADR-0011-upload-safety-gate.md @@ -1,9 +1,6 @@ # ADR-0011 — Safety Gate for Manual Dataset Uploads -**Status:** Accepted — "Now" (AWS-independent) slice implemented 2026-07-02, extended -2026-07-02 with a local interim content scan + manifest-drafting helper; **wired into the Gradio -UI 2026-07-06** (all three data-input paths — upload / URL / HF-dataset — now clear the gate before -the agent sees a file); only the AWS staging-bucket/encryption pieces remain deferred (see Plan) +**Status:** Accepted — "Now" (AWS-independent) slice implemented 2026-07-02; AWS pieces deferred (see Plan) **Date:** 2026-07-01 **Deciders:** Annie Voigt (project lead) **Driver:** OHSU security review Q4 — researchers should be able to supply their own datasets. @@ -64,91 +61,18 @@ the existing manifest/validation machinery. through the vetted `scanpy.read_h5ad` loader only — no `exec`/`eval`/`pickle`), and `register_upload` (admin-only promotion, `UPLOAD_ADMIN_IDS`, into the live registry). Every state transition persists an `UploadRecord` (with the de-id attestation + hash) through the always-on ADR-0008 audit sink. - End-to-end tests: `tests/test_upload_gate.py` (27). Auto-validation covers **h5ad** (full - five-check `validate_manifest_against_data`) and **flat matrices** (`.csv`/`.tsv`/`.txt` + `.gz`, - samples-as-rows/genes-as-columns via the vetted `pandas.read_csv`): a bare matrix has no `obs`, - so only the file-content checks that don't need metadata run (`data_level` + `feature_id_type`) - and the obs-dependent checks are recorded as explicit **caveats** on the record, not silently - passed — re-upload as an h5ad to validate grouping/contrasts. -- **Interim content scan (local, AWS-independent). ✅ Added 2026-07-02** as `src/uploads/scanning.py`, - wired as **gate 0** of `validate_upload` (runs on the staged object *before* the loaders). Two - layers: (1) an always-on, zero-dependency **structural magic-byte check** — an `.h5ad` must be an - HDF5 container, a `.gz` must be gzip, and a flat-text matrix must carry no executable/archive/pickle - leader, shebang, or NUL bytes; this catches a renamed ELF/Mach-O/PE/ZIP/pickle that clears the - suffix-only door gate, and is *always* a hard stop. (2) An **optional ClamAV pass** (`clamdscan`/ - `clamscan`, or any scanner named by `UPLOAD_SCAN_CMD`) — external AV *reading* the file, still no - `exec`/`eval`/`pickle`. Auto-detection tries the daemon client (`clamdscan`, fast when `clamd` is - warm) and **degrades to standalone `clamscan`** if the daemon is absent/misconfigured, rather than - reporting the file unscanned. When no AV is present the scan is recorded honestly as - `scan_status="skipped"` with a caveat (never reported as malware-scanned); set `UPLOAD_SCAN_REQUIRED=1` - to fail closed instead. Outcome persists on the `UploadRecord` (`scan_status`/`scanned_at`/ - `scan_detail`) through the ADR-0008 audit sink. Tests: `tests/test_upload_scan.py`. **Verified with a - real local ClamAV** (`brew install clamav` + `freshclam`, 3.6M sigs): a clean matrix passes, the - EICAR test file is flagged `infected` and blocked before the loaders run. -- **Manifest-drafting helper (local, AWS-independent). ✅ Added 2026-07-02** as `src/uploads/drafting.py` - (`draft_manifest`) — the friction mitigation from Consequences below. It opens the file with the same - vetted loaders and pre-fills a manifest skeleton (inferred `data_level`, `feature_id_type`, sample/ - feature counts, candidate `group_columns`) plus an explicit `todo` list of the fields a human must - still supply. It does **not** weaken the gate — a drafted manifest still has to clear `validate_upload` - and admin registration; it only removes the blank-page problem. A local Gradio panel can wrap it as - a thin editable-form shell. Tests: `tests/test_upload_drafting.py`. -- **UI wiring (local, AWS-independent). ✅ Added 2026-07-06** in `gradio_ui.py` - (`run_upload_gate`), called by the Upload-File / URL / HF-Dataset handlers. It runs - `stage_upload → scan_upload → draft_manifest → validate_upload` behind a required - de-identification checkbox, and only exposes the quarantined `staged_path` to the agent after the - security gates clear. - **Ordering correction (2026-07-29, from live verification of the prod Space).** The wiring - originally ran `draft_manifest` *first*, which meant the drafter parsed the file at its original - path before the type allow-list, the size cap, quarantine, or the structural magic-byte scan had - run — violating this ADR's own rule that nothing parses an upload until it is staged and scanned. - The visible symptom on the live Space: an ELF renamed `.csv` was refused, but by an incidental - numpy crash inside the drafter (`zero-size array to reduction operation maximum`) rather than by - the content scan, and a `.pdf` was refused as "could not read" rather than as a disallowed type. - `run_upload_gate` now stages with a `placeholder_manifest` (filename-derived, no file read), scans, - and only then drafts from the **quarantined** copy and attaches it via `attach_manifest`. A file - rejected after quarantine has its staged bytes deleted (`discard_upload`), leaving only the audit - record. The local test asserted merely that the ELF was "blocked", so it passed for the wrong - reason throughout; it now asserts the refusal comes from the content scan. **Session-use vs registration split (decision):** on this session-scoped, - single-file, read-only path the *security* envelope (attestation / type / size / structural + AV - scan / SHA-256 / never-exec) is **mandatory and blocks on failure**, but `validate_upload` - (manifest-vs-data consistency) is **advisory** — an ad-hoc uploader has no hand-authored manifest, - so a bare matrix (which fails the `group_columns`-non-empty schema rule) is still usable with a - visible "manifest not fully validated" warning rather than being blocked. Admin **registration** - into the shared registry (item 7) is unchanged and still required to make an upload a persistent, - listed dataset; a session upload is read by the vetted loaders for that session only and is never - added to the registry. Tests: `tests/test_upload_ui_gate.py`. -- **Deployed scan posture made explicit (`deploy/scan_posture.yaml`). ✅ Added 2026-07-29.** The AV - layer above is auto-detected from PATH, which left the *deployed* posture implicit — and wrong in - the optimistic direction during testing: a dev Mac with `brew install clamav` recorded - `scan_status="clean"`, while the prod Space records `skipped`. Verified 2026-07-29 against the - RUNNING prod Space at `7ef9d42`: `anne-voigt/Paper2Agent_decoupleRpy` is an `sdk: gradio` Space, - so `packages.txt` is its only apt channel and it contains no `clamav` — **no AV binary exists there, - and every upload records `skipped`.** `UPLOAD_SCAN_REQUIRED` was never set in any deploy config. - **Decision: keep structural-only and say so, rather than install ClamAV in the Space image.** - Rationale: (a) the AWS item below already supersedes a local-AV build by moving the managed pass - onto the encrypted staged object; (b) `apt install clamav` ships **no signature database**, so a - Space would need a ~1 GB `freshclam` download on every cold start, and a stale/failed refresh turns - `av_required` into a total upload outage on a cpu-basic box. `deploy/scan_posture.yaml` now declares - `posture: structural_only`, read by `src/uploads/posture.py`; `av_required` is the alternative value - and is equivalent to `UPLOAD_SCAN_REQUIRED=1` (both env knobs still override the file). Every record - is stamped with `scan_posture`, the skipped caveat states plainly that the upload is **NOT - malware-scanned**, a `clean` produced under a `structural_only` posture carries an extra caveat that - it is a host-local result the deployment does not guarantee, and the Gradio panel says - "structure-checked … this is not a malware scan" instead of the ambiguous "content-scanned". -- **At AWS (remaining):** staging bucket is a separate scoped prefix with its own encryption; the - ClamAV pass moves to run on the staged S3 object — at which point the posture file flips to - `av_required`. The basic type/size/quarantine gate **and** the local content scan above do **not** - require AWS — only the scoped/encrypted bucket does. + End-to-end tests: `tests/test_upload_gate.py` (24). Auto-validation currently covers `.h5ad`; + tabular uploads stage but stay quarantined pending a tabular loader. +- **At AWS:** staging bucket is a separate scoped prefix with its own encryption; the optional + malware scan runs on the staged object *before* validation. The basic type/size/quarantine gate + here does **not** require AWS; the malware scan does. ## Consequences - **Positive:** delivers the requested capability without opening an ingress hole — uploads inherit the same grounding gate as curated datasets, stay de-identified by attestation, are tamper-checked, and are never executable. Clean story for the review. -- **Cost / caveat:** a manifest requirement adds friction for uploaders — now mitigated by - `draft_manifest` (`src/uploads/drafting.py`), which drafts the manifest from the file + a few - prompts. Content scanning is now local: an always-on structural magic-byte check plus an optional - ClamAV pass. Full managed anti-malware (signature-updated, on the encrypted staging bucket) is still - AWS-dependent; until then, an upload that ran with no AV available carries an explicit `skipped` - scan caveat and must be described honestly as structurally-checked-but-not-malware-scanned rather - than malware-scanned — unless the deployment sets `UPLOAD_SCAN_REQUIRED` to fail closed. +- **Cost / caveat:** a manifest requirement adds friction for uploaders (mitigate with a UI that + drafts the manifest from the file + a few prompts). Full anti-malware coverage is AWS-dependent; + until then the gate is type/size/quarantine/validation/attestation, which should be stated + honestly rather than described as malware-scanned. diff --git a/docs/adr/ADR-0012-authentication-access-control.md b/docs/adr/ADR-0012-authentication-access-control.md deleted file mode 100644 index bbc5bd9cc782d1f8b5d814a9e0f2fc77249ba6da..0000000000000000000000000000000000000000 --- a/docs/adr/ADR-0012-authentication-access-control.md +++ /dev/null @@ -1,164 +0,0 @@ -# ADR-0012 — Authentication & Access Control on the Spaces - -**Status:** **Accepted — IMPLEMENTED, DEPLOYED to prod (both Spaces), and validated -end-to-end (2026-07-02).** Front-door HF OAuth + allow-list gate live on the -orchestrator (public); specialist flipped **private** + reached only via the -orchestrator's service token (anonymous access verified refused); authenticated -identity flows into the ADR-0008 trace. The **only** remaining piece is the OHSU -SSO/IdP provider cutover, which is **deferred by design** (an OHSU decision) and does -not block this "now" slice. *(Proposed → Accepted → Done 2026-07-02.)* -**Date:** 2026-07-02 -**Deciders:** Annie Voigt (project lead) -**Driver:** OHSU security review — the "Authentication & Authorization / Users" -section describes ~5 users, 1–2 admins, service accounts, and *no shared -accounts*. That is the intended model; today it is **not enforced in code**. -Both Spaces call Gradio `launch()` with no `auth=` and no login gate, so access -control is only whatever the Space's HuggingFace visibility setting provides. -**Related:** ADR-0008 (audit trace — records *what* happened but attaches no -authenticated identity), ADR-0011 (admin-only upload registration — assumes an -identity this ADR supplies), future OHSU SSO/IdP integration (the AWS-era target). - ---- - -## Context - -The system is two Gradio Spaces: - -- **`pdac-analysis-orchestrator`** — the user-facing front door. `app.py` → - `ui.build().launch(server_name="0.0.0.0", server_port=7860)`. No `auth`. -- **`Paper2Agent_decoupleRpy`** (specialist) — called by the orchestrator via - `gradio_client`. `GradioAgentUI.launch(share=False, **kwargs)` → - `app.queue(...).launch(...)`. No `auth`. It is also **directly reachable** as - its own Space, so it is an authentication bypass around the orchestrator. - -Neither app authenticates a user, and there is no per-user identity attached to -a request or to the ADR-0008 audit trace. The review's answers about admin -accounts, service accounts, and no-shared-accounts therefore have no technical -enforcement behind them. This is the largest AWS-independent exposure remaining. - -Two distinct problems: -1. **Front-door auth** — who may use the orchestrator at all. -2. **Specialist bypass** — the specialist must not be usable except *through* an - authenticated orchestrator (or under the same gate). - -## Decision - -Add an authentication gate to both Spaces now, using HuggingFace-native -identity (AWS-independent), structured so the later cutover to OHSU SSO/IdP is a -provider swap, not a redesign. - -1. **Front door — HF OAuth, allow-listed.** Put the orchestrator behind - HuggingFace OAuth (`hf_oauth: true` in the Space README metadata + - `gr.LoginButton` / the `gr.OAuthProfile` dependency), and authorize only an - explicit allow-list of HF usernames (config, not code — mirrors - `recipients.yaml` / `UPLOAD_ADMIN_IDS`). A logged-in user not on the list is - denied. This ties every session to a named identity that ADR-0008 can record. - - *Interim fallback if OAuth is deemed too heavy for the pilot:* Gradio native - `auth=` with per-user credentials threaded through the existing - `launch(**kwargs)` seam. This is weaker (credential-based, not identity- - federated, easy to share) and is explicitly a stopgap, not the target. - -2. **Close the specialist bypass — set the specialist Space to private + a - service token.** The specialist becomes a **private** Space; the orchestrator - authenticates to it as a **service account** (HF token, the "inter-service - token" already named in the review) via `gradio_client(..., hf_token=...)`. - Direct public access is removed; only the orchestrator (holding the token) - can reach it. No shared human accounts (consistent with Auth Q6). - -3. **Identity into the audit trace.** Once a request carries an authenticated - principal, thread it into `get_trace()` (`config` block) so ADR-0008 traces - record *who* ran each analysis — the missing link between the app-layer trace - and identity events. (App trace still ≠ full IdP log; see ADR-0008's two-layer - note.) - -4. **Roles.** Two roles only: **user** (run analyses) and **admin** (the 1–2 - accounts that register uploads per ADR-0011 and deploy). Role is an - allow-list attribute in config, not a separate auth system. - -## Plan - -- **Now (AWS-independent, ~2–4 days):** - 1. Orchestrator: add HF OAuth + allow-list gate; deny non-listed identities. - 2. Specialist: flip Space visibility to private; orchestrator authenticates - with a service token; verify direct anonymous access is refused. - 3. Thread the authenticated username into the ADR-0008 trace `config`. - 4. Add an `ADMIN_IDS` / allow-list config block (both Spaces) + a test that a - non-listed identity is rejected and an admin-only action refuses a user. -- **At OHSU SSO/IdP (deferred — OHSU decision):** replace the HF-OAuth provider - with OHSU SSO (OIDC/SAML) behind the same allow-list/role seam; the app change - is the provider, not the gate. This is the only piece that waits on OHSU. - -## Implementation status (2026-07-02) — COMPLETE - -**Specialist (`DecoupleRpy_Agent`) — deployed to prod:** -- **`src/core/access_control.py`** — the pure, provider-independent identity seam - (plan step 4). `Principal`, `role_for` / `is_authorized` / `is_admin`, - `resolve_principal`, and `principal_trace_fields`. Config via `ADMIN_IDS` - (admins) + `ALLOWED_IDS` (users), with `UPLOAD_ADMIN_IDS` (ADR-0011) honored as - admins for back-compat so there is one coherent admin set. **Fail-closed:** - empty config denies everyone; an unknown/blank identity has no role. This is - the seam OHSU SSO later reuses unchanged — only where the identity string comes - from changes. -- **Identity into the trace** (plan step 3) — `CodeAgent.get_trace()` `config` - block carries `principal` + `role` via `principal_trace_fields`, sourced from - `self.principal`. A hidden `principal` input on `/interact_with_agent` receives - the identity the orchestrator forwards; no forwarded identity records - `principal="anonymous"`, `role=None` (honest, never misattributed). -- **Tests** — `tests/test_access_control.py` (17). - -**Orchestrator (`pdac-analysis-orchestrator`) — deployed to prod:** -- **`access.py`** — front-door allow-list gate mirroring the specialist's env - contract, with an `ACCESS_CONTROL` enforcement toggle (fail-closed when on) so - the code could ship to the public Space without locking anyone out before cutover. -- **`README.md`** `hf_oauth: true`; **`gradio_ui.py`** `gr.LoginButton` + - `gr.OAuthProfile` gate in `_respond`; **`router.py`** forwards the authenticated - username as `principal` to the specialist (only when non-empty). Service-token - client side (`gradio_client(token=HF_TOKEN)`) was already wired. -- **Tests** — `tests/test_access.py` (20); full suite 38 green. - -**Architecture note (asymmetric on purpose):** the **orchestrator stays public** and -is gated at the *app layer* by HF OAuth + the allow-list (a private Space would push -access back to the HF-visibility layer the ADR is moving away from, and make OAuth -redundant). The **specialist is private** because it has no login of its own and must -not be directly reachable — it is an internal service reached only by the -orchestrator's service token. - -**Validated end-to-end on prod (2026-07-02):** a listed user signs in and gets a real -analysis (proving gate-allow + orchestrator→private-specialist via token + identity -forwarding in one shot); an anonymous request to the specialist Space is refused -(Hub API 401 / app 404, calibrated against a known-public Space returning 200); the -allow-list refuses a non-listed / anonymous user at the front door. - -**Deferred (the only open item):** the OHSU SSO/IdP provider cutover — swap HF OAuth -for OHSU OIDC/SAML behind this same `access_control` / `access` seam. Provider swap, -not a redesign; waits on OHSU. - -## Consequences - -- **Positive:** turns the review's stated user/admin/service-account model into - an enforced control; every session gains a named identity that flows into the - audit trace; the specialist stops being an open bypass; the SSO cutover is - scoped to a provider swap. -- **Cost / caveat:** HF OAuth requires each user to have (or make) a HuggingFace - account — acceptable for ~5 pilot users, but call it out; it is HF-account - identity, **not** OHSU-managed identity, until the SSO cutover. The Gradio - `auth=` fallback is credential-based and must not be presented as identity - federation. Setting the specialist private means the orchestrator's service - token becomes a secret to manage (rotate; never in the trace — see ADR-0013). -- **Honesty note for the review:** until this ships, state plainly that access - control is currently at the HF-Space-visibility layer only, and that app-level - authentication is the funded next step — do not imply the user/admin model is - already enforced. - -## Start-now checklist — ALL DONE (2026-07-02) -- [x] Orchestrator: HF OAuth + allow-list gate (deny non-listed). *(`access.py`, - `README.md` `hf_oauth`, `gradio_ui.py` gate — prod)* -- [x] Specialist: private visibility + orchestrator service-token auth; anonymous - direct access verified refused. *(HF console + `router.py` `token=HF_TOKEN`)* -- [x] Authenticated username threaded into ADR-0008 trace. *(`get_trace()` `config`)* -- [x] `ADMIN_IDS`/allow-list config + rejection tests (user vs admin). - *(`src/core/access_control.py`+`tests/test_access_control.py`; - `access.py`+`tests/test_access.py`)* - -**Deferred (not part of the now-slice):** OHSU SSO/IdP provider cutover — tracked for -the AWS/OHSU-managed-identity milestone. diff --git a/docs/adr/ADR-0013-audit-trace-redaction.md b/docs/adr/ADR-0013-audit-trace-redaction.md deleted file mode 100644 index 61fb9788b0690babb3a6943aeca7b582648839f8..0000000000000000000000000000000000000000 --- a/docs/adr/ADR-0013-audit-trace-redaction.md +++ /dev/null @@ -1,125 +0,0 @@ -# ADR-0013 — Audit-Trace Redaction (Secret / Credential / PII Scrubbing) - -**Status:** Accepted — implemented 2026-07-02 (AWS-independent; live once deployed) -**Date:** 2026-07-02 -**Deciders:** Annie Voigt (project lead) -**Driver:** OHSU security review — the always-on audit trace (ADR-0008) captures -the full prompt and the code the agent generated and executed. Anything a user -pastes, or that generated code prints, is persisted **verbatim** to the trace -store. Since that store *is* the audit control, a secret or identifier landing in -it is a leak into the very artifact meant to be trusted. -**Related:** ADR-0008 (always-on logging — defines the persist path this hooks), -ADR-0009 (S3 sink — same redacted payload lands there at cutover), ADR-0011 -(upload de-identification attestation — dataset content; this ADR is about -secrets/PII in prompts+code, a different surface), ADR-0012 (identity in trace — -must itself not over-collect). - ---- - -## Context - -`CodeAgent.get_trace()` returns -`{execution_time, config, messages, trace_logs}`, where `messages` is the full -message history (user prompts + model turns) and `trace_logs` includes the -generated code and captured stdout. `agent.run()` persists this on every live -run via `persist_trace_safe(get_log_sink(), run_id, self.get_trace())`. There is -**no redaction** anywhere in `src/logging_sink.py` or in `get_trace()`. - -Realistic leak paths into the trace: -- A user pastes an API key, token, or a credentialed URL into the chat prompt. -- Generated code echoes the environment (`print(os.environ)`), a connection - string, or a bearer token. -- The ADR-0012 orchestrator→specialist **service token** appears in an error - string or a debug print. -- Personal identifiers (email, name) in a prompt — the data is de-identified per - ADR-0011, but free-text prompts are not. - -The fail-open wrapper (`persist_trace_safe`) means a bad trace is written -silently, so there is no natural backstop. - -## Decision - -Add a deterministic, **no-LLM** redaction pass applied to the trace immediately -before it is persisted, in both the always-on sink path and the opt-in -`save_trace` file dump. - -- **A pure function `redact_trace(trace: dict) -> dict`** (new - `src/core/trace_redaction.py`, mirroring the shared-helper pattern of - `src/core/integrity.py`). It deep-copies and walks all string values in - `messages` + `trace_logs` and replaces matches with a typed placeholder - (`«REDACTED:anthropic_key»`, `«REDACTED:hf_token»`, etc.). -- **Pattern set (deterministic regex, high-precision):** - - Anthropic keys (`sk-ant-…`), HuggingFace tokens (`hf_…`), AWS access keys - (`AKIA…`) + secret-key-shaped high-entropy strings, generic `Bearer `, - OpenAI-style `sk-…`, and credentialed URLs (`https://user:pass@…`). - - Whole-value drop for obvious environment dumps (a dict/text blob containing - multiple `KEY=VALUE` env lines) → `«REDACTED:env_dump»`. - - Email addresses → `«REDACTED:email»` (PII; conservative, on by default). -- **Applied at one seam.** Hook `redact_trace` into `agent.run()` right before - `persist_trace_safe(...)` and before the file dump — so every sink (`local` / - `hf` / `s3`) and every path receives the redacted payload. The sinks stay - dumb; redaction is not per-sink. -- **Fail-closed on redaction, fail-open on logging.** If `redact_trace` itself - raises, persist a **minimal** trace (run_id + timestamp + "redaction_error") - rather than the raw payload — never write an unredacted trace, but still never - crash the run. -- **Size cap.** Truncate any single value over a configurable limit - (`TRACE_MAX_FIELD_CHARS`) so a pathological paste can't bloat the store. -- **Config, allow tuning:** `TRACE_REDACTION` (`on` default | `off` for local - debug only) + an extensible extra-patterns list; **off is never the prod - posture** and that is documented. - -Redaction runs on a copy; the in-memory trace the UI/eval harness reads is -unchanged, so no user-facing behavior changes — only what is *persisted*. - -## Plan - -- **Now (AWS-independent, ~2–3 days):** - 1. `src/core/trace_redaction.py` with the pattern set + `redact_trace`. - 2. Wire it into `agent.run()` before both persist paths. - 3. Tests (`tests/test_trace_redaction.py`): each pattern is scrubbed; a - planted `sk-ant-…` / `hf_…` / env-dump never reaches a stub sink; redaction - failure yields the minimal trace, not the raw one; clean traces pass through - unchanged. - 4. Document `TRACE_REDACTION` + the "off ≠ prod" note. -- **No AWS dependency at all** — this hardens the payload *before* it reaches any - sink, so it is complete independent of the S3 cutover, and the S3 sink - (ADR-0009) inherits it for free. - -## Consequences - -- **Positive:** the audit store can no longer silently capture credentials/PII; - closes the leak into the trust artifact itself; a single seam covers all - sinks; deterministic + testable, no model call, no latency of note. -- **Cost / caveat:** regex redaction is high-precision but not exhaustive — a - novel secret format can slip through, so this is defense-in-depth layered with - ADR-0012 (don't put the service token where it can be printed) and secret - hygiene (ADR-0014), not a guarantee. Over-eager patterns could redact - legitimate content (e.g. a gene identifier that looks token-shaped); keep - patterns anchored/high-entropy and cover with tests. State to the review that - redaction is best-effort scrubbing, not a proof of secret-free logs. -- **Honesty note:** this reduces *accidental* capture; it is not a substitute for - not exposing secrets to the agent in the first place. - -## Start-now checklist -- [x] `src/core/trace_redaction.py` (`redact_trace`, pattern set, size cap). -- [x] Hook before `persist_trace_safe` (`agent.run()`) + the `save_trace` file - dump (`WorkflowEngine.save_trace_to_file`, which also covers `agent.save_trace()`). -- [x] Fail-closed minimal-trace path on redaction error (`redact_trace_safe`). -- [x] `tests/test_trace_redaction.py` (20 tests, network-free) + `TRACE_REDACTION` - config doc (module docstring + Decision above). - -## Implementation notes (2026-07-02) -- Patterns landed: anthropic (`sk-ant-…`), hf (`hf_…`), AWS access key (`AKIA…`), - generic OpenAI `sk-…`, `Bearer …`, GitHub `gh[pousr]_…`, credentialed URL - (host preserved, `user:pass@` dropped), email, whole-value env-dump drop - (≥3 `UPPER_SNAKE=value` lines), and an entropy-gated 40+char base64 run for the - AWS *secret* key shape (`TRACE_REDACTION_EXTRA` adds operator regexes → - `«REDACTED:custom»`). -- Entropy gate (≥4.0 bits/char) on the 40+char matcher keeps repetitive/low-entropy - identifiers (e.g. a long gene-id run) from tripping the secret pattern. -- Size cap is `TRACE_MAX_FIELD_CHARS` (default 20 000); scrubbing runs *before* - truncation so a secret straddling the boundary is removed, not half-exposed. -- Both seams call `redact_trace_safe`, which fails **closed** to a minimal trace - (`run_id` + timestamp + `redaction_error`, no payload) and never raises, so the - fail-open sink wrapper still governs crash-safety. diff --git a/docs/adr/ADR-0014-ci-security-scanning.md b/docs/adr/ADR-0014-ci-security-scanning.md deleted file mode 100644 index 272fd730f7e39a52b7899ac0af7034c4ce3460e3..0000000000000000000000000000000000000000 --- a/docs/adr/ADR-0014-ci-security-scanning.md +++ /dev/null @@ -1,150 +0,0 @@ -# ADR-0014 — CI Security Scanning (Dependency, Static, Secret) - -**Status:** Accepted — in progress (specialist repo implemented 2026-07-02; -biodata-registry + orchestrator pending) -**Date:** 2026-07-02 -**Deciders:** Annie Voigt (project lead) -**Driver:** OHSU security review — no automated scanning exists today. There is -no `.github/workflows/` in the specialist, and no pip-audit / bandit / safety / -Dependabot / gitleaks anywhere in the three repos. The `tzlocal==5.4.2` -metadata-only-wheel break (found by hand during ADR-0007 validation, fixed -reactively in `05fca82`) is exactly the supply-chain class automated scanning -catches before it ships. -**Related:** ADR-0010 (dataset integrity — data supply chain; this is the *code* -supply chain), ADR-0012/0013 (secret hygiene — secret scanning here backstops -them), ADR-0007 (pinned sandbox image — scan the image too). - ---- - -## Context - -Three repos, three different deploy models, which changes where CI can run: - -- **`DecoupleRpy_Agent`** (specialist) — **has a GitHub repo** that auto-deploys - to the HF Space. GitHub Actions runs natively here. -- **`biodata-registry`** — **GitHub-only** pip package. GitHub Actions runs - natively. -- **`pdac-analysis-orchestrator`** — **no GitHub repo**; `origin` *is* the prod - HF Space (git push builds the Space). GitHub Actions has nowhere to run, so it - needs a different hook (pre-push + manual/scheduled scan). - -None of them run any dependency-vulnerability scan, static-analysis pass, or -secret scan. Dependencies are pinned (`requirements.txt`, `tests/ -requirements-test.txt`) but never checked against advisory databases, and -tokens flow through the code (HF write tokens, the ADR-0012 service token) with -no automated secret-scanning of commits or history. - -## Decision - -Add a standard three-part security scan — **dependencies, static analysis, -secrets** — to every repo, using the hook appropriate to that repo's deploy -model. All tooling is open-source and runs locally / in GitHub-hosted CI; **no -AWS and no OHSU decision required.** - -**Scan set (same three everywhere):** -1. **Dependencies — `pip-audit`** against `requirements*.txt` (PyPI advisory / - OSV). Fails on a known-vuln dependency; this is what would have surfaced a - bad `tzlocal` pin. Add **Dependabot** (GitHub repos) for automated bump PRs. -2. **Static analysis — `bandit`** over `src/` (common Python security - anti-patterns: `exec`/`eval`, `subprocess shell=True`, insecure temp files, - hardcoded secrets). Scoped/tuned so the *intended* sandboxed `exec` - (ADR-0007) is an acknowledged, annotated finding, not noise. -3. **Secrets — `gitleaks`** over the working tree **and full git history** - (tokens have flowed through these repos; history matters). Backstops - ADR-0012/0013. - -**Per-repo hook (deploy-model-aware):** -- **`DecoupleRpy_Agent` + `biodata-registry` (GitHub):** a - `.github/workflows/security.yml` running the three on every PR + a weekly - schedule; Dependabot config committed. -- **`pdac-analysis-orchestrator` (HF-only, no GitHub):** a `make security-scan` - target + a **pre-push git hook** running the same three locally before a push - builds the Space, plus optionally a **scheduled HF Job** (reusing the - `scripts/hf_job.sh` pattern already in `lit-agent`) so it also runs - unattended. The scan definition is shared so all repos run an identical check. -- **Shared config** so thresholds/allow-lists (e.g. the annotated ADR-0007 - `exec`) don't drift between repos. - -**Also scan the sandbox image (ADR-0007):** add a container-image vuln scan -(`trivy`, open-source) of `docker/sandbox.Dockerfile` to the specialist workflow -so the pinned image is checked, not just the Python deps. - -## Plan - -- **Now (AWS-independent, ~2–3 days):** - 1. Author the shared three-scan definition + a documented allow-list for known - accepted findings (the ADR-0007 `exec`). - 2. GitHub repos: commit `security.yml` (PR + weekly) + Dependabot config. - 3. Orchestrator: `make security-scan` + pre-push hook (+ optional scheduled HF - Job). - 4. Run once across all three, triage findings, fix or explicitly accept each, - and record the accepted set (so CI is green and every suppression is - justified — a clean artifact for the review). - 5. Add `trivy` on the sandbox image to the specialist workflow. -- **No AWS dependency.** Everything here is local tooling or GitHub-hosted - runners; nothing waits on the AWS account or an OHSU decision. - -## Consequences - -- **Positive:** turns "we pin dependencies" into "we pin *and* continuously - scan"; catches the next `tzlocal`-class break before deploy; secret scanning - backstops ADR-0012/0013; gives the review a concrete, always-on supply-chain - control across all three repos and the sandbox image. -- **Cost / caveat:** the orchestrator's HF-only model means its scan is a - pre-push hook / scheduled Job rather than blocking CI — a developer *can* - bypass a local hook, so pair it with the scheduled unattended run and say so - honestly (it is not an enforced merge gate the way the GitHub repos are). - Initial triage will surface a backlog to accept or fix; budget for that first - pass. Scanners produce false positives — the annotated allow-list keeps CI - meaningful rather than ignored. -- **Honesty note:** scanning reduces known-vulnerability and leaked-secret risk; - it does not prove the absence of either. It is one layer with ADR-0010 (data - supply chain), ADR-0012/0013 (secret handling), and ADR-0007 (execution - isolation). - -## Start-now checklist -- [x] Shared three-scan definition (pip-audit + bandit + gitleaks) + accepted- - findings allow-list. — `scripts/security_scan.sh` (4 stages incl. trivy), - `bandit.yaml` + `security/bandit-baseline.json`, `.gitleaks.toml`, - `security/ACCEPTED-FINDINGS.md`. -- [x] `security.yml` + Dependabot on `DecoupleRpy_Agent`. *(biodata-registry - still pending — separate repo; it reuses the same scan definition.)* -- [x] `make security-scan` + pre-push hook on the specialist. *(See deploy-model - correction below — the specialist is HF-only, so this local/pre-push path - is its enforced check, same as the orchestrator; the orchestrator itself - still pending.)* -- [x] `trivy` image scan on the sandbox Dockerfile. — clean (0 HIGH/CRITICAL). -- [~] First full run triaged; every suppression justified. — **done; CI is NOT - green because the first run surfaced two genuine open items** (below), - which is the intended outcome of a first pass, not a failure of it. - -## Implementation notes (2026-07-02, specialist repo) - -**Deploy-model correction.** The Context above states the specialist "has a -GitHub repo" where GitHub Actions runs natively. That is **not** true of the -current repo: `origin` *is* the HuggingFace Space (no GitHub remote), so -`.github/workflows/security.yml` will not execute on push — HF does not run -Actions. The specialist therefore has the **same** deploy model as the -orchestrator, and its enforced check is the **pre-push hook** (`make -install-hooks`) running the identical `scripts/security_scan.sh`, plus the -scheduled unattended run. The workflow + `dependabot.yml` are still committed so -that adding a GitHub mirror (or reusing them in `biodata-registry`, a real -GitHub repo) is zero extra work and the scan definition never diverges. - -**First-run findings (see `security/ACCEPTED-FINDINGS.md` for full triage):** -1. **Exposed Google API key in git history** — `AIzaSy…` hardcoded as a Gradio - textbox default in the initial commit (`155d8d7` `app.py:488`), removed from - HEAD in `acf6553` but still in published history. **Requires rotation** - (revoke in GCP console); left intentionally un-allowlisted so the scan keeps - failing until handled. This is exactly the history-scan value the ADR argued - for. -2. **Three fixable dependency CVEs** — `pillow 11.3.0` (→12.2.0), `langsmith - 0.8.16` (→0.8.18), `pydantic-settings 2.14.1` (→2.14.2). None are HF - `sdk_version`-locked, so all three are bump candidates (a separate, - test-gated dependency change + redeploy). - -Two gitleaks false positives (column-name kwargs in a precompute script) were -triaged and allowlisted. Bandit's 16 medium infra findings (0.0.0.0 dev bind, -`/tmp` working dirs, GEO-download urlopen) are captured in the committed baseline -so only *new* findings fail; the ADR-0007 sandboxed `exec` is inline-`# nosec` -annotated. diff --git a/gradio_ui.py b/gradio_ui.py index f72d5961ba9ea2342e819a7d38296eeaa47475b1..b6ef7fd806fcb2a89e8c6418c0eb8eccecce28c3 100644 --- a/gradio_ui.py +++ b/gradio_ui.py @@ -14,20 +14,19 @@ import sys import threading import time import traceback -from collections.abc import Generator from datetime import datetime +from typing import Generator import gradio as gr from gradio.themes.utils import fonts +from langchain_core.messages import HumanMessage, AIMessage from langchain_anthropic import ChatAnthropic -from langchain_core.messages import AIMessage, HumanMessage from agent import CodeAgent -from core.access_control import check_access from core.constants import DECOUPLER_DISCLAIMER from core.types import AgentConfig -from logging_sink import get_log_sink, persist_trace_safe from managers.hf_storage import HFResultsStorage +from logging_sink import get_log_sink, persist_trace_safe from ui_formatting import _UIFormattingMixin # --------------------------------------------------------------------------- @@ -37,9 +36,6 @@ from ui_formatting import _UIFormattingMixin _MCP_HTTP_PORT = 8765 _mcp_server_proc: subprocess.Popen | None = None -# Steps granted per press of the Continue button. -STEP_LIMIT_INCREMENT = 15 - def _port_open(port: int, host: str = "127.0.0.1") -> bool: try: @@ -104,494 +100,18 @@ def ensure_mcp_http_server() -> str: # port before committing to HTTP-vs-stdio. Catch a fast crash early. for i in range(15): if _mcp_server_proc.poll() is not None: - print( - f"[MCP] Server process exited early (code {_mcp_server_proc.returncode}) — " - "see logs above; prewarm will fall back to stdio" - ) + print(f"[MCP] Server process exited early (code {_mcp_server_proc.returncode}) — " + "see logs above; prewarm will fall back to stdio") return url if _port_open(_MCP_HTTP_PORT): - print(f"[MCP] HTTP server ready at {url} (took {i + 1}s)") + print(f"[MCP] HTTP server ready at {url} (took {i+1}s)") return url time.sleep(1) - print("[MCP] HTTP server not up in 15s — prewarm will keep waiting in the background") + print(f"[MCP] HTTP server not up in 15s — prewarm will keep waiting in the background") return url -# --------------------------------------------------------------------------- -# Upload safety gate (ADR-0011) — the UI wiring in front of src/uploads/. -# -# An uploaded/downloaded file is untrusted input. Before it is ever handed to -# the agent it must clear the *security* envelope of ADR-0011: -# 1. de-identification attestation (no PHI/PSI), -# 2. type allow-list + size cap + quarantine + SHA-256 (stage_upload), -# 3. structural magic-byte content scan (scan_upload) + an AV pass only where -# the deployment declares one (deploy/scan_posture.yaml; the Space does not). -# Those three are MANDATORY — a failure blocks the file entirely. -# -# validate_upload (manifest-vs-data consistency) is run too, but treated as -# ADVISORY for this session-scoped, single-file, read-only use: an ad-hoc -# uploader has no hand-authored manifest, so we auto-draft one and surface any -# validation gaps as warnings rather than blocking exploratory analysis. The -# admin-only registration step (promotion into the shared biodata-registry) is -# a separate governance action and is intentionally NOT part of this path — -# a session upload is read by the agent's vetted loaders for that session only, -# never added to the registry or exposed to other users. -# --------------------------------------------------------------------------- -def run_upload_gate(src_path, filename, session_state, deidentified): - """Route an upload/download through the ADR-0011 safety gate. - - Returns ``(session_state, status_markdown)``. On any *security* failure the - file is NOT made available (``uploaded_file`` stays unset). On a - manifest-validation-only gap the file is allowed for session use with a - visible warning. - """ - from src.uploads import ( - SCAN_INFECTED, - UploadRejected, - attach_manifest, - discard_upload, - draft_manifest, - placeholder_manifest, - scan_upload, - stage_upload, - validate_upload, - ) - from src.uploads.drafting import _slug - from src.uploads.records import STATUS_VALIDATED - - def _block(msg): - session_state.pop("uploaded_file", None) - session_state.pop("uploaded_filename", None) - session_state.pop("upload_record_id", None) - return session_state, msg - - # ── Gate 1: de-identification attestation (ADR-0011 item 6) ────────────── - if not deidentified: - return _block( - "⚠️ **Upload blocked.** Tick *“I confirm this data is de-identified" - " (no PHI/PSI)”* above before uploading — an upload without that" - " attestation is refused at the door (ADR-0011)." - ) - - uploader = session_state.get("principal") or "ui-upload" - dataset_id = f"upload_{_slug(filename)}" - - # A metadata workbook is not an analysable dataset on its own — it only has - # meaning joined to a counts matrix. Say so here rather than letting it - # quarantine successfully and then fail in the manifest drafter. - if filename.lower().endswith(".xlsx"): - return _block( - "⚠️ **An Excel workbook is metadata, not a dataset.** Use the" - " *Assemble from TSVs* tab, where the sheet is joined to a counts" - " matrix to build the analysis file." - ) - - # ── Gate 2: quarantine + type/size + SHA-256 (stage_upload) ────────────── - # Staged with a placeholder manifest, because drafting the real one means - # *parsing* the file, and ADR-0011 requires that no parser touch an - # unstaged, unscanned, un-type-checked file. The draft is attached below, - # after the content scan clears. - try: - record = stage_upload( - src_path, - uploader=uploader, - dataset_id=dataset_id, - manifest=placeholder_manifest(dataset_id), - deidentified=True, - ) - except UploadRejected as rej: - return _block("⚠️ **Upload rejected:** " + "; ".join(rej.record.errors)) - - # ── Gate 3: content scan (structural magic-byte + optional ClamAV) ─────── - # Runs on the quarantined copy *before* anything reads its contents, so a - # disguised binary (an ELF wearing a .csv suffix) is stopped here rather - # than surfacing later as an incidental parser crash. - record, _scan_report = scan_upload(record) - if record.scan_status == SCAN_INFECTED: - discard_upload(record, "blocked by content scan") - return _block("🛑 **Upload blocked by content scan:** " + record.scan_detail) - - # ── Auto-draft a manifest so the uploader faces no blank form ──────────── - # Reads the *quarantined* copy through the vetted loaders, never the original. - try: - draft = draft_manifest(record.staged_path, dataset_id=dataset_id) - except Exception as exc: # noqa: BLE001 — unreadable/unsupported file - discard_upload(record, f"could not draft a manifest: {exc}") - return _block(f"⚠️ **Upload blocked:** could not read `{filename}` — {exc}") - record = attach_manifest(record, draft.manifest) - - # ── Advisory: manifest-vs-data validation ─────────────────────────────── - # validate_upload re-checks scan status internally (idempotent) then runs the - # manifest/against-data gate. A validation gap does not block session use. - record, _val_report = validate_upload(record) - - session_state["uploaded_file"] = record.staged_path - session_state["uploaded_filename"] = filename - session_state["upload_record_id"] = record.upload_id - - lines = [f"✅ **Ready:** `{filename}`"] - if record.scan_status == "skipped": - lines.append( - "_Structurally checked, but **not malware-scanned** — this deployment" - " ships no anti-virus binary (declared posture `structural_only`, see" - " `deploy/scan_posture.yaml`). Treat this file as structurally-verified" - " only._" - ) - elif record.scan_status == "clean" and record.scan_posture == "structural_only": - lines.append( - "_Structurally checked; an anti-virus on this host also reported clean," - " but the deployment does not guarantee an AV pass (declared posture" - " `structural_only`) — production uploads are structure-checked only._" - ) - if record.status != STATUS_VALIDATED and record.errors: - lines.append( - "_Manifest not fully validated (auto-drafted): " - + "; ".join(record.errors) - + ". The agent will still load the file; confirm the analysis matches" - " your data._" - ) - elif record.caveats: - lines.append("_Caveats:_ " + "; ".join(record.caveats)) - return session_state, "\n\n".join(lines) - - -# --------------------------------------------------------------------------- -# Data-Input handlers (module level so they are unit-testable — they close over -# no UI state). All three funnel into run_upload_gate above. -# -# The URL and HF-Dataset paths must first materialise remote bytes on local disk -# before the gate can hash and quarantine them. That scratch copy is *ungated* -# untrusted input, so it goes to a private temporary directory that is deleted -# unconditionally once the gate has taken its own quarantined copy — it is never -# left behind in a persistent tmp/inputs/ dir for the agent to stumble onto. -# --------------------------------------------------------------------------- -def _incomplete_run_notice(reason: str | None, config) -> str: - """Explain why a run ended without an answer, and what actually helps. - - A run can end incomplete four ways, and only ONE of them is fixed by - granting more steps. The UI used to call all of them "Step limit reached", - which sent users to press Continue against a wall clock or a failing tool — - Continue would grant steps a timed-out run has no use for. - """ - step_budget = getattr(config, "max_steps", 15) - timeout_min = round(getattr(config, "timeout_seconds", 1200) / 60) - - if reason == "step_limit": - body = ( - f"⏸ Step limit reached ({step_budget} steps). " - f"Click Continue to give the agent {STEP_LIMIT_INCREMENT} more steps." - ) - elif reason == "timeout": - body = ( - f"⏱ Time limit reached (~{timeout_min} min). The agent was still " - "working, so Continue will not help — it grants more steps, not more " - "time. Re-run with a narrower question (one dataset, one contrast), or raise the " - "timeout if this analysis is genuinely long-running." - ) - elif reason == "error_limit": - body = ( - "⚠️ Stopped after repeated errors. The agent hit " - f"{getattr(config, 'retry_attempts', 3)} consecutive failures and gave up rather than " - "guess. The errors are shown above — Continue is unlikely to help until the underlying " - "failure is addressed." - ) - else: - body = ( - "⚠️ The run ended without an answer. The agent stopped without " - "producing a solution or any further code to run. Try rephrasing the question." - ) - - return ( - '
' - f"{body}
" - ) - - -def _clear_upload(session_state, msg): - session_state.pop("uploaded_file", None) - session_state.pop("uploaded_filename", None) - session_state.pop("upload_record_id", None) - return session_state, msg - - -def _gate_component_file(src_path, filename, uploader, dataset_id): - """Quarantine + content-scan ONE input file of a multi-file assembly. - - Returns the staged path. The component files (counts TSV, TPM TSV, metadata - workbook) are not datasets in their own right — no manifest is drafted and - no validation runs for them; the assembled h5ad goes through the full gate - afterwards. What matters here is that nothing reads a byte of them until - :func:`stage_upload` has type/size-checked and hashed the file and - :func:`scan_upload` has cleared its structure. - - Raises ``ValueError`` with a user-facing message on any gate failure. - """ - from src.uploads import ( - SCAN_INFECTED, - UploadRejected, - discard_upload, - placeholder_manifest, - scan_upload, - stage_upload, - ) - - try: - record = stage_upload( - src_path, - uploader=uploader, - dataset_id=dataset_id, - manifest=placeholder_manifest(dataset_id), - deidentified=True, - ) - except UploadRejected as rej: - raise ValueError(f"`{filename}` rejected: " + "; ".join(rej.record.errors)) from rej - - record, _report = scan_upload(record) - if record.scan_status == SCAN_INFECTED: - discard_upload(record, "blocked by content scan") - raise ValueError(f"`{filename}` blocked by content scan: {record.scan_detail}") - return record.staged_path - - -def run_assembly_gate( - counts_path, - metadata_path, - tpm_path, - session_state, - deidentified, - *, - sample_column=None, - skip_rows=None, - column_map=None, - value_maps=None, - group_column=None, - control_label=None, - treatment_label="shMyc", -): - """Build the analysis h5ad from raw delivery files, all inside the gate. - - Replaces the hand-run `scripts/assemble_myc_kd_kmc_mouse.py` pre-step: the - user drops the counts TSV (+ optional TPM TSV) and the metadata sheet, and - the assembly happens here. Each input is quarantined and content-scanned - *before* it is parsed; the assembled h5ad is then put through the ordinary - upload gate, so the session-visible file is a normal staged upload with its - own record, SHA-256, drafted manifest and advisory validation. - """ - import shutil as _shutil - import tempfile - - from src.uploads.assembly import AssemblyError, assemble_h5ad - from src.uploads.drafting import _slug - - def _block(msg): - session_state.pop("uploaded_file", None) - session_state.pop("uploaded_filename", None) - session_state.pop("upload_record_id", None) - return session_state, msg - - if not deidentified: - return _block( - "⚠️ **Assembly blocked.** Tick *“I confirm this data is de-identified" - " (no PHI/PSI)”* above first — an upload without that attestation is" - " refused at the door (ADR-0011)." - ) - if not counts_path or not metadata_path: - return _block("Provide at least a counts matrix **and** a metadata sheet.") - - uploader = session_state.get("principal") or "ui-upload" - base_id = f"upload_{_slug(os.path.basename(counts_path))}" - - try: - staged_counts = _gate_component_file( - counts_path, os.path.basename(counts_path), uploader, f"{base_id}_counts" - ) - staged_meta = _gate_component_file( - metadata_path, os.path.basename(metadata_path), uploader, f"{base_id}_metadata" - ) - staged_tpm = ( - _gate_component_file(tpm_path, os.path.basename(tpm_path), uploader, f"{base_id}_tpm") - if tpm_path - else None - ) - except ValueError as exc: - return _block(f"🛑 **Assembly blocked:** {exc}") - - workdir = tempfile.mkdtemp(prefix="assembled_") - try: - out_path = os.path.join(workdir, "assembled.h5ad") - try: - _adata, report = assemble_h5ad( - staged_counts, - staged_meta, - tpm_path=staged_tpm, - out_path=out_path, - sample_column=sample_column or None, - skip_rows=int(skip_rows) if str(skip_rows or "").strip() else None, - column_map=column_map or None, - value_maps=value_maps or None, - group_column=group_column or None, - control_label=control_label or None, - treatment_label=treatment_label or "shMyc", - staging_script="gradio_ui.run_assembly_gate", - ) - except AssemblyError as exc: - return _block(f"⚠️ **Could not assemble the dataset:** {exc}") - except Exception as exc: # noqa: BLE001 — unreadable/unsupported input - return _block(f"⚠️ **Could not read the supplied files:** {exc}") - - session_state, status = run_upload_gate( - out_path, "assembled.h5ad", session_state, deidentified - ) - finally: - _shutil.rmtree(workdir, ignore_errors=True) - - if "uploaded_file" not in session_state: - return session_state, status - - detail = [ - status, - f"_Assembled **{report['n_samples']} samples × {report['n_genes']} genes**" - f" from `{os.path.basename(counts_path)}`" - + (f" + `{os.path.basename(tpm_path)}` (TPM layer)" if tpm_path else "") - + f" + `{os.path.basename(metadata_path)}`._", - "_Groups:_ " - + "; ".join(f"**{col}** {counts}" for col, counts in report["obs_counts"].items()), - ] - if report["unmatched_samples"]: - detail.append( - f"_⚠️ {len(report['unmatched_samples'])} matrix sample(s) had no metadata row and" - f" were dropped: {', '.join(report['unmatched_samples'])}._" - ) - return session_state, "\n\n".join(detail) - - -def handle_assembly( - counts_file, - metadata_file, - tpm_file, - sample_column, - column_map, - value_maps, - group_column, - control_label, - treatment_label, - skip_rows, - deidentified, - session_state, -): - """Gradio binding for the *Assemble from TSVs* tab.""" - return run_assembly_gate( - counts_file, - metadata_file, - tpm_file, - session_state, - deidentified, - sample_column=sample_column, - skip_rows=skip_rows, - column_map=column_map, - value_maps=value_maps, - group_column=group_column, - control_label=control_label, - treatment_label=treatment_label, - ) - - -def handle_file_upload(file_path, deidentified, session_state): - """Gradio's own upload widget already wrote the file to a temp path.""" - if file_path is None: - return _clear_upload(session_state, "") - return run_upload_gate(file_path, os.path.basename(file_path), session_state, deidentified) - - -def handle_url_download(url, deidentified, session_state): - import gzip - import shutil as _shutil - import tempfile - - import requests - - from src.uploads.staging import _max_upload_bytes - - if not url or not url.strip(): - return session_state, "No URL provided" - url = url.strip() - if not url.lower().startswith(("http://", "https://")): - return _clear_upload(session_state, "Only http:// and https:// URLs are supported.") - filename = url.rstrip("/").split("/")[-1].split("?")[0] - if not filename or "." not in filename: - filename = "downloaded_data.bin" - - scratch = tempfile.mkdtemp(prefix="ungated_url_") - try: - dest = os.path.join(scratch, os.path.basename(filename)) - limit = _max_upload_bytes() - r = requests.get(url, stream=True, timeout=300) - r.raise_for_status() - if "text/html" in r.headers.get("Content-Type", ""): - return _clear_upload( - session_state, - "URL returned an HTML page, not a file. Make sure the URL points" - " directly to a file, not a directory.", - ) - # Enforce the size cap *while* streaming, so an oversized (or endless) - # response is abandoned rather than fully written and rejected after. - written = 0 - with open(dest, "wb") as f: - for chunk in r.iter_content(chunk_size=8192): - if not chunk: - continue - written += len(chunk) - if written > limit: - return _clear_upload( - session_state, - f"⚠️ **Download aborted:** the file exceeds the" - f" {limit}-byte upload limit (UPLOAD_MAX_BYTES).", - ) - f.write(chunk) - if filename.endswith(".gz") and not filename.endswith(".tar.gz"): - decompressed = filename[:-3] - decompressed_dest = os.path.join(scratch, decompressed) - with gzip.open(dest, "rb") as f_in, open(decompressed_dest, "wb") as f_out: - _shutil.copyfileobj(f_in, f_out) - os.remove(dest) - dest, filename = decompressed_dest, decompressed - return run_upload_gate(dest, filename, session_state, deidentified) - except Exception as e: # noqa: BLE001 — surfaced to the uploader - return _clear_upload(session_state, f"Download failed: {e}") - finally: - _shutil.rmtree(scratch, ignore_errors=True) - - -def handle_hf_dataset(repo_id, filepath, deidentified, session_state): - import shutil as _shutil - import tempfile - - if not repo_id or not repo_id.strip(): - return session_state, "No repo ID provided" - if not filepath or not filepath.strip(): - return session_state, "No file path provided" - - scratch = tempfile.mkdtemp(prefix="ungated_hf_") - try: - from huggingface_hub import hf_hub_download - - local_path = hf_hub_download( - repo_id=repo_id.strip(), - filename=filepath.strip(), - repo_type="dataset", - local_dir=scratch, - ) - return run_upload_gate( - local_path, os.path.basename(local_path), session_state, deidentified - ) - except Exception as e: # noqa: BLE001 — surfaced to the uploader - return _clear_upload(session_state, f"Failed to load from HF Dataset: {e}") - finally: - _shutil.rmtree(scratch, ignore_errors=True) - - class GradioAgentUI(_UIFormattingMixin): """ Gradio interface for interacting with the LangGraph ReAct Agent. @@ -603,8 +123,9 @@ class GradioAgentUI(_UIFormattingMixin): if model is None: # Prefer the standard ANTHROPIC_API_KEY; fall back to the legacy # mixed-case name some older Spaces still use. - api_key_anthropic = os.environ.get("ANTHROPIC_API_KEY") or os.environ.get( - "Anthropic_API_KEY" + api_key_anthropic = ( + os.environ.get("ANTHROPIC_API_KEY") + or os.environ.get("Anthropic_API_KEY") ) if not api_key_anthropic: raise RuntimeError( @@ -612,11 +133,18 @@ class GradioAgentUI(_UIFormattingMixin): "or Anthropic_API_KEY in the Space secrets." ) model = ChatAnthropic( - model="claude-sonnet-4-6", temperature=0, api_key=api_key_anthropic + model='claude-sonnet-4-6', + temperature=0, + api_key=api_key_anthropic ) if config is None: - config = AgentConfig(max_steps=15, retry_attempts=3, timeout_seconds=2700, verbose=True) + config = AgentConfig( + max_steps=15, + retry_attempts=3, + timeout_seconds=1200, + verbose=True + ) self.model = model self.config = config @@ -632,7 +160,6 @@ class GradioAgentUI(_UIFormattingMixin): except Exception as e: print(f"[log_sink] init failed ({e}); falling back to local sink") from logging_sink import LocalLogSink - self.log_sink = LocalLogSink() print(f"[log_sink] Using sink: {self.log_sink.name}") @@ -640,7 +167,7 @@ class GradioAgentUI(_UIFormattingMixin): # doesn't pay the 2-minute subprocess startup cost. # The seed agent's mcp_functions (stateless closures) are cached and # copied into each new session agent without re-spawning anything. - self._mcp_cache: dict = {} # tool_name → mcp_functions entry + self._mcp_cache: dict = {} # tool_name → mcp_functions entry self._mcp_discovery_errors: dict = {} # server_name -> error string, if discovery failed self._mcp_ready = threading.Event() # set when pre-warm finishes @@ -727,10 +254,8 @@ class GradioAgentUI(_UIFormattingMixin): if not seed.tool_manager.mcp_manager.mcp_functions: # HTTP discovery returned nothing — fall back to stdio so the # Space is still usable (just slower) rather than tool-less. - print( - "[mcp-prewarm] ⚠️ HTTP discovery returned 0 tools — " - "falling back to stdio add_mcp()" - ) + print("[mcp-prewarm] ⚠️ HTTP discovery returned 0 tools — " + "falling back to stdio add_mcp()") self._mcp_http_url = None self._probe_stdio_server() seed.add_mcp(mcp_config_path) @@ -751,17 +276,13 @@ class GradioAgentUI(_UIFormattingMixin): # add_mcp() didn't raise, but discovered zero tools — this is # just as broken as an exception (agent will have no analysis # tools), so log it just as loudly. - print( - "[mcp-prewarm] ⚠️ add_mcp() completed but discovered 0 MCP " - "tools — analysis requests will fail. Check server.py / " - "mcp_config.yaml on this Space. " - f"Discovery errors: {self._mcp_discovery_errors}" - ) + print("[mcp-prewarm] ⚠️ add_mcp() completed but discovered 0 MCP " + "tools — analysis requests will fail. Check server.py / " + "mcp_config.yaml on this Space. " + f"Discovery errors: {self._mcp_discovery_errors}") except Exception as e: - print( - "[mcp-prewarm] ⚠️ Pre-warm failed — will retry discovery on " - "first query. Full traceback:" - ) + print("[mcp-prewarm] ⚠️ Pre-warm failed — will retry discovery on " + "first query. Full traceback:") traceback.print_exc() self._mcp_discovery_errors["_prewarm"] = f"{type(e).__name__}: {e}" finally: @@ -828,8 +349,8 @@ class GradioAgentUI(_UIFormattingMixin): previous_plan = None step_timings = {} last_state = None - last_error = None # buffer transient step errors; only the last is - solution_shown = False # surfaced, and only if the run yields no solution + last_error = None # buffer transient step errors; only the last is + solution_shown = False # surfaced, and only if the run yields no solution # Monotonic step numbering for the UI, decoupled from the raw graph # step_count. The raw count is emitted twice per step (once for the @@ -839,10 +360,11 @@ class GradioAgentUI(_UIFormattingMixin): # the prior run on /handle_continue so the continuation keeps counting # up; reset to 0 for a fresh question. display_step = ( - getattr(agent.workflow_engine, "last_display_step", 0) if resume_messages else 0 + getattr(agent.workflow_engine, "last_display_step", 0) + if resume_messages else 0 ) - last_internal_step = None # raw step_count we last opened a header for - header_pending = False # a step started; flush its header before content + last_internal_step = None # raw step_count we last opened a header for + header_pending = False # a step started; flush its header before content header_drawn = bool(resume_messages) # leading
before the first step? # Snapshot existing figures so we can detect ones this run produces and @@ -876,29 +398,24 @@ class GradioAgentUI(_UIFormattingMixin): # header is gated on it, so a deduped/empty yield no longer # leaves an orphan "Step N" with no body beneath it. thinking_text = "" - if parts["thinking"] and len(parts["thinking"]) > 20: - _t = re.sub(r"(Thinking:|Plan:)\s*", "", parts["thinking"]).strip() - _t = re.sub(r"\n\s*\n\s*\n+", "\n\n", _t).strip() - _t = "\n".join(ln.strip() for ln in _t.split("\n") if ln.strip()) + if parts['thinking'] and len(parts['thinking']) > 20: + _t = re.sub(r'(Thinking:|Plan:)\s*', '', parts['thinking']).strip() + _t = re.sub(r'\n\s*\n\s*\n+', '\n\n', _t).strip() + _t = '\n'.join(ln.strip() for ln in _t.split('\n') if ln.strip()) if _t and hash(_t) not in displayed_reasoning: thinking_text = _t # Route an errored observation to the buffered-error path before # it can count as renderable content (see the long note below). - if parts["observation"]: - _obs = re.sub(r"^\s*Code Output:\s*", "", parts["observation"]).strip() - if _obs.startswith("Error:"): + if parts['observation']: + _obs = re.sub(r'^\s*Code Output:\s*', '', parts['observation']).strip() + if _obs.startswith('Error:'): last_error = _obs - parts["observation"] = None + parts['observation'] = None plan_changed = bool(current_plan and current_plan != previous_plan) - has_content = bool( - thinking_text - or plan_changed - or parts["code"] - or parts["observation"] - or parts["solution"] - ) + has_content = bool(thinking_text or plan_changed or parts['code'] + or parts['observation'] or parts['solution']) # One header per logical step. graph.stream emits a state after # both the generate node (step N) and the execute node (still @@ -916,15 +433,13 @@ class GradioAgentUI(_UIFormattingMixin): display_step += 1 separator = ( '
' - if header_drawn - else "" + 'margin: 20px 0;">' if header_drawn else '' ) header_drawn = True yield gr.ChatMessage( role="assistant", content=f"{separator}## Step {display_step}\n", - metadata={"status": "done"}, + metadata={"status": "done"} ) if thinking_text: @@ -933,35 +448,37 @@ class GradioAgentUI(_UIFormattingMixin):
{thinking_text}
""" yield gr.ChatMessage( - role="assistant", content=thinking_block, metadata={"status": "done"} + role="assistant", + content=thinking_block, + metadata={"status": "done"} ) displayed_reasoning.add(hash(thinking_text)) if current_plan and current_plan != previous_plan: - formatted_plan = ( - current_plan.replace("[ ]", "☐") - .replace("[✓]", "✅") - .replace("[✗]", "❌") - ) + formatted_plan = current_plan.replace('[ ]', '☐').replace('[✓]', '✅').replace('[✗]', '❌') plan_block = f"""
📋 Current Plan
{formatted_plan}
""" yield gr.ChatMessage( - role="assistant", content=plan_block, metadata={"status": "done"} + role="assistant", + content=plan_block, + metadata={"status": "done"} ) previous_plan = current_plan - if parts["code"]: + if parts['code']: code_block = f"""
⚡ Executing Code
```python -{parts["code"]} +{parts['code']} ```""" yield gr.ChatMessage( - role="assistant", content=code_block, metadata={"status": "done"} + role="assistant", + content=code_block, + metadata={"status": "done"} ) # An execution that raised is returned by the executor as @@ -969,8 +486,8 @@ class GradioAgentUI(_UIFormattingMixin): # buffered-error path above (parts['observation'] nulled) so it # never renders an alarming "Code Output: Error" block for a # hiccup the agent usually self-corrects on the next step. - if parts["observation"]: - truncated = self.truncate_output(parts["observation"]) + if parts['observation']: + truncated = self.truncate_output(parts['observation']) result_block = f"""
📊 Execution Result
@@ -979,30 +496,32 @@ class GradioAgentUI(_UIFormattingMixin): {truncated} ```""" yield gr.ChatMessage( - role="assistant", content=result_block, metadata={"status": "done"} + role="assistant", + content=result_block, + metadata={"status": "done"} ) - all_artifacts = self._extract_artifacts(parts["observation"]) + all_artifacts = self._extract_artifacts(parts['observation']) for desc, path in all_artifacts: - if path.endswith(".png") and os.path.exists(path): - with open(path, "rb") as f: + if path.endswith('.png') and os.path.exists(path): + with open(path, 'rb') as f: img_b64 = base64.b64encode(f.read()).decode() yield gr.ChatMessage( role="assistant", content=self._figure_html(desc, img_b64), - metadata={"status": "done"}, + metadata={"status": "done"} ) shown_fig_names.add(os.path.basename(path)) self.hf_storage.upload_artifacts(all_artifacts, run_id) - if parts["solution"]: + if parts['solution']: solution_shown = True # The standing decoupleR method limitations are appended # deterministically here, after the model's text — never # generated by the model — so they are identical on every # run and can never be softened or dropped. The model # writes only run-specific caveats. - solution_text = parts["solution"] + "\n\n" + DECOUPLER_DISCLAIMER + solution_text = parts['solution'] + "\n\n" + DECOUPLER_DISCLAIMER solution_block = f"""
✅ Final Solution
@@ -1048,22 +567,27 @@ class GradioAgentUI(_UIFormattingMixin):
""" yield gr.ChatMessage( - role="assistant", content=solution_block, metadata={"status": "done"} + role="assistant", + content=solution_block, + metadata={"status": "done"} ) - if parts["error"]: + if parts['error']: # Buffer, don't render. These are mostly transient # NameError/AttributeError that the CodeAgent hits and then # self-corrects on a later step — showing each one floods the # user with red boxes for problems that were already resolved. # The last error is surfaced after the loop only if the run # never produced a solution (i.e. it genuinely failed). - last_error = parts["error"] + last_error = parts['error'] - if parts["observation"] and step_count in step_timings: - footnote = f'
Step {display_step}
' + if parts['observation'] and step_count in step_timings: + duration = time.time() - step_timings[step_count] + footnote = f'
Step {display_step} | Duration: {duration:.2f}s
' yield gr.ChatMessage( - role="assistant", content=footnote, metadata={"status": "done"} + role="assistant", + content=footnote, + metadata={"status": "done"} ) # (The inter-step
divider is emitted as part of the next @@ -1096,7 +620,7 @@ class GradioAgentUI(_UIFormattingMixin): yield gr.ChatMessage( role="assistant", content=self._figure_html(desc, img_b64), - metadata={"status": "done"}, + metadata={"status": "done"} ) shown_fig_names.add(os.path.basename(path)) if new_figures: @@ -1111,7 +635,9 @@ class GradioAgentUI(_UIFormattingMixin):
{last_error}
""" yield gr.ChatMessage( - role="assistant", content=error_block, metadata={"status": "done"} + role="assistant", + content=error_block, + metadata={"status": "done"} ) except Exception as e: @@ -1119,57 +645,23 @@ class GradioAgentUI(_UIFormattingMixin):
💥 Critical Error
Error during agent execution: {str(e)}
""" - yield gr.ChatMessage(role="assistant", content=error_block, metadata={"status": "done"}) + yield gr.ChatMessage( + role="assistant", + content=error_block, + metadata={"status": "done"} + ) - def interact_with_agent( - self, - query: str, - chatbot_history: list, - session_state: dict, - principal: str = "", - profile: gr.OAuthProfile | None = None, - ) -> Generator: - """Handle interaction with the agent. - - ``principal`` is the authenticated end-user identity forwarded by the - orchestrator (ADR-0012), which reaches this endpoint via its service - token. ``profile`` is injected by Gradio from HF OAuth for a **direct** - human on this Space's own UI (it is not part of the API inputs, so the - orchestrator's ``/interact_with_agent`` call is unaffected). - - The effective caller is the direct human's OAuth username if present, - else the orchestrator-forwarded principal. When ``ACCESS_CONTROL`` is - enforced, a non-allow-listed or unauthenticated effective caller is - refused before any work — this closes the direct-UI side door around the - orchestrator's gate. Enforcement is OFF by default, so the existing - orchestrator route is byte-identical until an operator turns it on (and - then must list the forwarded identities here too). The effective identity - is recorded in the ADR-0008 audit trace via ``agent.set_principal``. - """ + def interact_with_agent(self, query: str, chatbot_history: list, session_state: dict) -> Generator: + """Handle interaction with the agent.""" original_query = query - # ADR-0012 app-layer gate. No-op when ACCESS_CONTROL enforcement is off. - username = getattr(profile, "username", None) if profile else None - effective_principal = username or principal - allowed, denial = check_access(effective_principal) - if not allowed: - chatbot_history = chatbot_history + [ - gr.ChatMessage(role="user", content=original_query, metadata={"status": "done"}), - gr.ChatMessage(role="assistant", content=denial, metadata={"status": "done"}), - ] - yield chatbot_history - return - # A signed-in human's identity supersedes an empty forwarded principal so - # the audit trace attributes the run to whoever actually ran it. - principal = effective_principal - uploaded_file = session_state.get("uploaded_file") if uploaded_file and os.path.exists(uploaded_file): filename = session_state.get("uploaded_filename", os.path.basename(uploaded_file)) is_geo = False if filename.endswith((".txt", ".tsv")): try: - with open(uploaded_file, encoding="utf-8", errors="replace") as _f: + with open(uploaded_file, "r", encoding="utf-8", errors="replace") as _f: is_geo = _f.readline().startswith("!") except Exception: pass @@ -1212,7 +704,6 @@ class GradioAgentUI(_UIFormattingMixin): # Re-register into the tool catalog so the agent can call them. for tool_name, tool_data in self._mcp_cache.items(): from managers.tools.tool_manager import ToolInfo, ToolSource - tool_info = ToolInfo( name=tool_name, description=tool_data.get("description", "MCP tool"), @@ -1225,9 +716,7 @@ class GradioAgentUI(_UIFormattingMixin): schema=None, ) mgr._tool_catalog[tool_name] = tool_info - print( - f"✅ Loaded {len(self._mcp_cache)} MCP tools from cache (no subprocess spawn)" - ) + print(f"✅ Loaded {len(self._mcp_cache)} MCP tools from cache (no subprocess spawn)") else: # Cache empty (pre-warm failed) — fall back to live discovery. # Prefer the persistent HTTP server if it's reachable; only drop @@ -1255,56 +744,37 @@ class GradioAgentUI(_UIFormattingMixin): mcp_tool_count = session_state["agent"].get_tool_statistics()["by_source"].get("mcp", 0) session_state["mcp_tool_count"] = mcp_tool_count if mcp_tool_count == 0: - print( - "⚠️ Session agent has 0 MCP tools — decoupleR analysis " - "tools are unavailable for this session." - ) + print("⚠️ Session agent has 0 MCP tools — decoupleR analysis " + "tools are unavailable for this session.") agent = session_state["agent"] - # ADR-0012: attach the authenticated caller (forwarded by the orchestrator) - # so it flows into the audit trace. Empty string -> None -> "anonymous". - agent.set_principal(principal or None) run_id = datetime.now().strftime("%Y%m%d_%H%M%S") try: - chatbot_history.append( - gr.ChatMessage(role="user", content=original_query, metadata={"status": "done"}) - ) + chatbot_history.append(gr.ChatMessage(role="user", content=original_query, metadata={"status": "done"})) yield chatbot_history if session_state.get("mcp_tool_count", 0) == 0: error_notice = ( '
' - "⚠️ Analysis tools unavailable. The decoupleR MCP " - "tools failed to load for this session, so I cannot run any real " - "analysis (TF activity, pathway/hallmark scoring, differential " - "expression, etc.) right now. Rather than guess at results, I'm " - "stopping here — please try again in a moment, or contact the " - "maintainer if this persists.
" - ) - chatbot_history.append( - gr.ChatMessage( - role="assistant", content=error_notice, metadata={"status": "done"} - ) + '⚠️ Analysis tools unavailable. The decoupleR MCP ' + 'tools failed to load for this session, so I cannot run any real ' + 'analysis (TF activity, pathway/hallmark scoring, differential ' + 'expression, etc.) right now. Rather than guess at results, I\'m ' + 'stopping here — please try again in a moment, or contact the ' + 'maintainer if this persists.' ) + chatbot_history.append(gr.ChatMessage(role="assistant", content=error_notice, metadata={"status": "done"})) yield chatbot_history return if uploaded_file and os.path.exists(uploaded_file): file_notice = f'
Using uploaded file: {filename}
' - chatbot_history.append( - gr.ChatMessage( - role="assistant", content=file_notice, metadata={"status": "done"} - ) - ) + chatbot_history.append(gr.ChatMessage(role="assistant", content=file_notice, metadata={"status": "done"})) yield chatbot_history - chatbot_history.append( - gr.ChatMessage( - role="assistant", content="🤔 Processing...", metadata={"status": "pending"} - ) - ) + chatbot_history.append(gr.ChatMessage(role="assistant", content="🤔 Processing...", metadata={"status": "pending"})) yield chatbot_history chatbot_history.pop() @@ -1313,10 +783,7 @@ class GradioAgentUI(_UIFormattingMixin): yield chatbot_history completed = getattr(agent.workflow_engine, "last_solution_shown", False) - reason = getattr(agent.workflow_engine, "last_end_reason", None) - # Only a genuine step exhaustion is fixed by granting more steps. - session_state["step_limit_hit"] = (not completed) and reason == "step_limit" - session_state["end_reason"] = reason + session_state["step_limit_hit"] = not completed session_state["last_run_id"] = run_id # ALWAYS-ON audit trace persistence via the configured sink. @@ -1332,18 +799,14 @@ class GradioAgentUI(_UIFormattingMixin): print("⚠️ Could not persist execution trace:") traceback.print_exc() - if not completed: - notice = _incomplete_run_notice(reason, agent.config) - chatbot_history.append( - gr.ChatMessage(role="assistant", content=notice, metadata={"status": "done"}) + if session_state["step_limit_hit"]: + notice = ( + '
' + '⏸ Step limit reached. Click Continue ' + 'to give the agent 15 more steps.
' ) - yield chatbot_history - - # Collapsible provenance panel: which analysis tools ran, which - # registered datasets were touched (GUI/observability TODO). - panel = self._what_happened_message(agent) - if panel is not None: - chatbot_history.append(panel) + chatbot_history.append(gr.ChatMessage(role="assistant", content=notice, metadata={"status": "done"})) yield chatbot_history saved = self.hf_storage.save_conversation(chatbot_history, original_query, run_id) @@ -1360,15 +823,10 @@ class GradioAgentUI(_UIFormattingMixin): pdf_remote = None try: raw_messages = agent.workflow_engine.last_state_messages or [] - current_run = self._current_question_messages(chatbot_history) - log_blocks = self._log_blocks(raw_messages, current_run) + log_blocks = self._log_blocks(raw_messages, chatbot_history) if log_blocks: - # Scoped to the current question: figures from earlier - # questions in the session must not reach this export. - images = self._extract_images(current_run) - pdf_path = self._write_pdf( - log_blocks, f"DecoupleRpy Full Run — {run_id}", images - ) + images = self._extract_images(chatbot_history) + pdf_path = self._write_pdf(log_blocks, f"DecoupleRpy Full Run — {run_id}", images) remote_name = f"full_run{os.path.splitext(pdf_path)[1] or '.pdf'}" if self.hf_storage.upload_run_file(pdf_path, run_id, remote_name): pdf_remote = remote_name @@ -1380,19 +838,13 @@ class GradioAgentUI(_UIFormattingMixin): pdf_line = "" if pdf_remote: pdf_url = f"https://huggingface.co/datasets/{self.hf_storage.repo_id}/resolve/main/runs/{run_id}/{pdf_remote}" - pdf_line = ( - f'
📄 Full run PDF: {pdf_remote}' - ) + pdf_line = f'
📄 Full run PDF: {pdf_remote}' log_notice = ( f'
' f'📁 Full run log saved: {hf_url}{pdf_line}
' ) - chatbot_history.append( - gr.ChatMessage( - role="assistant", content=log_notice, metadata={"status": "done"} - ) - ) + chatbot_history.append(gr.ChatMessage(role="assistant", content=log_notice, metadata={"status": "done"})) yield chatbot_history except Exception as e: @@ -1400,34 +852,11 @@ class GradioAgentUI(_UIFormattingMixin): gr.ChatMessage( role="assistant", content=f"Error: {str(e)}", - metadata={"title": "💥 Error", "status": "done"}, + metadata={"title": "💥 Error", "status": "done"} ) ) yield chatbot_history - def _what_happened_message(self, agent) -> "gr.ChatMessage | None": - """Build the collapsed 'what happened' provenance panel for a run. - - Never raises — display plumbing must not take the chat down. Returns - None when there is nothing to show (no steps / no analysis tools). - """ - try: - raw_messages = agent.workflow_engine.last_state_messages or [] - try: - from src.datasets.registry import list_available_datasets - - dataset_ids = [d["dataset_id"] for d in list_available_datasets()] - except Exception: - dataset_ids = [] - activity = self._run_activity(raw_messages, dataset_ids) - html = self._what_happened_html(activity) - if html is None: - return None - return gr.ChatMessage(role="assistant", content=html, metadata={"status": "done"}) - except Exception: - traceback.print_exc() - return None - def _arm_downloads(self, chatbot_history: list, session_state: dict): """Build export files at run completion and arm the download buttons. @@ -1436,12 +865,7 @@ class GradioAgentUI(_UIFormattingMixin): trace), then returns DownloadButton updates so each is a single reliable click. A button stays disabled if its file can't be built. """ - # Scope every export to the most recent question's run: a session - # accumulates questions, and sweeping the whole history embedded stale - # figures from earlier questions that could contradict the current - # run's tables (TODO 2026-08-11 #13). - current_run = self._current_question_messages(chatbot_history) - assessment = self._assessment_blocks(current_run) + assessment = self._assessment_blocks(chatbot_history) raw_messages = [] agent = session_state.get("agent") @@ -1450,7 +874,7 @@ class GradioAgentUI(_UIFormattingMixin): raw_messages = agent.workflow_engine.last_state_messages or [] except Exception: raw_messages = [] - log_blocks = self._log_blocks(raw_messages, current_run) + log_blocks = self._log_blocks(raw_messages, chatbot_history) def _safe(label, fn, *args): try: @@ -1459,16 +883,13 @@ class GradioAgentUI(_UIFormattingMixin): print(f"[export] {label} build failed: {exc}") return None - images = self._extract_images(current_run) + images = self._extract_images(chatbot_history) a_title = "DecoupleRpy Analysis" txt = _safe("txt", self._write_txt, assessment, a_title, images) if assessment else None docx = _safe("docx", self._write_docx, assessment, a_title, images) if assessment else None pdf = _safe("pdf", self._write_pdf, assessment, a_title, images) if assessment else None - log = ( - _safe("log", self._write_pdf, log_blocks, "DecoupleRpy — Full Generated Logic", images) - if log_blocks - else None - ) + log = _safe("log", self._write_pdf, log_blocks, + "DecoupleRpy — Full Generated Logic", images) if log_blocks else None return ( gr.DownloadButton(value=txt, interactive=txt is not None), @@ -1493,14 +914,14 @@ class GradioAgentUI(_UIFormattingMixin): def _gradio_theme(self): return gr.themes.Monochrome( - font=fonts.GoogleFont("Inter"), font_mono=fonts.GoogleFont("JetBrains Mono") + font=fonts.GoogleFont("Inter"), + font_mono=fonts.GoogleFont("JetBrains Mono") ) def create_app(self): """Create the Gradio app with sidebar layout.""" with gr.Blocks(fill_height=True, title=self.name) as demo: - demo.load( - js=""" + demo.load(js=""" () => { document.body.classList.remove('dark'); document.querySelector('gradio-app').classList.remove('dark'); @@ -1514,21 +935,9 @@ class GradioAgentUI(_UIFormattingMixin): brandingElements.forEach(el => el.style.display = 'none'); }, 100); } - """ - ) + """) session_state = gr.State({}) stored_messages = gr.State([]) - # ADR-0012: hidden channel for the authenticated caller. The UI leaves - # it blank (direct use is anonymous); the orchestrator sets it over - # gradio_client so the identity reaches the audit trace. - principal_input = gr.Textbox(value="", visible=False, label="principal") - - # ADR-0012: HuggingFace OAuth sign-in for direct use of this Space's - # UI. Requires `hf_oauth: true` in the README metadata. The injected - # gr.OAuthProfile is read in interact_with_agent, where the allow-list - # gate is enforced. Inert until OAuth is on + ACCESS_CONTROL=enforce, - # so it is safe to ship ahead of cutover. - gr.LoginButton() with gr.Row(): with gr.Column(scale=1): @@ -1540,87 +949,13 @@ class GradioAgentUI(_UIFormattingMixin): with gr.Group(): gr.Markdown("**Data Input (optional)**") - gr.Markdown( - "Uploads are " - "quarantined, structure-checked (file type verified against " - "its contents), and integrity-hashed before the agent sees " - "them (ADR-0011). This is not a malware scan — do not upload " - "files from an untrusted source. Data must be " - "de-identified." - ) - deid_checkbox = gr.Checkbox( - label="I confirm this data is de-identified (no PHI/PSI)", - value=False, - ) with gr.Tabs(): with gr.Tab("Upload File"): file_input = gr.File( - label="Upload .h5ad / .csv / .tsv / .txt (.gz ok)", - file_types=[ - ".h5ad", - ".csv", - ".tsv", - ".txt", - ".gz", - ], - type="filepath", - ) - with gr.Tab("Assemble from TSVs"): - gr.Markdown( - "" - "Drop a sequencing delivery as-is — a gene×sample" - " counts TSV, an optional TPM TSV, and the sample" - " metadata sheet — and the analysis file is built" - " here. Metadata must yield clone," - " arm, site and" - " mouse_id; use the mapping fields" - " below if the sheet names them differently." - "" - ) - asm_counts = gr.File( - label="Counts matrix (.tsv / .csv, genes × samples)", - file_types=[".tsv", ".csv", ".txt", ".gz"], - type="filepath", - ) - asm_tpm = gr.File( - label="TPM / abundance matrix (optional)", - file_types=[".tsv", ".csv", ".txt", ".gz"], - type="filepath", - ) - asm_meta = gr.File( - label="Sample metadata (.xlsx / .csv / .tsv)", - file_types=[".xlsx", ".csv", ".tsv", ".txt"], + label="Upload .h5ad or .csv", + file_types=[".h5ad", ".csv"], type="filepath", ) - with gr.Accordion("Metadata column mapping", open=False): - asm_sample_col = gr.Textbox( - label="Sample-id column", - placeholder="SampleName (default: first column)", - ) - asm_column_map = gr.Textbox( - label="Rename columns → obs", - placeholder="mouse_id=Mouse,site=Type", - ) - asm_value_maps = gr.Textbox( - label="Recode values (one per line)", - placeholder="site=Tumor:tumor\nsite=Met:liver_met", - lines=2, - ) - asm_group_col = gr.Textbox( - label="Group column (derives arm + clone)", - placeholder="Group", - ) - asm_control = gr.Textbox( - label="Control label", placeholder="shCntrl" - ) - asm_treatment = gr.Textbox( - label="Treatment label", value="shMyc" - ) - asm_skip_rows = gr.Textbox( - label="Header row offset", - placeholder="blank = auto-detect", - ) - asm_btn = gr.Button("Assemble dataset", size="sm") with gr.Tab("URL"): url_input = gr.Textbox( label="Public URL", @@ -1645,7 +980,7 @@ class GradioAgentUI(_UIFormattingMixin): lines=4, label="Query", placeholder="Enter your query here and press Enter or click Submit", - value="""Use decoupleRpy MCP to load a built-in RNA-seq dataset, perform preprocessing and differential expression analysis, then run transcription factor enrichment using CollecTRI. Summarize the key regulators.""", + value="""Use decoupleRpy MCP to load a built-in RNA-seq dataset, perform preprocessing and differential expression analysis, then run transcription factor enrichment using CollecTRI. Summarize the key regulators.""" ) submit_btn = gr.Button("Submit", variant="primary", size="lg") @@ -1655,25 +990,23 @@ class GradioAgentUI(_UIFormattingMixin): minimum=5, maximum=50, value=self.config.max_steps, - step=1, + step=1 ) temperature_input = gr.Slider( - label="Temperature", minimum=0.0, maximum=1.0, value=0, step=0.1 + label="Temperature", + minimum=0.0, + maximum=1.0, + value=0, + step=0.1 ) apply_config_btn = gr.Button("Apply Configuration", size="sm") with gr.Accordion("Example Queries", open=False): gr.Examples( examples=[ - [ - "Use decoupleRpy MCP to load a built-in RNA-seq dataset, perform preprocessing and differential expression analysis, then run transcription factor enrichment using CollecTRI. Summarize the key regulators." - ], - [ - "Load the uploaded dataset, run differential expression analysis between the two conditions, then perform TF enrichment with CollecTRI and pathway enrichment with PROGENy." - ], - [ - "Run hallmark gene set enrichment on the uploaded data and identify the most significantly activated and repressed pathways." - ], + ["Use decoupleRpy MCP to load a built-in RNA-seq dataset, perform preprocessing and differential expression analysis, then run transcription factor enrichment using CollecTRI. Summarize the key regulators."], + ["Load the uploaded dataset, run differential expression analysis between the two conditions, then perform TF enrichment with CollecTRI and pathway enrichment with PROGENy."], + ["Run hallmark gene set enrichment on the uploaded data and identify the most significantly activated and repressed pathways."], ], inputs=text_input, ) @@ -1700,18 +1033,10 @@ class GradioAgentUI(_UIFormattingMixin): with gr.Row(): copy_btn = gr.Button("📋 Copy text", size="sm") - export_txt_btn = gr.DownloadButton( - "⬇️ .txt", size="sm", value=None, interactive=False - ) - export_docx_btn = gr.DownloadButton( - "⬇️ .docx", size="sm", value=None, interactive=False - ) - export_pdf_btn = gr.DownloadButton( - "⬇️ .pdf", size="sm", value=None, interactive=False - ) - export_log_btn = gr.DownloadButton( - "⬇️ Full log (.pdf)", size="sm", value=None, interactive=False - ) + export_txt_btn = gr.DownloadButton("⬇️ .txt", size="sm", value=None, interactive=False) + export_docx_btn = gr.DownloadButton("⬇️ .docx", size="sm", value=None, interactive=False) + export_pdf_btn = gr.DownloadButton("⬇️ .pdf", size="sm", value=None, interactive=False) + export_log_btn = gr.DownloadButton("⬇️ Full log (.pdf)", size="sm", value=None, interactive=False) copy_box = gr.Textbox( label="Copy-friendly text (select all → Ctrl+C / use copy button)", @@ -1720,11 +1045,84 @@ class GradioAgentUI(_UIFormattingMixin): interactive=False, ) + def _inputs_dir(): + from pathlib import Path + d = Path(__file__).parent / "tmp" / "inputs" + d.mkdir(parents=True, exist_ok=True) + return d + + def handle_file_upload(file_path, session_state): + import shutil + if file_path is None: + session_state.pop("uploaded_file", None) + session_state.pop("uploaded_filename", None) + return session_state, "" + filename = os.path.basename(file_path) + dest = str(_inputs_dir() / filename) + shutil.copy2(file_path, dest) + session_state["uploaded_file"] = dest + session_state["uploaded_filename"] = filename + return session_state, f"Ready: {filename}" + + def handle_url_download(url, session_state): + import gzip + import requests + import shutil as _shutil + if not url or not url.strip(): + return session_state, "No URL provided" + url = url.strip() + filename = url.rstrip("/").split("/")[-1].split("?")[0] + if not filename or "." not in filename: + filename = "downloaded_data.bin" + dest = str(_inputs_dir() / filename) + try: + r = requests.get(url, stream=True, timeout=300) + r.raise_for_status() + ct = r.headers.get("Content-Type", "") + if "text/html" in ct: + return session_state, "URL returned an HTML page, not a file. Make sure the URL points directly to a file, not a directory." + with open(dest, "wb") as f: + for chunk in r.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + if filename.endswith(".gz") and not filename.endswith(".tar.gz"): + decompressed = filename[:-3] + decompressed_dest = str(_inputs_dir() / decompressed) + with gzip.open(dest, "rb") as f_in, open(decompressed_dest, "wb") as f_out: + _shutil.copyfileobj(f_in, f_out) + os.remove(dest) + dest, filename = decompressed_dest, decompressed + session_state["uploaded_file"] = dest + session_state["uploaded_filename"] = filename + return session_state, f"Ready: {filename}" + except Exception as e: + return session_state, f"Download failed: {e}" + + def handle_hf_dataset(repo_id, filepath, session_state): + if not repo_id or not repo_id.strip(): + return session_state, "No repo ID provided" + if not filepath or not filepath.strip(): + return session_state, "No file path provided" + try: + from huggingface_hub import hf_hub_download + local_path = hf_hub_download( + repo_id=repo_id.strip(), + filename=filepath.strip(), + repo_type="dataset", + local_dir=str(_inputs_dir()), + ) + filename = os.path.basename(local_path) + session_state["uploaded_file"] = local_path + session_state["uploaded_filename"] = filename + return session_state, f"Ready: {filename}" + except Exception as e: + return session_state, f"Failed to load from HF Dataset: {e}" + def update_config(max_steps, temperature, session_state): if "agent" in session_state: agent = session_state["agent"] agent.config.max_steps = max_steps - if hasattr(agent.model, "temperature"): + if hasattr(agent.model, 'temperature'): agent.model.temperature = temperature return "Configuration updated!" @@ -1739,7 +1137,7 @@ class GradioAgentUI(_UIFormattingMixin): return chatbot agent = session_state["agent"] - agent.config.max_steps += STEP_LIMIT_INCREMENT + agent.config.max_steps += 15 run_id = session_state.get("last_run_id", datetime.now().strftime("%Y%m%d_%H%M%S")) resume_messages = agent.workflow_engine.last_state_messages @@ -1750,104 +1148,52 @@ class GradioAgentUI(_UIFormattingMixin): yield chatbot completed = getattr(agent.workflow_engine, "last_solution_shown", False) - reason = getattr(agent.workflow_engine, "last_end_reason", None) - session_state["step_limit_hit"] = (not completed) and reason == "step_limit" - session_state["end_reason"] = reason - - if not completed: - notice = _incomplete_run_notice(reason, agent.config) - chatbot.append( - gr.ChatMessage( - role="assistant", content=notice, metadata={"status": "done"} - ) + session_state["step_limit_hit"] = not completed + + if session_state["step_limit_hit"]: + notice = ( + '
' + '⏸ Step limit reached again. Click Continue ' + 'to add 15 more steps.
' ) + chatbot.append(gr.ChatMessage(role="assistant", content=notice, metadata={"status": "done"})) yield chatbot - panel = self._what_happened_message(agent) - if panel is not None: - chatbot.append(panel) - yield chatbot - - file_input.change( - handle_file_upload, - [file_input, deid_checkbox, session_state], - [session_state, file_status], - ) - # Re-run the gate when the attestation is toggled, so ticking the box - # after picking a file re-validates without re-selecting it. - deid_checkbox.change( - handle_file_upload, - [file_input, deid_checkbox, session_state], - [session_state, file_status], - ) - asm_btn.click( - handle_assembly, - [ - asm_counts, - asm_meta, - asm_tpm, - asm_sample_col, - asm_column_map, - asm_value_maps, - asm_group_col, - asm_control, - asm_treatment, - asm_skip_rows, - deid_checkbox, - session_state, - ], - [session_state, file_status], - ) - url_btn.click( - handle_url_download, - [url_input, deid_checkbox, session_state], - [session_state, file_status], - ) - hf_btn.click( - handle_hf_dataset, - [hf_repo_input, hf_file_input, deid_checkbox, session_state], - [session_state, file_status], - ) + file_input.change(handle_file_upload, [file_input, session_state], [session_state, file_status]) + url_btn.click(handle_url_download, [url_input, session_state], [session_state, file_status]) + hf_btn.click(handle_hf_dataset, [hf_repo_input, hf_file_input, session_state], [session_state, file_status]) download_btns = [export_txt_btn, export_docx_btn, export_pdf_btn, export_log_btn] type_submit = text_input.submit( lambda x: (x, "", gr.Button(interactive=False)), - [text_input], - [stored_messages, text_input, submit_btn], + [text_input], [stored_messages, text_input, submit_btn] ) type_interact = type_submit.then( self.interact_with_agent, - [stored_messages, chatbot, session_state, principal_input], - [chatbot], + [stored_messages, chatbot, session_state], [chatbot] ) type_interact.then(self._arm_downloads, [chatbot, session_state], download_btns) type_interact.then(lambda: gr.Button(interactive=True), None, [submit_btn]) btn_submit = submit_btn.click( lambda x: (x, "", gr.Button(interactive=False)), - [text_input], - [stored_messages, text_input, submit_btn], + [text_input], [stored_messages, text_input, submit_btn] ) btn_interact = btn_submit.then( self.interact_with_agent, - [stored_messages, chatbot, session_state, principal_input], - [chatbot], + [stored_messages, chatbot, session_state], [chatbot] ) btn_interact.then(self._arm_downloads, [chatbot, session_state], download_btns) btn_interact.then(lambda: gr.Button(interactive=True), None, [submit_btn]) - apply_config_btn.click( - update_config, [max_steps_input, temperature_input, session_state], None - ) + apply_config_btn.click(update_config, [max_steps_input, temperature_input, session_state], None) clear_btn.click( clear_chat, [session_state], [chatbot, stored_messages, session_state] ).then( - lambda: tuple( - gr.DownloadButton(value=None, interactive=False) for _ in download_btns - ), - None, - download_btns, + lambda: tuple(gr.DownloadButton(value=None, interactive=False) for _ in download_btns), + None, download_btns ) # Export / copy buttons @@ -1863,14 +1209,12 @@ class GradioAgentUI(_UIFormattingMixin): # armed at run completion by _arm_downloads — a single reliable click, # with no recompute-on-click and no truncation. - continue_interact = continue_btn.click( - handle_continue, [chatbot, session_state], [chatbot] - ) + continue_interact = continue_btn.click(handle_continue, [chatbot, session_state], [chatbot]) continue_interact.then(self._arm_downloads, [chatbot, session_state], download_btns) stop_btn.click( lambda: gr.Button(interactive=True), outputs=[submit_btn], - cancels=[type_submit, type_interact, btn_submit, btn_interact, continue_interact], + cancels=[type_submit, type_interact, btn_submit, btn_interact, continue_interact] ) return demo @@ -1878,11 +1222,11 @@ class GradioAgentUI(_UIFormattingMixin): def launch(self, share: bool = False, **kwargs): """Launch the Gradio app.""" app = self.create_app() - kwargs.setdefault("server_name", "0.0.0.0") - kwargs.setdefault("server_port", 7860) - kwargs.setdefault("show_error", True) + kwargs.setdefault('server_name', '0.0.0.0') + kwargs.setdefault('server_port', 7860) + kwargs.setdefault('show_error', True) # Gradio 6.0 moved theme + css off the Blocks() constructor onto launch(). - kwargs.setdefault("theme", self._gradio_theme()) - kwargs.setdefault("css", self._GRADIO_CSS) + kwargs.setdefault('theme', self._gradio_theme()) + kwargs.setdefault('css', self._GRADIO_CSS) # gradio 6.0 removed the show_api argument from launch(). app.queue(max_size=10).launch(share=share, favicon_path=None, **kwargs) diff --git a/memory.md b/memory.md new file mode 100644 index 0000000000000000000000000000000000000000..74cc7fcbf097bd6dfc7ab35cedb701480301e700 --- /dev/null +++ b/memory.md @@ -0,0 +1,1940 @@ +# memory.md — DecoupleRpy Agent Working State + +This file tracks current status, recent decisions, and next steps. +Update it whenever meaningful work is completed or the direction changes. +Stable architectural facts belong in `CLAUDE.md`, not here. + +Last updated: 2026-07-02 + +--- + +## 2026-07-02 — ADR-0011 upload safety gate "Now" slice IMPLEMENTED (branch `feat/adr-0011-upload-gate`, NOT pushed — no deploy) + +Built the AWS-independent slice of ADR-0011 (manual-dataset-upload safety gate) as a +**standalone, non-agent-facing** Python package `src/uploads/` + a shared hash helper +`src/core/integrity.py`. An upload is quarantined-until-validated, never executable, and only an +admin can register it — the value is the gate, not the upload. + +- **Branch/worktree:** built off `origin/main` (`05fca82`, has the ADR-0008 audit sink) in an + isolated worktree so the unrelated `feat/sandbox-executor` work (incl. its uncommitted ADR-0009 + Appendix A edit) was left untouched. The ADR-0010/0011 docs (which only existed on the sandbox + branch's `cc0ad55`) were pulled onto this branch so the ADR travels with its implementation. +- **`src/core/integrity.py`** — `compute_sha256` / `verify_sha256` (streamed). Deliberately the + *shared* helper ADR-0010's on-load verification will reuse; ADR-0010 doc updated to point at it. +- **`src/uploads/`** — `stage_upload` (gate order: de-id attestation → manifest required → type + allow-list `.h5ad/.csv/.tsv/.txt` + gz → size limit → isolated per-upload quarantine dir + + SHA-256), `validate_upload` (manifest-schema via `validate_manifest` **then** + `validate_manifest_against_data` through the vetted `scanpy.read_h5ad` loader only — the + "never exec" guarantee is structural: no `exec`/`eval`/`pickle` anywhere, asserted by a test), + `register_upload` (admin-only, `UPLOAD_ADMIN_IDS` fail-closed; requires `validated`; writes the + manifest into a registered-overlay dir + `get_registry().register`). Every transition persists an + `UploadRecord` (de-id attestation + hash + status) through the always-on ADR-0008 log sink, so the + attestation lands in the durable audit trail. +- **Config resolved at call time** (`UPLOAD_STAGING_DIR`, `UPLOAD_REGISTERED_DIR`, `UPLOAD_MAX_BYTES`, + `UPLOAD_ADMIN_IDS`) — caught + fixed an import-time-constant bug that made env overrides inert and + leaked staging dirs into the repo root; tests are now hermetic (tmp dirs) and the default dirs are + gitignored. +- **Tests:** `tests/test_upload_gate.py` (24) green — type/size/manifest/attestation gates, + integrity record + tamper detection, validate happy-path + data-level mismatch stays quarantined, + tabular-not-yet-supported, admin refusal (non-admin, and register-before-validate), promotion into + the live registry, never-exec source scan, attestation persisted to the audit sink. Reused + validator/registry suites still green (133). **Limitation:** auto-validation is h5ad-only this + phase (tabular stages but stays quarantined); AWS staging-bucket + malware scan deferred, stated + honestly in the ADR. +- **No runtime/agent surface:** no MCP tool, no `server.py` change, no prompt change, **no Space + deploy/rebuild.** Local-only. + +--- + +## 2026-07-01 (later 3) — Always-on audit log sink (ADR-0008) MERGED to prod `main` + deployed + +Promoted the configurable, always-on execution-trace sink from `feat/configurable-log-sink` +to prod per ADR-0008 (OHSU security review — audit logging as a *production* control, not a +dev-only/in-memory artifact). Merged the whole branch (ADR-0008 core + ADR-0009 S3 sink, which +is inert unless `LOG_SINK=s3`) onto current `origin/main` and pushed to the prod Space. + +- `src/logging_sink.py` + always-on `persist_trace_safe(get_log_sink(), run_id, get_trace())` + in `agent.py`/`gradio_ui.py` now run on the live path (was opt-in `save_trace` only). +- Sink selected by env `LOG_SINK` (`local` default | `hf` | `s3`); prod posture is `hf`. + `persist_trace_safe` is fail-open — a logging error never crashes a run. +- ADR-0008 doc flipped `Proposed → Accepted` and shipped to `main`. +- Merge conflicts were `memory.md`/`TODO.md` only (status files, unioned); all code clean. + Tests: `tests/test_logging_sink.py` green (12 passed, 4 skipped). +- Done in an isolated worktree off `origin/main` on branch `deploy/adr-0008`, pushed with + `git push origin HEAD:main` (local `main` was checked out in another session's worktree). + +--- + +## 2026-07-01 (later 2) — Harden sc load/visualize against missing UMAP/leiden (MERGED to prod `main` `abc16da`; prod FACTORY-REBOOTED) + +`decoupler_load_and_visualize_data` (rna_sc.py) called `sc.pl.umap(color="leiden")` +unconditionally after load and crashed on any h5ad lacking `obsm['X_umap']` / +`obs['leiden']` — exactly the Loveless raw-count subsets (`gse155698_steele`, +`gse205013_werba`; biodata-registry 0.1.8): extracted as RAW COUNTS with the +atlas's corrected/scaled/integrated layers dropped → no embedding, no leiden. The +load tool is the Path-P entry point, so it would crash on first real use. (Was the +`task_8b2a1bdc` spin-off + the "Still open" harden bullet under the Loveless item.) + +- New `_ensure_umap(adata)` → `(color_key, note)`, mutates in place (caller holds + a `read_h5ad_cached` copy). Reuses a precomputed embedding+grouping unchanged + (pbmc3k demo path preserved); else runs a bounded + `normalize_total(1e4)+log1p` (only if the matrix looks like raw counts) → + `pca → neighbors → leiden → umap`. **Never raises** — on failure the UMAP plot + is skipped and the loaded AnnData + metadata are still returned, with the reason + in `message`. leiden uses `flavor="igraph"` (no `leidenalg` dep on the Space). +- New `_looks_like_raw_counts` — bounded, sparse-aware (`scipy.sparse.issparse`; a + plain numpy `.data` is a memoryview, not counts — the bug that failed the first + cut). +- Grouping detection `_UMAP_GROUPING_CANDIDATES` covers R `make.names` atlas cols + (`Clusters`, …), not just `leiden`. +- Sits ON TOP of the `_load_adata` seam already on `main`, so the hardening + the + hosted-URL loader ship together: the Loveless subsets now load AND visualize + end-to-end. Tests: `tests/test_rna_sc_load.py` (4 green; full rna_sc + loading-plan + suite 26 green). +- **Reconciliation note:** this branch was originally cut from an older `main` + (the executor-seam commit `b187b88`, which is NOT in `main`) and lacked the + Loveless loader. Reconciled by resetting the branch onto current `origin/main` + (`6cda24b`, loveless) and re-applying ONLY the UMAP hardening — the executor seam + was dropped (it lives on `feat/executor-seam`, unaffected). Old pre-reset commits + recoverable via reflog (`d46b0e9`). +- **DEPLOYED to prod (2026-07-01, user chose straight-to-prod):** opened HF **PR #1** + on the Space (no GitHub PR path — the `avoigt1121/Paper2Agent_DecoupleRpy` GitHub + repo has an unrelated history; `origin` IS prod), then **merged it into `main`** + (`6cda24b`→`abc16da`). Because prod carried the 0.1.8 Loveless re-pin whose FACTORY + rebuild had not been confirmed, triggered a **factory reboot** (`restart_space(factory_reboot=True)`) + so the 0.1.8 wheel + this code land together. Both prod + dev were RUNNING/cpu-basic + (quota not blocking). NOT dev-validated first. +- **Remaining:** confirm prod settles to RUNNING post-factory-reboot; then e2e-verify + the two Loveless sc datasets load + produce a UMAP end-to-end on prod. + +--- + +## 2026-07-01 (later) — Role-2 signature PUBLISHED + merged main + resolver auth (branch `feat/loveless-sc-serving`) + +Three things this session: +1. **Merged `origin/main` into the branch** (`ffc2a81`) — the branch now contains main's 0.1.7+0.1.8 + re-pins + the salvage docs commit. Only conflict was `memory.md` (resolved: kept my 3 Loveless + entries + main's Salvage entry). Done in an isolated **git worktree** because the primary checkout is + mid-work on `feat/sandbox-executor` (ADR-0007 executor — another session's uncommitted work, left + untouched). The worktree is under scratchpad; remove with `git worktree remove` when done. +2. **Published the Role-2 signature artifact.** `biodata-registry/scripts/ingest/loveless/derive_signatures.py` + (new) downloads the Steele subset (23,991 cells × 36,601 genes), runs `rank_genes_groups` (wilcoxon) + on `Clusters` → 14 per-cell-type marker signatures (683 gene rows, top-50/cluster, log2FC≥1, + padj≤0.05), and `--publish` uploaded `loveless/signatures/gse155698_steele_celltype_signatures.csv` + to `pdac-research-data`. Markers verified biologically sane (ACINAR→PRSS1/CLPS/CTRB1, + FIBROBLASTS→LUM/COL11A1/SFRP2, B CELLS→PAX5/IGHD/TCL1A, MYELOID→S100A8/A12, ENDOTHELIAL→SOX17/GPIHBP1). + `dataset_score_signature` scores it end-to-end (14 activities, 66.6% coverage on a synthetic cohort). + NB the derivation script lives in the **biodata-registry** repo — commit it there separately. +3. **Auth fixes so the private artifact is consumable:** `load_signature_net` now routes a path/URL + through `resolve_to_local_path` (was a bare `pd.read_csv`), and `resolve_to_local_path` now falls back + to the cached `huggingface-cli login` token (`huggingface_hub.get_token()`) when no `HF_TOKEN` env var + is set — so a private signature/h5ad URL loads both on the Space (env secret) and in local/dev. Tests: + `tests/test_signatures.py::TestLoadSignatureNetUrlResolution` (+ existing 42 green). + +**Spun off** the `decoupler_load_and_visualize_data` UMAP/leiden hardening as a separate session +(`task_8b2a1bdc`) — the subsets dropped integrated layers, so that tool will crash on first real load. + +--- + +## 2026-07-01 — Configurable, always-on execution-trace log sink (branch `feat/configurable-log-sink`) + +Prep for the future OHSU-managed AWS migration: made audit/execution-trace persistence +(a) **always-on on the live path** and (b) written through a **configurable sink** so the +destination is a config value, not hardcoded. Branch only — no deploy, no push. + +- **New `src/logging_sink.py`** — `LogSink` interface `persist_trace(run_id, trace) -> str | None` + with three implementations selected by env `LOG_SINK` (`local|hf|s3`, default `local`): + - **`local` (DEFAULT, always-on)** — writes the trace JSON to `LOG_SINK_LOCAL_DIR` + (default `./run_logs`) as `_trace.json`. + - **`hf`** — the existing HuggingFace-dataset behavior refactored behind the interface: + uploads `runs//trace.json` to `LOG_SINK_HF_DATASET` + (default `anne-voigt/decoupleRpy_results`), token from the existing + `decouplerpy_results_token` var; no token => no-op (returns None). `huggingface_hub` + imported lazily. + - **`s3`** — clearly-marked STUB (AWS migration target): raises `NotImplementedError` + with a helpful message naming the target key. `boto3` imported lazily so import never + breaks when it's absent. Sub-config: `LOG_SINK_S3_BUCKET`/`_REGION`/`_PREFIX`. + - `get_log_sink()` factory (unknown value => `ValueError`, fail loud) + `persist_trace_safe()` + wrapper that swallows any error so a logging failure NEVER crashes a request. +- **Always-on wiring:** + - `gradio_ui.py` — `__init__` builds `self.log_sink = get_log_sink()` (falls back to local + on a bad-env error); the streaming run path now calls `persist_trace_safe(self.log_sink, + run_id, agent.get_trace())` UNCONDITIONALLY after each run, independent of the existing HF + `save_conversation`/PDF block (which is untouched — the rich conversation.md/metadata.json/ + PDF "Full run log saved" link still works exactly as before). + - `src/agent.py` `run(...)` — persists the trace through the configured sink on EVERY run, + separate from the opt-in `save_trace=False` local-file dump (kept for backward compat). +- **What is captured is UNCHANGED** — the trace dict is `CodeAgent.get_trace()` + (`{execution_time, config, messages, trace_logs}`): prompts, tool invocations, generated code, + dataset-load steps. Only *that it's reliably persisted* changed. +- **Test:** `tests/test_logging_sink.py` (network/token/boto3-free, **13 passed**): sink selection + (local/hf/s3 + case-insensitive + unknown→ValueError + explicit-override), local sink writes a + real file + honors `LOG_SINK_LOCAL_DIR`, s3 stub raises NotImplementedError (with/without bucket), + `persist_trace_safe` swallows the stub error / returns a path on success / defaults run_id, + hf sink no-ops without a token. `py_compile` clean on all three edited files; verified no + lazy-import (boto3/huggingface_hub) leaks at module load. +- **Default with no new env vars = trace JSON on local disk under `./run_logs`.** `LOG_SINK=hf` + reproduces today's behavior; `LOG_SINK=s3` is the future stub. **Remaining for AWS:** implement + the boto3 `put_object` in `S3LogSink.persist_trace` + wire the S3 env/creds (TODO.md). + **(Superseded by the "later 3" entry above — this was merged + deployed to prod on 2026-07-01.)** + +--- + +## 2026-07-01 — Loveless artifacts LANDED (biodata-registry 0.1.8) + integrated on the agent (branch `feat/loveless-sc-serving`) + +biodata-registry 0.1.8 shipped the two Loveless-atlas single-cell subsets — `gse155698_steele` +(GSE155698) and `gse205013_werba` (GSE205013) — plus the `CROSS_RESOLUTION` gate. Re-pinned the agent +0.1.7→**0.1.8** (`requirements.in`/`.txt`, taken from `origin/main` `0e29c5e` wheel — the re-pin was +already on `origin/main`+prod via `e2be07b`; my feature branch had branched before it). Reinstalled in +`.venv`; registry now lists 22 datasets, both sc cohorts resolve `analysis_path=P`. + +**Key reconciliation:** the landed manifests declare `expression_source.type: **url**` (pointing at a +hosted `.h5ad` on `pdac-research-data`), NOT `type: h5ad`. My #6 code only used the sc loader for +`type == "h5ad"`, so a `type: url` sc manifest fell into the **bulk** flat-file loader +(`decoupler_load_url_counts`) — wrong for an AnnData. **Fixed `_build_loading_plan` to decide Path P by +MODALITY first** (before the source-type dispatch): a `sc_rnaseq`/`spatial_rnaseq` dataset always loads +via `decoupler_load_and_visualize_data` (URL-resolving `_load_adata`), never the bulk loader, regardless +of `url`/`h5ad`. Verified against the live manifests; regression test +`tests/test_loading_plan_h5ad.py::test_url_typed_sc_manifest_uses_sc_loader_not_bulk`. 165 pass across +touched suites (manifest-contract validates all 22). + +**Still open (not blockers to the agent code):** Role-2 derived-signature artifact not yet published +(`dataset_score_signature` waits on it); the 0.1.8 re-pin needs a **FACTORY rebuild** of the Space +(0.1.6 lesson) + e2e verify on `hf-dev`; `decoupler_load_and_visualize_data` will crash on a subset with +no precomputed UMAP/leiden (harden it); no pseudobulk-aggregation tool yet for the sample-level DE +contrast. **NB branch divergence:** `feat/loveless-sc-serving` is behind `origin/main` (which has the +0.1.7+0.1.8 re-pins + a docs commit) and ahead by my Loveless commits — reconcile at PR (memory.md will +conflict). + +--- + +## 2026-06-30 — Loveless sc serving: ADR-0006 #6 — h5ad source type + Path P (branch `feat/loveless-sc-serving`) + +Wired single-cell datasets into the registry loading plan. `manifest_schema` now adds `"h5ad"` to +`VALID_EXPRESSION_SOURCE_TYPES` and derives `analysis_path` **modality-first**: `sc_rnaseq` / +`spatial_rnaseq` → **"P"** (checked BEFORE data_level, so an sc `raw_counts` h5ad is never mislabeled +Path A). This matches biodata-registry 0.1.7's A/B/P — the agent had its own `analysis_path` property +that only knew A/B, so it would have mislabeled sc data without this. `_build_loading_plan` +(`src/tools/dataset_tools/_base.py`) gains an `h5ad` source branch (load via +`decoupler_load_and_visualize_data` → `read_h5ad_cached`) and a Path-P tail that emits per-cell +scoring + a pseudobulk note **instead of** the bulk DESeq2/limma contrast (`return steps` early — no +`dataset_validate_contrast` / `decoupler_differential_expression`). + +Docs updated: ADR-0006 item 6 ticked, CLAUDE.md (controlled vocab + Path A/B/**P** routing), TODO.md. +Tests: `tests/test_loading_plan_h5ad.py` (new) — sc/spatial→P, sc raw_counts NOT Path A, bulk +unaffected, no bulk DE tail, h5ad validates. (Committed `877c4f9`.) + +**Loader gap CLOSED (same session):** `rna_sc.py::_load_adata` now resolves a hosted/private h5ad +URL→local path through the shared authenticated resolver (`src/core/data_io.resolve_to_local_path`, +`HF_TOKEN`) — the same path the bulk tools use — so all 5 rna_sc tools load a `pdac-research-data` h5ad +end-to-end. Stable HF-cache file → `read_h5ad_cached` (parsed once per process); one-shot temp download +→ read directly + deleted (never pins memory). The early `Path.exists()` checks are now URL-aware. +`tests/test_rna_sc_loader.py` (8 pass). **Remaining = biodata-registry artifacts only** (manifest / +Steele-subset h5ad / signatures / `CROSS_RESOLUTION` gate) + prod RAM check + 0.1.7 re-pin & e2e verify. + +--- + +## 2026-06-29 — Loveless sc serving: ADR-0006 + agent-side scaffold (branch `feat/loveless-sc-serving`, `6e9f020`, pushed to `origin` — NOT main, no deploy) + +Started the **High** Loveless single-cell serving task (agent side). Design accepted as +[`ADR-0006`](docs/adr/ADR-0006-loveless-single-cell-serving.md): **two roles**, only the small one ever +live-computed, so the runtime stays bulk-like. +- **Role 1 (Steele subset, live, cached):** `src/tools/rna_sc.py` now loads via `read_h5ad_cached` + (was bare `sc.read_h5ad`) so the subset is parsed once per resident MCP process. Copy-on-read is + fine at the subset's MB scale; the full atlas stays off the live path on purpose. +- **Role 2 (full atlas, never live):** score offline-derived signatures against bulk cohorts on the + existing fast path. New `dataset_score_signature` MCP tool (mounted via `bulk_dataset_mcp`) + + `src/workflows/signatures.py` (load/validate `source/target/weight` net). Reuses the existing + engine via a relaxed gate: `score_bulk_samples_with_decoupler` now treats a caller-supplied + `_network` as a custom signature, so the built-in-resource check applies only when no network is + given (still runs before any download). + +**Tests:** `tests/test_signatures.py` (new) + `test_activity_scoring.py` updated for the relaxed gate → +69 passed locally (`.venv`, decoupler 2.1.6). Scaffold is **synthetic-tested only** — goes live once +biodata-registry produces the Loveless manifest(s) (Steele-subset `sc_rnaseq` h5ad + derived-signature +artifact) and the `CROSS_RESOLUTION` gate in `get_integration_plan`. Remaining in-repo work: add an +`h5ad`/single-cell `expression_source.type` to `_build_loading_plan`. See ADR-0006 action items 5–9 +and TODO.md. + +**Heads-up — the prepared 0.1.7 re-pin is still uncommitted (NOT mine):** `requirements.in`/ +`requirements.txt` carry the **biodata-registry 0.1.6 → 0.1.7** re-pin (`261db0e…` → `eef0406…` wheel) +in the working tree — this is the consumer-side pin for the 0.1.7 single-cell readiness (sc workflows, +`analysis_path` "P", `CROSS_RESOLUTION` gate) per SHOWCASE_STATUS. Deliberately **excluded** from the +Loveless commit (it's a separate deploy needing a **FACTORY rebuild** of the Space — 0.1.6 lesson). +Land it when deploying the Loveless work, not before. + +--- + +## 2026-06-29 — Salvage commit `8efd1be` ported file-by-file → FULLY SUPERSEDED; branch dropped + +Did the manual file-by-file port the `claude/magical-banach-a2d929` "wip(salvage)" commit +(`8efd1be`, Jun 22) asked for. Verdict: **nothing to port — all 10 files already re-implemented** +on current `main` + biodata-registry, in equal-or-better form: + +- semantic_audit `_BASE_INTERPRETATION_NOTE` → present identical (`semantic_audit.py:102`) +- `dataset_count_metadata_values` `prohibited_inferences`/`dataset_refusal_rules` → present + (`dataset_tools/metadata.py`, `_base.py`) +- writable-dir `_resolve_output_dir` (bulk) → present identical; rna writable-dir → present and + generalized to `_resolve_dir()` for INPUT/OUTPUT/GEO_CACHE (`rna/_base.py:34`) +- hf_storage structured conversation-save (Thinking/Plan/Code/Result/Solution + dict-or-object + `_content`/`_role`) → present, refined regex (`hf_storage.py:160`) +- prompts.yaml Moffitt examples (subtype counts; Classical-vs-Basal limma `subset_query "tumor_subtype != ''"`) → present +- test_moffitt_integration golden answer (89 Classical / 36 Basal) → present +- test_tool_response_schemas → present + expanded (all 9 salvage tests ⊂ main's 14) +- gradio_ui txt/docx/pdf export → present, reimplemented more completely + (`ui_formatting._write_docx`/pdf + `gradio_ui._arm_downloads`) +- gse71729_moffitt.yaml stroma_subtype col + decoding + **3 stroma contrasts** → present in biodata-registry + +Porting now would only risk re-introducing stale code on **deleted paths** (the salvage's `rna.py`/ +`dataset_tools.py`/`src/datasets/manifests/` predate the package splits + the manifests→biodata-registry +move). **Dropped the branch** — local + `origin` backup + its defunct `.claude/worktrees/` checkout. +Commit `8efd1be7b825a9ba3b5c45e19e3d611804cc91ec` stays recoverable via reflog/dangling (~90d) if ever +needed. Closes the "manual file-by-file port" follow-up. + +--- + +## 2026-06-26 — PERF: persistent MCP HTTP server **DEPLOYED TO DEV + PROD, confirmed live** (branch `perf/mcp-http-resident-server` → `hf-dev` + `origin` `58b961f`) + +**PROD deploy confirmed live (2026-06-29):** `git push origin HEAD:main` (`2d435a4..58b961f`, clean +ff — `origin/main` was an ancestor, nothing lost); prod Space **built and RUNNING on `58b961f`**, was +NOT paused (cpu-basic quota concern did not apply, no manual restart needed). Run logs show the HTTP +path: `[MCP] HTTP server ready … (took 7s)` → `Registering MCP tools over HTTP` → `Added 52 **remote** +MCP tools` → `Cached 52 MCP tools`, no "Pre-warm failed". **DEV** (`hf-dev` `984a525`) confirmed the +same way 2026-06-26. The "**remote**" wording (vs the earlier bare "Added 52 MCP tools") is the proof +tool calls now reuse the resident HTTP process instead of spawning a subprocess per call. Getting a +clean run took three fixes after the first dev push fell back to stdio: +1. **Botched commit** — the first `git commit` used `git add … ` + (outside the repo); git aborted the whole `add` atomically and staged nothing, so the core files + (agent.py/mcp_manager.py/tool_manager.py/cache.py/tools) were silently left out. The Space then + ran the new `gradio_ui` calling `seed.add_mcp_http()` against an agent with no such method → + `AttributeError` → stdio fallback. **Lesson: never mix an out-of-repo path into `git add`; check + `git show --stat` after committing.** Fixed in `984a525`. +2. **Pipe deadlock** — `ensure_mcp_http_server` launched the subprocess with `stdout=PIPE` and never + drained it; server.py's copious R-install/FastMCP output filled the ~64KB buffer and blocked the + child mid-startup so it never bound the port. Now inherits parent fds (output → HF log stream). +3. **Readiness budget** — the 60s `__init__` poll was too short for a cpu-basic cold start (two + server.py imports contending for 2 vCPUs). Now `__init__` records the intended endpoint + polls + briefly; `_prewarm_mcp` waits up to 240s via `_wait_for_port` (early-exits if the subproc dies) + before committing HTTP-vs-stdio, and skips the heavy raw stdio probe when going HTTP. + +Original change set: + +Killed the per-tool-call subprocess tax. Production wired MCP over **stdio**, so every tool call +spawned a fresh `python server.py` (re-import rpy2/scanpy/decoupler, mount 11 sub-servers, full MCP +handshake, re-read h5ad) — ~44s/step (XDI-001 = 1761s/40 steps). The HTTP fix existed in-tree but +had **zero call sites** (`ensure_mcp_http_server`, `add_mcp_http`). Now wired: + +- **`GradioAgentUI.__init__`** starts the resident `server.py --transport http` on :8765 once per + container (`ensure_mcp_http_server()`); health-checks the port. `_prewarm_mcp` + the per-session + rebuild register tools via **`add_mcp_http(url)`** instead of `add_mcp(stdio)`. **stdio kept as + fallback** if HTTP doesn't come up (or HTTP discovery returns 0 tools). +- **New `agent.add_mcp_http` / `tool_manager.add_mcp_http_server`** (delegate to existing + `mcp_manager.add_mcp_http`). Fixed a latent bug there: its discovery returned raw MCP `Tool` + objects but `_process_remote_server` expects dicts → `'Tool' object has no attribute 'get'`. Now + normalized to dicts (name/description/inputSchema). +- **In-memory AnnData cache** (`src/cache.read_h5ad_cached`, process-lifetime, keyed by + path+mtime+size, returns `.copy()` so concurrent sessions can't corrupt it; bounded to 8). Wired + into the hot DE-path reads (rna/analysis.py, rna/loaders.py, bulk_rnaseq/tools.py). Composes with + the GSE-key disk cache + `preload_datasets` (different keys, no double-load). +- **Instrumentation** `src/core/perf.py`: `[perf]` lines for generate/execute (agent.py) and each + MCP tool call (mcp_manager wrappers) → makes the win measurable in HF logs. + +**Verified locally** (transport, not the full agent loop): resident HTTP server up, **52 tools +registered over HTTP**, two tool calls in **26–43 ms** (vs tens of seconds stdio), resident process +count stayed at **1**, **zero** stdio `server.py` spawns. Full suite **947 passed / 55 skipped** + +new `tests/test_inmemory_adata_cache.py` (5 passed). Expected 30–60% cut on a ~29-min run, same +tools/results. **Live on `hf-dev` (dev) AND `origin` (prod), confirmed via run logs.** Next (optional): +a real multi-step DE run for wall-clock A/B (needs API credits) to quantify the actual % cut. + +--- + +## 2026-06-26 13:11 — XDI suite RE-RUN on PROD: 4/4 PASS, end-to-end on real data (`20260626_131117`) + +Ran the cross-dataset XDI suite (orchestrator `eval/cross_dataset.json`) against the **prod** +specialist (`anne-voigt/Paper2Agent_decoupleRpy`, forced via `HF_SPACE_DECOUPLERPY`) after the +user topped up Anthropic credits. **4/4 PASS**, routing 4/4 → specialist, no credit/timeout/error +cards, fabrication backstop never triggered (~52 min total). Verified against the real traces, not +just signal-counts: + +- **XDI-001 INTEGRATE_EARLY** (1256s) — pooled Moffitt+Puleo (249 samples, 14,008 shared genes), + harmonized subtypes, limma `~batch+subtype`, 2,148 sig genes. No redundant recompute/re-read + end-steps (consistent with the Fix-1 single-read rule). +- **XDI-002 INTEGRATE_LATE** (780s) — plan→late, per-cohort DE (DESeq2/limma), real + `decoupler_meta_analyze` (Stouffer) + per-TF Cochran's Q/I². The genuine prod re-validation of + the `0776267` late hardening (the credit-blocked 12:19 attempt did NOT count). +- **XDI-003 REFUSE_CONFOUNDED** (68s) — `dataset_get_integration_plan`→CONFOUNDED_DESIGN refusal + (Bailey labels absent from tcga_paad), offered single-cohort alternative; no fabricated combine. + Validates the load-bearing refuse path the `cdfee6f` Fix-2 guardrail protects. +- **XDI-004 INTEGRATE_EARLY_SCORING** (1031s) — pooled tcga+paca → ComBat-corrected matrix + (`tcga_paca_combat_corrected.h5ad`) → per-sample PROGENy via `dataset_score_bulk_samples`, + 275×14 matrix, ComBat-without-covariate caveat noted. Validates the `81f5db6` + `decoupler_pool_cohorts` BEHAVIOR on prod (exact tool name not confirmable — trace capped ~6KB). + +**⚠ Reconciles a stale risk note.** The SHOWCASE/memory heads say prod "lacks `HF_TOKEN` → 401 on +ALL private `pdac-research-data` loads, blocks every analysis" and that the 09:31 XDI-004 was +routing-only. This 13:11 run **executed end-to-end on real private data**: XDI-001/002 loaded with +NO 401; XDI-004 hit ONE 401 but recovered and produced full results. Most likely the loader-auth +fix promotion to prod (`5c5ca74`/`ce889e5`, entry below) fixed it, OR the h5ads served from Space +cache/persistent storage. **RESOLVED: user confirmed prod `HF_TOKEN` IS set (2026-06-26)** — so +the loads are real auth (not ephemeral cache), and the "prod lacks `HF_TOKEN` / blocks every +analysis" risk is STALE: prod executes end-to-end on private data. (XDI-004's single recovered 401 +was transient, not a hard block.) + +Caveat: clean efficiency step-count A/B not possible — the eval bank was revised since the pre-fix +run (XDI-001 is now Moffitt+Puleo, not tcga+paca). Results: orchestrator +`eval/results/20260626_131117_{raw,graded,report}.*`. Run from this session; not a git commit. + +--- + +## 2026-06-26 (later) — loader fix + prompt-wording reconcile PROMOTED to PROD (`origin` `5c5ca74`) + +On user go-ahead, pushed `main` to **prod** (`37c63d2`→`5c5ca74`, clean ff) and synced +dev. Two runtime deltas now on prod: `ce889e5` (metadata-tools loader-auth fix, validated +via the XDI-002 re-run) + `5c5ca74` (prompt point-3 reporting reconciled with +`decoupler_pool_cohorts`). The earlier "stale early-branch text" flag was partly wrong — +the user's `81f5db6` had already rewired the EARLY BRANCH to describe both +`decoupler_integrate_datasets` (contrast → batch covariate) and `decoupler_pool_cohorts` +(scoring → ComBat). The genuinely stale bit was POINT 3 (reporting), which still called +pooled activity scores "batch/cohort-confounded"; `5c5ca74` fixes that to: pooled scoring +via `decoupler_pool_cohorts` is ComBat-corrected (+ caveat), only raw pooling without it +is confounded. No `biodata-registry` re-pin → normal rebuild. Both metadata loader fix and +the late-meta-analyze prompt fix are now live on prod. + +--- + +## 2026-06-26 — Metadata tools loader-auth gap fixed + XDI-002 late path partially validated (`ce889e5`, dev) + +Re-ran the XDI-002 late cross-dataset eval (paca_au_rnaseq + paca_au_array, +Bailey squamous-vs-progenitor TF meta-analysis) against the dev Space to validate +the `0776267` meta-analyze prompt fix. Two findings: + +- **Prompt fix CONFIRMED working (as far as the run got).** Trace shows the agent + called `dataset_get_integration_plan([...], design_factor="membership.ordered", + test_group="Squamous", control_group="Pancreatic Progenitor")` FIRST → got + `mode="late"`, then correctly began the per-cohort SEPARATE path (group-size + checks + per-dataset loads) — NOT a hand-rolled sign-concordance. +- **Blocker (separate bug, now fixed): metadata tools didn't authenticate.** + `dataset_count_metadata_values` / `dataset_crosstab_metadata_values` / + `dataset_validate_manifest_against_data` did `Path(adata_path).exists()` on the + raw arg, so a private `pdac-research-data` h5ad URL returned "File not found" + and never authenticated — MISSED in the ce5168d/04b26ba loader-auth + centralization. The agent then fell back to an unauthenticated + `urllib.urlretrieve` that HUNG on the private repo → watchdog bailed at the + 10-min idle cap. Fixed (`ce889e5`): all three route through + `src.core.data_io.resolve_to_local_path` (HF_TOKEN auth + temp cleanup) like + the DE tool already does. Verified locally — fixed + `dataset_count_metadata_values` loads the private paca_au_rnaseq.h5ad and + returns real Bailey counts (Squamous 20 / Pancreatic Progenitor 21 / ADEX 9 / + Immunogenic 20; 70/92 labeled). Deployed **dev only** (`hf-dev`, `ce889e5`). + +- **XDI-002 RE-RUN = PASS (end-to-end, ~16 min on dev `f01f56b`).** With the loader + fix in place the agent completed the full late path: plan→`late` → DESeq2 (RNA-seq) + + limma (array) per cohort → CollecTRI per cohort → **`decoupler_meta_analyze` + (Stouffer)** combining 91 TFs → solution reports per-feature **Cochran's Q and I^2** + and explicitly flags the heterogeneous TFs (CDX2 I^2=72.9%, HOXD3 I^2=51.3%) as + inconsistent across platforms. NO hand-rolled sign-concordance. This is the direct + fix of the original XDI-002 FAIL (ad-hoc concordance, no Q/I^2). Both the + `0776267` prompt fix and the `ce889e5` loader fix are validated on dev. Result + saved `/tmp/xdi002_result.json`. + +NB the first run (pre-watchdog) exposed a router robustness gap: the gradio +streaming iterator has no client-side idle cap, so a stalled Space hung the +blocking dispatch ~16h. Watchdog (idle 600s / total 3000s) added in the eval +driver only, not the router. + +⚠ **Stale prompt text to fix:** the early-branch prompt (`0776267`, now on prod) +says "there is NO batch-aware activity-scoring tool (ComBat deferred)". The user's +`81f5db6` since added `decoupler_pool_cohorts` (ComBat per-sample scoring) — that +statement is now outdated and should be reconciled. + +--- + +## 2026-06-26 — PROD promotion: efficiency prompt fix + ComBat `decoupler_pool_cohorts` → prod (`origin` `81f5db6`) + +On user go-ahead, promoted `main` to **prod** (`git push origin main`, +`f621b4d`→`81f5db6`, clean fast-forward). Three commits now live on prod: +`cdfee6f` (step-count efficiency prompt edits) + `d24ef3b` (its docs) + `81f5db6` +(the ComBat `decoupler_pool_cohorts` scoring tool, server 54→55). The user chose +to include `81f5db6` even though its own commit had marked it "dev only". No +`biodata-registry` re-pin → **normal rebuild** (no factory rebuild). Prod Space +confirmed reachable (HTTP 200) and rebuilding (`RUNNING_APP_STARTING`, cpu-basic) +immediately after the push — NOT paused, so the earlier "prod paused on quota" +caveat did not apply. Follow-up: confirm the Space settles to RUNNING, then the +dev-validated XDI evals (XDI-001/002 step-count drop; XDI-004 pooled scoring) can +be re-run against prod. + +--- + +## 2026-06-25 (latest) — ComBat per-sample scoring tool `decoupler_pool_cohorts` (ADR-0001 item 10, dev only) + +Built the early-integration **scoring** arm: pool ≥2 cohorts + ComBat-correct into one +matrix for per-sample activity scoring. Closes the XDI-001 gap (pooled 258 samples → +CollecTRI ULM with batch applied *post-hoc* = cohort-confounded). + +**Premise correction first.** The task came in as "build the early pooling tool, it's +stubbed as not-built / falls back to late." That was **stale** — Mode A *DE* pooling +(`decoupler_integrate_datasets`, T7–T9) shipped 2026-06-24 (`a7a8caa`), is registered + +prompt-wired + deployed to prod, 22 tests green. The "not built yet" text survived only +in `CLAUDE.md:122` (now fixed). The genuine remaining gap was the **no-design-matrix +scoring path**: `decoupler_integrate_datasets` always runs a DE (needs a contrast), so a +pure pooled-activity-scoring request still hand-rolled load→align→concat→correct and +couldn't batch-correct the scores. User chose to build ComBat for it. + +**What shipped (additive, dev only):** +- `src/workflows/integration.py`: `batch_correct_for_scoring(adata, batch_key, data_level)` + — log-normalise (keyed on `poolable_data_level`: raw_counts→`normalize_total`+`log1p`; + tpm/fpkm→`log1p`; already-log→none, + a raw-counts safety detector), drop genes + constant within any batch, then `scanpy.pp.combat(key="batch")`. Guards: ≥2 samples/batch, + ≥2 non-constant genes. +- `src/tools/integration_tools.py`: `decoupler_pool_cohorts` (4th `integration_mcp` tool, + server **54→55**). Re-checks `get_integration_plan` (early-only; refuses/reroutes for + late/concordance/refuse exactly like `decoupler_integrate_datasets`), pools via the + existing `build_combined_anndata` (design_factor=None — no contrast), ComBat-corrects, + writes ONE corrected h5ad, **deletes the un-corrected intermediate** so it can't be + scored by mistake. Returns `output_path` → feed straight to `dataset_score_bulk_samples`. +- `prompts.yaml`: early branch IMPORTANT block rewired — was "there is NO batch-aware + per-sample scoring tool … flag batch/cohort-CONFOUNDED"; now "(2) pure per-sample + scoring → `decoupler_pool_cohorts(dataset_ids=[...])` → score the corrected matrix." + The (1) DE-contrast → `decoupler_integrate_datasets` path is unchanged; late/refuse + branches untouched. (Complements `cdfee6f`'s step-1 no-design_factor efficiency edit.) +- `tests/test_pool_cohorts_tool.py` (13: routing gate + REAL ComBat on synthetic data + + engine mechanics). `docs/adr/ADR-0001-*` item 10 marked done + decision subsection. + +**Key methodological decision — standard ComBat (`scanpy.pp.combat`), NOT ComBat-seq/ +`inmoose`.** The scoring path log-normalises anyway, so empirical-Bayes ComBat on +log-expression is correct; ComBat-seq's count-domain output only matters when corrected +*counts* feed a count model (DESeq2), and that path uses the **covariate** route, never +ComBat. So scanpy (already installed) suffices → **no new dependency** on the cpu-basic +Space. ComBat applied ONLY on this no-design-matrix path (ADR-0001: pre-correct-then-DE +inflates false positives). Runs without a biological covariate → if a group is confounded +with cohort it can also remove real between-cohort biology (surfaced as `interpretation_note`). +Covariate-preserving ComBat is a possible follow-up, not v1. + +**Validation:** py_compile clean; `prompts.yaml` parses + Jinja prompt renders; +`integration_mcp` + top-level `server.mcp` list the tool (55); `tests/` **947 passed / +55 skipped**; real-ComBat smoke test reduces batch separation, no NaN. + +**Deploy:** committed on `main`, pushed to **dev** (`hf-dev`) only. **Prod (`origin`) NOT +pushed** (still `f621b4d`) — left for the user. No requirements/`biodata-registry` change +→ normal rebuild (no factory rebuild). Follow-up: a live dev eval of a pooled per-sample +scoring request (e.g. XDI-001 reframed) to confirm the agent calls `decoupler_pool_cohorts` +→ `dataset_score_bulk_samples`. + +--- + +## 2026-06-25 (later) — Step-count efficiency: single re-read + no design_factor for no-contrast integration (`cdfee6f`, dev only) + +Latency lever from the 2026-06-25 diagnosis (multi-step compute Qs ~35-40 min, +~90% of it the serialized generate→execute step loop, ~35-45 LLM round-trips; +XDI-001 ~2314s / XDI-002 ~2197s in orchestrator `eval/results/20260625_102209_*`). +Two surgical `prompts.yaml` edits to cut wasted round-trips — **no tool/code changes**: + +- **Reporting Results rule 2 ("Re-read before you report")** tightened to a SINGLE + read of an ALREADY-SAVED results CSV: explicitly forbids recompute / re-derive / + re-save in that step and forbids repeating the re-read. Targets the ~3 redundant + end steps observed on XDI-001 (one step recomputed + re-saved the summary CSVs, + the next re-read + re-printed the same table, then the solution). +- **Cross-dataset integration step 1** now says pass design_factor/test_group/ + control_group ONLY when the user names a two-group contrast; for pooled / + per-sample "across all samples" scoring with NO contrast, call + `dataset_get_integration_plan([...])` with NO design_factor. Stops XDI-001's + refuse-then-retry cycle (it had called `design_factor="sample_type"` → mode=refuse + → re-called without it — a full wasted cycle). The load-bearing with-contrast + guidance (how confounded cross-cohort contrasts are correctly REFUSED) is + **unchanged** — only the no-contrast case was added. + +Pre-check before editing: dev Space tools confirmed loaded (the XDI-001 trace shows +real CollecTRI computation — MYC 24.4, real batch-comparison padj — not the 0-tools +NameError/fabrication failure mode). Verified `yaml.safe_load` parses and both +regions render correctly in the Jinja prompt. Deployed to **dev** (`hf-dev`, +`cdfee6f`, `f621b4d`→`cdfee6f`); **prod (`origin`) NOT pushed** (= `f621b4d`), left +for the user. Full live eval re-run (XDI-001/002, ~35-40 min each) deferred to the +user to confirm the step-count drop. (NB an unrelated `CLAUDE.md` one-line Mode-A +status edit was already in the working tree from another session — left uncommitted, +not part of `cdfee6f`.) + +--- + +## 2026-06-25 — Late-integration prompt hardened: meta_analyze + Q/I^2 mandatory (`0776267`, dev only) + +A 2026-06-25 cross-dataset eval (orchestrator +`eval/results/20260625_102209_*`) showed a "robust TFs across two cohorts" +question route correctly but the agent ran each cohort separately and combined +them with a **hand-rolled sign-concordance intersection** — it never called +`decoupler_meta_analyze` and never reported Cochran's Q / I^2. Fixed in the +cross-dataset section of `prompts.yaml` ("## Combining Two or More Datasets"): + +- **late + shared contrast** → run the SAME contrast per cohort, then combine + **only** with `decoupler_meta_analyze`; explicitly forbids hand-rolled + sign/intersection/fold-change agreement. **ALWAYS report per-feature Q / I^2** + from the tool output. Descriptive concordance is the fallback ONLY when no + shared contrast exists — stated explicitly ("no meta-analysis possible"). +- **early branch refreshed** (was stale — said the pooling tool "is not built + yet"; Mode A `decoupler_integrate_datasets` shipped `a7a8caa`). Now routes a + contrast request to `decoupler_integrate_datasets` (pooled batch-aware DE). + Clarifies there is **no batch-aware activity-scoring tool** (ComBat deferred, + ADR-0001 plan step 11): pool + batch-aware DE for a contrast, or run pooled + activity scoring and **flag it batch/cohort-confounded** — never silently. +- reporting rule (point 3) now requires Q / I^2 for late, batch-covariate / + confound note for early. + +Verified: `prompts.yaml` still parses (yaml.safe_load) and the Jinja system +prompt renders with all new rules present. Deployed to **dev** (`hf-dev`, +`0776267`) then promoted to **PROD** (`origin`, `791bb29`→`211cc90`, clean +fast-forward) on user go-ahead. Only runtime delta is `prompts.yaml`; no +`biodata-registry` re-pin, so a normal rebuild suffices (no factory-rebuild +needed this time). Prod already carried the loader-auth + concordance stack as +of the `791bb29` reconciliation, so this push is purely the prompt fix + docs. +Follow-up: re-run the 2026-06-25 cross-dataset eval (XDI-002 late) to confirm the +agent now calls `decoupler_meta_analyze` + reports Q/I^2 (user confirms dev +credits are OK; the earlier out-of-credits note no longer applies). + +--- + +## 2026-06-24 (later) — concordance VALIDATED live on dev + loader auth centralized (`ce5168d`, `04b26ba`) + +End-to-end validation of the 0.1.6 concordance gate on the dev Space (`hf-dev`), +driven via `gradio_client` 2.5.0 (py3.13 venv) against the private Space. + +- **Concordance routing confirmed LIVE.** A sibling-variant request (`gse205154_sears` + + `gse205154_sears_tmm`) → live `dataset_get_integration_plan` returns + `mode="concordance"` with `cohort_id`/`variant` in `per_dataset`; the agent states the + samples "must NOT be pooled or meta-analyzed" and routes to + `decoupler_normalization_concordance` — **never** `decoupler_meta_analyze`. +- **⚠ The 0.1.6 re-pin needed a FACTORY rebuild, not a normal one.** First validation + showed the live plan returning `late` with NO `cohort_id`/`variant` — the Space ran a + *stale pre-0.1.6 `biodata_registry`* (HF build/pip cache) despite `fb2091e` pinning the + 0.1.6 wheel. A factory rebuild (cache-bust) installed 0.1.6 and the gate went live. + **Prod promotion must factory-rebuild too**, not just `git push origin`. +- **Private-data HTTP 401 fixed by centralizing loader auth.** The h5ad loaders were + fragmented + unauthenticated, so private `pdac-research-data` h5ads 401'd. Added one + resolver `src/core/data_io.resolve_to_local_path` (hf_hub_download + `HF_TOKEN` for + private `huggingface.co` `/resolve/` URLs; urllib fallback) and routed + `decoupler_differential_expression` (was local-only `sc.read_h5ad`), Mode A + `_resolve_to_local` (was unauthenticated `urlretrieve`), and + `bulk_dataset_tools._resolve_to_local_path` through it. `ce5168d` (first patch) → + `04b26ba` (centralization, new `src/core/data_io.py`). Dev only (`hf-dev`). Proven: the + resolver downloads the real 85.4 MB private h5ad with auth; the Space run no longer + 401s/TaskGroup-errors on load. Needs the `HF_TOKEN` Space secret (read + `anne-voigt/pdac-research-data`) — set on dev. ADR-0005 keeps the data private, so + authenticating (not making public) is the correct fix. +- **⚠ BLOCKER: dev Space `ANTHROPIC_API_KEY` is OUT OF CREDITS** ("credit balance too low + to access the Anthropic API") → the agent can't finish a multi-step run, so the final + concordance numbers weren't captured. Top up credits, then re-run for the end-to-end + output. Code is validated up to that wall. + +--- + +## 2026-06-24 — ADR-0001 Phase 2 Mode A (early integration) BUILT + DEPLOYED (`a7a8caa`) + +Built T7–T9 (Mode A early integration) and deployed to dev + prod. `early` verdicts +now actually pool instead of falling back to `late`. + +- **T7 `src/workflows/integration.py`** — combined-AnnData builder: harmonise each + dataset to a gene-symbol axis (var.index if feature_id_type=gene_symbol, else a + SYMBOL/gene_symbol var col), intersect features, concat with a `batch` obs key = + dataset_id. Pure `combine_anndatas` (unit-testable) + loader `build_combined_anndata`. +- **T8 `batch_column` on `decoupler_differential_expression`** — DESeq2 `~batch+factor`; + new `run_limma_covariate` (`~batch+group`, stable `grouptest` coef) in microarray.py; + ttest+batch refused (no silent covariate drop). Default None = behaviour unchanged. +- **T9 `decoupler_integrate_datasets`** (3rd integration_mcp tool, server 52→53) — + re-checks get_integration_plan; pools + one batch-aware DE ONLY on mode=="early", + else refuses/reroutes (late→meta_analyze, concordance→concordance, refuse). Auto-picks + deseq2 (raw counts) / limma from poolable_data_level. +- **Tests:** +22 (test_integration_mode_a 9, test_de_batch_covariate 4, + test_integrate_datasets_tool 9). Full non-live suite **915 passed** (incl. a real + DESeq2 run + live run_limma_covariate). No regressions. +- **v1 limits:** gene-symbol axis only (no probe collapse/ortholog → those don't reach + `early` anyway); ComBat (step 11) deferred; ttest+batch refused. NB the 19-dataset + registry has few pairs that are BOTH poolable AND share a usable contrast, so most + real cross-dataset requests still hit the concordance/late/refuse reroutes. +- **Deploy:** branch `adr-0001-phase-2-mode-a` → **dev** (`hf-dev:main`, + `fb2091e..5cb2737`, user-confirmed RUNNING), then merged ff to `main` → **prod** + (`origin`, `fb2091e..5cb2737`, code-only rebuild). Additive / inert-unless-called, so + the prod promotion can't change existing behaviour. ADR plan + TODO checkboxes updated. +- **NB:** an uncommitted `src/tools/bulk_dataset_tools.py` change sits in the working + tree (parallel/other work, NOT Mode A) — left untouched, not deployed. + +## 2026-06-24 — ADRs reconciled & accepted (docs-only; committed `main`, NOT pushed) + +Landed the ADR status/checkbox reconciliation that `2f95179`/`4f3b2dd` left open. Docs +only, no code; held local to avoid an extra prod rebuild on top of the settling 0.1.6 +deploy (entry below). + +- **ADR-0003 Spaces Dev Mode → Accepted.** User confirmed Dev Mode is enabled on both dev + Spaces → action item 1 ticked. Open: DEPLOYMENT.md inner-loop note (sibling repo) + VS + Code/SSH config. +- **ADR-0004 ZeroGPU → Accepted** ("considered & deferred"). Specialist stays CPU-only; + item 1 ticked. Items 2–3 conditional (only if a PyTorch/CUDA single-cell/Visium port is + scheduled) → open. +- **ADR-0005 Private storage → Accepted.** User confirmed `pdac-research-data` is private → + item 1 ticked. Open: onboarding "promote to public" step + Data Studio workflow + + biodata-registry/memory.md note. +- **ADR-0001 checkboxes → shipped reality.** Phase 0 + Phase 1 (Mode B, T1–T6) ticked in the + main ADR + implementation-plan; Phase 2 (Mode A, T7–T9) + Phase 3 (T10–T11) left open. + Recorded the **concordance** 4th verdict (`mode="concordance"` + `DUPLICATE_COHORT`, 0.1.6) + as a "Phase 1.5" across main ADR / implementation-plan / spinoff-tasks (decision-matrix + already had it). Also folded in the prior session's pending ADR-0002 prod-status tweak. +- **Stale-premise correction:** the task brief assumed `fb2091e` was unpushed/inert; mid- + session `origin/main` had already advanced to `fb2091e` (prod deploy by the user) + memory + commit `0f20f51`. Corrected all ADR-0001 deploy wording from "inert/not-deployed" → + "deployed to prod" to match git. **I did not push anything.** + +## 2026-06-24 — biodata-registry 0.1.6 re-pin DEPLOYED — same-cohort concordance now live (`fb2091e`) + +The behavior blocker noted in the entry below is resolved: biodata-registry was +cut to **0.1.6** (release `261db0e`, wheel sha256 `c4a21878…`) and the agent +re-pinned to it, so the same-cohort concordance gate routes live. + +- **Re-pin (`fb2091e`):** `requirements.in` + `requirements.txt` bumped + 0.1.5 → 0.1.6 (URL `…/resolve/261db0e994eb2b5a7f3e8a40ca8d269cc89bda9b/biodata_registry-0.1.6-py3-none-any.whl`). + requirements.txt has no `--hash` line (plain `uv pip compile` output), so only + the commit-pinned URL changed. +- **What 0.1.6 changes:** `get_integration_plan` returns `mode="concordance"` for + sibling quantifications of one cohort (the GSE205154 TPM/counts/TMM trio, which + now share `cohort_id=gse205154`) instead of `late`; siblings mixed with + independent datasets refuse `DUPLICATE_COHORT`. So `dataset_get_integration_plan` + → concordance → the (already-built) `decoupler_normalization_concordance` tool, + NOT `decoupler_meta_analyze`. 0.1.5 returned `late` → meta-analyzed the identical + samples (the double-count bug). +- **Deployed:** pushed to **dev** (`hf-dev`) and **prod** (`origin`, + `c9b0d3a..fb2091e`). Dev confirmed **RUNNING on 0.1.6** by the user (after a + transient cold-start 500 cleared on its own); prod rebuilding on 0.1.6 at push + time — expect the same transient 500 during boot (wait/refresh, or pause→unpause + to clear an HF rollout wedge). +- **Verification:** the live private-Space query could NOT be driven from the + release session — the cached HF API token lacks access to the private dev Space + (git push works, the API 404s). Verified instead by: deterministic A/B + (0.1.5→`late` vs 0.1.6→`concordance` on the sibling pair), confirming + `dataset_get_integration_plan` is a pass-through to the registry function, all + 19 v0.1.6 manifests parsing clean through the agent's own `from_dict` (tolerates + the new `cohort_id`/`variant` keys), and the user confirming dev RUNNING. +- **This `memory.md` update is committed locally + NOT pushed** — to avoid a + second prod rebuild while the first settles. `TODO.md` left for the next commit + (it holds concurrent WIP). Pairs with biodata-registry `bb88ec0` + + SHOWCASE_STATUS.md. + +## 2026-06-24 — ADR docs reconciled + 3 new ADRs (`2f95179`, docs-only, NOT pushed) + +Documentation housekeeping only — no code, no deploy. Committed to `main`, not pushed. + +- **Stale ADR `Status:` headers reconciled** to shipped reality. ADR-0001 (+ its + decision-matrix / implementation-plan / spinoff-tasks) now reads: Mode B shipped to + prod 2026-06-19 (T1–T6); Mode A (T7–T9) unbuilt so `early` falls back to `late`; + same-cohort concordance built agent-side but inert pending the biodata-registry + 0.1.6 cut + re-pin. ADR-0002 (+ spinoff-tasks) now reads: Phase 1 + S1/S2/S3 shipped + (`0ac610f`). Previously all still said "Proposed" / "Phase 0" / "not yet dispatched". +- **Three PRO-subscription ADRs added** (were untracked; all Status: Proposed): + - **ADR-0003 Spaces Dev Mode** — enable on the two *dev* Spaces as the inner loop; 0/5 action items done. + - **ADR-0004 ZeroGPU** — considered & **deferred**: specialist stays CPU-only (rpy2/DESeq2/decoupler are non-CUDA); no build, just record the decision. + - **ADR-0005 Private storage** — treat PRO 1 TB as the durable private-by-default tier; confirm `pdac-research-data` is private + audit public files; 0/5 done. +- **One real behavior blocker, unchanged by this commit:** cut biodata-registry 0.1.6 + + re-pin the agent so the same-cohort concordance gate routes live (concordance code is + deployed but inert at the current 0.1.5 pin). + +## 2026-06-22 — CONSOLIDATION + DEV DEPLOY (main `0ac610f`) + +All in-flight work consolidated onto `main` and deployed to **both** the dev Space +(`hf-dev`, `97e2484..0ac610f`) and **prod** (`origin`, `fbfa153..c9b0d3a`, +fast-forward) per user request. **Prod Space picked up the push and went +`RUNNING_BUILDING`** — the cpu-basic quota did NOT block it (old `fbfa153` keeps +serving until the new build finishes; watch for RUNNING vs BUILD_ERROR — the +gradio-6 jump is the risk, though dev validated the same code). The dev Space is +private, so its runtime stage isn't readable via the unauthenticated HF API. + +- `feat/adr-0002-sanity-layer` (`b36d5ce`) merged into `main` via merge commit + `0ac610f` (no-ff; clean — the bulk-URL fix was byte-identical on both branches, + `674c230`/`e12ae0f`, and merged as a no-op). The 4 `.claude/worktrees/*` task + branches were already ancestors (nothing stranded); the stale + `claude/magical-banach` "wip salvage" (behind 148) was deliberately EXCLUDED. +- Consolidated content now on `main` + dev: ADR-0002 Layer-2 sanity layer + (Phase 1 + S1 + S2 contamination + S3 PROGENy/Hallmark parity), UI monotonic + step numbering (`e7585ef`), the same-cohort concordance routine (below), and the + bulk-dataset URL-loading fix. +- `git push hf-dev main:main` → dev Space rebuilding on `0ac610f` (97e2484..0ac610f, + fast-forward, no force). Full `tests/` suite green: 912 passed / 55 skipped. +- **Prod pushed (`fbfa153..c9b0d3a`) and build reached `RUNNING` on `c9b0d3a` ✅** + — prod is live on the consolidated set (gradio-6 build succeeded). Remaining: + concordance stays inert until the biodata-registry 0.1.6 wheel + re-pin (the plan + won't return `mode="concordance"` on 0.1.5) — do that next to activate it on both + Spaces. + +## 2026-06-22 session — same-cohort concordance routine (`decoupler_normalization_concordance`) — committed `b36d5ce`, on `main`, deployed to dev + +Built the agent-side routine that the registry's new `mode="concordance"` routes +to (the counterpart to the biodata-registry 0.1.6 same-cohort gate added this +session). This is the fix for the GSE205154 TPM-vs-TMM run that wrongly used +`decoupler_meta_analyze`: sibling variants are the same samples, so meta-analysis +double-counts them. The new tool *compares* instead of *combining*. + +- **New `src/workflows/concordance.py`** (pure numpy/pandas/scipy, no + decoupler/scanpy): `concordance_metrics(tables, dataset_ids, sig_threshold)` + returns `(summary, per_feature_df)`. Metrics over the shared effect field + (`score`, else `stat` for DE): pairwise Pearson/Spearman, sign-concordance, + per-feature effect spread (max−min), and significant-call overlap (per-variant + n_sig, intersection/union, Jaccard, per-variant-only, discordant count) when + `padj` is present. Aligns features the same way as the meta-analysis engine but + **never combines**. +- **New tool `decoupler_normalization_concordance`** in + `src/tools/integration_tools.py` (same `integration_mcp` sub-server as + `decoupler_meta_analyze`). Reuses `_spec_to_envelope` + the shared + result_type/contrast compatibility refusals; reads each variant's result file, + calls `concordance_metrics`, writes a per-feature `*_concordance.csv`, returns a + `mode="concordance"` summary. Refuses (does not raise) on arity/mismatch. +- **Routing docs:** `dataset_get_integration_plan` docstring now enumerates the + `concordance` mode (→ run each variant, compare with the new tool; do NOT + meta-analyze) and `DUPLICATE_COHORT`. `integration_tools` module docstring + the + two CLAUDE.md tool/workflow lists + the "Same-cohort variants" design note + updated (routine now built; only the registry wheel re-pin is pending). +- **Tests:** `tests/test_concordance.py` (9, pure math: identity→r=1/sign=1/0 + spread; sign-flip; padj overlap counts; score→stat fallback; 3-variant pairwise + keys; partial overlap; arity/`no shared field` raises) — **9 passed** locally + (scipy+langgraph installed in sandbox). `tests/test_concordance_tool.py` (6: + registration, enrichment pair, padj overlap, mismatched type/contrast, arity) — + validated via a **stubbed smoke** here (decoupler/scanpy/pydeseq2 too heavy to + install in-sandbox; the committed test runs against the real stack in CI). All + smoke assertions passed. +- **STILL PENDING:** (1) biodata-registry 0.1.6 wheel + re-pin (until then the + live plan returns `late` for sibling variants, so this tool isn't reached); (2) + commit + push this agent work (origin = prod HF Space — coordinate with the + paused-prod / gradio-6 state). No deploy done this session. + +--- + +## 2026-06-22 session — ADR-0002 Layer-2 result-aware sanity layer + S1 verification (COMMITTED on branch `feat/adr-0002-sanity-layer`, NOT pushed) + +New **second safety layer** for the specialist (ADR +`docs/adr/ADR-0002-result-aware-sanity-layer.md`). Motivated by a real +GSE205154 (Sears) Met-vs-Primary CollecTRI run that executed *correctly* yet was +misleading three ways: (1) top "active" TFs were hepatocyte master regulators +(HNF1A/HNF4A/PPARA/RXRA) = **normal-liver contamination** of liver-met FFPE, not +tumour biology; (2) E2F4 reported at **log2FC +19.22** (~610,000×) — a +duplicate-collapse artefact on a bounded TPM scale; (3) BRCA1 requested as a +"TF" but it has no CollecTRI regulon. Layer 1 (registry/`get_integration_plan`, +metadata-only, pre-compute) *can't* see these — they're only visible in the +**results**, and the manifest actually *sanctions* the contrast. Refusal would +also be wrong (the question is legitimate-but-confounded). So Layer 2 +**cautions, never refuses/blocks**. + +### Phase 1 (implemented; introduced with the ADR change) +- **New module `src/workflows/sanity_checks.py`** — pure functions over + DataFrames/lists, no network/decoupler import. Three checks + + `run_sanity_checks()` aggregator returning + `{warnings, n_warnings, max_severity, passed, summary}` (severity + info