Upload folder using huggingface_hub

.claude/settings.local.json

@@ -0,0 +1,45 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(.venv/bin/python -c ' *)",
+      "Bash(.venv/bin/python *)",
+      "Bash(git add *)",
+      "Bash(git commit -m ' *)",
+      "Bash(gh auth *)",
+      "Bash(gh repo *)",
+      "Bash(.venv/bin/streamlit run *)",
+      "Bash(.venv/bin/pip install *)",
+      "Bash(.venv/bin/pytest -x)",
+      "Bash(dot -V)",
+      "Bash(curl -s -o /dev/null -w \"HTTP %{http_code}\\\\n\" http://localhost:8765/)",
+      "Bash(curl -s http://localhost:8765/_stcore/health)",
+      "Bash(.venv/bin/pytest -q)",
+      "Bash(git push *)",
+      "Bash(curl -s -o /dev/null -w \"HTTP %{http_code}\\\\n\" http://localhost:8766/)",
+      "Bash(curl -s http://localhost:8766/_stcore/health)",
+      "Bash(.venv/bin/pytest tests/test_engine.py tests/test_airgap.py -q)",
+      "Bash(.venv/bin/uvicorn api.app:app *)",
+      "Bash(curl -s http://127.0.0.1:8000/health)",
+      "Bash(python -m json.tool)",
+      "Bash(curl -sX POST http://127.0.0.1:8000/reveal -H 'Content-Type: application/json' -d '{\"gene_ids\":[\"g07588\",\"g13403\",\"g05340\",\"g12032\"]}')",
+      "Bash(npm run *)",
+      "Bash(curl -s -o /tmp/lab.html -w \"HTTP %{http_code} \\(%{size_download}B\\)\\\\n\" http://127.0.0.1:3000/)",
+      "Read(//private/tmp/**)",
+      "Bash(curl -s -o /tmp/lab.html -w \"HTTP %{http_code} \\(%{size_download}B\\)\\\\n\" http://127.0.0.1:3001/)",
+      "Bash(npx tsc *)",
+      "Bash(curl *)",
+      "Bash(npm install *)",
+      "Bash(git restore *)",
+      "Bash(.venv/bin/pytest tests/test_engine_v2.py -q)",
+      "Bash(.venv/bin/pytest tests/test_api_airgap.py -q)",
+      "Bash(grep -n \"return \\(\\\\s*$\\\\|<main\\\\|<\\\\/main>\" web/app/Lab.tsx)",
+      "Bash(node *)",
+      "Bash(python3 -c ' *)",
+      "Bash(python3 -m json.tool)",
+      "Bash(grep -n \"^};$\\\\|prefilter_n: {\" web/app/paramHelpContent.tsx)",
+      "Bash(python3 *)",
+      "Bash(npx --prefix /Users/govindbalakrishnan/Govind_Projects/oncodsl/web tsc --noEmit -p /Users/govindbalakrishnan/Govind_Projects/oncodsl/web)",
+      "Bash(echo \"EXIT=$?\")"
+    ]
+  }
+}

.dockerignore

@@ -0,0 +1,12 @@
+.git
+.venv
+**/__pycache__
+**/*.pyc
+web/node_modules
+web/.next
+# Raw downloads are large and not needed at runtime; processed parquet IS needed.
+data/raw
+data/raw_hnsc
+data/raw_gse65858
+tests
+OncoDSL_cover_note.docx

.gitignore

@@ -0,0 +1,12 @@
+.venv/
+data/raw/
+data/processed/
+data/raw_hnsc/
+data/processed_hnsc/
+data/raw_gse65858/
+data/processed_gse65858/
+__pycache__/
+*.pyc
+.pytest_cache/
+.DS_Store
+.python-version

.pytest_cache/.gitignore

@@ -0,0 +1,2 @@
+# Created by pytest automatically.
+*

.pytest_cache/CACHEDIR.TAG

@@ -0,0 +1,4 @@
+Signature: 8a477f597d28d172789f06886806bc55
+# This file is a cache directory tag created by pytest.
+# For information about cache directory tags, see:
+#	https://bford.info/cachedir/spec.html

.pytest_cache/README.md

@@ -0,0 +1,8 @@
+# pytest cache directory #
+
+This directory contains data from the pytest's cache plugin,
+which provides the `--lf` and `--ff` options, as well as the `cache` fixture.
+
+**Do not** commit this to version control.
+
+See [the docs](https://docs.pytest.org/en/stable/how-to/cache.html) for more information.

.pytest_cache/v/cache/lastfailed

@@ -0,0 +1,3 @@
+{
+  "tests/test_api_airgap.py::test_residualize_against_rejected_on_supervised_target": true
+}

.pytest_cache/v/cache/nodeids

@@ -0,0 +1,136 @@
+[
+  "tests/test_airgap.py::test_anonymise_fails_loudly_on_unknown_gene",
+  "tests/test_airgap.py::test_anonymise_idempotent_with_existing_sealed_map",
+  "tests/test_airgap.py::test_anonymise_renames_to_opaque_ids_and_writes_sealed_map",
+  "tests/test_airgap.py::test_engine_does_not_import_sealed_map_or_reveal",
+  "tests/test_airgap.py::test_reveal_rejects_unknown_id",
+  "tests/test_airgap.py::test_reveal_round_trips",
+  "tests/test_airgap.py::test_search_only_accepts_anonymised_matrix",
+  "tests/test_api_airgap.py::test_coadread_rejects_hpv_target",
+  "tests/test_api_airgap.py::test_enable_search_false_pins_search_to_zero",
+  "tests/test_api_airgap.py::test_evaluate_rejects_hnsc_reference_in_coadread",
+  "tests/test_api_airgap.py::test_evaluate_rejects_unknown_reference_set",
+  "tests/test_api_airgap.py::test_evaluate_returns_only_supplied_ids",
+  "tests/test_api_airgap.py::test_hnsc_hpv_run_is_airgap_clean",
+  "tests/test_api_airgap.py::test_hnsc_rejects_msi_target",
+  "tests/test_api_airgap.py::test_lab_payloads_contain_only_opaque_ids[20-spec0]",
+  "tests/test_api_airgap.py::test_lab_payloads_contain_only_opaque_ids[20-spec1]",
+  "tests/test_api_airgap.py::test_lab_payloads_contain_only_opaque_ids[20-spec2]",
+  "tests/test_api_airgap.py::test_lab_payloads_contain_only_opaque_ids[None-spec0]",
+  "tests/test_api_airgap.py::test_lab_payloads_contain_only_opaque_ids[None-spec1]",
+  "tests/test_api_airgap.py::test_lab_payloads_contain_only_opaque_ids[None-spec2]",
+  "tests/test_api_airgap.py::test_lab_payloads_contain_only_opaque_ids[spec0]",
+  "tests/test_api_airgap.py::test_lab_payloads_contain_only_opaque_ids[spec1]",
+  "tests/test_api_airgap.py::test_lab_stream_replays_after_done_with_only_opaque_ids",
+  "tests/test_api_airgap.py::test_legacy_endpoints_still_present",
+  "tests/test_api_airgap.py::test_modules_endpoint_404_for_unknown_run",
+  "tests/test_api_airgap.py::test_modules_endpoint_rejects_unsupervised_run",
+  "tests/test_api_airgap.py::test_modules_endpoint_returns_425_while_running",
+  "tests/test_api_airgap.py::test_modules_endpoint_returns_opaque_only_modules",
+  "tests/test_api_airgap.py::test_operator_usage_endpoint_404_for_unknown_run",
+  "tests/test_api_airgap.py::test_operator_usage_endpoint_425_while_running",
+  "tests/test_api_airgap.py::test_operator_usage_endpoint_returns_opaque_counts",
+  "tests/test_api_airgap.py::test_population_endpoint_is_airgap_clean[v1]",
+  "tests/test_api_airgap.py::test_population_endpoint_is_airgap_clean[v2]",
+  "tests/test_api_airgap.py::test_post_runs_rejects_unsupported_objective",
+  "tests/test_api_airgap.py::test_rates_override_search_zero_pins_search_to_zero",
+  "tests/test_api_airgap.py::test_residualisation_chain_runs_and_stays_airgap_clean",
+  "tests/test_api_airgap.py::test_residualize_against_rejected_on_supervised_target",
+  "tests/test_api_airgap.py::test_residualize_against_unknown_id_returns_400",
+  "tests/test_api_airgap.py::test_residualize_against_works_on_supervised_targets",
+  "tests/test_api_airgap.py::test_transfer_endpoint_425_while_running",
+  "tests/test_api_airgap.py::test_transfer_endpoint_carries_only_winner_revealed_symbols",
+  "tests/test_api_airgap.py::test_transfer_endpoint_rejects_non_hpv_run",
+  "tests/test_api_airgap.py::test_unsupervised_run_emits_posthoc_alignment",
+  "tests/test_build.py::test_label_msi_thresholds",
+  "tests/test_build.py::test_normalise_stage",
+  "tests/test_build.py::test_parse_os_status",
+  "tests/test_build.py::test_read_clinical_happy_path",
+  "tests/test_build.py::test_read_clinical_missing_column_raises_loudly",
+  "tests/test_dsl.py::test_apply_returns_probabilities_indexed_like_input",
+  "tests/test_dsl.py::test_associate_pearson_perfect_negative",
+  "tests/test_dsl.py::test_associate_pearson_perfect_positive",
+  "tests/test_dsl.py::test_associate_spearman_monotone_only",
+  "tests/test_dsl.py::test_associate_unknown_kind_raises",
+  "tests/test_dsl.py::test_effect_drops_nans_and_reports_n_used",
+  "tests/test_dsl.py::test_effect_partial_correlation_removes_confounder",
+  "tests/test_dsl.py::test_fit_separable_features_score_above_random",
+  "tests/test_dsl.py::test_reduce_mean_collapses_rows",
+  "tests/test_dsl.py::test_reduce_rejects_unknown_agg",
+  "tests/test_dsl.py::test_reduce_supports_full_agg_vocab",
+  "tests/test_dsl.py::test_search_ranks_by_objective_and_takes_top_k",
+  "tests/test_dsl.py::test_search_rejects_non_opaque_columns",
+  "tests/test_dsl.py::test_select_missing_column_raises",
+  "tests/test_dsl.py::test_select_picks_columns_in_order",
+  "tests/test_dsl.py::test_split_partitions_by_predicate",
+  "tests/test_engine.py::test_baseline_uses_first_k",
+  "tests/test_engine.py::test_correlation_objective_pipeline_end_to_end",
+  "tests/test_engine.py::test_correlation_objective_prefilter_finds_informative",
+  "tests/test_engine.py::test_crossover_produces_valid_child",
+  "tests/test_engine.py::test_cv_auroc_strong_for_informative_program",
+  "tests/test_engine.py::test_fitness_penalises_size",
+  "tests/test_engine.py::test_mutate_preserves_invariants",
+  "tests/test_engine.py::test_permutation_null_is_around_chance",
+  "tests/test_engine.py::test_pipeline_end_to_end_on_synthetic",
+  "tests/test_engine.py::test_pipeline_no_prefilter_samples_full_pool",
+  "tests/test_engine.py::test_pipeline_rejects_named_columns",
+  "tests/test_engine.py::test_pipeline_unknown_objective_rejected_by_spec_builder",
+  "tests/test_engine.py::test_prefilter_surfaces_the_informative_features",
+  "tests/test_engine.py::test_program_state_shape",
+  "tests/test_engine.py::test_random_program_well_formed",
+  "tests/test_engine.py::test_run_gp_improves_or_holds_fitness_over_generations",
+  "tests/test_engine.py::test_streaming_pipeline_emits_one_event_per_generation",
+  "tests/test_engine_v2.py::test_combine_op_vocab_executes",
+  "tests/test_engine_v2.py::test_constant_or_nan_output_returns_worst_fitness",
+  "tests/test_engine_v2.py::test_crossover_returns_same_root_type",
+  "tests/test_engine_v2.py::test_default_run_unchanged_by_diversity_param",
+  "tests/test_engine_v2.py::test_effect_default_confounders_are_stage_and_age",
+  "tests/test_engine_v2.py::test_effect_extended_confounders_include_sex_and_race",
+  "tests/test_engine_v2.py::test_feature_ids_returns_union_across_leaves",
+  "tests/test_engine_v2.py::test_feature_set_rejects_empty",
+  "tests/test_engine_v2.py::test_fitness_floors_mismatched_target_to_worst",
+  "tests/test_engine_v2.py::test_fitness_floors_no_select_program_under_unsup",
+  "tests/test_engine_v2.py::test_full_grammar_operators_execute",
+  "tests/test_engine_v2.py::test_immigrant_fraction_injects_fresh_programs",
+  "tests/test_engine_v2.py::test_leakage_guard_floors_contaminated_single_gene_winner",
+  "tests/test_engine_v2.py::test_matrix_terminal_executes_to_full_matrix",
+  "tests/test_engine_v2.py::test_mutation_preserves_root_type_and_budget",
+  "tests/test_engine_v2.py::test_objective_from_spec_dispatch",
+  "tests/test_engine_v2.py::test_omni_auroc_is_orientation_agnostic",
+  "tests/test_engine_v2.py::test_oos_silhouette_kills_non_generalising_program",
+  "tests/test_engine_v2.py::test_oos_silhouette_rewards_consistent_split",
+  "tests/test_engine_v2.py::test_oos_silhouette_too_small_floored",
+  "tests/test_engine_v2.py::test_oos_silhouette_train_only_constant_floored",
+  "tests/test_engine_v2.py::test_peeloff_residualisation_uses_train_rows_only",
+  "tests/test_engine_v2.py::test_permutation_null_is_around_chance",
+  "tests/test_engine_v2.py::test_pipeline_end_to_end_produces_typed_repr_and_diverse_population",
+  "tests/test_engine_v2.py::test_pipeline_payloads_carry_only_opaque_ids",
+  "tests/test_engine_v2.py::test_pipeline_rejects_named_columns",
+  "tests/test_engine_v2.py::test_program_repr_is_a_tree_serialisation",
+  "tests/test_engine_v2.py::test_ramped_population_is_diverse",
+  "tests/test_engine_v2.py::test_random_program_obeys_depth_budget",
+  "tests/test_engine_v2.py::test_random_program_returns_vector_typed_tree",
+  "tests/test_engine_v2.py::test_reduce_supports_every_agg",
+  "tests/test_engine_v2.py::test_select_restricts_columns_to_feature_set",
+  "tests/test_engine_v2.py::test_survival_still_rejected_for_v2",
+  "tests/test_engine_v2.py::test_synthesis_binds_scoring_target_to_objective",
+  "tests/test_engine_v2.py::test_termination_guarantee_closes_at_depth_zero",
+  "tests/test_engine_v2.py::test_unsupervised_objective_accepted",
+  "tests/test_engine_v2.py::test_unsupervised_pipeline_strips_labels_from_exec_context",
+  "tests/test_engine_v2.py::test_unsupervised_programs_always_use_gene_select",
+  "tests/test_engine_v2.py::test_unsupervised_silhouette_kills_self_divide_outlier_split",
+  "tests/test_engine_v2.py::test_unsupervised_silhouette_rewards_clean_2_cluster_split",
+  "tests/test_engine_v2.py::test_unsupervised_synthesis_is_vector_only",
+  "tests/test_h1.py::test_h1_recovers_known_direction_on_synthetic_data",
+  "tests/test_hpv_rank.py::test_absent_gene_reported_gracefully",
+  "tests/test_hpv_rank.py::test_present_genes_get_finite_ranks_with_right_sign",
+  "tests/test_hpv_rank.py::test_zero_variance_gene_dropped_from_ranking",
+  "tests/test_tmb_rank.py::test_absent_gene_reported_gracefully",
+  "tests/test_tmb_rank.py::test_present_genes_get_finite_ranks_with_right_sign",
+  "tests/test_tmb_rank.py::test_zero_variance_gene_dropped_from_ranking",
+  "tests/test_transfer_gse65858.py::test_missing_symbols_are_reported_without_crashing",
+  "tests/test_transfer_gse65858.py::test_no_found_symbols_returns_graceful_none",
+  "tests/test_transfer_gse65858.py::test_noise_only_symbols_are_around_chance",
+  "tests/test_transfer_gse65858.py::test_returned_payload_contains_only_supplied_symbols",
+  "tests/test_transfer_gse65858.py::test_signal_gene_produces_high_auroc_and_small_p"
+]

.streamlit/config.toml

@@ -0,0 +1,7 @@
+[theme]
+base = "light"
+primaryColor = "#3A6B7E"
+backgroundColor = "#FAFAF7"
+secondaryBackgroundColor = "#F1EFEA"
+textColor = "#23303A"
+font = "sans serif"

Dockerfile

@@ -0,0 +1,48 @@
+# OncoDSL Lab — single-image deployment.
+# Runs the FastAPI engine, the Next.js front end, and a Caddy reverse proxy
+# (with a password gate) behind one HTTPS port. Data is baked into the image.
+
+# ---- Stage 1: build the Next.js front end ----
+FROM node:20-slim AS web
+WORKDIR /web
+COPY web/package*.json ./
+RUN npm ci || npm install
+COPY web/ ./
+RUN mkdir -p public
+# The browser calls the engine on the same origin, under /api.
+ENV NEXT_PUBLIC_API_URL=/api
+RUN npm run build
+
+# ---- Stage 2: runtime (python engine + node + caddy) ----
+FROM python:3.11-slim
+ENV PYTHONUNBUFFERED=1 NODE_ENV=production NEXT_TELEMETRY_DISABLED=1 \
+    HOME=/tmp XDG_DATA_HOME=/tmp/.local/share XDG_CONFIG_HOME=/tmp/.config
+WORKDIR /app
+
+# Node (to run `next start`) via NodeSource; Caddy copied from its official image.
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends curl ca-certificates \
+ && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
+ && apt-get install -y --no-install-recommends nodejs \
+ && rm -rf /var/lib/apt/lists/*
+COPY --from=caddy:2 /usr/bin/caddy /usr/bin/caddy
+
+# Python deps
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+
+# App code + baked data (data/processed* is NOT in .dockerignore)
+COPY . /app
+# Built front end from stage 1
+COPY --from=web /web/.next /app/web/.next
+COPY --from=web /web/node_modules /app/web/node_modules
+COPY --from=web /web/public /app/web/public
+
+COPY deploy/Caddyfile /etc/caddy/Caddyfile
+# HF Spaces runs as a non-root user (uid 1000); make the dirs it writes writable.
+RUN chmod +x /app/deploy/start.sh \
+ && chmod -R a+rwX /app/web/.next \
+ && mkdir -p /tmp/.local/share /tmp/.config && chmod -R 777 /tmp/.local /tmp/.config
+
+EXPOSE 7860
+CMD ["/app/deploy/start.sh"]

FINDINGS.md

@@ -0,0 +1,183 @@
+# OncoDSL — Findings
+
+A running log of what we've learned from the engine and the data. Each entry is meant to be honest and self-contained — usable in front of co-founders / collaborators.
+
+---
+
+## Finding 1 — Expression can surface MLH1, but not the rest of the MMR set, and that is a data-sensor limit, not an engine limit
+
+**Date:** 2026-06-23
+
+### The observation
+Running the diagnostic on the TMB objective (359 samples, 20,057 genes ranked by signed Spearman correlation of expression with mutation burden; most-negative = rank 1):
+
+| Gene | corr with TMB | rank / 20,057 | percentile |
+|------|---------------|----------------|------------|
+| **MLH1** | −0.31 | **270** | top **1.3%** |
+| PMS2 | −0.18 | 2,214 | top ~11% |
+| MSH2 | −0.05 | 7,516 | ~middle |
+| MSH6 | +0.09 | 15,257 | bottom ~24% (wrong direction) |
+
+Immune markers (CD8A, GZMA, PRF1) sit at the very bottom (ranks ~19,600–20,018, strongly *positive*) — exactly as expected, since they rise with mutation burden. The diagnostic is behaving correctly.
+
+### Why the four MMR genes don't all rank near the top
+"MMR is linked to MLH1 / MSH2 / MSH6 / PMS2" is true at the level of **protein function** — break any of the four proteins and DNA repair fails. But the ranking is built on **mRNA expression**, and **expression measures how much of a gene is transcribed, not whether the protein it makes works.** The gap is explained by *how each gene gets broken*:
+
+- **MLH1** is usually broken by **silencing** (promoter methylation) — the gene is turned *down*, so its expression drops. Expression is a faithful readout of its failure → MLH1 ranks in the top 1.3%.
+- **MSH2 / MSH6 / PMS2** are usually broken by **mutation** — a coding "typo" that produces a faulty protein. The gene is still transcribed at a normal level, so expression doesn't change. **Expression cannot see a typo** → no usable signal (MSH2 flat; MSH6 even slightly the wrong way).
+
+Analogy: trying to find broken spell-checkers by measuring how *loudly* each is running. MLH1's failure mode is "switched off" (volume drops — detectable). The others fail by "running at full volume but with a bug inside" (the volume meter shows nothing wrong). The fault is in the **sensor (expression data)**, not the search.
+
+### Why MLH1 is rank 270 and not rank 1
+Its signal is driven by the ~15% MSI-H minority and diluted across the MSS majority, so its cohort-wide correlation is moderate (−0.31). A cloud of **co-methylated "hitchhiker" genes** — silenced by the *same* methylation program (CIMP) that silences MLH1 — score even more strongly (top-10 ≈ −0.42 to −0.44). Correlation cannot distinguish the causal gene from its co-silenced neighbours.
+
+### Implications
+- **The engine and DSL are not at fault.** They faithfully found the genes whose expression most tracks TMB; MLH1 is right up there.
+- **There is a hard expression-only ceiling:** MSH2/MSH6/PMS2 are *invisible* to an expression matrix because their breakage doesn't change expression. No objective and no amount of compute recovers them from expression alone.
+- **To detect mutation-type breakage you need a different sensor** — the mutation calls (`data_mutations`) — which directly show "this tumour has a broken MSH2." That is a multi-modal extension (expression + mutations).
+- **The honest headline result:** *blind, with gene names hidden, the engine's mechanism objective ranks the causal gene MLH1 in the top ~1% of 20,000 genes* — and the genes above it are its own co-silenced methylation neighbourhood. That is a stronger and more defensible claim than forcing a 4/4 recovery the data cannot support.
+
+---
+
+## Finding 2 — The MSI-H / MSS biology we're testing against (confirmed mental model)
+
+The whole project rests on one causal cascade. The MMR genes are a "spell-checker" for DNA; their state drives everything downstream:
+
+```
+                 working spell-checker          broken spell-checker (dMMR)
+MMR activity     HIGH (MLH1/MSH2/MSH6/PMS2)      LOW
+   ↓                     ↓                              ↓
+phenotype        MSS (microsatellite stable)     MSI-H (microsatellite unstable)
+mutation burden  LOW TMB                          HIGH TMB  (errors pile up)
+immune (CD8A/    LOW (immune-cold)                HIGH (immune-hot — many
+GZMA/PRF1)                                        neoantigens attract T-cells)
+driver           cancer driven by OTHER           cancer driven by broken MMR
+                 mechanisms                       → responds well to immunotherapy
+```
+
+Caveat (see Finding 1): "MMR activity LOW" is only visible in *expression* for MLH1 (silencing); MSH2/MSH6/PMS2 fail by mutation, which expression can't see.
+
+## Finding 3 — Reframed success criterion: "MLH1 in the top ~1% blind"
+
+Forcing a 4/4 MMR recovery is not supported by expression data. The honest, defensible, *reproducible* claim is: blind, the engine's mechanism objective ranks MLH1 at 270 / 20,057 (top 1.3%). This rank is a property of the data, not a lucky run — fully reproducible across seeds/budgets, and far beyond chance. The genes above it are MLH1's own co-silenced (CIMP) neighbourhood, so the engine rediscovered the *methylation-silencing footprint* MLH1 sits in. Stronger and more honest than a forced 4/4.
+
+## Finding 4 — What DSL + GA on TCGA alone can prove (the case to make)
+
+The asset is not "find new biology" — it's "a tiny language + an evolutionary engine, working blind, rediscovers *known* biology from raw numbers, honestly." Both of Toni's proposed problems are *already-solved* questions, which makes them ideal **known-answer validations** of the method. Three claims, strongest first:
+
+1. **Generality across two independent problems with the SAME engine (flagship).** Same unchanged DSL + engine detects the MSI subtype in colorectal AND HPV status in head-&-neck/cervical (HPV+ has a strong, clean expression signature). Two cancers, two questions, both rediscovered blind = a *general* discovery engine, the real moat.
+2. **The engine as an honest instrument.** Changing the objective changes what's found (label-prediction → shortcuts; mechanism objective → MLH1 in the top 1%), and it surfaces its own limits (Finding 1). Interpretable, falsifiable hypotheses — not a black box.
+3. **Minimum-vocabulary discovery (Edgar's principle).** Winning programs are short, human-readable DSL; λ drives toward the smallest program that works.
+
+### Stage-2 objectives note
+The expression-sensor ceiling (Finding 1) is objective-independent — survival and unsupervised read the same expression matrix, so neither lifts the MMR ceiling. Of the two, **unsupervised** is the on-thesis one (find the dominant structure with no labels → plausibly rediscover the MSI-H axis without being told it exists). **Survival** is heavily confounded (stage/age/treatment), so it's a weaker, off-thesis "prognostic prediction" task. The real ceiling-breaker for mutation-type mechanisms is **multi-modal data** (expression + mutation calls).
+
+---
+
+## Finding 5 — HPV detection: the generality demo, and why it avoids the MMR wall
+
+### The biology (parallel to MSI)
+Where MSI is a broken internal spell-checker, HPV is an external hijacker. The virus makes two proteins, **E6** and **E7**, that disable the cell's two main brakes:
+
+```
+            HPV virus → E6 + E7
+   E6 removes p53                 E7 removes RB
+   (protein p53 · gene TP53)      (protein RB · gene RB1)
+        ↓                              ↓
+   HPV+ usually does NOT          RB gone → cell over-makes p16
+   mutate TP53 (virus does it)    (protein p16 · gene CDKN2A), futilely
+                                  → high p16 = the HPV+ marker
+
+              HPV+ (virus-driven)          HPV− (e.g. smoking-driven)
+CDKN2A/p16    ↑ HIGH                        ↓ LOW
+cell-cycle    ↑ HIGH (E2F targets)          lower / different
+immune        ↑ hotter                      cooler
+TP53          usually intact                ✗ MUTATED (smoking)
+outcome       responds well to therapy      worse response
+```
+
+### Genes vs proteins (they have two names each)
+| Protein | Gene | Role |
+|---|---|---|
+| RB (pRB) | RB1 | cell-cycle "brake" |
+| p53 | TP53 | "guardian" — repair/kill damaged cells |
+| p16 | CDKN2A | brake-booster that keeps RB engaged |
+
+### Why p16/CDKN2A rises (the feedback)
+Normally p16 keeps RB engaged, and engaged RB keeps p16 low — a balanced loop. E7 destroys RB, so the cell senses "no braking" and frantically over-produces p16 to compensate — but with RB gone it does nothing and **piles up**. High p16 is therefore the smoke-signal that RB has been removed → the reliable HPV+ marker.
+
+### Why cell-cycle (E2F) genes rise
+RB's normal job is to clamp shut a master "go" switch called **E2F**. E2F, when released, turns ON the whole batch of genes that copy DNA and divide the cell (the "cell-cycle / proliferation" genes — e.g. MCM family, PCNA, cyclins). HPV's E7 destroys RB → E2F is freed → those E2F-target genes switch ON (HIGH). So the *single* root cause (E7 removes RB) produces **two** independent, expression-visible readouts: p16/CDKN2A piles up (the futile brake) **and** the E2F cell-cycle program lights up. Having two strong, readable signals from one cause is part of why HPV is a clean expression target.
+
+### What TCGA shows / what the engine can use
+- **Expression (mRNA) — what the engine reads:** `CDKN2A`, `TP53`, `RB1` all have expression values. **CDKN2A/p16 expression is the usable HPV+ signal** (turned up). ✓
+- **Mutation (TP53) — separate file**, not in the expression matrix (same sensor issue as MMR).
+- **Proteins p53/p16/RB themselves — RPPA**, a separate modality we don't use.
+
+### Why HPV avoids the MMR wall
+The MMR wall was a sensor mismatch: 3 of 4 MMR genes fail by *mutation* (invisible to expression). HPV's defining signature is **over-expression** (p16 ↑, cell-cycle ↑, immune ↑) — squarely in expression. The sensor matches the signal, so a blind "HPV+ vs HPV−" run should give a clean, high held-out AUROC and recover the textbook markers (CDKN2A/p16, E2F/cell-cycle) — a *cleaner* win than MMR. Two honest notes: (1) the *cause* is the virus, not a host gene, so this is **detection**, not "recover a causal gene" — but the host-response markers ARE the clinically accepted answer; (2) the wall only reappears if you try to detect HPV− via its *TP53 mutation* (invisible) — you don't need to, since HPV+ has a strong positive expression footprint.
+
+### Cohort note
+TCGA **HNSC** (head & neck) is the cohort with a genuine HPV+/− mix (~15–25% HPV+) — best for a classification demo. **CESC** (cervical) is ~95%+ HPV+, so it's a positive control rather than a +/− contrast.
+
+---
+
+## Finding 6 — Unsupervised, fixed, discovers the KRT5 (basal/squamous) axis blind — and honestly reports it isn't MSI
+
+**Date:** 2026-06-23
+
+### The fix
+The first unsupervised attempts overfit: the objective selected the program with the cleanest split on the TRAINING data, so a flexible program over a small gene set could carve an in-sample split that didn't generalise (held-out silhouette of 1.000 from a `protected_div`-by-self outlier artifact, or −1.000 collapse). The fix: score the split **out-of-sample** — fit the winsorize bounds, standardisation, and 2-means **on the training fold only**, then assign held-out patients to those clusters and measure separation there — both during selection (`cv_score`) and in the final held-out (`evaluate_holdout`, now passed the training context). Plus: require unsupervised programs to `Select` genes (no whole-matrix `Reduce`), and restrict `Split` predicates to "score" (no clinical variables).
+
+### The result (a clean, honest run)
+- Winner: `Split(Reduce(Select(M,[g09331]),median), score)` — a single gene, **g09331 = KRT5**.
+- Held-out structure **0.933** (out-of-sample — it generalises), permutation **p 0.0050** (beats random programs), GENES 1.
+- Post-hoc alignment: MSI **AUROC 0.596** (≈ coin-flip), TMB |spearman| **0.116** (≈ 0).
+
+### Interpretation
+Blind and airgapped, the engine found a **real, generalising two-group split** of patients driven by **KRT5** — a basal/squamous epithelial marker that is strongly *bimodal* across the cohort. It rediscovered a genuine biological axis with **zero labels** — and the post-hoc check correctly reports it is **NOT** the MSI subtype. This is the honest-instrument story working end to end: it finds real structure, generalises it to unseen patients, beats chance, reveals the gene, **and tells you what the structure is and isn't**.
+
+### Why it isn't MSI (as predicted)
+The single strongest, cleanest, most-generalisable natural division in this expression data is the KRT5 (basal/squamous) axis, not MSI-H/MSS. MSI-H is a ~15% minority and a subtler, multi-gene signal — it can't out-separate a sharply bimodal single gene. So unsupervised honestly surfaces the dominant clean axis, not the subtype. (To recover MSI specifically, use the supervised MSI objective; to learn *what* the KRT5 split is — histology / mucinous vs adeno / tissue-of-origin — would need a clinical column, same post-hoc pattern.)
+
+---
+
+## Finding 7 — Why the engine isn't "finding MSI" (the consolidated answer)
+
+"Find MSI" means three different things, with three different answers. The failures aren't the engine being wrong — they're a mismatch between what's rewarded and what MSI actually is. The engine faithfully optimises exactly what you ask; it's an honest instrument that surfaces these truths instead of faking a result.
+
+### 1. "Tell MSI-H apart from MSS" → this WORKS
+Point the engine at the MSI label (supervised MSI-separation objective) and it separates the two groups on unseen patients at held-out AUROC ~0.89. Detecting the subtype is not the problem.
+
+### 2. "Recover the causal repair genes (MLH1, MSH2, MSH6, PMS2)" → mostly can't, two reasons
+- **The instrument can't see most of them.** We only have gene *expression* — how loudly each gene is "read out." Three of the four (MSH2, MSH6, PMS2) usually break by **mutation** — a typo in the gene's DNA letters. The cell still reads the recipe out just as loudly; only the protein it builds is broken. Expression measures *volume*, not whether the product works — so a mutation-broken gene looks perfectly normal. Only **MLH1** breaks by being **silenced** (switched off → volume drops), which expression *can* see.
+- **The visible one gets out-competed.** MLH1 *is* visible — top ~1% of genes on the TMB axis — but not the strongest. Hundreds of louder bystanders (immune/consequence genes, and genes co-silenced alongside MLH1) move with MSI even more sharply, so a "best predictor" search grabs those. The cause is present but buried.
+
+### 3. "Rediscover the MSI grouping with no labels (unsupervised)" → won't, wrong *shape*
+Unsupervised looks for the *cleanest two-group split*. MSI-H is (a) a **lopsided ~15% minority** — a small group inside a big crowd, not a balanced split — and (b) a **subtle signal** — many genes each shifting a little, no sharp gap. Single genes like KRT5 are sharply on/off and carve a far cleaner split, so the engine keeps finding those (KRT5, STARD6 …) and never the soft minority MSI axis. MSI simply isn't a clean cluster, so a clean-cluster-finder never lands on it.
+
+### The unifying idea + honest bottom line
+- Reward "any separator" → shortcut/consequence genes, not the cause.
+- Reward "cleanest split" → sharp single genes, not the subtle subtype.
+- And expression alone physically can't show 3 of the 4 causal genes.
+
+So: **detecting** MSI works (~0.89); **recovering the causal genes** is limited to MLH1 (the rest need *mutation* data, not expression — a data-modality limit, not an engine limit); and MSI **won't emerge unsupervised** because it isn't a clean cluster. The right tool depends on the question — and the engine is being honest about which questions this data can and can't answer.
+
+---
+
+## Finding 8 — HPV vs MSI: why HPV is the cleaner problem, the DSL suffices, and what "causal" means here
+
+### The three MSI problems mostly don't apply to HPV
+| MSI problem | HPV? |
+|---|---|
+| **Sensor** — expression can't see mutation-broken genes (3/4 MMR genes) | **N/A.** The HPV+ fingerprint is **over-expression** (p16/CDKN2A ↑, cell-cycle/E2F ↑) — exactly what expression measures. The sensor matches the signal. (HPV−'s TP53 *mutation* is invisible, but you don't need it — HPV+ has its own loud positive signature.) |
+| **Shortcut vs cause** — "any separator" grabs consequences, not the causal genes | **N/A.** The cause is the *virus* (E6/E7), not a host gene, so there's no host causal gene to miss; and the host-response markers you want (p16, cell-cycle) ARE the strongest separators *and* the clinically accepted answer. "Reward any separator" lands on the right genes. |
+| **Wrong shape for unsupervised** — MSI is a lopsided ~15% minority, subtle, no clean cluster | **Partially** (HPV+ is also a ~15–25% minority, and head-&-neck has its own sharp axes), **but moot** — the HPV+ signal is sharp and coordinated, and you run HPV **supervised** (detect HPV+ vs HPV−), where minority/shape doesn't matter and the strong signature gives a clean win. |
+
+Net: the two problems that actually sank MSI (sensor + shortcut-vs-cause) **don't apply** to HPV, because HPV's answer is a strong over-expression signature the data sees directly and the search rewards. HPV plays to the data's strengths instead of its blind spots — which is why it's the stronger generality demo.
+
+### Will the same DSL suffice? Yes.
+HPV detection is structurally identical to MSI separation (binary classification from expression). Reuse Select / Reduce / Combine / Fit / Associate / Effect / Split **unchanged — no new operators**. What changes is only plumbing: (1) load + anonymise the **HNSC** cohort (expression + HPV-status label), and (2) a new objective target `"hpv"` using the same binary-AUROC machinery as MSI. The demo is literally: *same DSL, same engine, same airgap — new cancer, new question.*
+
+### What causal link are we establishing with HPV? Honestly, none that's new.
+HPV detection **recovers the expression footprint of a cause that's already known** (virus → E6/E7 disable p53/RB → p16 & cell-cycle ↑). It is *recovery/validation*, not causal discovery — the same honest position as MSI: **TCGA is observational**, so neither proves causation; recovering MLH1 / p16 *corroborates* established biology. The project's causal reasoning lives elsewhere: (a) the DSL can *express* causal-style operations (`Effect` = confounder adjustment; mechanism-shaped objectives that target a cause's signature, not a label) — strong proxies on observational data, not proof; and (b) the genuine causal-discovery claim is **method-level** (the engine discovering causal structure in sparse domains, as on the maths/crypto problems). A *new* causal claim in cancer would need **interventional / treated-cohort data** (Claim B), which TCGA lacks. So HPV's value is **generality** (same engine, second cancer, known biology rediscovered blind), not new causation — and saying so plainly is a credibility strength, not a weakness.

Read docs/PROMPT_capability_ratio_test.md

@@ -0,0 +1,95 @@
+# Prompt — capability test: can the engine discover a planted gene ratio? (isolated)
+
+Build a **self-contained, known-answer capability test** (like the H1 fixture)
+that plants a synthetic target defined by a *ratio of two real genes*, rigs it so
+neither gene is useful alone, and checks whether the blind engine rediscovers the
+ratio. This is an engine unit test — NOT a biological claim, NOT a new UI
+objective, NOT a new dataset.
+
+**Isolation (non-negotiable):** live as a single new file `scripts/capability_ratio_test.py`
+(scripts/ is allowed to be biology-aware). It READS the existing processed matrix
+and reuses the existing sealed map via `airgap.reveal` for the final check. It
+writes NOTHING to disk, adds no dataset, no API route, no UI, does not modify the
+sealed map or any parquet. Deleting the file leaves zero trace. Do not touch
+`engine_v2/`, `dsl/`, `api/`, or `web/`.
+
+## Why a small panel (important)
+Two individually-uninformative genes give the search NO univariate gradient, so
+finding them among ~20,000 columns is an impossible needle-hunt — that tests
+search-at-scale, not composition. To test composition fairly, restrict the search
+to a small controlled panel: the two planted genes + ~50 random decoy genes.
+
+## Steps
+1. **Load real expression.** `cohort = Load("processed")` (colorectal). Keep
+   samples with complete expression. Work on the NAMED matrix here (scripts/ may
+   see gene symbols).
+2. **Pick a correlated gene pair A, B.** From highly-expressed, adequately-varying
+   genes, find a pair with high positive correlation (e.g. Pearson r ≥ 0.6) — a
+   ratio only hides the single-gene signal when A and B move together. (Search a
+   few hundred random candidate pairs; take the first that satisfies the r
+   threshold AND the step-4 checks. Make the pair configurable via CLI for
+   reproducibility.)
+3. **Define the planted target.** `zA, zB = zscore(A), zscore(B)`;
+   `signal = zA - zB` (the log-ratio direction); `y = (signal > median(signal)).astype(int)`.
+   y depends only on the A-vs-B balance, not on either level.
+4. **Validity checks (assert before running — this is what makes the test
+   meaningful):**
+   - single-gene AUROC(A) and AUROC(B) each within ~0.5 ± 0.07 (individually
+     ~useless). If not, pick another pair.
+   - the ratio itself, `max(AUROC(signal), 1-AUROC(signal))`, ≥ ~0.9 (jointly
+     strong).
+   - Print all three so the setup is auditable.
+5. **Build the search panel.** Columns = {A, B} + ~50 random decoy genes from the
+   matrix. **Control checks (print):** best single-gene AUROC over the whole panel
+   ≈ 0.5 (nothing wins alone), and AUROC of the plain mean of all panel genes ≈
+   0.5 (averaging doesn't work). These controls mean any high score can ONLY come
+   from composition.
+6. **Anonymise + run blind.** `M = anonymise(panel_expression)` (reuses the sealed
+   map; opaque IDs). Run `engine_v2.run_v2_pipeline(M, y, objective=<binary AUROC>,
+   ...)`:
+   - objective: `from engine_v2.fitness import V2Objective; V2Objective(target="msi",
+     binary=True)` — used purely as a binary-AUROC scorer; the "msi" name is just
+     the internal carrier, the values are the planted label.
+   - `prefilter_n=None` (panel is already small; must not univariate-filter out the
+     ~0.5 genes), `scalar_share_override=0.0` (force per-patient-score / Vector
+     programs so the winner is a readable Combine, not an Associate scalar),
+     `coherence_weight=0.0`, diversity on (`tournament_k=2, p_mutate=0.85,
+     immigrant_fraction=0.10`), a generous budget (e.g. `population_size=200,
+     n_generations=40`), and a small `n_permutations` (e.g. 100) — or skip the null.
+   - Run a few seeds (e.g. 3) and report each.
+7. **Grade against the known answer.** For each seed's winner
+   (`result["winning"]`): `reveal(winning["gene_ids"])` → symbols. PASS if the
+   winner's genes are exactly {A, B} (or contain both) AND `winning["holdout_score"]`
+   ≫ the panel's best single-gene AUROC (say ≥ 0.80 vs the ~0.5 controls) AND the
+   `program_repr` contains a `Combine(` with `protected_div` (or `sub`/`mul` — any
+   two-gene interaction that recovers the planted relationship; note which).
+   Print PASS/FAIL, the winner `program_repr`, the revealed symbols, the holdout
+   score, and the control numbers, per seed. Print an overall verdict (recovered
+   the interaction in k / n seeds).
+
+## Interpretation to print at the end (verbatim-ish)
+- PASS (recovered in most seeds): "The engine can discover a genuine two-gene
+  interaction blind — so when a real target (HPV) yields only averages, that's
+  because the biology doesn't need a ratio, not because the engine can't build
+  one."
+- FAIL: "The engine did not recover the planted interaction even on a small panel
+  — a real limitation to fix (raise Combine rate / diversity / budget) before
+  claiming the DSL composes."
+
+## CONSTRAINTS
+- Airgap intact: the engine sees only the anonymised opaque panel; symbols are
+  revealed once at the end via `airgap.reveal` (bounded — the winner's genes).
+  Nothing in `engine_v2` changes.
+- Isolated: one new file under `scripts/`, read-only w.r.t. data + sealed map,
+  no API/UI/dataset/registry change. Existing tests + runs untouched.
+- The point is capability, not forcing the answer: do NOT tell the engine which
+  genes matter or that the target is a ratio. Restricting to a panel and to
+  Vector programs is a fair framing, not a hint (it still must find WHICH two
+  genes and HOW to combine them).
+
+## Checkpoint
+- `python -m scripts.capability_ratio_test` prints: the planted pair's single-gene
+  AUROCs (~0.5), the ratio AUROC (~0.9+), the panel controls (best single ≈ 0.5,
+  panel-mean ≈ 0.5), then per-seed winner `program_repr` + revealed genes +
+  holdout score + PASS/FAIL, and an overall verdict.
+- No change to engine/API/UI/data; deleting the file leaves no trace.

Read docs/PROMPT_caps_rename_operator_usage.md

@@ -0,0 +1,45 @@
+# Prompt — raise caps, spell out "GP" as "genetic programming", and add DSL operator-usage
+
+Three changes, verified against the code (`web/app/Lab.tsx`, `api/app.py`, `engine_v2/gp.py`). Airgap untouched. `pytest` + `tsc` after.
+
+## 1. Raise the Generations / Population ceilings (web/app/Lab.tsx)
+The parameter list (~lines 1202-1208) caps `generations` at `max: 100` and `population` at `max: 500`. Raise both:
+- `generations` → `max: 1000`
+- `population` → `max: 3000`
+Keep them BOUNDED (not removed) so an accidental huge value can't hang the backend. Leave min/step and the other params unchanged.
+(Context, not a code requirement: with the current early convergence, more generations alone won't help much — the useful long run is "Maintain diversity" ON + larger population.)
+
+## 2. Spell out "GP" as "genetic programming" in all USER-FACING copy (web/app/Lab.tsx)
+Replace the abbreviation everywhere it appears in visible text — captions, tooltips, headings, labels, axis titles, sort buttons, table headers. Use "genetic programming" in prose; for the compact fitness label use **"Genetic-programming fitness"** (hyphenated; wraps fine). Known locations (search for `GP` to catch any others):
+- Parameters subtitle (~1222): "GP knobs…" → "Genetic-programming knobs…"
+- Run subtitle (~1374): "Starts a GP run on the backend." → "Starts a genetic-programming run on the backend."
+- Copy/tooltips at ~197, 227, 244, 2119, 2588-2591, 4012, 4586 → "genetic programming" / "the genetic-programming search".
+- The **"GP fitness"** label everywhere shown: the sort-key label (~4617 `["gp_fitness", "GP fitness"]`), the table header (~4682 `<th>GP fitness</th>`), the scatter axis title (~4233), the scatter heading (~4008 "Landscape — GP fitness × Synergy"), the median-guide label (~4185), and the tooltip dt (~4407) → **"Genetic-programming fitness"** (scatter heading → "Genetic-programming fitness × Synergy"; axis → "Genetic-programming fitness — what the engine preferred →").
+- Code COMMENTS that say "GP" (~286, 2160, 2237, 2435, 3800, 4156) are not user-facing — optional. **Do NOT change the internal field/key name `gp_fitness`** (API contract) — only human-readable labels.
+
+## 3. DSL operator-usage across the whole search (API + frontend)
+Show, after a run completes, how often each DSL operator (Select, Reduce, Combine, Split, Associate, Effect, Fit/Apply, Search) was used across **every candidate program in every generation** (Generations × population). Airgap-safe — operator names are DSL keywords, no gene names.
+
+### 3a. API — new endpoint `GET /runs/{run_id}/operator-usage`
+Mirror `_compute_module_ranking` (which already walks `run.log`):
+- Iterate every candidate in every persisted generation (`run.log[*].candidates`). Each candidate has a `program_repr` string (e.g. `Combine(Reduce(Select(M,[…]),min),Reduce(Select(M,[…]),var),protected_div)`; built in `engine_v2/gp.py` ~line 132).
+- Per operator, count occurrences of its token `Name(` in `program_repr` — `Select(`, `Reduce(`, `Combine(`, `Split(`, `Associate(`, `Effect(`, `FitApply(`, `Search(`. (Matching `Name(` avoids false hits; `MatrixTerminal`/`M` is not an operator.) Sum across ALL candidates across ALL generations.
+- Return per operator: `total_uses` (sum of occurrences) and `programs_using` (count of candidate-instances containing it ≥ once); plus run totals (`n_generations`, `n_candidates`). Note that persistent elites are counted once per generation they appear in (the "Generations × population" grid the user asked for).
+- Opaque/airgap-safe (operator keywords + integer counts only). Cache per run; 425 while running, 404 unknown.
+
+### 3b. Frontend — show it on the "DSL vocabulary" tiles
+After a run completes, fetch `/runs/{id}/operator-usage` and on each operator tile show its usage:
+- Under each description: **"used 12,431× — in 38% of programs"** (`total_uses`, and `programs_using / n_candidates`).
+- A small inline bar per tile (relative to the most-used operator) so the distribution is scannable — Select/Reduce dominate; Effect/Split/Fit-Apply/Search are rare or 0 (Search off by default → 0).
+- Before a run / while running, tiles show just descriptions (gate the counts to a completed run).
+- Tile-panel "?" / caption: "Counts every use of each operator across all programs the search tried (generations × population). Select and Reduce are the backbone; Effect, Split, Fit/Apply and Search appear only where the engine reached for them."
+
+## CONSTRAINTS
+- Airgap: the operator-usage endpoint emits keywords + counts only — no gene IDs/names. Engine/GP search untouched. `gp_fitness` data key unchanged.
+- Frontend reads the new endpoint; no change to the run flow.
+
+## Checkpoint
+- Generations accepts up to 1000, Population up to 3000.
+- No visible "GP" remains in the UI ("genetic programming" / "Genetic-programming fitness"); `gp_fitness` key unchanged.
+- `GET /runs/{id}/operator-usage` returns per-operator counts over all generations × population, opaque-safe; tiles show count + share + bar after a completed run (Search reads 0 when off).
+- `pytest` green; airgap untouched; `tsc` clean.

Read docs/PROMPT_cheap_diversity.md

@@ -0,0 +1,27 @@
+# Prompt — cheap diversity knobs (tournament size + mutation + random immigrants), curve as the test
+
+The fitness curve collapses early (best = median by ~gen 10, then flat) — premature convergence. Add the cheap diversity levers before considering the island model. Verified against the code. Defaults must preserve current behaviour exactly. `pytest` + `tsc` after.
+
+## Verified current state (engine_v2/gp.py)
+`run_gp_v2(..., population_size=150, tournament_k=3, elitism=5, p_mutate=0.7, ...)`. `_tournament_select` (line ~31) picks `k` random contenders and returns the best. Main loop (~121-204): evaluate → rank → carry `elitism` elites → fill the rest via `_tournament_select` ×2 + crossover + `mutate`. There is **no** random-immigrant mechanism today.
+
+## Part A — add the three levers to `run_gp_v2`
+1. **Tournament size** — `tournament_k` is already a param. Lowering it (3 → 2) reduces selection pressure → slower takeover → more diversity. Keep the default 3.
+2. **Mutation** — `p_mutate` is already a param (0.7). Allow raising it (e.g. 0.85). Keep the default 0.7.
+3. **Random immigrants (new)** — add `immigrant_fraction: float = 0.0`. When > 0, each generation reserve `round(immigrant_fraction * population_size)` slots in the new population for **fresh random programs** drawn from `ramped_population` (same rng/seed, same grammar/objective constraints as init) instead of crossover+mutation offspring. Fill them after the elites, before/among the offspring; never displace the elites. Default 0.0 ⇒ behaviour unchanged.
+
+## Part B — one UI toggle to drive them (so you can A/B the curve)
+- `RunRequest` gains `diversity: bool = False`. The `_worker` maps `diversity=True` → `tournament_k=2`, `p_mutate=0.85`, `immigrant_fraction=0.10` (tune if needed); `diversity=False` → current defaults unchanged.
+- Frontend: a Parameters checkbox **"Maintain diversity"** (default OFF). Tooltip in plain English: "Off: the population can collapse to near-clones early (the fitness curve's best and median lines meet and go flat). On: lowers selection pressure and injects fresh random programs each generation, so the population keeps exploring — watch the best-vs-median gap stay open longer."
+
+## Part C — the test is the curve
+The success criterion is visible in the Live view fitness curve: with **Maintain diversity ON**, the **best and median lines should stay separated for many more generations** (the population doesn't collapse to clones by ~gen 10), versus OFF where they meet early. Also confirm the winner's held-out AUROC doesn't meaningfully drop (diversity should preserve or improve detection, not hurt it). Re-run HPV coherence-on with the toggle off vs on and compare the curves.
+
+## CONSTRAINTS
+- Defaults preserve current behaviour byte-for-byte (immigrant_fraction 0, tournament_k 3, p_mutate 0.7) so existing runs are unchanged. No airgap/API-shape break beyond the additive `diversity` field. Airgap untouched (this is search-internal; opaque IDs throughout).
+
+## Checkpoint
+- `run_gp_v2` accepts `immigrant_fraction`; with it >0, fresh random programs enter each generation without displacing elites.
+- The "Maintain diversity" toggle (default off) flips tournament_k / p_mutate / immigrant_fraction.
+- With the toggle ON, the fitness curve's best-vs-median gap visibly persists longer; winner held-out is not degraded.
+- `pytest` green (defaults unchanged), `tsc` clean, airgap untouched.

Read docs/PROMPT_confound_pass.md

@@ -0,0 +1,50 @@
+# Prompt — confound pass: extend Effect's confounders + add site-stratified / purity-survival flags to the module ranking
+
+Goal: let us tell whether a gene/module separates HPV for a *mechanistic* reason or via a *confounder* (sex, ancestry, anatomic site, immune composition). Read the named code first; don't assume. Airgap stays absolute (clinical/technical covariates are named non-gene variables — allowed, same category as the existing stage/age; purity is derived in the validation layer, never in the engine). `pytest` + `tsc` after.
+
+## VERIFIED data availability (don't re-assume — checked against the repo)
+Raw HNSC clinical files (`data/raw_hnsc/data_clinical_patient.txt` / `_sample.txt`) contain: `SEX`, `RACE`, `ETHNICITY`, `AJCC_PATHOLOGIC_TUMOR_STAGE`, `AGE`, `ICD_O_3_SITE` (patient) and `TUMOR_TISSUE_SITE` / `ONCOTREE_CODE` (sample). The processed `clinical.parquet` (`build_hnsc.py` ~lines 110-117) currently keeps only `site`(=ONCOTREE_CODE), `age`, `sex`, `stage`, `hpv_status`. So:
+- **sex** — already in the parquet. ✓
+- **race / ethnicity** — in the raw files, NOT yet extracted. Need a build change.
+- **anatomic site for oropharynx** — `ONCOTREE_CODE` is insufficient; use `TUMOR_TISSUE_SITE` (sample) or `ICD_O_3_SITE` (patient). Need a build change.
+- **smoking** — NOT in this download at all. Do not attempt to adjust for it.
+- **tumor purity** — NOT in clinical. Must be derived from expression (see Part E).
+
+## Part A — extend the HNSC build to carry the confounders (data)
+Note (verified): `read_clinical_tsv` returns the FULL dataframe, so `RACE` / `TUMOR_TISSUE_SITE` / `ICD_O_3_SITE` are reachable via `merged.get(...)` — no need to add them to `HNSC_REQUIRED_*_COLS`. In `data_pipeline/build_hnsc.py` (the clinical dict ~lines 110-117): read and write into `clinical.parquet`:
+- `race` (from `RACE`; optionally also `ethnicity`).
+- `tissue_site` — a clean anatomic-site field from `TUMOR_TISSUE_SITE` (preferred) or `ICD_O_3_SITE`, plus a derived boolean `is_oropharynx` (map the oropharynx/tonsil/base-of-tongue values; print the distinct site values + counts so we can confirm the mapping). Keep the existing `site`(ONCOTREE) column too.
+- `sex` is already present — leave it.
+Print the non-null counts for race / tissue_site / is_oropharynx so missingness is visible. (Colorectal build unchanged.)
+
+## Part B — carry the new clinical fields through the API (no airgap change)
+`_prepare_lab_data` (api/app.py) currently subsets `cohort.clinical[["stage","age"]]`. For HNSC, widen the kept set to include `sex`, `race`, `tissue_site`/`is_oropharynx` (only the columns that exist; guard each). These are named clinical variables, not gene symbols — airgap-clean. Keep colorectal as-is. The returned `clinical` DataFrame is what both Effect (via the run's ctx) and the module endpoint (Part D/E) will read, so the new columns must be in the RETURN, not just kept somewhere upstream.
+
+## Part C — generalize Effect to a configurable confounder set (engine)
+`Effect.execute` (engine_v2/nodes.py ~436-471) hardcodes stage + age. Generalize it to adjust on a configurable list of confounders pulled from `ctx.clinical` (one-hot encode categoricals like sex/race/site; keep age continuous; dropna per the design matrix; guard rank/collinearity). **Default must stay stage + age** so existing MSI/TMB/HPV runs are byte-for-byte unchanged. When the extra clinical columns are present, allow Effect to include sex + race (NOT smoking — unavailable). The confounder set is fixed by config, not GP-selectable (the program shouldn't roam over arbitrary confounders). Update/extend the relevant engine_v2 tests.
+
+## Part D — site-stratified survival flag on the module ranking (validation layer)
+IMPORTANT (verified): `_compute_module_ranking` currently **discards clinical** — line ~1282 is `M_all, y_all, _clin, _extra = _prepare_lab_data(...)` with clinical thrown away as `_clin`. Stop discarding it: capture the clinical frame, and align `is_oropharynx` to the held-out patients (the same `test_ids` it already reconstructs) so you can subset. Then, for the HNSC/HPV case add per-module:
+- `combined_holdout_oropharynx`: the module's mean-aggregate AUROC recomputed **within the oropharynx subgroup only** (held-out patients with `is_oropharynx == True`), and a `survives_site` boolean = it stays close to the full-cohort combined AUROC (within a tolerance, e.g. drop < ~0.05) rather than collapsing toward 0.5. This tells us whether the module separates HPV beyond just marking oropharynx tissue.
+- Guard small subgroups (skip / mark "n too small" if the oropharynx held-out has too few per class).
+- Gate to HNSC; colorectal modules omit this gracefully.
+
+## Part E — purity-survival flag (validation layer; heaviest part — may be phased)
+Purity isn't in the data, so derive an **immune-infiltration proxy** in the validation layer (names allowed here, exactly like the rank diagnostics): per patient, the mean expression of a small standard immune-marker set (e.g. CD8A, GZMB, PRF1, CD3D, CD2 — a fixed curated list in one place). Since the matrix `M` is opaque-columned, resolve those marker SYMBOLS to their opaque IDs via the bounded reveal path (same as reference sets), then mean those columns of `M` per patient. Higher proxy = more infiltrate = lower purity. Then per module add:
+- `combined_holdout_highpurity`: the module's mean-aggregate AUROC recomputed within the **low-infiltration (high-purity) subset** of held-out patients (e.g. bottom tertile of the proxy), and a `survives_purity` boolean (same tolerance idea). This catches modules whose HPV signal is really an immune-composition artifact (they'll collapse in the high-purity subset).
+- This derivation lives in the validation/API layer — never in `engine/` or `engine_v2/`. If scope is tight, ship Parts A-D first and land Part E second; mark it clearly.
+
+## Part F — UI: surface the flags + the winner's-curse caveat (presentation)
+- In the Coordinated-modules table, show the survival flags per module (e.g. small "site ✓/✗" and "purity ✓/✗" chips, or the stratified AUROCs on expand). A module that survives both is a real candidate; one that collapses is a likely confound artifact.
+- Add to the modules "?" the winner's-curse caveat we agreed on, verbatim intent: *"These groups are scored by re-evaluating ~2,900 explored sets on the same small held-out set and showing the best — so the very top values are optimistically biased (the luckiest of thousands). The winning program was chosen by cross-validation, which guards against that, so trust it as the engine's pick even when a table row scores higher."*
+
+## CONSTRAINTS
+- Airgap: clinical confounders (sex/race/site) and the derived purity proxy are named non-gene variables / validation-layer computations — never enter `engine/` or `engine_v2/` as gene identities. Module payloads stay opaque-ID + scores + flags.
+- Defaults preserve current behaviour (Effect default = stage+age; non-HNSC unaffected). Colorectal + existing tests stay green.
+
+## Checkpoint
+- HNSC `clinical.parquet` carries race + tissue_site + is_oropharynx (with printed counts); sex already there.
+- Effect adjusts on a configurable set, default stage+age unchanged; with HNSC clinical present it can include sex+race.
+- Module ranking shows site-stratified (and, when Part E lands, purity) survival flags for HNSC/HPV; modules that are site/immune artifacts visibly collapse.
+- Modules "?" carries the winner's-curse caveat.
+- `pytest` green, `tsc` clean, airgap tests untouched.

Read docs/PROMPT_detection_vs_recovery.md

@@ -0,0 +1,43 @@
+# Prompt — separate detection from recovery, name AUROC, and kill the global-mean shortcut
+
+Motivating run (HPV detection, HNSC): held-out AUROC 0.953, p 0.005, 7 genes, winner =
+`Combine(Reduce(M, mean), Reduce(Select(M,[g14733,g16190,g20255,g06520,g07022,g17192,g02069]), mean), add)`.
+Revealed genes (REXO2, SLC35B3, ZNF598, GFPT1, FKBP9, SSBP1, C17orf98) overlap the cell_cycle set 0/7.
+
+Diagnosis: this is a **real detector** but **not** gene recovery. Its dominant term is `Reduce(M, mean)` — a global mean over the WHOLE matrix (no Select) — so it detects HPV from a bulk-expression signal, and the 7 "revealed" genes are a minor additive tail. Yet the UI currently (a) never names the metric AUROC, and (b) the verdict asserts "the engine recovered the HPV viral signature," conflating detection with recovery. Fix all three. Read the named files first; don't assume. Build (`tsc`) + `pytest` after.
+
+## PART 1 — Stop the verdict claiming gene recovery (presentation)
+The verdict must report **detection** (held-out + permutation) and must NOT assert the engine found the marker genes — recovery is judged separately in Reveal & evaluate.
+- `web/app/Lab.tsx` ~line 1632 (`HPVBreakdownContent`): replace
+  `"Real and useful — the engine recovered the HPV viral signature."`
+  with detection-only language, e.g. `"Real and useful — it separates HPV+ from HPV− tumours on patients it never saw."` Do NOT claim it recovered the signature/genes.
+- Add one plain sentence to the HPV (and MSI) breakdown making the distinction explicit, e.g. *"Whether it found the known marker genes is a separate question — see Reveal & evaluate below."*
+- Audit the other verdict/breakdown strings for the same conflation: the top-level success callout (~1406), `MSIBreakdownContent` (~1583), and the unsupervised "rediscovered the viral signature blind" line (~1752). The unsupervised one is a *post-hoc alignment* claim and may be legitimate when alignment is high — leave it if it's correctly gated, but make sure detection-objective verdicts (MSI/HPV) never claim gene recovery from held-out + p alone.
+
+## PART 2 — Name the metric "AUROC" everywhere held-out is shown (presentation)
+For the binary objectives (MSI, HPV) the held-out number IS an AUROC; the UI never says so.
+- `web/app/Lab.tsx` ~line 126 (`TIPS.heldOut`) and the per-target held-out tooltips / `FITNESS_*_BY_TARGET` records (~lines 314–409, 1561, 1610): for binary objectives state plainly "held-out **AUROC**" and keep the "0.5 = coin-flip, 1.0 = perfect" gloss. For the continuous (TMB) and unsupervised (silhouette) objectives keep their own metric names — don't mislabel those as AUROC.
+- The "Held-out" result card label and the "What does this mean?" panel should read "Held-out AUROC" for MSI/HPV runs. Keep it objective-aware (don't hardcode AUROC where the metric is spearman/silhouette).
+
+## PART 3 — Kill the global-mean shortcut: require Select before Reduce for EVERY objective (engine — the real fix)
+Today `engine_v2/synthesize.py` only wraps Matrix leaves in `Select` when `is_unsup` (`objective_target == "none"`). For MSI/TMB/HPV a bare `Reduce(MatrixTerminal, agg)` is legal — that's the whole-matrix global mean the winner used. Generalise the unsupervised "must be gene-based" rule to ALL objectives so no program can score via an un-Selected whole-matrix reduction.
+- Read `engine_v2/synthesize.py` (`_grow_matrix` ~83–126, `_grow_vector` depth-floor ~143–148, the Scalar depth-floor `Associate(Reduce(MatrixTerminal,"mean"),…)`, plus `random_program`/`ramped_population`/`mutate`) and `engine_v2/fitness.py`.
+- Make Select-wrapping **unconditional**: every Matrix leaf that feeds a Reduce is `Select(MatrixTerminal(), featureset)`, for every objective — not just unsup. Concretely, the Vector depth-floor becomes `Reduce(Select(MatrixTerminal(), …), agg)` for all targets, the `_grow_matrix` fall-through wraps in Select for all targets, and the Scalar depth-floor reduces a Select, not a bare MatrixTerminal. (Effectively: treat `require_select = True` for all objectives; keep the existing `Search` gating as-is.)
+- Belt-and-braces in `fitness.py`: floor any program that still contains a `Reduce` over a bare `MatrixTerminal` (no Select on that path) to the objective's worst score, so a stray one (e.g. from a replayed tree) can't win.
+- Airgap unchanged: Select uses opaque IDs only. Update any tests that assumed bare-`MatrixTerminal` programs exist.
+- Expected effect: detection AUROC may dip slightly, but winners become gene-specific and the reveal reflects the actual drivers — which is the point.
+
+## PART 4 — Show each revealed gene's individual rank in Reveal & evaluate (presentation + small API)
+So "0 of 7 in cell_cycle" is always readable as alternates-vs-bystanders.
+- In the `/evaluate` reveal path (api/app.py), where a single-gene rank diagnostic exists for the active (dataset, target) — HPV→`validate/hpv_rank`, colorectal TMB→`validate/tmb_rank` — also compute each revealed gene's **individual rank / total** and its single-gene AUROC (reuse the existing diagnostic machinery; reveal already happens server-side here, so this stays airgap-clean). Return rank + score per revealed symbol.
+- `web/app/Lab.tsx` Revealed-genes table: add a "Rank" column showing "rank N / total" (and the single-gene AUROC) per gene where available; omit gracefully for (MSI, unsupervised) where no single-gene diagnostic exists. Add a one-line "?" explaining: a high individual rank means that gene separates the label on its own (a real alternate marker); a low rank means it only helps in combination.
+
+## CONSTRAINTS
+- Colorectal behaviour, the airgap, and existing tests stay green; gene names never leak into engine/ or any opaque payload.
+- Presentation changes are objective- and dataset-aware (no MSI/AUROC wording leaking onto TMB/unsupervised).
+
+## Checkpoint
+- An HPV run's verdict says it *separates HPV+/−*, names *held-out AUROC*, and does NOT claim it "recovered the signature"; recovery is shown only by the reveal/overlap + per-gene ranks.
+- After the engine fix, re-run an HPV seed: the winner contains no bare `Reduce(M, …)` term (every Reduce sits on a Select), and the revealed genes are the actual drivers.
+- Revealed-genes table shows each gene's individual rank for HPV/TMB runs.
+- `tsc` clean; `pytest` green; airgap tests untouched and passing.

Read docs/PROMPT_dsl_airgap_h1.md

@@ -0,0 +1,52 @@
+# Next chunk — DSL operators, airgap harness, H1 verification, and UI
+
+Implement the DSL operators for real, the airgap harness, the H1 verification, and a UI that visualises both the DSL and the H1 outputs. Do NOT build the genetic-programming engine yet. Reuse `data/processed`; no data re-download.
+
+**Key concept — two views of the same matrix:** a NAMED matrix (columns = gene symbols) used by the H1 fixture, and an ANONYMISED matrix (columns = opaque IDs) used by the discovery path. The DSL operators are label-agnostic: `Select` picks columns by whatever labels the passed matrix has.
+
+## 1. dsl/ — implement as a tested module
+- `Load(source)` -> Cohort: expression matrix, clinical (stage, age, sex, survival), labels (msi_status, tmb).
+- `Select(matrix, feature_ids)` -> submatrix of those columns (works for symbols or opaque IDs).
+- `Reduce(matrix, agg="mean")` -> one score per patient. Support "mean" only.
+- `Split(cohort, predicate)` -> (subset_true, subset_false).
+- `Associate(a, b, kind="pearson")` -> scalar (pearson and spearman). Observational only.
+- `Effect(cause, effect, adjust)` -> the PARTIAL CORRELATION of cause vs effect given the adjustment covariates (stage as categorical, age as continuous), so it is directly comparable to the unadjusted Associate. Drop rows missing any confounder and report the n used. Return both the partial correlation (adjusted) and the unadjusted pearson. Document it as observational backdoor adjustment, only as good as the measured confounders.
+- `Search(matrix, objective, k)` -> best <=k feature_ids. THIS chunk: a SIMPLE deterministic baseline (rank single features by the objective, take top-k), clearly marked as a placeholder the GP engine replaces next chunk. Objective is a function of (feature scores, labels) only — never gene identity. Run on the usable MSI-H-vs-MSS cohort only.
+- `Fit(state, cohort)` -> logistic regression from the state to the MSI-H-vs-MSS label (usable cohort only); `Apply` -> predicted probabilities.
+
+Add unit tests for each (shapes + a tiny hand-checkable case).
+
+## 2. airgap/
+- `anonymise(expression_matrix)` -> columns renamed to opaque IDs (g00001…); write the symbol<->ID mapping to a SEALED file `data/processed/_sealed_gene_map.json`.
+- `reveal(feature_ids)` -> translate IDs back via the sealed map; the ONLY function allowed to read the map; used only in a final reveal step.
+- Enforcement test: (a) the discovery module does not import the sealed map or `reveal`; (b) at runtime, assert the matrix passed to `Search` has columns matching `^g\d+$` only (no symbols). The H1 fixture is exempt (it legitimately uses names).
+
+## 3. H1 verification
+Fixture — gene names allowed; known-answer check, not blind discovery. Separate, clearly-labelled module, on the named matrix, usable MSI-H-vs-MSS cohort, with MMR = [MLH1, MSH2, MSH6, PMS2] and immune = [CD8A, GZMA, PRF1]:
+- `mmr_score = Reduce(Select(M, MMR), mean)`; `immune_score = Reduce(Select(M, immune), mean)`.
+- Show MSI-H tumours have LOW mmr_score and HIGH immune_score vs MSS.
+- `Effect(mmr_score, immune_score, adjust={stage, age})` — report adjusted partial correlation vs unadjusted pearson, and the n used.
+- `Fit` on (mmr_score, immune_score) -> held-out AUROC and balanced accuracy (NOT raw accuracy).
+
+## 4. Streamlit — add TWO new sections
+Calm style; each with a one-line "How to read this".
+
+### (a) "The DSL" — visualise the language
+- Operator reference: a compact list of the eight operators (Load, Select, Reduce, Split, Associate, Effect, Search, Fit/Apply) with a one-line plain meaning each. Reuse glossary wording; don't restate the full glossary.
+- The H1 program as a composition diagram via `st.graphviz_chart` using a DOT string (add `graphviz` to requirements — a deliberate, allowed new dependency for visualisation; verify it renders). Build exactly this graph (rankdir=LR):
+  - "Expression matrix" -> "Select(MMR genes)" -> "Reduce(mean)" -> "MMR score"
+  - "Expression matrix" -> "Select(immune genes)" -> "Reduce(mean)" -> "immune score"
+  - "MMR score","immune score" -> "Effect(adjust: stage, age)"
+  - "MMR score","immune score" -> "Fit → MSI-H probability"
+  - Style: node shape=box, rounded; operator nodes fill #F1EFEA, border #3A6B7E; the two SCORE nodes (MMR score, immune score) are "meta-concepts" — fill #FBEFE2, border #BC6B2E; edges #9AA0A6; calm fonts. This shows scores are composed from primitives.
+- Also show the same program as its literal one-line composition in code, e.g. `mmr_score = Reduce(Select(M, MMR), "mean")`.
+- Caption: "The same operators compose into 'meta-concepts' (scores) and then a causal test — this is the H1 program; the blind-discovery version comes next."
+
+### (b) "H1 verification" — the outputs
+- mmr_score and immune_score distributions split by MSI status.
+- the Effect adjusted partial correlation vs unadjusted pearson, with the n used.
+- the H1 AUROC and balanced accuracy.
+- an airgap demo: show the anonymised matrix head (opaque IDs) and that `reveal()` recovers the real names.
+
+## Finally
+Run the tests, then tell me the H1 numbers and how to view the two new sections.

Read docs/PROMPT_enable_search.md

@@ -0,0 +1,28 @@
+# Prompt — enable the Search operator (turn on the full DSL)
+
+Search is currently gated off by default; turn it on so the engine can use the whole DSL. Verified the exact location. Airgap untouched (Search operates on opaque IDs). `pytest` + `tsc` after.
+
+## 1. Enable Search injection (engine_v2/synthesize.py)
+`DEFAULT_RATES` (~lines 51-55) has `"search": 0.0`. Set it to a **modest non-zero** rate so Search appears in the population without dominating or blowing up runtime:
+- `"search": 0.05`
+Keep Search's existing safety **caps** (k ≤ 4 selected, ≤ 200 candidate columns) — those bound each Search's cost; don't remove them. The injection gate at ~line 109 (`rates["search"] > 0`) already does the rest. Leave the other rates (split/fitapply/effect) unchanged.
+
+## 2. Update the "Search" tile copy (web/app/Lab.tsx, DSL vocabulary)
+The Search tile says "Nested feature search — available but off by default." Change it to reflect that it's now on, e.g. "Nested feature search — runs a small gene-ranking inside the program (bounded: ≤4 genes from ≤200 columns)." Remove the "off by default" wording.
+
+## 3. (Recommended) a per-run toggle so it can be turned off for speed
+Search is the one heavy operator — every Search node runs an inner gene-ranking, so runs will be slower. To keep control:
+- `RunRequest` gains `enable_search: bool = True`. The `_worker` maps `enable_search=False` → pass a rates override with `search: 0.0`; `True` → use the default (0.05). (Thread the rates override into `run_v2_pipeline*` → `synthesize`, which already accepts a `rates` dict, ~line 212/220.)
+- Frontend: a Parameters checkbox **"Enable Search operator"** (default ON). Tooltip: "Lets programs run a small built-in gene-search (bounded). Powerful but the heaviest operator — turn off for faster runs."
+- If this part is too much scope, at minimum do Parts 1-2 (enable by default); the toggle is the nice-to-have.
+
+## CONSTRAINTS
+- Airgap untouched — Search ranks opaque columns, no gene names. 
+- Caps stay (k ≤ 4, ≤ 200 cols) so cost is bounded.
+- Note for the user: expect somewhat slower runs now that Search can fire; lower its rate or use the toggle if it's too slow.
+
+## Checkpoint
+- `DEFAULT_RATES["search"] = 0.05`; an HPV coherence run now shows Search used > 0× in the DSL-vocabulary operator-usage panel.
+- The Search tile no longer says "off by default".
+- (If done) an "Enable Search operator" toggle (default ON) flips its rate.
+- `pytest` green; airgap untouched; `tsc` clean.

Read docs/PROMPT_final.md

@@ -0,0 +1,93 @@
+# Prompt — engine_v2 full DSL grammar + objective explainers (FINAL, run this one)
+
+Two things in one: (A) make the engine compose programs from the ENTIRE DSL, and (B) upgrade the objective "?" help. This is the single source of truth — it supersedes the earlier Effect/Split and full-DSL drafts; do not run those as well. Strongly-typed GP over the complete operator set. Keep the airgap (genes opaque). Keep existing objectives/presets/Streamlit working (simple programs remain a valid region, so prior results still reproduce).
+
+## 0. READ FIRST (don't assume)
+Read the engine_v2 grammar, interpreter, GP operators, program_repr, the shared graph parser, the available data (expression matrix, clinical fields, labels), and the existing objective + tooltip code. Verify types/signatures/names and build on what's there.
+
+# PART A — Full DSL grammar
+
+## A1. TYPES (nouns)
+Cohort, Matrix, Vector (per-patient score), FeatureSet (opaque gene IDs), Predicate, Scalar, Model, Outcome.
+
+## A2. OPERATORS (verbs) — ALL of them, typed
+- `Load` → Cohort: the implicit ROOT — expression Matrix (opaque gene columns) + clinical {stage, age, …} + labels {msi, tmb}. Always present as the starting data; matrix genes stay opaque.
+- `Select(Matrix, FeatureSet)` → Matrix. FeatureSet terminal = 1..genes_per_set opaque IDs (full pool, or the prefilter shortlist when prefilter is on).
+- `Reduce(Matrix, Agg)` → Vector. Agg ∈ {mean, median, max, min, var}.
+- `Combine(Vector, Vector, Op)` → Vector. Op ∈ {add, sub, mul, div(safe), mean}.
+- `Split(Cohort|Vector, Predicate)` → two branches → recombine per-patient → Vector. Predicate is NAME-BLIND (a threshold on a Reduce'd score, or a clinical variable like stage). Guards: min subgroup size, ONE level of Split only (no Split-in-Split), deterministic recombine.
+- `Associate(Vector, target, kind)` → Scalar. kind ∈ {pearson, spearman}. Observational association with a target column (msi/tmb via Load).
+- `Effect(Vector, target, adjust=[stage,age])` → Scalar. Adjusted association = residualize the Vector on the confounders, then associate. Observational backdoor adjustment — "only as good as the measured confounders" (label it so).
+- `Fit(Vector|Vectors, labels)` → Model ; `Apply(Model, Cohort)` → Vector (predicted probabilities). Logistic for the MSI label.
+- `Search(Matrix, objective, k)` → FeatureSet. SEE A4 — self-referential; must be bounded.
+
+## A3. OUTPUT + FITNESS (handle multiple terminal types)
+A valid program outputs Vector, Scalar, or Model. Normalise to a single fitness given the objective's target + direction; everything on TRAIN (k-fold), TEST touched once for the winner:
+- Vector → apply the objective metric (AUROC for msi; signed correlation for tmb).
+- Model → Apply → predicted Vector → metric.
+- Scalar from Associate/Effect against the objective's target → that statistic IS the fitness (re-evaluated held-out, in the objective's direction).
+- The objective stays MINIMAL: target + direction + honest held-out evaluation. Everything else (genes, aggregations, adjust?, branch?, fit?) lives in the program.
+- Permutation null re-evaluates the winning program (its exact structure) on permuted targets.
+
+## A4. Search inside programs — the self-referential case (bounded)
+`Search` is what the GP engine itself does, so a program containing `Search` is a nested/meta-search. To honour "the entire DSL," INCLUDE it but strictly bounded: small k, shallow nested budget, hard time/size caps, memoised, introduced at a LOW rate. Document it as the recursive/meta operator. If nested search exceeds caps → degenerate (worst fitness). If it proves too costly, FLAG it and gate behind a toggle — do not silently drop it.
+
+## A5. TYPED GP (over the full grammar)
+- Type-safe init (ramped half-and-half), crossover, and mutation — only combine/replace subtrees whose return types match.
+- TERMINATION GUARANTEE: every type must be closable within max-depth. Provide a minimal closer for each type (Vector via Reduce(Matrix-terminal, agg); Scalar via Associate(Vector, target); etc.) so init/mutation never produce an unclosable tree.
+- One seed governs ALL randomness (init, selection, crossover, mutation, FeatureSet sampling, Search).
+- Bloat control: λ penalises node count; enforce max-depth and max-nodes; Search and Split obey these caps.
+- Degenerate guards: empty FeatureSet forbidden; protected division; constant/NaN output → worst fitness; Split min subgroup size.
+
+## A6. VISIBILITY
+Initialization and mutation must introduce the richer operators (Effect/adjust, Split, Fit, and—rarely—Search) at a non-trivial rate, so the population genuinely contains programs using them (visible on tiles and in graphs). They survive into the winner only if they improve held-out fitness.
+
+## A7. PARSER / GRAPH / FORMAT HELP
+- Extend the ONE shared parser + React Flow renderer to draw every node type: Select, Reduce, Combine, Split (branch→merge), Associate, Effect (adjust wrapper), Fit/Apply, Search (nested). Reuse existing node styling; add distinct but on-theme styles for new ones.
+- Update the "Format & examples" panel to document the full grammar with one short annotated example per operator.
+
+## A8. CONSTRAINTS
+- Airgap test green: genes opaque everywhere (engine, program_repr, population endpoint); only clinical fields (stage/age) and labels (msi/tmb) are named; no gene symbols leak.
+- Existing engine_v2 objectives/presets/Streamlit untouched; simple Select→Reduce programs remain valid so prior results reproduce.
+- The full grammar is a vastly larger search space — recommend a bigger default budget and note runs will be messier early; surface the expected runtime increase.
+- All new UI pieces are Client Components.
+
+## A9. OBJECTIVE FIX (signed-negative TMB)
+If not already done: the mutation-burden (TMB) objective must reward the SIGNED negative correlation between the program's score and TMB (low score ↔ high TMB), NOT |spearman|. Update every UI label that shows "association (|spearman|)" accordingly. Permutation null re-evaluates the winner under this signed objective.
+
+# PART B — Objective "?" explainers (UI, rich click-to-open)
+
+Upgrade the four Objective "?" icons to the SAME rich click-to-open explainer used for the parameters (hover = short summary; click = detailed modal). Reuse that component; presentation-only. ALSO FIX the typo in the Mutation burden tooltip: "brokNA spell-checker" → "broken spell-checker".
+
+Shared intro line (top of each modal): "An objective is the rule that scores every program — the fitness the engine maximises. It's computed only from a program's per-patient output and a target column (never gene names), so it can't smuggle in the answer. It sets what 'good' means; the engine then composes DSL programs to satisfy it."
+
+### MSI separation
+SHORT: "Reward programs whose score separates MSI-H from MSS (measured by AUROC). It rewards ANY separator, so it often finds shortcut genes, not the cause."
+DETAILED:
+- What it optimises: a program whose per-patient score ranks MSI-H patients above MSS.
+- Represented as: { target: msi (the subtype label), metric: AUROC }. The engine sees only the program's score + the MSI label — no gene names.
+- AUROC: pick a random MSI-H and a random MSS patient; AUROC is the chance the score ranks the MSI-H one higher. 0.5 = coin flip, 1.0 = perfect. We use orientation-agnostic AUROC (a perfectly inverted score isn't discarded) and AUROC rather than accuracy because MSI-H is a ~15% minority.
+- Caveat: MSI-H and MSS differ in thousands of genes, so this rewards consequences and coincidences as readily as causes — it tends to grab shortcut genes, not the MMR cause.
+
+### Mutation burden
+SHORT: "Reward programs whose score is negatively associated with mutation burden (low score ↔ high TMB) — the broken spell-checker's fingerprint. A sharper proxy for the cause, but still association, not proof."
+DETAILED:
+- What it optimises: a program whose score goes DOWN as mutations go UP — the directional signature of a broken DNA-repair gene (switch it off → mutations accumulate). The "spell-checker" is the MMR set (MLH1, MSH2, MSH6, PMS2).
+- Represented as: { target: tmb (total mutation count), metric: correlation, direction: negative }. Engine sees only the program's score + the TMB numbers.
+- Why "negative" (not magnitude): we reward the SIGNED negative correlation. Magnitude (|spearman|) would also reward genes that go UP with TMB — anti-MMR distractors — diluting the target.
+- Correlation ≠ causation (honest): this is a mechanism-SHAPED association — a sharper proxy for the cause than predicting the label, but it does not prove causation. Co-regulated or confounded genes can mimic the same low-expression↔high-TMB pattern. Recovering MLH1 would corroborate known biology, not prove it.
+- Toward causal: the program itself can choose to ADJUST for confounders via the Effect operator (negative correlation after removing stage/age) — the most causal the observational data honestly allows. The engine decides whether adjusting helps; we don't hardcode it.
+
+### Survival (Stage 2)
+SHORT: "Would reward programs whose score relates to how long patients survive. Not built yet."
+DETAILED: "Target = survival outcome/time; metric = association with survival (e.g. log-rank / hazard). Would reward scores that track prognosis. Not built yet (Stage 2)."
+
+### Unsupervised (Stage 2)
+SHORT: "Would give no target at all — reward programs that capture the strongest natural structure, then you interpret what they found. Not built yet."
+DETAILED: "No target column. Reward = how much real structure the score captures (e.g. variance explained / cluster separation), checked for robustness; you interpret the discovery afterward. Not built yet (Stage 2)."
+
+### Shared footer note (small, in each modal)
+"The metric and target above can be written in the DSL itself (Associate / Effect / Fit against a Load'ed label); what stays outside the DSL is the compass — what to maximise, in which direction, judged honestly on held-out data."
+
+# CHECKPOINT
+Run once and show me: (a) a few program_repr from the population that use the newer operators (Effect, Split, Fit, and any Search); (b) the winner's program_repr — which operators it chose; and (c) confirm the four objective "?" modals open with the new content and the typo is fixed. Confirm the airgap test is green and existing presets/Streamlit still work.

Read docs/PROMPT_fitness_synergy_scatter.md

@@ -0,0 +1,36 @@
+# Prompt — "GP fitness × synergy" landscape scatter, with HPV-marker groups highlighted
+
+Add a scatter above the merged "Groups the engine explored" table: x = GP fitness, y = synergy, one dot per explored group, with the known-marker groups (cell_cycle / p16) and the winner highlighted. Frontend-only — all data is already on the module row. **Depends on PROMPT_merge_groups_table.md being in first** (it adds `gp_fitness` and the merged panel). Verified field refs below. No engine/API/airgap change. `tsc` after.
+
+## Data (per module, from the already-fetched `ModuleRanking.modules`)
+- **x = `m.gp_fitness`** (added by the merge prompt; skip a module if null).
+- **y = synergy = `m.combined_holdout − max(g.single_gene_metric over m.per_gene)`** (both fields exist: `combined_holdout` ~api.ts line 295, `per_gene[].single_gene_metric` ~287/300; ignore genes with null metric; skip the module if synergy can't be computed).
+- **category / colour:**
+  - **winner** → teal, drawn with a ring + "winner" label. Identify via the same unordered gene-set equality the table uses (`isWinnerSet`, ~Lab.tsx 3596).
+  - **p16 marker** (HNSC) / **immune** (CRC) → gold. From `m.ref_sets` (string[], ~api.ts 303) containing the single-marker set name.
+  - **cell_cycle** (HNSC) / **MMR** (CRC) → amber. From `m.ref_sets` containing the broader set name.
+  - **everything else** → faint grey background dot.
+  - (Use the dataset's reference-set names — the same ones `ref_sets` reports; don't hardcode HNSC-only.)
+
+## Rendering — use a canvas (performance)
+There are ~6,000 modules, so do NOT emit 6k SVG nodes. Draw the dots on an HTML5 `<canvas>`:
+- Plot the faint grey background dots first (small, low alpha), then the **amber/gold/teal highlighted dots on top** (larger), then the winner (ring) last so it's never occluded.
+- Axes, tick labels, the two quadrant guide lines (vertical at the **median GP fitness**, horizontal at **synergy = 0**), the quadrant captions ("real teamwork" / "ideal — ≈ empty" / "engine's lone-gene detectors" / "weak/junk"), axis titles ("GP fitness — what the engine preferred →" and "Synergy — teamwork beyond best gene ↑"), and a legend (all explored groups / winner & picks / cell_cycle / p16) — draw these as the chrome (either on the canvas, or as an SVG overlay aligned to the plot rect; pick one and keep coordinates consistent).
+- Axis ranges: auto-fit to the data (x over the gp_fitness range, y over the synergy range) with a little padding; label a few ticks.
+- Palette: reuse the existing three-way colours (winner teal `#3A6B7E`, single-marker set gold, broader set amber) — the same `RANK_COLORS`/palette the table badges use, for consistency.
+
+## Placement + gating
+- Render it directly **above** the merged "Groups the engine explored" table, in the same panel/section (a "landscape" view of the same groups). Consider a short caption: "Each dot is a group the engine explored. Bottom-right = the engine's lone-gene detectors; top = genuine teamwork. Highlighted dots show where the known HPV markers fall."
+- Gate to runs where both axes exist: supervised target + coherence on (same gate as the survival flags / synergy column). Omit gracefully otherwise.
+
+## Optional (nice-to-have, not required)
+- Hover on a highlighted dot → small tooltip with its revealed gene symbols (symbols are already available for displayed modules). If hit-testing on canvas is too much, skip it — the table below already lets the user inspect groups.
+
+## CONSTRAINTS
+- Frontend-only: synergy + category are derived from fields already on the wire; no engine/API/airgap change, no extra reveal (only already-revealed/displayed symbols if the optional tooltip is added).
+- Don't choke the page: ~6k points on canvas, not SVG. Highlighted dots on top.
+
+## Checkpoint
+- A scatter appears above the merged groups table: x = GP fitness, y = synergy, faint dots for all explored groups, amber cell_cycle / gold p16 / teal-ringed winner highlighted, quadrant guide lines + labels + legend.
+- You can see at a glance which quadrant the HPV-marker groups fall into vs the engine's bottom-right picks.
+- Renders smoothly with ~6k points (canvas). `tsc` clean; no API/airgap change.

Read docs/PROMPT_fix_peeloff_leakage.md

@@ -0,0 +1,42 @@
+# Prompt — fix the supervised peel-off leakage (residualize on train only)
+
+A peel-off (Axis 2) HPV run produced a winner — `FitApply(Reduce(Select(M,[g16970]),median),hpv)`, gene SPACA1 — with held-out AUROC **0.963**, while SPACA1's honest single-gene HPV AUROC is **0.510**. That's impossible for a single-gene monotonic program (its AUROC must equal the gene's AUROC on the same data), so the held-out is contaminated. Root cause: residualisation is fit on the FULL cohort before the train/test split, so for a target-correlated prior axis it leaks the target signal into the held-out features. Read the named code first; don't assume. After: `pytest` green, airgap tests untouched, MSI/HPV/unsup runs sane.
+
+## The bug (engine_v2/pipeline.py)
+`_residualise_matrix(M, residualize_scores)` (~line 77) is called at **line 254–255** (`run_v2_pipeline`) and **line 476–477** (`run_v2_pipeline_streaming`), in both cases BEFORE `make_split` (lines 261 / 483). So the OLS projection that removes the prior axis is fit using ALL patients — including the held-out set. When the prior axis (Axis 1) is correlated with the supervised target (HPV/MSI), this smears the target signal into the test features and inflates held-out AUROC. (For unsupervised it was harmless — no target to leak.)
+
+## Part 1 — Residualise inside the split, fit on TRAIN only
+Restructure both pipeline entry points so NO preprocessing is fit using test rows:
+1. First drop rows missing a prior score (defining the cohort is fine — not leakage), keeping M and y aligned.
+2. `make_split` into train/test.
+3. Fit the residualisation projection — the OLS of each gene column on `[intercept, *prior_scores]` — using **train rows only** → `beta_train`.
+4. Apply `beta_train` to transform train AND test features (test is residualised with train-fit coefficients, never its own).
+5. Prefilter, GP, and `evaluate_holdout` then run on these correctly-residualised train/test matrices.
+- Refactor `_residualise_matrix` to either fit-and-return-beta separately from apply, or take an explicit `fit_index` (train) and apply to the whole frame. The key invariant: **the projection coefficients are a function of training rows only.**
+- Persisted `winning.full_scores` (full-cohort scores for the next axis): compute by applying the winner program to the full cohort residualised with the SAME train-fit projection — keep the chain consistent, still no test-fit preprocessing.
+- Apply this train-only-fit discipline for ALL objectives (correct for supervised, harmless-but-cleaner for unsupervised). Update any unsupervised peel-off test that assumed residualise-before-split.
+
+## Part 1b — Close the FitApply train/test leak (lower priority, do while here)
+Separately, `FitApply.execute` (engine_v2/nodes.py ~line 500) fits its model on `ctx.labels.get(self.target)` — the labels of whatever context it runs in. During `evaluate_holdout` that's the TEST context, so FitApply currently trains on the held-out labels it's being graded against. It's benign *today* (FitApply always fits on a single scalar score, so the result is monotonic and AUROC is unchanged — this is NOT what produced the 0.963), but it breaks the held-out-honesty guarantee the moment FitApply takes more than one input. Fix it: FitApply must fit its model on TRAIN and apply the frozen model to TEST — never fit on the test labels. (Mechanism is yours: e.g. fit during the train pass and carry the fitted model, or pass a fit-context. Keep behaviour identical for the single-score monotonic case so existing AUROCs don't move.) Also note: only `target=="msi"` gets logistic regression; `"hpv"` falls through to the continuous OLS branch — harmless for AUROC (still monotonic) but tidy it to route binary HPV through the same binary path as MSI for clarity.
+
+## Part 2 — Leakage guard + regression test
+Add a backstop so this class of bug can't silently win again:
+- **Invariant test:** a program that is monotonic in a single gene (one `Select` of one gene, with only monotonic `Reduce`/`FitApply` on top, no `Combine`/`Split`) MUST have held-out AUROC equal — within a small tolerance (e.g. 0.02) — to that gene's single-gene held-out AUROC on the same test set. Add a test that constructs such a program on residualised data and asserts the equality holds after the Part-1 fix (it currently fails, ~0.96 vs ~0.51).
+- **Runtime guard (conservative):** for a winner that is single-gene-monotonic, if its held-out AUROC exceeds its constituent gene's single-gene held-out AUROC by more than the tolerance, treat it as contaminated — floor to the objective's worst score (or refuse it) rather than report it. Do NOT apply this to genuinely multi-gene/`Combine` programs, where synergy beyond single genes is legitimate.
+
+## Part 3 — REPRODUCE FIRST, then prove the fix (don't assume the cause)
+The exact mechanism that turns SPACA1 (single-gene AUROC 0.510) into a 0.963 held-out is NOT yet proven — it is most likely the residualise-before-split leak (possibly via a statistical *suppression* effect that the full-cohort fit amplifies onto the test rows), but confirm it empirically rather than assuming:
+1. **Reproduce deterministically first.** Re-run the user's exact case (HPV/HNSC, coherence ON, same seed, then "find next axis") on the CURRENT code and confirm you see the SPACA1 winner at held-out ≈0.96. Instrument: print resid-g16970's single-gene AUROC vs HPV on the test set both before and after the Part-1 fix.
+2. **Then prove the fix resolves THIS case.** After moving residualisation to train-only-fit, the same run must drop the SPACA1-shape winner's held-out to ≈ its honest single-gene AUROC (~0.51). If it does NOT fully collapse, the leak has another source — keep instrumenting (check `_make_full_ctx`, below) until the 0.96 is explained and gone. Do not close this out on the invariant test alone.
+3. **Verify the supervised full-scores path.** `_make_full_ctx` (engine_v2/pipeline.py ~line 110) is documented unsupervised-only ("Labels are always empty"), but supervised peel-off needs Axis 1's full-cohort scores to residualise against. Confirm how those scores are produced for a supervised winner and that it's correct (a winner containing `FitApply(...,hpv)` executed with empty labels returns its raw inner vector, not the fitted prediction — decide whether that's the intended residualisation target and document it). This may itself contribute to the inflation.
+- Sanity property: a supervised Axis 2 held-out should be **≤ Axis 1** (residualising removes target-aligned signal; you can't gain separation by removing it). Assert this on a real re-run.
+- Confirm MSI and unsupervised runs are unaffected in behaviour (aside from the now train-only residualisation), and the airgap tests stay green (payloads opaque; the fix touches only split/preprocessing order).
+
+## CONSTRAINTS
+- Airgap absolute; engine stays name-blind. The fix is split/preprocessing ordering + a guard — no new data crosses the wire.
+- Non-peel-off runs (no `residualize_scores`) must be byte-for-byte unchanged.
+
+## Checkpoint
+- Residualisation projection is fit on train rows only and applied to test; the residualise-before-split calls are gone.
+- The single-gene-monotonic invariant test passes (held-out ≈ single-gene AUROC); the SPACA1-shape case is no longer reported as a 0.96 win.
+- Supervised Axis 2 ≤ Axis 1 on a real re-run; MSI/HPV/unsup otherwise unchanged; `pytest` + airgap green.

Read docs/PROMPT_gp_top10_groups.md

@@ -0,0 +1,39 @@
+# Prompt — expand "The GP's group" to the GP's top 10 programs
+
+Right now the Result panel's "The GP's group" block shows only the #1 winner (its genes' individual ranks + confound survival). Expand it to the **GP's top 10 programs, ranked by the engine's own fitness** — each with the same treatment. Verified against the code. Presentation + frontend data-flow only; no engine/API/airgap change. `tsc` after.
+
+## What "top 10" means here (keep it distinct from Coordinated modules)
+Rank by the **GP's own fitness** (what the search actually preferred) — NOT by the combined-AUROC re-score. This is deliberately different from the Coordinated modules leaderboard (which re-scores all explored groups by a different metric). Label it so the two don't blur: e.g. heading "The GP's top programs" with a one-line note "ranked by the engine's own fitness — what the search preferred. (The Coordinated modules panel below re-scores all explored groups by a different metric.)"
+
+## Data sources (verified)
+- The GP's programs + fitness live in the persisted population: `GET /runs/{id}/population/{generation}` returns `candidates` each with `fitness`, `gene_ids`, `program_repr` (engine_v2/gp.py builds these). `GET /runs/{id}` exposes `generations_persisted`.
+- Per-gene single-gene ranks come from the full-rank diagnostic (`getFullRankDiagnostic(dataset, target)` → `diag.ranks`) — already fetched in the Result panel after the consolidation change.
+- Confound survival comes from the module ranking (`getModuleRanking(runId)`) — match a program to its module by **unordered gene-set equality** (the same notion `ModuleRankingPanel.isWinnerSet` uses).
+
+## Build the top-10 list (web/app/Lab.tsx, ResultPanel)
+1. Fetch the **last persisted generation's** population (`generations_persisted - 1`). Sort its `candidates` by `fitness` descending, dedupe by gene-set (unordered), take the top 10 distinct programs. (#1 should be the winner — keep it visually highlighted as today.)
+2. Render each of the 10 as a compact row reusing the existing "GP's group" treatment:
+   - **Genes, each on its own:** for each gene id, look it up in `diag.ranks` → `SYMBOL #rank / N` (reveal symbols via the existing bounded reveal; reveal only these displayed programs' genes — never the whole map).
+   - **Confound survival:** find the program's module in the module data by gene-set equality → render `SurvivalChips` (the pre→post `full → subgroup` chips). Omit gracefully where survival isn't available.
+   - Show the program's GP fitness (and gene count) so the ordering is legible.
+3. Keep it compact (10 rows); the winner (#1) stays highlighted. Bounded reveal: only the genes of these ≤10 programs.
+
+## Part — rewrite the Coordinated modules caption + "?" in plain language
+The current copy is dense and circular. Rewrite both, in plain English, to clearly say what this panel is and how it differs from "The GP's top programs" above.
+- Inline subtitle (web/app/Lab.tsx ~line 3861): replace with:
+  > "Not the engine's picks. After the run, this re-scores **every** gene group the engine tried — using one simple number (the group's average expression, measured on held-out patients) instead of how the engine judged groups during the search. So this list can rank groups differently from *The GP's top programs* above, and its #1 can even beat the engine's winner."
+- The "?" tooltip `TIPS.moduleRanking` (~line 217): replace with plain copy covering three things:
+  > "What this is: after the run finishes, we take every group of genes the engine explored, ignore how the engine combined them, and give each group one score — the average of its genes, on patients held out of training.
+  > How it differs from 'The GP's top programs' above: that panel is the engine's actual choices, ranked the way the engine judged them during the search. This panel is a separate, after-the-fact re-scoring with a simpler yardstick — so the order differs, and a group here scoring higher than the engine's winner does NOT mean the engine was wrong.
+  > Caveat: this tries ~2,900 groups on the same small held-out set and shows the best, so the very top scores are optimistically biased (the luckiest of thousands) — trust the engine's own picks above as the reliable choice."
+- Check the related line ~200 (the single-gene/highlighted tip that references the modules leaderboard) and make sure its wording is consistent with the above.
+
+## CONSTRAINTS
+- Presentation + frontend data-flow only; no engine/API/airgap change. Reveal stays bounded to the displayed programs' genes.
+- Do not re-introduce group/individual mixing: each program's *group* identity is the row; the per-gene `#rank` values are clearly that program's *individual* genes' solo ranks — labelled as such, exactly like the current single-program block.
+
+## Checkpoint
+- "The GP's group" becomes "The GP's top programs" showing 10 rows ranked by GP fitness, #1 = winner (highlighted).
+- Each row shows its genes' individual single-gene ranks + its confound-survival chips.
+- A one-line note distinguishes this (GP fitness) from the Coordinated modules leaderboard (combined-AUROC re-score).
+- Reveal bounded to the ≤10 displayed programs; `tsc` clean; no API/airgap change.

Read docs/PROMPT_gse65858_transfer_test.md

@@ -0,0 +1,211 @@
+# Prompt — external-cohort transfer test (validate the HPV winner on GSE65858)
+
+Add a second, independent HPV cohort (GEO **GSE65858**, ~270 head & neck tumours,
+HPV16 virus-typed) as a **reveal-side validation asset** and a **transfer test**:
+score the winning HNSC/HPV program — discovered blind on TCGA — on this
+independent cohort, and show the result in the Lab.
+
+This is **purely additive**. Do not change the engine, the search, the airgap
+seal, the existing datasets, or any current UI behaviour. GSE65858 is NEVER a
+runnable dataset — the engine must not search on it. It lives on the **named
+(reveal) side** of the airgap, exactly like `validate/` and the H1 fixture.
+
+`pytest` + `tsc` after. Keep the whole change behind the HNSC/HPV objective so
+colorectal is untouched.
+
+---
+
+## Airgap discipline (read first — this is the credibility of the whole feature)
+- GSE65858's expression matrix is keyed by **real gene symbols** and lives with
+  the other named-side code (`validate/`). It is NOT anonymised, gets NO sealed
+  map, and is NEVER passed to `engine`/`engine_v2`.
+- The ONLY thing that crosses from the blind side to GSE65858 is the winning
+  program's **gene symbols**, obtained via the existing bounded `reveal()` of
+  just the winner's opaque IDs — the same pattern `/evaluate` already uses. The
+  sealed map is never dumped; GSE65858's full gene list never crosses back into
+  the engine.
+- Net effect: the engine still discovers blind on TCGA; GSE65858 only ever sees
+  the small revealed gene set, on the reveal side. State this in the code
+  comments so the discipline is auditable.
+
+---
+
+## Part 1 — Data pipeline: ingest GSE65858 (GEO, not cBioPortal)
+
+GSE65858 is a **GEO** series (platform GPL10558, Illumina HumanHT-12 v4
+microarray) — a different source than the cBioPortal datahub, so this needs its
+own small download/build, mirroring the *structure* of
+`data_pipeline/download_hnsc.py` + `build_hnsc.py` (provenance printing, loud
+failures, parquet outputs) but with GEO-specific ingestion.
+
+Add schema constants to `data_pipeline/schema.py` next to the HNSC block:
+- `GSE65858_GEO_ID = "GSE65858"`
+- `GSE65858_SERIES_MATRIX_URL` = the GEO series-matrix gz
+  (`https://ftp.ncbi.nlm.nih.gov/geo/series/GSE65nnn/GSE65858/matrix/GSE65858_series_matrix.txt.gz`)
+- `GSE65858_PLATFORM_URL` = the GPL10558 annotation (probe → Symbol) — either the
+  GEO GPL soft/annot file, or note that the build can map probes → symbols via a
+  bundled/queried annotation. If a live platform fetch is awkward, allow a
+  manual-download fallback message like the HNSC downloader does.
+- `GSE65858_RAW_DIR = REPO_ROOT / "data" / "raw_gse65858"`
+- `GSE65858_PROCESSED_DIR = REPO_ROOT / "data" / "processed_gse65858"`
+
+`data_pipeline/download_gse65858.py` (mirror `download_hnsc.py`):
+- Fetch the series matrix gz (and platform annotation) into `raw_gse65858/`.
+  Skip if already present; loud manual-fallback instructions on failure (same
+  shape as HNSC).
+
+`data_pipeline/build_gse65858.py`:
+- Parse the series matrix: the `!Sample_*` header block carries the per-sample
+  metadata (including the HPV characteristics), and the matrix body is
+  probe × sample expression.
+- **HPV label**: derive `hpv_status ∈ {HPV+, HPV-}` from the sample
+  characteristics. Use the **transcriptionally-active virus** definition — HPV16
+  **DNA+ and RNA+ → HPV+**, everything else → **HPV-** (DNA+/RNA- behaves like
+  HPV- biologically). If the matrix exposes a single called HPV field, use it;
+  otherwise combine the DNA and RNA characteristic columns. Print which
+  field(s) the label came from (provenance), and the counts (HPV+ / HPV-).
+- **Probe → symbol**: map probes to HUGO symbols via the GPL10558 annotation;
+  drop probes with no symbol; when multiple probes map to one symbol, collapse
+  to a single row (mean, or max-mean-intensity probe — pick one and note it).
+- Write `data/processed_gse65858/{clinical.parquet, expression.parquet}`
+  (`expression` = symbols × samples; `clinical` = one row per sample with at
+  least `sample_id`, `hpv_status`). Gitignore `data/raw_gse65858/` and
+  `data/processed_gse65858/` (mirror the HNSC gitignore entries).
+- Print a COHORT summary like `build_hnsc.py` (total, HPV+ / HPV-, n genes).
+
+If any of the GEO parsing specifics are ambiguous at implement time, **stop and
+print exactly what was found** (available characteristic fields, first columns)
+rather than guessing a label — same discipline as the HNSC HPV-column finder.
+
+---
+
+## Part 2 — `validate/transfer_gse65858.py` (named side; gene symbols allowed here)
+
+A single pure, testable function — no network, no engine import:
+
+```
+def transfer_score(symbols: list[str], *, n_permutations: int = 1000, seed: int = 0)
+    -> dict
+```
+- Load `data/processed_gse65858/{expression,clinical}.parquet`.
+- Keep patients with a called HPV status and complete expression for the
+  requested symbols; intersect `symbols` with the cohort's genes (record
+  `found` / `missing`).
+- **Standardize each found gene within GSE65858** (z-score across patients),
+  take the **per-patient mean** across the found genes = the transfer score.
+  (This is the SAME aggregate the module-ranking "Combined AUROC" already uses;
+  standardizing per gene is the cross-platform fix so TCGA RNA-seq scale vs
+  array intensity doesn't matter.)
+- Metric: **orientation-agnostic AUROC** `max(AUROC, 1-AUROC)` of the transfer
+  score vs `y = (hpv_status == "HPV+")`.
+- **Permutation null**: shuffle `y` `n_permutations` times, recompute AUROC;
+  `p = fraction of nulls ≥ observed`.
+- Return: `{auroc, p, n, n_pos, n_neg, n_found, n_missing, found_symbols,
+  missing_symbols}`. Finite-guard everything (no NaN/Inf).
+- Tests in `tests/test_transfer_gse65858.py` use a **synthetic** GSE65858-shaped
+  fixture (no network): a signal gene that separates HPV+/- → AUROC ≫ 0.5 and
+  small p; a noise-only gene set → AUROC ≈ 0.5, p ≈ large.
+
+---
+
+## Part 3 — API: `GET /runs/{run_id}/transfer`
+
+Mirror the structure/guards of `GET /runs/{run_id}/modules`:
+- Gate to **`run.dataset == "hnsc"` and `run.target == "hpv"`** only; anything
+  else → 400 ("transfer validation is only defined for the HNSC/HPV objective").
+  425 while the run is still going, 404 unknown run.
+- Take the winner's opaque gene IDs from the run result
+  (`run.result["winning"]["gene_ids"]`). **`reveal()`** them to symbols (bounded
+  — only the winner's genes). Call `validate.transfer_gse65858.transfer_score(symbols)`.
+- Return the `transfer_score` dict plus a small header:
+  `{cohort: "GSE65858", platform: "Illumina HumanHT-12 v4 microarray",
+    n_cohort, source: "GEO"}`. Run everything through `_json_finite()`.
+- **Airgap payload rule**: the only gene NAMES in the response are the winner's
+  own revealed symbols (`found_symbols` / `missing_symbols`) — the same bounded
+  reveal `/evaluate` already returns. Do NOT include GSE65858's full gene list
+  or any sealed-map contents. Cache per run.
+- Lazy-import `validate.transfer_gse65858` inside the handler (like the other
+  diagnostics) so the API import graph doesn't pull it in eagerly.
+
+---
+
+## Part 4 — Frontend: an "Independent-cohort validation" panel in the Result section
+
+- `web/lib/api.ts`: add `TransferResult` interface + `getRunTransfer(runId)`.
+- New `<ExternalValidation>` component appended at the **end of the existing
+  Result section** — after the "Winning program" line, not between the metric
+  cards and the program. Everything currently in the Result panel (verdict
+  callout, "What does this mean?", the four held-out metric cards, the winning-
+  program line + Copy button) stays exactly as-is. Gated to
+  `dataset === "hnsc" && target === "hpv" && runId`. Fetch on run completion
+  (from the SSE `done` handler, like the modules/operator-usage fetches). Reuse
+  the existing `<SectionCard>` chrome, the palette tokens, the metric-card style,
+  the green verdict-callout style, and `<InfoTip>` — it must look native to the
+  current HPV Result section, not bolted on.
+- **Exact layout** (top → bottom; this mirrors the approved mockup):
+  1. SectionCard title "Independent-cohort validation" + `?`, subtitle:
+     *"The winning pattern was found blind on TCGA. Here it is applied,
+     unchanged, to a completely separate group of head & neck patients — a
+     different country, hospital, and measuring machine."*
+  2. Green verdict callout (reuse the held-out `ResultVerdict` green style):
+     *"It holds on strangers. The pattern still separates HPV+ from HPV− in an
+     independent cohort — AUROC {auroc}, and only {p×100}% of random labellings
+     do this well (p {p})."* Use the amber "found nothing reliable" variant when
+     `auroc < 0.75` or `p ≥ 0.05`.
+  3. Three metric cards (same style as held-out cards): **Independent AUROC**
+     `{auroc}` / sub "GSE65858 · {n} patients"; **Permutation p** `{p}` / sub
+     "beats random labels"; **Genes measurable** `{n_found} / {n_found+n_missing}`
+     / sub "of the winner's genes". Each with a plain `?`.
+  4. Amber cross-platform pill (reuse the amber tint): *"Validated across two
+     platforms — TCGA RNA-seq → GSE65858 microarray. Agreement across different
+     machines is a stronger result, not a weaker one."*
+  5. A **"How this validation works"** block (teal accent header) with a
+     three-step row of light-teal boxes: **1 · reveal** ("Unmask only the
+     winner's handful of genes — the one thing that crosses over."); **2 · match
+     & level** ("Find those genes in GSE65858 and put them on a common scale, so
+     RNA-seq vs microarray doesn't matter."); **3 · score** ("Average them per
+     patient and check how well that separates HPV+ from HPV− — against the
+     virus-typed labels.").
+  6. A teal airgap footnote with a lock icon: *"The airgap holds. The engine
+     never sees this cohort. Only the winning gene names cross over, once, at
+     the end — so a result here is genuine rediscovery, not a lookup."*
+  7. A muted one-liner: *"One honest note. This checks the gene signature the
+     winner chose. For a simple winner that is the whole program; for a
+     multi-part one it's the gene set rather than the exact arithmetic — which,
+     across two platforms, is the more honest thing to measure anyway."*
+  - The three metric-card `?` InfoTips: Independent AUROC = what an independent
+    cohort proves (kills "you overfit TCGA"), 0.5 coin-flip / 1.0 perfect;
+    Permutation p = how often random labels score this well; Genes measurable =
+    array cohorts don't carry every gene, so some of the winner's genes may be
+    absent — the score uses those that are present.
+- If the winner has too few genes present in GSE65858 (`n_found == 0`) or the
+  cohort load fails, render a graceful "couldn't validate externally" note — no
+  crash, no fake number.
+
+---
+
+## CONSTRAINTS
+- **Airgap absolute.** GSE65858 is named-side only (`validate/`, parquet). Engine
+  and `engine_v2` untouched; they never import the transfer module or the new
+  parquets. The only crossover is the winner's revealed symbols (bounded).
+- **Additive only.** No change to `DATASET_TARGETS`, `REFERENCE_SETS_BY_DATASET`,
+  the run flow, the existing objectives, or colorectal. GSE65858 does NOT appear
+  in the cancer selector and is NOT runnable.
+- Existing tests stay green; add the new transfer tests. The API airgap suite
+  must still pass — extend it with a test that the `/transfer` payload carries
+  only the winner's revealed symbols (no full GSE65858 gene dump, no sealed map).
+- Note for me (the user): GSE65858 uses the strict virus-active HPV+ definition
+  (DNA+ & RNA+); print the counts so I can sanity-check them.
+
+## Checkpoint
+- `python -m data_pipeline.download_gse65858 && python -m data_pipeline.build_gse65858`
+  writes `data/processed_gse65858/*.parquet` and prints HPV+/HPV- counts +
+  provenance.
+- `validate/transfer_gse65858.py::transfer_score(symbols)` returns AUROC + p +
+  found/missing on a synthetic fixture; `tests/test_transfer_gse65858.py` green.
+- `GET /runs/{id}/transfer` (HNSC/HPV run) returns AUROC + p + the winner's
+  revealed symbols only; 400 for non-HPV, 425 mid-run, 404 unknown.
+- Result panel shows the independent-cohort AUROC + p + two-platform note for an
+  HNSC/HPV run; hidden for colorectal.
+- `pytest` green; airgap suite green (incl. the new `/transfer` payload test);
+  `tsc` clean.

Read docs/PROMPT_hnsc_copy_rewrite.md

@@ -0,0 +1,96 @@
+# Prompt — rewrite all HNSC / HPV Lab copy for a non-specialist reader
+
+Copy-only pass across the Head & Neck (HPV) Lab UI so it reads self-explanatorily for co-founders with mixed cancer/AI background — like a dynamic document. No logic, engine, API, or airgap change; only visible text and tooltip/modal copy. `tsc` after. Apply the verbatim replacements below; where a string isn't listed, bring it into the same voice using the Voice spec.
+
+## Voice spec (apply everywhere)
+- **Plain, not dumbed-down.** Explain the idea in one clear sentence a smart non-expert gets; don't just delete jargon, translate it. First time a needed term appears, gloss it inline: *AUROC (0.5 = coin-flip, 1.0 = perfect)*, *held-out (patients the engine never trained on)*, *permutation p (how often random noise matches this)*.
+- **Cut implementation *plumbing*** — "Server-Sent Events", "worker thread", "on the backend". Say what it *means*, not how it's wired. BUT this is different from the *integrity* ideas (blind to gene names; scored on unseen patients; checked against chance) — those are the credibility of the whole project and MUST stay and be explained (see "Credibility points"). Replace jargon *words* ("opaque IDs", "airgapped", "residualise") with plain phrasing, but never drop the *concept*.
+- **Brevity and elegance.** One or two sentences per caption; one short paragraph per "?". If a sentence can go without loss, cut it.
+- **Consistent nouns:** the hidden gene codes = "hidden gene codes"; the engine = "the engine"; a program = "a program"; genetic-programming = spelled out.
+
+## Section-by-section replacements (verbatim)
+
+**Page header**
+- Subtitle → "The engine hunts for a gene-expression pattern that predicts a cancer trait — blind: gene names are hidden as codes so it can't lean on what we already know. You set the goal; it evolves a small program to hit it; then we reveal the genes it chose and check them against known biology."
+- Drop the "API: localhost:8000" line from the visible header (or move it out of the reader's way).
+
+**Cancer / problem**
+- Subtitle → "Which cancer to study. Picking one swaps the biology, the goals, and the reference genes below — the engine itself stays the same."
+
+**Objective**
+- Subtitle → "What you're asking the engine to find."
+- HPV detection card sub-label → keep "AUROC of HPV+ vs HPV−" but add the gloss in its "?" (below).
+- obj_hpv modal (rewrite to): "**HPV detection.** Head & neck cancers split into two kinds: those caused by the HPV virus (HPV+) and those that aren't (HPV−). The engine looks for a gene-expression pattern that tells the two apart. Score = **AUROC**: 0.5 is a coin-flip, 1.0 is perfect. Because only ~15% of tumours are HPV+, we use AUROC rather than plain accuracy. Important: this is *detecting a known viral fingerprint* (the virus switches specific genes on), not discovering a new cause — the cause is the virus itself."
+
+**What is HPV? (the biology)**
+- Keep the collapsible; its diagrams now carry the explanation. Caption (if any) → "How HPV drives these cancers — the two brakes it breaks, and why one gene (p16) becomes its fingerprint."
+
+**Parameters**
+- Subtitle → "How hard the engine searches. Bigger numbers explore more but run slower; results shift with the random seed, so try a few and compare."
+- Keep the "How these parameters relate" disclosure. Ensure each param "?" reads in plain language (Generations = how many rounds of evolution; Population = how many programs per round; Genes/set = how many genes a program may pick; etc.) — bring any that don't into the Voice spec.
+
+**DSL vocabulary**
+- Panel subtitle → "The building blocks a program can use. The engine combines them freely; you can nudge how often it reaches for the optional ones."
+- Keep the tile descriptions but ensure they're plain (e.g. Select = "Pick specific genes."; Reduce = "Combine those genes into one score per patient."; Combine = "Merge two scores into one."). Keep the rate "?" and Scalar-share copy already rewritten.
+
+**Run**
+- Subtitle → "Start the search." (drop "on the backend".)
+
+**Live view**
+- Subtitle → "Watch the engine improve, round by round." (cut the Server-Sent Events sentence entirely.)
+- Fitness-curve "?" → "Each round, the engine keeps the best programs and breeds variations. The solid line is the best program so far; the dashed line is the typical one. When they meet and flatten, the population has converged — it's stopped finding anything new."
+- Keep the axis title plain: "separation (AUROC — 0.5 coin-flip, 1.0 perfect)".
+
+**Population**
+- Subtitle → "Every program in this round. Darker = higher score; outlined = kept for the next round."
+
+**Program graph**
+- Subtitle → "The winning program, drawn as a tree: pick genes → turn them into a score → out comes an HPV+ probability."
+
+**Result**
+- Subtitle → "What the engine settled on, judged on patients it never trained on."
+- Metric card labels + "?":
+  - Held-out AUROC "?" → "How well the winner separates HPV+ from HPV− on patients it never saw during the search. 0.5 = coin-flip, 1.0 = perfect. This is the honest test that it didn't just memorise the training patients."
+  - Permutation p "?" → "How often a random program would score this well by luck. Small = unlikely to be a fluke."
+  - Nodes "?" → "How many steps the program has."
+  - Genes "?" → "How many genes the program uses."
+- Verdict copy: keep the plain "Real result: it holds up on unseen patients (X) and beats chance (p Y)." and the "What does this mean?" breakdown; ensure the success line says it *separates HPV+ from HPV−*, not "recovered the signature".
+- "The GP's group" / top-programs block heading → "The engine's top programs" with sub: "Its actual picks, ranked by how the search judged them. Each shows its genes and where each gene ranks on its own."
+
+**Known-marker recovery**
+- Subtitle → "Where the textbook HPV genes land on their own. The engine never sees this — it's how we check whether the known biology is even findable in the data."
+- Keep the chart title plain: "Where the known markers rank (1 = best single-gene separator)".
+
+**Groups the engine explored**
+- Subtitle → keep the plain four-lens version already written (GP fitness / Combined AUROC / Coherence / Synergy), but ensure each lens is glossed in one clause.
+- Column "?"s in plain language: Coherence = "how tightly the group's genes move together"; Synergy = "how much the group beats its single best gene (near 0 = one gene carrying passengers)".
+- "Survives" "?" → keep the plain site/purity explainer (already written).
+
+**The fitness × synergy scatter**
+- Heading + caption plain: "Every group the engine explored. Left–right = how much the engine liked it; up–down = how much the group beats its best single gene. Highlighted dots show where the known HPV genes fall."
+- Keep the quadrant captions short and plain.
+
+**Operator-usage (DSL tiles after a run)**
+- Caption → "How often the engine reached for each building block across the whole search. Select and Reduce are the backbone; the rest show up only where they helped."
+
+## Credibility points — KEEP and explain plainly (do NOT cut or minimise)
+These are what make the result believable and worth showing a co-founder — surface them clearly, in plain words. They are the opposite of superfluous.
+- **Blind discovery (the airgap).** The engine never sees gene names — only anonymous codes — so it can't lean on what's already known. Recovering real biology this way is a genuine *rediscovery*, not a lookup. Carry this in the page intro, at the reveal step, and in "Known-marker recovery" ("the engine never sees this"). Replace the jargon ("opaque IDs", "airgapped matrix") with plain phrasing — e.g. "gene names are hidden as codes so the engine can't cheat" — but keep the point front and centre.
+- **Honest scoring (held-out).** Every score is on patients the engine never trained on, so it can't just memorise the training set. Say this plainly at the Result.
+- **Not luck (permutation).** Scores are compared against random programs — a real result has to beat chance. Keep it.
+- **Reveal only at the end.** Gene names are unmasked once, *after* the search, purely to check the answer against known biology. Keep it — it's the punchline of the blind-discovery story.
+
+## What to cut (pure plumbing only)
+- "Every generation arrives over Server-Sent Events as the engine evolves." → the plain "Watch the engine improve, round by round." (keep the idea, drop the transport detail).
+- "on the backend" (Run).
+- The literal "API: …" line in the header.
+- The jargon *words* "opaque IDs", "airgapped", "residualise", "SSE", "worker thread" — but restate the *concept* in plain terms wherever it carries meaning (especially the airgap — see Credibility points). Never drop the idea, only the jargon.
+
+## CONSTRAINTS
+- Copy-only. No engine/API/airgap/logic change. Keep all existing behaviour, gates, and per-target/per-dataset switching — only the words change.
+- Where a string isn't named above, rewrite it to the Voice spec (plain, brief, glossed) — don't leave jargon behind.
+
+## Checkpoint
+- A non-specialist can scroll the HNSC/HPV Lab top-to-bottom and understand each section and every "?" without outside help.
+- No "Server-Sent Events", "opaque IDs", "airgapped", "residualise", "SSE" in visible copy; needed terms (AUROC, held-out, permutation, coherence, synergy) are glossed on first use.
+- `tsc` clean; no behaviour change.

Read docs/PROMPT_hpv_dataset.md

@@ -0,0 +1,59 @@
+# Prompt — Add HPV (head & neck) as a second dataset, with a top-level cancer selector
+
+Add the TCGA HNSC (head & neck) cohort and an HPV-detection objective, behind a new top-level "Cancer / problem" selector. The DSL, engine, airgap, parameters, live view, population, program graph, and result/verdict are SHARED and unchanged — only the *dataset-specific config* swaps (objectives, biology panel, reference sets, diagnostics, post-hoc targets). Read the named code first; don't assume. Do this in two stages.
+
+GUIDING PRINCIPLE: the engine is single. Everything that changes per cancer is config around it.
+
+=================================================================
+## STAGE 1 — Backend: the HNSC dataset + HPV objective
+=================================================================
+
+### 1a. Data pipeline (the main lift)
+- FIRST verify the data source: confirm the cBioPortal TCGA HNSC study id and that its clinical data carries **HPV status** (HPV+/HPV−). Mirror the COADREAD provenance approach documented in CLAUDE.md (LFS-resolved media URL). State the study id + HPV field you used.
+- Build a second processed cohort alongside the colorectal one: HNSC `expression.parquet` + `clinical`/labels including an **`hpv_status`** label (HPV+ / HPV−), restricted to samples with a called HPV status and complete expression. Put schema constants (an `HNSC_PROCESSED_DIR`, HNSC filenames, the HPV-status column + how +/− is derived) in `data_pipeline/schema.py` — do not hardcode in build. Ensure the build writes `hpv_status` into the labels so `cohort.labels["hpv_status"]` exists after Load.
+- LOADING — IMPORTANT: `dsl.Load` (operators.py ~line 62) only schema-resolves the exact string `"processed"`; ANY other string is treated as a *literal directory path*. So do NOT write `Load("processed_hnsc")`. Instead, EITHER (a) extend `Load` to map named sources (`"processed"`→COADREAD dir, `"processed_hnsc"`→`schema.HNSC_PROCESSED_DIR`), OR (b) pass the HNSC directory Path straight to `Load(hnsc_dir)` — Load's existing `else` branch already loads any directory containing clinical.parquet + expression.parquet, so option (b) needs no change to Load.
+
+### 1b. Objective
+- Add a binary objective for HPV: spec `{ target: "hpv", metric: "auroc" }`, reusing the SAME orientation-agnostic binary-AUROC machinery as MSI. Concretely: extend `V2Objective.target` (currently `Literal["msi","tmb","none"]`) to include `"hpv"` with `binary=True`; route its `score_vector` through the AUROC path and `worst_score`→0.5 (identical to MSI); add an `HPV_OBJECTIVE` and dispatch `hpv+auroc` in `engine_v2.fitness.objective_from_spec`. No new operators.
+
+### 1c. Data loading per (dataset, target)
+- `_prepare_lab_data` (api/app.py ~line 177) is currently keyed by target and always uses `Load("processed")` (colorectal). Generalise it to key by **(dataset, target)**: add a `dataset` argument; for `dataset="hnsc", target="hpv"`, load the HNSC cohort, anonymise its expression, and set `y` = (hpv_status == HPV+). Keep colorectal (`dataset="coadread"`, msi/tmb/none) exactly as-is.
+
+### 1d. API
+- `RunRequest` gains a `dataset: str` field (default "coadread"). `_worker` passes it through to `_prepare_lab_data`. Validate dataset+objective combinations (e.g. hpv only valid for hnsc; msi/tmb only for coadread).
+- `REFERENCE_SETS` (api/app.py ~line 106) becomes **per-dataset**: colorectal keeps MMR/immune; HNSC gets HPV reference sets — e.g. `"HPV_markers"` = {CDKN2A, …} and a `"cell_cycle"` / E2F-target set (define from a small curated list in one place, like MMR_GENES). `/evaluate` and any diagnostics select the reference sets for the active dataset.
+- Diagnostics: the MSI/TMB-rank diagnostics are colorectal-specific — gate them to that dataset; HNSC can have its own (or none) for now.
+- Airgap unchanged: anonymise HNSC expression; HPV status is a label (target), never a gene name. Extend the airgap tests to the HNSC path (no gene-symbol leak).
+
+=================================================================
+## STAGE 2 — Frontend: cancer selector + per-dataset config + full "?" coverage
+=================================================================
+
+### 2a. Top-level Cancer / problem selector
+- Add a segmented control (same style as the Objective control) ABOVE the Objective row: **Colorectal (TCGA COADREAD)** | **Head & Neck (TCGA HNSC)**. (No side "what swaps" panel — that was mockup annotation only.)
+- Drive everything from a **dataset registry**: each entry = `{ label, dataset_id, objectives, biologyPanel, referenceSets, diagnostics, postHocTargets, axisLabels }`. The page renders the selected entry's config around the unchanged engine. Selecting a cancer sets `dataset` on the run request and swaps the objective options, biology panel, reference sets, and diagnostics.
+
+### 2b. HPV biology panel — include the schematic
+- Add a "What is HPV? (the biology)" collapsed-by-default disclosure, mirroring the existing "What is MMR?" panel EXACTLY (same collapsible pattern). Embed the HPV reference SVG from repo root **`hpv_reference.svg`** (already created) verbatim, responsive (width 100%, height auto). Show it for the Head & Neck dataset; show the MMR panel for Colorectal.
+
+### 2c. HPV objective + rich "?" explainer
+- Add the "HPV detection" objective card for the HNSC dataset (objective spec `{target:"hpv", metric:"auroc"}`), plus "Unsupervised" (same engine).
+- Give it the SAME rich click-to-open ParamHelp modal as the other objectives, in plain English (no assumed knowledge). Content: what it optimises ("a score that separates HPV+ from HPV− tumours"); how it's scored (AUROC: 0.5 = coin-flip, 1.0 = perfect; minority ~15–25% HPV+, so AUROC not accuracy); represented as `{target: hpv, metric: AUROC}`; and the honest note that this is *detection/recovery* of a known viral signature (p16/CDKN2A, cell-cycle), not new causation. Use the same intro/footer pattern as the other objective modals.
+
+### 2d. Full "?" coverage — every objective, parameter, result metric, and chart axis
+This is a hard requirement, including for HPV:
+- **Objectives:** all have rich "?" explainers (above).
+- **Parameters:** shared engine — the existing ParamHelp "?" already cover them; no change.
+- **Result metrics** (Held-out, Permutation p, Nodes, Genes) and the **verdict / post-hoc** "?" and inline text must be objective-aware and read correctly for HPV (held-out = AUROC separation of HPV+/−; the post-hoc/verdict framing is the supervised one — "real and useful" vs "not reliable" — NOT the unsupervised two-questions framing).
+- **Charts:** the fitness-curve y-axis label + its "?" for the HPV objective = "separation (AUROC)" with the AUROC explanation (same as MSI). Ensure the objective-aware label/tooltip records include the `hpv` case.
+- All "?" text in plain English explaining the concept, consistent with the rest of the Lab.
+
+### 2e. Reveal & evaluate
+- For HNSC, the reference-set toggle shows the HPV sets (HPV_markers / cell_cycle) instead of MMR/immune; overlap + reveal work the same way. Expected win: revealing the winner shows it recovered CDKN2A/p16 + cell-cycle genes.
+
+## CONSTRAINTS
+- Colorectal behaviour, airgap tests, and existing runs unchanged.
+- The engine / DSL / parameters / live view / population / program graph are identical across datasets — only config swaps.
+
+## Checkpoint
+After Stage 1: a CLI/API HPV run on HNSC completes and separates HPV+/− (held-out AUROC should be high). After Stage 2: the cancer selector swaps to Head & Neck, the "What is HPV?" schematic shows, an HPV run gives a strong held-out AUROC with a plain verdict, and reveal recovers p16/CDKN2A + cell-cycle genes. Airgap tests green; colorectal unchanged.

Read docs/PROMPT_hpv_stage2.md

@@ -0,0 +1,41 @@
+# Prompt — HPV Stage 2 (frontend) + broaden the HPV reference set
+
+Stage 1 (HNSC dataset + HPV objective) is done and works (held-out AUROC ~0.92, p ~0.02). Now: the frontend — a top-level cancer selector, per-dataset config, the HPV schematic, the HPV objective with rich "?", and full "?" coverage — plus one small backend tweak to broaden the HPV reference set. Read the named code first; don't assume. Colorectal behaviour, airgap tests, and existing runs stay unchanged. The engine / DSL / parameters / live view / population / program graph are identical across datasets — only config swaps.
+
+## 0. Broaden the HPV reference set (small backend)
+The Stage-1 HPV reference set was p16-centric, so winners (which land in the broader proliferation footprint, e.g. DDB2/HAUS5) showed 0 overlap. In the per-dataset `REFERENCE_SETS` (api/app.py ~line 106), give HNSC TWO sets, defined from a single curated list (like MMR_GENES), NOT cherry-picked to match any winner:
+- `"p16"` = {CDKN2A} — the single canonical marker.
+- `"cell_cycle"` = a standard E2F-target / proliferation set, e.g. {MCM2, MCM3, MCM4, MCM5, MCM6, MCM7, PCNA, CDK1, CCNE1, CCNB1, CDC6, CDC20, MKI67, TOP2A, RRM2, TYMS, FOXM1, E2F1, BUB1, AURKB} (a recognised cell-cycle/E2F core — adjust to a standard set, don't tune to results).
+Keep colorectal's MMR/immune sets unchanged. `/evaluate` for HNSC offers these two.
+
+## 1. Top-level Cancer / problem selector + dataset registry
+- Add a segmented control (same style as the Objective control) ABOVE the Objective row: **Colorectal (TCGA COADREAD)** | **Head & Neck (TCGA HNSC)**. No side "what swaps" panel.
+- Drive everything from a **dataset registry**: each entry = `{ label, dataset_id, objectives, biologyPanel, referenceSets, axisLabels, postHocTargets }`. Selecting a cancer sets `dataset` on the run request and swaps the objective options, biology panel, and reference sets. Default = Colorectal (unchanged behaviour).
+  - coadread → objectives {MSI separation, Mutation burden, Unsupervised}; MMR biology panel; reference sets {MMR, immune}.
+  - hnsc → objectives {HPV detection, Unsupervised}; HPV biology panel; reference sets {p16, cell_cycle}.
+
+## 2. HPV biology panel (embed the schematic)
+- Add a "What is HPV? (the biology)" collapsed-by-default disclosure, mirroring the existing "What is MMR?" panel EXACTLY (same collapsible pattern). Embed the SVG at repo root **`hpv_reference.svg`** verbatim, responsive (width 100%, height auto). Show the HPV panel for hnsc, the MMR panel for coadread.
+
+## 3. HPV objective card + rich "?" explainer
+- Add the "HPV detection" objective card for hnsc (spec `{target:"hpv", metric:"auroc"}`), plus "Unsupervised".
+- Give it the SAME rich click-to-open ParamHelp modal as the other objectives (use the shared ObjectiveIntro + ObjectiveFooter pattern), in plain English:
+  - What it optimises: "a score that separates HPV+ from HPV− tumours."
+  - How it's scored: "AUROC — 0.5 = coin-flip, 1.0 = perfect. HPV+ is a ~15% minority, so we use AUROC, not accuracy."
+  - Represented as: `{ target: hpv, metric: AUROC }`.
+  - Honest note: "This is detection / recovery of a KNOWN viral signature (p16/CDKN2A and the cell-cycle program the virus switches on) — not new causation. The cause is the virus itself, not a host gene."
+
+## 4. Full "?" coverage — objective-aware for HPV (supervised framing)
+HPV is supervised binary classification — treat it like MSI, NOT like unsupervised:
+- **Fitness y-axis:** add `hpv` to `FITNESS_LABEL_BY_TARGET` (Lab.tsx ~line 215) → "separation (AUROC)"; and to the per-target fitness-curve tooltip record → the AUROC/separation explanation (reuse the MSI text).
+- **Result metrics + verdict:** the held-out = AUROC separation of HPV+/−; use the SUPERVISED verdict framing ("Real and useful" vs "Not reliable"), NOT the unsupervised two-questions framing, and NOT the unsupervised post-hoc-alignment block. Threshold: held-out ≥ 0.75 = generalises well (same as MSI).
+- All objective/parameter/result/chart "?" read correctly for an HPV run; parameters are shared (unchanged).
+
+## 5. Reveal & evaluate
+- For hnsc, the reference-set toggle shows **p16 | cell_cycle** (instead of MMR/immune); overlap + reveal work identically. Expected: the winner overlaps the cell_cycle set (the HPV+ proliferation footprint); p16 may or may not hit at a given seed.
+
+## CONSTRAINTS
+- Colorectal + airgap tests + existing runs unchanged. Only per-dataset config swaps around the single engine.
+
+## Checkpoint
+The cancer selector swaps to Head & Neck; "What is HPV?" shows the schematic; an HPV run shows held-out AUROC ~0.9 with a plain SUPERVISED verdict and a "separation (AUROC)" y-axis + correct "?"; Reveal offers p16 / cell_cycle and the winner overlaps cell_cycle. Colorectal unchanged; airgap tests green.

Read docs/PROMPT_iterative_discovery.md

@@ -0,0 +1,27 @@
+# Prompt — Iterative unsupervised discovery ("peel off axes")
+
+Add a "find the next axis" capability to the UNSUPERVISED objective: after a run, residualise the discovered split out of the data and re-run, so the engine surfaces a ranked list of independent axes (Axis 1 = KRT5, Axis 2 = …, each with its own held-out, p, revealed genes, and post-hoc MSI/TMB alignment). Read the named code first; don't assume. Airgap stays absolute (opaque IDs, no labels in the search). Only applies to the unsupervised objective (target "none").
+
+## Why this design
+To residualise against a prior axis we need that axis's per-patient score over the FULL cohort. Rather than re-parse a program_repr string back into a Node (no such parser exists), we STORE the winner's full-cohort score in the run result and reuse it. So:
+
+## PART A — engine_v2/pipeline.py
+1. Store full-cohort winner scores. The pipeline already re-executes the winner on ctx_test for held-out scores (`run_v2_pipeline_streaming`, ~lines 417–431). ALSO execute the winner on a FULL-cohort context (all patients, train+test; labels empty for unsup) and store, in the result's **`"winning"`** block (the existing nested dict at line ~462, NOT a new "winner" key), `full_scores` (per-patient, finite-guarded) and `full_sample_ids` (str ids aligned to them) — alongside the existing test-only `holdout_scores`/`holdout_sample_ids`. This is what the next run residualises against.
+2. Residualisation hook. Add an optional param to `run_v2_pipeline_streaming` (and run_v2_pipeline), e.g. `residualize_scores: pd.DataFrame | None` (index = sample id, columns = prior axis scores). When provided, IMMEDIATELY after loading M and BEFORE the train/test split, regress every gene column of M on `[intercept, *prior scores]` (vectorised OLS: `resid = M - P @ np.linalg.lstsq(P, M, rcond=None)[0]`, aligning P to M.index, guarding rank/NaN) and REPLACE M with the residuals (same opaque columns + index). Residualising before the split keeps train and test consistent. (No effect for MSI/TMB — only the unsupervised worker passes it.)
+
+## PART B — api/app.py
+3. `RunRequest` (~line 294) gains an optional `residualize_against: list[str] | None` = prior run_ids in this discovery chain.
+4. In `_worker` (~line 417), for an unsupervised run with `residualize_against` set: look up each prior run_id in the in-memory run store, pull its stored `run.result["winning"]["full_scores"]` + `["full_sample_ids"]`, assemble a DataFrame aligned by sample id, and pass it to the pipeline as `residualize_scores`. Note `_json_finite` may have turned non-finite scores into `null` — drop those patients (or treat as NaN) when aligning. Validate the run_ids exist and were unsupervised; ignore the field for MSI/TMB. (In-memory store ⇒ a chain is valid within a server session; note this.)
+5. Keep all payloads opaque-ID-only; extend the airgap test to the residualisation path (scores are opaque-derived numbers — no labels, no gene names leak).
+
+## PART C — Frontend (web/app/Lab.tsx)
+6. After an unsupervised run completes, show a "Find next axis →" button. Clicking it POSTs a new run with the SAME params + unsupervised objective and `residualize_against` = the run_ids of ALL axes discovered so far in this chain.
+7. Accumulate results into a "Discovered axes" stack rendered below the Result: Axis 1, Axis 2, … Each axis is a compact card reusing the existing verdict + post-hoc rendering — held-out score, permutation p, revealed gene(s), and "aligns with MSI at AUROC … / TMB …". So the user reads a ranked list of independent axes the engine found blind, each named by its post-hoc alignment.
+8. The chain resets when the user starts a fresh run (not "next axis").
+
+## Honesty / scope
+- This is a discovery HEURISTIC: linear residualisation removes the linear component of each prior axis; deeper non-linear structure may remain. State that in the "?" for the axes stack.
+- It does not guarantee MSI ever appears — the value is the ranked, blind, honestly-labelled list of axes (and the chance that a deeper axis turns out to be MSI).
+
+## Checkpoint
+Run unsupervised → Axis 1 should be KRT5. Hit "Find next axis" → Axis 2 should be a DIFFERENT gene/axis (orthogonal to KRT5), with its own held-out/p and a post-hoc MSI/TMB read. Confirm airgap tests stay green and MSI/TMB runs are unaffected.

Read docs/PROMPT_lab_stage1.md

@@ -0,0 +1,37 @@
+# Stage 1 — React/Next.js "Lab" (MVP) on the existing FastAPI backend
+
+Stage 1 — MVP of the React/Next.js "Lab" on top of the existing FastAPI backend. Reuse the DSL, airgap harness, GP engine, and data. Do NOT modify or delete the Streamlit app — it stays as a working fallback. Keep the airgap test green and extend it to the new endpoints.
+
+**Allowed new dependencies:** backend — `sse-starlette`. Frontend — `next`, `react`, `react-dom`, `typescript`, `tailwindcss`, `recharts`. No others.
+
+## PART 1 — Backend (extend api/)
+- **Objective spec** (structured, airgap-clean — references columns, NEVER gene names). Stage 1 supports exactly TWO valid objectives — do NOT expose a free cross-product of target/metric/direction (most combinations are meaningless, e.g. AUROC against continuous TMB):
+  1. `{ target: "msi", metric: "auroc" }` — separate MSI-H from MSS by AUROC (= the existing `msi_separation` preset; direction not applicable).
+  2. `{ target: "tmb", metric: "correlation", direction: "neg" }` — genes whose LOW expression score tracks HIGH mutation burden (= the existing `mutation_burden` preset).
+  The UI offers these as two named choices. `survival` and `none` (unsupervised) are Stage 2 — show disabled with a "coming soon" note.
+- **Permutation null is computed FOR THE CHOSEN OBJECTIVE** as part of the run (shuffle the relevant target ~200× and re-run the fast baseline under the same objective); include `permutation_p` in the result.
+- **Params:** `{ generations, population, genes_per_set (≤8), max_sets (1–2), lambda, seed }`.
+- `POST /runs` { objective_spec, params } -> start a GP run as a **BACKGROUND WORKER THREAD** (the GP is CPU-bound — it must NOT run on the async event loop). Return `{ run_id }`. The worker pushes one event **PER GENERATION** onto a per-run thread-safe queue: `{ generation, best_fitness, median_fitness, candidates: [ top ~12: { id, gene_ids:[opaque], program_repr, fitness, survived:bool } ] }`, then a final `{ done: true, result: {...} }` sentinel.
+- `GET /runs/{id}/stream` -> Server-Sent Events via `sse-starlette` EventSourceResponse; drain the queue and yield each generation event, then the done/result event. Correct SSE headers; no buffering. The async SSE generator MUST NOT block the event loop — bridge the worker thread to async safely (e.g. the worker feeds an `asyncio.Queue` via `loop.call_soon_threadsafe`, or the endpoint reads the thread-safe queue via `run_in_executor`).
+- `GET /runs/{id}` -> status + accumulated log (polling fallback). `GET /runs/{id}/result` -> final result (anonymised).
+- `POST /reveal` { gene_ids } -> { symbols } (existing isolated map-reader). `POST /evaluate` { gene_ids, reference_set } -> `{ revealed:[{id,symbol,matched:bool}], overlap_count }` where reference_set is a server-defined name: `"MMR"` = {MLH1,MSH2,MSH6,PMS2}, `"immune"` = {CD8A,GZMA,PRF1} (extensible). `/evaluate` and `/reveal` are the ONLY map-readers; keep them in the isolated reveal module. The frontend never receives the whole map — only the chosen program's revealed names + overlap.
+- **Airgap:** `/runs`, `/runs/{id}`, `/stream`, `/result` payloads contain ONLY opaque gene IDs. Extend the airgap test to assert this. Ensure CORS allows http://localhost:3000 (add it if not already configured).
+- **Do not break the Streamlit app:** keep every existing endpoint Streamlit depends on (e.g. `/run/a`, `/run/b`, `/result/a`, `/result/b`) working unchanged — ADD the new lab endpoints alongside. Generalising the engine must not alter the existing `msi_separation` / `mutation_burden` presets or their outputs.
+- run_id store can be in-memory for now (single-user local; ephemeral on restart — fine for MVP).
+
+## PART 2 — Frontend (new Next.js app in web/)
+- Scaffold a Next.js (App Router) + TypeScript + Tailwind app in `web/`. Put the calm palette in the Tailwind theme as tokens: bg #FAFAF7, ink #23303A, accent #3A6B7E, highlight #BC6B2E, muted #6E7F8C, border #ECEAE4, card #FCFBF8. API base URL from env `NEXT_PUBLIC_API_URL` (default http://localhost:8000).
+- One "Lab" page. It is INTERACTIVE (state, SSE, event handlers) so it (and its interactive components) **MUST be Client Components (`"use client"`)**. Sections:
+  1. **Objective builder:** segmented controls / selects for target (MSI / TMB), direction, metric. Show survival/none disabled ("Stage 2").
+  2. **Params:** inputs/sliders for generations, population, genes-per-set, lambda, seed.
+  3. **"Run" button** -> POST /runs -> open the SSE stream for the run_id.
+  4. **Live view** (Stage 1 = simple, not React Flow yet): a fitness curve (Recharts line) that appends each generation; a population grid of the top ~12 candidate cards showing opaque gene IDs + fitness, with survivors in the accent and discarded ones muted at low opacity. Y-axis label reflects the metric ("separation (AUROC)" or "association").
+  5. **Result panel:** winning program (program_repr with opaque IDs), held-out AUROC/association, permutation p, baseline.
+  6. **Evaluator:** a reference-set select (MMR / immune) + "Reveal & evaluate" button -> POST /reveal + POST /evaluate -> show the revealed genes (IDs -> names), matched-reference genes highlighted in #BC6B2E, and the overlap count.
+- Clean, spacious, calm — elegant but functional. NO React Flow, NO animations, NO free-text objective this stage (all Stage 2).
+
+## PART 3 — Run instructions
+Document both, e.g. backend `uvicorn api.app:app --reload`; frontend `cd web && npm install && npm run dev` (localhost:3000). Confirm the Streamlit app still runs unchanged.
+
+## Checkpoint
+After PART 1, show me the new endpoints working (curl /runs then /stream). Then build PART 2 and tell me how to start the lab.

Read docs/PROMPT_liveview_fixes.md

@@ -0,0 +1,33 @@
+# Prompt — Live view fixes (fitness curve explainers, legend, axis, generation navigation)
+
+Fix the Live view: clearer "?" explainers on the fitness curve, a centered y-axis label, a legend you can actually read, no overlapping labels, and fast generation navigation. Presentation-only — no logic/science changes. Reuse the existing InfoTip and calm theme. READ the chart + generation-stepper code first (Recharts or custom SVG) — don't assume names.
+
+## 1. Y-axis "?" (negative association)
+Add/clarify the "?" for the y-axis with this text (verbatim): "This shows how strongly the program's score moves OPPOSITE to mutation burden — low score where mutations are high (the broken-spell-checker pattern). 0 = no relationship; higher = a stronger opposite-direction link; the top = score and mutation count move almost perfectly oppositely. The underlying correlation is negative (e.g. −0.49); we plot its strength (0.49) so up = better, like every fitness curve. (For the MSI objective this axis instead shows separation / AUROC.)"
+
+## 2. "best vs median" "?"
+Set its text to (verbatim): "Each generation has a whole population of programs, each with a fitness. 'best' is the single highest-scoring program that generation (the champion); 'generation median' is the middle one (half score higher, half lower). Best shows the ceiling and whether it's still climbing; median shows whether the WHOLE population is improving. When median rises toward best (as here), good structure is spreading — the population is converging. A persistent gap means a few standouts still lead a varied crowd."
+
+## 3. Center the y-axis title
+Center the rotated y-axis title vertically along the axis (it currently sits at the top). Recharts: YAxis label position "insideLeft", angle -90, style { textAnchor: "middle" }, dy centered. Custom SVG: rotated <text> at the plot's vertical midpoint, text-anchor middle. Keep the label objective-dependent ("negative association" for TMB / "separation (AUROC)" for MSI).
+
+## 4. Make the two lines distinguishable — on the chart AND in the legend
+- best = SOLID line, accent #3A6B7E, slightly thicker.
+- generation median = DASHED line, muted #6E7F8C.
+- The legend swatches must show the SAME solid-vs-dashed styles (not two identical dots), each beside its label.
+
+## 5. Fix the label collision
+The x-axis title "generation" currently overlaps the "median" legend entry. Increase the chart's bottom margin and lay them out on SEPARATE rows with clear spacing — e.g. tick numbers, then the "generation" axis title, then the legend below it (or legend above the title) with a gap. Nothing should overlap.
+
+## 6. Easier generation navigation
+Replace the click-only "‹ Generation N / total ›" stepper with a fast navigator (so you can jump from 1 to 60 in one move, not 60 clicks):
+- A draggable SLIDER (range input) spanning all available generations — drag to jump anywhere instantly; show the current generation number live while dragging.
+- Keep the ‹ › arrows for ±1 fine steps, and add jump-to-first (|‹) and jump-to-latest (›|) buttons.
+- A small numeric input to type a generation and go there directly.
+- Bind the max to generations available so far (it grows during a live run; "latest" tracks the newest).
+- PERFORMANCE: while dragging, do NOT fetch on every intermediate value — update the displayed number live but fetch the population only on release (or throttle to ~1 fetch / 150ms). Otherwise it hammers /runs/{id}/population.
+- Accessible: slider has an aria-label and value text; when focused, ← → move ±1 and Home/End jump to first/latest.
+Reuse the calm theme; keep it compact above the population grid.
+
+## Confirm
+Both "?" open with the new text; the y-axis title is vertically centered (and objective-dependent); best (solid) and median (dashed) are clearly tellable apart in chart and legend; "generation" and the legend no longer overlap; and you can jump to any generation via the slider/input without clicking through each one, with no fetch storm while dragging. These are Client Components.

Read docs/PROMPT_liveview_objectives.md

@@ -0,0 +1,56 @@
+# Prompt — Live-view fixes + objective-modal clarity (run this one)
+
+Five fixes, all targeting existing code (don't rebuild what's there). Read the named spots first. Keep MSI/TMB behaviour + airgap tests intact.
+
+## 1. Strengthen the unsupervised metric (engine_v2/fitness.py, `_score_silhouette`, ~lines 104–130)
+It z-scores the output (line 117) and rejects clusters < 10% (line 123). A `protected_div`-by-self program makes ~1 for most patients and a few huge values → a "perfect" outlier split (held-out silhouette 1.0 that aligns with nothing).
+- WINSORIZE first: clip the score to its 2.5–97.5 percentile range. Do NOT rank-transform (it erases genuine bimodal gaps).
+- ORDERING MATTERS: the existing `std == 0` guard (line 114) runs on the RAW score — the self-division case has LARGE raw std, so it slips through. After winsorizing, recompute mean/std on the WINSORIZED score and z-score with those; if the winsorized std < 1e-9 → return `worst_score()`. (Clipping collapses the blow-up to near-constant, so this is what actually catches it.)
+- Raise the minimum cluster size from 10% to ~30%: `min_cluster = max(10, int(np.ceil(0.30 * n)))`.
+Re-run should no longer return ~1.0 from a self-dividing program.
+
+## 2. Shorten the TMB y-axis label (Lab.tsx `FITNESS_LABEL_BY_TARGET`, ~line 217)
+Change `tmb` from "negative association (signed −spearman with TMB)" to "neg. association with TMB". Leave `msi` ("separation (AUROC)") as-is; optionally make `none` read "cluster separation (0–1)". Ensure the rotated title is fully visible / vertically centred. (The label is ALREADY objective-aware — don't rebuild that.)
+
+## 3. Make the fitness-curve TOOLTIP objective-aware (the real bug)
+`TIPS.fitnessCurve` (~line 85) is a SINGLE, TMB-centric string (used by the InfoTip at ~line 871). The y-axis already selects by the active objective via `fitnessLabel` (line 890) — make the tooltip do the same: build a per-target record and select with the same target value (e.g. `FITNESS_TIP_BY_TARGET[target]`), replacing the single `TIPS.fitnessCurve` usage. Three verbatim texts:
+- msi: "This shows how well the best program SEPARATES the two subtypes (MSI-H vs MSS), measured by AUROC: 0.5 = coin-flip (no separation), 1.0 = perfect; higher = a cleaner split. We use AUROC because MSI-H is only ~15% of patients, so it isn't fooled by always guessing the majority."
+- tmb: "This shows how strongly the program's score moves OPPOSITE to mutation burden — low score where mutations are high (the broken-spell-checker pattern). 0 = no relationship; higher = a stronger opposite-direction link; the top = score and mutation count move almost perfectly oppositely. The underlying correlation is negative (e.g. −0.49); we plot its strength (0.49) so up = better."
+- none: "There is no target here. This measures how cleanly the program's score splits patients into TWO groups: 0 ≈ no real split, 1 = two clean, well-separated clusters. After the run we check what the split lines up with (MSI? TMB?) — that's the post-hoc alignment in the Result panel."
+
+## 4. Add tooltips to the NODES and GENES stat cards
+`StatCard` already takes a `tip` prop (~line 1054) — just pass it:
+- NODES: "How many operators (Select / Reduce / Combine / …) the program is built from — its size/complexity."
+- GENES: "How many distinct genes the program uses."
+
+## 5. Rewrite the objective `?` modals for clarity (web/app/paramHelpContent.tsx)
+
+### 5a. Fix the shared `ObjectiveIntro` (~line 462) — it currently says "a target column", which is FALSE for unsupervised.
+Replace its text with (verbatim): "An objective is the rule that scores every program — the fitness the engine maximises. It's computed only from the program's per-patient output (and, for the supervised objectives, a target column such as MSI or TMB — never gene names). It sets what 'good' means; the engine then composes DSL programs to satisfy it."
+
+### 5b. The shared `ObjectiveFooter` (~line 474) is target-centric — KEEP it for MSI and TMB, but do NOT render it for unsupervised (unsupervised has no target / no Associate/Effect). Give unsupervised its own footer (see 5e).
+
+### 5c. obj_msi.detailed — rewrite the Items to (verbatim, keep the <ObjectiveIntro/> … <ObjectiveFooter/> wrapper):
+- What it optimises: "a score that ranks MSI-H patients above MSS — i.e. tells the two subtypes apart."
+- How it's scored (AUROC): "pick one random MSI-H and one random MSS patient; AUROC is the chance the score puts the MSI-H one higher. 0.5 = coin-flip (no separation), 1.0 = perfect. We use AUROC, not plain accuracy, because MSI-H is only ~15% of patients — 'always guess MSS' would look 85% accurate while separating nothing."
+- Represented as: `{ target: msi, metric: AUROC }` — "the engine sees only the score + the MSI label, never gene names."
+- The catch: "it rewards ANY separator. MSI-H and MSS differ in thousands of genes, so it usually grabs easy 'shortcut' genes (consequences or coincidences), not the causal MMR genes."
+
+### 5d. obj_tmb.detailed — rewrite the Items to (verbatim, keep wrapper + <ObjectiveFooter/>):
+- What it optimises: "a score that goes DOWN as mutation count goes UP — the fingerprint of a broken repair gene (switch it off → mutations pile up). The repair genes are the MMR set: MLH1, MSH2, MSH6, PMS2."
+- Represented as: `{ target: tmb, metric: correlation, direction: negative }` — "the engine sees only the score + the TMB numbers."
+- Why 'negative' (not just 'related'): "we reward the score being NEGATIVELY correlated with TMB. Rewarding any correlation would also pick genes that RISE with mutations — the opposite of the repair signature."
+- Correlation isn't causation (honest): "this is a mechanism-shaped association — a much better proxy for the cause than predicting the label, but it doesn't prove causation. Genes silenced alongside the repair genes can mimic the same low-expression↔high-TMB pattern."
+- Toward causal: "the program can choose to ADJUST for confounders (age, stage) via the Effect operator — the most causal move the observational data honestly allows. The engine decides whether it helps; we don't hardcode it."
+
+### 5e. obj_unsupervised.detailed — replace the single paragraph + shared footer with these Items (verbatim) and a CUSTOM footer (NOT <ObjectiveFooter/>):
+- No target at all: "Unlike the other objectives, this one is given NO target — only the gene numbers. The search never sees MSI, TMB, or any label."
+- What it optimises: "a score that divides patients into two as-cleanly-separated-as-possible groups (measured by cluster separation). It is NOT told what the groups should be."
+- Shape vs aim: "it always produces a two-group split, but it doesn't aim at MSS/dMMR — or anything. It finds whatever the strongest natural division in the data is."
+- How we read it: "after the run we check what the discovered split lines up with — MSI? TMB? — the 'post-hoc alignment' in the Result panel."
+- The win: "if the strongest natural split turns out to BE the MSS-vs-dMMR divide, it aligns with MSI at high AUROC → the engine rediscovered the subtype without ever being told it exists."
+- Honest expectation: "it may instead land on a different dominant axis (e.g. immune hot vs cold) that only partly overlaps MSI. MSI-H is only ~15% of patients, so a clean recovery isn't guaranteed — the result is HOW MUCH the blind split overlaps MSS/dMMR."
+- Custom footer (italic, muted, in place of ObjectiveFooter): "The only thing outside the DSL here is 'find the cleanest split, judged honestly on held-out data' — there is no target, and the program can't see one. That's exactly what makes a match with MSI meaningful: the engine wasn't told to look for it."
+
+## Confirm
+Unsupervised no longer returns a degenerate ~1.0 silhouette; the fitness-curve tooltip matches the active objective; TMB y-axis label fits; NODES & GENES have tooltips; the three objective modals read clearly; and the unsupervised modal no longer shows the target-centric intro/footer.

Read docs/PROMPT_merge_groups_table.md

@@ -0,0 +1,33 @@
+# Prompt — merge "GP's top programs" + "Coordinated modules" into ONE sortable "Groups the engine explored" table
+
+Replace the two overlapping panels with a single table of the groups the GP explored, sortable by four lenses: **GP fitness** (the engine's own pick order), **Combined AUROC**, **Coherence**, **Synergy**. Drop the separate "GP's top programs" block. This supersedes PROMPT_synergy_lens.md and the top-10 panel from PROMPT_gp_top10_groups.md. Read the named code first; verified line refs below. Airgap unchanged. `pytest` + `tsc` after.
+
+GUIDING IDEA: the groups in this table ARE the GP's explored groups (harvested from the population). "GP fitness" sort = the engine's own preference (its picks); the other three are re-score lenses. One table, same groups, selectable lens — this dissolves the "is this from the GP or not" confusion.
+
+## Part 1 — API: add GP fitness per module
+In `api/app.py` `_compute_module_ranking` (~line 1244): it already harvests distinct gene-sets from the run's population (`run.log` candidates), and each candidate carries `fitness`. While harvesting, record per gene-set the **max GP fitness** among candidates with that set; add `gp_fitness: float | None` to each module dict in the payload. No other endpoint change. (Airgap unchanged — fitness is a number, no gene names.)
+- `web/lib/api.ts`: add `gp_fitness?: number | null` to the module type (`ModuleRanking` per-module entry, near `combined_holdout` ~line 295).
+
+## Part 2 — Frontend: one merged table (enhance `ModuleRankingPanel`, ~3521-4100)
+- **Sorts (four).** Extend `SortKey` (currently `"combined" | "coherence"`, ~3940) to add `"gp_fitness"` and `"synergy"`. Wire all four into the sort comparator (~3975-3984) and the sort-button row (~4089-4098). Labels: "GP fitness", "Combined AUROC", "Coherence", "Synergy". **Default sort = `gp_fitness`** (so the default view is the engine's own preference order — what the old top-programs panel showed).
+- **Columns.** Add **GP fitness** (from the new field) and **Synergy** columns alongside the existing Combined AUROC / Coherence / Size / Genes / Survives. `synergy = m.combined_holdout − max(g.single_gene_metric over m.per_gene)`; guard nulls → "—" and sort to the bottom (both fields already on the row).
+- **Keep everything the panel already does:** winner badge (`isWinnerSet`, ~3596) on the winner's gene-set row; SurvivalChips (pre→post `full → subgroup`); inline gene symbols (auto-revealed per page); expand → per-gene single-gene AUROC + rank; pagination.
+- **Retitle + caption (plain English):** title e.g. **"Groups the engine explored"**. Caption: "Every gene group the engine tried during the search. Sort by **GP fitness** (what the engine itself preferred — its picks), **Combined AUROC** (the group's average score on held-out patients), **Coherence** (how tightly the genes move together), or **Synergy** (how much the group beats its best single gene). The winning program is badged. The Survives flags show whether a group still separates HPV under the confound checks."
+- **"?" tooltip:** explain the four lenses plainly, and keep the winner's-curse caveat for the re-score lenses: "Combined AUROC / Coherence / Synergy re-score ~2,900 groups on the same small held-out set, so their very top values are optimistically biased — the **GP fitness** order (and the badged winner) is the engine's own, cross-validated pick; trust it as the reliable choice."
+
+## Part 3 — remove the now-duplicate "GP's top programs" block
+In `ResultPanel` (~line 1983): remove the "The GP's top programs" block (the top-10-by-fitness list added previously). Its job is now done by sorting the merged table by GP fitness, with per-gene ranks available on expand and the winner badged. Keep the metric cards (Held-out, Permutation p, Nodes, Genes) as the headline. Add one line under the cards pointing down: "The winning program is badged in **Groups the engine explored** below." Remove any fetches in ResultPanel that existed only to feed the deleted block (the merged table fetches its own data); leave the rest intact.
+
+## Part 4 — leave the other panels alone
+- "Known-marker recovery" (single-gene markers-only diagnostic) is unchanged.
+- No engine/GP-search change; this is presentation + one additive API field.
+
+## CONSTRAINTS
+- Airgap absolute: module payload stays opaque-ID + scores + `gp_fitness` + survival; symbols revealed only for displayed rows (bounded, as today). Engine/GP search untouched (detection unaffected).
+- Defaults: colorectal + non-HNSC behaviour unchanged; existing tests + airgap tests green.
+
+## Checkpoint
+- One "Groups the engine explored" table with four sorts (GP fitness default), GP-fitness + Synergy columns added, winner badged, survival chips, expand-for-per-gene-ranks, pagination.
+- Sorting by GP fitness reproduces the engine's preference order (the old top-programs list); sorting by Synergy sinks ZFR2+passenger groups and surfaces additive ones.
+- The separate "GP's top programs" block is gone; Result cards keep the headline + a pointer to the table.
+- `pytest` green, `tsc` clean, airgap untouched.

Read docs/PROMPT_mmr_unsup.md

@@ -0,0 +1,60 @@
+# Prompt — MMR biology panel + MLH1 success reframe + drop Survival / add Unsupervised
+
+Three changes. Read the named files first and reuse what's there; don't assume. Keep the airgap tests green and existing presets working.
+
+================================================================
+## PART 1 — Collapsible "What is MMR?" biology panel in the Lab
+================================================================
+Add a collapsed-by-default disclosure to the Lab that explains the MMR/MSI biology, mirroring the existing collapsible in `web/app/ParameterFlow.tsx` ("How these parameters relate") — same open/close pattern, calm styling, `aria-expanded`, collapsed on first render, no layout shift when closed.
+
+- Title: "What is MMR? (the biology behind MSI)". Place it just under the Objective card (near the MSI/Mutation-burden choices).
+- Content: embed the reference SVG at repo root `mmr_reference.svg` (already created). Render it inline (responsive: width 100%, height auto via its viewBox) so it never overflows the card. If you inline the markup rather than fetch the file, copy it verbatim from `mmr_reference.svg`.
+- Presentation only; no engine/API changes in this part.
+
+================================================================
+## PART 2 — Reframe TMB success to "MLH1 in the top X% blind"
+================================================================
+The Reference-gene diagnostic (Lab.tsx ~line 1217–1294, fetched from `/diagnostic/tmb-rank`) already returns each MMR gene's rank/N. Use that SAME data — do not refetch differently.
+
+- When the active objective is **TMB** (mutation burden) and the diagnostic has loaded, show a prominent SUCCESS callout in the Result / Reveal area (accent styling, not the muted zero-state):
+  "✓ Blind, MLH1 ranks {rank} / {N} (top {pct}%) on the TMB objective — the causal gene surfaced near the top without ever seeing gene names."
+  where `pct = round(rank / N * 100, 1)` from the diagnostic's MLH1 row.
+- Pair it with the existing "0 of N winning genes are in the MMR set" line so that line no longer reads as failure — add one muted sentence beneath it: "Winning programs rarely ARE the MMR genes (many co-silenced genes outrank them) — but MLH1 itself sits in the top {pct}%. See the diagnostic below."
+- Only render this for the TMB objective (the rank is a TMB-axis property). If the diagnostic hasn't loaded or MLH1 is "not present", omit the callout gracefully.
+- Presentation only — purely surfacing existing diagnostic numbers.
+
+================================================================
+## PART 3 — Drop Survival; implement Unsupervised (engine_v2)
+================================================================
+
+### 3a. Backend — a real unsupervised objective in engine_v2
+Read `engine_v2/fitness.py` (the `V2Objective` dataclass, `objective_from_spec`, `cv_score`, `evaluate_holdout`, `fitness_fn`), `engine_v2/pipeline.py`, `engine_v2/gp.py`, and `engine_v2/synthesize.py`.
+
+- Add an unsupervised objective spec: `{ target: "none", metric: "structure" }`. Extend `objective_from_spec` to return a new `UNSUP_OBJECTIVE` for it (keep msi+auroc and tmb+correlation as they are). `survival` must still RAISE (unsupported) — do not add it.
+- The unsupervised objective scores a program's per-patient Vector by INTRINSIC STRUCTURE, with NO label:
+  - `score_vector(scores, y=None)` = the separation of a 2-cluster split of the standardised score — use the silhouette of a 2-means (k=2) clustering on the 1-D standardised score (range −1..1; higher = a cleaner two-group split). `binary = False`. `worst_score()` returns a finite floor (−1.0). `fitness_label()` = "structure (2-cluster separation)".
+  - Guards (reuse the degenerate handling already in `_vector_from_program`): standardise the score first; require each cluster ≥ max(10, 10% of n); constant/NaN/too-small-cluster → worst.
+- Thread "no label" through scoring: `cv_score` / `evaluate_holdout` / `fitness_fn` must accept `y = None` for the unsupervised objective and never index `y` (the structure metric ignores it). The KFold path (non-binary) already exists — just skip the `y[te]` access when y is None and score on `scores[te]` alone.
+- **Airgap-style guarantee for unsupervised:** during an unsupervised run the engine must see NO labels. Build the ExecContext with `labels = {}` (no msi/tmb). Consequently the label-using operators can't be used, so for the unsupervised objective the synthesiser must generate **Vector-only programs** (Select / Reduce / Combine / Split) — set the rates so Scalar-roots (Associate/Effect) and FitApply are OFF (rate 0) and `scalar_share = 0`. (Programs that would need a target simply aren't produced.)
+- **Permutation null for unsupervised** (the winner-fixed target-shuffle does NOT apply — there's no target): compute the winner's structure score, then build a null distribution from N random programs (sample N random Vector-only programs, score each on the held-out context), and report p = fraction of random programs with structure ≥ the winner. State this in the result.
+
+### 3b. Post-hoc interpretation — "what did it find?" (map-reading side only)
+After an unsupervised run, in the reveal/evaluate / diagnostic module (the side allowed to read labels — NEVER the engine), compare the winner's discovered 2-group split to the known labels:
+- report AUROC of the discovered split vs the MSI label, and |spearman| of the score vs TMB, plus the revealed top genes.
+This answers "the unsupervised structure turned out to align with MSI / TMB". Labels are used ONLY here, after the search — not during it.
+
+### 3c. API
+- `api/app.py` `ObjectiveSpecModel` (~line 250) + the v2 dispatch (`v2_objective_from_spec`, ~336/383–389): accept `{target:"none", metric:"structure"}` and route it to the unsupervised objective. Airgap payloads unchanged (opaque IDs only). For an unsupervised run, `/runs` must build the worker's context with no labels (3a).
+
+### 3d. Frontend (web/app/Lab.tsx + paramHelpContent.tsx)
+- **Remove the Survival objective card entirely** (Lab.tsx ~line 524) and its ParamHelp modal entry (`paramHelpContent.tsx` ~line 444).
+- **Enable the Unsupervised card** (Lab.tsx ~line 530): remove the disabled/"Stage 2" state; set its objective spec to `{ target: "none", metric: "structure" }` (add it alongside the `msi`/`tmb` specs at Lab.tsx ~line 199). Sub-label: "no target — find the strongest natural split".
+- Rewrite the Unsupervised ParamHelp modal (`paramHelpContent.tsx` ~line 461) to the real behaviour: "No target column at all. The engine searches for a program whose score splits patients into two clean groups (measured by cluster separation), then — afterwards — we check what that split lines up with (MSI? TMB?). It can rediscover the subtype without ever being told it exists. The search itself sees no labels."
+- Live view: when objective is unsupervised, the fitness y-axis label = "structure (cluster separation)".
+- Result panel for unsupervised: show the post-hoc interpretation from 3b ("the discovered split aligns with MSI at AUROC … / TMB |spearman| …") and the permutation p.
+
+### 3e. Tests
+- Update `tests/test_engine_v2.py` / `tests/test_api_airgap.py`: unsupervised spec now ACCEPTED by v2; `survival` still rejected. Add a test that an unsupervised run's ExecContext carries no labels (the engine can't peek). Keep the airgap gene-leak tests green.
+
+## Checkpoint
+Show me: (1) the collapsible MMR panel renders the diagram; (2) a TMB run shows the "MLH1 in top X%" callout from the diagnostic; (3) an unsupervised run completes, the engine context has no labels, and the post-hoc step reports how the discovered split aligns with MSI/TMB. Confirm airgap tests + existing presets still pass.

Read docs/PROMPT_module_ranking.md

@@ -0,0 +1,40 @@
+# Prompt — ranked list of coordinated gene modules (coherence runs)
+
+When "Prefer coordinated gene modules" (coherence) is on, show a ranked list of the coordinated gene **modules** the run explored — ordered by the module's **combined held-out AUROC** (the group's aggregate score vs the label, on unseen patients), with each module's coherence and its individual gene AUROCs/ranks shown. This lets a real co-expressed program (e.g. cell_cycle) surface as a GROUP even when no single gene is the top separator. Read the named code first; don't assume. Airgap stays absolute. `pytest` + `tsc` after.
+
+## 0. Why / honesty
+A module's combined AUROC MUST be measured on the **held-out** split (the same train/test split the run used). Larger groups can trivially fit the training patients, so only the held-out group-AUROC is meaningful. The aggregate is parameter-free (mean of the module's genes → AUROC vs label), so there's no model to fit and no leak. Modules were *explored by the GP* so they carry selection bias; the held-out combined AUROC is exactly the honest check on that.
+
+## 1. Harvest the modules from the run
+IMPORTANT — verified data shape: engine_v2 persists each candidate with a **flat `gene_ids` list** (`engine_v2/gp.py` ~line 136, `list(population[i].feature_ids())`) plus a typed `program_repr` string. It does NOT store separated per-`Select` gene-sets. So:
+- **Default (v1 of this feature): a "module" = a candidate's full distinct `gene_ids` set.** Iterate the persisted population across all generations (`GET /runs/{id}/population/{gen}` → `candidates`, which is the FULL population, not the SSE-trimmed top-12; harvest every generation that's persisted), take each candidate's `gene_ids`, dedupe modules as unordered opaque-ID sets, keep size as a column.
+- **Optional (only if cleanly splitting multi-Select programs matters): parse `program_repr`** to extract individual `Select` sets — the frontend already has a parser (`web/lib/programRepr.ts`); a backend parser would be new work. Don't do this unless the flat-set version proves insufficient.
+- Drop singletons if you want modules ≥2 genes; exact-set dedupe is fine for v1 (skip Jaccard merging unless needed).
+
+## 2. Score each module (all on the run's train/test split)
+First reproduce the run's EXACT held-out split: the module-scoring endpoint must rebuild the same train/test partition the run used — `make_split(M.index, y, test_size, random_state=seed, stratify=objective.binary)` with the run's stored `seed` (and matching `test_size`/stratify), over the anonymised matrix from `_prepare_lab_data(target, dataset)`. If the split doesn't match, "held-out" is meaningless. For every distinct module compute, on that split:
+- **Combined held-out AUROC** — aggregate the module's genes into one per-patient score with the **mean** (matching `Reduce(mean)`), compute orientation-agnostic AUROC vs the label on the **held-out** patients. This is the primary sort key. (For TMB/continuous targets use the run's metric, e.g. |spearman|, not AUROC.)
+- **Coherence** — mean absolute pairwise correlation among the module's genes, on **train** (the same quantity the coherence prior rewards). Shows *why* it's a module.
+- **Size** — number of genes.
+- **Per-gene** — each gene's single-gene AUROC and rank/N from the existing diagnostic (`/diagnostic/...-rank`), so the user sees the individual strengths inside the group.
+- Add an endpoint (or extend the result endpoint) that returns this module ranking for a given run, opaque-ID only for the ranking itself.
+
+## 3. UI — ranked modules
+- Show this section only when coherence was on for the run (gate on the run's coherence flag).
+- Render a ranked list (cards or a table), default sorted by **combined held-out AUROC** descending, with columns: combined AUROC · coherence · size. Let the user re-sort by coherence.
+- Each module expands to its genes: revealed **symbol**, single-gene AUROC, rank/N — reuse the highlighted-genes styling. Mark genes that are in a reference set (p16 / cell_cycle) with the existing colour scheme, so a cell_cycle-heavy module is visually obvious.
+- Header copy + a "?" explaining plainly: "Each row is a group of genes that move together (a coordinated module the engine explored). Combined AUROC = how well the group's average score separates the label on patients it never saw. A group can beat its individual genes — that's the point of looking at modules."
+- Place it near the result, after the single-gene ranking. It does NOT replace the winner/ranking; it's an additional view.
+
+## 4. Airgap
+- The module ranking itself crosses the wire as **opaque IDs + scores**. Reveal **symbols only for the modules actually displayed** (bounded — same discipline as revealing a winner; never the whole map). If the list is long, reveal lazily per expanded module.
+- Combined AUROC and coherence are computed on the anonymised matrix; no names needed to rank.
+
+## CONSTRAINTS
+- Engine name-blind; reveal only the displayed modules' genes. No change to the GP search itself ��� this is a post-run analysis over the persisted population.
+- Held-out combined AUROC only; never in-sample. No model fitting in the aggregate (mean only) so there's nothing to leak.
+
+## Checkpoint
+- A coherence-on HPV run shows a ranked list of modules by combined held-out AUROC; expanding a module reveals its genes with per-gene AUROC/rank and reference-set marking.
+- A cell_cycle-heavy module (if the run explored one) appears with a high combined AUROC even though its genes aren't individually top — demonstrating the group view.
+- Ranking payload is opaque-only; revealed symbols are bounded to displayed modules; airgap tests green; `tsc`/`pytest` clean.

Read docs/PROMPT_modules_highlight_symbols.md

@@ -0,0 +1,23 @@
+# Prompt — colour-code reference genes in Coordinated modules + fix missing symbols
+
+Two presentation fixes to the Coordinated-modules table. Verified against the code: `_compute_module_ranking` (api/app.py ~lines 1244–1457) returns each module as `{gene_ids (opaque), size, combined_holdout, coherence, per_gene:[{id,single_gene_metric,rank,total}]}` — **opaque IDs only, no symbols and no reference-set flag**. The table's "—" cells are module genes that were never revealed (e.g. `g09647` = LIG1); they resolve fine (HNSC reuses the single colorectal sealed map, its genes a subset), they're just not fetched. Read the current `_compute_module_ranking` and the modules UI component first. Airgap stays absolute; `pytest`/`tsc` after.
+
+## 1. Flag + colour-code reference-set membership on EVERY module row (so no expanding needed)
+- In `_compute_module_ranking`, resolve the **active dataset's reference sets** to opaque IDs ONCE (HNSC → `p16`={CDKN2A}, `cell_cycle`=the 20-gene set; CRC → `MMR`, `immune`). Reuse the existing reference→opaque resolution already used by `/diagnostic/full-rank` (its `reference_marks`) / the highlighted-genes table — don't re-implement. This is a bounded reveal of a known small set, not the whole map.
+- For each module add `ref_sets: list[str]` = the reference-set names whose opaque IDs intersect that module's `gene_ids` (e.g. `["cell_cycle"]`, `["p16","cell_cycle"]`, or `[]`).
+- Frontend modules table: on each **collapsed** row, render a small badge per matched set and tint the row using the existing three-way colour scheme (winner accent / p16 deep-gold / cell_cycle muted-amber — the same palette the highlighted-genes table uses). A module containing a p16 or cell_cycle gene must be scannable in the ranked list without expanding it.
+- Airgap note: only a boolean membership (derived from the bounded reference reveal) crosses the wire here — no module gene NAMES are sent for the collapsed list.
+
+## 2. Fix the missing symbols ("—") on expand
+- The per-gene `symbol` is blank because module genes outside the already-revealed subset are never translated. When a module is **expanded**, reveal that module's `gene_ids` via the existing `/reveal` endpoint (bounded — only that module's ~2–5 genes) and show the symbol on each per-gene row. `g09647` should then read **LIG1**.
+- Reveal **lazily, per expanded module only**. Do NOT reveal all modules' genes up front — there are ~2,900 modules and that would approach translating the whole map (airgap violation).
+- Keep the per-gene single-gene AUROC / rank already returned; just add the revealed symbol alongside. Where a gene has no rank (not in the diagnostic series) leave its metric blank but still show its symbol.
+
+## CONSTRAINTS
+- Airgap: reference-membership flag = bounded reference-set reveal; per-module symbols = bounded on-expand reveal. Never translate the full map; the collapsed ranking stays opaque + scores + ref_sets booleans.
+- No change to module harvesting, scoring, or sorting — purely additive (ref_sets field + lazy symbol reveal + row colouring).
+
+## Checkpoint
+- Collapsed module rows show a p16 / cell_cycle (or MMR / immune) badge and matching row tint wherever the module contains such a gene — visible without expanding.
+- Expanding any module shows a symbol for every gene (e.g. LIG1), no "—".
+- Ranking payload stays opaque + ref_sets booleans; symbols revealed only for expanded modules; airgap tests green; `tsc`/`pytest` clean.

Read docs/PROMPT_param_help.md

@@ -0,0 +1,206 @@
+# Prompt — Rich parameter help (click-to-open explainers with diagrams)
+
+Upgrade the "?" help for the 8 Parameters in the Lab from a one-line tooltip into a rich, click-to-open explainer (text + examples + diagrams), aimed at a non-expert reviewer. Presentation/read-only — no logic changes. Reuse the calm theme (bg #FAFAF7, card #FCFBF8, ink #23303A, muted #6E7F8C, border #ECEAE4, accent #3A6B7E, highlight #BC6B2E) and the existing monospace for gene IDs.
+
+## Interaction
+- Build ONE reusable component `<ParamHelp short="..." detailed={...} />` so wording can't drift. The "?" button:
+  - on **hover/focus**, shows the SHORT one-liner (a small tooltip — keep the current text as the short summary);
+  - on **click**, opens a MODAL dialog with the DETAILED content (text + any diagram).
+- Modal: `role="dialog"`, `aria-modal="true"`, labelled by its title; ~640px wide on desktop, full-width with padding and vertical scroll on narrow screens; close on X, Esc, and backdrop click; return focus to the "?" button on close. Calm card styling (border #ECEAE4, fill #FCFBF8, ~14px radius, generous padding). Headings in #23303A, body readable (~14px), muted captions #6E7F8C.
+- Apply this ONLY to the 8 Parameters "?" icons. Leave other "?" icons elsewhere (objectives, run id/status) as the existing InfoTip — do not touch them.
+- Diagrams are embedded as small SVG components, `width:100%; height:auto` (viewBox scales). Each diagram is used once, in its own parameter.
+
+## Detailed content per parameter
+For each: SHORT = hover text; DETAILED = modal body (use the wording below; render line breaks and bullet/example blocks cleanly).
+
+### Generations
+SHORT: "How many rounds of 'keep the best, breed, mutate' the engine runs. More rounds refine further, but take longer."
+DETAILED:
+- What it is: one "generation" is a single round of the engine's loop — score every program, keep the best, then breed and mutate them into the next batch. Generations is how many rounds it runs.
+- How it works: the engine starts with random programs (mostly poor) and each round nudges the population toward better ones — like selective breeding.
+- More vs fewer: more rounds refine the winners further, but with diminishing returns once it has converged. The harder part is usually *discovering* the right genes in the first place (driven by Population and mutation), not polishing — so extra generations help less once the curve flattens.
+- Typical: 30–60.
+- Small inline illustration: a 4-step loop "random programs → score each → keep the best → breed & mutate → (back to score)", calm style, arrows; the loop arrow labelled "× Generations".
+
+### Population
+SHORT: "How many candidate programs compete in each round. A 'program' is a small pipeline built by composing the DSL operators (Select → Reduce → Fit) over the gene data. More programs explores more options, but is slower."
+DETAILED:
+- What it is: how many candidate programs compete each round. A "program" is a small pipeline built from the DSL operators — Select (pick genes) → Reduce (average them into one score) → Fit (use the score to separate the groups).
+- Why it matters: a bigger population samples more genes and structures each round, so it's the main lever for *discovery* — finding the right genes at all. Generations then refine what was found.
+- Example — three random programs in a population (opaque gene IDs, monospace):
+  - `average(g04823, g11201, g00917) → score`
+  - `average(g15522, g02013) − average(g08840, g00231) → score`
+  - `average(g07788, g13002, g05340) → score`
+- More vs fewer: more explores more options, but each round is slower.
+- Typical: 150–300.
+
+### Genes / set
+SHORT: "The most genes a single score may use. Smaller keeps programs simple and readable."
+DETAILED:
+- What it is: a "set" is the group of genes averaged together into a single score. Genes/set is the maximum size of that group.
+- Example: `Select(g05347, g00048, g06271) → average → one score` (uses 3; the cap might be 8).
+- Smaller vs larger: smaller keeps each score simple and readable; larger lets one score blend more genes.
+- It pairs with Max sets: Genes/set is the *width* of each group; Max sets is *how many* groups.
+
+### Max sets
+SHORT: "How many separate gene-scores a program may combine (1 or 2). 2 lets it build a small 'score made of scores'."
+DETAILED:
+- What it is: how many *separate* gene groups a program may build and then combine (1 or 2). They use different genes — not the same ones reused.
+- Example (2 sets):
+  - Set 1: `average(g05347, g00048, g06271) → scoreA`
+  - Set 2: `average(g15522, g02013) → scoreB`
+  - Combine: `scoreA − scoreB → final score`
+- Why two: to capture a *contrast* — e.g. one group that's low in MSI-H and one that's high; the gap between them can separate better than either alone (like "repair activity minus immune activity"). One set can't express that; two can.
+- If one group is enough, the engine can still use 1.
+- Diagram (small): two short chains — `matrix → Select → Reduce → scoreA` and `matrix → Select → Reduce → scoreB` — both feeding a `Combine (sub)` node, then a final `score`. Verbs = accent-bordered boxes, scores = amber pills, calm style.
+
+### λ (size penalty)
+SHORT: "How hard the engine is penalised for using more genes. Higher pushes it toward fewer genes (simpler answers)."
+DETAILED:
+- What it is: a "price per gene" that discourages bloated programs. Every program is graded on a single number:
+  - `fitness = separation − λ × (number of genes)`
+  - so the engine ranks by accuracy *minus* a size tax.
+- The key idea — λ is the admission price each gene must beat: with λ = 0.005, a gene is only worth keeping if it adds more than 0.005 of separation.
+- Worked example: a 3-gene program scores 0.90 → net 0.90 − 0.015 = 0.885. Add a 4th gene that lifts it to 0.903 (only +0.003) → net 0.883, *lower*, so it's rejected. A 4th gene that lifts it to 0.91 (+0.01) → net 0.890, *higher*, so it's kept.
+- Turning the knob: λ = 0 → genes free → bloated, overfit programs. λ small (0.005) → trims useless genes, keeps useful ones. λ large → very lean programs, but may drop useful genes.
+- Diagram: the accuracy-vs-genes trade-off (embed the SVG in "DIAGRAMS" below).
+
+### Seed
+SHORT: "The starting point for the engine's randomness. The same seed reproduces the exact same run; change it to see a different run."
+DETAILED:
+- Why randomness: the space of possible programs is astronomically large (picking even 8 genes out of 20,000 is ~10^29 combinations), so the engine can't try them all — it explores with randomness: random starting programs, random mutations, random breeding.
+- What the seed is: computers don't make true randomness; a formula generates a sequence of numbers, each from the previous one. The seed is the *starting number* fed into that formula — everything random flows from it. (One common formula: `next = (1664525 × current + 1013904223) mod 2³²`; with seed 42 the first value is 1,083,814,273, then mapped onto a gene position.)
+- What it does: think of the seed as one fixed list of dice rolls used in order across the whole run — early rolls pick the starting genes, later rolls drive mutations. Same seed → same list → identical run. Different seed → a different list → a different run.
+- Why vary it: because the search has luck in it, run a few seeds (42, 7, 123) and see whether the same genes keep appearing — that tells you a finding is real, not a fluke.
+- Diagram: the seed-as-a-fixed-list-of-dice-rolls (embed the SVG below).
+
+### Permutations
+SHORT: "How many times we re-run on deliberately scrambled labels to check the result isn't luck. The real result must beat these chance runs (that's the 'permutation p')."
+DETAILED:
+- What it checks: whether the winning score is real signal or could have come up by luck. The engine searches so hard that some programs separate the groups well by pure coincidence, so a high score alone isn't proof.
+- How it works: shuffle the MSI-H / MSS labels across patients at random — this breaks any real gene↔label link, creating a "no-signal world". Re-score there: any score above 0.5 is pure chance. Repeat many times (this knob = how many) to build a picture of what luck looks like.
+- Reading it: permutation p = the fraction of shuffled runs that scored ≥ the real winner. p = 0.005 means only 0.5% of chance runs matched it → very unlikely a fluke. Conventionally p < 0.05 is the "unlikely to be luck" line.
+- The knob: with 200 shuffles the smallest p you can report is about 1/200 ≈ 0.005; more permutations give a finer, more trustworthy p.
+- Diagram: the "luck pile" vs the real result (embed the SVG below).
+
+### Prefilter top-N
+SHORT: "By default the engine searches all ~20,000 genes, so nothing is pre-excluded. Turning this on narrows to the N most promising genes first — faster, but it can drop a real gene that only shows signal in combination."
+DETAILED:
+- Default (off): the engine searches all ~20,000 genes, so nothing is pre-excluded — the most honest setting for a discovery demo.
+- Turning it on: first narrows to the N genes most individually associated with the target (computed name-blind, on the training split only), then searches within that shortlist. Faster, because the space is ~10× smaller.
+- The trade-off: speed and focus vs completeness. A univariate shortlist can drop a gene that only matters *in combination* (no signal on its own), and it sets a ceiling — if a gene isn't in the shortlist, the engine can never find it.
+- Small inline illustration (funnel): "~20,000 genes → keep the N most individually associated → search within N", with a muted note "off by default = no funnel".
+
+## DIAGRAMS (embed verbatim; wrap each so it scales: width:100%, height auto)
+
+### Seed diagram
+```svg
+<svg viewBox="0 0 720 200" xmlns="http://www.w3.org/2000/svg" role="img" font-family="ui-sans-serif,system-ui,sans-serif">
+<title>The seed as one fixed list of dice rolls used in order across the whole run</title>
+<desc>Seed 42 produces a fixed sequence of dice rolls; early rolls pick initial genes, later rolls drive a mutation. Same seed gives the same run.</desc>
+<rect x="40" y="6" width="74" height="22" rx="11" fill="#3A6B7E"/>
+<text x="77" y="21" text-anchor="middle" font-size="12" font-weight="700" fill="#fff">Seed 42</text>
+<text x="124" y="22" font-size="12.5" fill="#6E7F8C">→ one fixed list of random rolls, used in order:</text>
+<rect x="50" y="55" width="48" height="48" rx="9" fill="#FCFBF8" stroke="#3A6B7E" stroke-width="1.5"/>
+<circle cx="62" cy="67" r="3.5" fill="#23303A"/><circle cx="74" cy="79" r="3.5" fill="#23303A"/><circle cx="86" cy="91" r="3.5" fill="#23303A"/>
+<rect x="112" y="55" width="48" height="48" rx="9" fill="#FCFBF8" stroke="#3A6B7E" stroke-width="1.5"/>
+<circle cx="124" cy="67" r="3.5" fill="#23303A"/><circle cx="148" cy="67" r="3.5" fill="#23303A"/><circle cx="136" cy="79" r="3.5" fill="#23303A"/><circle cx="124" cy="91" r="3.5" fill="#23303A"/><circle cx="148" cy="91" r="3.5" fill="#23303A"/>
+<rect x="174" y="55" width="48" height="48" rx="9" fill="#FCFBF8" stroke="#3A6B7E" stroke-width="1.5"/>
+<circle cx="186" cy="67" r="3.5" fill="#23303A"/><circle cx="210" cy="91" r="3.5" fill="#23303A"/>
+<rect x="236" y="55" width="48" height="48" rx="9" fill="#FCFBF8" stroke="#3A6B7E" stroke-width="1.5"/>
+<circle cx="248" cy="67" r="3.5" fill="#23303A"/><circle cx="248" cy="79" r="3.5" fill="#23303A"/><circle cx="248" cy="91" r="3.5" fill="#23303A"/><circle cx="272" cy="67" r="3.5" fill="#23303A"/><circle cx="272" cy="79" r="3.5" fill="#23303A"/><circle cx="272" cy="91" r="3.5" fill="#23303A"/>
+<line x1="294" y1="52" x2="294" y2="106" stroke="#C7CDD1" stroke-width="1.5" stroke-dasharray="4 4"/>
+<rect x="314" y="55" width="48" height="48" rx="9" fill="#FCFBF8" stroke="#3A6B7E" stroke-width="1.5"/>
+<circle cx="338" cy="79" r="3.5" fill="#23303A"/>
+<rect x="376" y="55" width="48" height="48" rx="9" fill="#FCFBF8" stroke="#3A6B7E" stroke-width="1.5"/>
+<circle cx="388" cy="67" r="3.5" fill="#23303A"/><circle cx="412" cy="67" r="3.5" fill="#23303A"/><circle cx="388" cy="91" r="3.5" fill="#23303A"/><circle cx="412" cy="91" r="3.5" fill="#23303A"/>
+<text x="446" y="86" font-size="22" fill="#B9B6AE">…</text>
+<path d="M50,110 L50,116 L284,116 L284,110" fill="none" stroke="#9AA0A6" stroke-width="1.2"/>
+<text x="167" y="132" text-anchor="middle" font-size="11.5" fill="#2C5563"><tspan font-weight="700">Generation 0</tspan> — pick the initial programs' genes</text>
+<path d="M314,110 L314,116 L424,116 L424,110" fill="none" stroke="#9AA0A6" stroke-width="1.2"/>
+<text x="369" y="132" text-anchor="middle" font-size="11.5" fill="#8A4E20"><tspan font-weight="700">Generation 5</tspan> — a mutation</text>
+<text x="294" y="150" text-anchor="middle" font-size="10.5" fill="#6E7F8C">… many rolls later …</text>
+<line x1="40" y1="166" x2="680" y2="166" stroke="#ECEAE4"/>
+<text x="40" y="186" font-size="12" fill="#23303A"><tspan font-weight="700">Same seed</tspan> → same list → same run.    <tspan fill="#6E7F8C">Change the seed → a different list → a different run.</tspan></text>
+</svg>
+```
+
+### λ trade-off diagram
+```svg
+<svg viewBox="0 0 720 410" xmlns="http://www.w3.org/2000/svg" role="img" font-family="ui-sans-serif,system-ui,sans-serif">
+<title>How lambda picks the best program size: net fitness = separation minus a per-gene penalty</title>
+<desc>Raw separation rises then plateaus; net fitness subtracts lambda times genes and peaks at an intermediate size; a larger lambda moves the peak to fewer genes.</desc>
+<g stroke="#ECEAE4" stroke-width="1">
+<line x1="70" y1="340" x2="660" y2="340"/><line x1="70" y1="273" x2="660" y2="273"/><line x1="70" y1="207" x2="660" y2="207"/><line x1="70" y1="140" x2="660" y2="140"/><line x1="70" y1="73" x2="660" y2="73"/>
+</g>
+<g font-size="10" fill="#6E7F8C" text-anchor="end">
+<text x="62" y="344">0.5</text><text x="62" y="277">0.6</text><text x="62" y="211">0.7</text><text x="62" y="144">0.8</text><text x="62" y="77">0.9</text>
+</g>
+<line x1="70" y1="40" x2="70" y2="340" stroke="#D9D6CE" stroke-width="1.5"/>
+<line x1="70" y1="340" x2="660" y2="340" stroke="#D9D6CE" stroke-width="1.5"/>
+<g font-size="10" fill="#6E7F8C" text-anchor="middle">
+<text x="70" y="356">0</text><text x="154" y="356">2</text><text x="238" y="356">4</text><text x="322" y="356">6</text><text x="406" y="356">8</text><text x="491" y="356">10</text><text x="575" y="356">12</text><text x="659" y="356">14</text>
+</g>
+<text x="365" y="378" text-anchor="middle" font-size="12" fill="#23303A">number of genes in the program</text>
+<text x="20" y="190" text-anchor="middle" font-size="12" fill="#23303A" transform="rotate(-90 20,190)">separation (accuracy)</text>
+<polyline fill="none" stroke="#3A6B7E" stroke-width="2.5" points="70,340 112,227 154,159 196,117 238,92 280,77 322,67 364,62 406,59 449,57 491,55 575,54 659,53"/>
+<text x="500" y="44" font-size="11.5" font-weight="700" fill="#3A6B7E">raw separation</text>
+<text x="500" y="59" font-size="10.5" fill="#6E7F8C">rises fast, then plateaus —</text>
+<text x="500" y="72" font-size="10.5" fill="#6E7F8C">extra genes barely help</text>
+<polyline fill="none" stroke="#BC6B2E" stroke-width="2.5" points="70,340 112,231 154,165 196,127 238,105 280,93 322,87 364,85 406,85 449,87 491,89 575,94 659,100"/>
+<line x1="364" y1="85" x2="364" y2="340" stroke="#BC6B2E" stroke-width="1" stroke-dasharray="3 3" opacity="0.5"/>
+<circle cx="364" cy="85" r="5" fill="#BC6B2E"/>
+<text x="376" y="120" font-size="11" font-weight="700" fill="#8A4E20">peak (λ = 0.005)</text>
+<text x="376" y="134" font-size="10.5" fill="#8A4E20">engine keeps ~7 genes</text>
+<polyline fill="none" stroke="#D79A6A" stroke-width="2.2" stroke-dasharray="6 4" points="70,340 112,247 154,199 196,177 238,172 280,177 322,187 364,202 406,219 449,237 491,255"/>
+<line x1="238" y1="172" x2="238" y2="340" stroke="#D79A6A" stroke-width="1" stroke-dasharray="3 3" opacity="0.5"/>
+<circle cx="238" cy="172" r="5" fill="#D79A6A"/>
+<text x="250" y="200" font-size="10.5" font-weight="700" fill="#B07636">higher λ = 0.03</text>
+<text x="250" y="214" font-size="10.5" fill="#B07636">peak shifts left → ~4 genes</text>
+<g font-size="11" fill="#23303A">
+<line x1="80" y1="398" x2="104" y2="398" stroke="#3A6B7E" stroke-width="2.5"/><text x="110" y="402">raw separation</text>
+<line x1="240" y1="398" x2="264" y2="398" stroke="#BC6B2E" stroke-width="2.5"/><text x="270" y="402">net fitness (λ=0.005)</text>
+<line x1="450" y1="398" x2="474" y2="398" stroke="#D79A6A" stroke-width="2.2" stroke-dasharray="6 4"/><text x="480" y="402">net fitness (λ=0.03)</text>
+</g>
+</svg>
+```
+
+### Permutations diagram
+```svg
+<svg viewBox="0 0 700 380" xmlns="http://www.w3.org/2000/svg" role="img" font-family="ui-sans-serif,system-ui,sans-serif">
+<title>Permutation null: scores from 200 shuffled-label runs versus the real result</title>
+<desc>A histogram of 200 shuffled-label scores clusters near 0.58, well below the real winning score of 0.89; no shuffle reaches it, giving a permutation p of about 0.005.</desc>
+<line x1="70" y1="50" x2="70" y2="320" stroke="#D9D6CE" stroke-width="1.5"/>
+<line x1="70" y1="320" x2="640" y2="320" stroke="#D9D6CE" stroke-width="1.5"/>
+<g font-size="10" fill="#6E7F8C" text-anchor="middle">
+<text x="70" y="336">0.4</text><text x="162" y="336">0.5</text><text x="253" y="336">0.6</text><text x="345" y="336">0.7</text><text x="437" y="336">0.8</text><text x="528" y="336">0.9</text><text x="620" y="336">1.0</text>
+</g>
+<text x="355" y="358" text-anchor="middle" font-size="12" fill="#23303A">score (separation / AUROC)</text>
+<text x="20" y="185" text-anchor="middle" font-size="12" fill="#23303A" transform="rotate(-90 20,185)">number of shuffled runs</text>
+<line x1="162" y1="60" x2="162" y2="320" stroke="#C7CDD1" stroke-width="1" stroke-dasharray="4 4"/>
+<text x="162" y="72" text-anchor="middle" font-size="9.5" fill="#9AA0A6">0.5 = guessing</text>
+<g fill="#CFE0E4">
+<rect x="110.4" y="303.1" width="20" height="16.9"/><rect x="133.3" y="277.8" width="20" height="42.2"/><rect x="156.3" y="235.6" width="20" height="84.4"/><rect x="179.2" y="176.6" width="20" height="143.4"/><rect x="202.1" y="109.1" width="20" height="210.9"/><rect x="225" y="58.4" width="20" height="261.6"/><rect x="247.9" y="92.2" width="20" height="227.8"/><rect x="270.8" y="159.7" width="20" height="160.3"/><rect x="293.75" y="218.75" width="20" height="101.25"/><rect x="316.7" y="260.9" width="20" height="59.1"/><rect x="339.6" y="286.25" width="20" height="33.75"/><rect x="362.5" y="303.1" width="20" height="16.9"/>
+</g>
+<text x="150" y="44" font-size="11.5" font-weight="700" fill="#3A6B7E">200 shuffled-label runs (chance)</text>
+<text x="150" y="100" font-size="10.5" fill="#6E7F8C">if there were no real signal,</text>
+<text x="150" y="113" font-size="10.5" fill="#6E7F8C">scores land in here</text>
+<text x="455" y="250" text-anchor="middle" font-size="10.5" fill="#9AA0A6">no shuffle ever</text>
+<text x="455" y="263" text-anchor="middle" font-size="10.5" fill="#9AA0A6">gets this far →</text>
+<line x1="519" y1="60" x2="519" y2="320" stroke="#BC6B2E" stroke-width="2.5"/>
+<circle cx="519" cy="60" r="5" fill="#BC6B2E"/>
+<text x="519" y="48" text-anchor="middle" font-size="12" font-weight="700" fill="#8A4E20">real result: 0.89</text>
+<text x="519" y="300" text-anchor="middle" font-size="11" font-weight="700" fill="#8A4E20">p ≈ 0.005</text>
+</svg>
+```
+
+## "How these parameters relate" chart
+- Keep the neat (HTML/flex, wrapping) version already built.
+- Enhancement: make each parameter NAME in the chart's pills a button that opens that parameter's ParamHelp modal (same content). So the chart becomes a map into the detailed help.
+
+## Constraints
+- Presentation-only; do not change any GP logic or other tooltips.
+- One reusable `ParamHelp` component + one reusable modal; diagrams are separate small components used once each.
+- Accessible: "?" is a real button (keyboard-activatable); modal has role/aria, Esc + backdrop + X close, focus trap and focus return.
+- Responsive: modal scrolls on small screens; diagrams scale to container width.
+- These are Client Components ("use client") since they manage open/close state.

Read docs/PROMPT_polish_tooltips_graph.md

@@ -0,0 +1,39 @@
+# Prompt — Lab polish: plain-English "?" copy, no all-caps, fix the clipped program graph
+
+Three presentation-only fixes to the Lab (web/). No engine, API, DSL, or airgap changes. Read the named files first; don't assume. After each part, build (`tsc`/`next build`) and confirm clean.
+
+## 1. Audit EVERY "?" so it's clear to a layperson and correct per objective (including HPV)
+The reader is not a cancer expert or an ML expert. Every tooltip and every click-to-open modal must explain its concept in plain English, define any term it uses, and read correctly for whichever objective/dataset is active.
+
+Where the copy lives (read all of these):
+- `web/app/Lab.tsx` — the `TIPS` record (short InfoTip strings), `FITNESS_TIP_BY_TARGET`, `FITNESS_LABEL_BY_TARGET`, and every inline `tip=` / `<InfoTip>` usage (objective cards, parameters, run/live/result/evaluator labels, discovered-axes, rank panels).
+- `web/app/paramHelpContent.tsx` — `PARAM_TIPS` (short + detailed), the objective modals `obj_msi`, `obj_tmb`, `obj_unsupervised`, `obj_hpv`, and the shared `ObjectiveIntro` / `ObjectiveFooter` / breakdown components.
+
+Requirements:
+- **Plain language.** No undefined jargon. If a term is unavoidable (AUROC, permutation, held-out, silhouette, residualise), give a one-line plain gloss in the same tip — e.g. "held-out = patients the engine never saw during the search, used to check the result isn't a fluke."
+- **Objective- and dataset-aware.** Any tip shown during an HPV run must talk about HPV+/HPV− separation, not MSI or TMB. Sweep for tips that hardcode MSI/TMB wording and make them branch on the active objective (the per-target records already exist for the fitness curve — extend the same pattern wherever a result/verdict/axis/chart tip is objective-specific). Confirm each of the four objectives (MSI, TMB, Unsupervised, HPV) shows tips that match what that run actually produces.
+- **HPV specifically.** The HPV "?" should make clear, in plain words: the engine is *detecting a known viral signature* (high CDKN2A/p16 and the cell-cycle program the virus switches on), not discovering new causation — the cause is the virus itself. Keep `obj_hpv` consistent with that.
+- Keep tone and length consistent with the existing good tips; don't bloat.
+
+## 2. Never write in all caps (anywhere in the Lab UI)
+All-caps text is currently produced two ways — fix both:
+- **Tailwind `uppercase` classes.** In `web/app/Lab.tsx` there are many label/eyebrow elements using `uppercase tracking-wide` (e.g. lines ~1757, 1799, 1850, 2110, 2119, 2248, 2290, 2298, 2342, 2386, 2394, 2436 — and any others). Remove the `uppercase` utility from every one so labels render in their natural case. Keep `tracking-wide`/sizing if you like, just drop the uppercasing. Then make sure the underlying strings read well in sentence case (e.g. "Held-out", "Permutation p", "Revealed genes", "Winning program", "Post-hoc alignment", "Opaque ID", "Symbol", "Discovered axes", "Paste a program", "Nodes", "Genes").
+- **`textTransform: "uppercase"`** in `web/app/ProgramGraph.tsx` (~line 196, the Tier label). Remove it; the labels are already literal strings "Tier-1" / "Tier-2 program" — render them as-is (Title case), not uppercased.
+- Grep the whole `web/app` tree for any remaining `uppercase` class, `textTransform`, or literal ALL-CAPS string constants in UI copy and convert them to sentence/Title case. No visible UI text should be all-caps after this.
+
+## 3. Fix the program graph being clipped
+Symptom (see screenshot): for a two-tier (`Combine`) program the dashed "Tier-2 program" wrapper and the lower "Tier-1" group are cut off — the graph overflows on the right and the bottom is sliced by the "Paste a program" section.
+
+In `web/app/ProgramGraph.tsx` (the `ProgramGraph` component, ~lines 725-760) and `ProgramGraph.css`:
+- The container height is `Math.max(220, height)` where `height` is the laid-out bounding box. Confirm that `height` for the `Combine`/Tier-2 case includes the full Tier-2 wrapper (label padding + both Tier-1 lanes + the Combine/Output column). If the wrapper's bbox isn't fully accounted for, the container is too short and clips the bottom.
+- `fitView` only fits on first mount; when the selected program changes (winner ↔ candidate, or a paste) it doesn't re-fit, so a larger graph overflows instead of scaling down. Make `fitView` re-run whenever the program changes — e.g. give `<ReactFlow>` a `key` derived from the program (so it remounts), or call `fitView()` from an `onInit`/effect keyed on `props`. 
+- The `minZoom={0.4}` floor stops `fitView` from shrinking a large 2-tier graph enough to fit, so it clips. Lower the floor (e.g. `minZoom={0.2}`) so the whole graph always scales to fit the card width.
+- Ensure the card/section wrapping the graph has no fixed height or `overflow: hidden` that crops it, and that there's vertical separation from the "Paste a program" block below.
+
+Result: the entire program (both tiers, all Select/Reduce/Combine/Split nodes) is always fully visible inside the card, scaled to fit, for single-tier and two-tier programs alike.
+
+## Checkpoint
+- Every "?" reads in plain English and is correct for each of the four objectives; an HPV run shows HPV-framed tips (no MSI/TMB leftovers).
+- No all-caps text anywhere in the Lab.
+- A two-tier program renders fully inside the Program graph card with nothing clipped; switching candidates re-fits.
+- `tsc` clean; colorectal/HPV runs and airgap tests unaffected.

Read docs/PROMPT_program_graph.md

@@ -0,0 +1,60 @@
+# Prompt — Program graph + paste-to-draw tool (with format help & copy)
+
+Build/refine the Lab's "Program graph" feature: render any DSL program as a node-graph (React Flow), with a paste-to-draw mini-tool as the primary way to view a program, a detailed format guide, and easy ways to obtain a program's text. Presentation/read-only — do not change the GP science or the airgap. It MUST look like it belongs to the existing Lab (same cards, palette, fonts, InfoTip). React Flow is an allowed frontend dependency.
+
+## 0. READ FIRST (don't assume)
+Open a real run and read the ACTUAL shapes: GET /runs/{id}/result and one candidate object. Confirm the exact `program_repr` text format and how structure is represented (which operators appear, how genes are grouped). Build the parser around what you actually see, and make the Format guide (Section 4) describe the REAL operators/aggregations — the lists below are the expected set; verify and adjust to match.
+- If per-set / tree structure isn't recoverable from the payload, add a structured field (opaque IDs ONLY), keep every existing field, and extend the airgap test.
+
+## 1. PRINCIPLES
+- Airgap absolute: graphs show opaque g##### IDs until an explicit reveal; pasted programs are opaque-only.
+- Keep the page light: ONE graph at a time in a single graph area — the winning program by default, replaced by whatever is drawn next. Never render a graph per candidate.
+- ONE shared parser: `program_repr` text → render model, used by the winner render, the paste tool, AND tile clicks. Do not write two parsers. The parser must tolerate the documented format with or without spaces (trim/normalise input).
+
+## 2. RENDERING (React Flow, aligned with the app)
+- Wrap in the SAME section card the other sections use (1px #ECEAE4 border, #FCFBF8 fill, ~14–16px radius, generous padding) with a bold #23303A heading "Program graph", the existing <InfoTip> "?" beside it, and a one-line #6E7F8C caption.
+- Palette tokens only: bg #FAFAF7, card #FCFBF8, ink #23303A, muted #6E7F8C, border #ECEAE4, accent #3A6B7E, highlight #BC6B2E. Same font stack; render opaque gene IDs in the SAME monospace as the run id.
+- Custom node styles (match the app's cards): verbs (Select, Reduce, Combine, Fit) = #FCFBF8 rounded box, 1px #3A6B7E border, #23303A label, optional #6E7F8C subtitle. Data nouns (M / "Expression matrix") = subtle grey pill (#F1EFEA fill, #ECEAE4 border). Scores = amber pill (#FBF1E6 fill, #BC6B2E border). Output node (label matches the objective — "MSI-H probability" or "association") = filled #BC6B2E, white text. Edges = thin #B9B6AE smoothstep with arrowheads; no animated edges.
+- Tier containers when a program combines scores: light teal group box (#F1F6F7 fill, thin #3A6B7E border, "Tier-1" label) per sub-score, wrapped in a dashed #BC6B2E "Tier-2 program" box. A single-score program needs no dashed wrapper.
+- TURN OFF React Flow default chrome: no <Background> (plain #FAFAF7/transparent canvas); hide connection handles (.react-flow__handle { opacity:0 }); remove node selection outline/box-shadow; no MiniMap; omit <Controls> or restyle to the muted palette; proOptions={{ hideAttribution: true }}; use fitView. Deterministic left-to-right layout (manual x/y) — no extra auto-layout library.
+- The graph ALWAYS roots in the DSL primitives (M → Select → Reduce → … → output) and draws ARBITRARY trees of the real operators — not a fixed skeleton.
+
+## 3. PASTE-TO-DRAW (primary feature)
+- Inside the card: a monospace textarea labelled "Paste a program", a "Draw" button, and the single graph area below.
+- On Draw, parse the textarea with the shared parser and render into the graph area, identically styled.
+- Invalid/unparseable input → a friendly inline message in muted text (#6E7F8C), no crash.
+- Pasted programs render with opaque IDs only; Reveal/evaluate is NOT available for arbitrary pasted text (only for actual-run winners) — state this in the hint.
+- The textarea PLACEHOLDER is the well-formed example below (note: every Reduce wraps a Select).
+
+## 4. FORMAT & EXAMPLES (detailed; collapsed-by-default panel under the textarea, plus the "?")
+Document the syntax with this content (adjust operator names to the real grammar from Section 0):
+- What a program is: a recipe that turns gene expression into ONE score per patient.
+- The pieces:
+  - `M` — the expression matrix (all patients × genes); the starting data.
+  - `Select(M, [g#####, ...])` — pick a group of genes → a smaller matrix.
+  - `Reduce(<matrix>, <agg>)` — collapse that group into one score per patient. `<agg>` = mean | median | max | min | var.
+  - `Combine(<scoreA>, <scoreB>, <op>)` — merge two scores into one. `<op>` = add | sub | mul | div (safe) | mean.
+  - Output: one score per patient, used to separate the groups.
+- Rules: gene IDs are the opaque `g#####` IDs shown on the tiles and in the result; list a Select's genes in [square brackets], comma-separated; spaces are optional; every Reduce should wrap a Select(...) so it reduces a chosen group, not the whole matrix.
+- Annotated example (also the placeholder):
+  `Combine( Reduce(Select(M,[g05347,g00048]), mean), Reduce(Select(M,[g06271]), max), sub )`
+  → "average g05347 & g00048 → scoreA; take the max of g06271 → scoreB; final score = scoreA − scoreB."
+
+## 5. HOW TO GET A PASTEABLE PROGRAM
+Make it trivial to obtain a program in this exact text form:
+- Result panel: a "Copy program" button next to the winning program that copies its `program_repr` to the clipboard (navigator.clipboard; degrade gracefully if unavailable — e.g. a select-the-text fallback).
+- Population tiles: clicking a tile loads its program into the textarea AND draws it; ALSO add a small copy icon on each tile (or on hover) that copies that tile's `program_repr`.
+- A one-line muted hint under the textarea (always visible): "Paste a program here, or click any tile to load it. Use Copy to grab a program's text."
+- Every copied/loaded string is the raw `program_repr` (opaque IDs only) — the exact string the parser accepts — so copy → paste round-trips and draws identically.
+
+## 6. REVEAL
+- After "Reveal & evaluate" (run winner only), swap opaque IDs for real gene names in every Select node, highlighting matched reference genes in #BC6B2E. Do not attempt reveal on arbitrary pasted programs.
+
+## 7. WHERE
+- Result panel: render the WINNING program's graph by default.
+- Population tiles drive the same single graph area (load on click) — no separate per-tile graphs.
+
+## CONSTRAINTS
+- Airgap test green (including any new structured field); existing endpoints / Streamlit / live curve untouched.
+- One shared parser; one graph area; parser tolerant of spacing.
+- The graph, paste tool, format panel, and copy buttons are Client Components ("use client").

Read docs/PROMPT_ranking_coherence_peeloff.md

@@ -0,0 +1,44 @@
+# Prompt — rebuild the Lab result around the full gene ranking, add a coherence prior + peel-off, retire the Unsupervised card
+
+Five coordinated changes. The engine stays blind (opaque IDs only); the airgap is absolute. Read the named code first; don't assume. After: `pytest` green, `tsc` clean, airgap tests untouched and passing. Colorectal + HPV behaviour otherwise unchanged.
+
+GUIDING PRINCIPLE: nothing here may put the answer into the engine. The coherence prior is a generic "prefer coordinated gene modules" bias that names no gene/pathway. Peel-off just removes the last axis and re-runs. The full ranking is computed on the ANONYMISED matrix; gene names are revealed only for a handful of highlighted genes, never the whole map.
+
+## 1. Retire the Unsupervised objective from the UI (keep the backend)
+- Remove the **Unsupervised** objective card from `ObjectiveBuilder` / `DATASET_REGISTRY` in `web/app/Lab.tsx` for both datasets, so the UI shows only: Colorectal → {MSI separation, Mutation burden}; Head & Neck → {HPV detection}.
+- Do NOT delete `UNSUP_OBJECTIVE`, its pipeline branches, or its tests — leave the engine capability and tests intact; just stop surfacing the card. (It's conceptually distinct and tested; keep it.)
+- Anywhere the frontend special-cased `target === "none"` for verdict/post-hoc/axes, leave those code paths working but unreachable from the UI.
+
+## 2. Coherence prior — a toggle on the supervised objectives (engine + UI)
+A generic structural bias toward gene sets that MOVE TOGETHER (co-expressed modules), not lone sharp genes. Names no gene, so it's airgap-safe.
+- **Engine** (`engine_v2/fitness.py`): add an optional `coherence_weight: float = 0.0`. When > 0, a program's fitness becomes `separation_metric + coherence_weight * coherence(selected_genes)`, where `coherence` = the mean absolute pairwise correlation among the program's Select'd opaque columns, computed on the anonymised matrix (no names). A program with <2 genes gets coherence 0 (neutral). Keep the existing worst-case floors and the metric's own orientation. Default 0.0 ⇒ current behaviour exactly.
+- **API** (`api/app.py`): `RunRequest` gains `coherence: bool = False` (or a small float); `_worker` threads it into the pipeline as `coherence_weight` (pick one sensible default weight when the toggle is on — start modest so separation still dominates).
+- **UI** (`web/app/Lab.tsx`): one checkbox toggle in Parameters, "Prefer coordinated gene modules", default OFF. Rich "?" in plain English: *"Off: the engine rewards whatever genes separate the label best, even a single lucky gene. On: it also rewards genes that move together across patients — a real co-expressed program — so it tends to surface coordinated modules (e.g. a cell-cycle program) rather than one sharp correlate. It names no pathway; it just prefers coordinated signals."*
+
+## 3. Peel-off ("find next axis") on the supervised objectives (engine + API + UI)
+Today residualisation is unsupervised-only. Extend it so HPV/MSI/TMB runs can enumerate axes too.
+- **Engine/pipeline**: the `residualize_scores` hook already residualises M before the split; make sure it runs for supervised targets too (it's currently a no-op for msi/tmb). Persist `winning.full_scores` + `full_sample_ids` for supervised winners (as already done for unsup) so the next axis can residualise against them.
+- **API**: allow `residualize_against` for supervised runs (currently gated to unsup). Validate the prior run_ids are the SAME dataset+objective; pull their `full_scores`; pass them through. Keep payloads opaque-only; extend the airgap test to the supervised residualisation path.
+- **UI**: show the existing "Find next axis →" button + Discovered Axes stack after ANY run (not just unsup). Each axis card reuses the verdict + held-out + p + revealed-winner rendering, and now also shows the axis winner's position in the ranking (§4). Rich "?": *"Removes the signal the last run found, then re-runs — so the engine surfaces the next strongest independent program. Use it to enumerate the coordinated axes that separate the label, strongest first, instead of seeing only the loudest one. Linear residualisation (removes the linear component); the chain is in-memory for this session."*
+
+## 4. New Result: the full single-gene ranking, with the winner + reference genes marked
+Replace the current Reveal/overlap/diagnostic panels with ONE ranking result. Keep the existing verdict callout and the Held-out AUROC / Permutation p / Nodes / Genes metric cards (that's the blind result) ABOVE it.
+- **Data (airgap-clean):** add an endpoint (or extend the diagnostic) that returns, for the active (dataset, target, run's train split), the FULL ranking of all genes by single-gene separation — as `[{opaque_id, score, rank}]` for every column — computed on the ANONYMISED matrix vs the label (no names on the wire). Then reveal SYMBOLS only for: (a) each gene in the GP winner's program, (b) the dataset's reference-set genes (p16 / cell_cycle for HNSC; MMR / immune for CRC). Use the existing single-ID reveal path — never translate the whole map.
+- **View:** render a **rank track** — a horizontal axis from rank 1 (best separator) to N (≈20,218) — with marked, labelled ticks for: the GP winner gene(s) (accent), p16/CDKN2A, and each cell_cycle gene. Below it, a compact, searchable/virtualised list of the ranking by opaque ID + score (so the full list is browsable without a 20k-row table), with the highlighted genes pinned to the top and showing their revealed symbol + rank + score. Make explicit in copy that the winner is the engine's blind pick and the marked reference genes show where the known biology actually ranks.
+- This makes the detection-vs-recovery story literal: you see the winner near the top, and the canonical markers scattered down the ranking (e.g. MCM5 high, CCNE1 low).
+
+## 5. State the GP-vs-diagnostic difference in the UI
+Add this as a short note / "?" beside the ranking result (verbatim intent, plain English):
+> The GP isn't an exhaustive top-1 ranker — it's a stochastic search. The diagnostic ranking scores all ~20,218 genes and sorts them; the GP does not. It samples programs, mutates, recombines, and keeps what survives tournaments — exploring only a subset of the space. So it finds an excellent solution, not provably the best one: it may never have tried the rank-1 gene, or tried it in a generation that got out-competed, and settled into a neighbouring solution. Evolution finds a very good answer, not the global maximum.
+
+## CONSTRAINTS
+- Airgap absolute: the full ranking is opaque-ID + score only; symbols revealed only for the winner + reference genes via the single-ID reveal path. The engine never sees names; `engine/`+`engine_v2/` untouched by the reveal path.
+- Coherence names no gene/pathway. Peel-off encodes no answer.
+- Defaults preserve current behaviour (coherence OFF; no residualise) so existing runs are unchanged.
+
+## Checkpoint
+- UI objectives: CRC {MSI, Mutation burden}; HNSC {HPV detection}. No Unsupervised card; its tests still pass.
+- Coherence toggle (default off) changes fitness only when on; with it on, HPV winners trend toward coordinated modules.
+- "Find next axis" works on an HPV run and yields an orthogonal Axis 2; payloads stay opaque; airgap tests green.
+- The Result shows the full opaque ranking with the winner + p16 + cell_cycle marked; only those symbols are revealed; the GP-vs-diagnostic note is present.
+- `pytest` green, `tsc` clean.

Read docs/PROMPT_ranking_ui_cleanup.md

@@ -0,0 +1,39 @@
+# Prompt — clean up the ranking Result UI
+
+Presentation-only polish to the new full-ranking Result in `web/app/Lab.tsx`. No engine/API/airgap changes. Read the current Result/ranking components first. `tsc` clean after.
+
+## 1. Distinguish the three sources visually (winner / p16 / cell_cycle)
+Right now the winner is one colour and p16 + cell_cycle are both the same amber, so p16 is lost in the cell-cycle block.
+- Use THREE distinct colours, consistently in both the Highlighted-genes table and the rank chart, with a small legend:
+  - **winner** — accent teal (`#3A6B7E`).
+  - **p16 (CDKN2A)** — its own distinct colour (e.g. a deep gold/`#B8860B`-ish or the project's `#BC6B2E`), visually separate from cell_cycle.
+  - **cell_cycle** — a lighter/muted amber so the set reads as a group but doesn't collide with p16.
+- Keep the "Source" text label too. p16 should be immediately findable among the cell_cycle rows.
+
+## 2. Add a percentile next to the rank (Highlighted-genes table)
+- Next to `rank / N`, show the percentile, e.g. `6 / 20,218 · top 0.03%`, `99 / 20,218 · top 0.5%`, `15,918 / 20,218 · bottom 21%`. Compute `top% = rank / N * 100` (and show "bottom (100−top)%" when it's past the midpoint, or just always "top X%", pick the clearer one — top% for the strong ones, and let the big numbers read as low). Round sensibly (2 sig figs).
+- This column is the headline: it shows the winner + canonical markers clustered at the very top and a few cell_cycle genes (CCNE1, CDC20, BUB1) near the bottom.
+
+## 3. Fix the rank chart — make it readable (log scale + colour)
+The current linear 1→N axis crushes every highlighted gene (ranks 6–500) into the far left while CCNE1 sits alone at the right — unreadable.
+- Switch the rank axis to a **log scale**, gridlines/ticks at 1, 10, 100, 1,000, 10,000, (N). This spreads the top cluster out.
+- Plot each highlighted gene as a labelled dot/lollipop at its rank, coloured by the three-way source scheme above, with a legend.
+- De-clutter labels: stagger vertically / use short leader lines so the symbols (ZFR2, MCM5, MCM2, MCM6, CDKN2A…) don't overlap near the top.
+- Title it clearly, e.g. "Where the winner and known markers rank (1 = best single-gene separator, log scale)".
+- If a clean labelled chart for ~22 points is impractical, a horizontal lollipop chart (one row per highlighted gene, sorted by rank, bar = AUROC, coloured by source) is an acceptable alternative — but the log-rank axis is preferred.
+
+## 4. Demote the "Full ranking (browse by opaque ID)" table
+It has little analytic value (opaque IDs are unreadable; revealing all names would break the airgap). Its only purpose is to show the ranking is computed blind.
+- Move it into a **collapsed-by-default disclosure** (same collapsible pattern as the biology panels), labelled e.g. "Raw anonymous ranking — proof the ranking is computed blind on opaque IDs". Keep the opaque-ID search box inside it.
+- Add a one-line caption stating that purpose. Don't show it expanded in the main result flow.
+
+## 5. General tidy
+- Tighten spacing between the metric cards, the rank chart, the highlighted table, and the (now-collapsed) raw ranking so the result reads top-to-bottom: verdict → metric cards → rank chart → highlighted-genes table → collapsed raw ranking.
+- Ensure the Highlighted-genes "?" explains, plainly: the winner is the engine's blind pick; the p16 and cell_cycle rows show where the *known* markers actually rank as single-gene separators; a high rank (low %) = a strong standalone marker, a low rank = only useful in combination.
+
+## Checkpoint
+- p16 is visually distinct from cell_cycle everywhere; legend present.
+- Highlighted table shows rank + percentile.
+- The rank chart is log-scaled, colour-coded, labels legible.
+- The opaque-ID table is collapsed by default with an airgap-proof caption.
+- `tsc` clean; no engine/API/airgap changes.

Read docs/PROMPT_result_views_consolidation.md

@@ -0,0 +1,36 @@
+# Prompt — consolidate the Result views: GP's group vs known-marker individuals vs exploration
+
+Presentation refactor so the three panels map cleanly to three questions, with no group/individual mixing. Verified against the code. No engine/airgap change; one small frontend data-flow change. `tsc` after. (Supersedes the standalone "survival chip pre→post" prompt — Part A folds it in.)
+
+Target end-state, three cleanly-separated panels:
+- **Result panel = the GP's chosen group** (the winner) — its combined held-out, its genes' *individual* ranks, and its confound survival, all together.
+- **Single-gene ranking panel = known markers only** (p16, cell_cycle) — a recovery diagnostic, no winner, no group concept.
+- **Coordinated modules = the explored-group leaderboard** + survival flags — explicitly not the GP's pick.
+
+## Part A — survival chips show pre → post (full → subgroup)
+In `SurvivalChips` (web/app/Lab.tsx ~3999-4060), the `chip(label, survives, value, tooltip)` helper renders only the subgroup AUROC. Change each chip to show **full → subgroup** using the full-cohort `module.combined_holdout` as the "pre": e.g. `site ✓ 0.97 → 0.95`, optionally with the delta `(−0.02)`; purity uses `combined_holdout_highpurity` as post. Keep the existing "n too small" / "—" rendering (no arrow when the subgroup value is null). The tooltip already explains the 0.05 tolerance — no change needed there.
+
+## Part B — single-gene ranking panel = known markers only (remove the winner)
+In `RankingResult` (~line 2187):
+- In `pinnedRows` (~2297-2329), STOP adding the `kind:"winner"` rows (the loop over `diag.ranks` that pushes `winnerOpaqueSet` matches, ~2306-2317). Keep only the reference rows (p16 / cell_cycle). The Highlighted-genes table then contains only known markers.
+- In the lollipop chart ("Where the winner and known markers rank"), remove the **winner** series + its legend entry, and retitle to "Where the known markers rank." Keep the p16 + cell_cycle dots.
+- Net: this whole panel is now unambiguously individual genes from the known reference lists — a recovery diagnostic with no winner and no group concept.
+
+## Part C — Result panel shows the GP's group as a self-contained story
+`ResultPanel` (~line 1983) currently takes only `{ result }` and renders the metric cards from `result.winning`. Give it what it needs to show the winner's individual gene ranks + confound survival next to the existing cards:
+- Pass `dataset` and `target` (and the run id) into `ResultPanel` from the parent (the Lab component already has them — it passes them to `RankingResult`).
+- In `ResultPanel`, fetch the full-rank diagnostic (`getFullRankDiagnostic(dataset, target)`) and the module ranking (`getModuleRanking(runId)`) — the same calls `RankingResult` / `ModuleRankingPanel` already make; the endpoints are cached, so a second fetch is fine. (If you prefer, lift both fetches to the parent and pass `diag` + `moduleData` as props to all three panels — either is acceptable, but ResultPanel must end up with both.)
+- Render a compact **"Winning program"** block under/next to the metric cards:
+  - **Its genes, each on its own:** look up each winner gene id (`result.winning.gene_ids`) in `diag.ranks` and show `SYMBOL #rank / N` (e.g. `ZFR2 #6 / 20,218 · LIG1 #36 / 20,218`). Reveal the symbols via the existing bounded reveal (the winner's symbols are already revealed elsewhere — reuse that).
+  - **Its confound survival:** find the winner's module in `moduleData` using the SAME gene-set equality `ModuleRankingPanel` uses (`isWinnerSet`, ~3596) and render its survival via `SurvivalChips` (so it gets the Part-A pre→post treatment). Gate to runs where survival exists (HNSC/HPV coherence runs); omit gracefully otherwise.
+- Frame it in copy: "This is the group the engine actually chose. Below: how each of its genes ranks on its own, and whether the group survives the confound checks." This makes the Result panel the self-contained "GP's group" view, distinct from the known-marker diagnostic (Part B) and the exploration leaderboard (Coordinated modules).
+
+## CONSTRAINTS
+- Pure presentation / data-flow; no engine, API, or airgap change. Reveal stays bounded (winner genes + reference sets only). Defaults and non-HNSC behaviour unchanged.
+- After this: no single table mixes a group entry with individual-gene rows; the winner appears as a *group* only in the Result panel (and badged in the exploration table), and as *individual genes* only via its ranks inside that same Result block — clearly labelled.
+
+## Checkpoint
+- Survival chips read `full → subgroup` (e.g. `0.976 → 0.90 ✗`).
+- The single-gene ranking panel shows only p16 + cell_cycle (no winner rows/dots; retitled).
+- The Result panel shows the winner's combined held-out (existing), its genes' individual ranks, and its site/purity survival chips — one self-contained "GP's group" story.
+- `tsc` clean; no API/airgap changes.

Read docs/PROMPT_scatter_hover.md

@@ -0,0 +1,35 @@
+# Prompt — hover tooltips on the fitness × synergy scatter (highlighted dots)
+
+Add hover to the scatter so the **highlighted** dots (winner, p16, cell_cycle) show their genes + key scores. Frontend-only. **Depends on PROMPT_fitness_synergy_scatter.md being in** (it draws the canvas). Verified the data is all on the module row. No engine/API/airgap change. `tsc` after.
+
+## Scope — hover only the highlighted dots
+Hit-test only the highlighted points (winner + cell_cycle + p16/immune groups) — that's a few dozen, so it's trivially fast, and it keeps the reveal bounded. Do NOT hover-test the ~6,000 faint background dots (they're not revealed, and revealing all of them would break the airgap).
+
+## Hit-testing on the canvas
+- When drawing the highlighted dots, also record each one's screen position + its module: `{ cx, cy, module }` in a ref array.
+- On `mousemove` over the canvas, find the nearest highlighted point within ~8px of the cursor; if found, show the tooltip for that module; on mouseout / none-in-range, hide it.
+- Tooltip = an absolutely-positioned HTML div over the chart container, following the cursor (clamp to stay on-screen).
+
+## Tooltip content
+For the hovered group show:
+- **Genes** (revealed symbols), e.g. `MCM5, MCM2` (winner: `C11orf85, ZFR2`).
+- **Source** tag: winner / p16 / cell_cycle (or immune / MMR for CRC) — colour-matched to the dot.
+- **GP fitness** = `m.gp_fitness`
+- **Combined AUROC** = `m.combined_holdout`
+- **Coherence** = `m.coherence`
+- **Synergy** = `m.combined_holdout − max(per_gene single_gene_metric)` (the same value the table/column uses; reuse the shared helper, don't recompute inconsistently).
+- (Optionally size = `m.size`.)
+Format every number through the existing `fmtFit` so non-finite shows "—".
+
+## Revealing the gene symbols (bounded)
+- The highlighted groups may include modules not on the current table page, so their symbols may not be revealed yet. On scatter mount, batch-reveal the **union of the highlighted groups' `gene_ids`** in one `postReveal` call (winner + all `ref_sets`-tagged groups). This is bounded (a few dozen groups × a few genes — ~100-200 ids), same discipline as revealing the reference sets. Cache `symbolByOpaque` and use it in the tooltip.
+- Never reveal the background groups' genes.
+
+## CONSTRAINTS
+- Frontend-only; scores are already on the row, symbols via a bounded reveal of the highlighted groups only. No engine/API/airgap change.
+- Keep hover restricted to highlighted dots (fast + bounded reveal).
+
+## Checkpoint
+- Hovering a winner / p16 / cell_cycle dot shows a tooltip with its gene symbols + GP fitness, Combined AUROC, Coherence, Synergy (and source).
+- Background dots are not hover-targets; no full-map reveal.
+- `tsc` clean; no API/airgap change.

Read docs/PROMPT_separate_group_individual_views.md

@@ -0,0 +1,34 @@
+# Prompt — stop mixing group-of-genes and individual-gene views (coherence-run UI)
+
+Presentation-only cleanup of the coherence-run result UI in `web/app/Lab.tsx`. Verified against the code. The rule: **a table that shows single-gene AUROC contains only individual genes; a table that shows a group's combined AUROC contains only groups. Never mix the two notions in one table.** No engine/API change. `tsc` after.
+
+## The problem (verified)
+The "Result: full single-gene ranking" card (~line 2345) builds `pinnedRows` (~2297–2329) that mix `kind:"winner"` — the winning *program's* genes (a GROUP, e.g. ZFR2+LIG1) — with `kind:"reference"` (p16/cell_cycle individual genes), all rendered with **single-gene** AUROC/rank. So a group is shown as individual rows labelled "winner," reading as if each gene won on its own. Meanwhile the winning program's actual *group* score (held-out AUROC) lives only in the Result metric cards, and the Coordinated-modules table (~3485+) ranks groups by a *different* metric, so its #1 (e.g. ZFR2+WDR76) disagrees with the winner — which currently looks contradictory.
+
+## 1. Make the single-gene ranking table individual-only
+- Keep the winner's constituent genes in this table (their individual ranks are useful — that's how we see ZFR2 is rank 6), but **drop the group framing**: change the winner rows from a `"winner"` source label to a neutral per-gene tag like **"in winning program"**. Every row in this table is then unambiguously *one individual gene scored on its own*.
+- Add one line of copy (or the "?"): "These are the single-gene ranks of individual genes — including the genes inside the winning program and the reference markers. The winning program's combined (group) score is in the Result cards above and the Coordinated modules below."
+- Do NOT show any group/combined score in this table.
+
+## 2. Keep all group scores in the group views, and connect them
+- The winning program's **combined held-out AUROC** stays in the Result metric cards (group score) — unchanged.
+- In the **Coordinated modules** table, tag the module whose gene-set exactly equals the winner's gene-set with a **"winner"** badge. The frontend already has the winner's opaque IDs (`winnerIds` / `winnerOpaqueSet`, ~2285) — match a module's `gene_ids` set against it (unordered equality) and badge it. This makes "the winner is module #N here" visible, so the two panels stop looking contradictory.
+
+## 3. Label what each panel measures (the "?" copy)
+- Single-gene ranking "?": "where each *individual* gene ranks on its own, by single-gene AUROC."
+- Coordinated modules "?": "every *group* the engine explored, re-scored by the group's average separation on held-out — a different number than the search used, so the #1 module can differ from, and even beat, the winning program."
+This directly resolves the "winner (LIG1+ZFR2) vs module #1 (ZFR2+WDR76)" confusion: they're different metrics, now said plainly.
+
+## 4. Auto-reveal module symbols inline (kill the per-row click)
+- In the modules table, symbols are currently revealed only on expand, one `/reveal` per row (`symbolsByIx`, ~3502–3556) — the "Reveal genes" click you do every time. Instead: show the gene symbols inline on each *displayed* module row, auto-revealed for the visible page. Paginate/window the table (~25–50 rows/page) and reveal that page's `gene_ids` in ONE batched `/reveal` call; reveal the next page as the user scrolls/pages. **Never reveal all ~2,900 at once** (airgap bound — only the visible page). Expand then only shows per-gene single-gene AUROC/rank detail; drop the per-row "Reveal genes" button in favour of a row click / chevron.
+
+## CONSTRAINTS
+- Pure presentation: no engine/API/airgap changes. Reveal stays bounded to the visible page; the collapsed module colouring still comes from the server `ref_sets` flag.
+- After this, no single table contains both a group entry and individual-gene rows under the same score column.
+
+## Checkpoint
+- The single-gene ranking table shows only individual genes (winner genes tagged "in winning program," not "winner"); no group score appears in it.
+- The winning program's combined score appears only in the Result cards; the Coordinated modules table badges the winner's own gene-set row.
+- Both panels' "?" state which metric they use, and explain why module #1 can differ from the winner.
+- Module symbols show inline without per-row clicking; paging reveals subsequent pages in batched calls; never all at once.
+- `tsc` clean; airgap tests untouched.

Read docs/PROMPT_stability_panel.md

@@ -0,0 +1,97 @@
+# Prompt — "Stability across seeds" panel in the Lab (HNSC/HPV)
+
+Add a multi-seed stability panel to the Lab that reruns the HPV search across
+several seeds and shows (a) how tightly the result holds and (b) which genes
+recur across independent searches. **Frontend-only** — reuse the existing run,
+transfer, reveal, and full-rank endpoints. No engine / API / airgap change.
+`tsc` after.
+
+Gated to `dataset === "hnsc" && target === "hpv"` (hidden otherwise). Everything
+this needs already exists on the wire; do not add a backend endpoint.
+
+---
+
+## What it does
+For each seed in a list (default `[1, 3, 7, 11, 13, 17, 23, 29]`), fire a normal
+HPV run with that seed, wait for it to finish, then pull its held-out result and
+its independent-cohort transfer. Aggregate across seeds into one panel.
+
+## Data flow (all existing endpoints)
+- Fire a run: `postRun({ objective_spec: OBJECTIVE_PRESETS.hpv, params: {
+  ...params, seed }, engine: "v2", dataset, coherence, diversity, ...ratesDiff })`
+  — i.e. the SAME body `launchRun()` builds, only `params.seed` overridden per
+  seed. Use the user's current Parameters / coherence / diversity / DSL rates so
+  the sweep matches their single-run config.
+- Wait for completion **without disturbing the main Live view / Result state**.
+  Write a dedicated helper `awaitRunDone(runId): Promise<RunResult>` that opens
+  an `EventSource(${API_URL}/runs/${runId}/stream)`, resolves on the `done`
+  event (parse `RunResult`), rejects on `error`, and closes the stream — it must
+  NOT call `setGenerations` / `setResult` / `setStatus` (those belong to the
+  main single-run UI). (Polling `getRunStatus` until done then `getRunResult` is
+  an acceptable alternative.)
+- Per seed collect: **`winning.holdout_score`** (this is the field the Result
+  panel's "Held-out AUROC" card and the verdict display — use it, NOT
+  `holdout_auroc`, so the stability dots match the numbers already on screen) and
+  `winning.permutation_p` (from the RunResult), then `getRunTransfer(runId)` →
+  `auroc`, `p`, `n`, `found_symbols`, `missing_symbols`. If transfer fails (e.g.
+  GSE65858 not built), store null for the transfer fields and keep the held-out
+  row — never crash the sweep.
+- After all seeds: call `getFullRankDiagnostic(dataset, target)` once and build a
+  `symbol → set_name` map from `reference_marks` (p16 / cell_cycle) to tag genes.
+
+## Recurrence (the key output)
+- A seed's winner symbols come from `postReveal(winning.gene_ids)` — a bounded
+  reveal of just that winner's genes (same discipline `/transfer` uses). Source
+  it from the winner, not the transfer payload, so recurrence still works even if
+  a seed's transfer call failed. (`found_symbols ∪ missing_symbols` from the
+  transfer is an equivalent set when transfer succeeds.)
+- Tally: for every symbol, count in how many seeds it appears. Sort desc. Tag
+  each with its reference set via the map (else "alternate"/"—"). Genes in only
+  1 seed collapse into a single "passengers · 1 seed each" row (show the count of
+  such genes).
+
+## UI (new `<StabilityPanel>` — match the approved mockup)
+Reuse `<SectionCard>`, `<InfoTip>`, `fmtFit`, and the palette tokens so it looks
+native. Place it as its own SectionCard in the Result area, after the
+independent-cohort validation panel. Gated to hnsc/hpv.
+
+- Header + its own **"Run stability check"** button (independent of the main Run
+  button) + a seed-list input (comma or space separated, default the 8 above).
+  While sweeping: disable both this button and the main Run button, show
+  "Running seed {k} of {N}…", and offer a **Stop** button that sets a cancel
+  flag checked between seeds (finish the current seed, then halt).
+- **Section 1 — "Does the result hold across seeds?"** Two horizontal dot-strips
+  on a 0.5 → 1.0 axis (label "0.5 coin-flip" left, "1.0" right): Held-out AUROC
+  (accent-teal dots) and Independent AUROC (amber dots), one dot per completed
+  seed, with a light min–max band behind the dots. Caption with the two ranges
+  (`min–max`, via `fmtFit`) and a one-line plain read ("tight + high = stable,
+  not a lucky seed; the independent cohort costs a little, as expected").
+- **Section 2 — "Do the same genes keep coming back?"** Recurrence bars: symbol
+  | bar (width = seeds-appeared / N) | "{k} / {N} · {set}". Colour by tag:
+  cell_cycle amber, p16 gold, alternate teal, passengers grey. Caption: recurring
+  cell-cycle / proliferation genes = real signal; one-off passengers = noise.
+- **Section 3 — per-seed table**: seed | held-out AUROC | p | independent AUROC |
+  p | genes found/total. Route every number through `fmtFit` ("—" for null).
+- Panel `?` (InfoTip): what multi-seed proves — that the result isn't one lucky
+  random start (spread), and that the same biology recurs across independent
+  searches (recurrence); note each seed is a full run so the sweep takes a while.
+
+## CONSTRAINTS
+- Frontend-only. No engine/API/airgap change. Reuse `postRun`, the SSE stream,
+  `getRunResult`, `getRunTransfer`, `getFullRankDiagnostic`. The only gene names
+  shown are the winner's revealed symbols (already bounded via `/transfer`) and
+  the reference marks (bounded known set) — no new reveal surface.
+- The sweep must not overwrite the main single-run Result / Live view; it keeps
+  its own state (`stabilityRows`, `stabilityStatus`, `stabilityProgress`,
+  `stabilityCancel`).
+- Runs are sequential (the backend runs one at a time); show progress, allow
+  Stop. Graceful nulls if a transfer fails.
+
+## Checkpoint
+- On an HNSC/HPV run, a "Stability across seeds" panel appears with a "Run
+  stability check" button; clicking it sweeps the seeds, showing live progress.
+- When done: two dot-strips (held-out + independent AUROC, one dot per seed with
+  the range), a gene-recurrence bar list tagged by reference set, and a per-seed
+  table. Hidden for colorectal.
+- Main single-run Result/Live view is untouched while the sweep runs.
+- `tsc` clean; no behaviour change to existing runs; airgap untouched.

Read docs/PROMPT_survival_encoding_and_help.md

@@ -0,0 +1,34 @@
+# Prompt — encode site-survival on the scatter + plain-language "Survives" help
+
+Two presentation fixes around the confound-survival flags. Frontend-only; the data (`survives_site`, `survives_purity`, the subgroup AUROCs) is already on the module row. No engine/API/airgap change. `tsc` after.
+
+## Part 1 — encode site-survival on the scatter (don't hard-remove)
+The fitness × synergy scatter currently plots all highlighted groups (winner / cell_cycle / p16) the same, regardless of whether they passed the site confound check. Distinguish them, and let the user declutter:
+- **Encode site-survival by fill**, keeping colour = category (winner teal / cell_cycle amber / p16 gold):
+  - `survives_site === true` → **solid** dot (as now).
+  - `survives_site === false` → **hollow / outline-only** (or clearly faded) dot — so failed-site groups read as "less trustworthy" at a glance.
+  - `survives_site == null` (n too small) → solid but with a thin dashed outline, or just treat as solid; keep it simple.
+- (Encode **site** only — purity is mostly "—" here, so it carries no signal; don't encode it.)
+- Add a small **toggle** above/beside the scatter: **"Hide groups that fail the site check"** (default OFF). When on, drop the `survives_site === false` highlighted dots from the plot (background grey dots are unaffected).
+- Update the scatter legend/caption to note: "solid = survives the site check; hollow = fails it (its signal is partly location)."
+
+## Part 2 — add a plain-language "?" on the "Survives" column
+In the merged table, the "Survives" column header (`<th>Survives</th>`, ~Lab.tsx 3741) needs a clear "?" (rewrite `TIPS.moduleSurvival`, or attach a help trigger right on that header). Use this plain-English copy (anyone should understand it):
+
+> **What "Survives" checks:** whether a group still separates HPV when you take away a possible *confounder* — something that travels with HPV but isn't HPV biology.
+>
+> **Site.** HPV+ tumours are mostly in the oropharynx (back of the throat), so a gene could look like an "HPV gene" just by marking that location. The site check re-scores the group using **only oropharynx patients** (everyone the same location). The two numbers are full-cohort → oropharynx-only.
+>
+> **Purity.** A tumour sample is a mix of cancer cells and immune cells; HPV+ tumours carry more immune cells, so a gene could look like an "HPV gene" just by marking immune content. The purity check re-scores using only the **"purest" (least-immune) tumours**. It's usually "—" here because those tumours include almost no HPV+ patients, so there's nothing to test.
+>
+> **✓** = held up when the confounder was held constant (likely real signal). **✗** = dropped past the tolerance (part of it was the confounder). **—** = couldn't test (too few patients).
+
+- Keep the existing per-chip tooltips (the `full → subgroup` detail) unchanged; this is the column-level explainer.
+
+## CONSTRAINTS
+- Frontend-only; uses `survives_site` already on the row. No engine/API/airgap change.
+
+## Checkpoint
+- Scatter: failed-site highlighted dots render hollow/faded; a "Hide groups that fail the site check" toggle removes them; legend explains solid vs hollow.
+- The "Survives" column has a plain-language "?" explaining site, purity, ✓, ✗, and —.
+- `tsc` clean.

Read docs/PROMPT_target_fix.md

@@ -0,0 +1,24 @@
+# Prompt — Bind the scoring target to the objective (bug fix) + update objective explainers
+
+A run produced this "TMB winner": `Associate(Reduce(Select(M,[…]),median), msi, spearman)` — it scored against the MSI label under the TMB objective. The engine was able to choose its own target because labels were exposed as operands. Fix that, and update the objective help text that now describes the old (wrong) behaviour. Read the grammar / interpreter / program_repr / objective + tooltip code first; don't assume names. Keep the airgap green and existing presets/Streamlit working.
+
+## 1. BIND THE TARGET (engine fix)
+- REMOVE the target labels (msi, tmb) as selectable operands/terminals in the grammar. The engine must not reference a label as data anywhere in a program.
+- The program produces a per-patient score (Vector) from genes (+ optional Effect adjustment on confounders). The terminal scoring/fitting step's TARGET is BOUND to the active objective's target, not chosen:
+  - MSI objective → AUROC of the score vs msi.
+  - TMB objective → signed-negative correlation of the score vs tmb.
+  - The engine may still choose: raw (Associate) vs adjusted (Effect, residualize on stage/age), and correlation kind (pearson/spearman). It may NOT choose the target.
+  - Same rule for Fit: it trains on the objective's target only, evaluated held-out.
+- Confounders stage/age remain available ONLY as Effect adjustment inputs and as Split predicates — never as prediction targets.
+- program_repr shows the bound objective target (e.g. `Associate(score, <tmb>, spearman)` under the TMB objective), not a free argument.
+- Add a guard/test: assert no program references any label column as a free operand, and that the scoring target always equals the active objective's target. Keep the airgap test green.
+
+## 2. UPDATE THE OBJECTIVE EXPLAINERS (UI) to match
+The shared footer note in the objective "?" modals currently says the target can be written in the DSL "against a Load'ed label" — that is now FALSE (the program can't reference the target). Replace that shared footer note (in all four objective modals) with, verbatim:
+
+"What the program chooses is how to build the score and how to compare it — a raw association (Associate) or a confounder-adjusted one (Effect), plus the correlation kind. What stays outside the DSL is the compass: the target it's scored against, in which direction, judged honestly on held-out data. The program can't pick the target — that would let the answer into the language."
+
+Leave the rest of each modal's content unchanged (the "Represented as", "Why negative", "Correlation ≠ causation", and "Toward causal" sections remain correct).
+
+## Checkpoint
+Re-run the TMB objective and show me: (a) the winner's program_repr now scores against TMB (not msi); (b) the guard/test passes; (c) the Mutation-burden "?" modal shows the updated footer. Confirm the airgap test is green and existing presets/Streamlit still work.

Read docs/PROMPT_ui_fixes_default_axes_tooltip.md

@@ -0,0 +1,35 @@
+# Prompt — four small UI fixes (default cohort, remove peel-off panel, fix two clipped bits)
+
+All frontend-only, verified line refs in `web/app/Lab.tsx`. No engine/API/airgap change. `tsc` after.
+
+## 1. Make Head & Neck the default cohort
+- `const [dataset, setDatasetState] = useState<DatasetId>("coadread")` (~line 541) → default to **`"hnsc"`**.
+- Make sure the initial selected **objective** matches the default dataset's first objective (HPV detection) — i.e. on first load the page shows Head & Neck + HPV detection, not a colorectal objective. (Mirror the logic at ~line 592 that picks `DATASET_REGISTRY[next].objectives[0].key` on dataset change; apply the same to the initial objective state.)
+- Colorectal stays selectable; only the default changes.
+
+## 2. Remove the "Discovered axes" / "Find next axis" panel from the UI
+Peel-off only makes sense for the unsupervised objective (which isn't a UI objective anymore); on a supervised HPV run it re-finds the same signal and shows nonsensical "Aligns with MSI / TMB" text. Remove it from the page:
+- Drop the `<DiscoveredAxes …/>` render (~line 867). Keep the `DiscoveredAxes` component definition (~884) and the backend untouched — just don't render it. (Equivalently, gate its render to `target === "none"`, which never occurs in the UI.)
+
+## 3. Fix the clipped "Cancer / problem" card subtitle
+The dataset cards show "(TCGA COADREAD" / "(TCGA HNSC" with the closing paren cut off. In `CancerSelector` (~line 1043), the subtitle line is being clipped (truncate / overflow / fixed width). Let it display in full — allow it to wrap or size to content so the complete "(TCGA COADREAD)" / "(TCGA HNSC)" shows, including the closing paren. (Confirm the source strings in `DATASET_REGISTRY` ~379/389 are complete — `longLabel` has the paren — so this is a CSS clipping fix, not a missing character.)
+
+## 4. Fix the "Survives" "?" tooltip cutting off
+The Survives column "?" copy is multi-paragraph and overflows/clips as a hover tooltip (see screenshot — text is cut on the right). A hover tooltip is the wrong vehicle for that much text. Fix it one of these ways (prefer the first):
+- **Convert this "?" to a click-to-open modal** using the existing rich ParamHelp modal infrastructure (the same pattern the objective/parameter rich explainers use), with the plain-language site/purity/✓/✗/— copy. Modals don't clip.
+- If keeping it a hover tooltip: render it in a portal at the document root with a sensible `max-width` and **viewport clamping** (flip/shift so it never overflows the right edge), and ensure no ancestor `overflow: hidden` crops it.
+- While here, make the shared "?" tooltip component clamp to the viewport generally, so no other long tooltip clips either.
+
+## 5. Swap the "What is HPV? (the biology)" diagram to the new card-wall figures
+Two repo-root SVGs have been updated/added (literal colours, same convention as the old `hpv_reference.svg`): **`hpv_reference.svg`** (rewritten — the "HPV INFECTION — the virus dismantles both brakes" card wall, viewBox `0 0 920 660`) and **`normal_cell_reference.svg`** (new — "NORMAL CELL — how division is controlled", viewBox `0 0 920 900`). Update `web/app/HPVBiologyPanel.tsx` so the panel shows the **normal-cell diagram first, then the HPV diagram** (normal baseline → the viral attack):
+- The panel inlines its SVG as hand-translated JSX (`HPVDiagram()` ~line 54). Replace `HPVDiagram`'s body with the **new** `hpv_reference.svg` contents, and add a sibling component (e.g. `NormalCellDiagram`) inlining `normal_cell_reference.svg`, rendered ABOVE the HPV one inside the panel.
+- Translate SVG attributes to JSX exactly as the existing component does: `font-family`→`fontFamily`, `font-size`→`fontSize`, `font-weight`→`fontWeight`, `stroke-width`→`strokeWidth`, `text-anchor`→`textAnchor`, `stroke-dasharray`→`strokeDasharray`, `marker-end`→`markerEnd`, etc. Each `<svg>` keeps `viewBox` + `style={{ width: "100%", height: "auto", display: "block" }}` so it scales without overflow (same wrapper as today).
+- Keep the panel's collapsible "What is HPV? (the biology)" framing unchanged — only the figure changes.
+
+## Checkpoint
+- Page loads on **Head & Neck + HPV detection** by default; Colorectal still selectable.
+- The "What is HPV?" panel shows the new normal-cell + HPV card-wall diagrams (scaled, no overflow).
+- No "Discovered axes" / "Find next axis" panel in the UI.
+- The Cancer/problem card subtitles show in full (closing paren visible).
+- The Survives "?" opens without clipping (modal, or clamped tooltip).
+- `tsc` clean; no engine/API/airgap change.

airgap/__init__.py

@@ -0,0 +1,17 @@
+"""Airgap layer for OncoDSL.
+
+Sits between the data layer (real gene symbols) and the engine (opaque IDs).
+The GP engine sees only an `anonymise()`d view of the expression matrix so the
+search can't lean on known gene-name priors — the MSI signature, if it emerges,
+must be rediscovered from the data.
+
+Public API:
+- `anonymise(matrix)` — rename columns to opaque IDs g00001…; write the
+  symbol<->ID map to `data/processed/_sealed_gene_map.json` on first call.
+- `reveal(feature_ids)` — translate opaque IDs back. THE ONLY function allowed
+  to open the sealed map. Must not be imported by the engine.
+"""
+
+from airgap.seal import OPAQUE_ID_RE, SEALED_PATH, anonymise, reveal
+
+__all__ = ["anonymise", "reveal", "SEALED_PATH", "OPAQUE_ID_RE"]

airgap/seal.py

@@ -0,0 +1,85 @@
+"""Implementation of the airgap (gene-name anonymisation).
+
+`anonymise(matrix)` renames every column to a deterministic opaque ID (g00001…)
+and writes the symbol<->ID mapping to a SEALED file the GP engine must never
+read. `reveal(feature_ids)` is the ONLY function allowed to open that file;
+it is used in a final unmask step inside the H1 fixture / viewer.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+import pandas as pd
+
+from data_pipeline import schema
+
+SEALED_PATH = schema.PROCESSED_DIR / "_sealed_gene_map.json"
+ID_FORMAT = "g{:05d}"
+OPAQUE_ID_RE = re.compile(r"^g\d+$")
+
+
+def _read_sealed() -> dict:
+    with SEALED_PATH.open() as f:
+        return json.load(f)
+
+
+def _write_sealed(payload: dict) -> None:
+    SEALED_PATH.parent.mkdir(parents=True, exist_ok=True)
+    with SEALED_PATH.open("w") as f:
+        json.dump(payload, f, indent=2, sort_keys=True)
+
+
+def anonymise(matrix: pd.DataFrame) -> pd.DataFrame:
+    """Return `matrix` with columns renamed to opaque IDs g00001, g00002, …
+
+    The first call writes the sealed symbol<->ID map. Later calls reuse it; if
+    the matrix's columns are not all covered by the existing map (e.g. you
+    rebuilt expression with a different gene set), the call fails loudly so a
+    stale airgap can't silently leak information through reordered IDs.
+    """
+    cols = list(matrix.columns)
+    if SEALED_PATH.exists():
+        sealed = _read_sealed()
+        id_to_symbol = sealed["id_to_symbol"]
+        symbol_to_id = {v: k for k, v in id_to_symbol.items()}
+        unknown = [c for c in cols if c not in symbol_to_id]
+        if unknown:
+            raise ValueError(
+                f"airgap.anonymise: {len(unknown)} matrix columns are not in the "
+                f"sealed map at {SEALED_PATH} (e.g. {unknown[:5]}). Delete the "
+                "sealed file to re-seal against the current matrix."
+            )
+    else:
+        symbols_sorted = sorted(cols)
+        id_to_symbol = {ID_FORMAT.format(i + 1): s for i, s in enumerate(symbols_sorted)}
+        symbol_to_id = {v: k for k, v in id_to_symbol.items()}
+        _write_sealed({
+            "id_to_symbol": id_to_symbol,
+            "n_genes": len(id_to_symbol),
+            "id_format": ID_FORMAT,
+        })
+
+    renamed = matrix.rename(columns={s: symbol_to_id[s] for s in cols})
+    return renamed.reindex(sorted(renamed.columns), axis=1)
+
+
+def reveal(feature_ids: list[str]) -> list[str]:
+    """Translate opaque IDs back to real gene symbols. Sealed-map-only entry point.
+
+    No other module in the codebase is permitted to open the sealed map. This
+    function is reserved for a final reveal step in the H1 fixture or the
+    viewer — it must never be imported by the engine.
+    """
+    if not SEALED_PATH.exists():
+        raise FileNotFoundError(
+            f"airgap.reveal: sealed map not yet written at {SEALED_PATH}; "
+            "call anonymise() at least once first."
+        )
+    id_to_symbol = _read_sealed()["id_to_symbol"]
+    missing = [fid for fid in feature_ids if fid not in id_to_symbol]
+    if missing:
+        raise KeyError(f"airgap.reveal: unknown opaque IDs: {missing[:5]}")
+    return [id_to_symbol[fid] for fid in feature_ids]

api/__init__.py

@@ -0,0 +1,5 @@
+"""FastAPI surface for OncoDSL — serves the persisted H2 artefacts.
+
+Keeping this thin and framework-agnostic so the same endpoints can drive
+either the project's Streamlit viewer or a future React/Next.js front end.
+"""