diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000000000000000000000000000000000000..15cf960b06374f2ae439a3a5247bf780e37f2908 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,45 @@ +{ + "permissions": { + "allow": [ + "Bash(.venv/bin/python -c ' *)", + "Bash(.venv/bin/python *)", + "Bash(git add *)", + "Bash(git commit -m ' *)", + "Bash(gh auth *)", + "Bash(gh repo *)", + "Bash(.venv/bin/streamlit run *)", + "Bash(.venv/bin/pip install *)", + "Bash(.venv/bin/pytest -x)", + "Bash(dot -V)", + "Bash(curl -s -o /dev/null -w \"HTTP %{http_code}\\\\n\" http://localhost:8765/)", + "Bash(curl -s http://localhost:8765/_stcore/health)", + "Bash(.venv/bin/pytest -q)", + "Bash(git push *)", + "Bash(curl -s -o /dev/null -w \"HTTP %{http_code}\\\\n\" http://localhost:8766/)", + "Bash(curl -s http://localhost:8766/_stcore/health)", + "Bash(.venv/bin/pytest tests/test_engine.py tests/test_airgap.py -q)", + "Bash(.venv/bin/uvicorn api.app:app *)", + "Bash(curl -s http://127.0.0.1:8000/health)", + "Bash(python -m json.tool)", + "Bash(curl -sX POST http://127.0.0.1:8000/reveal -H 'Content-Type: application/json' -d '{\"gene_ids\":[\"g07588\",\"g13403\",\"g05340\",\"g12032\"]}')", + "Bash(npm run *)", + "Bash(curl -s -o /tmp/lab.html -w \"HTTP %{http_code} \\(%{size_download}B\\)\\\\n\" http://127.0.0.1:3000/)", + "Read(//private/tmp/**)", + "Bash(curl -s -o /tmp/lab.html -w \"HTTP %{http_code} \\(%{size_download}B\\)\\\\n\" http://127.0.0.1:3001/)", + "Bash(npx tsc *)", + "Bash(curl *)", + "Bash(npm install *)", + "Bash(git restore *)", + "Bash(.venv/bin/pytest tests/test_engine_v2.py -q)", + "Bash(.venv/bin/pytest tests/test_api_airgap.py -q)", + "Bash(grep -n \"return \\(\\\\s*$\\\\|\" web/app/Lab.tsx)", + "Bash(node *)", + "Bash(python3 -c ' *)", + "Bash(python3 -m json.tool)", + "Bash(grep -n \"^};$\\\\|prefilter_n: {\" web/app/paramHelpContent.tsx)", + "Bash(python3 *)", + "Bash(npx --prefix /Users/govindbalakrishnan/Govind_Projects/oncodsl/web tsc --noEmit -p /Users/govindbalakrishnan/Govind_Projects/oncodsl/web)", + "Bash(echo \"EXIT=$?\")" + ] + } +} diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..6777bc154d2ebc2ccdd0840b0b257ba4fddf5a00 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +.git +.venv +**/__pycache__ +**/*.pyc +web/node_modules +web/.next +# Raw downloads are large and not needed at runtime; processed parquet IS needed. +data/raw +data/raw_hnsc +data/raw_gse65858 +tests +OncoDSL_cover_note.docx diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..78bd8d8dce79a2932b87b73ab68bf06f8c6d9036 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +.venv/ +data/raw/ +data/processed/ +data/raw_hnsc/ +data/processed_hnsc/ +data/raw_gse65858/ +data/processed_gse65858/ +__pycache__/ +*.pyc +.pytest_cache/ +.DS_Store +.python-version diff --git a/.pytest_cache/.gitignore b/.pytest_cache/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..bc1a1f6167d09c909aad37280b760bb715d0f1da --- /dev/null +++ b/.pytest_cache/.gitignore @@ -0,0 +1,2 @@ +# Created by pytest automatically. +* diff --git a/.pytest_cache/CACHEDIR.TAG b/.pytest_cache/CACHEDIR.TAG new file mode 100644 index 0000000000000000000000000000000000000000..fce15ad7eaa74e5682b644c84efb75334c112f95 --- /dev/null +++ b/.pytest_cache/CACHEDIR.TAG @@ -0,0 +1,4 @@ +Signature: 8a477f597d28d172789f06886806bc55 +# This file is a cache directory tag created by pytest. +# For information about cache directory tags, see: +# https://bford.info/cachedir/spec.html diff --git a/.pytest_cache/README.md b/.pytest_cache/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b89018ced91c0a8af7f3f23ce8901870da89f3a0 --- /dev/null +++ b/.pytest_cache/README.md @@ -0,0 +1,8 @@ +# pytest cache directory # + +This directory contains data from the pytest's cache plugin, +which provides the `--lf` and `--ff` options, as well as the `cache` fixture. + +**Do not** commit this to version control. + +See [the docs](https://docs.pytest.org/en/stable/how-to/cache.html) for more information. diff --git a/.pytest_cache/v/cache/lastfailed b/.pytest_cache/v/cache/lastfailed new file mode 100644 index 0000000000000000000000000000000000000000..44506d80e1e2614b20ebfb0c1e6a3a5c0ee9a9ca --- /dev/null +++ b/.pytest_cache/v/cache/lastfailed @@ -0,0 +1,3 @@ +{ + "tests/test_api_airgap.py::test_residualize_against_rejected_on_supervised_target": true +} \ No newline at end of file diff --git a/.pytest_cache/v/cache/nodeids b/.pytest_cache/v/cache/nodeids new file mode 100644 index 0000000000000000000000000000000000000000..aa3c23a3005da77adb28755415d4e86172b493f2 --- /dev/null +++ b/.pytest_cache/v/cache/nodeids @@ -0,0 +1,136 @@ +[ + "tests/test_airgap.py::test_anonymise_fails_loudly_on_unknown_gene", + "tests/test_airgap.py::test_anonymise_idempotent_with_existing_sealed_map", + "tests/test_airgap.py::test_anonymise_renames_to_opaque_ids_and_writes_sealed_map", + "tests/test_airgap.py::test_engine_does_not_import_sealed_map_or_reveal", + "tests/test_airgap.py::test_reveal_rejects_unknown_id", + "tests/test_airgap.py::test_reveal_round_trips", + "tests/test_airgap.py::test_search_only_accepts_anonymised_matrix", + "tests/test_api_airgap.py::test_coadread_rejects_hpv_target", + "tests/test_api_airgap.py::test_enable_search_false_pins_search_to_zero", + "tests/test_api_airgap.py::test_evaluate_rejects_hnsc_reference_in_coadread", + "tests/test_api_airgap.py::test_evaluate_rejects_unknown_reference_set", + "tests/test_api_airgap.py::test_evaluate_returns_only_supplied_ids", + "tests/test_api_airgap.py::test_hnsc_hpv_run_is_airgap_clean", + "tests/test_api_airgap.py::test_hnsc_rejects_msi_target", + "tests/test_api_airgap.py::test_lab_payloads_contain_only_opaque_ids[20-spec0]", + "tests/test_api_airgap.py::test_lab_payloads_contain_only_opaque_ids[20-spec1]", + "tests/test_api_airgap.py::test_lab_payloads_contain_only_opaque_ids[20-spec2]", + "tests/test_api_airgap.py::test_lab_payloads_contain_only_opaque_ids[None-spec0]", + "tests/test_api_airgap.py::test_lab_payloads_contain_only_opaque_ids[None-spec1]", + "tests/test_api_airgap.py::test_lab_payloads_contain_only_opaque_ids[None-spec2]", + "tests/test_api_airgap.py::test_lab_payloads_contain_only_opaque_ids[spec0]", + "tests/test_api_airgap.py::test_lab_payloads_contain_only_opaque_ids[spec1]", + "tests/test_api_airgap.py::test_lab_stream_replays_after_done_with_only_opaque_ids", + "tests/test_api_airgap.py::test_legacy_endpoints_still_present", + "tests/test_api_airgap.py::test_modules_endpoint_404_for_unknown_run", + "tests/test_api_airgap.py::test_modules_endpoint_rejects_unsupervised_run", + "tests/test_api_airgap.py::test_modules_endpoint_returns_425_while_running", + "tests/test_api_airgap.py::test_modules_endpoint_returns_opaque_only_modules", + "tests/test_api_airgap.py::test_operator_usage_endpoint_404_for_unknown_run", + "tests/test_api_airgap.py::test_operator_usage_endpoint_425_while_running", + "tests/test_api_airgap.py::test_operator_usage_endpoint_returns_opaque_counts", + "tests/test_api_airgap.py::test_population_endpoint_is_airgap_clean[v1]", + "tests/test_api_airgap.py::test_population_endpoint_is_airgap_clean[v2]", + "tests/test_api_airgap.py::test_post_runs_rejects_unsupported_objective", + "tests/test_api_airgap.py::test_rates_override_search_zero_pins_search_to_zero", + "tests/test_api_airgap.py::test_residualisation_chain_runs_and_stays_airgap_clean", + "tests/test_api_airgap.py::test_residualize_against_rejected_on_supervised_target", + "tests/test_api_airgap.py::test_residualize_against_unknown_id_returns_400", + "tests/test_api_airgap.py::test_residualize_against_works_on_supervised_targets", + "tests/test_api_airgap.py::test_transfer_endpoint_425_while_running", + "tests/test_api_airgap.py::test_transfer_endpoint_carries_only_winner_revealed_symbols", + "tests/test_api_airgap.py::test_transfer_endpoint_rejects_non_hpv_run", + "tests/test_api_airgap.py::test_unsupervised_run_emits_posthoc_alignment", + "tests/test_build.py::test_label_msi_thresholds", + "tests/test_build.py::test_normalise_stage", + "tests/test_build.py::test_parse_os_status", + "tests/test_build.py::test_read_clinical_happy_path", + "tests/test_build.py::test_read_clinical_missing_column_raises_loudly", + "tests/test_dsl.py::test_apply_returns_probabilities_indexed_like_input", + "tests/test_dsl.py::test_associate_pearson_perfect_negative", + "tests/test_dsl.py::test_associate_pearson_perfect_positive", + "tests/test_dsl.py::test_associate_spearman_monotone_only", + "tests/test_dsl.py::test_associate_unknown_kind_raises", + "tests/test_dsl.py::test_effect_drops_nans_and_reports_n_used", + "tests/test_dsl.py::test_effect_partial_correlation_removes_confounder", + "tests/test_dsl.py::test_fit_separable_features_score_above_random", + "tests/test_dsl.py::test_reduce_mean_collapses_rows", + "tests/test_dsl.py::test_reduce_rejects_unknown_agg", + "tests/test_dsl.py::test_reduce_supports_full_agg_vocab", + "tests/test_dsl.py::test_search_ranks_by_objective_and_takes_top_k", + "tests/test_dsl.py::test_search_rejects_non_opaque_columns", + "tests/test_dsl.py::test_select_missing_column_raises", + "tests/test_dsl.py::test_select_picks_columns_in_order", + "tests/test_dsl.py::test_split_partitions_by_predicate", + "tests/test_engine.py::test_baseline_uses_first_k", + "tests/test_engine.py::test_correlation_objective_pipeline_end_to_end", + "tests/test_engine.py::test_correlation_objective_prefilter_finds_informative", + "tests/test_engine.py::test_crossover_produces_valid_child", + "tests/test_engine.py::test_cv_auroc_strong_for_informative_program", + "tests/test_engine.py::test_fitness_penalises_size", + "tests/test_engine.py::test_mutate_preserves_invariants", + "tests/test_engine.py::test_permutation_null_is_around_chance", + "tests/test_engine.py::test_pipeline_end_to_end_on_synthetic", + "tests/test_engine.py::test_pipeline_no_prefilter_samples_full_pool", + "tests/test_engine.py::test_pipeline_rejects_named_columns", + "tests/test_engine.py::test_pipeline_unknown_objective_rejected_by_spec_builder", + "tests/test_engine.py::test_prefilter_surfaces_the_informative_features", + "tests/test_engine.py::test_program_state_shape", + "tests/test_engine.py::test_random_program_well_formed", + "tests/test_engine.py::test_run_gp_improves_or_holds_fitness_over_generations", + "tests/test_engine.py::test_streaming_pipeline_emits_one_event_per_generation", + "tests/test_engine_v2.py::test_combine_op_vocab_executes", + "tests/test_engine_v2.py::test_constant_or_nan_output_returns_worst_fitness", + "tests/test_engine_v2.py::test_crossover_returns_same_root_type", + "tests/test_engine_v2.py::test_default_run_unchanged_by_diversity_param", + "tests/test_engine_v2.py::test_effect_default_confounders_are_stage_and_age", + "tests/test_engine_v2.py::test_effect_extended_confounders_include_sex_and_race", + "tests/test_engine_v2.py::test_feature_ids_returns_union_across_leaves", + "tests/test_engine_v2.py::test_feature_set_rejects_empty", + "tests/test_engine_v2.py::test_fitness_floors_mismatched_target_to_worst", + "tests/test_engine_v2.py::test_fitness_floors_no_select_program_under_unsup", + "tests/test_engine_v2.py::test_full_grammar_operators_execute", + "tests/test_engine_v2.py::test_immigrant_fraction_injects_fresh_programs", + "tests/test_engine_v2.py::test_leakage_guard_floors_contaminated_single_gene_winner", + "tests/test_engine_v2.py::test_matrix_terminal_executes_to_full_matrix", + "tests/test_engine_v2.py::test_mutation_preserves_root_type_and_budget", + "tests/test_engine_v2.py::test_objective_from_spec_dispatch", + "tests/test_engine_v2.py::test_omni_auroc_is_orientation_agnostic", + "tests/test_engine_v2.py::test_oos_silhouette_kills_non_generalising_program", + "tests/test_engine_v2.py::test_oos_silhouette_rewards_consistent_split", + "tests/test_engine_v2.py::test_oos_silhouette_too_small_floored", + "tests/test_engine_v2.py::test_oos_silhouette_train_only_constant_floored", + "tests/test_engine_v2.py::test_peeloff_residualisation_uses_train_rows_only", + "tests/test_engine_v2.py::test_permutation_null_is_around_chance", + "tests/test_engine_v2.py::test_pipeline_end_to_end_produces_typed_repr_and_diverse_population", + "tests/test_engine_v2.py::test_pipeline_payloads_carry_only_opaque_ids", + "tests/test_engine_v2.py::test_pipeline_rejects_named_columns", + "tests/test_engine_v2.py::test_program_repr_is_a_tree_serialisation", + "tests/test_engine_v2.py::test_ramped_population_is_diverse", + "tests/test_engine_v2.py::test_random_program_obeys_depth_budget", + "tests/test_engine_v2.py::test_random_program_returns_vector_typed_tree", + "tests/test_engine_v2.py::test_reduce_supports_every_agg", + "tests/test_engine_v2.py::test_select_restricts_columns_to_feature_set", + "tests/test_engine_v2.py::test_survival_still_rejected_for_v2", + "tests/test_engine_v2.py::test_synthesis_binds_scoring_target_to_objective", + "tests/test_engine_v2.py::test_termination_guarantee_closes_at_depth_zero", + "tests/test_engine_v2.py::test_unsupervised_objective_accepted", + "tests/test_engine_v2.py::test_unsupervised_pipeline_strips_labels_from_exec_context", + "tests/test_engine_v2.py::test_unsupervised_programs_always_use_gene_select", + "tests/test_engine_v2.py::test_unsupervised_silhouette_kills_self_divide_outlier_split", + "tests/test_engine_v2.py::test_unsupervised_silhouette_rewards_clean_2_cluster_split", + "tests/test_engine_v2.py::test_unsupervised_synthesis_is_vector_only", + "tests/test_h1.py::test_h1_recovers_known_direction_on_synthetic_data", + "tests/test_hpv_rank.py::test_absent_gene_reported_gracefully", + "tests/test_hpv_rank.py::test_present_genes_get_finite_ranks_with_right_sign", + "tests/test_hpv_rank.py::test_zero_variance_gene_dropped_from_ranking", + "tests/test_tmb_rank.py::test_absent_gene_reported_gracefully", + "tests/test_tmb_rank.py::test_present_genes_get_finite_ranks_with_right_sign", + "tests/test_tmb_rank.py::test_zero_variance_gene_dropped_from_ranking", + "tests/test_transfer_gse65858.py::test_missing_symbols_are_reported_without_crashing", + "tests/test_transfer_gse65858.py::test_no_found_symbols_returns_graceful_none", + "tests/test_transfer_gse65858.py::test_noise_only_symbols_are_around_chance", + "tests/test_transfer_gse65858.py::test_returned_payload_contains_only_supplied_symbols", + "tests/test_transfer_gse65858.py::test_signal_gene_produces_high_auroc_and_small_p" +] \ No newline at end of file diff --git a/.streamlit/config.toml b/.streamlit/config.toml new file mode 100644 index 0000000000000000000000000000000000000000..dad70a473db3367e6bb793641643f80007cf22f6 --- /dev/null +++ b/.streamlit/config.toml @@ -0,0 +1,7 @@ +[theme] +base = "light" +primaryColor = "#3A6B7E" +backgroundColor = "#FAFAF7" +secondaryBackgroundColor = "#F1EFEA" +textColor = "#23303A" +font = "sans serif" diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000000000000000000000000000000000000..0c6e38094f6f1bc19c9243ff5b85f5035af03732 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,2086 @@ +# OncoDSL — context for Claude + +Long-term goal: a genetic-programming engine that, over a small DSL of data +operations, **rediscovers the MSI / mismatch-repair gene signature from TCGA +colorectal data blind**. The engine must not see real gene names — an "airgap" +layer between the data layer and the engine hides them. + +## Chunk status + +Built in chunks. Current state: + +- **Chunk 1 (done):** project scaffold + cBioPortal data load + Streamlit + sanity viewer. See `README.md`. +- **Chunk 2 (done):** DSL operators (`Load`, `Select`, `Reduce`, `Split`, + `Associate`, `Effect`, `Search`, `Fit`, `Apply`) live in `dsl/`. Airgap + (`anonymise` / `reveal` + sealed map at `data/processed/_sealed_gene_map.json`) + lives in `airgap/`. H1 verification (known-answer MMR + immune check on the + NAMED matrix) lives in `validate/h1.py`. Viewer is split into two top-level + tabs — **Dataset** (the five sanity panels) and **Hypothesis 1** (intro, + DSL reference with nouns + verbs, graphviz composition diagram with + `Given:` input nodes, then verification outputs in this order: dynamic + pass/fail Conclusion → score distributions → Effect → Fit → collapsed + "How to read this", plus an airgap demo). Glossary expander sits above + the tabs. +- **Chunk 3 (done):** GP engine in `engine/` — strict airgap; composes + Select/Reduce/Fit programs over the anonymised matrix with prefilter + + tournament-selection GP + permutation null. Orchestrator at + `scripts/run_h2.py` (Load → anonymise → engine → persist artefacts). + Artefacts at `data/processed/h2/{evolution_log,result}.json`, both + anonymised. FastAPI surface in `api/app.py` (`/health`, `/run`, + `/result`, `/reveal`) — `/reveal` is the only endpoint allowed to read + the sealed map and translates only the supplied IDs. Viewer gains a + "Hypothesis 2" tab (5 cards: hypothesis → setup → evolve replay → + result → reveal + computed conclusion) that fetches from the API. + Validation lives in `validate/h2.py` (reveal + MMR overlap). +- **Chunk 4 (done — Stage 1):** React/Next.js "Lab" at `web/` + (App Router + TS + Tailwind + Recharts). Adds a second selectable + engine objective — `CorrelationObjective(direction)` for continuous + targets like TMB — alongside `BinaryAUROCObjective` for MSI. New API: + `POST /runs` (worker thread + SSE bridge), `GET /runs/{id}`, + `GET /runs/{id}/stream` (`sse-starlette`), `GET /runs/{id}/result`, + `POST /evaluate` (reveal supplied IDs + score overlap with `"MMR"` or + `"immune"`). Lab page is one Client Component with 5 sections: + objective builder (survival / unsupervised disabled, Stage 2) → + parameter inputs → Run → live fitness curve + population grid + (survivors solid accent, discarded faded) → result + reveal & + evaluate. Reuses the same engine — `scripts/run_h2.py`, the legacy + `/run /result /reveal /health` endpoints, and the Streamlit viewer + are untouched. +- **Chunk 4 follow-ups (done):** + - **Prefilter optional + bigger GP budget.** Engine + `prefilter_n: int | None`; `None` means the GP samples its + initial-population AND its mutation operator from the full opaque + column set (~17k after NaN drop), not from a univariate shortlist. + `_prepare` returns `gp_pool` + `baseline_genes` separately — the + baseline keeps using the univariate top-K so it stays a fair sanity + check. API accepts `prefilter_n: null`. Lab UI defaults to OFF (60 + gens / 300 pop) with an "Off (all genes) / On" toggle; turning On + flips back to 30 / 150 / `prefilter_n=2000`. `scripts/run_h2.py` + untouched (explicit `--prefilter-n 2000`). + - **InfoTips everywhere.** Single `` Client Component drives + every "?" in the Lab (objective cards, parameters, run / live / + result / evaluator labels + sub-labels). Verbatim tooltip copy + lives in the `OBJECTIVE_TIPS` / `PARAM_TIPS` / `TIPS` records at + the top of `Lab.tsx` — never inline. + - **Program graph (React Flow).** `web/app/ProgramGraph.tsx` reads + `feature_sets` straight from the winner / candidate payload (no + backend change) and renders the DSL program as a node graph: + Tier-1 lanes [Expression matrix → Select(opaque IDs) → Reduce + (mean) → Score pill] per gene-set, then [Score(s) → Fit → output + node]. Tier-1 = light-teal group box, Tier-2 = dashed `#BC6B2E` + wrapper (omitted for single-set programs). Custom node types only, + palette tokens only; React Flow chrome stripped via + `ProgramGraph.css` (`opacity:0` handles, no Background, no + MiniMap, no Controls, attribution hidden, scroll passes through). + The graph card defaults to the winning program; clicking any + candidate in the population grid swaps the graph to that candidate + in place. After the Evaluator runs, Select-node pills show real + symbols and matched reference genes are filled in the project + highlight (`#BC6B2E`). +- **Chunk 5 (done — engine_v2 + full-population UI):** + - **engine_v2/** — strongly-typed program synthesis. Programs are + trees over a five-type grammar (`Matrix`, `Vector`, `FeatureSet`, + `Agg`, `Op`). Nodes: `MatrixTerminal`, `Select(Matrix,FeatureSet) + →Matrix`, `Reduce(Matrix,Agg)→Vector`, `Combine(Vector,Vector,Op) + →Vector`. `Reduce.agg ∈ {mean, median, max, min, var}`, + `Combine.op ∈ {add, sub, mul, protected_div, mean}`. Output type = + `Vector`. Split / Effect deferred to v2.1. Ramped half-and-half + init; tournament + elitism; subtree crossover with strict type + matching; subtree + point mutation; depth/node budgets. Termination + guarantee: depth-floor closes Matrix to `MatrixTerminal` and Vector + to `Reduce(MatrixTerminal, agg)`, so no tree ever has an open slot. + Reduce.agg vocab also widened in `dsl.Reduce` itself. + - **Two v2 objectives, with finite worst-case fallbacks**: + `BinaryAUROCObjective` (MSI/MSS — orientation-agnostic + `max(AUROC, 1−AUROC)`, worst = 0.5) and the v2 `CorrelationObjective` + (TMB — `−spearman(score, TMB)`, worst = 0.0). Degenerate programs + (empty FeatureSet, constant/NaN output) get the worst score; they + never leave the API as NaN/Infinity. + - **Permutation null is winner-fixed in v2:** hold the winning + program fixed and shuffle the target N times; p = fraction of + nulls ≥ observed. The old "rerun the univariate baseline on + shuffled labels" null does NOT transfer to v2 and was dropped. + - **API**: `POST /runs` accepts `engine: "v1" | "v2"`. v2 worker uses + `engine_v2.run_v2_pipeline_streaming`. Full population persisted + per generation in `run.log[i].candidates` (both engines); SSE + stream keeps the bandwidth budget light by trimming to top-12. + `GET /runs/{id}` exposes `generations_persisted`; full per-gen + populations come from `GET /runs/{id}/population/{generation}` + (404 for unpersisted indices, never 500). All endpoint returns, + the in-memory `run.result`, and the SSE payloads pass through + `_json_finite()` — NaN / ±∞ → `None`, so `JSON.parse` never breaks. + - **Frontend (web/)**: Lab now posts `engine: "v2"`. Replaced the + old top-12 grid with `PopulationTiles` (full per-gen population, + fitness-tinted teal ramp, "Nn·dD" structure signature, generation + stepper bounded to `generations_persisted`). New + `lib/programRepr.ts` is the SHARED parser for the typed + `program_repr` (also tolerates the legacy v1 `Fit(…)` shape). + `ProgramGraph.tsx` was rewritten to consume the parsed tree and + now renders arbitrary Select/Reduce/Combine/M shapes; same custom + nodes, same `ProgramGraph.css` chrome strip. `PasteToDraw.tsx` + feeds arbitrary `program_repr` text into the same renderer. + `ParameterFlow.tsx` adds a collapsed "How these parameters + relate" disclosure under the Parameters card caption — a centered + single-column HTML/flex diagram (max-width 460px, width 100%, + never overflows): step boxes, colour-grouped space/effort/ + validation pills that wrap on narrow widths, plain text `↓` + arrows, amber final "permutation p" box. Generations pill uses a + `↻ ` text prefix (not an icon glyph). A single + `lib/fmt.ts::fmtFit()` formats every fitness/score across tiles, + cards, tooltips, the result panel, and the live curve — falling + back to "—" if a non-finite value ever reaches the UI; tiles also + use `fitnessForOrder()` so any non-finite sinks to the bottom of + the grid with the lightest tint. +- **Chunk 5 follow-ups (done — UX polish, no GP / API changes):** + - **Rich Parameter help.** New `` Client + Component replaces the small InfoTip on the 8 Parameters labels. + Hover/focus still shows the SHORT one-liner; click opens a modal + dialog with text + diagrams. `ParamHelpProvider` mounts a single + portal-modal at the Lab root so any trigger can open it (the + ParameterFlow chart's pill names are also ParamHelp triggers). + Registry of detailed content lives in `web/app/paramHelpContent.tsx`; + six diagrams (Generations loop, MaxSets, Lambda trade-off, Seed + dice, Permutations histogram, Prefilter funnel) live under + `web/app/diagrams/`. Modal is role="dialog" aria-modal="true" with + Esc / X / backdrop close, focus trap, and focus return. + - **Program graph + paste-to-draw.** `ProgramGraph.tsx` now adds + Tier-1 (light teal `#F1F6F7` / `#3A6B7E`) and Tier-2 (dashed + `#BC6B2E`) wrapper nodes when the parsed root is `Combine` (single- + score programs skip wrappers). `PasteToDraw.tsx` is now controlled + (text lifted to Lab so tile clicks AND the winner-default both + write into it), placeholder fixed to a well-formed example, hint + spells out the Reveal restriction for pasted text, and a collapsed + "Format & examples" disclosure documents the grammar (`M`, + `Select`, `Reduce` aggs, `Combine` ops, rules, annotated example). + - **Copy & load.** New `web/app/CopyButton.tsx` (`navigator.clipboard` + with `execCommand("copy")` fallback) drives a "Copy program" button + next to the winner in the Result panel and an icon-only copy + overlay on every tile in `PopulationTiles`. Tile body uses + `role="button" tabIndex={0}` so the inner icon-button can stop + propagation; clicking the body loads the program into the textarea + AND the graph. +- **Chunk 6 (done — full DSL in engine_v2 + objective explainers):** + - **Full DSL grammar in engine_v2.** Typed grammar grew from + {Matrix, Vector} to {Matrix, Vector, Scalar, Model}. New nodes: + `Split(Vector, Predicate)→Vector` (predicates: `score` = above / + below median, `stage_late` = AJCC III/IV; one level only, + mean-centre per branch, min subgroup guard); + `Associate(Vector, target, kind)→Scalar` and + `Effect(Vector, target, kind)→Scalar` (Effect residualises on + stage + age — observational backdoor adjustment, "only as good as + the measured confounders"); `FitApply(Vector, target)→Vector` + (Fit a LR/OLS on the score, Apply in place); `Search(Matrix, k) + →Matrix` (bounded univariate nested ranker, capped at k ≤ 4 and + 200 cols, **gated OFF by default** per A4 — visible in the + grammar but rate 0 until cost is acceptable). + - **ExecContext** threads through every `Node.execute(ctx)` — + bundles `M` (opaque), `clinical` (named: stage, age only), and + `labels` (named: msi, tmb only). The pipeline and worker plumb + clinical + the OTHER target through so a v2 MSI run can still + Associate / Effect / FitApply against TMB and vice versa. + - **Fitness handles every output type.** Vector → objective metric + on per-patient scores; Scalar (Associate / Effect) IS the + fitness, oriented per objective (|·| for MSI, `-x` for TMB); + Model → routed through FitApply which auto-Applies and emits a + Vector. Worst-case floor still applies (0.5 MSI, 0.0 TMB). + - **Typed GP** extended with type-safe init / crossover / mutation + for every new node; ramped half-and-half now spawns ~20% + Scalar-rooted programs so Associate / Effect appear in the + population; rates dict configures Effect / Split / FitApply / + Search injection. Termination guarantee: Matrix → MatrixTerminal, + Vector → Reduce(MatrixTerminal, agg), Scalar → + Associate(Reduce(MatrixTerminal, "mean"), target, "spearman"). + - **Parser + renderer + format guide.** `web/lib/programRepr.ts` + parses every new operator (spaces tolerated). `ProgramGraph.tsx` + renders Split / Search / FitApply / Associate / Effect as + accent-bordered verb nodes (same theme). PasteToDraw's "Format & + examples" disclosure documents the full grammar with one short + annotated example per operator. + - **Objective modals.** The four objective `?` icons (MSI / TMB / + Survival / Unsupervised) now use the same rich ParamHelp modal + infrastructure as the parameters — verbatim SHORT + DETAILED + text per the prompt, shared intro about "an objective is the + rule that scores every program" and shared footer about + Associate / Effect / Fit being expressible in the DSL itself. + Old `OBJECTIVE_TIPS` constant removed; the "brokNA spell-checker" + typo is gone. + - **A9.** UI fitness label for TMB switched from `"association + (|spearman|)"` to `"negative association (signed −spearman with + TMB)"`; LiveView and held-out copy reworded to match. Engine + fitness already used signed −spearman. + - **Airgap stays absolute.** Only `stage`, `age`, `msi`, `tmb` + flow through engine_v2 as named values; gene IDs remain opaque + everywhere (program_repr, populations, API payloads). Airgap + tests parametrised over both engines × both prefilter modes + stay green. +- **Chunk 6 follow-up (done — scoring target bound to the objective):** + A bug from chunk 6: under the TMB objective an engine_v2 run could + produce `Associate(Reduce(...), msi, spearman)` — the program freely + picked its own target. Fixed by removing the target as a selectable + operand: + - `engine_v2.synthesize` now threads `objective_target` through + `_grow_scalar`, `_grow_vector`, `random_program`, + `ramped_population`, and `mutate`. Every `Associate` / `Effect` / + `FitApply` is constructed with `target = objective.target`. + - Point mutation no longer carries a `fit_target` spot and the + `scalar_kind` spot no longer flips the target — only the + correlation kind (pearson / spearman) is mutable. Crossover never + touched targets (subtree swaps preserve them). + - `engine_v2.fitness._check_target_binding(program, expected)` walks + every Associate / Effect / FitApply node and asserts + `n.target == expected`. `fitness_fn` calls it first and floors any + mismatch to `WORST_FITNESS`, so a stray target — even from a + replayed tree — can never win. + - `program_repr` still shows the bound target (e.g. + `Associate(score, tmb, spearman)` under the TMB objective) so the + graph and paste-to-draw renderers don't change. + - The shared footer in all four objective `?` modals now reads + verbatim: *"What the program chooses is how to build the score and + how to compare it — a raw association (Associate) or a + confounder-adjusted one (Effect), plus the correlation kind. What + stays outside the DSL is the compass: the target it's scored + against, in which direction, judged honestly on held-out data. + The program can't pick the target — that would let the answer + into the language."* + - New tests: `test_synthesis_binds_scoring_target_to_objective` + (init + 600 mutations × both objectives — every node binds to the + active target) and `test_fitness_floors_mismatched_target_to_worst` + (hand-crafted bug-shape program is floored to `WORST_FITNESS`). + 80 / 80 pytest pass; the API airgap and gene-symbol-leak tests are + untouched. +- **Chunk 6 follow-up (done — Live view polish, presentation-only):** + - **Fitness curve chart.** Recharts' default `` is gone; in + its place a custom `` HTML row sits below the chart + with SVG-line swatches that mirror the actual strokes (solid + accent for best, dashed muted for median). The X-axis title + ("generation") is pulled OUT of the SVG and rendered as a centred + caption between the chart and the legend so the three rows + (ticks → axis title → legend) never overlap. Y-axis label uses a + `` that rotates `` at + `viewBox.y + viewBox.height/2` with `text-anchor="middle"` — the + title now sits beside the *middle* of the axis, not the top. + Lines: best solid `#3A6B7E` strokeWidth 2.5, median dashed (4 4) + `#6E7F8C` strokeWidth 1.5. + - **InfoTip copy.** `TIPS.fitnessCurve` and `TIPS.bestVsMedian` + rewritten verbatim — the y-axis tip explains negative-association + strength (with the MSI/AUROC fallback parenthetical); the + best-vs-median tip explains convergence vs. persistent gap. + - **Generation navigator (PopulationTiles).** Replaced the + click-only `‹ Gen N ›` stepper with: `|‹` first / `‹` prev / a + draggable range slider / `›` next / `›|` latest / a numeric input + with a "go" button. Fetch decoupled from display via two states: + `draftGen` updates on every input event for live number, but the + `/runs/{id}/population/{gen}` fetch is keyed on `committedGen`, + which only updates on `pointerup`/`pointercancel` or a 150 ms + throttle while dragging. Keyboard: native left/right arrows + commit immediately; custom Home/End jump to first/latest. ARIA: + `aria-label="Generation"`, `aria-valuemin/max/now`, + `aria-valuetext`, and a polite live region for the "Generation N + / total" label. + - **DSL vocabulary tiles cover the full grammar.** The Lab's "DSL + vocabulary" panel now lists all eight engine_v2 verbs (Select, + Reduce, Combine, Split, Associate, Effect, Fit/Apply, Search) so + the panel matches what programs in the population can actually + contain — earlier it only showed the three Vector-shape verbs and + silently omitted Scalar/Model-shape and gated ops. Presentation + only; engine grammar unchanged. +- **Chunk 6 follow-up (done — TMB-rank diagnostic, MMR biology panel, + MLH1 success reframe, real Unsupervised objective, live-view polish, + gene-based unsupervised synthesis):** + - **TMB-rank diagnostic.** New `validate/tmb_rank.py` and a small + `GET /diagnostic/tmb-rank` API endpoint. Mirrors `api._prepare_lab_data` + target="tmb" exactly (Load('processed') filtered to tmb.notna() & + expression-complete rows) and computes signed Spearman per gene + vs TMB on the NAMED matrix (vectorised rank-Pearson, sub-second). + Reports each MMR / IMMUNE gene's rank/N + the top-10 + most-negatively-correlated genes. On the live TCGA cohort: MLH1 + sits at rank 270 / 20057 (top 1.3%); PMS2 at 2214 (11%); MSH2/MSH6 + not in expression (mutational, not silenced). The diagnostic lives + in `validate/` (allowed gene names) — the structural engine + airgap test still scans only `engine/`. 3 new tests in + `tests/test_tmb_rank.py`. + - **MMR biology panel.** `web/app/MMRBiologyPanel.tsx` mirrors + `ParameterFlow.tsx`'s collapsible exactly (button + chevron + + aria-expanded + useId-panel-id + closed-by-default). Body inlines + `mmr_reference.svg` verbatim (cause→effect: MMR → MSS / MSI-H → immune + response, plus the gene-expression visibility readout). Mounted + between the Objective card and Parameters card in `Lab.tsx`. + - **MLH1 success callout (TMB).** `/diagnostic/tmb-rank` is now + fetched at the Evaluator level (lifted out of the inner + `TMBRankPanel`) so the same response also drives an accent SUCCESS + callout above the overlap table — *"Blind, MLH1 ranks 270 / 20057 + (top 1.3%) on the TMB objective — the causal gene surfaced near + the top without ever seeing gene names."* `OverlapSummary` gains + an optional muted secondary line so the "0 of N winning genes" read + no longer reads as flat failure. + - **Real Unsupervised objective (engine_v2).** New `UNSUP_OBJECTIVE` + with `target="none"`, `metric="structure"`. Fitness = silhouette of + a 2-means split on standardised 1-D score (range [-1, 1], worst + floored at -1.0). Vector-only programs: `V2Objective.synthesis_overrides()` + returns `{rates: {split:.1, fitapply:0, effect:0, search:0}, + scalar_share:0}` for unsup, plumbed via `gp.py` into + `ramped_population` / `mutate`. `_build_ctxs` strips all labels + when target='none' (asserted) so the engine literally cannot see + msi/tmb during search. New `unsup_random_null` in `permutation.py` + — N random Vector-only programs scored on held-out (no target to + shuffle); `permutation_summary.null_kind` distinguishes the two + nulls in the result. + - **Post-hoc alignment.** Worker thread for unsup target runs + `_compute_unsup_posthoc` AFTER the GP finishes: orientation-agnostic + AUROC of winner scores vs MSI (held-out subset, gated n≥10 per class) + and `|spearman|` vs TMB. Lives in `api/_worker` (allowed labels) — + never in engine_v2. New `result.posthoc = {msi_auroc, tmb_abs_spearman, + n_holdout, n_msi_held, n_tmb_held}` rendered in `` + inside the ResultPanel. + - **Frontend — drop Survival, add Unsupervised end-to-end.** `Target` + widened to `"msi" | "tmb" | "none"`; `OBJECTIVE_PRESETS.none`, + `FITNESS_LABEL_BY_TARGET.none`. Survival entry removed from + `ObjectiveBuilder` + `paramHelpContent.tsx`. For unsup runs the + Evaluator hides the reference-set toggle / overlap / matched + column (no overlap concept) and falls back to the post-hoc block. + The `RunResult` interface in `web/lib/api.ts` gains `posthoc?: + Posthoc`. + - **Live-view + objective-modal clarity (presentation-only).** + Shortened y-axis titles (`tmb: "neg. association with TMB"`, + `none: "cluster separation (0–1)"`). Replaced the single TMB-centric + `TIPS.fitnessCurve` with `FITNESS_TIP_BY_TARGET: Record` and plumbed `target` into `LiveView` so the InfoTip matches + the y-axis title's objective-awareness. Added `TIPS.nodes` and + `TIPS.genes` and passed them to the NODES + GENES `Metric` cards + in the ResultPanel. Rewrote `ObjectiveIntro` so it no longer + asserts "a target column" unconditionally; rewrote `obj_msi` / + `obj_tmb` / `obj_unsupervised` modal bodies verbatim per prompt. + `obj_unsupervised` no longer renders the target-centric + `ObjectiveFooter` — new `UnsupObjectiveFooter` ("no target, can't + pick one") in its place. + - **Silhouette guard hardened (engine_v2/fitness.py + `_score_silhouette`).** Winsorize the score to its 2.5–97.5 + percentile range, then recompute std on the winsorized vector; + if `std_w < 1e-9` → worst floor. This catches the + `protected_div`-by-self failure mode (huge raw std + a small + handful of outliers → a 98:2 "perfect" split with silhouette ~1). + Min-cluster floor also raised from 10% to 30% of n. + `test_unsupervised_silhouette_kills_self_divide_outlier_split` + locks the fix. + - **Gene-based unsupervised synthesis (engine_v2/synthesize.py).** + Earlier unsup winners could be no-Select trees that did the + splitting work via `Split(..., predicate="stage_late")` — GENES=0, + silhouette 1.0 trivially, no actual gene discovery. Fixed by: + `_grow_matrix` now takes `objective_target`; under unsup every + Matrix leaf is `Select(MatrixTerminal(), sampled FeatureSet)` + (never bare `MatrixTerminal`), `Search` is gated off, and the + `_grow_vector` depth-floor becomes `Reduce(Select(M, [...]), agg)`. + `_grow_vector`'s Split branch forces `predicate="score"` under + unsup; point mutation's `predicate` spot is suppressed entirely + for unsup so a `"score"` can't be flipped to `"stage_late"` mid-run. + Belt-and-braces fitness floor: `fitness_fn` and `evaluate_holdout` + return `WORST_FITNESS` when `objective.target == "none"` and + `program.feature_ids() == []`. Two new tests + (`test_unsupervised_programs_always_use_gene_select` over 120 + programs × 20 mutations, and `test_fitness_floors_no_select_program_under_unsup` + on the literal bug-shape program). + - **Verification.** pytest 94/94, tsc clean. Live unsupervised + TCGA smoke (10 gens × 60 pop, prefilter 2000) across four seeds + shows every winner is gene-based (1–5 genes), Split predicate is + `score` (no stage_late), held-out silhouette ≈ 0.55–0.60 (not the + trivial 1.0), and the discovered split aligns blind with MSI at + AUROC up to **0.774** — the engine rediscovered the MSI subtype + without ever seeing the label. +- **Chunk 6 follow-up (done — generalisation-aware unsup silhouette + + plain-reading verdict):** + - **OOS silhouette in `cv_score` AND `evaluate_holdout`.** Earlier + unsup runs scored silhouette IN-SAMPLE inside CV (the KMeans was + fit on the same fold-test points it then evaluated) — so programs + that overfit looked great during selection and collapsed on the + final held-out (often to -1.0 worst floor). New + `V2Objective._score_oos_silhouette(train_scores, test_scores)` + fits the winsorize bounds, mean/std, and KMeans centres on TRAIN + scores only; test scores are clipped + standardised with the + train statistics and assigned to the train centres via + `KMeans.predict`. Guards live on the TEST side (≥30% per cluster, + near-constant after clipping → worst). `cv_score` per fold now + uses `(tr, te)` not just `te`. `evaluate_holdout` gained an + optional `ctx_train` kwarg; under unsup it executes the program + on TRAIN and TEST then calls the OOS scorer. MSI / TMB paths + ignore the new kwarg (those metrics are already on the held-out + vector by construction). + - **OOS null too.** `unsup_random_null` gained `ctx_train` and + forwards it into `evaluate_holdout` — otherwise the random + Vector-only nulls would be scored in-sample while the winner is + scored OOS, deflating the p-value. The pipeline now plumbs + `ctx_train` into both the winner's `evaluate_holdout` and the + null-distribution loop. + - **Plain-reading verdict at the top of the Result panel.** New + `` rendered first inside `ResultPanel`. Per-target + thresholds (MSI 0.75, TMB 0.30, unsup 0.30) plus `p < 0.05` gate + a green "Real result: it holds up on unseen patients (X) and + beats chance (p Y)" callout vs an amber "This run found nothing + reliable: held-out X (fell apart on unseen patients) · p Y (a + random program beats it ~N% of the time). Don't read the genes + as a discovery." For unsup the green callout appends one of three + tails based on `posthoc.msi_auroc` (≥0.75 → "rediscovered the + subtype"; <0.75 → "doesn't line up with MSI"; null → "too few + held-out labels"). Constants `HOLDOUT_THRESHOLD`, + `P_VALUE_SIGNIFICANT`, `POSTHOC_MSI_ALIGN` at the top of + `Lab.tsx` are tunable. + - **Honest unsup numbers.** With OOS silhouette as the selection + metric the GP now selects for *consistent* two-cluster structure + rather than splits that happen to align with MSI via in-sample + overfitting. seed=13 used to report in-sample silhouette 0.6 + + MSI-AUROC 0.774 (alignment was partly an in-sample artefact); + now it reports OOS silhouette 0.900, p 0.024, MSI-AUROC 0.546 — + a real generalising cluster split that doesn't happen to be MSI. + The verdict surfaces that mismatch in plain words. + - **Four new tests** in `tests/test_engine_v2.py`: + `test_oos_silhouette_rewards_consistent_split` (train+test both + bimodal → ≥0.70), `test_oos_silhouette_kills_non_generalising_program` + (train bimodal + test noise → drops >0.3 below in-sample), + `test_oos_silhouette_train_only_constant_floored`, + `test_oos_silhouette_too_small_floored`. pytest 98/98 pass. +- **Chunk 6 follow-up (done — "What does this mean?" verdict + breakdown, presentation-only):** + - The verdict callout now has a collapsed-by-default disclosure + underneath it (mirrors `ParameterFlow.tsx`'s collapsible — button + + chevron + `aria-expanded` + `aria-controls` + `useId`-keyed + panel + closed-on-first-render). + - Expanded content branches on `objective_spec.target`: + `` and `` use a one- + question shape ("Is this result real and useful?") with two + bullets — held-out separation/association and beats-chance — + plus a `→ Real and useful.` / `→ Not reliable…` capstone tied to + the same generalises/significant booleans the headline uses. + `` uses the two-question shape from the + prompt verbatim ("Is the split real?" / "Is that split the MSI + subtype?") with the hair-colour analogy at the bottom. All + numbers route through `fmtFit` so non-finite renders as em-dash. +- **Chunk 6 follow-up (done — iterative unsupervised discovery, + "peel off axes"):** + - **Engine.** `engine_v2/pipeline.py` gained + `_residualise_matrix(M, residualize_scores)` — vectorised OLS via + `np.linalg.lstsq` over `[intercept, *priors]`. Rows missing a + prior score are dropped BEFORE the train/test split so train and + test see the same residualised feature space. Both + `run_v2_pipeline` and `run_v2_pipeline_streaming` gained + `residualize_scores: pd.DataFrame | None = None` (no-op for + MSI/TMB; only the unsup API worker sets it). Both functions now + emit `winning.full_scores` + `winning.full_sample_ids` (full- + cohort per-patient scores) alongside `holdout_scores` — these + feed the chain's next residualisation step. + - **API.** `RunRequest.residualize_against: list[str] | None`. + `_assemble_residualize_df` validates each prior (must exist + be + done + `target=="none"` + have `full_scores`); turns + `_json_finite`'s `None` back into `NaN` so the pipeline's NaN + filter drops those patients during residualisation. `post_runs` + validates the chain BEFORE spawning the worker — supervised + target + priors → HTTP 400; unknown / wrong-target / no-scores + prior → HTTP 400 with the specific reason. `_worker` gained + `residualize_against`; only assembles + forwards the DataFrame + for unsup runs. Chain is in-memory only — valid within a single + server session. + - **Frontend.** Lab.tsx grew an `axes: {run_id, result}[]` state. + `start()` refactored into `launchRun(opts?)` so a fresh Run + (`launchRun()`, clears `axes`) and `findNextAxis()` + (`launchRun({residualize_against: axes.map(a=>a.run_id)})`, + preserves the chain) share the same SSE wiring. The SSE `done` + handler appends unsup results to `axes`. New `` + section under the Evaluator renders an ordered stack of compact + ``s (accent border when both verdict gates pass; + holdout / p / revealed opaque IDs / post-hoc MSI AUROC + TMB + |spearman| in one line) plus a "Find next axis →" button. The + `TIPS.discoveredAxes` tooltip flags the linear-residualisation + heuristic and the in-memory-chain caveat. + - **Tests.** Extended `tests/test_api_airgap.py`: + `test_unsupervised_run_emits_posthoc_alignment` now asserts + `winning.full_scores` + `winning.full_sample_ids` are populated; + new `test_residualisation_chain_runs_and_stays_airgap_clean` + posts axis 1 then axis 2 with `residualize_against=[axis1]` and + confirms the second payload is opaque-only and carries its own + `full_scores`; new `test_residualize_against_unknown_id_returns_400` + + `test_residualize_against_rejected_on_supervised_target`. 101 / + 101 pytest pass. + - **Verified live on TCGA** (seed=13, 6 gens × 50 pop, prefilter + 2000): + - Axis 1 (5 genes, no overlap with MSI): holdout 0.597, p 0.065, + MSI-AUROC 0.552, TMB-|spearman| 0.009. + - Axis 2 (residualised against Axis 1; 5 different genes): + holdout 0.653, p 0.097, **MSI-AUROC 0.706**, TMB-|spearman| + 0.248. Zero shared genes between the two axes — the + residualisation freed the search to land on an orthogonal + direction that's more MSI-aligned than the first. +- **Chunk 7 (done — second dataset: HNSC + HPV detection):** + - **Dataset pipeline.** New `data_pipeline/{download_hnsc,build_hnsc}.py` + mirror the colorectal scripts for TCGA HNSC PanCancer Atlas + (`hnsc_tcga_pan_can_atlas_2018`, LFS-resolved media URL). The + build script derives `hpv_status` ∈ {HPV+, HPV−} from whichever + column carries it: first `data_clinical_sample.txt::HPV_STATUS*`, + then `data_clinical_patient.txt::HPV_STATUS`, then the `SUBTYPE` + suffix fallback (e.g. `HNSC_HPV+` / `HNSC_HPV-`). On the real + cohort the resolved source is `patient.SUBTYPE`. Writes + `data/processed_hnsc/{clinical,expression}.parquet` (gitignored). + HNSC schema constants live in `data_pipeline/schema.py` next to + the colorectal ones; final usable cohort is 487 samples (72 HPV+ / + 415 HPV−). + - **Loader.** `dsl.Load` already handled directory paths via its + `else` branch; the only change is that `Cohort.labels` now picks + up `hpv_status` if present. `Load(schema.HNSC_PROCESSED_DIR)` + returns the HNSC cohort with `hpv_status` in `cohort.labels`. + - **HPV objective (engine_v2).** `V2Objective.target` widened to + `Literal["msi","tmb","none","hpv"]`. New `HPV_OBJECTIVE` + (binary=True, worst=0.5) routes through the SAME + orientation-agnostic AUROC machinery as MSI; `score_scalar` + mirrors MSI's `abs(value)` orientation. `objective_from_spec` + dispatches `hpv + auroc/auroc_omni`. No new operators; no + other engine surface changes. + - **API per-(dataset, target).** `_prepare_lab_data(target, dataset)` + keyed by `f"{dataset}:{target}"`. New `(hnsc, hpv)` branch loads + HNSC, restricts to called HPV samples + complete expression, + anonymises, sets `y = (hpv_status == "HPV+").astype(int)`. New + `(hnsc, none)` unsup branch keeps every patient with complete + expression and carries the HPV label aside via `extra_labels` so + `_compute_unsup_posthoc` can compute orientation-agnostic + `hpv_auroc` (mirrors the `msi`/`tmb` branches; new key + `Posthoc.hpv_auroc?`). `RunRequest.dataset` + `Run.dataset` + + server-side `DATASET_TARGETS = {coadread: {msi,tmb,none}, + hnsc: {hpv,none}}` validation rejects mismatched combos with a + 400 at the API boundary before any data prep. + - **Per-dataset reference sets.** Replaced flat `REFERENCE_SETS` + with `REFERENCE_SETS_BY_DATASET`. Colorectal keeps MMR + immune; + HNSC ships `p16 = ["CDKN2A"]` and a 20-gene standard + cell-cycle / E2F-target core (MCM2–7 / PCNA / CDK1 / CCNE1 / + CCNB1 / CDC6 / CDC20 / MKI67 / TOP2A / RRM2 / TYMS / FOXM1 / + E2F1 / BUB1 / AURKB). `EvaluateRequest.dataset` is required and + cross-dataset reference-set requests return 400. + - **Frontend dataset registry.** `Target` widened to include `hpv`; + new `DatasetId = "coadread" | "hnsc"`. `DATASET_REGISTRY` in + `Lab.tsx` drives objective cards, biology panel, and reference-set + keys per cancer. A `` segmented control mounts + above the Objective row. New `` mirrors + `` exactly and inlines `hpv_reference.svg` + verbatim (RB / E2F / p16 / p53 / E6+E7 explainer + readout + table + airgap-side disclosure). Switching cancers resets the + chain + result state and snaps the objective to the dataset's + default. + - **HPV verdict + "?" coverage.** `FITNESS_LABEL_BY_TARGET.hpv = + "separation (AUROC)"`; `FITNESS_TIP_BY_TARGET.hpv` reuses the + MSI-style AUROC explanation worded for HPV+/−; + `HOLDOUT_THRESHOLD.hpv = 0.75`; `OBJECTIVE_PRESETS.hpv = + {target:"hpv", metric:"auroc"}`. New `` adds + a supervised one-question shape ("Is this result real and + useful?") — no two-question / hair-colour shape for HPV. Two new + paramHelp modals: `obj_hpv` (verbatim per prompt: what it + optimises, how it's scored, represented as, honest "detection / + recovery of a known viral signature — not new causation" note). + `` + `` + + `` all branch on which named label the cohort + carries (HPV vs MSI/TMB) so HNSC unsup runs surface a real + alignment number instead of "(too few held-out labels)". + - **Per-dataset Reveal.** `` reads its reference-set + toggle keys from `DATASET_REGISTRY[dataset].refSetKeys`. HNSC + shows **p16 | cell_cycle**; colorectal stays on MMR | immune. + `` + `` gated to + `dataset === "coadread" && target === "tmb"`. The "Unsupervised + run — no reference set" footnote reads dataset-agnostically. + - **Live verification (TCGA).** MSI seed 7 — held-out 0.959, p 0.048. + HPV seed 7 — held-out 0.869, p 0.048. HPV seed 11 — held-out + **0.970**, p **0.024**, winner overlaps `cell_cycle` via **MCM5**. + HPV seed 17 — held-out 0.957, MCM5 again. HPV seed 3 — E2F2 + RPA2 + (cell-cycle/E2F + DNA replication, biologically on-target but not + in the literature set). Colorectal + airgap tests untouched and + green. + - **Airgap tests extended.** `tests/test_api_airgap.py` fake_prep + handles `(target, dataset)`; new tests: + `test_hnsc_hpv_run_is_airgap_clean`, `test_hnsc_rejects_msi_target`, + `test_coadread_rejects_hpv_target`, + `test_evaluate_rejects_hnsc_reference_in_coadread`. 105 / 105 + pytest pass. +- **Chunk 7 follow-up (done — HPV-marker-rank diagnostic):** + Mirror of the TMB-rank diagnostic for HNSC. New `validate/hpv_rank.py` + ranks every gene by single-gene orientation-agnostic AUROC vs the + HPV+/HPV− label on the engine's TRAIN split (`make_split(seed=42, + test_size=0.3, stratify=True)` so the diagnostic never reads test + data — same discipline the engine uses). Vectorised + Mann-Whitney / rank-sum formula: + `auroc = (S_pos − n_pos·(n_pos+1)/2) / (n_pos·n_neg)` then + `max(AUROC, 1−AUROC)`; sub-second over ~20k genes. New + `GET /diagnostic/hpv-rank` endpoint mirrors `/diagnostic/tmb-rank` + (lazy import, `_HPV_RANK_CACHE`, `_json_finite()`-wrapped). Three + synthetic-cohort tests in `tests/test_hpv_rank.py`. Frontend: + `HPVRankDiagnostic` interface + `getHPVRankDiagnostic()` in + `web/lib/api.ts`; new `` in `Lab.tsx` reuses the + existing `` chrome and renders **p16 + cell-cycle + top-N + separators** with the prompt's verbatim caption, gated to + `dataset === "hnsc" && target === "hpv"`. `TIPS.hpvRank` documents + the heuristic. Live TCGA cohort: N(TRAIN) = 340 (HPV+ 50 / HPV− 290), + 20,218 genes ranked. **CDKN2A AUROC 0.877, rank 99 / 20,218 (top + 0.5%)** — meaningfully recoverable but 98 genes outrank it. Best + cell-cycle ranks: MCM5 (22), MCM2 (28), MCM6 (38), MCM3 (72), + PCNA (78), E2F1 (99, tied with CDKN2A). Top-10 led by TCAM1P / + C19orf57 / ARHGEF33 / STAG3 / SMC1B (meiosis-cohesin) + RPA2 + (DNA replication — what the engine actually picked at seed 3). 108 / + 108 pytest pass; airgap unaffected (engine_v2 + dsl + engine + untouched; structural scan still only walks `engine/`). +- **Chunk 7 follow-up (done — UX polish: plain "?" copy, no all-caps, + program-graph clipping):** Presentation-only sweep across the Lab. + - **Objective-aware tooltips.** Added four per-target / per-dataset + records next to `FITNESS_TIP_BY_TARGET`: + `HELD_OUT_TIP_BY_TARGET`, `PERMUTATION_P_TIP_BY_TARGET`, + `REFERENCE_SET_TIP_BY_DATASET`, `POSTHOC_TIP_BY_DATASET`, + `DISCOVERED_AXES_TIP_BY_DATASET`. Held-out / permutation p / + posthoc / discovered-axes / reference-set `?` all switch on the + active objective or dataset so HPV runs no longer see MSI/TMB + wording. `` and `` receive + `dataset` (or infer it from `posthoc.hpv_auroc` presence) and look + up the right copy. + - **Plain English everywhere.** Rewrote the verbose / typo'd + legacy `TIPS.heldOut`, `TIPS.permutationP`, `TIPS.posthoc`, + `TIPS.discoveredAxes`, `TIPS.referenceSet` to crisp sentences with + one-line glosses for jargon (AUROC, held-out, permutation). Fixed + the `seed` PARAM_TIPS typo ("xact" → "the exact"). The unsupervised + fitness-curve tip no longer hardcodes "(MSI? TMB?)" — it now says + "the named label this cohort carries". + - **ObjectiveIntro / UnsupObjectiveFooter dataset-aware.** + `ObjectiveIntro` now lists MSI / TMB / HPV (was MSI / TMB only). + `UnsupObjectiveFooter` mentions both MSI (colorectal) and HPV + (head & neck) as legitimate downstream alignments instead of MSI + only. + - **No all-caps anywhere.** Stripped Tailwind's `uppercase` utility + from all 13 eyebrow / table-header sites in `Lab.tsx` + + `PasteToDraw.tsx`. Removed `textTransform: "uppercase"` from the + Tier label in `ProgramGraph.tsx`. Upcased the underlying literal + strings that were relying on the CSS (`Winning program`, + `Revealed genes`, `Opaque ID` / `Symbol` / `Matched` table + headers, rank-table `Symbol` / `Correlation` / `Rank / N` / + `AUROC`). `grep -rn "uppercase\|textTransform" web/app` returns + zero hits. + - **Program-graph clipping fix.** `ProgramGraph.tsx` gained a + remount `key={fitKey}` derived from `width × height × node-count + × edge-count` so ReactFlow's one-shot `fitView` re-runs every + time the laid-out program changes (winner ↔ candidate click, + paste, dataset swap). Lowered `minZoom` from 0.4 to 0.2 so + `fitView` can scale a full two-tier graph down to fit the card + width. Confirmed no `overflow: hidden` on the wrapping + ``. `` now dispatches by + target — `MSI-H probability` / `HPV+ probability` / + `TMB association` / `cluster score` instead of MSI-only. +- **Chunk 7 follow-up (done — detection vs recovery + AUROC naming + + ban whole-matrix Reduce + per-gene rank in Reveal):** + - **Engine.** Motivating live run produced + `Combine(Reduce(M, mean), Reduce(Select(M, …), mean), add)` — a + bare global-mean detector with a 7-gene additive tail; held-out + AUROC 0.953 with `0/7` cell-cycle overlap. The dominant term + detected HPV from bulk expression, not from gene choice. Fix: + `engine_v2/synthesize.py` makes the Select-wrapping rule + **unconditional** for every objective (was unsup-only). Every + Matrix leaf is now `Select(MatrixTerminal, FeatureSet)`; the + Vector depth-floor returns `Reduce(Select(M, …), agg)`; the + bare-MatrixTerminal fall-through is gone. Belt-and-braces in + `engine_v2/fitness.py`: new `_has_bare_matrix_reduce(program)` + walks the tree and any Reduce on a bare MatrixTerminal floors to + `WORST_FITNESS` for every objective (was unsup-only). The + `no Select → WORST_FITNESS` floor is also no longer gated to + unsup. Two test fixups: the closed-Matrix leaf at depth 0 is now + `Select(MatrixTerminal, …)`; the depth-budget contract is loosened + by 1 because the mandatory Select adds a level. Live re-run on + real HNSC (seed 7): winner is now `Reduce(Select(M,[g17705]),max)` + — a single-gene detector, AUROC 0.964, no bare `Reduce(M,…)`. + - **Detection vs recovery (presentation).** `HPVBreakdownContent` + + `MSIBreakdownContent` success copy changed from "the engine + recovered the HPV viral signature" / "Real and useful." to + "Real and useful — it separates HPV+ from HPV− (or MSI-H from + MSS) on patients it never saw." Both breakdowns gained a muted + secondary line — *"This is a detection result. Whether the + engine found the known marker genes (CDKN2A/p16, the cell-cycle + program / MMR / immune) is a separate question — see Reveal & + evaluate below."* The top-level callout already used detection- + only wording. Unsupervised "rediscovered the viral signature + blind" stays — that's a post-hoc *alignment* claim, gated on + `posthoc.hpv_auroc ≥ 0.75`. + - **Name AUROC where the metric IS AUROC.** New + `HELD_OUT_LABEL_BY_TARGET` — Held-out card label is `Held-out + AUROC` for MSI / HPV, `Held-out (|spearman|)` for TMB, + `Held-out (silhouette)` for unsup. `HELD_OUT_TIP_BY_TARGET` + rewritten to name AUROC explicitly with a one-line gloss for + MSI/HPV and to call out "Not AUROC: TMB is a continuous label / + there's no label here" for TMB/unsup. Held-out bullet inside + each breakdown switched `Held-out separation` → `Held-out AUROC` + with the same gloss. + - **Per-revealed-gene rank in Reveal & evaluate.** `/evaluate` + request gains optional `target`; response rows gain optional + `rank` / `total` / `single_gene_metric` / `metric_kind` + (`"auroc"` for HNSC/HPV, `"spearman"` for coadread/TMB). New + `_gene_rank_lookup(dataset, target)` precomputes a + `{symbol → rank/metric}` map on first call by reusing + `validate/hpv_rank.py`'s `_auroc_per_column` (on the engine's + TRAIN slice) and `validate/tmb_rank.py`'s `_spearman_per_column`; + cached for the process lifetime. Frontend: `EvaluateRow` gained + the optional fields; `postEvaluate` passes `target`; the + Revealed-genes table now shows a "Single-gene rank" column with + `{rank} / {total}` plus `AUROC X.XXX` (HPV) or `ρ X.XXX` (TMB). + Column appears only when at least one row carries a rank — MSI + and unsupervised omit gracefully. New `TIPS.singleGeneRank` + explains the column: "rank near 1 = real alternate marker; high + rank = only helps in combination." + - **Verified live.** HPV seed 11 on real HNSC (10 gens × 80 pop): + winner `Reduce(Select(M,[g06264,g08575,g10783,g14446,g18989, + g19553,g05351,g10525]),mean)`, held-out AUROC 0.937, p 0.024. + Cell-cycle overlap **0/8**, BUT every revealed gene is a top-2000 + single-gene HPV separator on its own: FANCI (rank 996, AUROC + 0.779), INPP5B (332, 0.830), MIR924HG (237, 0.841), RAD1 (359, + 0.825), UBD (1808, 0.746), WHSC1 (402, 0.821), DYRK1A (246, + 0.840), MCCC1 (1334, 0.763). The story is now legible: real + alternate markers (DNA-damage-response / chromatin players), not + the canonical MCM / PCNA shortcut. 108 / 108 pytest pass; tsc + clean; airgap tests untouched. +- **Chunk 7 follow-up (done — coherence prior + supervised peel-off + + full-ranking Result + Unsup card retired):** Five coordinated + landings that rebuild the Lab's reveal story around the full single- + gene ranking. + - **Coherence prior (engine).** New + `engine_v2.fitness._coherence_score(program, ctx)` returns the + mean absolute pairwise correlation among the program's Select'd + opaque columns (range [0, 1]; names no gene or pathway). + `fitness_fn` gained `coherence_weight: float = 0.0`; when > 0, + fitness becomes `base − λ·n_nodes + w · coherence`. Threaded + through `gp.py` + both `pipeline.py` entry points with default 0 + so existing runs stay byte-for-byte unchanged. API: new + `RunRequest.coherence: bool = False`; `_worker` translates to + `COHERENCE_DEFAULT_WEIGHT = 0.10` (modest enough that separation + still dominates). Frontend: a Parameters checkbox "Prefer + coordinated gene modules" — default OFF, posted in the run body. + `TIPS.coherence` carries the prompt's verbatim copy. + - **Peel-off for supervised objectives.** Both pipeline entry + points now persist `winning.full_scores` + `full_sample_ids` for + every objective (the `is_unsup` gate is gone in both spots). + `api.app.post_runs` no longer requires `target=="none"` for + `residualize_against`; `_assemble_residualize_df` validates that + every prior shares both `(dataset, target)` with the new run. + Frontend `` mounts after ANY run (not just unsup) + and the SSE `done` handler appends to the chain for every + objective. Live verified on HNSC HPV seed 11: Axis 1 → AUROC + 0.941 / 3 genes; Axis 2 (residualised) → AUROC 0.875 / 2 + different genes; zero overlap. Updated airgap test: + `test_residualize_against_works_on_supervised_targets` posts MSI + Axis 1 → Axis 2 and asserts payload stays opaque + carries + full_scores. + - **Full-ranking endpoint.** New `GET /diagnostic/full-rank?dataset= + &target=` returns `{n_samples, n_pos, n_neg, n_genes, metric_kind, + ranks: [{opaque_id, score, rank}, …], reference_marks: [{opaque_id, + symbol, set_name, rank, score}, …]}`. The `ranks` list is + opaque-only (~20k rows). `reference_marks` is the small reference + gene set (CDKN2A + 20 cell-cycle for HNSC; MMR + immune for CRC), + looked up via the sealed map on the API side and revealed up-front + so the rank track can label them — the whole map never crosses + the wire. Computed on the same TRAIN slice the engine sees + (`make_split(seed=42, test_size=0.3, stratify=binary)`), using + the rank-sum AUROC formula for binary targets and signed Spearman + for TMB. Unsupervised → 404 (no label to rank against). Cached + per (dataset, target) for the process lifetime. + - **New `` UI.** Replaces the prior + Evaluator + TMBRankPanel + HPVRankPanel + MLH1SuccessCallout for + supervised targets. Renders, top-to-bottom: + - Rank chart: a labelled lollipop chart on a log-rank axis + (gridlines at 1 / 10 / 100 / 1k / 10k / N), with the winner + gene(s) as accent-teal dots and each reference gene as a + colour-coded dot. Labels stagger across 4 rows to avoid + collisions near the top. + - Highlighted-genes table with three-way colour-coded "Source" + column (winner / p16 / cell_cycle for HNSC; winner / immune / + MMR for CRC) — p16 reads visibly distinct from cell_cycle + via a separate deep-gold palette. Columns: Source · Symbol · + Opaque ID · Rank/N · Percentile · metric. Percentile shows + "top 0.5%" for high ranks and flips to "bottom 21%" past the + midpoint (2 sig figs). + - Collapsed `` ("Raw anonymous ranking — + proof the ranking is computed blind on opaque IDs") wraps the + browsable opaque-ID list + search box (top-200 with substring + filter). Caption restates the airgap purpose. + - GP-vs-diagnostic note (verbatim per prompt) closes the panel. + - **Unsup card hidden, backend kept.** `DATASET_REGISTRY` drops + the `none` objective entry for both datasets — UI now shows + coadread → {MSI, Mutation burden} and hnsc → {HPV detection}. + `UNSUP_OBJECTIVE`, the unsup pipeline branches, the + `target === "none"` frontend code paths (verdict / breakdown / + post-hoc / Evaluator fallback), and `tests/test_engine_v2.py` + stay intact — just unreachable from the UI. + - **108 / 108 pytest pass; tsc clean; airgap tests still green.** +- **Chunk 7 follow-up (done — Ranking UI polish):** Presentation-only + pass on ``. New three-way `RANK_COLORS` palette + (winner accent teal / p16 deep gold / cell_cycle muted amber, plus + immune/MMR aliases for CRC) drives every coloured surface — chart + dots, table backgrounds, "Source" badges. Highlighted-genes table + gains a "Percentile" column (`top X%` / `bottom X%` with 2 sig figs) + so CDKN2A reads "99 / 20,218 · top 0.5%" and CCNE1 reads + "15,918 / 20,218 · bottom 21%". `` rewritten with a log + axis (gridlines at 1 / 10 / 100 / 1k / 10k / N), a 4-row label + stagger that avoids 60px-radius collisions, and an inline + three-colour legend in the chart header. Raw opaque-ID browse + table moved into a collapsible disclosure ("Raw anonymous ranking + — proof the ranking is computed blind on opaque IDs"), closed by + default. Page reads verdict → metric cards → log chart → highlighted + table → collapsed raw → GP-vs-diagnostic note. tsc clean; no + engine/API/airgap changes. +- **Chunk 7 follow-up (done — peel-off leakage fix: train-only + residualisation + FitApply train-fit + leakage guard):** A peel-off + Axis 2 HPV run produced + `FitApply(Reduce(Select(M,[g16970]),median),hpv)` (SPACA1) at + held-out AUROC 0.963 — impossible for a single-gene monotonic + program whose gene's honest single-gene AUROC is 0.510. Root + cause: residualisation was fit on the FULL cohort BEFORE + `make_split`, so the OLS projection saw the held-out rows and + smeared target signal into them. + - **`engine_v2/pipeline.py`** — `_residualise_matrix` replaced + with three helpers: `_align_priors` (drops rows missing a + prior — defining the cohort, not leakage), `_fit_residualise_beta` + (OLS of each gene on `[intercept, *priors]` fit on TRAIN rows + only), `_apply_residualise` (applies train-fit β to the full + M so train + test sit in the same residualised space without + test ever being seen by the fit). Both `run_v2_pipeline` and + `run_v2_pipeline_streaming` reordered: align → split → fit β + on `M.loc[split.train_ids]` → apply β to full M → prefilter / + GP / `evaluate_holdout`. `winning.full_scores` re-executes the + winner on the train-only-residualised M with empty labels (the + raw inner score for FitApply winners — monotonic with the + fitted prediction in the 1-D case, so the chain stays + consistent). + - **`engine_v2/nodes.py`** — `ExecContext` gained + `fit_ctx: ExecContext | None = None`. `FitApply.execute` now + runs the inner subtree on `fit_ctx` (train) to fit LR/OLS and + applies the FROZEN model to the test inputs (no fitting on + test labels). Binary `hpv` now routes through the same + logistic branch as `msi`. Identical behaviour in the single- + score monotonic case (LR remains monotonic) so existing AUROCs + don't move; the discipline is in place for any future multi- + input FitApply. + - **`engine_v2/fitness.py`** — `LEAKAGE_TOLERANCE = 0.02`. + `evaluate_holdout` wraps the test `ExecContext` with + `fit_ctx=ctx_train` when `ctx_train` is provided, then runs a + backstop check: if the program is `_single_gene_monotonic` + (one gene; no Combine/Split) AND its held-out AUROC exceeds + the gene's single-gene AUROC by > tolerance, floor to + `WORST_FITNESS`. Genuine multi-gene synergy is unaffected. + - **Tests** — `tests/test_engine_v2.py` gained + `test_peeloff_residualisation_uses_train_rows_only` + (synthetic cohort + target-correlated prior; asserts the + single-gene-monotonic held-out AUROC is within tolerance of + the gene's honest single-gene AUROC under the new pipeline) + and `test_leakage_guard_floors_contaminated_single_gene_winner` + (guard sanity-check on an honest program). 114 / 114 pytest + pass; airgap suite untouched. + - **Verified live (TCGA HNSC, HPV + coherence ON).** Across + seeds {3, 7, 11, 13} every Axis 2 is multi-gene, no + SPACA1-style runaway wins, and Axis 2 held-out ≤ Axis 1 + + tolerance: seed 11 → Axis 1 AUROC 0.941 / 3 genes → Axis 2 + AUROC 0.923 / 8 genes (zero gene overlap). The + residualise-before-split mechanism is closed. +- **Chunk 7 follow-up (done — coordinated modules: ranked group + view + ref-set highlights):** Coherence-on runs now expose a + ranked list of the coordinated gene modules the GP explored, + sorted by combined held-out AUROC — so a real co-expressed + program (e.g. cell_cycle) surfaces as a GROUP even when no + single gene is the top separator. + - **API.** New `GET /runs/{run_id}/modules` endpoint + (`api/app.py::_compute_module_ranking`). Reproduces the run's + EXACT train/test split via the persisted `full_sample_ids` / + `holdout_sample_ids` so "held-out" actually is held-out; + harvests distinct candidates' `gene_ids` sets (≥ 2 genes) + across every persisted generation (engine_v2 already stores a + flat `gene_ids` list per candidate); for each module computes + **combined held-out AUROC** = orientation-agnostic AUROC of + the per-patient mean across the module's genes vs the held- + out label (parameter-free — nothing to fit, nothing to leak; + `|spearman|` for TMB), **coherence** = mean abs pairwise + correlation on TRAIN (the same quantity the prior rewards), + and a per-gene single-gene AUROC + rank/N. Each module also + carries **`ref_sets: list[str]`** — the dataset's reference + sets (HNSC: p16 / cell_cycle; CRC: MMR / immune) whose + opaque IDs intersect that module, resolved ONCE via the + sealed map (bounded reveal of a small known set; same + pattern `/diagnostic/full-rank` uses for `reference_marks`). + Sorted by combined held-out desc; non-finite sinks to the + bottom. `Run` dataclass gained `coherence: bool`; `/runs/{id}` + now surfaces it (along with `dataset`). Endpoint returns 425 + while running, 404 for unknown, 400 for unsupervised runs + (no target to evaluate against). + - **Frontend.** New `` rendered after + ``, gated to `target !== "none" && coherence + && runId`. Sortable header (Combined AUROC / Coherence); + each collapsed row carries a small **badge per matched + reference set** and is **tinted** using the existing + three-way `RANK_COLORS` palette (winner accent teal / + single-marker sets like p16/immune deep gold / broader + proliferation sets like cell_cycle/MMR muted amber) — so a + cell_cycle-heavy module is scannable without expanding. + Expanding a row calls **`POST /reveal`** with just that + module's gene IDs (bounded — never the whole map) so every + per-gene row gets a symbol (no more "—" for module genes + outside the already-revealed subset, e.g. `g09647 → LIG1`). + Expanded view gains a "Source" column tinted per-symbol's + reference-set membership. + - **Airgap.** Module ranking carries opaque IDs + scores + + `ref_sets` set NAMES only — no gene symbols. Reference-set + membership comes from the same bounded sealed-map lookup + `/diagnostic/full-rank` already uses. Per-module symbols + revealed lazily per expanded module via `/reveal`. New + airgap tests in `tests/test_api_airgap.py`: + `test_modules_endpoint_returns_opaque_only_modules` (assert + `ref_sets` list present, opaque IDs only, sorted desc by + combined AUROC), plus rejection tests for unsupervised / + unknown / mid-run scenarios. 114 / 114 pytest, airgap suite + 25 / 25, `tsc --noEmit` clean. + - **Verified live (HNSC HPV, seed 11, coherence ON).** 303 + distinct modules; the rank-1 module (size 11, combined + held-out AUROC **0.978**) carries the cell_cycle badge and + beats the winner's 0.941. Expanding it resolves all 11 + symbols on the wire: MCM2, CENPQ, FAM111B, KIF2C, NDC80, + RAD1, USP1, REXO2, ING4, KIAA1407, MMP17 — a proliferation + / DNA-replication module the single-gene picture missed. + Exactly the "group view can out-score its individual genes" + demonstration the design called for. +- **Chunk 7 follow-up (done — separate group vs individual views in + the coherence-run UI):** Presentation-only cleanup so no single + table mixes group and individual scores under the same column. + - **Single-gene "Highlighted genes" table** retags winner-source + rows from `"winner"` → `"in winning program"` and adds a one-line + caption underneath: *"These are single-gene ranks of individual + genes — including the genes inside the winning program and the + reference markers. The winning program's combined (group) score is + in the Result cards above and the Coordinated modules below."* + The winner's combined held-out is unchanged in the Result cards. + - **InfoTip copy rewritten** for both panels. `TIPS.rankingResult` + + `TIPS.rankingHighlighted` lead with INDIVIDUAL; `TIPS.module + Ranking` leads with GROUP and explicitly states that the modules' + re-scoring (mean across the gene-set vs label on held-out) is a + different number than the GP search used, so the #1 module can + differ from — and even beat — the winning program. + - **Coordinated-modules table** now tags the module whose gene-set + is exactly equal to the winner's `gene_ids` (unordered set + equality) with a `winner` badge in the existing accent-teal + palette. That row sits at its scored rank, not at the top, so the + user can see what rank the winner occupies under the group metric + (e.g. seed 11 / HPV: winner program holdout 0.9411 vs the same + gene-set as a module → 0.9153 → rank 41; #1 module 0.9778 — three + different numbers, three honest meanings). + - **Paginated auto-reveal** replaces the "Reveal genes" per-row + click. `MODULE_PAGE_SIZE = 25`; symbols cached in a + `Record` keyed by opaque ID and ONE batched + `/reveal` call per page (bounded by ≤ 25 × `max_genes_per_set` + IDs — never the whole map). Page-1 of the HNSC seed 11 run pulls + 99 unique IDs out of ~20k. New `` (|‹ ‹ page n/N › ›|) + + a row click (▶ / ▼) toggles the expanded per-gene-metrics view. + Sort or fresh-run resets page + collapses any open row. Inline + symbol list truncates at 8 with a `· +N` overflow chip; expanding + shows them all. The expanded `` reads from the + shared `symbolByOpaque` cache (no second fetch). + - **No engine/API/airgap changes.** `tsc --noEmit` clean; + `pytest -q` 114 / 114; airgap suite untouched. +- **Chunk 7 follow-up (done — confound pass: configurable Effect + confounders + site / purity survival flags on module ranking):** + Lets us tell whether a gene/module separates HPV for a *mechanistic* + reason or via a *confounder* (sex / race / anatomic site / immune + composition). Airgap stays absolute — clinical confounders (sex / + race / oropharynx flag) and the derived purity proxy are named + non-gene variables in the data + validation layer; engine_v2 only + ever sees opaque gene IDs. + - **HNSC build.** `data_pipeline/build_hnsc.py` now writes `race`, + `ethnicity`, `tissue_site` (raw TUMOR_TISSUE_SITE), + `icd_o_3_site` (ICD-O-3 topography from the patient file), and a + derived **`is_oropharynx`** boolean (SEER mapping: ICD-O-3 prefix + C01/C09/C10 + exact C02.4/C05.1/C05.2). Prints non-null counts + and the top ICD-O-3 distribution so the mapping is auditable. On + the live cohort: 72 oropharynx / 415 not / 0 NaN among 487 called + samples; HPV+ concentrates in the oropharynx (49/72 = 68%) vs + HPV- (23/415 = 5.5%) — the confound is real and large. + - **dsl.Load** widens its allowed clinical columns to include + `race`, `ethnicity`, `tissue_site`, `icd_o_3_site`, `is_oropharynx` + — filtered per cohort (coadread's parquet doesn't have these, so + nothing changes there). + - **API `_prepare_lab_data`** widens the HNSC clinical subset to + `(stage, age, sex, race, tissue_site, icd_o_3_site, is_oropharynx)` + when present (skips columns not in the frame). Coadread unchanged. + - **`Effect` is now configurable.** `ExecContext` gained + `confounders: tuple[str, ...] = ("stage", "age")` — default + preserves byte-for-byte legacy behaviour (MSI / TMB / HPV runs + unchanged). `Effect.execute` reads `ctx.confounders` and builds + a one-hot-encoded design matrix from any columns present in + `ctx.clinical` (age continuous, everything else categorical; + missing values drop the row). Pipeline + worker plumb the + confounder set in — HNSC v2 runs use + `("stage","age","sex","race")` automatically when those columns + are in clinical. Smoking deliberately NOT included (not in this + download). New tests: `test_effect_default_confounders_are_stage_and_age` + + `test_effect_extended_confounders_include_sex_and_race` + (asserts the extended path runs and unknown column names like + "smoking" are silently skipped). + - **Module ranking: site-stratified survival flag.** + `_compute_module_ranking` no longer discards `clinical`. For + HNSC/HPV runs it slices `is_oropharynx` to the held-out test + patients, recomputes each module's mean-aggregate AUROC within + the oropharynx subgroup, and emits `combined_holdout_oropharynx` + + `survives_site` (True iff stratified AUROC ≥ full − 0.05). + Modules whose "HPV signal" was really an oropharynx-tissue + marker visibly collapse here. + - **Module ranking: purity-proxy survival flag.** Immune- + infiltration proxy = per-patient mean expression of CD8A / GZMB / + PRF1 / CD3D / CD2 (`HPV_IMMUNE_PROXY_GENES`, curated once in + `api/app.py`), resolved to opaque IDs via the sealed map — same + bounded-reveal pattern `/diagnostic/full-rank` uses for + `reference_marks`; never leaves the API layer. Bottom tertile + of the proxy on TEST = high-purity subset; each module gets + `combined_holdout_highpurity` + `survives_purity`. On the live + HNSC cohort the high-purity TEST subset is ~49 patients but with + only ~2 HPV+ (class imbalance — HPV+ tumours have more immune + infiltrate), below the n_pos ≥ 5 guard, so the flag honestly + reports "n too small" rather than a misleading AUROC. + - **Module endpoint payload** gains `subgroups: { site, purity }` + metadata (kind / n / tolerance / n_proxy_genes) at the top level + so the UI can render the column label + an explanatory caption. + Opaque-only on the wire — no gene NAMES enter the module + payload. + - **Frontend.** New `` renders compact + `site ✓ 0.92` / `site ✗ 0.84` / `purity —` chips on each module + row (gated to whichever subgroups the server emitted; non-HNSC / + non-HPV runs don't show the column at all). Expanding a module + appends `` — full vs oropharynx vs + high-purity AUROCs with n + per-class counts. New `TIPS.module + Survival` `?` explains the flags. `TIPS.moduleRanking` gained + the verbatim winner's-curse caveat: *"these groups are scored by + re-evaluating ~2,900 explored sets on the same small held-out + set and showing the best — so the very top values are + optimistically biased (the luckiest of thousands). The winning + program was chosen by cross-validation, which guards against + that, so trust it as the engine's pick even when a table row + scores higher."* + - **Verification.** `pytest -q` 116 / 116 (was 114; +2 Effect + confounder tests). Airgap suite 25 / 25 green — module payload + still opaque-only. `tsc --noEmit` clean. Live HNSC seed-11 + coherence-on: 303 modules, 234 survive the oropharynx subgroup + ✓, 69 collapse ✗, 0 n/a; purity all n/a (class imbalance, as + expected). The winner module (the 3-gene Combine — full AUROC + 0.915 in this module-metric view) survives the oropharynx test + (subgroup AUROC 0.919) — it's genuine HPV signal that holds + within oropharynx, not a tissue marker. +- **Chunk 7 follow-up (done — Result views consolidation: GP's + group vs known markers vs explored leaderboard):** Presentation + refactor so the three Lab Result panels map cleanly to three + questions and no single table mixes a group entry with individual- + gene rows. No engine / API / airgap change. + - **Survival chips show full → subgroup.** `` (was: + `site ✓ 0.95`) now renders `site ✓ 0.97 → 0.95 (−0.02)` so the + user sees both the pre and post AUROC plus the drop. Pre = + `module.combined_holdout`; post = the stratified AUROC. The + "n too small" path drops the arrow and renders `site —` + (unchanged). Tooltip wording updated to spell out the full-cohort + → subgroup pair. + - **Known-marker recovery panel = reference markers only.** + ``'s `pinnedRows` no longer pushes the winner's + `kind:"winner"` rows; only reference-set rows survive. SectionCard + title becomes **"Known-marker recovery"**, the heading flips from + *"Highlighted genes"* to *"Known markers"*, the lollipop chart + auto-titles to *"Where the known markers rank"* (and drops the + winner legend swatch) when `winnerOpaqueIds=[]`. The captions + and `TIPS.rankingResult` / `TIPS.rankingHighlighted` rewritten to + state this is a known-marker recovery diagnostic and point users + to the Result panel for the winner's group view. The raw-anonymous + list still tints rows that match the winner — that's per-row + highlighting, not a pinned-source mix. + - **Result panel = self-contained "GP's group" story.** + `` now takes `dataset` / `target` / `runId` / + `coherence` from the Lab parent and fetches + `getFullRankDiagnostic` (per-gene single-gene ranks) + + `getRunModules` (for the winner's survival chips) — same + endpoints already used elsewhere; the server caches them. + Reveals the winner's symbols via `postReveal(winner.gene_ids)` + (bounded — only the winner). New `` rendered + under the metric cards: + - **Its genes, each on its own:** for each `winning.gene_ids`, + looks up its rank in `diag.ranks` and shows + `SYMBOL #rank / N` (e.g. `RNF32 #123 / 20,218 · PCBD2 #79 / + 20,218 · RPS10P7 #2344 / 20,218`). + - **Confound survival:** finds the winner's own module in + `moduleData` via unordered gene-set equality (same notion + `::isWinnerSet` uses), then renders + `` — automatically inheriting the Part-A + pre→post pairing. Gated to runs that emit survival metadata + (HNSC/HPV coherence-on); omitted gracefully otherwise. + - Block InfoTip frames the three-panel layout: *"This is the + group the engine actually chose. Below: how each of its genes + ranks on its own, and whether the group survives the confound + checks. The known-marker recovery diagnostic and the explored- + group leaderboard live in the next two panels — different + questions."* + - **Verification.** `tsc --noEmit` clean; `pytest -q` 116/116 + (no Python changes). Live HNSC seed-11 coherence-on: winner + RNF32+PCBD2+RPS10P7 AUROC 0.941, the three per-gene ranks resolve + (#123 / #79 / #2344), winner module survives the oropharynx + subgroup (site ✓ 0.915 → 0.919) and reports purity n/a (class + imbalance). The known-marker panel shows only MCM5 #22 / MCM2 #28 + / MCM6 #38 / … with no winner dots/rows. The Coordinated-modules + table still badges the winner's own set at its scored rank. +- **Chunk 7 follow-up (done — Result panel: GP's top-10 programs + + plain-language modules copy):** Expanded the single-row "GP's + group" block into the engine's top-10 programs ranked by its OWN + fitness, and rewrote the Coordinated-modules copy in plain + English. Presentation + frontend data-flow only. + - **TopProgramsBlock** replaces `` in ``. + `` now fetches `getRunStatus(runId)` → last + `generations_persisted - 1` → `getRunPopulation(runId, lastGen)`, + dedupes the `candidates` by unordered UNIQUE-gene-set (engine_v2 + programs can carry repeated genes across multiple Selects), keeps + the highest-fitness representative per set, sorts by fitness + desc, and takes the top-10. Each row reuses the existing + treatment: per-gene `SYMBOL #rank / N` (looked up in `diag.ranks`) + and `` for the program's matching module (gene-set + equality the same notion `ModuleRankingPanel::isWinnerSet` uses). + #1 is the winner and is highlighted with the accent-teal palette + + a `winner` chip. + - **Bounded reveal**. The reveal cache (`symbolsByOpaque`) is + populated by a single batched `postReveal` over the UNION of all + top-10 programs' gene_ids — never the whole map. Live HNSC seed- + 11 coherence-on: 15 unique opaque IDs across the top-10. + - **Modules subtitle + InfoTip rewritten in plain English.** The + Coordinated-modules `` subtitle is now: *"Not the + engine's picks. After the run, this re-scores every gene group + the engine tried — using one simple number (the group's average + expression, measured on held-out patients) instead of how the + engine judged groups during the search. So this list can rank + groups differently from 'The GP's top programs' above, and its + #1 can even beat the engine's winner."* `TIPS.moduleRanking` + rewritten verbatim to the three-paragraph "What this is / How + it differs / Caveat" structure from the prompt; the winner's- + curse paragraph now reads as plain advice ("trust the engine's + own picks above as the reliable choice") rather than statistical + jargon. `TIPS.rankingResult` reworded to point at the GP's top + programs and the after-the-fact re-scoring panel by name. + - **Verification.** `tsc --noEmit` clean; `pytest -q` 116/116. + Live HNSC seed-11 coherence-on: GP top-10 by fitness has the + winner (RNF32+PCBD2+RPS10P7) at #1 with GP fitness 0.924; runner- + ups are small 2-3-gene programs around the same core. Module + ranking by combined held-out AUROC has an 11-gene cell-cycle- + flavoured set at #1 (combined 0.978) — a different list, exactly + the divergence the new copy makes explicit. +- **Chunk 7 follow-up (done — cheap diversity knobs):** Premature + convergence was visible in the fitness curve (best = median by + ~gen 10, then flat). Added three cheap diversity levers behind one + "Maintain diversity" toggle. Defaults preserve behaviour byte-for- + byte; the airgap is untouched (search-internal). + - **`run_gp_v2` gains `immigrant_fraction: float = 0.0`.** When > 0, + `round(immigrant_fraction * population_size)` slots in the new + generation are filled with fresh programs drawn from + `ramped_population` (same rng, same grammar/objective/depth + constraints as init), inserted AFTER the elites and BEFORE the + crossover/mutation offspring — never displacing elites. Default + 0.0 ⇒ no immigrants ⇒ behaviour unchanged. `tournament_k` and + `p_mutate` were already parameters; raising mutation and + lowering selection pressure are paired with immigrants via the + same toggle. + - **Pipeline + worker plumbing.** `run_v2_pipeline` and + `run_v2_pipeline_streaming` accept the new + `immigrant_fraction: float = 0.0`. `RunRequest` gains + `diversity: bool = False`. `Run` dataclass gains a `diversity` + field; surfaced in `/runs/{id}`. `_worker` maps `diversity=True` + → `tournament_k=2`, `p_mutate=0.85`, `immigrant_fraction=0.10`; + `False` → current `(3, 0.7, 0.0)`. Two new tests in + `tests/test_engine_v2.py`: + `test_default_run_unchanged_by_diversity_param` (passing + `immigrant_fraction=0.0` explicitly produces an identical best- + fitness trajectory to omitting the arg) and + `test_immigrant_fraction_injects_fresh_programs` (with `frac>0` + the last generation contains ≥1 immigrant — empty-parents and + non-survived — and more distinct `program_repr` strings than the + baseline). + - **Frontend.** New `` copy. ParamsControls gets + a "Maintain diversity" checkbox under the coherence toggle; + plain-English caption: *"Default off. Lowers selection pressure + and injects fresh random programs each generation so the + population keeps exploring — watch the best-vs-median gap in + the fitness curve stay open longer."* Lab state + + `postRun(body.diversity)` plumbed end-to-end. + - **Verification.** `pytest -q` 118/118 (was 116; +2 diversity + tests). `tsc --noEmit` clean. Airgap suite untouched. Live + HNSC HPV seed-11 coherence-on, 20 generations, A/B: + - **OFF**: gap collapses 0.242 → 0.002 by gen 19 (full + convergence). Held-out 0.977, p 0.048. + - **ON**: gap stays 0.13-0.18 the entire run (mean 0.149 vs + OFF 0.041; last-gen 0.139 vs OFF 0.002). Held-out 0.948, + p 0.048 — detection preserved while the population stays + exploratory. +- **Chunk 7 follow-up (done — merge "GP's top programs" + + "Coordinated modules" into one sortable "Groups the engine + explored" table):** Replaced the two overlapping panels with a + single table over the SAME groups (harvested from the persisted + population), selectable by four sort lenses. One additive API + field, presentation refactor; airgap unchanged. + - **API.** `_compute_module_ranking` now records the max GP fitness + per gene-set while harvesting candidates and emits + `gp_fitness: float | None` on each module dict. Fitness is a + number — no gene names cross the wire. `RankedModule` in + `web/lib/api.ts` gains `gp_fitness?: number | null`. + - **Frontend — merged table.** `` retitled + **"Groups the engine explored"**. `SortKey` extends to + `"gp_fitness" | "combined" | "coherence" | "synergy"`. Default + sort = **gp_fitness** so the default view IS the engine's own + preference order (replacing the deleted top-programs panel). + Two new columns: **GP fitness** (per-row) and **Synergy** = + `combined_holdout − max(single_gene_metric over per_gene)` — high + Synergy = real additive lift; low/negative = "best gene + + passengers" pattern. Sort buttons relabelled accordingly. + Subtitle rewritten in plain English (verbatim per prompt); + `TIPS.moduleRanking` rewritten to explain all four lenses + keep + the winner's-curse caveat for the re-score lenses. + - **Frontend — drop TopProgramsBlock.** `` no longer + fetches `getRunStatus` / `getRunPopulation` / `postReveal` / + `getFullRankDiagnostic` / `getRunModules`; the entire + `TopProgramsBlock` function definition and its supporting state + are gone. Result-panel signature simplifies to + `{ result, dataset, target, runId, coherence }` with `dataset` / + `runId` only used to gate the new pointer copy under the metric + cards: *"The winning program is badged in **Groups the engine + explored** below — sort by GP fitness to see the engine's full + preference order, or by Combined AUROC / Coherence / Synergy to + re-score the same groups by a different lens."* Pointer is gated + to `target !== "none" && coherence` (the same gate the merged + table uses). Unused imports (`Candidate`, `getRunStatus`, + `getRunPopulation`) removed. + - **Verification.** `pytest -q` 118/118 (no Python tests changed — + the new field is additive, the old payload tests still pass). + `tsc --noEmit` clean. Airgap suite untouched. Live HNSC seed-11 + coherence-on confirms the four lenses give visibly distinct + orderings on the same module set: GP fitness #1 = winner + (RNF32+PCBD2+RPS10P7, gp 0.924); Combined-AUROC #1 = 11-gene + cell-cycle module (combined 0.978); Coherence #1 = a tightly + co-expressed 2-gene set; Synergy #1 = a 3-gene group with + +0.154 lift above its best single gene. +- **Chunk 7 follow-up (done — "GP fitness × Synergy" landscape + scatter):** Added a canvas scatter above the merged Groups table + so the user can SEE which quadrant each explored group falls into + at a glance. Frontend-only; no engine / API / airgap change. + - **``** (~340 LOC in `web/app/Lab.tsx`). + x = `m.gp_fitness`, y = `synergyOf(m)` (lifted to a module-scope + helper so the table + scatter share one definition). + Categorisation via `categoriseModule(m, winnerSet)` → + `winner | p16 | immune | cell_cycle | MMR | other` (single- + marker sets win over broader sets; winner takes priority over + membership so the ring is always on top). + - **Canvas for the dots, SVG overlay for the chrome.** ~6k modules + in larger runs would choke as SVG nodes; canvas paints faint + grey background dots first, then amber/gold highlighted dots, + then the winner as a teal-ringed filled centre on top so it + never gets occluded. HiDPI: `canvas.width = clientWidth × dpr` + + `ctx.setTransform(dpr, 0, 0, dpr, 0, 0)`. Plot rect responsive + via a `ResizeObserver` on the wrapper div. + - **Quadrant story.** Vertical guide at the median GP fitness, + horizontal guide at synergy = 0 (orange dashed). Four captions + in the corners: "real teamwork (unexplored by the engine)" / + "ideal — usually empty (engine + teamwork)" / "weak / junk" / + "engine's lone-gene detectors". Axis titles: *"GP fitness — what + the engine preferred →"* and *"Synergy — teamwork beyond best + gene ↑"*. Dataset-aware legend (HNSC shows p16 + cell_cycle; + CRC shows immune + MMR). + - **Gated** to `target !== "none"` + coherence-on (same gate as + the survival flags / synergy column). Mounted between the + survival-flag caption and the Groups table inside + `` so it shares one SectionCard frame with + the table. + - **Verification.** `tsc --noEmit` clean; `pytest -q` 118/118 (no + Python changes). Live HNSC seed-11 coherence-on: 303 plottable + modules; winner @ `gp=0.924 / syn=0.032`; 21 dots in the + bottom-right "engine's lone-gene detectors" quadrant; 20 in the + top-left "real teamwork" quadrant the engine didn't prefer; + 3 cell_cycle-flagged dots cluster at low GP (~0.58, the engine + barely scored them) with slightly-negative synergy — exactly + the "best gene + passengers" pattern the scatter is meant to + expose. Airgap untouched (synergy + category derived from + fields already on the wire; no extra reveal). +- **Chunk 7 follow-up (done — hover tooltips on the scatter, + highlighted dots only):** Added hit-tested tooltips to the + fitness × synergy scatter so the user can read the gene-set and + scores of any highlighted dot without scrolling the table. + Frontend-only; airgap intact. + - **Hit-test scope.** When painting, `` + now records the screen position of every HIGHLIGHTED dot + (`winner` + `cell_cycle` + `MMR` + `p16` + `immune`) into a + `hitsRef.current: Hit[]` array. The faint grey background dots + are deliberately NOT in this array — they are not revealed and + are not hit-tested. Selection: `mousemove` walks the array and + picks the nearest hit within ~8px (radius² = 64); `mouseleave` + clears. + - **Bounded reveal.** On mount / when `modules` changes, the + scatter takes the UNION of all highlighted modules' `gene_ids` + and issues ONE batched `postReveal` call — never the whole map. + On live HNSC seed-11 coherence-on: **31 unique opaque IDs** + revealed out of ~20k (winner 3 + 21 cell_cycle-tagged modules, + plus a handful of overlapping genes). Cached in + `highlightSymbols: Record` so re-renders are + free. + - **``** absolutely-positioned over the chart + container. Renders: source chip (winner / p16 / cell_cycle / + immune / MMR — coloured to match the dot), `n genes`, the + revealed gene symbols (falls back to `(revealing…)` while the + bounded call is in flight), and a 2×4 grid of GP fitness, + Combined AUROC, Coherence, Synergy via `fmtFit` (non-finite → + em-dash). Synergy uses the shared `synergyOf` helper so the + tooltip, the table column, and the y-axis stay consistent. + Position clamps to the chart rect so dots near the edge don't + push the box off-screen. + - **Verification.** `tsc --noEmit` clean; `pytest -q` 118/118 (no + Python changes). Live HNSC seed-11 coherence-on: hovering the + winner ring shows `RNF32, PCBD2, RPS10P7 · GP 0.924 · Combined + 0.915 · Coherence 0.474 · Synergy 0.032`; hovering a + cell_cycle dot shows e.g. `CDC6, …` with its scores. Background + dots are not hover-targets, no full-map reveal. +- **Chunk 7 follow-up (done — encode site survival on the scatter + + plain-language Survives explainer):** Two presentation fixes + around the confound flags. Frontend-only; the data + (`survives_site` / `survives_purity` / subgroup AUROCs) is already + on the wire. + - **Scatter site-survival encoding.** Highlighted dots in + `` are now drawn: + - **solid** (fill + stroke) when `survives_site === true` (or + `null` / "n too small" — keeps the simple default visible), + - **hollow** (white fill + coloured stroke) when + `survives_site === false` — so failed-site groups read as + "less trustworthy" at a glance. + Only **site** is encoded; purity is mostly `null` on this + cohort and carries no signal. The category colour (winner teal + / cell_cycle amber / p16 gold) is preserved. The winner ring + uses the same encoding when it fails the check. + - **Hide-failed toggle.** Added a small "Hide groups that fail + the site check" checkbox in the scatter header (default OFF). + On: `survives_site === false` highlighted dots are dropped + from both the canvas paint AND the hit-test array (no + accidental hover tooltips on hidden dots). Background grey + dots are unaffected (not site-tested). + - **Caption + legend updated.** The italic caption gains + "Solid dots survive the site check; hollow dots fail it (their + signal is partly location)." The legend gains a "hollow = fails + site check" entry. + - **Plain-language "Survives" column ?.** `TIPS.moduleSurvival` + rewritten verbatim per the prompt: + > What "Survives" checks: whether a group still separates HPV + > when you take away a possible confounder — something that + > travels with HPV but isn't HPV biology. + > + > Site / Purity each get a one-paragraph plain-English + > explanation; ✓ / ✗ / — get a one-line legend each. + The tip is now anchored on the **Survives column header** + (the ``) where it's most discoverable. Per-chip pre→post + tooltips on the rows are unchanged. + - **Verification.** `tsc --noEmit` clean; `pytest -q` 118/118 + (no Python changes). Live HNSC seed-11 coherence-on shows + 234 of 303 modules survive site / 69 fail; the 4 highlighted + dots (winner + 3 cell_cycle) all survive on this seed, so the + encoding renders solid — the failed-dot path is exercised in + the general case where a tagged module collapses. +- **Chunk 7 follow-up (done — five small UI fixes: default cohort, + remove peel-off, fix two clipped widgets, swap biology figure):** + Presentation-only sweep over Lab; frontend-only. No engine / API / + airgap change. + 1. **Default to Head & Neck + HPV detection.** `dataset` state in + Lab now defaults to `"hnsc"`, and `target` to + `DATASET_REGISTRY.hnsc.objectives[0].key` so first load lands + on the HPV demo of the workflow. Colorectal stays selectable; + only the default changed. + 2. **Hide the Discovered axes panel.** `` is now + gated to `target === "none"` (never true in the UI — the + unsupervised objective isn't a UI objective anymore). On + supervised runs the panel was re-finding the same signal and + showing nonsensical "Aligns with MSI / TMB" text. The component + definition + backend chain are untouched in case a developer + flag wants to surface it again. + 3. **Fix the "(TCGA COADREAD" subtitle clipping.** Not a CSS + issue — the old regex `/^[(]|[)]$/g.replace` was stripping the + trailing `)`. Replaced with + `entry.longLabel.replace(new RegExp('^${entry.label}\\s*'), '').trim()` + so the parenthesised tail renders intact, e.g. `(TCGA HNSC)`. + 4. **"Survives" `?` is a click-to-open modal now.** Multi-paragraph + copy didn't fit a hover tooltip — clipped on the right. + `paramHelpContent.tsx::ParamKey` extended with `"module_survival"` + and a new `PARAM_HELP.module_survival` entry carrying the + prompt's verbatim site / purity / ✓ / ✗ / — explainer. The + `Survives` (and the table-header caption above the + table) now uses `` instead + of ``. Per-chip `full → subgroup` row tooltips are + unchanged. + 5. **HPVBiologyPanel — new card-wall figures.** `HPVBiologyPanel.tsx` + now renders TWO inlined SVGs: `` (the new + repo-root `normal_cell_reference.svg`, viewBox 920×900: RB1 / + p16 / RB-E2F cycle / E2F / genes E2F turns on / protein + production / p53) followed by `` (the rewritten + repo-root `hpv_reference.svg`, viewBox 920×660: HOW HPV TAKES + OVER → HOW E7 ATTACKS RB · HOW E6 ATTACKS p53 → p16 BECOMES + THE HPV+ MARKER · WHY HPV DOESN'T MUTATE THE GENES). The + normal-cell baseline renders first so the reader sees the brake + before they see it dismantled. SVG attributes hand-translated to + JSX exactly as the existing component did (`font-family → + fontFamily`, `font-weight → fontWeight={n}`, `text-anchor → + textAnchor`, `marker-end → markerEnd`, etc.); each `` keeps + `viewBox` + a width:100% / height:auto wrapper so it scales + without overflow. + - **Verification.** `tsc --noEmit` clean; `pytest -q` 118/118 (no + Python changes). Lab first-load lands on HNSC + HPV; biology + panel shows the new normal-cell + HPV figures (both fully + visible, no overflow); the Discovered-axes panel is gone for + HPV / MSI / TMB runs; cancer-card subtitles read "(TCGA HNSC)" + with the closing paren; the Survives "?" now opens a click- + modal with the four-paragraph site / purity explainer that + never clips. +- **Chunk 7 follow-up (done — raised caps, "GP" → "genetic + programming", DSL operator-usage):** Three changes across the + Lab + API surface. + - **Generations / Population caps raised.** Lab's NumField config + bumps Generations max 100 → 1000 and Population max 500 → 3000; + server-side `RunParamsModel.generations / population` Field + bounds raised to match. Bounded — not removed — so an + accidental huge value still can't hang the backend. + - **"GP" spelled out everywhere user-facing.** Every visible + occurrence in `Lab.tsx` replaced: "GP knobs" → "Genetic- + programming knobs"; "Starts a GP run" → "Starts a genetic- + programming run"; the compact label "GP fitness" → **"Genetic- + programming fitness"** at the sort button, table header, + scatter heading + axis title + median guide + tooltip dt + + module-ranking subtitle, and TIPS prose. The internal data + key `gp_fitness` is UNCHANGED (API contract). Code comments + still say "GP" by choice — they're not user-facing. + - **DSL operator-usage endpoint + tiles.** + - New `GET /runs/{run_id}/operator-usage` walks every candidate + in every persisted generation, counts each operator token + (`Select(`, `Reduce(`, `Combine(`, `Split(`, `Associate(`, + `Effect(`, `FitApply(`, `Search(`) in the `program_repr` + strings, and emits per-operator + `{name, total_uses, programs_using}` + run totals + (`n_generations`, `n_candidates`). Cached per run. 425 while + running, 404 unknown, 500 on error. Opaque-safe by + construction — operator keywords + integer counts only, no + gene IDs / symbols ever in the payload. Three new airgap + tests (`test_operator_usage_endpoint_returns_opaque_counts`, + `_425_while_running`, `_404_for_unknown_run`). + - Frontend: `getRunOperatorUsage(runId)` fires from the SSE + `done` handler; Lab caches the result in `operatorUsage` + state. New `` driver swaps each + `` for an enhanced version that adds a + `used N× — in P% of programs` line and a tile-relative bar + (height 1, accent fill, white-grey track) under the + description. Before / during a run, tiles render just the + hints (no counts). A caption appears beneath the grid after + a completed run. + - **Verification.** `pytest -q` 121/121 (+3 new airgap tests). + `tsc --noEmit` clean. Live HNSC seed-11 coherence-on, 10×80: + Reduce in 100% of programs (grammar floor), Select 90.6% + (1098 uses), Combine 21.2% (216), Split 4.1%, Associate 2.4%, + Effect 1.0%, Fit/Apply 3.6%, **Search 0** (off by default). + Airgap-clean payload — keywords + counts only. +- **Chunk 7 follow-up (done — turn Search on by default + add a + per-run gate):** Search was the one DSL operator still gated off + (`DEFAULT_RATES["search"] = 0.0`). Turn it on at a modest rate so + the engine can use the full DSL, and add a UI toggle so it can be + flipped off when speed matters (every Search node runs an inner + gene ranking). + - **Engine.** `engine_v2/synthesize.DEFAULT_RATES["search"]` flips + 0.0 → **0.05**. Caps stay (`SEARCH_MAX_K`, ≤ 200 candidate + columns) so each Search's cost is bounded. The existing + `is_unsup` gate at the injection site still forbids Search on + unsupervised runs. + - **`run_gp_v2` gains `rates_override: dict | None = None`.** Inside + `run_gp_v2` the override is merged on top of the objective's + synthesis-overrides (`UNSUP` etc.) — caller's keys win — and the + merged dict is threaded into every `ramped_population` / `mutate` + call. Both pipeline entry points (`run_v2_pipeline`, + `run_v2_pipeline_streaming`) expose `rates_override` and forward + it through. + - **API.** `RunRequest` gains `enable_search: bool = True`. `Run` + dataclass tracks it; `/runs/{id}` surfaces it. `_worker` builds + `rates_override = None` when on, or `{**DEFAULT_RATES, + "search": 0.0}` when off, and passes it into the pipeline. The + override is a complete dict (overlaid on DEFAULT_RATES) so the + inner `synthesize._grow_*` code can index by key without a + fallback. + - **Frontend.** New `` copy. ParamsControls + gains an **"Enable Search operator"** checkbox under the + diversity toggle (default ON). Lab state + `postRun.body + .enable_search` plumbed end-to-end. DSL-vocabulary tile for + Search now reads *"Nested feature search — runs a small + gene-ranking inside the program (bounded: ≤ 4 genes from + ≤ 200 columns)."* — the "off by default" wording is gone. + - **Tests.** `tests/test_api_airgap.py::test_enable_search_false_pins_search_to_zero` + verifies the toggle: with `enable_search=False` the operator- + usage payload reports Search at 0 total uses / 0 programs, and + `/runs/{id}` surfaces the flag. The existing + `test_operator_usage_endpoint_returns_opaque_counts` relaxed + its old "Search must be 0" assertion (it's now `>= 0` — Search + may or may not fire on a tiny synthetic 3×12 run). + - **Verification.** `pytest -q` 122/122 (+1). `tsc --noEmit` + clean. Live HNSC seed-11 coherence-on, 10 × 80: + - **Search ON** (default): Search used **20×** across 20 / 800 + programs (~2.5%); winner AUROC 0.909, p 0.048. + - **Search OFF**: Search used **0×** / 0 programs; winner + AUROC 0.941, p 0.048. +- **Chunk 7 follow-up (done — editable DSL rates in the vocabulary + panel + drop the Search checkbox + reorder DSL before Run):** + Make every optional DSL operator's injection rate editable in the + DSL vocabulary section (pre-filled with engine defaults), fold the + Search checkbox into a Search-rate input (0 = off), and move the + panel above the Run section. Airgap untouched. + - **API.** `RunRequest.enable_search` is gone; replaced by a flat + `RunRequest.rates_override: dict[str, float] | None`. Keys we + accept: `split`, `effect`, `fitapply`, `search`, `scalar_share`. + Missing keys keep their engine defaults — an empty / missing + override reproduces current behaviour byte-for-byte. `Run` + dataclass stores the dict; `/runs/{id}` surfaces it. + `_worker` splits the flat dict back into `rates_override` + (overlay on `engine_v2.synthesize.DEFAULT_RATES`) and + `scalar_share_override`, then forwards both to the pipeline. + - **Engine.** `run_gp_v2` gains + `scalar_share_override: float | None = None`; merged with the + objective's overrides via an `effective_scalar_share` that wins + over `overrides.get("scalar_share")`. Both pipeline entry + points expose the new param and forward it. + - **Frontend.** New `DSL_DEFAULT_RATES` constant mirrors the + engine defaults (Split 0.10 / Effect 0.40 / Fit/Apply 0.10 / + Search 0.05 / scalar_share 0.20). `` + rewritten: structural operators (Select / Reduce / Combine / + Associate) render with an "always available" tag; optional + operators (Split / Effect / Fit/Apply / Search) render with a + small `{copied ? "Copied" : label} + + ); +} + +function CopyGlyph() { + return ( + + + + + ); +} + +function CheckGlyph() { + return ( + + + + ); +} + +function legacyCopy(text: string): boolean { + try { + const ta = document.createElement("textarea"); + ta.value = text; + ta.setAttribute("readonly", ""); + ta.style.position = "fixed"; + ta.style.opacity = "0"; + document.body.appendChild(ta); + ta.select(); + const ok = document.execCommand("copy"); + document.body.removeChild(ta); + return ok; + } catch { + return false; + } +} diff --git a/web/app/HPVBiologyPanel.tsx b/web/app/HPVBiologyPanel.tsx new file mode 100644 index 0000000000000000000000000000000000000000..e67219b4510355366ae156412578f57075c143f1 --- /dev/null +++ b/web/app/HPVBiologyPanel.tsx @@ -0,0 +1,297 @@ +"use client"; + +import { useId, useState } from "react"; + +/** + * "What is HPV? (the biology)" — collapsed disclosure that mirrors + * MMRBiologyPanel.tsx exactly. Closed on first render; the embedded + * SVGs (verbatim from repo-root `normal_cell_reference.svg` and + * `hpv_reference.svg`) scale via their viewBoxes so the panel never + * overflows. + * + * Renders the NORMAL CELL diagram first (baseline — how division is + * controlled), then the HPV diagram (the viral attack), so the reader + * sees the brake before they see it dismantled. + * + * Presentation only. + */ +export default function HPVBiologyPanel() { + const [open, setOpen] = useState(false); + const panelId = useId(); + return ( +
+ + {open && ( +
+

+ How HPV drives these cancers — the two brakes it breaks, and + why one gene (p16) becomes its fingerprint. +

+ + +
+ )} +
+ ); +} + +// Verbatim contents of repo-root `normal_cell_reference.svg`. Wrapped +// in a width:100% / height:auto container so the SVG's viewBox handles +// scaling — no horizontal overflow at narrow widths. +function NormalCellDiagram() { + return ( +
+ + Normal cell — how division is controlled + + A wall of cards covering RB1 the brake, p16 keeping the brake on, + how RB and E2F interact through a phosphate toggle, what E2F is, + the genes E2F switches on, protein production, and p53 the damage + guard. + + + + + NORMAL CELL — how division is controlled + + + {/* RB1 */} + + RB1 — THE BRAKE + RB1 is a gene in every normal human cell. + Its RB protein is present permanently. + A phosphate tag toggles it ON / OFF — a brake + that’s always installed; the cell presses or + releases it. + JOB: stop the cell dividing until it’s + genuinely supposed to. + + {/* p16 */} + + p16 — KEEPS THE BRAKE ON + p16 blocks the enzyme CDK. + With CDK blocked, no phosphate goes onto RB, + so RB stays active — the brake stays on. + It’s a protein; its gene is CDKN2A. + + + {/* RB & E2F interact */} + + + HOW RB & E2F INTERACT — the brake cycle + + + • E2F sits parked on the DNA at the start switches of the division genes, ready to fire. + + + • RB binds directly onto E2F, clamping it shut — the division genes stay OFF. + + + • A legitimate “time to divide” signal makes the enzyme CDK stick a phosphate tag onto RB. + + + • Phosphate ON → RB changes shape → it can no longer hold E2F → RB lets E2F go (cell divides). + + + • After division, phosphate OFF → RB returns to shape → it re-grips E2F (brake back on). + + + {/* E2F */} + + E2F — THE “GO” SWITCH + A protein called a transcription factor (its job: + switch other genes on by binding to DNA). + When free, it latches onto DNA and turns on the + whole batch of genes needed to copy DNA and + divide. + It docks at a gene’s start switch (its promoter) + and signals “read this now”, which lets the + machinery make the protein. + It recruits RNA polymerase. + + {/* Genes E2F turns on */} + + GENES E2F TURNS ON + DNA-copying machinery, e.g. MCM2–7 & PCNA: + • MCM2–7 — the “unzipper” that opens the DNA. + • PCNA — clamp that holds the copier onto + the DNA strand. + Raw materials: enzymes that make the DNA + “letters” to copy with. + E2F → genes ON → their proteins are built → + those proteins copy the genome & split the cell. + + + {/* Protein production */} + + PROTEIN PRODUCTION + RNA polymerase reads the gene’s DNA. + ↓ + Writes a working copy of the recipe (mRNA). + ↓ + Ribosomes read the copy and assemble the + protein. + + {/* p53 */} + + p53 — THE DAMAGE GUARD + A protein; its gene is TP53. + When DNA is damaged or something’s wrong, + p53 steps in and either: + (a) halts the cell and orders a repair, or + (b) if damage is too severe, triggers the + cell to self-destruct. + + + + + + + +
+ ); +} + +// Verbatim contents of repo-root `hpv_reference.svg` (the rewritten +// card-wall figure — "HPV INFECTION: the virus dismantles both +// brakes"). Wrapped the same way as the normal-cell diagram. +function HPVDiagram() { + return ( +
+ + HPV infection — how E6 and E7 dismantle both brakes + + Cards covering how HPV takes over the cell, how E7 attacks RB, how + p16 becomes the HPV-positive marker, how E6 attacks p53, and why + HPV destroys the proteins rather than mutating the genes. + + + + + HPV INFECTION — the virus dismantles both brakes + + + {/* Overview */} + + HOW HPV TAKES OVER + + The HPV virus infects a human cell and slips in its viral genes. + + + It gets the cell’s own machinery to read those genes and build the proteins they code for — + + + two of which are E6 & E7 (“E” = early). The human cell builds the proteins HPV wants. + + + E7 → attacks the RB brake. E6 → attacks the p53 guard. + + + + + + {/* E7 attacks RB */} + + HOW E7 ATTACKS RB + RB has a “pocket” it uses to grip E2F. + E7 carries a grappling hook shaped to fit that + same pocket. + E7 grabs RB exactly where RB grabs E2F → + E2F is pried off and freed. + Then E7 recruits the cell’s disposal machinery + and marks RB for destruction — sent to the shredder. + + {/* E6 attacks p53 */} + + HOW E6 ATTACKS p53 + E6 recruits a host enzyme called E6AP (a + tagging enzyme). + E6 + E6AP latch onto p53 and tag it for + destruction. + The cell can no longer halt, repair, or self-destruct. + Damaged cells survive and keep dividing → + mutations and genome instability pile up. + + + + + {/* p16 marker */} + + p16 BECOMES THE HPV+ MARKER + Normally, active RB keeps p16 low (a feedback loop). + Lose RB → that feedback is gone → p16 piles up. + But the p16 is useless now: with RB already + destroyed, it has nothing left to protect. + + High p16 (gene CDKN2A) = the HPV+ marker we measure. + + + {/* why not mutate */} + + WHY HPV DOESN’T MUTATE THE GENES + The TP53 and RB1 genes stay intact — + HPV destroys the p53 and RB proteins directly. + By contrast, smoking-driven HPV− tumours + disable p53 the other way — by mutating the + TP53 gene itself. + + + + + + + +
+ ); +} diff --git a/web/app/InfoTip.tsx b/web/app/InfoTip.tsx new file mode 100644 index 0000000000000000000000000000000000000000..33b24d2ffdc64152c6e0583ed93f91d2402f144c --- /dev/null +++ b/web/app/InfoTip.tsx @@ -0,0 +1,79 @@ +"use client"; + +import { useEffect, useId, useRef, useState } from "react"; + +interface Props { + text: string; + label?: string; +} + +/** + * Muted "?" trigger with a calm popover. + * + * Opens on hover, on focus (keyboard tab), and on click/tap (touch). Closes + * on outside click, Escape, or blur. The popover is aria-labelled to the + * trigger so screen readers announce it as a tooltip. + */ +export default function InfoTip({ text, label = "More info" }: Props) { + const [open, setOpen] = useState(false); + const wrapperRef = useRef(null); + const tooltipId = useId(); + + useEffect(() => { + if (!open) return; + function onDown(e: MouseEvent | TouchEvent) { + if ( + wrapperRef.current && + !wrapperRef.current.contains(e.target as Node) + ) { + setOpen(false); + } + } + function onKey(e: KeyboardEvent) { + if (e.key === "Escape") setOpen(false); + } + document.addEventListener("mousedown", onDown); + document.addEventListener("touchstart", onDown); + document.addEventListener("keydown", onKey); + return () => { + document.removeEventListener("mousedown", onDown); + document.removeEventListener("touchstart", onDown); + document.removeEventListener("keydown", onKey); + }; + }, [open]); + + return ( + setOpen(true)} + onMouseLeave={() => setOpen(false)} + > + + {open && ( + + {text} + + )} + + ); +} diff --git a/web/app/Lab.tsx b/web/app/Lab.tsx new file mode 100644 index 0000000000000000000000000000000000000000..f3252ea0dde70910cc127e0344c503bad70af4c1 --- /dev/null +++ b/web/app/Lab.tsx @@ -0,0 +1,6779 @@ +"use client"; + +import { Fragment, useEffect, useId, useMemo, useRef, useState } from "react"; +import { + CartesianGrid, + Line, + LineChart, + ResponsiveContainer, + Tooltip, + XAxis, + YAxis, +} from "recharts"; +import { + API_URL, + DatasetId, + EvaluateResponse, + FullRankDiagnostic, + GeneRankRow, + GenerationEvent, + HPVRankDiagnostic, + ObjectiveSpec, + ModuleRanking, + RankedModule, + ReferenceMark, + RunParams, + RunResult, + TMBRankDiagnostic, + Target, + getFullRankDiagnostic, + getHPVRankDiagnostic, + getRunModules, + getRunOperatorUsage, + getRunTransfer, + TransferResult, + OperatorUsage, + getTMBRankDiagnostic, + postEvaluate, + postReveal, + postRun, +} from "../lib/api"; +import InfoTip from "./InfoTip"; +import ProgramGraph from "./ProgramGraph"; +import PopulationTiles from "./PopulationTiles"; +import PasteToDraw from "./PasteToDraw"; +import ParameterFlow from "./ParameterFlow"; +import MMRBiologyPanel from "./MMRBiologyPanel"; +import HPVBiologyPanel from "./HPVBiologyPanel"; +import ParamHelp from "./ParamHelp"; +import ParamHelpProvider from "./ParamHelpProvider"; +import CopyButton from "./CopyButton"; +import { + REFERENCE_SETS_BY_DATASET, + ReferenceSetKey, + refSetsFor, +} from "../lib/referenceSets"; +import { fmtFit } from "../lib/fmt"; +import { ParamKey } from "./paramHelpContent"; + +// ---------- Tooltip copy (verbatim, never edited inline) ------------------- +// Objective tooltips moved to paramHelpContent.tsx (obj_msi / obj_tmb / +// obj_survival / obj_unsupervised) so the SHORT + DETAILED + diagrams +// share the single ParamHelp infrastructure. + +const PARAM_TIPS: Record = { + generations: + "How many rounds of evolution the engine runs. Each round it " + + "keeps the best programs and breeds variations. More rounds refine " + + "further, but take longer.", + population: + "How many programs the engine tries in each round. More programs " + + "explore more options, but the round takes longer.", + genes_per_set: + "The most genes a single score is allowed to use. Smaller keeps " + + "programs simple and easy to read.", + max_sets: + "How many separate gene-scores a program is allowed to combine (1 " + + "or 2). Two lets it build a small “score made of scores.”", + lambda: + "How much the engine is penalised for using more genes. Higher pushes " + + "it toward fewer genes (simpler answers).", + seed: + "The starting point for the engine’s randomness. The same seed " + + "reproduces the exact same run; change it to see a different one.", + prefilter_n: + "By default the engine searches all ~20,000 genes, so nothing is " + + "pre-excluded. Turning this on narrows to the N most promising genes " + + "first — faster, but it can drop a real gene that only helps in " + + "combination.", + permutations: + "How many times we re-run the search on deliberately scrambled " + + "labels — a chance baseline. The real result must beat these random " + + "runs (that’s the permutation p).", +}; + +const TIPS = { + run: + "Starts the search with the settings above.", + runId: + "Just a label for this run, so its results can be fetched.", + status: + "Where the run is: queued → running → done.", + // NB: fitness-curve InfoTip is now objective-aware — see + // FITNESS_TIP_BY_TARGET below. This single string was only correct + // for TMB and was misleading on MSI / unsup runs. + nodes: + "How many steps the program has.", + genes: + "How many genes the program uses.", + bestVsMedian: + "Each round, the engine keeps the best programs and breeds " + + "variations. The solid line is the best program so far; the dashed " + + "line is the typical one. When they meet and flatten, the " + + "population has converged — it’s stopped finding anything " + + "new.", + populationGrid: + "Every program in this round. Darker = higher score; outlined = " + + "kept for the next round.", + survivedVsDiscarded: + "Solid cards survived to breed the next generation; faded cards were " + + "culled for scoring worse.", + opaqueIds: + "Anonymous stand-in labels for real genes. The engine works blind " + + "and never sees gene names until we reveal them at the end — so it " + + "can’t lean on what’s already known about them.", + winningProgram: + "The best program found, showing which (still-anonymous) genes it " + + "uses and how it combines them.", + // Held-out and permutation-p have target-aware variants (see the + // BY_TARGET records below the TIPS block). These plain strings stay + // for any legacy call site. + heldOut: + "The winner's score on patients the engine never saw during the " + + "search (the held-out set). It tests that the result isn't just " + + "memorising the training data.", + permutationP: + "How often pure chance matches this score across many random " + + "comparisons. Small (e.g. below 0.05) = unlikely to be a fluke.", + baseline: + "A simple, no-frills method's score, for comparison — the evolved " + + "program should beat it.", + // Reference-set tip has a dataset-aware variant (see + // REFERENCE_SET_TIP_BY_DATASET). This is the colorectal default. + referenceSet: + "A known gene list to check the answer against. MMR = the 4 DNA " + + "repair / 'spell-checker' genes (MLH1, MSH2, MSH6, PMS2); " + + "immune = immune-attack genes (CD8A, GZMA, PRF1).", + revealEvaluateButton: + "Unlocks the real gene names for the winning program and checks how " + + "many match the chosen known list.", + revealedGenes: + "The real names behind the program's gene IDs — shown only now, " + + "after the blind search is done.", + overlap: + "How many of the engine's genes appear in the known list (matches " + + "highlighted). More overlap = it rediscovered known biology on its " + + "own.", + programGraph: + "The winning program drawn as a graph: boxes are operators (verbs), " + + "pills are data and scores. Tier-1 turns genes into a score; Tier-2 " + + "combines scores into the prediction.", + dslVocabulary: + "Rate = how often the engine reaches for an operator when it " + + "builds or mutates a program — roughly the chance, at each " + + "eligible spot, that it picks that operator instead of a simpler " + + "one. 0.1 ≈ 1 in 10; 0.05 ≈ 1 in 20; 0 = never. Higher means the " + + "operator is tried in more programs (more exploration) — it does " + + "NOT mean the operator is better; selection still decides whether " + + "it survives into the winners. (Effect's rate is its share of the " + + "correlation-style programs versus Associate.)", + // Discovered-axes + post-hoc tips have dataset-aware variants below. + // The plain strings below are the colorectal defaults. + discoveredAxes: + "After each unsupervised run, click \"Find next axis\" to peel the " + + "discovered split out of the data (linear residualisation — we " + + "subtract its signal from every gene) and search again. The chain " + + "is a ranked, blind list of independent directions in the gene-" + + "expression matrix — each named only afterwards by how well it " + + "lines up with a known label. The residualisation is LINEAR, so " + + "deeper non-linear structure may remain. The chain is in-memory " + + "only — it resets on server restart or a fresh Run.", + posthoc: + "After the unsupervised search finishes, the discovered split is " + + "compared to the named labels the engine never saw. AUROC = how " + + "well the score sorts the two groups apart (0.5 = no relationship, " + + "1.0 = a perfect match).", + tmbRank: + "Rank every gene by signed Spearman with TMB on the engine's TMB " + + "cohort, ascending (most-negative = rank 1, since the TMB objective " + + "rewards the most-negative association). Where each known gene " + + "lands tells you whether the signal is even there: MMR genes near " + + "rank 1 = findable from expression alone; high rank = the signal " + + "isn't strongly present in this matrix.", + rankingResult: + "Where each KNOWN marker ranks on its own. Every gene in the " + + "matrix is scored as a single-gene separator of the label on the " + + "engine's TRAIN split (~20k opaque columns), sorted strongest " + + "first; this panel pins the dataset's reference markers (p16 / " + + "cell_cycle for HNSC; MMR / immune for CRC) so you can see how " + + "recoverable the canonical biology is — a recovery diagnostic. " + + "The engine's actual picks (the genetic-programming top programs, their genes, " + + "their group scores, their confound survival) live in the Result " + + "panel above; the after-the-fact re-scoring of every explored " + + "group is the Coordinated modules panel below.", + rankingHighlighted: + "Rows pinned here are the dataset's KNOWN markers, revealed by " + + "name. p16 / cell_cycle (HNSC) or MMR / immune (CRC). Each row is " + + "one gene scored alone — a high rank (low %, near 1) = a strong " + + "standalone marker; a low rank (high %, far down the list) = a " + + "gene that only helps in combination with others. The winning " + + "program's constituent genes live in the Result panel above as " + + "part of its self-contained group view; they are deliberately " + + "NOT pinned in this known-marker recovery diagnostic.", + coherence: + "Off: the engine rewards whatever genes separate the label best, " + + "even a single lucky gene. On: it also rewards genes that move " + + "together across patients — a real co-expressed program — so it " + + "tends to surface coordinated modules (e.g. a cell-cycle program) " + + "rather than one sharp correlate. It names no pathway; it just " + + "prefers coordinated signals.", + diversity: + "Off: the population can collapse to near-clones early (the " + + "fitness curve's best and median lines meet and go flat). On: " + + "lowers selection pressure and injects fresh random programs " + + "each generation, so the population keeps exploring — watch the " + + "best-vs-median gap stay open longer. Held-out detection should " + + "be preserved or improved, not hurt.", + dslRates: + "How often the engine reaches for each OPTIONAL operator while " + + "growing a program. Each rate is in [0, 1]; setting one to 0 " + + "removes that operator from the search. " + + "Search is the heaviest operator (every Search node runs an " + + "inner gene-ranking) — set its rate to 0 for faster runs. " + + "Scalar share = the fraction of programs the engine seeds with a " + + "Scalar root (Associate / Effect) versus a Vector root.", + moduleRanking: + "Every gene group the engine tried during the search. Four sort " + + "lenses:\n\n" + + "• Genetic-programming fitness — the engine's OWN preference order, what the search " + + "actually picked (its cross-validated objective). The default " + + "sort.\n" + + "• Combined AUROC — the group's mean across its genes, scored on " + + "patients held out of training. A simpler yardstick.\n" + + "• Coherence — mean absolute pairwise correlation among the " + + "group's genes on TRAIN. High = the genes move together (a real " + + "module); low = an arbitrary set.\n" + + "• Synergy — combined held-out minus the best single gene's " + + "score. High = the GROUP beats its strongest constituent (a real " + + "additive effect); low / negative = a “best gene + passengers” " + + "pattern.\n\n" + + "The badged row is the winning program. The Survives flags show " + + "whether each group still separates the label inside the " + + "stratified confound subgroups (HNSC/HPV).\n\n" + + "Caveat: Combined AUROC / Coherence / Synergy re-score ~2,900 " + + "groups on the same small held-out set, so their very top values " + + "are optimistically biased (the luckiest of thousands). The Genetic-programming " + + "fitness order (and the badged winner) is the engine's own, " + + "cross-validated pick — trust it as the reliable choice.", + moduleSurvival: + "What “Survives” checks: whether a group still separates HPV when " + + "you take away a possible confounder — something that travels with " + + "HPV but isn't HPV biology.\n\n" + + "Site. HPV+ tumours are mostly in the oropharynx (back of the " + + "throat), so a gene could look like an \"HPV gene\" just by " + + "marking that location. The site check re-scores the group using " + + "only oropharynx patients (everyone the same location). The two " + + "numbers are full-cohort → oropharynx-only.\n\n" + + "Purity. A tumour sample is a mix of cancer cells and immune " + + "cells; HPV+ tumours carry more immune cells, so a gene could " + + "look like an \"HPV gene\" just by marking immune content. The " + + "purity check re-scores using only the \"purest\" (least-immune) " + + "tumours. It's usually \"—\" here because those tumours include " + + "almost no HPV+ patients, so there's nothing to test.\n\n" + + "✓ = held up when the confounder was held constant (likely real " + + "signal). ✗ = dropped past the tolerance (part of it was the " + + "confounder). — = couldn't test (too few patients).", + singleGeneRank: + "Each gene's rank as a SINGLE-gene separator of the active target " + + "(out of ~20,000). Low rank (near 1) = the gene separates the label " + + "on its own — a real alternate marker. High rank = it only helps in " + + "combination with others. AUROC for binary targets (MSI/HPV), signed " + + "Spearman (ρ) for TMB.", + hpvRank: + "Rank every gene by single-gene HPV+/HPV− AUROC on the engine's " + + "HNSC training split, descending (best separator = rank 1; " + + "orientation-agnostic, so a gene that goes UP or DOWN in HPV+ " + + "scores equally). Where each known HPV marker lands tells you " + + "whether the signal is even there: CDKN2A / cell-cycle genes near " + + "rank 1 = recoverable as single features; high rank = out-competed " + + "by other separators (the engine may still combine them).", +}; + +// ---------- Top-level page ------------------------------------------------- + +type Status = "idle" | "running" | "done" | "error"; + +// Two profiles: prefilter OFF is the default (integrity over speed) so +// we bump the GP budget; prefilter ON keeps the snappier 30/150 defaults. +const DEFAULTS_NO_PREFILTER: RunParams = { + generations: 60, + population: 300, + genes_per_set: 8, + max_sets: 2, + lambda: 0.005, + seed: 42, + prefilter_n: null, + permutations: 200, +}; + +const DEFAULTS_WITH_PREFILTER: RunParams = { + generations: 30, + population: 150, + genes_per_set: 8, + max_sets: 2, + lambda: 0.005, + seed: 42, + prefilter_n: 2000, + permutations: 200, +}; + +const DEFAULT_PARAMS: RunParams = { + generations: 60, + population: 300, + genes_per_set: 8, + max_sets: 2, + lambda: 0.005, + seed: 42, + prefilter_n: null, + permutations: 200, +}; + +// Engine-side defaults for the editable DSL injection rates and the +// scalar-share field. Mirror engine_v2/synthesize.py:DEFAULT_RATES + +// ramped_population's default scalar_share=0.20. Mandatory operators +// (Select / Reduce / Combine; Associate is the leftover of the +// Scalar branch) get no rate — they're structural. +type DslRates = { + split: number; + effect: number; + fitapply: number; + search: number; + scalar_share: number; +}; + +const DSL_DEFAULT_RATES: DslRates = { + split: 0.10, + effect: 0.40, + fitapply: 0.10, + search: 0.05, + scalar_share: 0.20, +}; + +const OBJECTIVE_PRESETS: Record = { + msi: { target: "msi", metric: "auroc" }, + tmb: { target: "tmb", metric: "correlation", direction: "neg" }, + none: { target: "none", metric: "structure" }, + hpv: { target: "hpv", metric: "auroc" }, +}; + +// Short y-axis titles, calibrated to fit the rotated centred label +// inside the chart without overflowing. The actual fitness numbers are +// signed (TMB) / [0,1] (MSI) / [-1,1] (unsup) — see engine_v2's +// V2Objective.fitness_label() for the verbose form used in logs. +const FITNESS_LABEL_BY_TARGET: Record = { + msi: "separation (AUROC — 0.5 coin-flip, 1.0 perfect)", + tmb: "neg. association with TMB", + none: "cluster separation (0–1)", + hpv: "separation (AUROC — 0.5 coin-flip, 1.0 perfect)", +}; + +// Per-target "this is a real result" thresholds. The ResultVerdict at +// the top of the Result panel turns green only when the winner's +// held-out clears its target's bar AND the permutation p is < 0.05. +// Defaults are tunable; bump if real-world runs ask. +const HOLDOUT_THRESHOLD: Record = { + msi: 0.75, + tmb: 0.30, + none: 0.30, + hpv: 0.75, +}; +const P_VALUE_SIGNIFICANT = 0.05; +const POSTHOC_MSI_ALIGN = 0.75; +const POSTHOC_HPV_ALIGN = 0.75; + +// ---------- Dataset registry -------------------------------------------- +// One registry entry per supported cancer / study. The engine, DSL, +// parameters, live view, population, program graph, and verdict are +// IDENTICAL across datasets — only the loader (in api/_prepare_lab_data) +// and the per-dataset config below swap. +interface ObjectiveCard { + key: Target; + label: string; + sub: string; +} + +interface DatasetEntry { + label: string; + longLabel: string; + objectives: ObjectiveCard[]; // ordered: first entry = default target + BiologyPanel: () => JSX.Element; + refSetKeys: ReferenceSetKey[]; +} + +// The "none" (Unsupervised) objective is intentionally absent from the +// UI registry — the backend capability + tests are intact (see +// UNSUP_OBJECTIVE), it's just not surfaced as a card. The frontend +// code paths for target==="none" stay working but unreachable from +// here. +const DATASET_REGISTRY: Record = { + coadread: { + label: "Colorectal", + longLabel: "Colorectal (TCGA COADREAD)", + objectives: [ + { key: "msi", label: "MSI separation", sub: "AUROC of MSI-H vs MSS" }, + { key: "tmb", label: "Mutation burden", sub: "negative correlation with TMB" }, + ], + BiologyPanel: MMRBiologyPanel, + refSetKeys: refSetsFor("coadread"), + }, + hnsc: { + label: "Head & Neck", + longLabel: "Head & Neck (TCGA HNSC)", + objectives: [ + { key: "hpv", label: "HPV detection", sub: "AUROC of HPV+ vs HPV−" }, + ], + BiologyPanel: HPVBiologyPanel, + refSetKeys: refSetsFor("hnsc"), + }, +}; + +// Per-target tooltip for the fitness-curve InfoTip. Replaces the old +// single TMB-centric TIPS.fitnessCurve string — now the tooltip mirrors +// the y-axis title's objective-awareness. +const FITNESS_TIP_BY_TARGET: Record = { + hpv: + "This shows how well the best program SEPARATES HPV+ from HPV− " + + "tumours, measured by AUROC: 0.5 = coin-flip (no separation), " + + "1.0 = perfect; higher = a cleaner split. We use AUROC because " + + "HPV+ is only ~15% of patients, so it isn't fooled by always " + + "guessing the majority.", + msi: + "This shows how well the best program SEPARATES the two subtypes " + + "(MSI-H vs MSS), measured by AUROC: 0.5 = coin-flip (no separation), " + + "1.0 = perfect; higher = a cleaner split. We use AUROC because MSI-H " + + "is only ~15% of patients, so it isn't fooled by always guessing the " + + "majority.", + tmb: + "This shows how strongly the program's score moves OPPOSITE to " + + "mutation burden — low score where mutations are high (the " + + "broken-spell-checker pattern). 0 = no relationship; higher = a " + + "stronger opposite-direction link; the top = score and mutation " + + "count move almost perfectly oppositely. The underlying correlation " + + "is negative (e.g. −0.49); we plot its strength (0.49) so up = better.", + none: + "There is no target here. This measures how cleanly the program's " + + "score splits patients into TWO groups: 0 ≈ no real split, 1 = two " + + "clean, well-separated clusters. After the run we check what that " + + "split lines up with on the named label this cohort carries — " + + "that's the post-hoc alignment in the Result panel.", +}; + +// Held-out card label per objective. Binary objectives ARE an AUROC; +// say so. TMB is a (signed Spearman) correlation; unsup is silhouette. +const HELD_OUT_LABEL_BY_TARGET: Record = { + msi: "Held-out AUROC", + hpv: "Held-out AUROC", + tmb: "Held-out (|spearman|)", + none: "Held-out (silhouette)", +}; + +// Held-out tooltip per objective — same data, but the explanation of +// what 'held-out' MEANS for the active metric differs. +const HELD_OUT_TIP_BY_TARGET: Record = { + msi: + "Held-out AUROC — how well the winner sorts MSI-H apart from MSS " + + "on patients it never saw during the search. AUROC is the chance " + + "the score ranks a random MSI-H patient above a random MSS " + + "patient: 0.5 = coin-flip, 1.0 = perfect. Tests the result isn't " + + "just memorising the training data.", + hpv: + "How well the winner separates HPV+ from HPV− on patients it " + + "never saw during the search. 0.5 = coin-flip, 1.0 = perfect. " + + "This is the honest test that it didn’t just memorise the " + + "training patients.", + tmb: + "Held-out |spearman| — the strength of the negative link between " + + "the winner's score and mutation burden on patients it never saw. " + + "0 = no link, higher = stronger (low score tracks high TMB). " + + "Not AUROC: TMB is a continuous label, so the metric is a " + + "correlation. Tests the result isn't just memorising training data.", + none: + "Held-out silhouette — how cleanly the winner splits unseen " + + "patients into two groups. Scale: −1 = no real split, ~1 = two " + + "very clean groups. The engine is told nothing about which two " + + "groups; the held-out score checks that the split it found holds " + + "up on new patients. Not AUROC: there's no label to compare to.", +}; + +// Permutation-p tooltip per objective — the null shape changes for +// unsupervised (random programs, no labels to shuffle). +const PERMUTATION_P_TIP_BY_TARGET: Record = { + msi: + "How often pure chance matches this score across many runs with " + + "the MSI labels shuffled. Small (e.g. below 0.05) = unlikely to " + + "be a fluke.", + hpv: + "How often a random program would score this well by luck. Small " + + "(e.g. below 0.05) = unlikely to be a fluke.", + tmb: + "How often pure chance matches this score across many runs with " + + "the TMB values shuffled. Small (e.g. below 0.05) = unlikely to " + + "be a fluke.", + none: + "How often a random gene-based program produces a split this clean " + + "on the same patients. Small (e.g. below 0.05) = the winner is " + + "unlikely to be a fluke (no labels are shuffled here because the " + + "unsupervised search never saw any).", +}; + +// Reference-set + post-hoc + discovered-axes tips swap with the active +// dataset, since the reference vocabulary and the named labels differ +// between colorectal (MMR/immune, MSI/TMB) and head & neck (p16 / +// cell-cycle, HPV). +const REFERENCE_SET_TIP_BY_DATASET: Record = { + coadread: + "A known gene list to check the answer against. MMR = the 4 DNA " + + "repair / 'spell-checker' genes (MLH1, MSH2, MSH6, PMS2); " + + "immune = immune-attack genes (CD8A, GZMA, PRF1).", + hnsc: + "A known gene list to check the answer against. p16 = CDKN2A — " + + "the canonical HPV+ surrogate marker pathologists use; cell-cycle " + + "= a standard set of DNA-replication / E2F-target genes that go " + + "UP when HPV-E7 wrecks the RB brake.", +}; + +const POSTHOC_TIP_BY_DATASET: Record = { + coadread: + "After the unsupervised search finishes, the discovered split is " + + "compared to the MSI and TMB labels the engine never saw. AUROC " + + "tells us whether the split tracks the MSI subtype; |spearman| " + + "with TMB tells us whether the score moves with mutation burden. " + + "The search itself was label-free.", + hnsc: + "After the unsupervised search finishes, the discovered split is " + + "compared to the HPV label the engine never saw. AUROC tells us " + + "whether the split tracks the HPV+ subtype: 0.5 = no relationship, " + + "1.0 = a perfect match. The search itself was label-free.", +}; + +const DISCOVERED_AXES_TIP_BY_DATASET: Record = { + coadread: + "After each unsupervised run, click \"Find next axis\" to peel the " + + "discovered split out of the data (linear residualisation — we " + + "subtract its signal from every gene) and search again. The chain " + + "is a ranked, blind list of independent directions in the gene-" + + "expression matrix — each named only afterwards by how well it " + + "lines up with MSI / TMB. The residualisation is LINEAR, so " + + "deeper non-linear structure may remain. The chain is in-memory " + + "only — it resets on server restart or a fresh Run.", + hnsc: + "After each unsupervised run, click \"Find next axis\" to peel the " + + "discovered split out of the data (linear residualisation — we " + + "subtract its signal from every gene) and search again. The chain " + + "is a ranked, blind list of independent directions in the gene-" + + "expression matrix — each named only afterwards by how well it " + + "lines up with HPV. The residualisation is LINEAR, so deeper non-" + + "linear structure may remain. The chain is in-memory only — it " + + "resets on server restart or a fresh Run.", +}; + +export default function Lab() { + // Default to Head & Neck + HPV detection on first load. Colorectal + // stays selectable; first-load just lands on the dataset whose + // detection objective is the live demo of the workflow. + const [dataset, setDatasetState] = useState("hnsc"); + const [target, setTarget] = useState( + DATASET_REGISTRY.hnsc.objectives[0].key, + ); + const [params, setParams] = useState(DEFAULT_PARAMS); + const [coherence, setCoherence] = useState(false); + const [diversity, setDiversity] = useState(false); + // Editable DSL injection rates. Only changed keys travel on the + // wire so the default run is byte-for-byte unchanged. The DslRates + // type is hoisted to module scope so the StabilityPanel (and any + // other consumer) can accept it as a prop. + const [dslRates, setDslRates] = useState({ ...DSL_DEFAULT_RATES }); + const [status, setStatus] = useState("idle"); + const [errorMsg, setErrorMsg] = useState(null); + const [runId, setRunId] = useState(null); + const [generations, setGenerations] = useState([]); + const [result, setResult] = useState(null); + const [operatorUsage, setOperatorUsage] = useState(null); + // Graph view state — single graph area, swapped by winner / candidate / + // pasted text. The string is a program_repr that the shared parser + // turns into a tree. + const [graphProgramRepr, setGraphProgramRepr] = useState(null); + const [graphSourceLabel, setGraphSourceLabel] = useState("winner"); + const [graphSelectedCandidateId, setGraphSelectedCandidateId] = useState(null); + // Textarea text lives at the Lab level so tile clicks AND the paste tool + // share the same controlled string. + const [pasteText, setPasteText] = useState(""); + const [revealData, setRevealData] = useState(null); + // Iterative-unsupervised discovery: the ordered chain of axes + // discovered so far (Axis 1, Axis 2, …). Each new unsup result — + // whether it's the first run or a "Find next axis" follow-up — is + // appended. A fresh Run resets the chain. + const [axes, setAxes] = useState<{ run_id: string; result: RunResult }[]>( + [], + ); + const esRef = useRef(null); + + useEffect(() => { + return () => { + esRef.current?.close(); + }; + }, []); + + // When a fresh result arrives, default the graph to the winning program. + useEffect(() => { + if (result) { + setGraphProgramRepr(result.winning.program_repr); + setPasteText(result.winning.program_repr); + setGraphSourceLabel("winner"); + setGraphSelectedCandidateId(null); + } + }, [result]); + + // Switching the cancer resets the chain + per-dataset state and + // snaps the objective to the new dataset's default (first card) so + // the request body never carries a stale (dataset, target) combo. + function setDataset(next: DatasetId) { + if (next === dataset) return; + setDatasetState(next); + const firstObjective = DATASET_REGISTRY[next].objectives[0].key; + setTarget(firstObjective); + setResult(null); + setOperatorUsage(null); + setGenerations([]); + setRevealData(null); + setRunId(null); + setAxes([]); + setGraphProgramRepr(null); + setPasteText(""); + setGraphSelectedCandidateId(null); + } + + async function launchRun( + opts: { residualize_against?: string[] } = {}, + ) { + const chained = + !!opts.residualize_against && opts.residualize_against.length > 0; + setStatus("running"); + setErrorMsg(null); + setGenerations([]); + setResult(null); + setOperatorUsage(null); + setRunId(null); + setRevealData(null); + setGraphProgramRepr(null); + setPasteText(""); + setGraphSelectedCandidateId(null); + // Fresh runs reset the chain; "Find next axis" preserves it. + if (!chained) setAxes([]); + try { + // Only the keys the user actually changed travel on the wire — + // an untouched DSL config reproduces the engine's defaults + // byte-for-byte. + const ratesDiff: Record = {}; + (Object.keys(DSL_DEFAULT_RATES) as (keyof DslRates)[]).forEach((k) => { + if (dslRates[k] !== DSL_DEFAULT_RATES[k]) { + ratesDiff[k] = dslRates[k]; + } + }); + const body: Parameters[0] = { + objective_spec: OBJECTIVE_PRESETS[target], + params, + engine: "v2", + dataset, + coherence, + diversity, + }; + if (Object.keys(ratesDiff).length > 0) body.rates_override = ratesDiff; + if (chained) body.residualize_against = opts.residualize_against; + const { run_id } = await postRun(body); + setRunId(run_id); + const es = new EventSource(`${API_URL}/runs/${run_id}/stream`); + esRef.current?.close(); + esRef.current = es; + es.addEventListener("generation", (ev) => { + const data = JSON.parse((ev as MessageEvent).data) as GenerationEvent; + setGenerations((prev) => [...prev, data]); + }); + es.addEventListener("done", (ev) => { + const data = JSON.parse((ev as MessageEvent).data) as RunResult; + setResult(data); + setStatus("done"); + // Peel-off chain: append every result so the user sees a + // ranked stack of axes — supervised objectives now too. + setAxes((prev) => [...prev, { run_id, result: data }]); + // Fetch the operator-usage payload now that the run is done. + // Cached server-side; opaque-safe (keywords + counts only). + // Non-fatal — the DSL vocabulary tiles just render without + // counts on failure. + getRunOperatorUsage(run_id) + .then((u) => setOperatorUsage(u)) + .catch(() => setOperatorUsage(null)); + es.close(); + }); + es.addEventListener("error", (ev) => { + try { + const data = JSON.parse((ev as MessageEvent).data) as { + detail?: string; + }; + setErrorMsg(data.detail ?? "stream error"); + } catch { + setErrorMsg("stream error"); + } + setStatus("error"); + es.close(); + }); + es.onerror = () => { + if (status === "running") { + // Only flag if we haven't received a clean done already. + setErrorMsg((m) => m ?? "lost connection to /stream"); + setStatus((s) => (s === "running" ? "error" : s)); + es.close(); + } + }; + } catch (e) { + setErrorMsg(e instanceof Error ? e.message : String(e)); + setStatus("error"); + } + } + + async function start() { + await launchRun(); + } + + async function findNextAxis() { + await launchRun({ + residualize_against: axes.map((a) => a.run_id), + }); + } + + const fitnessLabel = FITNESS_LABEL_BY_TARGET[target]; + + return ( + +
+
+

+ OncoDSL Lab +

+

+ The engine hunts for a gene-expression pattern that predicts a + cancer trait — blind: gene names are hidden as codes so it + can’t lean on what we already know. You set the goal; it + evolves a small program to hit it; then we reveal the genes it + chose and check them against known biology. +

+
+ +
+ + + + + + {/* + DSL vocabulary moved ABOVE the Run section now that its rate + inputs are interactive — they're config the user edits + before pressing Run. + */} + + + {operatorUsage && ( +

+ How often the engine reached for each building block across + the whole search. Select and Reduce are the backbone; the + rest show up only where they helped. +

+ )} + + + + + + {runId && status !== "idle" && ( + + { + setGraphProgramRepr(c.program_repr); + setPasteText(c.program_repr); + setGraphSourceLabel(`candidate ${c.id}`); + setGraphSelectedCandidateId(c.id); + }} + /> + + )} + + + {graphProgramRepr ? ( + + ) : ( +
+ No program drawn yet. +
+ )} +
+ { + setGraphProgramRepr(repr); + setGraphSourceLabel(label); + setGraphSelectedCandidateId(null); + }} + /> +
+ + + {result && ( + + )} + {result && target !== "none" && ( + + )} + {result && target !== "none" && coherence && runId && ( + + )} + {result && target === "none" && ( + + )} + {/* + DiscoveredAxes (peel-off chain) is no longer rendered. + Peel-off only makes sense for the unsupervised objective — + on supervised HPV / MSI / TMB runs "Find next axis" + re-finds the same signal and the "aligns with MSI / TMB" + framing is nonsensical. The unsupervised objective isn't + a UI objective anymore, so this never fires. Gate kept as + `target === "none"` for documentation; the component + definition + backend chain remain untouched in case we + want to surface it again behind a developer flag. + */} + {target === "none" && axes.length > 0 && ( + + )} +
+
+ + ); +} + +// --------------------------------------------------------------------------- +// Iterative unsupervised discovery: the "peel off axes" stack. +// --------------------------------------------------------------------------- + +function DiscoveredAxes({ + axes, + status, + dataset, + onFindNext, +}: { + axes: { run_id: string; result: RunResult }[]; + status: Status; + dataset: DatasetId; + onFindNext: () => void; +}) { + const busy = status === "running"; + return ( + +
    + {axes.map((a, i) => ( + + ))} +
+
+ + + Chains within this session only. Start a fresh Run to reset. + +
+ + ); +} + +function AxisCard({ index, result }: { index: number; result: RunResult }) { + const w = result.winning; + const target = result.objective_spec.target as Target; + const threshold = HOLDOUT_THRESHOLD[target]; + const generalises = + Number.isFinite(w.holdout_score) && w.holdout_score >= threshold; + const significant = + Number.isFinite(w.permutation_p) && w.permutation_p < P_VALUE_SIGNIFICANT; + const success = generalises && significant; + const posthoc = result.posthoc; + const msi = posthoc?.msi_auroc; + const tmb = posthoc?.tmb_abs_spearman; + + return ( +
  • +
    +
    Axis {index}
    +
    + held-out {fmtFit(w.holdout_score, 3)} · p {fmtFit(w.permutation_p, 4)} + {!success && ( + (not above the verdict threshold) + )} +
    +
    +
    + {w.gene_ids.length > 0 ? w.gene_ids.join(", ") : "(no genes)"} +
    +
    + Aligns with{" "} + MSI{" "} + {msi != null && Number.isFinite(msi) ? ( + <> + at AUROC {fmtFit(msi, 3)} + {msi >= POSTHOC_MSI_ALIGN && ( + — matches the subtype + )} + + ) : ( + — (too few labels) + )} + ; TMB |spearman|{" "} + {tmb != null && Number.isFinite(tmb) ? ( + {fmtFit(tmb, 3)} + ) : ( + — + )} + . +
    +
    • + ); +} + +// Optional-operator rate-input keys recognised by the DSL panel. +// Mandatory operators (Select / Reduce / Combine; Associate is the +// leftover of the Scalar branch) have no editable rate. +type DslRateKey = "split" | "effect" | "fitapply" | "search"; + +function DSLVerbCard({ + name, + hint, + usage, + maxUses, + nCandidates, + rateKey, + rateValue, + rateDefault, + onRate, + structuralNote, +}: { + name: string; + hint: string; + usage: { total_uses: number; programs_using: number } | null; + maxUses: number; + nCandidates: number; + // When `rateKey` is set the tile renders a rate input; otherwise + // it shows `structuralNote` (always-on operators). + rateKey?: DslRateKey; + rateValue?: number; + rateDefault?: number; + onRate?: (k: DslRateKey, v: number) => void; + structuralNote?: string; +}) { + const showUsage = usage != null && nCandidates > 0; + const barFrac = + showUsage && maxUses > 0 ? usage!.total_uses / maxUses : 0; + const sharePct = + showUsage && nCandidates > 0 + ? Math.round((usage!.programs_using / nCandidates) * 100) + : 0; + const showRate = rateKey != null && onRate != null; + const isDefault = + showRate && rateValue != null && rateDefault != null + && Math.abs(rateValue - rateDefault) < 1e-9; + return ( +
    +
    +
    + {name} +
    + {showRate ? ( + + ) : structuralNote ? ( + {structuralNote} + ) : null} +
    +
    {hint}
    + {showUsage && ( + <> +
    + used{" "} + + {usage!.total_uses.toLocaleString()} + + × — in{" "} + + {sharePct}% + {" "} + of programs +
    +
    +
    0 ? "#3A6B7E" : "#C9CCC4", + transition: "width 200ms ease", + }} + /> +
    + + )} +
    + ); +} + +function DSLVocabularyTiles({ + usage, + rates, + onRates, +}: { + usage: OperatorUsage | null; + rates: { + split: number; effect: number; fitapply: number; search: number; + scalar_share: number; + }; + onRates: React.Dispatch>; +}) { + // Mandatory (structural) operators come first; the engine always + // closes Vector to Reduce(Select(M, …), agg) and Combine glues + // scores together, so they have no rate. + const STRUCTURAL: { name: string; hint: string }[] = [ + { name: "Select", hint: "Pick specific genes." }, + { name: "Reduce", hint: "Combine those genes into one score per patient (average, max, etc.)." }, + { name: "Combine", hint: "Merge two scores into one." }, + { name: "Associate", hint: "Ask how strongly the score correlates with the target — the default of the two correlation-style options." }, + ]; + const OPTIONAL: { name: string; hint: string; key: DslRateKey }[] = [ + { name: "Split", key: "split", hint: "Split patients into two subgroups, score each, then bring them back together." }, + { name: "Effect", key: "effect", hint: "Same correlation as Associate, but after removing the effect of things like stage, age, sex, and race." }, + { name: "Fit/Apply", key: "fitapply", hint: "Fit a small model that maps the score to the target, then use it." }, + { name: "Search", key: "search", hint: "Runs a tiny gene-ranker inside the program (up to 4 genes, from at most 200 candidates). Rate 0 turns it off." }, + ]; + const byName = new Map(); + if (usage) for (const op of usage.operators) byName.set(op.name, op); + const maxUses = usage + ? usage.operators.reduce((m, op) => Math.max(m, op.total_uses), 0) + : 0; + const nCandidates = usage?.n_candidates ?? 0; + + function setRate(k: DslRateKey | "scalar_share", v: number) { + onRates((prev) => ({ ...prev, [k]: v })); + } + + // For the "Set all equal" button: split the (default) sum of the + // four optional rates evenly across them so the average grammar + // mix stays near the defaults. (Default sum = 0.10 + 0.40 + 0.10 + // + 0.05 = 0.65 → ~0.16 each.) + const equalEach = + (DSL_DEFAULT_RATES.split + DSL_DEFAULT_RATES.effect + + DSL_DEFAULT_RATES.fitapply + DSL_DEFAULT_RATES.search) / 4; + + const isDefault = ( + rates.split === DSL_DEFAULT_RATES.split + && rates.effect === DSL_DEFAULT_RATES.effect + && rates.fitapply === DSL_DEFAULT_RATES.fitapply + && rates.search === DSL_DEFAULT_RATES.search + && rates.scalar_share === DSL_DEFAULT_RATES.scalar_share + ); + + return ( + <> +
    + {STRUCTURAL.map((v) => ( + + ))} + {OPTIONAL.map((v) => ( + setRate(k, val)} + /> + ))} +
    + +
    + + + how many starting programs are built around Associate / + Effect (which score a gene set by how well it correlates + with the target) instead of giving each patient a score. + 0.20 ≈ 1 in 5; 0 = never. + + + + + + +
    + + ); +} + +function buildRevealMap( + data: EvaluateResponse | null, +): Record | undefined { + if (!data) return undefined; + const out: Record = {}; + for (const row of data.revealed) { + out[row.id] = { symbol: row.symbol, matched: row.matched }; + } + return out; +} + +// ---------- Building blocks ------------------------------------------------ + +function SectionCard({ + title, + titleTip, + subtitle, + children, +}: { + title: string; + titleTip?: string; + subtitle?: string; + children: React.ReactNode; +}) { + return ( +
    +
    +

    + {title} + {titleTip && } +

    + {subtitle && ( +

    {subtitle}

    + )} +
    + {children} +
    + ); +} + +// Top-level cancer / problem selector. Mirrors the segmented style of +// the Objective control. Swapping a cancer swaps the objective options +// + biology panel + reference sets via the dataset registry; the engine +// underneath is identical. +function CancerSelector({ + dataset, + onDataset, +}: { + dataset: DatasetId; + onDataset: (d: DatasetId) => void; +}) { + const entries = Object.entries(DATASET_REGISTRY) as [DatasetId, DatasetEntry][]; + return ( + +
    = 3 ? "sm:grid-cols-3" : "sm:grid-cols-2") + } + > + {entries.map(([id, entry]) => { + const selected = id === dataset; + return ( + + ); + })} +
    +     + ); +} + +function BiologyPanelForDataset({ dataset }: { dataset: DatasetId }) { + const Panel = DATASET_REGISTRY[dataset].BiologyPanel; + return ; +} + +function ObjectiveBuilder({ + dataset, + target, + onTarget, +}: { + dataset: DatasetId; + target: Target; + onTarget: (t: Target) => void; +}) { + const options = DATASET_REGISTRY[dataset].objectives; + const cols = options.length >= 3 ? "sm:grid-cols-3" : "sm:grid-cols-2"; + + return ( + +
    + {options.map((o) => { + const selected = o.key === target; + const paramKey = ( + o.key === "none" ? "obj_unsupervised" : `obj_${o.key}` + ) as ParamKey; + return ( +
    + + + + +
    + ); + })} +
    +     + ); +} + +type NumField = { + key: Exclude; + label: string; + min: number; + max: number; + step: number; +}; + +function ParamsControls({ + params, + onParams, + coherence, + onCoherence, + diversity, + onDiversity, +}: { + params: RunParams; + onParams: (p: RunParams) => void; + coherence: boolean; + onCoherence: (c: boolean) => void; + diversity: boolean; + onDiversity: (d: boolean) => void; +}) { + const fields: NumField[] = [ + { key: "generations", label: "Generations", min: 2, max: 1000, step: 1 }, + { key: "population", label: "Population", min: 10, max: 3000, step: 10 }, + { key: "genes_per_set", label: "Genes / set", min: 2, max: 8, step: 1 }, + { key: "max_sets", label: "Max sets", min: 1, max: 2, step: 1 }, + { key: "lambda", label: "λ (size penalty)", min: 0, max: 0.05, step: 0.001 }, + { key: "seed", label: "Seed", min: 0, max: 9999, step: 1 }, + { key: "permutations", label: "Permutations", min: 10, max: 500, step: 10 }, + ]; + const prefilterOff = params.prefilter_n === null; + + function setPrefilterOff() { + onParams({ ...DEFAULTS_NO_PREFILTER, seed: params.seed }); + } + function setPrefilterOn() { + onParams({ ...DEFAULTS_WITH_PREFILTER, seed: params.seed }); + } + + return ( + + + {prefilterOff && ( +

    + Searching all genes is a bigger space — higher population/generations + help, and results vary by seed, so try a few seeds and compare. +

    + )} +
    + {fields.map((f) => ( + + ))} + +
    + + Prefilter top-N + + +
    + + +
    + {!prefilterOff && ( + + onParams({ + ...params, + prefilter_n: Number.parseInt(e.target.value, 10), + }) + } + className="mt-2 rounded-md border border-border bg-white px-3 py-2 text-sm text-ink focus:border-accent focus:outline-none" + /> + )} +
    +
    +
    + onCoherence(e.target.checked)} + className="mt-0.5 h-4 w-4 cursor-pointer accent-accent" + /> + +
    +
    + onDiversity(e.target.checked)} + className="mt-0.5 h-4 w-4 cursor-pointer accent-accent" + /> + +
    + {/* + The Search-operator toggle moved into the DSL vocabulary + panel as an editable rate (rate = 0 means disabled). See + below. + */} +     + ); +} + +function RunBar({ + status, + onRun, + runId, + error, +}: { + status: Status; + onRun: () => void; + runId: string | null; + error: string | null; +}) { + const isRunning = status === "running"; + return ( + +
    + +
    + {runId && ( + + + run id + + + {runId} + · + + status + + + {status} + + )} +
    +
    + {error && ( +

    + {error} +

    + )} +     + ); +} + +// Centered Y-axis label: rotated placed at the plot's vertical +// midpoint via the YAxis viewBox. Default Recharts label sits at the top +// of the axis with `position="insideLeft"`; this puts it where readers +// expect — beside the middle of the axis. +function CenteredYAxisLabel({ + value, + viewBox, +}: { + value: string; + // Recharts injects viewBox automatically when the component is passed + // as a label. + viewBox?: { x: number; y: number; width: number; height: number }; +}) { + if (!viewBox) return null; + const cx = viewBox.x - 4; + const cy = viewBox.y + viewBox.height / 2; + return ( + + {value} + + ); +} + +// Custom legend rendered OUTSIDE the chart SVG so it lives on its own +// row, below the X-axis title. Swatches mirror the line styles exactly: +// solid accent for best, dashed muted for median. +function ChartLegend() { + return ( +
    + + + best + + + + generation median + +
    + ); +} + +function LiveView({ + generations, + fitnessLabel, + target, + status, +}: { + generations: GenerationEvent[]; + fitnessLabel: string; + target: Target; + status: Status; +}) { + if (generations.length === 0 && status !== "running") return null; + const series = generations + .map((g) => ({ + generation: g.generation, + best: + typeof g.best_fitness === "number" && Number.isFinite(g.best_fitness) + ? Number(g.best_fitness.toFixed(4)) + : null, + median: + typeof g.median_fitness === "number" && Number.isFinite(g.median_fitness) + ? Number(g.median_fitness.toFixed(4)) + : null, + })); + const latest = generations[generations.length - 1]; + return ( + +
    +

    + Fitness curve + +

    + + best vs median + + +
    +
    + + + + + } + domain={["auto", "auto"]} + /> + + + + + +
    +
    generation
    + + + {latest && ( +
    + + Generation {latest.generation} · best{" "} + + {fmtFit(latest.best_fitness, 4)} + {" "} + · median{" "} + + {fmtFit(latest.median_fitness, 4)} + + + + gene codes + + +
    + )} +     + ); +} + +// Plain-reading verdict at the top of the Result panel. Green when the +// run cleared its objective's held-out bar AND beat chance (p < 0.05); +// amber otherwise. Unsupervised appends the post-hoc MSI-alignment read. +function ResultVerdict({ result }: { result: RunResult }) { + const target = result.objective_spec.target as Target; + const holdout = result.winning.holdout_score; + const p = result.winning.permutation_p; + const posthoc = result.posthoc; + + const threshold = HOLDOUT_THRESHOLD[target]; + const generalises = Number.isFinite(holdout) && holdout >= threshold; + const significant = Number.isFinite(p) && p < P_VALUE_SIGNIFICANT; + const success = generalises && significant; + const pctChance = Number.isFinite(p) ? Math.round((p as number) * 100) : null; + + const callout = success ? ( +
    + + ✓ + +

    + Real result: it holds up on unseen patients ({fmtFit(holdout, 3)}) + and beats chance (p {fmtFit(p, 4)}). + {target === "none" && } +

    +
    + ) : ( +
    + + ! + +

    + This run found nothing reliable: held-out {fmtFit(holdout, 3)}{" "} + (fell apart on unseen patients) · p {fmtFit(p, 4)}{" "} + {pctChance != null && ( + <>(a random program beats it ~{pctChance}% of the time) + )} + . Don’t read the genes as a discovery. +

    +
    + ); + + return ( +
    + {callout} + +
    + ); +} + +// "What does this mean?" — expandable plain-language explanation under +// the verdict headline. Closed by default; mirrors ParameterFlow's +// disclosure pattern (button + chevron + aria-expanded + useId panel). +// Content branches on objective_spec.target since MSI/TMB are +// single-question (held-out IS the answer) while unsupervised is +// two-question (split is real vs. is it MSI). +function VerdictBreakdown({ + target, + holdout, + p, + posthoc, + success, + generalises, + significant, +}: { + target: Target; + holdout: number; + p: number; + posthoc: RunResult["posthoc"] | undefined; + success: boolean; + generalises: boolean; + significant: boolean; +}) { + const [open, setOpen] = useState(false); + const panelId = useId(); + + return ( +
    + + {open && ( +
    + {target === "none" ? ( + + ) : target === "tmb" ? ( + + ) : target === "hpv" ? ( + + ) : ( + + )} +
    + )} +
    + ); +} + +function MSIBreakdownContent({ + holdout, + p, + generalises, + significant, + success, +}: { + holdout: number; + p: number; + generalises: boolean; + significant: boolean; + success: boolean; +}) { + const pctChance = Number.isFinite(p) ? Math.round(p * 100) : null; + return ( + <> +

    Is this result real and useful?

    +
      +
    • + Held-out AUROC{" "} + ({fmtFit(holdout, 3)}). On patients it never trained on, how well + the score tells MSI-H from MSS. AUROC is the chance the score + ranks a random MSI-H patient above a random MSS patient: 0.5 = + coin-flip, 1.0 = perfect.{" "} + + {generalises + ? "Strong." + : "Weak — barely better than guessing."} + +
      • +
    • + Beats chance (p {fmtFit(p, 4)}). + Only about {pctChance ?? "—"}% of shuffled-label runs match this — so{" "} + + {significant + ? "unlikely a fluke." + : "it could be luck."} + +
      • +
    +

    + →{" "} + {success + ? "Real and useful — it separates MSI-H from MSS on patients it never saw." + : "Not reliable — don’t read the genes as a discovery."} +

    +

    + This is a detection result. Whether the engine found the known + MMR / immune marker genes is a separate question — see Reveal + & evaluate below. +

    + + ); +} + +function HPVBreakdownContent({ + holdout, + p, + generalises, + significant, + success, +}: { + holdout: number; + p: number; + generalises: boolean; + significant: boolean; + success: boolean; +}) { + const pctChance = Number.isFinite(p) ? Math.round(p * 100) : null; + return ( + <> +

    Is this result real and useful?

    +
      +
    • + Held-out AUROC{" "} + ({fmtFit(holdout, 3)}). On patients it never trained on, how well + the score tells HPV+ from HPV− tumours. AUROC is the chance the + score ranks a random HPV+ patient above a random HPV− patient: + 0.5 = coin-flip, 1.0 = perfect.{" "} + + {generalises + ? "Strong." + : "Weak — barely better than guessing."} + +
      • +
    • + Beats chance (p {fmtFit(p, 4)}). + Only about {pctChance ?? "—"}% of shuffled-label runs match this — so{" "} + + {significant + ? "unlikely a fluke." + : "it could be luck."} + +
      • +
    +

    + →{" "} + {success + ? "Real and useful — it separates HPV+ from HPV− tumours on patients it never saw." + : "Not reliable — don’t read the genes as a discovery."} +

    +

    + This is a detection result. Whether the engine found the known + marker genes (CDKN2A/p16, the cell-cycle program) is a separate + question — see Reveal & evaluate below for the gene overlap + and each revealed gene's individual rank. +

    + + ); +} + +function TMBBreakdownContent({ + holdout, + p, + generalises, + significant, + success, +}: { + holdout: number; + p: number; + generalises: boolean; + significant: boolean; + success: boolean; +}) { + const pctChance = Number.isFinite(p) ? Math.round(p * 100) : null; + return ( + <> +

    Is this result real and useful?

    +
      +
    • + Held-out association{" "} + ({fmtFit(holdout, 3)}). On patients it never trained on, the + strength of the negative link between the score and mutation + burden. Scale: 0 = no link, higher = stronger; ≥ 0.30 = a real + link.{" "} + + {generalises + ? "Strong." + : "Weak — barely better than guessing."} + +
      • +
    • + Beats chance (p {fmtFit(p, 4)}). + Only about {pctChance ?? "—"}% of shuffled-label runs match this — so{" "} + + {significant + ? "unlikely a fluke." + : "it could be luck."} + +
      • +
    +

    + →{" "} + {success + ? "Real and useful." + : "Not reliable — don’t read the genes as a discovery."} +

    + + ); +} + +function UnsupBreakdownContent({ + holdout, + p, + posthoc, + success, +}: { + holdout: number; + p: number; + posthoc: RunResult["posthoc"] | undefined; + success: boolean; +}) { + const pctChance = Number.isFinite(p) ? Math.round(p * 100) : null; + const hpv = posthoc?.hpv_auroc; + const msi = posthoc?.msi_auroc; + const tmb = posthoc?.tmb_abs_spearman; + const hpvKnown = hpv != null && Number.isFinite(hpv); + const msiKnown = msi != null && Number.isFinite(msi); + // HNSC unsup carries hpv; coadread unsup carries msi+tmb. Each cohort + // shows the alignment question that's actually answerable. + const showHPV = hpvKnown; + const hpvAligned = hpvKnown && (hpv as number) >= POSTHOC_HPV_ALIGN; + const msiAligned = msiKnown && (msi as number) >= POSTHOC_MSI_ALIGN; + + return ( + <> +

    + This is answering two separate questions. +

    + +

    1) Is the split real?

    +
      +
    • + Holds up on unseen patients{" "} + ({fmtFit(holdout, 3)}). The engine found the split on one set of + patients, then it still cleanly separated a different set it never + saw. Scale: −1 = no real split, ~1 = two very clean, + well-separated groups. +
      • +
    • + Beats chance (p {fmtFit(p, 4)}). + Only about {pctChance ?? "—"}% of random programs produce a split + this clean — so it’s very unlikely to be a fluke. +
      • +
    +

    + → Together:{" "} + {success + ? "yes, it found a real, reproducible way to split patients into two groups." + : "no — the split didn’t hold up on unseen patients or could be luck. Don’t read the genes as a discovery."} +

    + + {showHPV ? ( + <> +

    + 2) Is that split the HPV+ subtype? +

    +
      +
    • + Match to HPV:{" "} + {fmtFit(hpv, 3)}. Scale: 0.5 = no relationship (coin-flip), + 1.0 = a perfect match.{" "} + + {hpvAligned + ? "So this split largely IS the HPV+ subtype — the engine rediscovered the viral signature blind." + : "So this split is NOT the HPV+ subtype — it’s a real grouping about something else (see the revealed genes)."} + +
      • +
    + + ) : ( + <> +

    2) Is that split the MSI subtype?

    +
      +
    • + Match to MSI:{" "} + {msiKnown ? fmtFit(msi, 3) : "—"}. Scale: 0.5 = no relationship + (coin-flip), 1.0 = a perfect match.{" "} + {msiKnown ? ( + + {msiAligned + ? "So this split largely IS MSI — the engine rediscovered the subtype blind." + : "So this split is NOT the MSI subtype — it’s a real grouping about something else (see the revealed genes)."} + + ) : ( + + (Too few held-out labels to check alignment.) + + )} +
      • +
    • + (Match to mutation burden, TMB:{" "} + {tmb != null && Number.isFinite(tmb) ? fmtFit(tmb, 3) : "—"} — 0 = + unrelated, 1 = move together perfectly.) +
      • +
    + + )} + +

    + Key idea: a split can be completely real and still be about a + different thing than you hoped — like sorting a crowd cleanly by + hair colour when you wanted left- vs right-handed. +

    + + ); +} + +function UnsupAlignmentTail({ + posthoc, +}: { + posthoc: RunResult["posthoc"] | undefined; +}) { + if (!posthoc) return null; + // Pick whichever supervised label the cohort actually carries. + // HNSC unsup runs report hpv_auroc; coadread unsup runs report msi_auroc. + const hpv = posthoc.hpv_auroc; + if (hpv != null && Number.isFinite(hpv)) { + if (hpv >= POSTHOC_HPV_ALIGN) { + return ( + <> + {" "}and the blind split matches HPV (AUROC {fmtFit(hpv, 3)}) — it + rediscovered the viral subtype. + + ); + } + return ( + <> + {" "}but the split doesn’t line up with HPV (AUROC{" "} + {fmtFit(hpv, 3)}). + + ); + } + const auroc = posthoc.msi_auroc; + if (auroc == null || !Number.isFinite(auroc)) { + return <> (too few held-out labels to check alignment); + } + if (auroc >= POSTHOC_MSI_ALIGN) { + return ( + <> + {" "}and the blind split matches MSI (AUROC {fmtFit(auroc, 3)}) — it + rediscovered the subtype. + + ); + } + return ( + <> + {" "}but the split doesn’t line up with MSI (AUROC{" "} + {fmtFit(auroc, 3)}). + + ); +} + +function ResultPanel({ + result, + dataset, + target, + runId, + coherence, + diversity, + params, + dslRates, +}: { + result: RunResult; + dataset: DatasetId; + target: Target; + runId: string | null; + coherence: boolean; + diversity: boolean; + params: RunParams; + dslRates: DslRates; +}) { + const w = result.winning; + const b = result.baseline; + const t = result.objective_spec.target as Target; + // The merged Groups table is gated to coherence-on supervised runs + // — same gate uses. Only point at it when it + // will actually render. + const pointToGroups = target !== "none" && coherence; + + return ( + + +
    + + + {b ? ( + + ) : ( + + )} + +
    + {pointToGroups && ( +

    + The winning program is badged in{" "} + + Groups the engine explored + {" "} + below — that panel is the engine’s actual picks + ranked by how the search judged them, and each row shows + its genes and where each gene ranks on its own. +

    + )} +
    +
    +
    + Winning program + +
    + +
    +
    {w.program_repr}
    +
    + {w.feature_sets && w.feature_sets.length > 0 && ( +
    + feature sets:{" "} + + {w.feature_sets.map((s) => `[${s.join(", ")}]`).join(" ")} + +
    + )} + + {result.posthoc && result.objective_spec.target === "none" && ( + + )} + + {dataset === "hnsc" && target === "hpv" && runId && ( + + )} + {dataset === "hnsc" && target === "hpv" && ( + + )} +     + ); +} + + +// Post-hoc alignment block for an unsupervised run: show how the +// discovered cluster split lines up with the MSI label and TMB, +// computed AFTER the GP finished by the api worker (engine remains +// blind during the search). +function PosthocAlignment({ posthoc }: { posthoc: NonNullable }) { + // HNSC unsup runs carry hpv_auroc; coadread unsup runs carry msi / + // tmb. Pick the alignment that has actual data — neither cohort has + // both, so showing the empty one is just noise. + const showHPV = posthoc.hpv_auroc != null && Number.isFinite(posthoc.hpv_auroc); + const tipKey: DatasetId = showHPV ? "hnsc" : "coadread"; + + return ( +
    +
    + Post-hoc alignment + +
    + {showHPV ? ( +

    + The discovered 2-cluster split aligns with{" "} + HPV at AUROC{" "} + {fmtFit(posthoc.hpv_auroc, 3)}. +

    + ) : ( +

    + The discovered 2-cluster split aligns with{" "} + MSI{" "} + {posthoc.msi_auroc != null ? ( + <> + at AUROC{" "} + {fmtFit(posthoc.msi_auroc, 3)} + + ) : ( + — (too few held-out labels) + )} + ; the score’s |spearman| with{" "} + TMB is{" "} + {posthoc.tmb_abs_spearman != null ? ( + {fmtFit(posthoc.tmb_abs_spearman, 3)} + ) : ( + — (insufficient TMB) + )} + . +

    + )} +

    + {posthoc.n_holdout} held-out patients · labels were never visible to + the engine during search. +

    +
    + ); +} + + +// Independent-cohort validation panel — GSE65858 transfer test for +// HNSC/HPV runs. Fetches /runs/{id}/transfer once the run is done; +// renders a green "holds on strangers" verdict callout (or an amber +// "found nothing reliable" one), three metric cards, an amber +// cross-platform pill, a three-step "how this works" strip, an airgap +// footnote, and one honest note. Only surfaces symbols that were +// already revealed on the blind side (the winner's own genes). +function ExternalValidation({ runId }: { runId: string }) { + const [data, setData] = useState(null); + const [error, setError] = useState(null); + const [loading, setLoading] = useState(true); + useEffect(() => { + let cancelled = false; + setLoading(true); + setError(null); + setData(null); + getRunTransfer(runId) + .then((d) => { + if (!cancelled) setData(d); + }) + .catch((e) => { + if (!cancelled) setError(e instanceof Error ? e.message : String(e)); + }) + .finally(() => { + if (!cancelled) setLoading(false); + }); + return () => { + cancelled = true; + }; + }, [runId]); + + return ( +
    +
    + Independent-cohort validation + +
    +

    + The winning pattern was found blind on TCGA. Here it’s + applied, unchanged, to a completely separate group of head & + neck patients — a different country, hospital, and measuring + machine. +

    + + {loading && ( +

    running the transfer test…

    + )} + {!loading && (error || !data) && ( +

    + Couldn’t validate externally: {error ?? "unknown error"}. + The main result stands — this extra check is available once + the GSE65858 cohort is built (see + + data_pipeline/build_gse65858 + + ). +

    + )} + + {!loading && data && data.n_found === 0 && ( +

    + Couldn’t validate externally — none of the winner’s + genes are measured on GSE65858’s platform (Illumina + HumanHT-12 v4 microarray). The main result stands; this extra + check is only meaningful when at least some of the winner’s + genes carry over. +

    + )} + + {!loading && data && data.n_found > 0 && ( + + )} +
    + ); +} + +function ExternalValidationBody({ data }: { data: TransferResult }) { + const isReal = + data.auroc != null + && data.p != null + && data.auroc >= 0.75 + && data.p < 0.05; + const pPct = + data.p != null && Number.isFinite(data.p) + ? Math.max(0.01, Math.min(99.99, data.p * 100)) + : null; + + return ( + <> +
    + + {isReal ? "✓" : "?"} + +

    + {isReal ? ( + <> + It holds on strangers.{" "} + The pattern still separates HPV+ from HPV− in an + independent cohort — AUROC{" "} + {fmtFit(data.auroc, 3)}, + and only {pPct != null ? pPct.toFixed(1) : "—"}% of random + labellings do this well (p{" "} + {fmtFit(data.p, 3)}). + + ) : ( + <> + Not confirmed on strangers.{" "} + On the independent cohort the pattern reached AUROC{" "} + {fmtFit(data.auroc, 3)} + {data.p != null && ( + <> + {" "}with p{" "} + {fmtFit(data.p, 3)} + + )}{" "} + — below the “holds up” bar. The main result + stands; treat the transfer step as inconclusive here. + + )} +

    +
    + +
    + + + +
    + +
    + Validated across two platforms.{" "} + TCGA RNA-seq → GSE65858 microarray. Agreement across different + machines is a stronger result, not a weaker one. +
    + +
    +
    + How this validation works +
    +
    + {[ + { + n: "1", + head: "reveal", + body: "Unmask only the winner's handful of genes — the one thing that crosses over.", + }, + { + n: "2", + head: "match & level", + body: "Find those genes in GSE65858 and put them on a common scale, so RNA-seq vs microarray doesn't matter.", + }, + { + n: "3", + head: "score", + body: "Average them per patient and check how well that separates HPV+ from HPV− — against the virus-typed labels.", + }, + ].map((step) => ( +
    +
    + {step.n} · {step.head} +
    +
    {step.body}
    +
    + ))} +
    +
    + +
    + 🔒 +

    + The airgap holds. The + engine never sees this cohort. Only the winning gene names + cross over, once, at the end — so a result here is genuine + rediscovery, not a lookup. +

    +
    + +

    + One honest note. This checks the gene signature the + winner chose. For a simple winner that is the whole program; + for a multi-part one it’s the gene set rather than the + exact arithmetic — which, across two platforms, is the more + honest thing to measure anyway. +

    + + ); +} + + +// --------------------------------------------------------------------------- +// Stability across seeds. Reruns the HPV pipeline across a list of +// seeds and shows (a) how tightly the result holds and (b) which +// genes recur across independent searches. Frontend-only — reuses +// postRun / SSE / getRunResult / getRunTransfer / getFullRankDiagnostic; +// no new endpoint. Sequential (backend runs one at a time). Gated to +// HNSC/HPV by the caller. +// --------------------------------------------------------------------------- + +const STABILITY_DEFAULT_SEEDS = [1, 3, 7, 11, 13, 17, 23, 29]; + +type SeedRow = { + seed: number; + holdout: number | null; + permutation_p: number | null; + symbols: string[]; + transfer: TransferResult | null; + error: string | null; +}; + +type StabilityStatus = "idle" | "running" | "done" | "error"; + +function parseSeedList(input: string): number[] { + return Array.from( + new Set( + input + .split(/[\s,]+/) + .map((s) => s.trim()) + .filter((s) => s.length > 0) + .map((s) => Number(s)) + .filter((n) => Number.isFinite(n) && Number.isInteger(n) && n >= 0 && n < 1_000_000_000), + ), + ); +} + +/** Fire a run's SSE stream and resolve on ``done``. Does NOT touch the + * main single-run state (setResult / setGenerations / setStatus). */ +function awaitRunDone(runId: string): Promise { + return new Promise((resolve, reject) => { + const es = new EventSource(`${API_URL}/runs/${runId}/stream`); + let settled = false; + const finish = (done: () => void) => { + if (settled) return; + settled = true; + try { es.close(); } catch { /* noop */ } + done(); + }; + es.addEventListener("done", (ev) => { + finish(() => { + try { + const data = JSON.parse((ev as MessageEvent).data) as RunResult; + resolve(data); + } catch (e) { + reject(e instanceof Error ? e : new Error(String(e))); + } + }); + }); + es.addEventListener("error", (ev) => { + finish(() => { + try { + const data = JSON.parse((ev as MessageEvent).data) as { detail?: string }; + reject(new Error(data.detail ?? "run error")); + } catch { + // Some SSE errors don't carry a JSON body — fall back to a + // generic message. + reject(new Error("stream error")); + } + }); + }); + // Note: EventSource emits "error" for network hiccups too. We treat + // it as a hard reject — the sweep records the error and continues + // with the next seed. + }); +} + +function StabilityPanel({ + dataset, + target, + params, + coherence, + diversity, + dslRates, +}: { + dataset: DatasetId; + target: Target; + params: RunParams; + coherence: boolean; + diversity: boolean; + dslRates: DslRates; +}) { + const [seedInput, setSeedInput] = useState( + STABILITY_DEFAULT_SEEDS.join(", "), + ); + const [status, setStatus] = useState("idle"); + const [progress, setProgress] = useState<{ k: number; total: number } | null>(null); + const [rows, setRows] = useState([]); + const [errorMsg, setErrorMsg] = useState(null); + const cancelRef = useRef(false); + const [tagBySymbol, setTagBySymbol] = useState>({}); + + const running = status === "running"; + + // A global signal the outer Run bar can honour to disable itself + // while a sweep is in flight. We set it on window so it doesn't + // require plumbing changes; the RunBar reads it via the same-file + // check below. + useEffect(() => { + if (typeof window === "undefined") return; + (window as unknown as { __stabilitySweepActive?: boolean }).__stabilitySweepActive = running; + }, [running]); + + async function runSweep() { + const seeds = parseSeedList(seedInput); + if (seeds.length === 0) { + setErrorMsg("Enter at least one seed."); + return; + } + setStatus("running"); + setErrorMsg(null); + setRows([]); + setProgress({ k: 0, total: seeds.length }); + cancelRef.current = false; + + // Only send DSL rates that differ from defaults — same discipline + // launchRun() uses so unaltered runs stay byte-for-byte identical + // to the current defaults. + const ratesDiff: Record = {}; + (Object.keys(DSL_DEFAULT_RATES) as (keyof DslRates)[]).forEach((k) => { + if (dslRates[k] !== DSL_DEFAULT_RATES[k]) ratesDiff[k] = dslRates[k]; + }); + + const collected: SeedRow[] = []; + for (let i = 0; i < seeds.length; i++) { + if (cancelRef.current) break; + const seed = seeds[i]; + setProgress({ k: i + 1, total: seeds.length }); + let row: SeedRow = { + seed, + holdout: null, + permutation_p: null, + symbols: [], + transfer: null, + error: null, + }; + try { + const body: Parameters[0] = { + objective_spec: OBJECTIVE_PRESETS[target], + params: { ...params, seed }, + engine: "v2", + dataset, + coherence, + diversity, + }; + if (Object.keys(ratesDiff).length > 0) body.rates_override = ratesDiff; + const { run_id } = await postRun(body); + const result = await awaitRunDone(run_id); + row.holdout = result.winning.holdout_score; + row.permutation_p = result.winning.permutation_p; + // Reveal only this seed's winner's genes (bounded — same + // discipline as /transfer). + const winnerIds = result.winning.gene_ids ?? []; + if (winnerIds.length > 0) { + try { + const r = await postReveal(winnerIds); + row.symbols = r.symbols + .filter((s: string) => typeof s === "string" && s.length > 0); + } catch { + // Non-fatal — keep the row without symbols. + } + } + // Transfer test. Nulls on failure (e.g. GSE65858 not built); + // never crash the sweep. + try { + row.transfer = await getRunTransfer(run_id); + } catch { + row.transfer = null; + } + } catch (e) { + row.error = e instanceof Error ? e.message : String(e); + } + collected.push(row); + setRows([...collected]); + } + + // Tag genes by reference set (via reference_marks from + // /diagnostic/full-rank — bounded, already-revealed known set). + try { + const diag = await getFullRankDiagnostic(dataset, target); + const tags: Record = {}; + for (const m of diag.reference_marks ?? []) { + // p16 wins over cell_cycle if a symbol is in both. + if (!tags[m.symbol] || m.set_name === "p16") { + tags[m.symbol] = m.set_name; + } + } + setTagBySymbol(tags); + } catch { + setTagBySymbol({}); + } + + setProgress(null); + setStatus(cancelRef.current ? "idle" : "done"); + cancelRef.current = false; + } + + function stopSweep() { + cancelRef.current = true; + } + + return ( +
    +
    + Stability across seeds + +
    +

    + Rerun the HPV search from different random starting points and see + whether the answer holds — the same held-out AUROC bracket, the + same genes coming back. +

    + + + + {rows.length > 0 && ( + + )} +
    + ); +} + +function StabilityControls({ + seedInput, + onSeedInput, + running, + onStart, + onStop, + progress, + error, +}: { + seedInput: string; + onSeedInput: (s: string) => void; + running: boolean; + onStart: () => void; + onStop: () => void; + progress: { k: number; total: number } | null; + error: string | null; +}) { + return ( +
    + +
    + {!running ? ( + + ) : ( + + )} + {progress && ( + + Running seed {progress.k} of {progress.total}… + + )} +
    + {error && ( +

    + {error} +

    + )} +
    + ); +} + +function StabilityResults({ + rows, + tagBySymbol, + done, +}: { + rows: SeedRow[]; + tagBySymbol: Record; + done: boolean; +}) { + // ----- Section 1: dot strips ----- + const holdouts = rows.map((r) => r.holdout).filter((v): v is number => v != null); + const xfers = rows + .map((r) => r.transfer?.auroc ?? null) + .filter((v): v is number => v != null); + const heldRange: [number, number] | null = + holdouts.length > 0 ? [Math.min(...holdouts), Math.max(...holdouts)] : null; + const xferRange: [number, number] | null = + xfers.length > 0 ? [Math.min(...xfers), Math.max(...xfers)] : null; + + // ----- Section 2: recurrence ----- + const symbolCounts: Record = {}; + for (const r of rows) { + // dedup within a winner + const uniq = Array.from(new Set(r.symbols)); + for (const s of uniq) symbolCounts[s] = (symbolCounts[s] ?? 0) + 1; + } + const N = rows.length; + type RecEntry = { symbol: string; count: number; tag: string }; + const rec: RecEntry[] = Object.entries(symbolCounts) + .map(([symbol, count]) => ({ + symbol, + count, + tag: tagBySymbol[symbol] ?? "alternate", + })) + .sort((a, b) => b.count - a.count || a.symbol.localeCompare(b.symbol)); + const recurring = rec.filter((e) => e.count >= 2); + const passengers = rec.filter((e) => e.count === 1); + + return ( +
    + {done && ( + 0 + && rows.every( + (r) => r.permutation_p != null && r.permutation_p < 0.05, + ) + } + allIndepSig={ + rows.length > 0 + && rows.every( + (r) => r.transfer?.p != null && r.transfer.p < 0.05, + ) + } + topGeneEntry={rec.find((e) => e.count >= 2) ?? null} + nOnce={passengers.length} + /> + )} + + + +
    + ); +} + +// Deterministic, LLM-free plain-language summary composed from the +// completed sweep's aggregates. Rendered above the dot strips when +// the sweep finishes. Border/tone: green if perfStable && geneStable; +// teal if perfStable && !geneStable; amber otherwise. +function StabilitySummaryCallout({ + nSeeds, + heldRange, + xferRange, + allHeldSig, + allIndepSig, + topGeneEntry, + nOnce, +}: { + nSeeds: number; + heldRange: [number, number] | null; + xferRange: [number, number] | null; + allHeldSig: boolean; + allIndepSig: boolean; + topGeneEntry: { symbol: string; count: number; tag: string } | null; + nOnce: number; +}) { + if (nSeeds === 0) return null; + const heldMin = heldRange ? heldRange[0] : null; + const heldMax = heldRange ? heldRange[1] : null; + const indepMin = xferRange ? xferRange[0] : null; + const indepMax = xferRange ? xferRange[1] : null; + const perfStable = heldMin != null && heldMin >= 0.75 && allHeldSig; + const topGene = topGeneEntry?.symbol ?? null; + const topCount = topGeneEntry?.count ?? 0; + const geneStable = topGene != null && topCount >= Math.ceil(nSeeds / 2); + + // Tone. + let border = "#B3402F"; + let bg = "#FBE6E6"; + let fg = "#8A3225"; + if (perfStable && geneStable) { + border = "#2F6E4C"; + bg = "#E3F2EA"; + fg = "#1F4A34"; + } else if (perfStable && !geneStable) { + border = "#3A6B7E"; + bg = "#EAF0F2"; + fg = "#2C5563"; + } + + // Line 1 — performance. + const heldPart = + heldMin != null && heldMax != null + ? `held-out AUROC ${fmtFit(heldMin, 3)}–${fmtFit(heldMax, 3)}` + : "held-out AUROC not available"; + const indepPart = + indepMin != null && indepMax != null + ? ` on the independent cohort (${fmtFit(indepMin, 3)}–${fmtFit(indepMax, 3)})` + : ""; + const chanceTail = allIndepSig + ? ", and every run beat chance." + : ", and most runs beat chance on the independent cohort."; + const line1 = `Across ${nSeeds} independent searches, the engine detected HPV every time — ${heldPart}${indepPart}${chanceTail}`; + + // Line 2 — gene recurrence. + let line2: string; + if (geneStable && topGene) { + line2 = `The same genes kept coming back — ${topGene} in ${topCount}/${nSeeds} runs — which is strong evidence the signature itself is real, not a fluke of one search.`; + } else if (topGene) { + line2 = `But the specific genes differed almost every run — the most repeated was ${topGene} (${topCount}/${nSeeds}), and ${nOnce} genes appeared only once. That's the mark of a strong but redundant signal: many different gene sets separate HPV equally well.`; + } else { + // No gene appeared more than once at all. + line2 = `But the specific genes differed every run — no single gene showed up in more than one seed's winner, and ${nOnce} genes appeared only once. That's the mark of a strong but redundant signal: many different gene sets separate HPV equally well.`; + } + + const line3 = geneStable + ? null + : `So trust the detection, but don't read any single run's gene list as THE gene list — for a specific, stable gene program you'd need a target where the answer is concentrated in a few genes.`; + + return ( +
    +

    {line1}

    +

    {line2}

    + {line3 &&

    {line3}

    } +
    + ); +} + +function StabilityDotStrips({ + heldRange, + xferRange, + rows, +}: { + heldRange: [number, number] | null; + xferRange: [number, number] | null; + rows: SeedRow[]; +}) { + return ( +
    +
    + Does the result hold across seeds? +
    + r.holdout)} + color="#3A6B7E" + band="#D9E4EA" + /> + r.transfer?.auroc ?? null)} + color="#BC6B2E" + band="#F5E5D4" + /> +

    + Tight and high across seeds = the result isn’t a lucky + random start. The independent cohort usually costs a little — + that’s expected across countries, hospitals, and platforms. +

    +
    + ); +} + +function DotStrip({ + label, + sublabel, + values, + color, + band, +}: { + label: string; + sublabel: string; + values: (number | null)[]; + color: string; + band: string; +}) { + const W = 640; + const H = 44; + const padL = 40; + const padR = 40; + const innerW = W - padL - padR; + const AXIS_MIN = 0.5; + const AXIS_MAX = 1.0; + const y = 26; + function xOf(v: number): number { + const t = Math.max(0, Math.min(1, (v - AXIS_MIN) / (AXIS_MAX - AXIS_MIN))); + return padL + t * innerW; + } + const finite = values.filter((v): v is number => v != null && Number.isFinite(v)); + const vmin = finite.length > 0 ? Math.min(...finite) : null; + const vmax = finite.length > 0 ? Math.max(...finite) : null; + return ( +
    +
    + + {label}{" "} + {sublabel} + + + {finite.length} of {values.length} seeds + +
    + + {/* min–max band */} + {vmin != null && vmax != null && ( + + )} + {/* baseline */} + + {/* tick labels: 0.5, 0.75, 1.0 */} + {[0.5, 0.75, 1.0].map((t) => ( + + + + {t === 0.5 ? "0.5 coin-flip" : t.toFixed(2)} + + + ))} + {/* dots */} + {values.map((v, i) => ( + v != null && Number.isFinite(v) ? ( + + ) : null + ))} + +
    + ); +} + +const REC_COLOURS: Record = { + cell_cycle: { fill: "#F5E5D4", stroke: "#BC6B2E", label: "cell_cycle" }, + p16: { fill: "#FBF0CC", stroke: "#B8860B", label: "p16" }, + alternate: { fill: "#E6F0F3", stroke: "#3A6B7E", label: "alternate" }, + passengers: { fill: "#EDEDED", stroke: "#9AA0A6", label: "passengers" }, +}; + +function StabilityRecurrence({ + recurring, + passengers, + totalSeeds, +}: { + recurring: { symbol: string; count: number; tag: string }[]; + passengers: { symbol: string; count: number; tag: string }[]; + totalSeeds: number; +}) { + return ( +
    +
    + Do the same genes keep coming back? +
    +

    + Recurring proliferation / cell-cycle genes = real signal. + Genes that appear only once are usually passengers. +

    +
    + {recurring.length === 0 && ( +

    + No gene appeared in more than one seed’s winner. +

    + )} + {recurring.map((r) => { + const pal = REC_COLOURS[r.tag] ?? REC_COLOURS.alternate; + const pct = Math.round((r.count / Math.max(1, totalSeeds)) * 100); + return ( +
    + {r.symbol} +
    +
    +
    + + {r.count} / {totalSeeds} + + + {r.tag === "alternate" ? "alternate" : r.tag} + +
    + ); + })} + {passengers.length > 0 && ( +
    + passengers +
    +
    +
    + 1 / {totalSeeds} + p.symbol) + .join(", ") + + (passengers.length > 40 ? ", …" : "") + } + > + {passengers.length} gene{passengers.length === 1 ? "" : "s"} · 1 seed each + +
    + )} +
    +
    + ); +} + +function StabilityTable({ rows }: { rows: SeedRow[] }) { + return ( +
    +
    + Per-seed detail +
    +
    + + + + + + + + + + + + + {rows.map((r) => { + const t = r.transfer; + const total = t ? t.n_found + t.n_missing : r.symbols.length; + const found = t ? t.n_found : r.symbols.length; + return ( + + + + + + + + + ); + })} + +
    seedheld-out AUROCpindep. AUROCindep. pgenes found / total
    {r.seed} + {fmtFit(r.holdout, 3)} + + {fmtFit(r.permutation_p, 3)} + + {fmtFit(t?.auroc ?? null, 3)} + + {fmtFit(t?.p ?? null, 3)} + + {r.error ? ( + error + ) : ( + `${found} / ${total || "—"}` + )} +
    +
    +
    + ); +} + + +function Metric({ + label, + value, + tip, + sub, +}: { + label: string; + value: string; + tip?: string; + sub?: string; +}) { + return ( +
    +
    + {label} + {tip && } +
    +
    {value}
    + {sub && ( +
    {sub}
    + )} +
    + ); +} + +// --------------------------------------------------------------------------- +// RankingResult — the consolidated reveal panel for supervised +// objectives. Shows the FULL single-gene ranking computed on the +// engine's TRAIN split (~20k opaque rows), with the GP winner gene(s) +// and the dataset's reference-set genes marked along a rank track and +// pinned to the top of a browsable list. Symbols are revealed ONLY for +// those highlighted rows via the single-ID /evaluate reveal path. +// Airgap-clean: the full list never carries symbols on the wire. +// --------------------------------------------------------------------------- + +// Three-way colour scheme for the ranking result. The winner stays in +// the project's accent teal; reference sets get their own distinct +// colours so e.g. p16 (a single canonical marker) reads separately +// from the broader cell-cycle group. Falls back to the muted-amber +// "set" colour for any future set name. +interface RankPalette { + stroke: string; + fill: string; + bg: string; +} + +const RANK_COLORS: Record & { + winner: RankPalette; + p16: RankPalette; + immune: RankPalette; + cell_cycle: RankPalette; + MMR: RankPalette; +} = { + winner: { stroke: "#3A6B7E", fill: "#3A6B7E", bg: "#EAF0F2" }, + // p16 / immune — the "single canonical marker" slot — deep gold. + p16: { stroke: "#B8860B", fill: "#B8860B", bg: "#FBF0CC" }, + immune: { stroke: "#B8860B", fill: "#B8860B", bg: "#FBF0CC" }, + // cell_cycle / MMR — the "broader set" slot — muted amber. + cell_cycle: { stroke: "#A75D2A", fill: "#D9A074", bg: "#FBF1E6" }, + MMR: { stroke: "#A75D2A", fill: "#D9A074", bg: "#FBF1E6" }, +}; + +function colorForSet(setName: string | undefined): RankPalette { + if (!setName) return RANK_COLORS.cell_cycle; + return RANK_COLORS[setName] ?? RANK_COLORS.cell_cycle; +} + +// "top 0.5%" / "top 21%" / "bottom 4.9%" — two sig figs. Use "top X%" +// for ranks in the upper half; flip to "bottom Y%" once past the +// midpoint so very-poor ranks read as low. +function rankPercentile(rank: number, n: number): string { + if (n <= 0) return "—"; + const topPct = (rank / n) * 100; + if (topPct <= 50) return `top ${formatPct(topPct)}%`; + const bottomPct = 100 - topPct; + return `bottom ${formatPct(bottomPct)}%`; +} + +function formatPct(p: number): string { + // 2 sig figs across the whole range. + if (p >= 10) return p.toFixed(0); + if (p >= 1) return p.toFixed(1); + if (p >= 0.1) return p.toFixed(2); + return p.toFixed(3); +} + +interface RevealedWinnerRow { + opaque_id: string; + symbol: string; + rank: number | null; + score: number | null; +} + +function RankingResult({ + result, + dataset, + target, + onRevealed, +}: { + result: RunResult; + dataset: DatasetId; + target: Target; + onRevealed?: (r: EvaluateResponse) => void; +}) { + const [diag, setDiag] = useState(null); + const [diagError, setDiagError] = useState(null); + const [diagLoading, setDiagLoading] = useState(false); + + const [winnerReveal, setWinnerReveal] = useState([]); + const [refKey, setRefKey] = useState(""); + const [search, setSearch] = useState(""); + + // Fetch the full opaque ranking once per (dataset, target). + useEffect(() => { + let cancelled = false; + setDiagLoading(true); + setDiagError(null); + setDiag(null); + getFullRankDiagnostic(dataset, target) + .then((d) => { + if (!cancelled) setDiag(d); + }) + .catch((e) => { + if (!cancelled) + setDiagError(e instanceof Error ? e.message : String(e)); + }) + .finally(() => { + if (!cancelled) setDiagLoading(false); + }); + return () => { + cancelled = true; + }; + }, [dataset, target]); + + // Build a map from opaque_id → rank/score from the full diagnostic. + const rankByOpaque = useMemo(() => { + const m = new Map(); + if (diag) { + for (const r of diag.ranks) + m.set(r.opaque_id, { rank: r.rank, score: r.score }); + } + return m; + }, [diag]); + + // Reveal the winner's opaque IDs → symbols, plus look up their rank + // in the diagnostic for the rank-track marks. Uses the single-ID + // /evaluate path — never the whole map. + const winnerIds = result.winning.gene_ids; + const refKeysForDataset = useMemo( + () => DATASET_REGISTRY[dataset].refSetKeys, + [dataset], + ); + useEffect(() => { + if (refKey === "" || !refKeysForDataset.includes(refKey as never)) { + setRefKey(refKeysForDataset[0] as string); + } + }, [refKeysForDataset, refKey]); + + useEffect(() => { + let cancelled = false; + if (!winnerIds.length || !diag) { + setWinnerReveal([]); + return; + } + const referenceSet = (refKey || refKeysForDataset[0]) as string; + postEvaluate({ + gene_ids: winnerIds, + reference_set: referenceSet, + dataset, + target, + }) + .then((r) => { + if (cancelled) return; + const rows: RevealedWinnerRow[] = r.revealed.map((row) => { + const lookup = rankByOpaque.get(row.id); + return { + opaque_id: row.id, + symbol: row.symbol, + rank: lookup?.rank ?? row.rank ?? null, + score: + lookup?.score ?? + (row.single_gene_metric != null + ? row.single_gene_metric + : null), + }; + }); + setWinnerReveal(rows); + if (onRevealed) onRevealed(r); + }) + .catch(() => { + if (!cancelled) setWinnerReveal([]); + }); + return () => { + cancelled = true; + }; + }, [winnerIds, diag, dataset, target, refKey, rankByOpaque, onRevealed, refKeysForDataset]); + + if (diagLoading && !diag) { + return ( + +

    Computing the full ranking…

    +     + ); + } + if (diagError || !diag) { + return ( + +

    + {diagError ?? "No ranking available for this (dataset, target)."} +

    +     + ); + } + + const winnerOpaqueSet = new Set(winnerIds); + const winnerSymbolByOpaque = new Map( + winnerReveal.map((w) => [w.opaque_id, w.symbol]), + ); + const referenceByOpaque = new Map(); + for (const m of diag.reference_marks) referenceByOpaque.set(m.opaque_id, m); + + const metricLabel = + diag.metric_kind === "auroc" ? "AUROC" : "signed Spearman ρ"; + + // Pinned rows: known reference markers only (p16 / cell_cycle for + // HNSC; MMR / immune for CRC). The winner's constituent genes used + // to be pinned here too, but that mixed the GP's GROUP into a table + // of INDIVIDUALLY-scored reference markers — confusing. Winner + // genes + their ranks now live in the Result panel itself + // (self-contained "GP's group" view); this panel is the + // known-marker recovery diagnostic. + const pinnedRows: { + opaque_id: string; + symbol: string; + rank: number; + score: number; + kind: "reference"; + set_name?: string; + }[] = []; + for (const m of [...diag.reference_marks].sort((a, b) => a.rank - b.rank)) { + pinnedRows.push({ + opaque_id: m.opaque_id, + symbol: m.symbol, + rank: m.rank, + score: m.score, + kind: "reference", + set_name: m.set_name, + }); + } + + // Browsable list: top-200 of the ranks PLUS any rows that match the + // search filter — pin the highlighted rows above. (Anti-pattern to + // render 20k s; this is more than enough for browsing.) + const TOP_N = 200; + const trimmedSearch = search.trim().toLowerCase(); + let listRows: FullRankDiagnostic["ranks"]; + if (trimmedSearch.length > 0) { + listRows = diag.ranks.filter((r) => + r.opaque_id.toLowerCase().includes(trimmedSearch), + ); + } else { + listRows = diag.ranks.slice(0, TOP_N); + } + + return ( + 0 + ? ` (${ + target === "hpv" ? "HPV+" : "MSI-H" + } ${diag.n_pos} / ${ + target === "hpv" ? "HPV−" : "MSS" + } ${diag.n_neg}.)` + : "") + } + titleTip={TIPS.rankingResult} + > + + + {/* Pinned highlighted rows */} +
    +
    + Known markers + +
    +
    + + + + + + + + + + + + + {pinnedRows.length === 0 && ( + + + + )} + {pinnedRows.map((r) => { + const palette = colorForSet(r.set_name); + const label = r.set_name ?? "reference"; + return ( + + + + + + + + + ); + })} + +
    SourceSymbolGene codeRank / NPercentile{metricLabel}
    + No reference markers configured for this (dataset, target). +
    + + {label} + + + {r.symbol} + {r.opaque_id} + {r.rank.toLocaleString()} / {diag.n_genes.toLocaleString()} + + {rankPercentile(r.rank, diag.n_genes)} + + {fmtFit(r.score, 3)} +
    +
    +

    + A recovery diagnostic — where each known marker{" "} + (p16, cell_cycle for HNSC; MMR, immune for CRC) ranks as a + single-gene separator. The winning program (its genes, its + group score, its confound survival) lives in the Result + panel above. +

    +
    + + {/* Raw-anonymous-ranking — collapsed disclosure */} + + +

    + The genetic-programming search isn’t an exhaustive top-1 + ranker — it’s stochastic. The diagnostic ranking scores + all{" "}~{diag.n_genes.toLocaleString()} genes and sorts them; + genetic programming does not. It samples programs, mutates, + recombines, and keeps + what survives tournaments — exploring only a subset of the + space. So it finds an excellent solution, not provably the best + one: it may never have tried the rank-1 gene, or tried it in a + generation that got out-competed, and settled into a + neighbouring solution. Evolution finds a very good answer, not + the global maximum. +

    +     + ); +} + +// Collapsed disclosure for the raw opaque-ID ranking. Only purpose +// is to demonstrate the ranking IS computed blind — the analytic +// content lives in the highlighted table + the rank chart. +function RawRankingDisclosure({ + diag, + search, + setSearch, + winnerOpaqueSet, + winnerSymbolByOpaque, + referenceByOpaque, + listRows, + trimmedSearch, + topN, + metricLabel, +}: { + diag: FullRankDiagnostic; + search: string; + setSearch: (s: string) => void; + winnerOpaqueSet: Set; + winnerSymbolByOpaque: Map; + referenceByOpaque: Map; + listRows: FullRankDiagnostic["ranks"]; + trimmedSearch: string; + topN: number; + metricLabel: string; +}) { + const [open, setOpen] = useState(false); + const panelId = useId(); + return ( +
    + + {open && ( +
    +

    + The ranking above is computed only on the anonymous gene + codes. The full list below stays anonymous on purpose — + revealing every name would defeat the whole blind-search + point. This view is just to show that the ranking really + is blind. +

    +
    + + Full ranking (browse by gene code) + + setSearch(e.target.value)} + className="ml-auto w-56 rounded-md border border-border bg-white px-2 py-1 text-xs focus:border-accent focus:outline-none" + /> +
    +
    +
    + + + + + + + + + + {listRows.map((r) => { + const isWinner = winnerOpaqueSet.has(r.opaque_id); + const refMark = referenceByOpaque.get(r.opaque_id); + const palette = isWinner + ? RANK_COLORS.winner + : refMark + ? colorForSet(refMark.set_name) + : null; + return ( + + + + + + ); + })} + {listRows.length === 0 && ( + + + + )} + +
    RankGene code{metricLabel}
    + {r.rank.toLocaleString()} + + {r.opaque_id} + {isWinner && ( + + ← winner ( + {winnerSymbolByOpaque.get(r.opaque_id) ?? "…"}) + + )} + {!isWinner && refMark && ( + + ← {refMark.set_name}: {refMark.symbol} + + )} + + {fmtFit(r.score, 3)} +
    + No matches for “{search}”. +
    +
    + {trimmedSearch.length === 0 && ( +

    + Showing the top {topN} of {diag.n_genes.toLocaleString()}. Use + the search box to find any gene code. +

    + )} +
    +
    + )} +
    + ); +} + +// Where the winner and known markers rank (1 = best single-gene +// separator, log scale). Log-rank axis spreads the top cluster out; +// each highlighted gene is a labelled dot, coloured by its source +// (winner / p16 / cell_cycle, or winner / MMR / immune for CRC). +// Labels stagger vertically with leader lines so they don't overlap. +function RankTrack({ + diag, + winnerOpaqueIds, + winnerReveal, +}: { + diag: FullRankDiagnostic; + winnerOpaqueIds: string[]; + winnerReveal: RevealedWinnerRow[]; +}) { + const W = 760; + const padL = 56; + const padR = 40; + const padTop = 30; + const padBot = 40; + const innerW = W - padL - padR; + + const labelByOpaque = new Map( + winnerReveal.map((w) => [w.opaque_id, w.symbol]), + ); + const rankByOpaque = new Map(diag.ranks.map((r) => [r.opaque_id, r.rank])); + + // Collect every highlighted point (winner + reference) with its + // colour palette and source label. + type Mark = { + key: string; + symbol: string; + rank: number; + palette: RankPalette; + kind: "winner" | "reference"; + set_name?: string; + }; + const marks: Mark[] = []; + for (const id of winnerOpaqueIds) { + const rank = rankByOpaque.get(id); + if (rank == null) continue; + marks.push({ + key: `w-${id}`, + symbol: labelByOpaque.get(id) ?? "…", + rank, + palette: RANK_COLORS.winner, + kind: "winner", + }); + } + const winnerOpaqueSet = new Set(winnerOpaqueIds); + for (const m of diag.reference_marks) { + if (winnerOpaqueSet.has(m.opaque_id)) continue; + marks.push({ + key: `r-${m.opaque_id}`, + symbol: m.symbol, + rank: m.rank, + palette: colorForSet(m.set_name), + kind: "reference", + set_name: m.set_name, + }); + } + // Sort by rank ascending so labels stagger predictably left → right. + marks.sort((a, b) => a.rank - b.rank); + + // Log scale: x = padL + (log10(rank) / log10(N)) * innerW. + const logN = Math.log10(Math.max(2, diag.n_genes)); + function rankToX(r: number): number { + const lr = Math.log10(Math.max(1, r)); + return padL + (lr / logN) * innerW; + } + + // Gridline ticks at 1, 10, 100, 1k, 10k, N (skipping ones too close + // to N). + const tickRanks: number[] = [1, 10, 100, 1000, 10000].filter( + (t) => t < diag.n_genes, + ); + tickRanks.push(diag.n_genes); + const tickLabels: Record = { + 1: "1", + 10: "10", + 100: "100", + 1000: "1k", + 10000: "10k", + }; + tickLabels[diag.n_genes] = diag.n_genes.toLocaleString(); + + // Collision-free label placement. Each label is `textAnchor="middle"` + // around its dot's x position, so its bounding box spans + // [x - w/2, x + w/2] where w ≈ text.length × fontSize × 0.6. + // Walk marks left-to-right (they're already sorted by rank) and + // greedily place each label in the LOWEST row whose previous + // label's right edge + a min gap is ≤ this label's left edge; if + // no row fits, open a new one. No fixed cap — the SVG height + // expands so nothing clips. With this in place the MCM / + // AURKB / CCNB1 clusters render cleanly even when several genes + // share nearby ranks. + const LABEL_FONT_SIZE = 10; + const LABEL_CHAR_W = LABEL_FONT_SIZE * 0.6; + const LABEL_GAP_PX = 6; + const ROW_GAP = 14; + const rowLastRight: number[] = []; + const rowOfLabel: number[] = new Array(marks.length).fill(0); + marks.forEach((m, i) => { + const x = rankToX(m.rank); + const w = Math.max(LABEL_FONT_SIZE, m.symbol.length * LABEL_CHAR_W); + const left = x - w / 2; + const right = x + w / 2; + let chosen = -1; + for (let r = 0; r < rowLastRight.length; r++) { + if (rowLastRight[r] + LABEL_GAP_PX <= left) { + chosen = r; + break; + } + } + if (chosen === -1) { + chosen = rowLastRight.length; + rowLastRight.push(right); + } else { + rowLastRight[chosen] = right; + } + rowOfLabel[i] = chosen; + }); + const numRows = Math.max(1, rowLastRight.length); + // Labels stack from padTop downward; the axis sits below the + // stack with a small breathing gap so leader lines from the + // top-row labels are clearly readable. + const LABEL_TO_AXIS_GAP = 24; + const labelStackH = numRows * ROW_GAP; + const axisY = padTop + labelStackH + LABEL_TO_AXIS_GAP; + const H = axisY + padBot; + // Convert row index → label baseline y (one row below the top). + const labelY: number[] = rowOfLabel.map( + (r) => padTop + (r + 1) * ROW_GAP, + ); + + const showWinner = winnerOpaqueIds.length > 0; + const legendItems = [ + ...(showWinner + ? [{ label: "winner", palette: RANK_COLORS.winner }] + : []), + ...Array.from( + new Set(diag.reference_marks.map((m) => m.set_name)), + ).map((name) => ({ label: name, palette: colorForSet(name) })), + ]; + + return ( +
    +
    +

    + {showWinner + ? "Where the winner and known markers rank" + : "Where the known markers rank"} + + (1 = best single-gene separator) + +

    +
    + {legendItems.map((it) => ( + + + {it.label} + + ))} +
    +
    + + {/* gridlines + tick labels */} + {tickRanks.map((t) => { + const x = rankToX(t); + return ( + + + + {tickLabels[t]} + + + ); + })} + {/* axis line */} + + + best single-gene separator → + + + ← worst + + + {/* highlighted points: lollipop with leader line + dot + label */} + {marks.map((m, i) => { + const x = rankToX(m.rank); + const ly = labelY[i]; + return ( + + + + + {m.symbol} + + + ); + })} + +

    + Each dot is one known-marker gene at its single-gene rank on the + engine’s training split. Log axis stretches the top so a + recoverable marker reads cleanly; weak ones sit at the right. + A marker near rank 1 means the canonical biology is findable + from expression; a marker far down the axis means it only + helps in combination. +

    +
    + ); +} + +function Evaluator({ + geneIds, + fitnessLabel, + dataset, + target, + onRevealed, +}: { + geneIds: string[]; + fitnessLabel: string; + dataset: DatasetId; + target: Target; + onRevealed?: (r: EvaluateResponse) => void; +}) { + const refKeys = DATASET_REGISTRY[dataset].refSetKeys; + const [ref, setRef] = useState(refKeys[0]); + + // Switching cancers swaps the available reference sets — keep `ref` + // valid for the active dataset. + useEffect(() => { + if (!refKeys.includes(ref)) { + setRef(refKeys[0]); + } + }, [refKeys, ref]); + + // ``data`` is the raw last /evaluate response. We re-derive overlap + // locally against the toggled reference set, so switching MMR ↔ immune + // is instant and never needs another API round-trip. + const [data, setData] = useState(null); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + + // TMB-rank diagnostic lifted from TMBRankPanel so its data also drives + // the MLH1 success callout and the OverlapSummary's muted secondary + // sentence. One fetch, three consumers. Colorectal-only. + const [diagnostic, setDiagnostic] = useState(null); + const [diagError, setDiagError] = useState(null); + const [diagLoading, setDiagLoading] = useState(false); + useEffect(() => { + if (dataset !== "coadread" || target !== "tmb") { + setDiagnostic(null); + setDiagError(null); + setDiagLoading(false); + return; + } + let cancelled = false; + setDiagLoading(true); + setDiagError(null); + getTMBRankDiagnostic() + .then((d) => { + if (!cancelled) setDiagnostic(d); + }) + .catch((e) => { + if (!cancelled) + setDiagError(e instanceof Error ? e.message : String(e)); + }) + .finally(() => { + if (!cancelled) setDiagLoading(false); + }); + return () => { + cancelled = true; + }; + }, [dataset, target]); + + // HPV-rank diagnostic — sibling of TMB-rank, gated to HNSC + HPV. + const [hpvDiag, setHpvDiag] = useState(null); + const [hpvDiagError, setHpvDiagError] = useState(null); + const [hpvDiagLoading, setHpvDiagLoading] = useState(false); + useEffect(() => { + if (dataset !== "hnsc" || target !== "hpv") { + setHpvDiag(null); + setHpvDiagError(null); + setHpvDiagLoading(false); + return; + } + let cancelled = false; + setHpvDiagLoading(true); + setHpvDiagError(null); + getHPVRankDiagnostic() + .then((d) => { + if (!cancelled) setHpvDiag(d); + }) + .catch((e) => { + if (!cancelled) + setHpvDiagError(e instanceof Error ? e.message : String(e)); + }) + .finally(() => { + if (!cancelled) setHpvDiagLoading(false); + }); + return () => { + cancelled = true; + }; + }, [dataset, target]); + const mlh1Row = useMemo(() => { + if (!diagnostic) return undefined; + return diagnostic.mmr.find((r) => r.symbol === "MLH1"); + }, [diagnostic]); + + async function go() { + setBusy(true); + setError(null); + try { + const r = await postEvaluate({ + gene_ids: geneIds, + reference_set: ref, + dataset, + target, + }); + setData(r); + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + } finally { + setBusy(false); + } + } + + // Derive matches locally so the toggle always changes something — + // even when overlap is 0 the visible reference list, the count line, + // and the table rows all update. + const effectiveData = useMemo(() => { + if (!data) return null; + const datasetSets = REFERENCE_SETS_BY_DATASET[dataset] as Record< + string, + readonly string[] + >; + const memberSet = new Set(datasetSets[ref] ?? []); + const revealed = data.revealed.map((row) => ({ + ...row, + matched: memberSet.has(row.symbol), + })); + return { + revealed, + overlap_count: revealed.filter((r) => r.matched).length, + reference_set: ref, + }; + }, [data, ref, dataset]); + + // Push the re-derived matches up to Lab so the ProgramGraph's Select + // nodes re-tint the matched gene pills on toggle as well. + useEffect(() => { + if (effectiveData && onRevealed) onRevealed(effectiveData); + }, [effectiveData, onRevealed]); + + const sortedRows = useMemo(() => { + if (!effectiveData) return null; + return [...effectiveData.revealed].sort( + (a, b) => Number(b.matched) - Number(a.matched), + ); + }, [effectiveData]); + + const matchedSymbols = useMemo( + () => effectiveData?.revealed.filter((r) => r.matched).map((r) => r.symbol) ?? [], + [effectiveData], + ); + + return ( + +
    + {target !== "none" && ( +
    + + reference set + + + {refKeys.map((r) => ( + + ))} +
    + )} +
    + + +
    + objective: {fitnessLabel} +
    + + {target !== "none" && ( +
    + Checking the winning genes against this set:{" "} + + {( + REFERENCE_SETS_BY_DATASET[dataset] as Record + )[ref]?.join(", ") ?? ""} + +
    + )} + {target === "none" && ( +

    + Unsupervised run — no reference set; the post-hoc alignment (above, + in Result) shows how the discovered split lines up with the + named label this cohort carries. +

    + )} + + {error && ( +

    + {error} +

    + )} + + {dataset === "coadread" && target === "tmb" && mlh1Row?.present && ( + + )} + + {effectiveData && sortedRows && ( +
    + {target !== "none" && ( +
    + + +
    + )} +
    + Revealed genes + +
    + {/* Single-gene rank column is populated only for (HNSC,HPV) + and (coadread,TMB) — the (dataset, target) pairs with a + single-gene diagnostic. Computed server-side via /evaluate + (airgap-clean: reveal already happens there). */} + {(() => { + const showRank = sortedRows.some((r) => r.rank != null); + return ( +
    + + + + + + {target !== "none" && ( + + )} + {showRank && ( + + )} + + + + {sortedRows.map((row) => ( + + + + {target !== "none" && ( + + )} + {showRank && ( + + )} + + ))} + +
    Gene codeSymbolMatched + + Single-gene rank + + +
    {row.id} + {row.symbol} + + {row.matched ? ( + + ✓ + + ) : ( + + — + + )} + + {row.rank != null && row.total != null ? ( + <> + {row.rank} + / {row.total.toLocaleString()} + {row.single_gene_metric != null && ( + + {row.metric_kind === "auroc" + ? `AUROC ${fmtFit(row.single_gene_metric, 3)}` + : `ρ ${fmtFit(row.single_gene_metric, 3)}`} + + )} + + ) : ( + — + )} +
    +
    + ); + })()} +
    + )} + + {dataset === "coadread" && target === "tmb" && ( + + )} + {dataset === "hnsc" && target === "hpv" && ( + + )} +     + ); +} + +// Accent SUCCESS callout for the TMB objective: MLH1's rank on the TMB +// negative-association axis. Surfaces what's already in the diagnostic +// data, so the "0 of N winning genes in MMR" line stops reading as a +// flat failure. +function MLH1SuccessCallout({ + mlh1, + totalGenes, +}: { + mlh1: GeneRankRow; + totalGenes: number; +}) { + if (!mlh1.present || mlh1.rank == null) return null; + const pct = Math.round((mlh1.rank / totalGenes) * 1000) / 10; + return ( +
    + + ✓ + +

    + Blind, MLH1 ranks{" "} + + {mlh1.rank} / {totalGenes.toLocaleString()} + {" "} + (top {pct}%) on the TMB objective — the causal gene surfaced near the + top without ever seeing gene names. +

    +
    + ); +} + +// TMB-rank diagnostic: where each MMR / IMMUNE gene lands when every +// gene is ranked by signed Spearman with TMB on the engine's TMB cohort. +// "rank near 1 = findable; high rank = the signal isn't in the +// expression data." Data is fetched by the parent Evaluator and passed +// in so the success callout and OverlapSummary share the same response. +function TMBRankPanel({ + data, + loading, + error, +}: { + data: TMBRankDiagnostic | null; + loading: boolean; + error: string | null; +}) { + return ( +
    +
    + Reference-gene diagnostic + +
    +

    + Where each known gene sits on the engine's TMB target — rank near 1 = findable; + high rank = the signal isn't in the expression data. +

    + + {loading && ( +

    computing ranks…

    + )} + {error && ( +

    + {error} +

    + )} + + {data && ( +
    +

    + {data.n_samples} samples · + {data.n_genes.toLocaleString()} genes + ranked. +

    + +
    + + +
    + +
    +
    + Top {data.top_negative.length} most-negatively-correlated genes +
    +

    + What outranks MLH1 — every gene above MLH1's rank on the TMB axis. +

    +
    + + + + + + + + + + {data.top_negative.map((r) => ( + + + + + + ))} + +
    SymbolCorrelationRank / N
    {r.symbol} + {r.corr == null ? "—" : r.corr.toFixed(4)} + + {r.rank == null ? "—" : `${r.rank} / ${data.n_genes}`} +
    +
    +
    +
    + )} +
    + ); +} + +// HNSC-side mirror of . Reads /diagnostic/hpv-rank: per- +// gene single-gene HPV+/HPV− AUROC on the engine's TRAIN split. Shows +// CDKN2A (p16), each cell-cycle / E2F-target gene, and the top single- +// gene separators. Caption matches the prompt verbatim. +function HPVRankPanel({ + data, + loading, + error, +}: { + data: HPVRankDiagnostic | null; + loading: boolean; + error: string | null; +}) { + return ( +
    +
    + Reference-gene diagnostic + +
    +

    + Where each known HPV marker sits as a single-gene HPV+/HPV− + separator — rank near 1 = recoverable; high rank = out-competed. +

    + + {loading && ( +

    computing ranks…

    + )} + {error && ( +

    + {error} +

    + )} + + {data && ( +
    +

    + {data.n_samples}{" "} + train samples (HPV+ {data.n_pos}{" "} + / HPV− {data.n_neg}) · + {data.n_genes.toLocaleString()} genes + ranked. +

    + +
    + + +
    + +
    +
    + Top {data.top_separators.length} single-gene HPV+/HPV− separators +
    +

    + What out-competes CDKN2A as a single feature on this cohort. +

    +
    + + + + + + + + + + {data.top_separators.map((r) => ( + + + + + + ))} + +
    SymbolAUROCRank / N
    {r.symbol} + {r.corr == null ? "—" : r.corr.toFixed(4)} + + {r.rank == null ? "—" : `${r.rank} / ${data.n_genes}`} +
    +
    +
    +
    + )} +
    + ); +} + +// Synergy = how much the group's combined held-out beats its best +// single gene's metric. Captures the additive lift the GROUP gives +// beyond its strongest constituent — high synergy = a real group +// effect, low / negative = a "best gene + passengers" pattern. +// Lifted to module scope so both the merged Groups table and the +// landscape scatter share one source of truth. +function synergyOf(m: RankedModule): number | null { + if (m.combined_holdout == null || !Number.isFinite(m.combined_holdout)) { + return null; + } + let best: number | null = null; + for (const g of m.per_gene) { + const v = g.single_gene_metric; + if (v == null || !Number.isFinite(v)) continue; + if (best == null || v > best) best = v; + } + if (best == null) return null; + return m.combined_holdout - best; +} + +// Category buckets a module by its ref_sets membership. Single-marker +// sets (p16 / immune) win over broader proliferation sets +// (cell_cycle / MMR) so a module with both gets the more specific +// colour. "other" = the faint grey background dot. +type ModuleCategory = "winner" | "p16" | "immune" | "cell_cycle" | "MMR" | "other"; + +function categoriseModule( + m: RankedModule, winnerSet: Set, +): ModuleCategory { + // Winner takes priority over reference-set membership so the + // ringed dot is visible even if the winner happens to overlap + // a known set. + if (m.gene_ids.length === winnerSet.size) { + let ok = true; + for (const g of m.gene_ids) if (!winnerSet.has(g)) { ok = false; break; } + if (ok) return "winner"; + } + if (m.ref_sets.includes("p16")) return "p16"; + if (m.ref_sets.includes("immune")) return "immune"; + if (m.ref_sets.includes("cell_cycle")) return "cell_cycle"; + if (m.ref_sets.includes("MMR")) return "MMR"; + return "other"; +} + +// Canvas-based landscape scatter — ~6k dots, so SVG is too heavy. +// Renders the dots on and the chrome (axes, gridlines, +// quadrant guides, labels, legend) as an absolutely-positioned SVG +// overlay sharing the same plot rect. +function FitnessSynergyScatter({ + modules, + winnerGeneIds, + dataset, +}: { + modules: RankedModule[]; + winnerGeneIds: string[]; + dataset: DatasetId; +}) { + const wrapperRef = useRef(null); + const canvasRef = useRef(null); + const [width, setWidth] = useState(720); + const HEIGHT = 320; + const PAD = useMemo( + () => ({ top: 28, right: 16, bottom: 44, left: 56 }), + [], + ); + + // Hit-test record: screen positions of HIGHLIGHTED dots only + // (winner + ref-set members). Populated when the canvas paints + // and consulted by the mousemove handler. The ~6k background dots + // are deliberately not in here — they're never revealed and would + // be a waste to hit-test. + type Hit = { cx: number; cy: number; cat: ModuleCategory; m: RankedModule }; + const hitsRef = useRef([]); + const [hover, setHover] = useState<{ + cat: ModuleCategory; m: RankedModule; cx: number; cy: number; + } | null>(null); + // "Hide failed-site groups" toggle: when on, highlighted dots whose + // group did NOT survive the oropharynx-subgroup check are dropped + // from the plot AND from the hit-test. Background dots are + // unaffected (they're not site-tested anyway). + const [hideFailedSite, setHideFailedSite] = useState(false); + + // Track the wrapper width so the canvas + overlay stay sized to + // the column the panel renders into. + useEffect(() => { + const el = wrapperRef.current; + if (!el) return; + function update() { + if (!el) return; + setWidth(Math.max(320, el.clientWidth)); + } + update(); + const ro = new ResizeObserver(update); + ro.observe(el); + return () => ro.disconnect(); + }, []); + + // Project + categorise once per `modules`. + const winnerSet = useMemo>( + () => new Set(winnerGeneIds), [winnerGeneIds], + ); + const points = useMemo(() => { + type P = { + x: number; + y: number; + cat: ModuleCategory; + m: RankedModule; + }; + const out: P[] = []; + for (const m of modules) { + const x = m.gp_fitness; + const y = synergyOf(m); + if (x == null || !Number.isFinite(x)) continue; + if (y == null || !Number.isFinite(y)) continue; + out.push({ x, y, cat: categoriseModule(m, winnerSet), m }); + } + return out; + }, [modules, winnerSet]); + + // Bounded reveal: highlighted groups only (winner + ref-set members). + // A few dozen modules × a few genes ≈ ~100-200 ids — fine for a + // single batched /reveal. Background dots are deliberately left + // opaque; revealing all of them would defeat the airgap. + const highlightedGeneIds = useMemo(() => { + const ids = new Set(); + for (const p of points) { + if (p.cat === "other") continue; + for (const g of p.m.gene_ids) ids.add(g); + } + return Array.from(ids); + }, [points]); + const [highlightSymbols, setHighlightSymbols] = useState>( + {}, + ); + useEffect(() => { + if (highlightedGeneIds.length === 0) return; + const needed = highlightedGeneIds.filter((g) => !(g in highlightSymbols)); + if (needed.length === 0) return; + let cancelled = false; + postReveal(needed) + .then((r) => { + if (cancelled) return; + const additions: Record = {}; + needed.forEach((gid, i) => { + additions[gid] = r.symbols[i] ?? ""; + }); + setHighlightSymbols((prev) => ({ ...prev, ...additions })); + }) + .catch(() => { + // Non-fatal — tooltips fall back to opaque IDs. + }); + return () => { + cancelled = true; + }; + // Key on the joined opaque-id list so re-renders without a real + // change in highlighted modules don't re-fetch. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [highlightedGeneIds.join("·")]); + + const stats = useMemo(() => { + if (points.length === 0) { + return null; + } + let xMin = Infinity, xMax = -Infinity; + let yMin = Infinity, yMax = -Infinity; + const xs: number[] = []; + for (const p of points) { + if (p.x < xMin) xMin = p.x; + if (p.x > xMax) xMax = p.x; + if (p.y < yMin) yMin = p.y; + if (p.y > yMax) yMax = p.y; + xs.push(p.x); + } + // Pad ranges a bit; force y range to span 0 so the synergy=0 + // guide line is always visible. + const xPad = Math.max(0.005, (xMax - xMin) * 0.05); + const yPad = Math.max(0.005, (yMax - yMin) * 0.08); + const x0 = xMin - xPad; + const x1 = xMax + xPad; + const y0 = Math.min(yMin - yPad, 0); + const y1 = Math.max(yMax + yPad, 0); + // Median GP fitness for the vertical quadrant guide. + const sorted = xs.slice().sort((a, b) => a - b); + const mid = sorted.length === 0 + ? 0 + : sorted.length % 2 === 1 + ? sorted[(sorted.length - 1) >> 1] + : 0.5 * (sorted[sorted.length / 2 - 1] + sorted[sorted.length / 2]); + return { x0, x1, y0, y1, medianX: mid }; + }, [points]); + + const plotW = Math.max(1, width - PAD.left - PAD.right); + const plotH = Math.max(1, HEIGHT - PAD.top - PAD.bottom); + + function project(x: number, y: number, s: NonNullable) { + const px = + PAD.left + ((x - s.x0) / (s.x1 - s.x0)) * plotW; + const py = + PAD.top + plotH - ((y - s.y0) / (s.y1 - s.y0)) * plotH; + return [px, py] as const; + } + + // Paint the canvas whenever data / dimensions change. + useEffect(() => { + const canvas = canvasRef.current; + if (!canvas || !stats) return; + const s = stats; // narrowed for the closure + const dpr = (typeof window !== "undefined" && window.devicePixelRatio) || 1; + canvas.width = Math.round(width * dpr); + canvas.height = Math.round(HEIGHT * dpr); + canvas.style.width = `${width}px`; + canvas.style.height = `${HEIGHT}px`; + const ctx = canvas.getContext("2d"); + if (!ctx) return; + ctx.setTransform(dpr, 0, 0, dpr, 0, 0); + ctx.clearRect(0, 0, width, HEIGHT); + + // Faint background dots first (the "other" group), then the + // highlighted ones on top, then the winner last (drawn as a + // ring) so nothing occludes it. + const layers: Record = { + other: [], + cell_cycle: [], + MMR: [], + p16: [], + immune: [], + winner: [], + }; + for (const p of points) layers[p.cat].push({ x: p.x, y: p.y, m: p.m }); + + // Reset the hit-test array on every repaint; the dots' screen + // positions move with the canvas size. + const hits: Hit[] = []; + + // Background layer: faint grey dots, no site encoding (the data + // doesn't apply — "other" modules aren't reference-tagged anyway). + function paintBackground( + pts: { x: number; y: number; m: RankedModule }[], + fill: string, + radius: number, + alpha: number, + ) { + ctx!.save(); + ctx!.globalAlpha = alpha; + ctx!.fillStyle = fill; + for (const pt of pts) { + const [px, py] = project(pt.x, pt.y, s); + ctx!.beginPath(); + ctx!.arc(px, py, radius, 0, Math.PI * 2); + ctx!.fill(); + } + ctx!.restore(); + } + + // Highlighted layer: encode site-survival on the FILL. Solid = + // survives the oropharynx check (or n too small to test); hollow + // (white fill + coloured stroke) = failed it — signal is at least + // partly location. Hide failed dots when the user has flipped the + // toggle; they're also removed from the hit-test array. + function paintHighlighted( + pts: { x: number; y: number; m: RankedModule }[], + cat: ModuleCategory, + fill: string, + stroke: string, + radius: number, + alpha: number, + ) { + ctx!.save(); + ctx!.globalAlpha = alpha; + for (const pt of pts) { + const failsSite = pt.m.survives_site === false; + if (hideFailedSite && failsSite) continue; + const [px, py] = project(pt.x, pt.y, s); + ctx!.beginPath(); + ctx!.arc(px, py, radius, 0, Math.PI * 2); + ctx!.fillStyle = failsSite ? "#FFFFFF" : fill; + ctx!.fill(); + ctx!.strokeStyle = stroke; + ctx!.lineWidth = failsSite ? 1.6 : 1.2; + ctx!.stroke(); + hits.push({ cx: px, cy: py, cat, m: pt.m }); + } + ctx!.restore(); + } + + const palP16 = RANK_COLORS.p16; + const palImm = RANK_COLORS.immune; + const palCC = RANK_COLORS.cell_cycle; + const palMMR = RANK_COLORS.MMR; + const palWin = RANK_COLORS.winner; + + paintBackground(layers.other, "#A8AEB5", 1.4, 0.18); + paintHighlighted(layers.cell_cycle, "cell_cycle", palCC.fill, palCC.stroke, 3.4, 0.85); + paintHighlighted(layers.MMR, "MMR", palMMR.fill, palMMR.stroke, 3.4, 0.85); + paintHighlighted(layers.p16, "p16", palP16.fill, palP16.stroke, 4.0, 0.95); + paintHighlighted(layers.immune, "immune", palImm.fill, palImm.stroke, 4.0, 0.95); + // Winner — a teal ring at radius 7.5 with a coloured centre at + // radius 4.5. If the winner's group failed the site check (rare + // — the engine usually picks survives_site=true), hollow the + // centre too (white fill) so the encoding is consistent. + for (const pt of layers.winner) { + const failsSite = pt.m.survives_site === false; + if (hideFailedSite && failsSite) continue; + const [px, py] = project(pt.x, pt.y, s); + ctx.save(); + ctx.fillStyle = failsSite ? "#FFFFFF" : palWin.fill; + ctx.beginPath(); + ctx.arc(px, py, 4.5, 0, Math.PI * 2); + ctx.fill(); + ctx.lineWidth = 2; + ctx.strokeStyle = palWin.stroke; + ctx.beginPath(); + ctx.arc(px, py, 4.5, 0, Math.PI * 2); + ctx.stroke(); + ctx.beginPath(); + ctx.arc(px, py, 7.5, 0, Math.PI * 2); + ctx.stroke(); + ctx.restore(); + hits.push({ cx: px, cy: py, cat: "winner", m: pt.m }); + } + hitsRef.current = hits; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [points, stats, width, plotW, plotH, hideFailedSite]); + + if (!stats || points.length === 0) { + return null; + } + + // Build axis ticks: 4 evenly-spaced ticks on each axis, formatted + // to 2 decimals. + function ticks(a: number, b: number, n: number): number[] { + const out: number[] = []; + for (let i = 0; i <= n; i++) out.push(a + (i / n) * (b - a)); + return out; + } + const xTicks = ticks(stats.x0, stats.x1, 4); + const yTicks = ticks(stats.y0, stats.y1, 4); + const medianXpx = project(stats.medianX, 0, stats)[0]; + const zeroYpx = project(0, 0, stats)[1]; + + // Quadrant captions — placed inside each quadrant near a corner. + const captions = [ + { + x: PAD.left + 6, + y: PAD.top + 14, + text: "real teamwork", + sub: "(unexplored by the engine)", + anchor: "start" as const, + }, + { + x: PAD.left + plotW - 6, + y: PAD.top + 14, + text: "ideal — usually empty", + sub: "(engine + teamwork)", + anchor: "end" as const, + }, + { + x: PAD.left + 6, + y: PAD.top + plotH - 8, + text: "weak / junk", + sub: "", + anchor: "start" as const, + }, + { + x: PAD.left + plotW - 6, + y: PAD.top + plotH - 8, + text: "engine's lone-gene detectors", + sub: "", + anchor: "end" as const, + }, + ]; + + // Per-dataset legend entries (single-marker first, broader second). + const legend: { label: string; pal: RankPalette; ring?: boolean }[] = [ + { label: "all explored groups", pal: { stroke: "#7B838C", fill: "#A8AEB5", bg: "#FFFFFF" } }, + { label: "winner & picks", pal: RANK_COLORS.winner, ring: true }, + ]; + if (dataset === "hnsc") { + legend.push({ label: "p16 (CDKN2A)", pal: RANK_COLORS.p16 }); + legend.push({ label: "cell_cycle", pal: RANK_COLORS.cell_cycle }); + } else { + legend.push({ label: "immune", pal: RANK_COLORS.immune }); + legend.push({ label: "MMR", pal: RANK_COLORS.MMR }); + } + + return ( +
    +
    +
    + Every group the engine explored + +
    + +
    +

    + Every group the engine explored. Left–right = how much the + engine liked it; up–down = how much the group beats its best + single gene. Highlighted dots show where the known HPV genes + fall. + + Solid{" "} + dots survive the site check;{" "} + hollow{" "} + dots fail it (their signal is partly location). + +

    + +
    { + const el = wrapperRef.current; + if (!el) return; + const rect = el.getBoundingClientRect(); + const cx = e.clientX - rect.left; + const cy = e.clientY - rect.top; + let nearest: Hit | null = null; + let best = 8 * 8; // pick within ~8px + for (const h of hitsRef.current) { + const dx = h.cx - cx; + const dy = h.cy - cy; + const d2 = dx * dx + dy * dy; + if (d2 < best) { + best = d2; + nearest = h; + } + } + if (nearest) { + setHover({ + cat: nearest.cat, + m: nearest.m, + cx: nearest.cx, + cy: nearest.cy, + }); + } else if (hover) { + setHover(null); + } + }} + onMouseLeave={() => setHover(null)} + > + + + {/* Plot frame */} + + + {/* Y gridlines + tick labels */} + {yTicks.map((t, i) => { + const [, py] = project(stats.x0, t, stats); + return ( + + + + {t.toFixed(2)} + + + ); + })} + {/* X gridlines + tick labels */} + {xTicks.map((t, i) => { + const [px] = project(t, stats.y0, stats); + return ( + + + + {t.toFixed(2)} + + + ); + })} + + {/* Quadrant guides: median GP fitness (vertical) + synergy=0 + (horizontal). */} + + + + median genetic-programming fitness + + + synergy = 0 + + + {/* Quadrant captions */} + {captions.map((c, i) => ( + + + {c.text} + + {c.sub && ( + + {c.sub} + + )} + + ))} + + {/* Axis titles */} + + Genetic-programming fitness — what the engine preferred → + + + Synergy — teamwork beyond best gene ↑ + + + + {hover && ( + + )} +
    + + {/* Legend */} +
    + {legend.map((it) => ( + + {it.ring ? ( + + ) : ( + + )} + {it.label} + + ))} + + + hollow = fails site check + +
    +
    + ); +} + +// Hover tooltip for the fitness × synergy scatter. Renders next to +// the hovered HIGHLIGHTED dot — never for the faint background dots, +// since those aren't revealed. Clamps so it stays inside the chart +// rect even when the dot is near the edge. +function ScatterTooltip({ + hover, + symbolByOpaque, + plotW, + plotH, + pad, + width, + height, +}: { + hover: { + cat: ModuleCategory; + m: RankedModule; + cx: number; + cy: number; + }; + symbolByOpaque: Record; + plotW: number; + plotH: number; + pad: { top: number; right: number; bottom: number; left: number }; + width: number; + height: number; +}) { + const { m, cat, cx, cy } = hover; + // Palette per category. "winner" uses the teal accent; ref-set + // categories reuse the existing RANK_COLORS map; "other" should + // not appear (we don't hit-test it). + const pal = + cat === "winner" + ? RANK_COLORS.winner + : cat === "p16" + ? RANK_COLORS.p16 + : cat === "immune" + ? RANK_COLORS.immune + : cat === "cell_cycle" + ? RANK_COLORS.cell_cycle + : RANK_COLORS.MMR; + const label = + cat === "winner" ? "winner" : (cat as string); + + // Tooltip dimensions are approximate; we clamp the corner so the + // box always stays inside the chart rect. Position 12px to the + // right of the dot, biased up; flip if it would overflow. + const TT_W = 220; + const TT_H = 100; + let left = cx + 14; + let top = cy - 14 - TT_H; + const leftMax = pad.left + plotW - TT_W - 4; + if (left > leftMax) left = Math.max(pad.left + 4, cx - TT_W - 14); + if (top < pad.top + 4) top = cy + 14; + if (top + TT_H > pad.top + plotH + 4) top = Math.max(pad.top + 4, top - TT_H); + // Final clamp to the wrapper bounds. + left = Math.min(Math.max(left, 4), width - TT_W - 4); + top = Math.min(Math.max(top, 4), height - TT_H - 4); + + const syn = synergyOf(m); + const symbols = m.gene_ids.map((g) => { + const s = symbolByOpaque[g]; + return s && s.length > 0 ? s : g; + }); + const allRevealed = m.gene_ids.every( + (g) => g in symbolByOpaque && symbolByOpaque[g].length > 0, + ); + + return ( +
    +
    + + {label} + + {m.size} gene{m.size === 1 ? "" : "s"} +
    +
    + {allRevealed ? symbols.join(", ") : "(revealing…)"} +
    +
    +
    Genetic-programming fitness
    +
    + {fmtFit(m.gp_fitness ?? null, 3)} +
    +
    Combined AUROC
    +
    + {fmtFit(m.combined_holdout, 3)} +
    +
    Coherence
    +
    + {fmtFit(m.coherence, 3)} +
    +
    Synergy
    +
    + {fmtFit(syn, 3)} +
    +
    +
    + ); +} + +// Ranked-modules panel: shown for coherence-on supervised runs. Each +// row is a coordinated gene module the engine explored, scored by +// combined held-out AUROC on the SAME train/test split the run used. +// Symbols are auto-revealed for the visible page in one batched +// /reveal call — bounded by page size, never the whole map. +const MODULE_PAGE_SIZE = 25; + +function ModuleRankingPanel({ + runId, + dataset, + target, + winnerGeneIds, +}: { + runId: string; + dataset: DatasetId; + target: Target; + winnerGeneIds: string[]; +}) { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + type SortKey = "gp_fitness" | "combined" | "coherence" | "synergy"; + // Default sort: the engine's own preference order (what the search + // actually picked). The other three are after-the-fact re-score + // lenses. + const [sortKey, setSortKey] = useState("gp_fitness"); + const [page, setPage] = useState(0); + // Symbol cache keyed by opaque ID — accumulates across pages so + // re-visiting a page is free. + const [symbolByOpaque, setSymbolByOpaque] = useState>( + {}, + ); + const [expandedIx, setExpandedIx] = useState(null); + + useEffect(() => { + let cancelled = false; + setLoading(true); + setError(null); + setData(null); + setSymbolByOpaque({}); + setExpandedIx(null); + setPage(0); + getRunModules(runId) + .then((d) => { + if (!cancelled) setData(d); + }) + .catch((e) => { + if (!cancelled) setError(e instanceof Error ? e.message : String(e)); + }) + .finally(() => { + if (!cancelled) setLoading(false); + }); + return () => { + cancelled = true; + }; + }, [runId]); + + // Synergy / set categorisation are shared with the scatter; lift to + // top-level helpers (defined just above this file's module-panel + // section) so both views share one source of truth. + + const sortedModules = useMemo(() => { + if (!data) return []; + const xs = [...data.modules]; + if (sortKey === "coherence") { + xs.sort((a, b) => (b.coherence ?? -Infinity) - (a.coherence ?? -Infinity)); + } else if (sortKey === "gp_fitness") { + xs.sort( + (a, b) => (b.gp_fitness ?? -Infinity) - (a.gp_fitness ?? -Infinity), + ); + } else if (sortKey === "synergy") { + xs.sort( + (a, b) => (synergyOf(b) ?? -Infinity) - (synergyOf(a) ?? -Infinity), + ); + } else { + xs.sort( + (a, b) => + (b.combined_holdout ?? -Infinity) - (a.combined_holdout ?? -Infinity), + ); + } + return xs; + // synergyOf depends only on RankedModule's stable per-gene fields, + // so we leave it out of the dep list. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [data, sortKey]); + + // Re-sort or re-load resets the page + collapses any open row so an + // expanded "rank N" row doesn't strand once the underlying ranking + // shifts. + useEffect(() => { + setPage(0); + setExpandedIx(null); + }, [sortKey, data]); + + // Winner-set equality: unordered set of opaque IDs equal to the + // winner's gene_ids. Used to badge that exact module — connects the + // group view back to the Result cards. + const winnerSet = useMemo>(() => new Set(winnerGeneIds), [ + winnerGeneIds, + ]); + function isWinnerSet(geneIds: string[]): boolean { + if (geneIds.length !== winnerSet.size) return false; + for (const g of geneIds) if (!winnerSet.has(g)) return false; + return true; + } + + const pageStart = page * MODULE_PAGE_SIZE; + const pageEnd = Math.min(pageStart + MODULE_PAGE_SIZE, sortedModules.length); + const pageModules = sortedModules.slice(pageStart, pageEnd); + const nPages = Math.max( + 1, + Math.ceil(sortedModules.length / MODULE_PAGE_SIZE), + ); + + // Auto-reveal the visible page's symbols in ONE batched /reveal — + // bounded to ≤ MODULE_PAGE_SIZE × max_genes_per_set IDs. Skips + // already-cached IDs. + useEffect(() => { + if (pageModules.length === 0) return; + const needed = new Set(); + for (const m of pageModules) { + for (const g of m.gene_ids) { + if (!(g in symbolByOpaque)) needed.add(g); + } + } + if (needed.size === 0) return; + const ids = Array.from(needed); + let cancelled = false; + postReveal(ids) + .then((r) => { + if (cancelled) return; + const additions: Record = {}; + ids.forEach((gid, i) => { + additions[gid] = r.symbols[i] ?? ""; + }); + setSymbolByOpaque((prev) => ({ ...prev, ...additions })); + }) + .catch((e) => { + // Non-fatal — symbols just stay "(revealing…)" for this batch. + console.error("module page reveal failed", e); + }); + return () => { + cancelled = true; + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [page, sortKey, data]); + + function toggle(localIx: number) { + const globalIx = pageStart + localIx; + setExpandedIx(globalIx === expandedIx ? null : globalIx); + } + + return ( + + {loading && ( +

    computing module ranks…

    + )} + {error && ( +

    + {error} +

    + )} + + {data && ( + <> +
    + + {data.n_modules}{" "} + distinct modules · train{" "} + {data.n_train}, test{" "} + {data.n_test} + +
    + sort: + {( + [ + ["gp_fitness", "Genetic-programming fitness"], + ["combined", "Combined AUROC"], + ["coherence", "Coherence"], + ["synergy", "Synergy"], + ] as [SortKey, string][] + ).map(([k, label]) => ( + + ))} +
    +
    + + {(data.subgroups?.site || data.subgroups?.purity) && ( +
    + + Survival flags shown for the held-out subgroups — + {data.subgroups?.site && ( + <> + {" site "} + + ({data.subgroups.site.kind}, n={data.subgroups.site.n}) + + + )} + {data.subgroups?.site && data.subgroups?.purity && " · "} + {data.subgroups?.purity && ( + <> + {" purity "} + + ({data.subgroups.purity.kind}, n={data.subgroups.purity.n}) + + + )} + . + + +
    + )} + + + +
    + + + + + + + + + + + + {(data.subgroups?.site || data.subgroups?.purity) && ( + + )} + + + + {pageModules.map((m, localIx) => { + const globalIx = pageStart + localIx; + const open = expandedIx === globalIx; + // Tint by the most-specific matched set: single-marker + // sets (p16 / immune — deep gold) win over broader + // proliferation sets (cell_cycle / MMR — muted amber). + const primarySet = + m.ref_sets.find((s) => s === "p16" || s === "immune") ?? + m.ref_sets.find( + (s) => s === "cell_cycle" || s === "MMR", + ) ?? + m.ref_sets[0]; + const rowBg = primarySet + ? colorForSet(primarySet).bg + : undefined; + const winner = isWinnerSet(m.gene_ids); + return ( + + toggle(localIx)} + aria-expanded={open} + > + + + + + + + + + {(data.subgroups?.site || data.subgroups?.purity) && ( + + )} + + {open && ( + + + + )} + + ); + })} + +
    #Genetic-programming fitness + {data.metric_kind === "auroc" + ? "Combined AUROC" + : "Combined |ρ|"} + CoherenceSynergySizeGenesTags + + Survives + {/* Multi-paragraph copy doesn't fit a hover + tooltip — use the rich modal infrastructure + so it never clips. */} + + +
    + + {open ? "▼" : "▶"} + + {globalIx + 1} + + {fmtFit(m.gp_fitness ?? null, 3)} + + {fmtFit(m.combined_holdout, 3)} + + {fmtFit(m.coherence, 3)} + + {fmtFit(synergyOf(m), 3)} + + {m.size} + + + +
    + {winner && ( + + winner + + )} + {m.ref_sets.map((s) => { + const pal = colorForSet(s); + return ( + + {s} + + ); + })} +
    +     		+ +
    + + {m.best_program_repr && ( + + )} + {(data.subgroups?.site || data.subgroups?.purity) && ( + + )} +
    +
    + + + + )} +     + ); +} + +function ModulePager({ + page, + nPages, + pageStart, + pageEnd, + total, + onPage, +}: { + page: number; + nPages: number; + pageStart: number; + pageEnd: number; + total: number; + onPage: (p: number) => void; +}) { + if (nPages <= 1) return null; + return ( +
    + + {(pageStart + 1).toLocaleString()}–{pageEnd.toLocaleString()} + of {total.toLocaleString()} + +
    + + + + page {page + 1} / {nPages} + + + +
    +
    + ); +} + +function ModuleSymbolList({ + geneIds, + symbolByOpaque, + datasetRefSets, +}: { + geneIds: string[]; + symbolByOpaque: Record; + datasetRefSets: Record; +}) { + const setsBySymbol = useMemo>(() => { + const out = new Map(); + for (const [name, syms] of Object.entries(datasetRefSets)) { + for (const s of syms ?? []) { + if (!out.has(s)) out.set(s, name); + } + } + return out; + }, [datasetRefSets]); + + // Truncate long modules inline; expanding the row shows them all. + const MAX_INLINE = 8; + const head = geneIds.slice(0, MAX_INLINE); + const overflow = Math.max(0, geneIds.length - MAX_INLINE); + return ( + + {head.map((gid, i) => { + const sym = symbolByOpaque[gid]; + const setName = sym ? setsBySymbol.get(sym) : undefined; + const pal = setName ? colorForSet(setName) : null; + return ( + + + {sym && sym.length > 0 + ? sym + : sym === "" + ? "—" + : "(revealing…)"} + + {i < head.length - 1 ? ", " : null} + + ); + })} + {overflow > 0 && ( + · +{overflow} + )} + + ); +} + +// Compact ✓ / ✗ / "n too small" chips for a module's site + purity +// survival flags. Both are HNSC/HPV-only — for other datasets we don't +// render the column at all. +function SurvivalChips({ + module, + subgroups, +}: { + module: RankedModule; + subgroups: ModuleRanking["subgroups"]; +}) { + // Full-cohort baseline ("pre" in the pre→post pairing). Reused by + // every chip so the user sees what each subgroup is being compared to. + const pre = module.combined_holdout; + function chip( + label: string, + survives: boolean | null | undefined, + post: number | null | undefined, + title: string, + ) { + let bg = "#F4F2EE"; + let stroke = "#6E7F8C"; + let glyph: string; + if (survives === true) { + bg = "#E3F2EA"; + stroke = "#2F6E4C"; + glyph = "✓"; + } else if (survives === false) { + bg = "#FBE6E6"; + stroke = "#A64242"; + glyph = "✗"; + } else { + // n too small / not applicable + bg = "#F4F2EE"; + stroke = "#9AA0A6"; + glyph = "—"; + } + // pre → post pair. When the subgroup AUROC is null ("n too small") + // we drop the arrow + post; the "—" glyph already signals N/A. + const preText = + pre != null && Number.isFinite(pre) ? fmtFit(pre, 2) : null; + const postText = + post != null && Number.isFinite(post) ? fmtFit(post, 2) : null; + let valueText = ""; + if (preText && postText) { + const delta = pre! - post!; + const deltaText = + Number.isFinite(delta) + ? ` (${delta >= 0 ? "−" : "+"}${Math.abs(delta).toFixed(2)})` + : ""; + valueText = ` ${preText} → ${postText}${deltaText}`; + } else if (preText && !postText) { + valueText = ` ${preText} → —`; + } else if (postText && !preText) { + valueText = ` ${postText}`; + } + return ( + + {label} {glyph}{valueText} + + ); + } + return ( +
    + {subgroups?.site && chip( + "site", + module.survives_site, + module.combined_holdout_oropharynx, + module.survives_site === null || module.survives_site === undefined + ? `Site (oropharynx) subgroup too small to score honestly (n=${module.n_holdout_oropharynx ?? 0}).` + : `Module recomputed within the oropharynx held-out subgroup: full-cohort AUROC ${fmtFit(pre, 3)} → subgroup AUROC ${fmtFit(module.combined_holdout_oropharynx, 3)} over n=${module.n_holdout_oropharynx ?? 0} (HPV+ ${module.n_pos_oropharynx ?? 0} / HPV− ${module.n_neg_oropharynx ?? 0}). Survives = within 0.05 of the full-cohort AUROC.`, + )} + {subgroups?.purity && chip( + "purity", + module.survives_purity, + module.combined_holdout_highpurity, + module.survives_purity === null || module.survives_purity === undefined + ? `High-purity (bottom-tertile immune proxy) subgroup too small to score honestly (n=${module.n_holdout_highpurity ?? 0}; HPV+ ${module.n_pos_highpurity ?? 0} / HPV− ${module.n_neg_highpurity ?? 0}).` + : `Module recomputed within the high-purity (bottom-tertile immune proxy) held-out subgroup: full-cohort AUROC ${fmtFit(pre, 3)} → subgroup AUROC ${fmtFit(module.combined_holdout_highpurity, 3)} over n=${module.n_holdout_highpurity ?? 0} (HPV+ ${module.n_pos_highpurity ?? 0} / HPV− ${module.n_neg_highpurity ?? 0}). Survives = within 0.05 of the full-cohort AUROC.`, + )} +
    + ); +} + +// Expanded-row strip: the stratified AUROCs in numbers next to the +// per-gene metrics table, so users can see exactly how far the module +// dropped in each subgroup. +function StratifiedAUROCStrip({ + module, + subgroups, + metricKind, +}: { + module: RankedModule; + subgroups: ModuleRanking["subgroups"]; + metricKind: "auroc" | "spearman"; +}) { + const label = metricKind === "auroc" ? "AUROC" : "|ρ|"; + function row( + name: string, + value: number | null | undefined, + n: number | undefined, + n_pos: number | undefined, + n_neg: number | undefined, + ) { + if (n == null && value == null) return null; + return ( +
    + {name} + + {label} {fmtFit(value, 3)} + + + n={n ?? 0} + {metricKind === "auroc" && n_pos != null && n_neg != null + ? ` (HPV+ ${n_pos} / HPV− ${n_neg})` + : ""} + +
    + ); + } + return ( +
    +
    + Stratified held-out AUROCs (survival check) +
    +
    + full + + {label} {fmtFit(module.combined_holdout, 3)} + +
    + {subgroups?.site && row( + "oropharynx", + module.combined_holdout_oropharynx, + module.n_holdout_oropharynx, + module.n_pos_oropharynx, + module.n_neg_oropharynx, + )} + {subgroups?.purity && row( + "high purity", + module.combined_holdout_highpurity, + module.n_holdout_highpurity, + module.n_pos_highpurity, + module.n_neg_highpurity, + )} +
    + ); +} + +// The actual best-fitness program tree for a Groups-table row. +// Re-uses the shared renderer (same one the Result +// panel uses for the winner) so the visual matches. The module's +// genes were already revealed in the table-row expand, so the tree +// can show their symbols too — no additional reveal needed. +function ModuleBestProgram({ + programRepr, + outputLabel, + symbolByOpaque, +}: { + programRepr: string; + outputLabel: string; + symbolByOpaque: Record; +}) { + // Adapt the flat opaque→symbol map to the {symbol, matched} shape + // ProgramGraph expects. `matched` is false here — the reference- + // set highlight is for the known-marker recovery diagnostic, not + // for a group's own gene tree. + const reveal = useMemo< + Record + >(() => { + const out: Record = {}; + for (const [opq, sym] of Object.entries(symbolByOpaque)) { + if (sym && sym.length > 0) { + out[opq] = { symbol: sym, matched: false }; + } + } + return out; + }, [symbolByOpaque]); + + return ( +
    +
    + This group’s best program +
    +

    + The actual tree of the candidate that earned the + Genetic-programming fitness above (argmax over the + persisted population for this gene-set). +

    +
    + +
    +
    +
    + program (raw) +
    + +
    +
    + {programRepr} +
    +
    + ); +} + +function ModuleGeneTable({ + module, + symbolByOpaque, + datasetRefSets, + metricKind, +}: { + module: RankedModule; + symbolByOpaque: Record; + datasetRefSets: Record; + metricKind: "auroc" | "spearman"; +}) { + // Symbol → ref-set names lookup so each row can be tinted by the + // specific set its gene belongs to (p16 deep gold vs cell_cycle + // muted amber, etc). + const setsBySymbol = useMemo>(() => { + const out = new Map(); + for (const [name, syms] of Object.entries(datasetRefSets)) { + for (const s of syms ?? []) { + // First-write-wins; the priority below also normalises this + // so p16 (single canonical) outranks cell_cycle for a gene + // that's in both (CDKN2A isn't, but a future split could be). + if (!out.has(s)) out.set(s, name); + } + } + return out; + }, [datasetRefSets]); + + return ( +
    + + + + + + + + + + + + {module.per_gene.map((g) => { + const sym = symbolByOpaque[g.id]; + const setName = sym ? setsBySymbol.get(sym) : undefined; + const pal = setName ? colorForSet(setName) : null; + return ( + + + + + + + + ); + })} + +
    Opaque IDSymbolSource + {metricKind === "auroc" ? "Single-gene AUROC" : "Single-gene ρ"} + Rank / N
    {g.id} + {sym && sym.length > 0 + ? sym + : sym === "" + ? "—" + : "(revealing…)"} + + {setName ? ( + + {setName} + + ) : ( + — + )} + + {fmtFit(g.single_gene_metric, 3)} + + {g.rank != null && g.total != null + ? `${g.rank} / ${g.total.toLocaleString()}` + : "—"} +
    +
    + ); +} + +function RankList({ + title, + subtitle, + rows, + total, +}: { + title: string; + subtitle: string; + rows: GeneRankRow[]; + total: number; +}) { + return ( +
    +
    + {title} +
    +

    {subtitle}

    +
      + {rows.map((r) => ( +
    • + {r.symbol} + {r.present ? ( + + {r.corr!.toFixed(4)} + {" · "} + {r.rank} + {` / ${total}`} + + ) : ( + not present + )} +
      • + ))} +
    +
    + ); +} + +function OverlapSummary({ + k, + n, + setName, + matchedSymbols, + mlh1, + totalGenes, +}: { + k: number; + n: number; + setName: ReferenceSetKey; + matchedSymbols: string[]; + mlh1?: GeneRankRow; + totalGenes?: number; +}) { + // For the TMB objective, soften the overlap line: programs rarely + // contain MMR genes literally, but the diagnostic shows MLH1 itself + // sits near the top of the negative-TMB ranking. + const mlh1Tail = + mlh1?.present && mlh1.rank != null && totalGenes + ? ( + + Winning programs rarely ARE the MMR genes (many co-silenced genes + outrank them) — but MLH1 itself sits in the top{" "} + {Math.round((mlh1.rank / totalGenes) * 1000) / 10}%. See the + diagnostic below. + + ) + : null; + + if (k === 0) { + return ( + + + No matches — none of the {n} winning gene{n === 1 ? "" : "s"}{" "} + {n === 1 ? "is" : "are"} in the{" "} + {setName} set. + + {mlh1Tail} + + ); + } + return ( + + + {k} of {n} winning genes{" "} + {k === 1 ? "is" : "are"} in the{" "} + {setName} set:{" "} + {matchedSymbols.join(", ")} + + {mlh1Tail} + + ); +} diff --git a/web/app/MMRBiologyPanel.tsx b/web/app/MMRBiologyPanel.tsx new file mode 100644 index 0000000000000000000000000000000000000000..723fa9ffbf23a2a6ea6196ae7cbd90fe70d8e159 --- /dev/null +++ b/web/app/MMRBiologyPanel.tsx @@ -0,0 +1,185 @@ +"use client"; + +import { useId, useState } from "react"; + +/** + * "What is MMR? (the biology behind MSI)" — a collapsed disclosure that + * mirrors ParameterFlow.tsx's open/close pattern exactly. Closed on + * first render; the embedded SVG (verbatim from repo-root + * `mmr_reference.svg`) scales via its viewBox so the panel never + * overflows. + * + * Presentation only. + */ +export default function MMRBiologyPanel() { + const [open, setOpen] = useState(false); + const panelId = useId(); + return ( +
    + + {open && ( +
    + +
    + )} +
    + ); +} + +// Verbatim contents of repo-root `mmr_reference.svg`. Wrapped in a +// width:100% / height:auto container so the SVG's viewBox handles +// scaling — no horizontal overflow at narrow widths. +function MMRDiagram() { + return ( +
    + + MMR in cancer — a reference + + MMR (the cause) at top; high MMR activity gives MSS, low gives + MSI-H; each state then drives the immune response (MSS cold, + MSI-H hot). Plus an MSI-H/MSS readout and what gene expression + can detect. + + + + + + + + + + 1 · The biology{" "} + + (top → bottom = cause → effect) + + + + + + MMR — the spell-checker + + + MLH1 · MSH2 · MSH6 · PMS2 + + + + + MSS — spell-checker WORKING + + + stable DNA · low TMB + + + + + MSI-H — BROKEN (dMMR) + + + unstable DNA · HIGH TMB + + + → responds to immunotherapy + + + + + Immune system + + + CD8A · GZMA · PRF1 (T-cell attack) + + + + MMR HIGH + + MMR LOW + + + immune LOW + + immune HIGH + + + 2 · MSI-H vs MSS readout + + + + MSI-H (broken) + + + MSS (working) + + + MMR activity + LOW + normal / HIGH + TMB (mutation count) + ↑ HIGH + ↓ LOW + immune attack (CD8A…) + ↑ hot + cold + + immunotherapy + responds well + poor + + + + 3 · What's in TCGA, and what the engine uses + + + + Gene expression — the engine reads THIS. + {" "}Visible: immune ↑ (CD8A…), and MLH1 ↓ (it is + + + silenced → findable). TMB & MSI come in as per-patient labels. + + + + Invisible to expression: MSH2 / MSH6 / PMS2 break by MUTATION, not silencing — their + + + expression doesn't change, so they can't be found from expression. (Why only MLH1 surfaces.) + + +
    + ); +} diff --git a/web/app/ParamHelp.tsx b/web/app/ParamHelp.tsx new file mode 100644 index 0000000000000000000000000000000000000000..5227d9b304a6f7a3d541155b36639e1ca47706d3 --- /dev/null +++ b/web/app/ParamHelp.tsx @@ -0,0 +1,88 @@ +"use client"; + +import { useEffect, useId, useRef, useState } from "react"; +import { PARAM_HELP, ParamKey } from "./paramHelpContent"; +import { useParamHelp } from "./ParamHelpProvider"; + +interface Props { + paramKey: ParamKey; + label?: string; +} + +/** + * "?" trigger for a Parameters help entry. + * + * - Hover / focus shows the SHORT one-liner as a small popover tooltip + * (same look-and-feel as the existing InfoTip). + * - Click opens the rich detailed modal via ParamHelpProvider. + * + * On modal close, focus returns to this button. + */ +export default function ParamHelp({ paramKey, label }: Props) { + const [tipOpen, setTipOpen] = useState(false); + const wrapperRef = useRef(null); + const buttonRef = useRef(null); + const tooltipId = useId(); + const meta = PARAM_HELP[paramKey]; + const { open } = useParamHelp(); + + useEffect(() => { + if (!tipOpen) return; + function onDown(e: MouseEvent | TouchEvent) { + if (wrapperRef.current && !wrapperRef.current.contains(e.target as Node)) { + setTipOpen(false); + } + } + function onKey(e: KeyboardEvent) { + if (e.key === "Escape") setTipOpen(false); + } + document.addEventListener("mousedown", onDown); + document.addEventListener("touchstart", onDown); + document.addEventListener("keydown", onKey); + return () => { + document.removeEventListener("mousedown", onDown); + document.removeEventListener("touchstart", onDown); + document.removeEventListener("keydown", onKey); + }; + }, [tipOpen]); + + return ( + setTipOpen(true)} + onMouseLeave={() => setTipOpen(false)} + > + + {tipOpen && ( + + {meta.short} + + click for details + + + )} + + ); +} diff --git a/web/app/ParamHelpModal.tsx b/web/app/ParamHelpModal.tsx new file mode 100644 index 0000000000000000000000000000000000000000..37b1d5617bf5abc0eb85a3aafe65ca038c9022e1 --- /dev/null +++ b/web/app/ParamHelpModal.tsx @@ -0,0 +1,142 @@ +"use client"; + +import { useEffect, useId, useRef } from "react"; +import { createPortal } from "react-dom"; +import { ParamHelpEntry } from "./paramHelpContent"; + +interface Props { + entry: ParamHelpEntry; + onClose: () => void; +} + +const FOCUSABLE = + 'a[href],button:not([disabled]),textarea,input,select,[tabindex]:not([tabindex="-1"])'; + +/** + * Centered dialog with backdrop. Closes on X, Esc, or backdrop click. + * Focus is trapped inside the modal and returns to the trigger on + * close (the provider handles the return). + */ +export default function ParamHelpModal({ entry, onClose }: Props) { + const titleId = useId(); + const containerRef = useRef(null); + + // Lock body scroll while open. + useEffect(() => { + const prev = document.body.style.overflow; + document.body.style.overflow = "hidden"; + return () => { + document.body.style.overflow = prev; + }; + }, []); + + // Esc + focus trap. + useEffect(() => { + function onKey(e: KeyboardEvent) { + if (e.key === "Escape") { + e.stopPropagation(); + onClose(); + return; + } + if (e.key === "Tab" && containerRef.current) { + const focusables = containerRef.current.querySelectorAll( + FOCUSABLE, + ); + if (focusables.length === 0) return; + const first = focusables[0]; + const last = focusables[focusables.length - 1]; + const active = document.activeElement as HTMLElement | null; + if (e.shiftKey) { + if (active === first || !containerRef.current.contains(active)) { + e.preventDefault(); + last.focus(); + } + } else { + if (active === last) { + e.preventDefault(); + first.focus(); + } + } + } + } + document.addEventListener("keydown", onKey); + return () => document.removeEventListener("keydown", onKey); + }, [onClose]); + + // Focus the close button once mounted. + useEffect(() => { + const f = containerRef.current?.querySelector(FOCUSABLE); + f?.focus(); + }, []); + + if (typeof document === "undefined") return null; + + const node = ( +
    { + if (e.target === e.currentTarget) onClose(); + }} + style={{ + position: "fixed", + inset: 0, + zIndex: 100, + background: "rgba(35,48,58,0.32)", + display: "flex", + alignItems: "flex-start", + justifyContent: "center", + padding: "min(8vh,72px) 16px", + overflowY: "auto", + }} + > +
    +
    +

    + {entry.title} +

    + +
    +
    + {entry.detailed} +
    +
    +
    + ); + + return createPortal(node, document.body); +} diff --git a/web/app/ParamHelpProvider.tsx b/web/app/ParamHelpProvider.tsx new file mode 100644 index 0000000000000000000000000000000000000000..07b267382dccc402b9e9f5ce1bc4b4ca63a4460a --- /dev/null +++ b/web/app/ParamHelpProvider.tsx @@ -0,0 +1,67 @@ +"use client"; + +import { + createContext, + ReactNode, + useCallback, + useContext, + useRef, + useState, +} from "react"; +import { PARAM_HELP, ParamKey } from "./paramHelpContent"; +import ParamHelpModal from "./ParamHelpModal"; + +type Ctx = { + open: (key: ParamKey, returnFocusTo?: HTMLElement | null) => void; + close: () => void; + current: ParamKey | null; +}; + +const ParamHelpCtx = createContext(null); + +export function useParamHelp(): Ctx { + const ctx = useContext(ParamHelpCtx); + if (!ctx) { + throw new Error( + "useParamHelp(): ParamHelpProvider missing — wrap the page once near the root.", + ); + } + return ctx; +} + +export default function ParamHelpProvider({ children }: { children: ReactNode }) { + const [current, setCurrent] = useState(null); + const returnRef = useRef(null); + + const open = useCallback( + (key: ParamKey, returnFocusTo: HTMLElement | null = null) => { + returnRef.current = returnFocusTo; + setCurrent(key); + }, + [], + ); + + const close = useCallback(() => { + setCurrent(null); + const target = returnRef.current; + if (target) { + // Defer focus to the next tick so the modal has unmounted. + setTimeout(() => { + try { + target.focus(); + } catch { + /* ignore */ + } + }, 0); + } + }, []); + + return ( + + {children} + {current && ( + + )} + + ); +} diff --git a/web/app/ParameterFlow.tsx b/web/app/ParameterFlow.tsx new file mode 100644 index 0000000000000000000000000000000000000000..fc45d55969e18d53c16325dedb05ed37c22eb010 --- /dev/null +++ b/web/app/ParameterFlow.tsx @@ -0,0 +1,367 @@ +"use client"; + +import { useId, useRef, useState } from "react"; +import { ParamKey } from "./paramHelpContent"; +import { useParamHelp } from "./ParamHelpProvider"; + +/** + * "How these parameters relate" — a collapsed disclosure with a clean + * HTML/flex pipeline diagram inside. Presentation only. + * + * Centered single-column layout, max-width 460px, width 100% — never + * wider than the panel, never overflowing. Each step is a neutral box; + * between consecutive boxes sits a coloured "pill" naming the + * parameter(s) that govern that transition, with a calm ↓ between + * pill and the next box. + */ +export default function ParameterFlow() { + const [open, setOpen] = useState(false); + const panelId = useId(); + return ( +
    + + {open && ( +
    + + +
    + )} +
    + ); +} + +// ---------- diagram model ------------------------------------------------ + +type PillKind = "SPACE" | "EFFORT" | "VALIDATION"; + +interface Box { + title: string; + sub?: string; + amber?: boolean; +} + +/** A pill is a list of inline fragments: either a plain string or a + * {bold} fragment so we can highlight just the parameter name. The + * bold fragments carry an optional ``paramKey`` — when present, that + * fragment is rendered as a button that opens the ParamHelp modal. */ +type Frag = string | { bold: string; paramKey?: ParamKey }; +interface Pill { + kind: PillKind; + /** Optional small text prefix (e.g. "↻ " for the loop pill). */ + prefix?: string; + fragments: Frag[]; +} + +const PILL_STYLES: Record = { + SPACE: { bg: "#E7EFF1", fg: "#2C5563" }, + EFFORT: { bg: "#F6ECE0", fg: "#8A4E20" }, + VALIDATION: { bg: "#ECEBE6", fg: "#5A6670" }, +}; + +const BOXES: Box[] = [ + { title: "~20,000 genes", sub: "every gene measured" }, + { title: "Gene pool — N genes", sub: "the shortlist the engine draws from" }, + { title: "One program", sub: "Select → Reduce → Fit" }, + { title: "One generation", sub: "every program scored & ranked" }, + { title: "Winning program" }, + { title: "permutation p", sub: "could this be luck?", amber: true }, +]; + +const PILLS: Pill[] = [ + { + kind: "SPACE", + fragments: [ + { bold: "Prefilter top-N", paramKey: "prefilter_n" }, + " — keep the N most promising genes (off by default = all ~20,000)", + ], + }, + { + kind: "SPACE", + fragments: [ + { bold: "Max sets", paramKey: "max_sets" }, + " × ", + { bold: "Genes/set", paramKey: "genes_per_set" }, + " — pick 1–2 sets, each ≤ G genes", + ], + }, + { + kind: "EFFORT", + fragments: [ + { bold: "Population", paramKey: "population" }, + " — programs compete each round · ", + { bold: "λ", paramKey: "lambda" }, + " — taxes extra genes", + ], + }, + { + kind: "EFFORT", + prefix: "↻ ", + fragments: [ + "× ", + { bold: "Generations", paramKey: "generations" }, + " — breed the best, repeat · ", + { bold: "Seed", paramKey: "seed" }, + " — fixes the randomness", + ], + }, + { + kind: "VALIDATION", + fragments: [ + { bold: "Permutations", paramKey: "permutations" }, + " — re-run on N shuffled-label sets", + ], + }, +]; + +// ---------- render ------------------------------------------------------- + +function Pipeline() { + return ( +
    + {BOXES.map((b, i) => ( +
    + + {i < PILLS.length && ( + <> + + + + )} +
    + ))} +
    + ); +} + +function BoxRow({ box }: { box: Box }) { + return ( +
    +
    + {box.title} +
    + {box.sub && ( +
    + {box.sub} +
    + )} +
    + ); +} + +function PillRow({ pill }: { pill: Pill }) { + const { bg, fg } = PILL_STYLES[pill.kind]; + return ( +
    +
    + {pill.prefix && ( + + {pill.prefix} + + )} + {pill.fragments.map((f, i) => { + if (typeof f === "string") return {f}; + if (!f.paramKey) { + return ( + + {f.bold} + + ); + } + return ; + })} +
    +
    + ); +} + +function ParamPillButton({ + bold, + paramKey, + colour, +}: { + bold: string; + paramKey: ParamKey; + colour: string; +}) { + const ref = useRef(null); + const { open } = useParamHelp(); + return ( + + ); +} + +function Arrow() { + return ( +
    + ↓ +
    + ); +} + +function Legend() { + return ( +
    + + teal = where it searches + + + amber = how hard it searches + + + grey = validation + +
    + ); +} + +function LegendItem({ + fill, + ink, + children, +}: { + fill: string; + ink: string; + children: React.ReactNode; +}) { + return ( + + + {children} + + ); +} diff --git a/web/app/PasteToDraw.tsx b/web/app/PasteToDraw.tsx new file mode 100644 index 0000000000000000000000000000000000000000..1cb3eea9b85a75d2f96562d7b129267a67159870 --- /dev/null +++ b/web/app/PasteToDraw.tsx @@ -0,0 +1,212 @@ +"use client"; + +import { useId, useState } from "react"; +import { safeParseProgramRepr } from "../lib/programRepr"; + +interface Props { + text: string; + onTextChange: (next: string) => void; + onDraw: (repr: string, label: string) => void; +} + +const PLACEHOLDER = + "Combine(Reduce(Select(M,[g05347,g00048]),mean),Reduce(Select(M,[g06271]),max),sub)"; + +export default function PasteToDraw({ text, onTextChange, onDraw }: Props) { + const [error, setError] = useState(null); + const [showFormat, setShowFormat] = useState(false); + const panelId = useId(); + + function draw() { + const trimmed = text.trim(); + if (!trimmed) { + setError("Paste a program first."); + return; + } + const { ok, error: pErr } = safeParseProgramRepr(trimmed); + if (!ok) { + setError(`Couldn't parse: ${pErr}`); + return; + } + setError(null); + onDraw(trimmed, "pasted program"); + } + + return ( +
    + +