| { |
| "gate": 2, |
| "phase": "Phase 1b - Probe faithfulness [block-3 operating layer]", |
| "status": "BORDERLINE", |
| "status_detail": "Strict rule (rho>=0.5) NOT met: best rho=0.48 < 0.50 [CALIBRATE]. Per spec, borderline = FAIL pending more data / threshold calibration. Faithfulness signal is real (p~0, monotone) but sub-threshold.", |
| "metrics": [ |
| { |
| "name": "spearman_coverage_drop_vs_detection_drop_rankme", |
| "modality": "LIDC-IDRI(test)", "budget": null, |
| "value": 0.480, "ci95": null, "test": "Spearman_pooled_per_slice_ratio (n=6788)", |
| "threshold": 0.50, "threshold_status": "CALIBRATE", |
| "passed": false, |
| "detail": "RankMe coverage. rho=0.480, p~0. Monotone across ratios. Just below 0.50." |
| }, |
| { |
| "name": "spearman_coverage_drop_vs_detection_drop_coding_rate", |
| "modality": "LIDC-IDRI(test)", "budget": null, |
| "value": 0.478, "ci95": null, "test": "Spearman_pooled_per_slice_ratio (n=6788)", |
| "threshold": 0.50, "threshold_status": "CALIBRATE", |
| "passed": false, |
| "detail": "Coding-rate surrogate (the spec's RankMe fallback). rho=0.478, p~0. Also sub-threshold; both coverage forms agree." |
| }, |
| { |
| "name": "monotonicity_across_ratios", |
| "modality": "LIDC-IDRI(test)", "budget": null, |
| "value": 1.0, "ci95": null, "test": "per-ratio means increasing", |
| "threshold": 1.0, "threshold_status": "FIXED", "passed": true, |
| "detail": "sens_drop {0.087,0.184,0.391,0.586} and delta_C both monotone increasing across prune ratios {0.1,0.25,0.5,0.75}." |
| } |
| ], |
| "thresholds_locked_ref": null, |
| "seeds": [0], |
| "data_gaps": [], |
| "eval_summary": { |
| "modality": "LIDC-IDRI", "probe": "logistic regression on block-3 patch tokens, trained on val (294k patches, 2813 lesion), EVAL-ONLY", |
| "eval_split": "test", "pruning": "random subsets at ratios {0.1,0.25,0.5,0.75}, 2 draws/slice", |
| "detection_metric": "full-denominator lesion-patch sensitivity (a pruned-away lesion patch counts as a miss)", |
| "per_ratio_sens_drop": {"0.1": 0.087, "0.25": 0.184, "0.5": 0.391, "0.75": 0.586}, |
| "metrics_artifact": "processed/covtoken/gate2_metrics.json" |
| }, |
| "decision_rule": "PASS iff best coverage rho>=0.5, p<0.05, monotone. Borderline (just under 0.5) = FAIL pending more data per IMPLEMENTATION_SPEC.", |
| "decision": "BORDERLINE / soft-FAIL. Coverage drop is a SIGNIFICANT, MONOTONE proxy for lesion-detection drop (rho~0.48, p~0, both RankMe and coding-rate), but the pooled correlation is fractionally below the 0.50 [CALIBRATE] convention. The threshold is a calibration constant, not a fixed decision constant; Phase 1b is where it would be set data-driven. Not overclaimed as PASS.", |
| "robustness_check": { |
| "energy_coverage_rho": 0.461, |
| "finding": "Tested a third coverage (energy = sum ||P_L z||^2, additive in tokens). ALL THREE coverages give rho ~0.46-0.48 (rankme 0.480, coding 0.478, energy 0.461). The ~0.48 ceiling is INVARIANT to the coverage definition, so it is NOT a rank-vs-energy aggregation artifact. Root cause is the RANDOM-pruning protocol: at a fixed ratio, whether the 1-3 lesion patches survive a random keep-set is nearly independent of total coverage lost, so within-ratio coupling is weak; the across-ratio relationship is perfectly monotone (p~0).", |
| "conclusion": "Coverage is moderately faithful and NOT blind to lesion loss (monotone, p~0, rho~0.48), robustly across coverage definitions, just under the arbitrary [CALIBRATE] 0.50 bar." |
| }, |
| "options": [ |
| "Calibrate the [CALIBRATE] 0.50 threshold in Phase 1b against the saliency/random baseline coupling (the spec's intended step); rho~0.48 may clear a properly-calibrated bar.", |
| "Report as a robust noted limitation: coverage is strongly faithful across pruning ratios (monotone) and moderately at the pooled (slice,ratio) level (rho~0.48), invariant to coverage form." |
| ], |
| "human_signoff": null |
| } |
|
|
