File size: 4,771 Bytes
40f7928 84193a6 40f7928 84193a6 40f7928 84193a6 40f7928 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 | {
"_doc": "VCBench reference values: capability matrix + reference test cases. Any drift in these values triggers test failure.",
"_tolerance_default": 0.001,
"dim_a": {
"geneformer_v2_316m_ft_d": {
"additive_evaluable_71": {"PRR": 0.627},
"full_106": {"PRR": 0.6267, "DES": 0.8778}
},
"geneformer_v2_316m_zs_d_matched_regime": {
"full_107": {"PRR": 0.239}
},
"scgpt_ft": {
"additive_evaluable_71": {"PRR": 0.5445, "DES": 0.8718},
"full_107": {"PRR": 0.5025, "DES": 0.8439},
"novel_36": {"PRR": 0.4196, "DES": 0.7889}
},
"transcriptformer_zs_d": {
"additive_evaluable_71": {"PRR": -0.165},
"full_107": {"PRR": -0.174},
"novel_36": {"PRR": -0.192}
},
"arc_state_ft": {
"full_107": {"PRR": 0.402, "DES": 0.751, "_role": "CANONICAL — used for VC Level decisions; real-control anchor is the principled cross-model convention"},
"full_107_pred_anchor": {"PRR": 0.4076, "DES": 0.7846, "_role": "FOR cell-eval cross-validation ONLY — reproduces upstream cell-eval pearson_delta. NOT a competing canonical number. NOT used for VC Level decisions."},
"_canonical_convention": "real",
"_convention_notes": "Real-anchor is canonical because cross-model benchmarking requires a shared baseline — letting each model anchor on its own predicted control turns the metric into a per-model quantity, defeating the spec's shared-baseline premise. Pred-anchor exists solely so VCBench can numerically reconcile against upstream cell-eval reports.",
"_provenance": "Trained on the Norman GEARS split (seed=1, 139 train / 107 held-out test) with arc-state v0.10.2. Cross-validated under matched anchor conventions: vcbench.evaluate_dim_a(control_anchor='pred') = 0.4076 vs cell-eval pearson_delta = 0.4076 (agree to numerical precision). Under the canonical real-control anchor PRR = 0.402. Pred-anchor is the cell-eval cross-validation convention."
},
"additive_baseline": {
"additive_evaluable_71": {"PRR": 0.8903, "DES": 0.9986}
},
"mean_baseline": {
"full_107": {"PRR": 0.5786, "DES": 0.8706}
},
"no_change_baseline": {
"full_107": {"PRR": 0.0, "DES": 0.0}
}
},
"dim_b": {
"common_set_aggregate_macroF1": {
"pca_knn": {"value": 0.4968, "tissues_evaluated": 5},
"geneformer": {"value": 0.171, "tissues_evaluated": 5},
"scgpt": {"value": 0.123, "tissues_evaluated": 5},
"uce": {"value": 0.379, "tissues_evaluated": 2, "tissues": ["heart", "brain"]},
"transcriptformer": {"value": 0.351, "tissues_evaluated": 2, "tissues": ["lung", "liver"]}
},
"per_tissue_common_set_macroF1": {
"pca_knn": {"lung": 0.4649, "liver": 0.4460, "heart": 0.5658, "kidney": 0.3265, "brain": 0.6806},
"geneformer": {"lung": 0.0188, "liver": 0.1719, "heart": 0.4833, "kidney": 0.0639, "brain": 0.1156},
"scgpt": {"lung": 0.0594, "liver": 0.0071, "heart": 0.4795, "kidney": 0.0040, "brain": 0.0626},
"uce": {"heart": 0.4580, "brain": 0.2999},
"transcriptformer": {"lung": 0.2060, "liver": 0.4954}
}
},
"dim_c": {
"table2": {
"geneformer": {"AUROC": 0.626, "AUPRC": 0.001, "EPR": 0.000},
"scgpt": {"AUROC": 0.519, "AUPRC": 0.003, "EPR": 20.05},
"co_expression": {"AUROC": 0.558, "AUPRC": 0.004, "EPR": 15.50},
"degree_null": {"AUROC": 0.500, "AUPRC": 0.0003, "EPR": 1.13},
"pyscenic": {"AUROC": 0.501, "AUPRC": 0.0011, "EPR": 3.50}
}
},
"dim_d": {
"table2_pearson": {
"transcriptformer": 0.232,
"uce": 0.132,
"scgpt": 0.064,
"geneformer": 0.001,
"mean_celltype": 0.152,
"scLinear": 0.129
}
},
"dim_e": {
"table2_kendall_tau_b": {
"uce": 0.136,
"transcriptformer": 0.041,
"geneformer": -0.017,
"scgpt": -0.057,
"pca_dpt": 0.190
},
"scgpt_weinreb_temporal_inversion": -0.103,
"transcriptformer_weinreb_bootstrap_std": 0.078
},
"dim_g_spread_error": {
"geneformer_norman_ft": {"rho": -0.119, "p": 0.225, "n_perturbations": 106},
"scgpt_norman_ft": {"rho": 0.131, "p": 0.177, "n_perturbations": 107}
},
"vc_levels": {
"geneformer_v2_316m": {"level": 1, "binding_dim": "A"},
"scgpt_fine_tuned": {"level": 1, "binding_dim": "A"},
"uce_33_layer": {"level": 1, "binding_dim": "E"},
"transcriptformer": {"level": 2, "binding_dim": "D"},
"arc_state": {"level": 1, "binding_dim": "A"}
}
}
|