File size: 5,403 Bytes
72b8bbc a306cdc 72b8bbc ebec40b a306cdc ebec40b 72b8bbc a306cdc 72b8bbc a306cdc ebec40b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 | """Unit tests for Hub metrics discovery helpers (no network)."""
from __future__ import annotations
import pandas as pd
from data import (
condition_family,
condition_label,
condition_order_index,
disagreement_by_slice,
disagreement_summary,
encoder_comparator_frame,
generative_ladder_frame,
slice_accuracy_frame,
)
def test_rfdt_labels_and_family() -> None:
assert condition_label("rfdt_base") == "RFDT Base"
assert condition_label("rfdt_lora") == "RFDT-LoRA"
assert condition_family("rfdt_base") == "rfdt"
assert condition_family("rfdt_lora") == "rfdt"
assert condition_family("lora") == "learned"
assert condition_family("base") == "frozen"
def test_encoder_labels_and_family() -> None:
assert condition_label("encoder_ft") == "Encoder-FT"
assert condition_family("encoder_ft") == "encoder"
def test_unknown_condition_gets_fallback_label() -> None:
assert condition_label("future_method_v2") == "Future Method V2"
assert condition_family("future_method_v2") == "other"
def test_rfdt_sorts_after_cartridge() -> None:
assert condition_order_index("cartridge") < condition_order_index("rfdt_base")
assert condition_order_index("rfdt_base") < condition_order_index("rfdt_lora")
assert condition_order_index("rfdt_lora") < condition_order_index("encoder_ft")
def test_generative_ladder_drops_encoder_rows() -> None:
df = pd.DataFrame(
[
{"model_key": "qwen3-8b", "condition": "lora", "family": "learned"},
{
"model_key": "modernbert-base",
"condition": "encoder_ft",
"family": "encoder",
},
]
)
out = generative_ladder_frame(df)
assert list(out["model_key"]) == ["qwen3-8b"]
def test_encoder_comparator_keeps_rfdt_and_modernbert() -> None:
df = pd.DataFrame(
[
{
"model_key": "qwen3-8b",
"condition": "rfdt_lora",
"family": "rfdt",
},
{
"model_key": "qwen3-8b",
"condition": "lora",
"family": "learned",
},
{
"model_key": "modernbert-base",
"condition": "encoder_ft",
"family": "encoder",
},
]
)
out = encoder_comparator_frame(df)
assert set(out["condition"]) == {"rfdt_lora", "encoder_ft"}
assert "lora" not in set(out["condition"])
def test_slice_accuracy_frame_long_form() -> None:
df = pd.DataFrame(
[
{
"model": "Qwen3-8B",
"model_key": "qwen3-8b",
"condition": "rfdt_lora",
"condition_label": "RFDT-LoRA",
"slice_acc_allowed_base": 1.0,
"slice_n_allowed_base": 8,
"slice_acc_denied_edge": 0.5,
"slice_n_denied_edge": 66,
"slice_acc_allowed_edge": None,
"slice_n_allowed_edge": None,
"slice_acc_denied_base": 0.9,
"slice_n_denied_base": 12,
},
{
"model": "ModernBERT-base",
"model_key": "modernbert-base",
"condition": "encoder_ft",
"condition_label": "Encoder-FT",
"slice_acc_allowed_base": 1.0,
"slice_n_allowed_base": 8,
"slice_acc_denied_edge": 0.4,
"slice_n_denied_edge": 66,
"slice_acc_allowed_edge": 0.9,
"slice_n_allowed_edge": 43,
"slice_acc_denied_base": 0.0,
"slice_n_denied_base": 12,
},
]
)
long = slice_accuracy_frame(df)
assert not long.empty
assert set(long["query_type"]) <= {
"allowed_base",
"allowed_edge",
"denied_base",
"denied_edge",
}
assert "Qwen3-8B · RFDT-LoRA" in set(long["method"])
assert long.loc[
(long["method"].str.contains("ModernBERT"))
& (long["query_type"] == "denied_edge"),
"accuracy",
].iloc[0] == 0.4
def test_disagreement_summary_rates() -> None:
disagree = pd.DataFrame(
[
{
"model": "Qwen3-8B",
"model_key": "qwen3-8b",
"query_type": "denied_edge",
"outcome": "rfdt_only",
},
{
"model": "Qwen3-8B",
"model_key": "qwen3-8b",
"query_type": "denied_edge",
"outcome": "encoder_only",
},
{
"model": "Qwen3-8B",
"model_key": "qwen3-8b",
"query_type": "allowed_edge",
"outcome": "both_correct",
},
{
"model": "Qwen3-8B",
"model_key": "qwen3-8b",
"query_type": "allowed_edge",
"outcome": "both_correct",
},
]
)
summary = disagreement_summary(disagree)
assert abs(summary.loc[summary["outcome"] == "both_correct", "rate"].iloc[0] - 0.5) < 1e-9
by_slice = disagreement_by_slice(disagree)
denied = by_slice[by_slice["query_type"] == "denied_edge"].iloc[0]
assert abs(denied["net_rfdt_edge"] - 0.0) < 1e-9
assert abs(denied["disagree_rate"] - 1.0) < 1e-9
|