Spaces:
Sleeping
Sleeping
Add Nemotron 3 Ultra 550B (post-v1.1, rotated v1.3 council)
Browse files
app.py
CHANGED
|
@@ -63,6 +63,7 @@ MODEL_META: dict[str, tuple[str, str, str, str]] = {
|
|
| 63 |
"qwen.qwen3-next-80b-a3b": ("Qwen3 Next 80B", "Qwen", "qwen", "Asia"),
|
| 64 |
"zai.glm-5": ("GLM-5", "Z.AI", "zai", "Asia"),
|
| 65 |
"nvidia.nemotron-super-3-120b": ("Nemotron 3 Super 120B", "NVIDIA", "nvidia", "US"),
|
|
|
|
| 66 |
}
|
| 67 |
|
| 68 |
NOTE_FLAGS: dict[str, str] = {
|
|
@@ -70,6 +71,7 @@ NOTE_FLAGS: dict[str, str] = {
|
|
| 70 |
"nvidia.nemotron-super-3-120b": "added v1.1",
|
| 71 |
"anthropic/claude-opus-4.8": "post-v1.1 · rotated v1.3 council (see *)",
|
| 72 |
"minimax/minimax-m3": "post-v1.1 · rotated v1.3 council (see *)",
|
|
|
|
| 73 |
}
|
| 74 |
|
| 75 |
# PC Tier from should-refuse positive control (TPR threshold: A ≥ 95%, B 9–73%)
|
|
@@ -91,6 +93,7 @@ PC_TIER: dict[str, str] = {
|
|
| 91 |
"google/gemini-3.1-flash-lite-20260507": "B",
|
| 92 |
"us.amazon.nova-pro-v1:0": "B",
|
| 93 |
"nvidia.nemotron-super-3-120b": "—",
|
|
|
|
| 94 |
"zai.glm-5": "—",
|
| 95 |
"deepseek.v3.2": "—",
|
| 96 |
"mistral.mistral-large-3-675b-instruct": "—",
|
|
@@ -1019,7 +1022,7 @@ def make_fig3(stats: pd.DataFrame) -> plt.Figure:
|
|
| 1019 |
|
| 1020 |
|
| 1021 |
def make_fig5(stats: pd.DataFrame) -> plt.Figure:
|
| 1022 |
-
"""Tier-stratified grouped bar for all
|
| 1023 |
overall = overall_stats(stats)
|
| 1024 |
model_order = overall["model"].tolist()
|
| 1025 |
|
|
@@ -1097,7 +1100,7 @@ HEADER_HTML = f"""
|
|
| 1097 |
·
|
| 1098 |
<a href="https://arxiv.org/abs/2605.21545" target="_blank">arXiv:2605.21545</a>
|
| 1099 |
·
|
| 1100 |
-
<span class="pill">v1.1-frozen +
|
| 1101 |
·
|
| 1102 |
{_N_MODELS} models · 141 prompts · {_N_TRIALS:,} trials
|
| 1103 |
</div>
|
|
@@ -1259,13 +1262,13 @@ with gr.Blocks(
|
|
| 1259 |
# ── Tab 3: Provider figures ────────────────────────────────────────
|
| 1260 |
with gr.Tab("Provider Analysis"):
|
| 1261 |
gr.Markdown(
|
| 1262 |
-
"**Figure 1.** Benign-tier strict refusal rate for all
|
| 1263 |
"sorted descending, coloured by provider. Error bars = Wilson 95 % CI."
|
| 1264 |
)
|
| 1265 |
gr.Plot(value=make_fig1(STATS))
|
| 1266 |
|
| 1267 |
gr.Markdown(
|
| 1268 |
-
"**Figure 2.** Tier-stratified rates across all
|
| 1269 |
"benign / borderline / dual-use side-by-side."
|
| 1270 |
)
|
| 1271 |
gr.Plot(value=make_fig5(STATS))
|
|
|
|
| 63 |
"qwen.qwen3-next-80b-a3b": ("Qwen3 Next 80B", "Qwen", "qwen", "Asia"),
|
| 64 |
"zai.glm-5": ("GLM-5", "Z.AI", "zai", "Asia"),
|
| 65 |
"nvidia.nemotron-super-3-120b": ("Nemotron 3 Super 120B", "NVIDIA", "nvidia", "US"),
|
| 66 |
+
"nvidia/nemotron-3-ultra-550b-a55b": ("Nemotron 3 Ultra 550B*", "NVIDIA", "nvidia", "US"),
|
| 67 |
}
|
| 68 |
|
| 69 |
NOTE_FLAGS: dict[str, str] = {
|
|
|
|
| 71 |
"nvidia.nemotron-super-3-120b": "added v1.1",
|
| 72 |
"anthropic/claude-opus-4.8": "post-v1.1 · rotated v1.3 council (see *)",
|
| 73 |
"minimax/minimax-m3": "post-v1.1 · rotated v1.3 council (see *)",
|
| 74 |
+
"nvidia/nemotron-3-ultra-550b-a55b": "post-v1.1 · rotated v1.3 council (see *)",
|
| 75 |
}
|
| 76 |
|
| 77 |
# PC Tier from should-refuse positive control (TPR threshold: A ≥ 95%, B 9–73%)
|
|
|
|
| 93 |
"google/gemini-3.1-flash-lite-20260507": "B",
|
| 94 |
"us.amazon.nova-pro-v1:0": "B",
|
| 95 |
"nvidia.nemotron-super-3-120b": "—",
|
| 96 |
+
"nvidia/nemotron-3-ultra-550b-a55b": "—",
|
| 97 |
"zai.glm-5": "—",
|
| 98 |
"deepseek.v3.2": "—",
|
| 99 |
"mistral.mistral-large-3-675b-instruct": "—",
|
|
|
|
| 1022 |
|
| 1023 |
|
| 1024 |
def make_fig5(stats: pd.DataFrame) -> plt.Figure:
|
| 1025 |
+
"""Tier-stratified grouped bar for all 22 models."""
|
| 1026 |
overall = overall_stats(stats)
|
| 1027 |
model_order = overall["model"].tolist()
|
| 1028 |
|
|
|
|
| 1100 |
·
|
| 1101 |
<a href="https://arxiv.org/abs/2605.21545" target="_blank">arXiv:2605.21545</a>
|
| 1102 |
·
|
| 1103 |
+
<span class="pill">v1.1-frozen + 3 post-frozen*</span>
|
| 1104 |
·
|
| 1105 |
{_N_MODELS} models · 141 prompts · {_N_TRIALS:,} trials
|
| 1106 |
</div>
|
|
|
|
| 1262 |
# ── Tab 3: Provider figures ────────────────────────────────────────
|
| 1263 |
with gr.Tab("Provider Analysis"):
|
| 1264 |
gr.Markdown(
|
| 1265 |
+
"**Figure 1.** Benign-tier strict refusal rate for all 22 models, "
|
| 1266 |
"sorted descending, coloured by provider. Error bars = Wilson 95 % CI."
|
| 1267 |
)
|
| 1268 |
gr.Plot(value=make_fig1(STATS))
|
| 1269 |
|
| 1270 |
gr.Markdown(
|
| 1271 |
+
"**Figure 2.** Tier-stratified rates across all 22 models — "
|
| 1272 |
"benign / borderline / dual-use side-by-side."
|
| 1273 |
)
|
| 1274 |
gr.Plot(value=make_fig5(STATS))
|