VibeCodingScientist commited on
Commit
4e06b0a
·
verified ·
1 Parent(s): 0ecf605

Add Nemotron 3 Ultra 550B (post-v1.1, rotated v1.3 council)

Browse files
Files changed (1) hide show
  1. app.py +7 -4
app.py CHANGED
@@ -63,6 +63,7 @@ MODEL_META: dict[str, tuple[str, str, str, str]] = {
63
  "qwen.qwen3-next-80b-a3b": ("Qwen3 Next 80B", "Qwen", "qwen", "Asia"),
64
  "zai.glm-5": ("GLM-5", "Z.AI", "zai", "Asia"),
65
  "nvidia.nemotron-super-3-120b": ("Nemotron 3 Super 120B", "NVIDIA", "nvidia", "US"),
 
66
  }
67
 
68
  NOTE_FLAGS: dict[str, str] = {
@@ -70,6 +71,7 @@ NOTE_FLAGS: dict[str, str] = {
70
  "nvidia.nemotron-super-3-120b": "added v1.1",
71
  "anthropic/claude-opus-4.8": "post-v1.1 · rotated v1.3 council (see *)",
72
  "minimax/minimax-m3": "post-v1.1 · rotated v1.3 council (see *)",
 
73
  }
74
 
75
  # PC Tier from should-refuse positive control (TPR threshold: A ≥ 95%, B 9–73%)
@@ -91,6 +93,7 @@ PC_TIER: dict[str, str] = {
91
  "google/gemini-3.1-flash-lite-20260507": "B",
92
  "us.amazon.nova-pro-v1:0": "B",
93
  "nvidia.nemotron-super-3-120b": "—",
 
94
  "zai.glm-5": "—",
95
  "deepseek.v3.2": "—",
96
  "mistral.mistral-large-3-675b-instruct": "—",
@@ -1019,7 +1022,7 @@ def make_fig3(stats: pd.DataFrame) -> plt.Figure:
1019
 
1020
 
1021
  def make_fig5(stats: pd.DataFrame) -> plt.Figure:
1022
- """Tier-stratified grouped bar for all 21 models."""
1023
  overall = overall_stats(stats)
1024
  model_order = overall["model"].tolist()
1025
 
@@ -1097,7 +1100,7 @@ HEADER_HTML = f"""
1097
   · 
1098
  <a href="https://arxiv.org/abs/2605.21545" target="_blank">arXiv:2605.21545</a>
1099
  &nbsp;·&nbsp;
1100
- <span class="pill">v1.1-frozen + Opus 4.8* + MiniMax M3*</span>
1101
  &nbsp;·&nbsp;
1102
  {_N_MODELS} models &nbsp;·&nbsp; 141 prompts &nbsp;·&nbsp; {_N_TRIALS:,} trials
1103
  </div>
@@ -1259,13 +1262,13 @@ with gr.Blocks(
1259
  # ── Tab 3: Provider figures ────────────────────────────────────────
1260
  with gr.Tab("Provider Analysis"):
1261
  gr.Markdown(
1262
- "**Figure 1.** Benign-tier strict refusal rate for all 21 models, "
1263
  "sorted descending, coloured by provider. Error bars = Wilson 95 % CI."
1264
  )
1265
  gr.Plot(value=make_fig1(STATS))
1266
 
1267
  gr.Markdown(
1268
- "**Figure 2.** Tier-stratified rates across all 21 models — "
1269
  "benign / borderline / dual-use side-by-side."
1270
  )
1271
  gr.Plot(value=make_fig5(STATS))
 
63
  "qwen.qwen3-next-80b-a3b": ("Qwen3 Next 80B", "Qwen", "qwen", "Asia"),
64
  "zai.glm-5": ("GLM-5", "Z.AI", "zai", "Asia"),
65
  "nvidia.nemotron-super-3-120b": ("Nemotron 3 Super 120B", "NVIDIA", "nvidia", "US"),
66
+ "nvidia/nemotron-3-ultra-550b-a55b": ("Nemotron 3 Ultra 550B*", "NVIDIA", "nvidia", "US"),
67
  }
68
 
69
  NOTE_FLAGS: dict[str, str] = {
 
71
  "nvidia.nemotron-super-3-120b": "added v1.1",
72
  "anthropic/claude-opus-4.8": "post-v1.1 · rotated v1.3 council (see *)",
73
  "minimax/minimax-m3": "post-v1.1 · rotated v1.3 council (see *)",
74
+ "nvidia/nemotron-3-ultra-550b-a55b": "post-v1.1 · rotated v1.3 council (see *)",
75
  }
76
 
77
  # PC Tier from should-refuse positive control (TPR threshold: A ≥ 95%, B 9–73%)
 
93
  "google/gemini-3.1-flash-lite-20260507": "B",
94
  "us.amazon.nova-pro-v1:0": "B",
95
  "nvidia.nemotron-super-3-120b": "—",
96
+ "nvidia/nemotron-3-ultra-550b-a55b": "—",
97
  "zai.glm-5": "—",
98
  "deepseek.v3.2": "—",
99
  "mistral.mistral-large-3-675b-instruct": "—",
 
1022
 
1023
 
1024
  def make_fig5(stats: pd.DataFrame) -> plt.Figure:
1025
+ """Tier-stratified grouped bar for all 22 models."""
1026
  overall = overall_stats(stats)
1027
  model_order = overall["model"].tolist()
1028
 
 
1100
  &nbsp;·&nbsp;
1101
  <a href="https://arxiv.org/abs/2605.21545" target="_blank">arXiv:2605.21545</a>
1102
  &nbsp;·&nbsp;
1103
+ <span class="pill">v1.1-frozen + 3 post-frozen*</span>
1104
  &nbsp;·&nbsp;
1105
  {_N_MODELS} models &nbsp;·&nbsp; 141 prompts &nbsp;·&nbsp; {_N_TRIALS:,} trials
1106
  </div>
 
1262
  # ── Tab 3: Provider figures ────────────────────────────────────────
1263
  with gr.Tab("Provider Analysis"):
1264
  gr.Markdown(
1265
+ "**Figure 1.** Benign-tier strict refusal rate for all 22 models, "
1266
  "sorted descending, coloured by provider. Error bars = Wilson 95 % CI."
1267
  )
1268
  gr.Plot(value=make_fig1(STATS))
1269
 
1270
  gr.Markdown(
1271
+ "**Figure 2.** Tier-stratified rates across all 22 models — "
1272
  "benign / borderline / dual-use side-by-side."
1273
  )
1274
  gr.Plot(value=make_fig5(STATS))