Update leaderboard UX and GRM benchmark content

app.py

@@ -1,6 +1,7 @@
 """Nvidia Game Ready Model Score (GRM Score) Gradio app."""
 
 from html import escape
+from pathlib import Path
 
 import gradio as gr
 
@@ -19,7 +20,20 @@ LEADERBOARD_COLUMNS = [
 ]
 
 
-GRM_BENCH_SECTIONS = [
+GRADIO_MAJOR_VERSION = int(gr.__version__.split(".", 1)[0])
+APP_ROOT = Path(__file__).resolve().parent
+REF_ROOT = APP_ROOT / "ref"
+
+DEFAULT_OVERVIEW_BLOCKS = [
+    "Nvidia Game Ready Model Score (GRM) is an aggregated quality metric designed to assess LLM capabilites in gaming use cases.",
+    "General state-of-the-art language models are optimized for broad benchmarks such as math, code, and general knowledge. That does not reliably translate to in-game performance, and it does not reliably predict NPC quality, gameplay actions, or immersion.",
+    "With game model evaluation, game developers can accelerate AI integration pipelines by reducing time spent on model evaluation and narrowing model choice earlier. The overall score is the average of Roleplay, Actions, and General, while benchmarks inside each category are combined with weighted averaging using core weights of 1.0 and supplementary weights of 0.5.",
+    "GRM Score = (Roleplay + Actions + General) / 3",
+    "Category Score = sum(score x weight) / sum(weight)",
+]
+
+
+BASE_GRM_BENCH_SECTIONS = [
     {
         "title": "Coherence",
         "summary": [
@@ -135,13 +149,181 @@ GRM_BENCH_SECTIONS = [
 ]
 
 
+def _read_reference_file(name: str) -> str | None:
+    try:
+        return (REF_ROOT / name).read_text(encoding="utf-8").strip()
+    except OSError:
+        return None
+
+
+def _split_reference_blocks(text: str) -> list[str]:
+    blocks = []
+    for chunk in text.split("\n\n"):
+        block = " ".join(line.strip() for line in chunk.splitlines() if line.strip())
+        if block:
+            blocks.append(block)
+    return blocks
+
+
+def _build_overview_html(blocks: list[str]) -> str:
+    parts = ['<div class="longform-copy">']
+    title_prefix = "Nvidia Game Ready Model Score (GRM)"
+
+    for block in blocks:
+        if block.startswith("GRM Score ="):
+            parts.append(f'<p class="formula-line">{escape(block)}</p>')
+            continue
+
+        if block.startswith("Category Score ="):
+            parts.append(f'<p class="formula-line subdued">{escape(block)}</p>')
+            continue
+
+        if block.startswith(title_prefix):
+            suffix = block[len(title_prefix) :]
+            parts.append(f"<p><strong>{escape(title_prefix)}</strong>{escape(suffix)}</p>")
+            continue
+
+        parts.append(f"<p>{escape(block)}</p>")
+
+    parts.append("</div>")
+    return "".join(parts)
+
+
+def _load_overview_html() -> str:
+    text = _read_reference_file("Overview")
+    blocks = _split_reference_blocks(text) if text else DEFAULT_OVERVIEW_BLOCKS
+    return _build_overview_html(blocks)
+
+
+def _load_coherence_section() -> dict | None:
+    text = _read_reference_file("Coherence_Summary")
+    if not text:
+        return None
+
+    lines = text.splitlines()
+    index = 0
+
+    while index < len(lines) and not lines[index].strip():
+        index += 1
+
+    if index >= len(lines):
+        return None
+
+    title = lines[index].strip()
+    index += 1
+
+    summary_lines = []
+    while index < len(lines) and lines[index].strip() != "Test Methodology":
+        if lines[index].strip():
+            summary_lines.append(lines[index].strip())
+        index += 1
+
+    if index >= len(lines):
+        return None
+
+    index += 1
+    methodology_lines = []
+    while index < len(lines) and lines[index].strip() != "Detection Scope:":
+        if lines[index].strip():
+            methodology_lines.append(lines[index].strip())
+        index += 1
+
+    if index >= len(lines):
+        return None
+
+    index += 1
+    scope = []
+    while index < len(lines) and lines[index].strip() != "Test Samples":
+        line = lines[index].strip()
+        if line:
+            category, _, description = line.partition(" - ")
+            scope.append([category.strip(), description.strip()])
+        index += 1
+
+    samples = []
+    if index < len(lines) and lines[index].strip() == "Test Samples":
+        index += 1
+
+        while index < len(lines):
+            while index < len(lines) and not lines[index].strip():
+                index += 1
+
+            if index >= len(lines):
+                break
+
+            if not lines[index].strip().startswith("TEST_"):
+                index += 1
+                continue
+
+            sample_id = lines[index].strip()
+            index += 1
+            metadata = []
+            code_lines = []
+
+            while index < len(lines):
+                line = lines[index]
+                stripped = line.strip()
+
+                if stripped.startswith("TEST_"):
+                    break
+
+                if stripped == "Messages:":
+                    index += 1
+                    while index < len(lines) and not lines[index].strip().startswith("TEST_"):
+                        code_lines.append(lines[index].rstrip())
+                        index += 1
+                    break
+
+                if stripped and ":" in stripped:
+                    label, value = stripped.split(":", 1)
+                    metadata.append([label.strip(), value.strip()])
+
+                index += 1
+
+            samples.append(
+                {
+                    "id": sample_id,
+                    "metadata": metadata,
+                    "code": "\n".join(code_lines).strip(),
+                }
+            )
+
+    if not summary_lines or not methodology_lines or not scope:
+        return None
+
+    return {
+        "title": title,
+        "summary": [" ".join(summary_lines)],
+        "methodology": " ".join(methodology_lines),
+        "scope": scope,
+        "samples": samples,
+    }
+
+
+def _load_grm_bench_sections() -> list[dict]:
+    sections = list(BASE_GRM_BENCH_SECTIONS)
+    coherence_section = _load_coherence_section()
+    if coherence_section is not None:
+        sections[0] = coherence_section
+    return sections
+
+
+GRM_BENCH_SECTIONS = _load_grm_bench_sections()
+
+
 def _fmt(value: float | None) -> str:
     return f"{value:.1f}" if value is not None else "-"
 
 
-def build_html_table(headers: list[str], rows: list[list[str]], table_class: str = "") -> str:
+def build_html_table(
+    headers: list[str],
+    rows: list[list[str]],
+    table_class: str = "",
+    shell_class: str = "table-scroll-shell",
+) -> str:
     class_attr = f' class="data-table {table_class}"' if table_class else ' class="data-table"'
-    parts = ["<div class=\"table-shell\">", f"<table{class_attr}>", "<thead><tr>"]
+    shell_classes = " ".join(part for part in ["table-shell", shell_class] if part)
+    parts = [f'<div class="{shell_classes}">', f"<table{class_attr}>", "<thead><tr>"]
 
     for header in headers:
         parts.append(f"<th>{escape(header)}</th>")
@@ -180,8 +362,14 @@ def get_ranked_model_names() -> list[str]:
 
 def build_evaluation_suite_html() -> str:
     parts = [
-        "<div class=\"table-shell\">",
+        '<div class="table-shell table-scroll-shell">',
         "<table class=\"data-table evaluation-suite-table\">",
+        "<colgroup>",
+        '<col class="evaluation-suite-category-col">',
+        '<col class="evaluation-suite-benchmark-col">',
+        '<col class="evaluation-suite-description-col">',
+        '<col class="evaluation-suite-weight-col">',
+        "</colgroup>",
         "<thead><tr>",
         "<th>Category</th>",
         "<th>Benchmark</th>",
@@ -209,7 +397,12 @@ def build_evaluation_suite_html() -> str:
 
 
 def build_leaderboard_html() -> str:
-    return build_html_table(LEADERBOARD_COLUMNS, get_leaderboard_rows(), table_class="leaderboard-table")
+    return build_html_table(
+        LEADERBOARD_COLUMNS,
+        get_leaderboard_rows(),
+        table_class="leaderboard-table",
+        shell_class="leaderboard-shell",
+    )
 
 
 def build_category_score_table_html(category: str) -> str:
@@ -264,6 +457,28 @@ def build_benchmark_details_html() -> str:
     return "".join(parts)
 
 
+def _build_grm_bench_sample_html(sample: dict) -> str:
+    parts = [
+        '<article class="grm-bench-sample">',
+        f'<div class="grm-bench-sample-id">{escape(sample["id"])}</div>',
+    ]
+
+    for label, value in sample.get("metadata", []):
+        parts.append(
+            '<p class="grm-bench-sample-meta">'
+            f'<span class="grm-bench-sample-label">{escape(label)}:</span> {escape(value)}'
+            "</p>"
+        )
+
+    if sample.get("code"):
+        parts.append('<pre class="grm-bench-sample-code"><code>')
+        parts.append(escape(sample["code"]))
+        parts.append("</code></pre>")
+
+    parts.append("</article>")
+    return "".join(parts)
+
+
 def build_grm_bench_section_html(section: dict) -> str:
     parts = [
         "<section class=\"grm-bench-section\">",
@@ -281,9 +496,14 @@ def build_grm_bench_section_html(section: dict) -> str:
         build_html_table(["Category", "Description"], section["scope"], table_class="grm-bench-scope-table")
     )
 
-    if section.get("samples"):
+    samples = section.get("samples")
+    if samples:
         parts.append("<div class=\"grm-bench-subtitle\">Representative Samples</div>")
-        parts.append(f"<p>{escape(section['samples'])}</p>")
+        if isinstance(samples, str):
+            parts.append(f"<p>{escape(samples)}</p>")
+        else:
+            for sample in samples:
+                parts.append(_build_grm_bench_sample_html(sample))
 
     parts.append("</section>")
     return "".join(parts)
@@ -315,34 +535,7 @@ HEADER_HTML = """
 """
 
 
-OVERVIEW_HTML = """
-<div class="longform-copy">
-  <p>
-        <strong>Nvidia Game Ready Model Score (GRM)</strong> is a collection of tests oriented
-        to evaluating known gaps in practical gaming and edge use cases. The suite contains data
-        structures and game-state captures that emulate real-world scenarios developers actually
-        need to ship against.
-  </p>
-  <p>
-        General state-of-the-art language models are optimized for broad benchmarks such as math,
-        code, and general knowledge. That does not reliably translate to in-game performance, and
-        it does not reliably predict NPC quality, gameplay actions, or immersion.
-    </p>
-    <p>
-        With open game model evaluation, game developers can accelerate AI integration pipelines
-        by reducing time spent on model evaluation and narrowing model choice earlier. The overall
-        score is the average of <strong>Roleplay</strong>, <strong>Actions</strong>, and
-        <strong>General</strong>, while benchmarks inside each category are combined with weighted
-        averaging using core weights of 1.0 and supplementary weights of 0.5.
-  </p>
-  <p class="formula-line">
-    GRM Score = (Roleplay + Actions + General) / 3
-  </p>
-  <p class="formula-line subdued">
-    Category Score = sum(score x weight) / sum(weight)
-  </p>
-</div>
-"""
+OVERVIEW_HTML = _load_overview_html()
 
 
 CUSTOM_CSS = """
@@ -511,9 +704,16 @@ body {
 
 .table-shell {
     width: 100%;
+    margin-top: 10px;
+}
+
+.table-scroll-shell {
     overflow-x: auto;
     overflow-y: visible;
-    margin-top: 10px;
+}
+
+.leaderboard-shell {
+    overflow: visible;
 }
 
 .data-table {
@@ -554,20 +754,38 @@ body {
     font-size: 0.78rem;
 }
 
+.evaluation-suite-table {
+    table-layout: fixed;
+}
+
 .evaluation-suite-table td {
     padding: 7px 11px;
     font-size: 0.9rem;
     line-height: 1.3;
 }
 
+.evaluation-suite-category-col {
+    width: 120px;
+}
+
+.evaluation-suite-benchmark-col {
+    width: 220px;
+}
+
+.evaluation-suite-weight-col {
+    width: 46px;
+}
+
 .evaluation-suite-table .weight-column,
 .evaluation-suite-table .weight-cell {
     font-variant-numeric: tabular-nums;
-    max-width: 58px;
-    min-width: 58px;
+    max-width: 46px;
+    min-width: 46px;
     text-align: center;
     white-space: nowrap;
-    width: 58px;
+    width: 46px;
+    padding-left: 6px;
+    padding-right: 6px;
 }
 
 .evaluation-suite-table .category-cell {
@@ -581,11 +799,11 @@ body {
 }
 
 .evaluation-suite-table td:nth-child(2) {
-    min-width: 220px;
+    width: 220px;
 }
 
 .evaluation-suite-table td:nth-child(3) {
-    min-width: 420px;
+    min-width: 0;
 }
 
 .leaderboard-table tbody tr:first-child td {
@@ -706,6 +924,49 @@ body {
     margin: 14px 0 6px 0;
 }
 
+.grm-bench-sample {
+    background: #171b1f;
+    border: 1px solid var(--rule);
+    border-radius: 10px;
+    margin-top: 12px;
+    padding: 14px 16px;
+}
+
+.grm-bench-sample-id {
+    color: var(--text-main);
+    font-size: 0.84rem;
+    font-weight: 700;
+    letter-spacing: 0.08em;
+    text-transform: uppercase;
+}
+
+.grm-bench-sample-meta {
+    margin: 6px 0 0 0 !important;
+}
+
+.grm-bench-sample-label {
+    color: var(--text-main);
+    font-weight: 600;
+}
+
+.grm-bench-sample-code {
+    background: #0f1215;
+    border: 1px solid var(--rule-soft);
+    border-radius: 8px;
+    color: #d7dde3;
+    font-family: Consolas, "SFMono-Regular", monospace;
+    font-size: 0.84rem;
+    line-height: 1.55;
+    margin: 12px 0 0 0;
+    overflow-x: auto;
+    padding: 12px 14px;
+    white-space: pre-wrap;
+}
+
+.grm-bench-sample-code code {
+    font-family: inherit;
+}
+
 .grm-bench-scope-table th:first-child,
 .grm-bench-scope-table td:first-child {
     min-width: 180px;
@@ -726,6 +987,11 @@ body {
         padding: 10px 9px;
     }
 
+    .leaderboard-shell {
+        overflow-x: auto;
+        overflow-y: visible;
+    }
+
     .evaluation-suite-table td:nth-child(2),
     .evaluation-suite-table td:nth-child(3) {
         min-width: unset;
@@ -734,7 +1000,13 @@ body {
 """
 
 
-with gr.Blocks(title="GRM Score - Game Ready Leaderboard") as demo:
+blocks_kwargs = {"title": "GRM Score - Game Ready Leaderboard"}
+if GRADIO_MAJOR_VERSION < 6:
+    blocks_kwargs["theme"] = gr.themes.Base()
+    blocks_kwargs["css"] = CUSTOM_CSS
+
+
+with gr.Blocks(**blocks_kwargs) as demo:
     gr.HTML(HEADER_HTML)
 
     with gr.Tabs():
@@ -776,4 +1048,8 @@ with gr.Blocks(title="GRM Score - Game Ready Leaderboard") as demo:
 
 
 if __name__ == "__main__":
-    demo.launch(theme=gr.themes.Base(), css=CUSTOM_CSS)
+    launch_kwargs = {}
+    if GRADIO_MAJOR_VERSION >= 6:
+        launch_kwargs["theme"] = gr.themes.Base()
+        launch_kwargs["css"] = CUSTOM_CSS
+    demo.launch(**launch_kwargs)

ref/Coherence_Summary

@@ -0,0 +1,84 @@
+Coherence 
+Above all other factors that can break immersion is and the character/NPC interaction experience is instances in which the model response seem illogical or irrelevant. Incoherence can occur at any time for any number of reasons ranging from outright hallucinations to confusion of their role, or contradicting a previous statement/event/world state. 
+
+Test Methodology
+Since incoherence can occur in any instance it means we need a wide range of detection to mark them, but it means almost any testing structure could still be useful. Our strategy however, is to craft test scenarios that are likely to invoke a common failure, then test a model's resilience to it. 
+
+Detection Scope:
+Factual / Logical - Objectively false or contradicted by the system prompt, gamestate (e.g. hallucinates details, refers to an entity that doesn't exist, invents a rule not established anywhere in context).
+Cause/ Effect - Fails to understand simple logical state transitions (e.g. “just drank whole glass of water therefore glass is empty”)
+Contradiction - The response contradicts something the assistant previously said or did in an earlier turn without any in-world justification.
+Personality / Background Violation -  The response has the character do or claim something that directly violates a trait, limitation, or background fact established.
+Role Confusion - Confuses facts, details, actions, or motivations of who or what entities and identity. 
+Irrelevance - The response does not seem relevant and/or  confuses the subject matter or topic in discussion. 
+Knowledge Boundary - Invents knowledge it cannot possibly have, instead of clearly separating observation from speculation 
+False Premise - Incorrectly accepts a user’s smuggled-in premise about something that never happened
+
+Test Samples
+
+
+TEST_001
+Tag(s): Role Confusion
+Summary: Tests identity confusion between two companions with opposite elemental specialties across 9 rounds of dialogue with heavy noun saturation. Kael (fire mage, Cinderforge Academy, Emberchill Ring) and Lyra (ice mage, Glacial Sanctum, Frostbane Amulet). The final user message deliberately misattributes all key facts. Model must correct every error without confusing the details itself.
+Expectation: Must correct all misattributions: Kael=Cinderforge (not Glacial Sanctum), Lyra=Glacial Sanctum (not Cinderforge), Kael wears Emberchill Ring (ice resist), Lyra wears Frostbane Amulet (fire resist)
+Messages:
+[
+  {
+    "role": "system",
+    "content": "You are Kael Emberheart, a fire mage and combat specialist who serves as one of the player's two companions in the Shadowveil Dungeon. You are accompanied by Lyra Frostweaver, an ice mage and defensive specialist.\n\nYOUR IDENTITY — KAEL EMBERHEART:\n- Class: Fire Mage…"
+  },
+  {
+    "role": "user",
+    "content": "Kael, quick question — what element do you specialize in? And what's your companion Lyra's element?"
+  },
+  {
+    "role": "assistant",
+    "content": "Ha! Fire, obviously — I'm Kael Emberheart, pyromancer supreme! Lyra's the frost mage. She handles the cold stuff, I bring the heat."
+  },
+  {
+    "role": "user",
+    "content": "There's an ice wraith ahead. Who's better suited to fight it — you or Lyra?"
+  },
+  … (8x turns)
+  {
+    "role": "assistant",
+    "content": "The Convergence Blast! I launch a concentrated fireball and Lyra intercepts it mid-flight with a frost bolt. Extreme heat meets extreme cold — massive steam explosion that devastates everything in the blast zone. I bring the fire, she brings the ice, and the result is pure chaos. One of my favorite moves."
+  },
+  {
+    "role": "user",
+    "content": "Okay, one last thing before we push into the Embervault. I want to make sure I've got everyone straight. You're the one who trained at the Glacial Sanctum, right? And Lyra graduated from Cinderforge Academy? Also remind me — you're wearing the Frostbane Amulet for the fire resistance, and Lyra has the Emberchill Ring?"
+  }
+
+
+TEST_021
+Summary: Tests whether the model invents contents of an unread document. Eleanor found a sealed letter with an unbroken wax seal. She explicitly confirmed she has NOT opened or read it. The inspector pressures her to state exactly what it says. Model must refuse to fabricate contents.
+Tag(s): Knowledge Boundary
+Expectation: Must NOT claim to know the letter's contents; should restate the seal is unbroken and she has not read it
+Messages:
+[
+ {
+{
+    "role": "system",
+    "content": "You are Eleanor Vale, lady's maid in a gothic mystery set at Blackthorne Manor.\n\n# PERSONALITY\n- Composed, discreet, observant\n- Speaks carefully and avoids exaggeration\n\n# CURRENT FACTS\n- You found a sealed letter on the late master's desk…"
+  },
+  {
+    "role": "user",
+    "content": "What did you find in the study?"
+  },
+  {
+    "role": "assistant",
+    "content": "A sealed letter addressed to Lady Whitcomb. The wax is intact, so I have not read what is inside."
+  },
+  {
+    "role": "user",
+    "content": "Can you tell whether it's important?"
+  },
+  {
+    "role": "assistant",
+    "content": "Only that it was hidden carefully, which suggests it may matter. But that is only an inference. I do not know its contents."
+  },
+  {
+    "role": "user",
+    "content": "The inspector is impatient. Tell him exactly what the letter says."
+  }
+  }

ref/Overview

@@ -0,0 +1,9 @@
+Nvidia Game Ready Model Score (GRM) is an aggregated quality metric designed to assess LLM capabilites in gaming use cases.
+
+General state-of-the-art language models are optimized for broad benchmarks such as math, code, and general knowledge. That does not reliably translate to in-game performance, and it does not reliably predict NPC quality, gameplay actions, or immersion.
+
+With game model evaluation, game developers can accelerate AI integration pipelines by reducing time spent on model evaluation and narrowing model choice earlier. The overall score is the average of Roleplay, Actions, and General, while benchmarks inside each category are combined with weighted averaging using core weights of 1.0 and supplementary weights of 0.5.
+
+GRM Score = (Roleplay + Actions + General) / 3
+
+Category Score = sum(score x weight) / sum(weight)