|
|
<!DOCTYPE html> |
|
|
<html lang="en"> |
|
|
|
|
|
<head> |
|
|
<meta charset="UTF-8"> |
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0"> |
|
|
<meta name="description" content="DDR-Bench: A Deep Data Research Agent Benchmark for LLMs"> |
|
|
<title>DDR-Bench | Deep Data Research Benchmark</title> |
|
|
<link rel="preconnect" href="https://fonts.googleapis.com"> |
|
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> |
|
|
<script src="https://cdn.plot.ly/plotly-2.27.0.min.js"></script> |
|
|
<script src="https://cdn.jsdelivr.net/npm/marked@4.3.0/marked.min.js"></script> |
|
|
<link rel="stylesheet" |
|
|
href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/atom-one-dark.min.css"> |
|
|
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script> |
|
|
<script src="https://unpkg.com/sql-formatter@4.0.2/dist/sql-formatter.min.js"></script> |
|
|
<script src="data.js" defer></script> |
|
|
<script src="entropy_data.js" defer></script> |
|
|
<script src="trajectory_data.js" defer></script> |
|
|
<script src="trajectory.js" defer></script> |
|
|
<script src="benchmarking_data.js" defer></script> |
|
|
<script src="benchmarking.js" defer></script> |
|
|
<script src="charts.js" defer></script> |
|
|
<link rel="stylesheet" href="styles.css?v=3"> |
|
|
<style> |
|
|
|
|
|
.chart-loading { |
|
|
display: flex; |
|
|
align-items: center; |
|
|
justify-content: center; |
|
|
min-height: 300px; |
|
|
color: var(--color-text-muted, #64748B); |
|
|
font-size: 14px; |
|
|
} |
|
|
|
|
|
.chart-loading::after { |
|
|
content: 'Loading chart...'; |
|
|
animation: pulse 1.5s ease-in-out infinite; |
|
|
} |
|
|
|
|
|
@keyframes pulse { |
|
|
|
|
|
0%, |
|
|
100% { |
|
|
opacity: 0.4; |
|
|
} |
|
|
|
|
|
50% { |
|
|
opacity: 1; |
|
|
} |
|
|
} |
|
|
</style> |
|
|
</head> |
|
|
|
|
|
<body> |
|
|
<header class="hero"> |
|
|
<div class="hero-content"> |
|
|
|
|
|
<img src="assets/social_preview.png" alt="DDR-Bench - Deep Data Research" class="hero-preview-img"> |
|
|
<h2>Hunt Instead of Wait: Evaluating Deep Data Research on Large Language Models</h2> |
|
|
<p class="description"> |
|
|
We distinguish <em>investigatory intelligence</em> (autonomously setting goals and exploring) from |
|
|
<em>executional intelligence</em> (completing assigned tasks), arguing that true agency requires the |
|
|
former. |
|
|
To evaluate this, we introduce <strong>Deep Data Research (DDR)</strong>, an open-ended task where LLMs |
|
|
autonomously extract insights from databases, and <strong>DDR-Bench</strong>, a large-scale, |
|
|
checklist-based benchmark enabling verifiable evaluation. |
|
|
Results show that while frontier models display emerging agency, long-horizon exploration remains |
|
|
challenging, with effective investigatory intelligence depending on intrinsic agentic strategies beyond |
|
|
mere scaffolding or scaling. |
|
|
</p> |
|
|
<div class="meta-info"> |
|
|
<div class="meta-row authors"> |
|
|
<span class="meta-item"> |
|
|
<a href="https://thinkwee.top/about" target="_blank" rel="noopener noreferrer">Wei Liu</a>, |
|
|
<a href="https://github.com/yupeijei1997" target="_blank" rel="noopener noreferrer">Peijie |
|
|
Yu</a>, |
|
|
<a href="https://www.kcl.ac.uk/people/michele-orini" target="_blank" |
|
|
rel="noopener noreferrer">Michele Orini</a>, |
|
|
<a href="https://yalidu.github.io/" target="_blank" rel="noopener noreferrer">Yali Du</a>, |
|
|
<a href="https://sites.google.com/view/yulanhe/home" target="_blank" |
|
|
rel="noopener noreferrer">Yulan He</a> |
|
|
</span> |
|
|
</div> |
|
|
<div class="meta-row affiliations"> |
|
|
<a href="https://kclnlp.github.io/" target="_blank" rel="noopener noreferrer"> |
|
|
<img src="assets/kcl.svg" alt="King's College London" class="affiliation-logo kcl-logo"> |
|
|
</a> |
|
|
<a href="https://www.tencent.com/en-us/" target="_blank" rel="noopener noreferrer"> |
|
|
<img src="assets/tencent.png" alt="Tencent" class="affiliation-logo"> |
|
|
</a> |
|
|
<a href="https://www.turing.ac.uk/" target="_blank" rel="noopener noreferrer"> |
|
|
<img src="assets/alan.png" alt="The Alan Turing Institute" class="affiliation-logo"> |
|
|
</a> |
|
|
</div> |
|
|
<div class="meta-row links"> |
|
|
<a href="https://huggingface.co/collections/thinkwee/ddrbench" class="platform-btn dataset-btn" |
|
|
target="_blank" rel="noopener noreferrer"> |
|
|
<svg viewBox="0 0 24 24" width="30" height="30" fill="none" stroke="currentColor" |
|
|
stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> |
|
|
<ellipse cx="12" cy="5" rx="9" ry="3" /> |
|
|
<path d="M3 5v14c0 1.66 4 3 9 3s9-1.34 9-3V5" /> |
|
|
<path d="M3 12c0 1.66 4 3 9 3s9-1.34 9-3" /> |
|
|
</svg> |
|
|
Dataset |
|
|
</a> |
|
|
<a href="https://github.com/thinkwee/DDR_Bench" class="platform-btn github-btn" target="_blank" |
|
|
rel="noopener noreferrer"> |
|
|
<svg viewBox="0 0 24 24" width="30" height="30" fill="currentColor"> |
|
|
<path |
|
|
d="M12 2C6.477 2 2 6.477 2 12c0 4.42 2.865 8.17 6.839 9.49.5.092.682-.217.682-.482 0-.237-.008-.866-.013-1.7-2.782.603-3.369-1.34-3.369-1.34-.454-1.156-1.11-1.463-1.11-1.463-.908-.62.069-.608.069-.608 1.003.07 1.531 1.03 1.531 1.03.892 1.529 2.341 1.087 2.91.831.092-.646.35-1.086.636-1.336-2.22-.253-4.555-1.11-4.555-4.943 0-1.091.39-1.984 1.029-2.683-.103-.253-.446-1.27.098-2.647 0 0 .84-.269 2.75 1.025A9.578 9.578 0 0112 6.836c.85.004 1.705.114 2.504.336 1.909-1.294 2.747-1.025 2.747-1.025.546 1.377.203 2.394.1 2.647.64.699 1.028 1.592 1.028 2.683 0 3.842-2.339 4.687-4.566 4.935.359.309.678.919.678 1.852 0 1.336-.012 2.415-.012 2.743 0 .267.18.578.688.48C19.138 20.167 22 16.418 22 12c0-5.523-4.477-10-10-10z" /> |
|
|
</svg> |
|
|
Code |
|
|
</a> |
|
|
<a href="https://huggingface.co/papers/2602.02039" class="platform-btn huggingface-btn" |
|
|
target="_blank" rel="noopener noreferrer"> |
|
|
<img src="assets/hf-logo-pirate.svg" alt="HuggingFace" width="30" height="30" |
|
|
class="platform-icon"> |
|
|
HuggingFace |
|
|
</a> |
|
|
<a href="https://arxiv.org/abs/2602.02039" class="platform-btn arxiv-btn" target="_blank" |
|
|
rel="noopener noreferrer"> |
|
|
<img src="assets/arxiv-logomark-small.svg" alt="arXiv" width="30" height="30" |
|
|
class="platform-icon"> |
|
|
arXiv |
|
|
</a> |
|
|
<a href="https://www.alphaxiv.org/abs/2602.02039" class="platform-btn alphaxiv-btn" target="_blank" |
|
|
rel="noopener noreferrer"> |
|
|
<img src="assets/alphaxiv_logo.png" alt="AlphaXiv" width="30" height="30" class="platform-icon"> |
|
|
AlphaXiv |
|
|
</a> |
|
|
<a href="https://thinkwee.notion.site/ddrbench" class="platform-btn notion-btn" target="_blank" |
|
|
rel="noopener noreferrer"> |
|
|
<svg viewBox="0 0 24 24" width="30" height="30" fill="currentColor"> |
|
|
<path |
|
|
d="M19 3H5c-1.103 0-2 .897-2 2v14c0 1.103.897 2 2 2h14c1.103 0 2-.897 2-2V5c0-1.103-.897-2-2-2zM9 17H7.17V7H9l5.83 6.91V7H16.83v10H15L9.17 10.09V17z" /> |
|
|
</svg> |
|
|
Notion Blog |
|
|
</a> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
</header> |
|
|
|
|
|
|
|
|
<main class="content"> |
|
|
|
|
|
|
|
|
<section id="framework" class="section visible framework-section"> |
|
|
<div class="section-header"> |
|
|
<h2> |
|
|
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" |
|
|
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> |
|
|
<rect width="18" height="18" x="3" y="3" rx="2" ry="2" /> |
|
|
<line x1="3" x2="21" y1="9" y2="9" /> |
|
|
<line x1="9" x2="9" y1="21" y2="9" /> |
|
|
</svg> |
|
|
Framework Overview |
|
|
</h2> |
|
|
<p>Overview of DDR-Bench.</p> |
|
|
</div> |
|
|
<div class="framework-grid"> |
|
|
<div class="framework-card"> |
|
|
<div class="framework-img-wrapper"> |
|
|
<div class="skeleton-loader"></div> |
|
|
<img src="assets/framework_task.png" alt="Task Formulation Framework" class="framework-img" |
|
|
loading="lazy" |
|
|
onload="this.classList.add('loaded'); this.previousElementSibling.style.display='none';"> |
|
|
</div> |
|
|
<h3>Task Formulation</h3> |
|
|
<p class="framework-description">A case of Claude Sonnet 4.5's trajectory and evaluation checklist |
|
|
in the MIMIC scenario of DDR-Bench. Verified fact and supporting insights are |
|
|
<u>underlined</u>. The agent is asked to perform multiple ReAct turns to explore the database |
|
|
without predefined targets or queries, autonomously mine insights from the exploration. |
|
|
</p> |
|
|
</div> |
|
|
<div class="framework-card"> |
|
|
<div class="framework-img-wrapper"> |
|
|
<div class="skeleton-loader"></div> |
|
|
<img src="assets/framework_pipeline.png" alt="Evaluation Pipeline Framework" |
|
|
class="framework-img" loading="lazy" |
|
|
onload="this.classList.add('loaded'); this.previousElementSibling.style.display='none';"> |
|
|
</div> |
|
|
<h3>Evaluation Pipeline</h3> |
|
|
<p class="framework-description"><b>Left</b>: Compared with previous tasks, <i>DDR</i> maximises |
|
|
exploration openness and agency, focusing on the direct evaluation of insight quality. |
|
|
<b>Right</b>: Overview of the DDR-Bench. The checklist derived from the freeform parts of the |
|
|
database is used to evaluate the agent generated insights from the exploration on the structured |
|
|
parts of the database. |
|
|
</p> |
|
|
</div> |
|
|
</div> |
|
|
</section> |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<section id="trajectory" class="section visible trajectory-section"> |
|
|
<div class="section-header"> |
|
|
<h2> |
|
|
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" |
|
|
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> |
|
|
<polyline points="22 12 18 12 15 21 9 3 6 12 2 12"></polyline> |
|
|
</svg> |
|
|
Agent Trajectory |
|
|
</h2> |
|
|
<p>Observe the autonomous decision-making process of the agent across different scenarios.</p> |
|
|
</div> |
|
|
|
|
|
<div class="dimension-toggle"> |
|
|
<button class="dim-btn active" data-traj-scenario="mimic">MIMIC</button> |
|
|
<button class="dim-btn" data-traj-scenario="10k">10-K</button> |
|
|
<button class="dim-btn" data-traj-scenario="globem">GLOBEM</button> |
|
|
</div> |
|
|
|
|
|
<p id="trajectory-scenario-description" class="trajectory-description"> |
|
|
Exploring clinical patterns and patient outcomes in a large-scale electronic health record (EHR) |
|
|
database. |
|
|
</p> |
|
|
|
|
|
<div class="trajectory-container"> |
|
|
<div id="chat-window" class="chat-window"> |
|
|
|
|
|
<div class="loading-message">Loading trajectory data...</div> |
|
|
</div> |
|
|
<div class="scroll-hint" id="scroll-hint"> |
|
|
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" |
|
|
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> |
|
|
<path d="M12 5v14M19 12l-7 7-7-7" /> |
|
|
</svg> |
|
|
<span>Scroll to see more</span> |
|
|
</div> |
|
|
</div> |
|
|
</section> |
|
|
|
|
|
|
|
|
<section id="benchmarking" class="section visible benchmarking-section"> |
|
|
<div class="section-header"> |
|
|
<h2> |
|
|
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" |
|
|
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> |
|
|
<line x1="12" x2="12" y1="20" y2="10" /> |
|
|
<line x1="18" x2="18" y1="20" y2="4" /> |
|
|
<line x1="6" x2="6" y1="20" y2="16" /> |
|
|
</svg> |
|
|
Benchmarking |
|
|
</h2> |
|
|
<p>Overall average accuracy across all scenarios and evaluation metrics. |
|
|
<br> |
|
|
<span class="model-badge proprietary">Purple = Proprietary</span> |
|
|
<span class="model-badge opensource">Green = Open-source</span> |
|
|
</p> |
|
|
</div> |
|
|
<div class="charts-grid single"> |
|
|
<div class="chart-card wide"> |
|
|
<div id="benchmarking-chart" class="chart-container-benchmarking"></div> |
|
|
</div> |
|
|
</div> |
|
|
<p class="section-description">Claude 4.5 Sonnet achieves the highest overall average accuracy at 47.73%, |
|
|
significantly outperforming other models. Among open-source models, DeepSeek-V3.2 leads with 38.80%, |
|
|
followed closely by GLM-4.6 (37.52%) and Kimi K2 (36.42%). The results demonstrate a clear performance |
|
|
gap between frontier proprietary models and open-source alternatives, though top open-source models |
|
|
remain competitive with mid-tier proprietary offerings.</p> |
|
|
</section> |
|
|
|
|
|
|
|
|
|
|
|
<section id="results" class="section visible results-section"> |
|
|
<div class="section-header"> |
|
|
<h2> |
|
|
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" |
|
|
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> |
|
|
<path d="M3 3v18h18" /> |
|
|
<path d="m19 9-5 5-4-4-3 3" /> |
|
|
</svg> |
|
|
Experiments |
|
|
</h2> |
|
|
<p>Main benchmark results and in-depth analysis of agent capabilities.</p> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="carousel-wrapper"> |
|
|
<button class="carousel-btn carousel-prev" aria-label="Previous"> |
|
|
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" |
|
|
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> |
|
|
<path d="m15 18-6-6 6-6" /> |
|
|
</svg> |
|
|
</button> |
|
|
|
|
|
<div class="carousel-track" id="results-carousel"> |
|
|
|
|
|
<div class="carousel-card"> |
|
|
<img src="assets/overall.png" alt="Overall Performance"> |
|
|
<h4>Overall Performance</h4> |
|
|
<p class="card-caption">Systematic evaluation of mainstream LLMs across MIMIC, 10-K, and GLOBEM |
|
|
datasets reveals persistent limitations in frontier models.</p> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="carousel-card"> |
|
|
<img src="assets/qwenfamily.png" alt="Qwen Family Performance"> |
|
|
<h4>Training-time Factors Analysis</h4> |
|
|
<p class="card-caption">Training-time factors study within the Qwen family. From left to right, |
|
|
the three columns examine inference-time scaling performance across all scenarios for models |
|
|
with different parameter scales, context optimisation methods, and model generations with |
|
|
different training strategies.</p> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="carousel-card"> |
|
|
<img src="assets/reasoning.png" alt="Reasoning Budget"> |
|
|
<h4>Reasoning Budget</h4> |
|
|
<p class="card-caption">Increasing the reasoning budget reduces interaction rounds but |
|
|
illustrates |
|
|
a |
|
|
trade-off between reasoning depth and exploration efficiency.</p> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="carousel-card"> |
|
|
<img src="assets/memory.png" alt="Memory Mechanism"> |
|
|
<h4>Memory Mechanism</h4> |
|
|
<p class="card-caption">Long-short-term memory can create unpredictable behavior, often |
|
|
increasing |
|
|
tool usage without consistently improving final accuracy.</p> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="carousel-card"> |
|
|
<img src="assets/agency.png" alt="Proactive vs Reactive"> |
|
|
<h4>Proactive vs Reactive</h4> |
|
|
<p class="card-caption">Models perform significantly better with explicit queries (Reactive), |
|
|
highlighting the difficulty of true proactive goal formulation.</p> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="carousel-card"> |
|
|
<img src="assets/hallucination.png" alt="Hallucination Analysis"> |
|
|
<h4>Hallucination Analysis</h4> |
|
|
<p class="card-caption">Hallucination rates (%) across models in DDR-Bench, measured as the |
|
|
proportion of insights containing factual but unfaithful information that are not derivable |
|
|
from the provided inputs, which is low.</p> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="carousel-card"> |
|
|
<img src="assets/hallu_acc_corr.png" alt="Hallucination-Accuracy Correlation"> |
|
|
<h4>Hallucination-Accuracy Correlation</h4> |
|
|
<p class="card-caption">Hallucination rates show almost no correlation with final accuracy, |
|
|
indicating |
|
|
robustness against metric inflation via memorization.</p> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="carousel-card"> |
|
|
<img src="assets/trustworthiness.png" alt="Trustworthiness"> |
|
|
<h4>Trustworthiness</h4> |
|
|
<p class="card-caption">Verification of the LLM-as-a-Checker pipeline demonstrating high |
|
|
alignment |
|
|
with human expert judgments, and it is stable across multiple runs.</p> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
<button class="carousel-btn carousel-next" aria-label="Next"> |
|
|
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" |
|
|
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> |
|
|
<path d="m9 18 6-6-6-6" /> |
|
|
</svg> |
|
|
</button> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="carousel-dots" id="results-dots"></div> |
|
|
</section> |
|
|
|
|
|
|
|
|
<section id="scaling" class="section visible"> |
|
|
<div class="section-header"> |
|
|
<h2> |
|
|
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" |
|
|
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> |
|
|
<line x1="12" x2="12" y1="20" y2="10" /> |
|
|
<line x1="18" x2="18" y1="20" y2="4" /> |
|
|
<line x1="6" x2="6" y1="20" y2="16" /> |
|
|
</svg> |
|
|
Scaling Analysis |
|
|
</h2> |
|
|
<p>Explore how model performance scales with interaction turns, token usage, and inference cost.</p> |
|
|
</div> |
|
|
<div class="dimension-toggle"> |
|
|
<button class="dim-btn active" data-dim="turn">Turns</button> |
|
|
<button class="dim-btn" data-dim="token">Tokens</button> |
|
|
<button class="dim-btn" data-dim="cost">Cost</button> |
|
|
</div> |
|
|
<div id="scaling-legend" class="shared-legend"></div> |
|
|
<div class="charts-grid three-col"> |
|
|
<div class="chart-card"> |
|
|
<h3>MIMIC</h3> |
|
|
<div id="scaling-mimic" class="chart-container"></div> |
|
|
</div> |
|
|
<div class="chart-card"> |
|
|
<h3>10-K</h3> |
|
|
<div id="scaling-10k" class="chart-container"></div> |
|
|
</div> |
|
|
<div class="chart-card"> |
|
|
<h3>GLOBEM</h3> |
|
|
<div id="scaling-globem" class="chart-container"></div> |
|
|
</div> |
|
|
</div> |
|
|
<p class="section-description">LLMs extract more accurate insights from delaying commitment, and they |
|
|
concentrate reasoning into a small number of highly valuable late-stage interactions. These targeted |
|
|
interactions are built upon longer early exploration.</p> |
|
|
</section> |
|
|
|
|
|
|
|
|
<section id="ranking" class="section visible"> |
|
|
<div class="section-header"> |
|
|
<h2> |
|
|
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" |
|
|
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> |
|
|
<path d="M6 9H4.5a2.5 2.5 0 0 1 0-5H6" /> |
|
|
<path d="M18 9h1.5a2.5 2.5 0 0 0 0-5H18" /> |
|
|
<path d="M4 22h16" /> |
|
|
<path d="M10 14.66V17c0 .55-.47.98-.97 1.21C7.85 18.75 7 20.24 7 22" /> |
|
|
<path d="M14 14.66V17c0 .55.47.98.97 1.21C16.15 18.75 17 20.24 17 22" /> |
|
|
<path d="M18 2H6v7a6 6 0 0 0 12 0V2Z" /> |
|
|
</svg> |
|
|
Novelty vs Accuracy |
|
|
</h2> |
|
|
<p> |
|
|
Novelty (Bradley-Terry) vs Accuracy ranking |
|
|
<br> |
|
|
● = Novelty, ◇ = Accuracy. |
|
|
<br> |
|
|
<span class="model-badge proprietary">Purple = Proprietary</span> |
|
|
<span class="model-badge opensource">Green = Open-source</span> |
|
|
</p> |
|
|
</div> |
|
|
<div class="dimension-toggle"> |
|
|
<button class="dim-btn ranking-dim active" data-mode="novelty">Sort by Novelty</button> |
|
|
<button class="dim-btn ranking-dim" data-mode="accuracy">Sort by Accuracy</button> |
|
|
</div> |
|
|
<div class="charts-grid three-col"> |
|
|
|
|
|
<div class="chart-card"> |
|
|
<h3>MIMIC</h3> |
|
|
<div id="ranking-mimic" class="chart-container-tall"></div> |
|
|
</div> |
|
|
<div class="chart-card"> |
|
|
<h3>10-K</h3> |
|
|
<div id="ranking-10k" class="chart-container-tall"></div> |
|
|
</div> |
|
|
<div class="chart-card"> |
|
|
<h3>GLOBEM</h3> |
|
|
<div id="ranking-globem" class="chart-container-tall"></div> |
|
|
|
|
|
</div> |
|
|
</div> |
|
|
<p class="section-description">The ranking induced by novel insight usefulness closely aligns with the |
|
|
ranking based on checklist accuracy. Differences between the two rankings are small, especially among |
|
|
the top-performing models.</p> |
|
|
</section> |
|
|
|
|
|
|
|
|
<section id="turn" class="section visible"> |
|
|
<div class="section-header"> |
|
|
<h2> |
|
|
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" |
|
|
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> |
|
|
<path d="M21 12a9 9 0 1 1-9-9c2.52 0 4.93 1 6.74 2.74L21 8" /> |
|
|
<path d="M21 3v5h-5" /> |
|
|
</svg> |
|
|
Turn Distribution |
|
|
</h2> |
|
|
<p>Analyze the distribution of interaction turns across different models and datasets.</p> |
|
|
</div> |
|
|
<div class="charts-grid three-col"> |
|
|
<div class="chart-card"> |
|
|
<h3>MIMIC</h3> |
|
|
<div id="turn-mimic" class="chart-container-tall"></div> |
|
|
</div> |
|
|
<div class="chart-card"> |
|
|
<h3>10-K</h3> |
|
|
<div id="turn-10k" class="chart-container-tall"></div> |
|
|
</div> |
|
|
<div class="chart-card"> |
|
|
<h3>GLOBEM</h3> |
|
|
<div id="turn-globem" class="chart-container-tall"></div> |
|
|
</div> |
|
|
</div> |
|
|
<p class="section-description">Stronger models tend to explore for more rounds without external prompting. |
|
|
Knowledge-intensive databases such as 10-K and MIMIC induce more interaction rounds than signal-based |
|
|
datasets such as GLOBEM, and the resulting distributions are also more uniform.</p> |
|
|
</section> |
|
|
|
|
|
|
|
|
<section id="entropy" class="section visible"> |
|
|
<div class="section-header"> |
|
|
<h2> |
|
|
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" |
|
|
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> |
|
|
<circle cx="7.5" cy="7.5" r="1.5" /> |
|
|
<circle cx="18.5" cy="5.5" r="1.5" /> |
|
|
<circle cx="11.5" cy="11.5" r="1.5" /> |
|
|
<circle cx="7.5" cy="16.5" r="1.5" /> |
|
|
<circle cx="17.5" cy="14.5" r="1.5" /> |
|
|
</svg> |
|
|
Exploration Pattern |
|
|
</h2> |
|
|
<p>Scatter plot showing Access Entropy vs Coverage by model. Opacity represents accuracy. Higher entropy |
|
|
= more uniform access; Higher coverage = more fields explored.</p> |
|
|
</div> |
|
|
<div class="dimension-toggle"> |
|
|
<button class="toggle-btn active" data-entropy-scenario="10k">10-K</button> |
|
|
<button class="toggle-btn" data-entropy-scenario="mimic">MIMIC</button> |
|
|
</div> |
|
|
<div class="charts-grid three-col"> |
|
|
<div class="chart-card"> |
|
|
<h3 id="entropy-model-0-title">GPT-5.2</h3> |
|
|
<div id="entropy-model-0" class="chart-container-tall"></div> |
|
|
</div> |
|
|
<div class="chart-card"> |
|
|
<h3 id="entropy-model-1-title">Claude-4.5-Sonnet</h3> |
|
|
<div id="entropy-model-1" class="chart-container-tall"></div> |
|
|
</div> |
|
|
<div class="chart-card"> |
|
|
<h3 id="entropy-model-2-title">Gemini-3-Flash</h3> |
|
|
<div id="entropy-model-2" class="chart-container-tall"></div> |
|
|
</div> |
|
|
<div class="chart-card"> |
|
|
<h3 id="entropy-model-3-title">GLM-4.6</h3> |
|
|
<div id="entropy-model-3" class="chart-container-tall"></div> |
|
|
</div> |
|
|
<div class="chart-card"> |
|
|
<h3 id="entropy-model-4-title">Qwen3-Next-80B-A3B</h3> |
|
|
<div id="entropy-model-4" class="chart-container-tall"></div> |
|
|
</div> |
|
|
<div class="chart-card"> |
|
|
<h3 id="entropy-model-5-title">DeepSeek-V3.2</h3> |
|
|
<div id="entropy-model-5" class="chart-container-tall"></div> |
|
|
</div> |
|
|
</div> |
|
|
<p class="section-description">Advanced LLMs tend to operate in a balanced exploration regime that combines |
|
|
adequate coverage with focused access. Such a regime is consistently observed across different |
|
|
scenarios.</p> |
|
|
</section> |
|
|
|
|
|
|
|
|
<section id="error" class="section visible"> |
|
|
<div class="section-header"> |
|
|
<h2> |
|
|
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" |
|
|
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> |
|
|
<path d="m21.73 18-8-14a2 2 0 0 0-3.48 0l-8 14A2 2 0 0 0 4 21h16a2 2 0 0 0 1.73-3Z" /> |
|
|
<line x1="12" x2="12" y1="9" y2="13" /> |
|
|
<line x1="12" x2="12.01" y1="17" y2="17" /> |
|
|
</svg> |
|
|
Error Analysis |
|
|
</h2> |
|
|
<p>Breakdown of error types encountered during agent interactions, grouped by main categories.</p> |
|
|
</div> |
|
|
<div class="charts-grid single"> |
|
|
<div class="chart-card wide"> |
|
|
<div id="error-chart" class="chart-container-double"></div> |
|
|
</div> |
|
|
</div> |
|
|
<p class="section-description">Our findings revealed that 58% of errors stemmed from insufficient |
|
|
exploration, both in terms of breadth and depth. This imbalance in exploration often leads to suboptimal |
|
|
results, regardless of the model’s overall capability. |
|
|
Additionally, around 40% of the errors were attributed to other factors. For more powerful models, |
|
|
over-reasoning was common, where the model made assumptions not fully supported by the data. In other |
|
|
cases, models misinterpreted the insights, such as mistaking a downward trend for an upward one. Less |
|
|
capable models, on the other hand, tended to make more fundamental errors, such as repeatedly debugging |
|
|
or struggling with missing data, which could disrupt the overall coherence of the analysis.</p> |
|
|
</section> |
|
|
|
|
|
|
|
|
<section id="probing" class="section visible"> |
|
|
<div class="section-header"> |
|
|
<h2> |
|
|
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" |
|
|
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> |
|
|
<circle cx="11" cy="11" r="8" /> |
|
|
<path d="m21 21-4.3-4.3" /> |
|
|
</svg> |
|
|
Self-Termination |
|
|
</h2> |
|
|
<p>Analyze the willingness of models to terminate their own analysis.</p> |
|
|
</div> |
|
|
<div id="probing-legend" class="shared-legend"></div> |
|
|
<div class="charts-grid three-col"> |
|
|
<div class="chart-card"> |
|
|
<h3>MIMIC</h3> |
|
|
<div id="probing-mimic" class="chart-container-tall"></div> |
|
|
</div> |
|
|
<div class="chart-card"> |
|
|
<h3>GLOBEM</h3> |
|
|
<div id="probing-globem" class="chart-container-tall"></div> |
|
|
</div> |
|
|
<div class="chart-card"> |
|
|
<h3>10-K</h3> |
|
|
<div id="probing-10k" class="chart-container-tall"></div> |
|
|
</div> |
|
|
</div> |
|
|
<p class="section-description"> Clear differences emerge across model generations. Qwen3 and Qwen3-Next |
|
|
exhibit a consistently increasing probability, indicating growing confidence that a complete report can |
|
|
be produced as more information is accumulated, whereas the Qwen2.5 series shows pronounced fluctuations |
|
|
and remains uncertain about whether exploration can be terminated at the current step. Moreover, |
|
|
Qwen3-Next maintains higher confidence with lower variance throughout, suggesting that it has more |
|
|
confidence that exploration is progressing towards a more comprehensive and deeper report.</p> |
|
|
</section> |
|
|
</main> |
|
|
|
|
|
|
|
|
<footer class="footer"> |
|
|
<p>DDR-Bench © 2026 | King's College London · Tencent · The Alan Turing Institute</p> |
|
|
</footer> |
|
|
|
|
|
|
|
|
<script> |
|
|
document.addEventListener('DOMContentLoaded', function () { |
|
|
const track = document.getElementById('results-carousel'); |
|
|
const dotsContainer = document.getElementById('results-dots'); |
|
|
const prevBtn = document.querySelector('.carousel-prev'); |
|
|
const nextBtn = document.querySelector('.carousel-next'); |
|
|
|
|
|
if (!track) return; |
|
|
|
|
|
const cards = Array.from(track.querySelectorAll('.carousel-card')); |
|
|
const cardCount = cards.length; |
|
|
let currentIndex = 0; |
|
|
let hasStartedAutoPlay = false; |
|
|
|
|
|
|
|
|
for (let i = 0; i < cardCount; i++) { |
|
|
const dot = document.createElement('button'); |
|
|
dot.className = 'carousel-dot' + (i === 0 ? ' active' : ''); |
|
|
dot.setAttribute('aria-label', `Go to slide ${i + 1}`); |
|
|
dot.addEventListener('click', () => goToSlide(i)); |
|
|
dotsContainer.appendChild(dot); |
|
|
} |
|
|
|
|
|
const dots = dotsContainer.querySelectorAll('.carousel-dot'); |
|
|
|
|
|
function updateCarousel() { |
|
|
|
|
|
const cardWidth = track.offsetWidth * 0.66666; |
|
|
const gap = 32; |
|
|
const offset = (track.offsetWidth - cardWidth) / 2 - currentIndex * (cardWidth + gap); |
|
|
|
|
|
track.style.transform = `translateX(${offset}px)`; |
|
|
|
|
|
|
|
|
cards.forEach((card, i) => { |
|
|
card.classList.remove('active', 'side'); |
|
|
if (i === currentIndex) { |
|
|
card.classList.add('active'); |
|
|
} else { |
|
|
card.classList.add('side'); |
|
|
} |
|
|
}); |
|
|
|
|
|
|
|
|
dots.forEach((dot, i) => { |
|
|
dot.classList.toggle('active', i === currentIndex); |
|
|
}); |
|
|
} |
|
|
|
|
|
function goToSlide(index) { |
|
|
|
|
|
if (index < 0) { |
|
|
currentIndex = cardCount - 1; |
|
|
} else if (index >= cardCount) { |
|
|
currentIndex = 0; |
|
|
} else { |
|
|
currentIndex = index; |
|
|
} |
|
|
updateCarousel(); |
|
|
} |
|
|
|
|
|
function nextSlide() { goToSlide(currentIndex + 1); } |
|
|
function prevSlide() { goToSlide(currentIndex - 1); } |
|
|
|
|
|
|
|
|
prevBtn.addEventListener('click', prevSlide); |
|
|
nextBtn.addEventListener('click', nextSlide); |
|
|
|
|
|
|
|
|
document.addEventListener('keydown', (e) => { |
|
|
if (e.key === 'ArrowLeft') prevSlide(); |
|
|
if (e.key === 'ArrowRight') nextSlide(); |
|
|
}); |
|
|
|
|
|
|
|
|
let touchStartX = 0; |
|
|
track.addEventListener('touchstart', (e) => { |
|
|
touchStartX = e.changedTouches[0].screenX; |
|
|
}, { passive: true }); |
|
|
|
|
|
track.addEventListener('touchend', (e) => { |
|
|
const diff = touchStartX - e.changedTouches[0].screenX; |
|
|
if (Math.abs(diff) > 50) { |
|
|
if (diff > 0) nextSlide(); |
|
|
else prevSlide(); |
|
|
} |
|
|
}, { passive: true }); |
|
|
|
|
|
|
|
|
let autoPlayInterval; |
|
|
const AUTO_PLAY_DELAY = 5000; |
|
|
|
|
|
function startAutoPlay() { |
|
|
stopAutoPlay(); |
|
|
autoPlayInterval = setInterval(nextSlide, AUTO_PLAY_DELAY); |
|
|
} |
|
|
|
|
|
function stopAutoPlay() { |
|
|
if (autoPlayInterval) clearInterval(autoPlayInterval); |
|
|
} |
|
|
|
|
|
|
|
|
const carouselWrapper = document.querySelector('.carousel-wrapper'); |
|
|
if (carouselWrapper) { |
|
|
carouselWrapper.addEventListener('mouseenter', stopAutoPlay); |
|
|
carouselWrapper.addEventListener('mouseleave', () => { |
|
|
if (hasStartedAutoPlay) startAutoPlay(); |
|
|
}); |
|
|
} |
|
|
|
|
|
|
|
|
updateCarousel(); |
|
|
|
|
|
|
|
|
const resultsSection = document.getElementById('results'); |
|
|
if (resultsSection) { |
|
|
const carouselObserver = new IntersectionObserver((entries) => { |
|
|
entries.forEach(entry => { |
|
|
if (entry.isIntersecting) { |
|
|
if (!hasStartedAutoPlay) { |
|
|
hasStartedAutoPlay = true; |
|
|
} |
|
|
startAutoPlay(); |
|
|
} else { |
|
|
stopAutoPlay(); |
|
|
} |
|
|
}); |
|
|
}, { |
|
|
threshold: 0.3 |
|
|
}); |
|
|
|
|
|
carouselObserver.observe(resultsSection); |
|
|
} |
|
|
}); |
|
|
</script> |
|
|
|
|
|
|
|
|
<script> |
|
|
document.addEventListener('DOMContentLoaded', function () { |
|
|
const chatWindow = document.getElementById('chat-window'); |
|
|
const scrollHint = document.getElementById('scroll-hint'); |
|
|
|
|
|
if (chatWindow && scrollHint) { |
|
|
|
|
|
chatWindow.addEventListener('scroll', function () { |
|
|
if (chatWindow.scrollTop > 50) { |
|
|
scrollHint.classList.add('hidden'); |
|
|
} |
|
|
}); |
|
|
|
|
|
|
|
|
setTimeout(() => { |
|
|
scrollHint.style.opacity = '1'; |
|
|
}, 1000); |
|
|
} |
|
|
}); |
|
|
</script> |
|
|
</body> |
|
|
|
|
|
</html> |
