Spaces:
Running
Running
| <html class="bg-white dark:bg-zinc-900" lang="en" data-theme="light"> <head><meta charset="UTF-8"><meta http-equiv="Content-Security-Policy" content="upgrade-insecure-requests; block-all-mixed-content"><meta name="description" content="PRISM - A multi-dimensional benchmark for evaluating LLM peer reviewers across depth, novelty, flaw detection, and constructiveness."><meta name="viewport" content="width=device-width"><meta name="generator" content="Astro v6.4.7"><link rel="canonical" href="https://prism-benchmark.github.io/"><link rel="icon" type="image/svg+xml" href="/favicon.svg"><link rel="describedby" type="text/html" href="https://arxiv.org/abs/2605.26730" title="PRISM paper on arXiv"><link rel="service-doc" type="text/html" href="/demo" title="PRISM interactive demo"><link rel="api-catalog" type="application/json" href="/.well-known/api-catalog" title="PRISM resource catalog for agents"><link rel="alternate" type="text/markdown" href="/index.md" title="PRISM paper — markdown version for agents"><meta property="og:title" content="PRISM: A Multi-Dimensional Benchmark for Evaluating LLM Peer Reviewers"><meta property="og:description" content="PRISM - A multi-dimensional benchmark for evaluating LLM peer reviewers across depth, novelty, flaw detection, and constructiveness."><meta property="og:type" content="website"><meta property="og:image" content="/screenshot-light.png"><style>@font-face{font-family:"Noto Sans-e26573b19e8d949d";src:url("/_astro/fonts/1d7aab50fda97bb3.woff2") format("woff2");font-display:swap;unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD;font-weight:100 900;font-style:italic;}@font-face{font-family:"Noto Sans-e26573b19e8d949d";src:url("/_astro/fonts/1e5097bbf9c9d577.woff2") format("woff2");font-display:swap;unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD;font-weight:100 900;font-style:normal;}@font-face{font-family:"Noto Sans-e26573b19e8d949d fallback: Arial";src:local("Arial");font-display:swap;font-weight:100 900;font-style:italic;size-adjust:100.269%;ascent-override:106.6132%;descent-override:29.2214%;line-gap-override:0%;}@font-face{font-family:"Noto Sans-e26573b19e8d949d fallback: Arial";src:local("Arial");font-display:swap;font-weight:100 900;font-style:normal;size-adjust:100.269%;ascent-override:106.6132%;descent-override:29.2214%;line-gap-override:0%;}:root{--font-noto-sans:"Noto Sans-e26573b19e8d949d","Noto Sans-e26573b19e8d949d fallback: Arial",sans-serif;}</style><link rel="preload" href="/_astro/fonts/1e5097bbf9c9d577.woff2" as="font" type="font/woff2" crossorigin><title>PRISM Benchmark: Evaluating LLM Peer Reviewers</title><script type="application/ld+json"> | |
| { | |
| "@context": "https://schema.org", | |
| "@type": "ScholarlyArticle", | |
| "name": "PRISM: A Multi-Dimensional Benchmark for Evaluating LLM Peer Reviewers", | |
| "headline": "PRISM: A Multi-Dimensional Benchmark for Evaluating LLM Peer Reviewers", | |
| "url": "https://prism-benchmark.github.io/", | |
| "description": "PRISM is a multi-dimensional benchmark for evaluating LLM-generated and human reviews across depth, novelty, flaw identification, and constructiveness.", | |
| "sameAs": "https://arxiv.org/abs/2605.26730", | |
| "author": [ | |
| { "@type": "Person", "name": "Ngoc Phan Phuoc Loc" }, | |
| { "@type": "Person", "name": "Toan Huynh La Viet" }, | |
| { "@type": "Person", "name": "Thanh Tran Khanh" }, | |
| { "@type": "Person", "name": "Duy A Nguyen" }, | |
| { "@type": "Person", "name": "Tuan Anh Nguyen Pham" }, | |
| { "@type": "Person", "name": "Thanh Nguyen" }, | |
| { "@type": "Person", "name": "Nitesh V. Chawla" }, | |
| { "@type": "Person", "name": "Wray Buntine" }, | |
| { "@type": "Person", "name": "Kok-Seng Wong" }, | |
| { "@type": "Person", "name": "Khoa D. Doan" }, | |
| { "@type": "Person", "name": "Binh T. Nguyen" } | |
| ], | |
| "publisher": { | |
| "@type": "Organization", | |
| "name": "VinUniversity", | |
| "url": "https://vinuni.edu.vn" | |
| }, | |
| "datePublished": "2026-05-26" | |
| } | |
| </script><!-- GoatCounter Analytics (self-hosted to avoid ad blockers) --><script data-goatcounter="https://prism-benchmark.goatcounter.com/count" async src="/count.js"></script><!-- Markdown for Agents: register SW that negotiates Accept: text/markdown | |
| to the pre-built /index.md companion with Content-Type text/markdown | |
| and x-markdown-tokens. Browsers without SW support still see HTML. --><script> | |
| if ("serviceWorker" in navigator) { | |
| navigator.serviceWorker.register("/sw.js").catch(() => {}); | |
| } | |
| </script><link rel="stylesheet" href="/_astro/index.DDIg3bR4.css"><link rel="stylesheet" href="/_astro/global.CiXSV0zR.css"></head> <body class="prose prose-zinc dark:prose-invert prose-a:text-blue-500 prose-a:dark:text-blue-300 prose-a:no-underline prose-a:hover:underline prose-a:font-normal prose-code:bg-zinc-200 prose-code:dark:bg-zinc-800 prose-code:text-zinc-800 prose-code:dark:text-zinc-200 prose-code:px-1.5 prose-code:py-0.5 prose-code:rounded prose-code:font-medium prose-code:before:content-none prose-code:after:content-none max-w-none pt-6 pb-4 font-(family-name:--font-noto-sans)"> <header class="prism-hero mx-auto mb-8 w-[75%] max-w-none px-6 text-center"> <div class="prism-hero-shell"> <div class="prism-hero-lockup" aria-label="PRISM"> <img src="/_astro/prism-logo.DAMhA8P8_Z1EuKwc.webp" alt="PRISM LLM Reviewer Benchmark Logo" loading="eager" decoding="sync" width="1338" height="563" class="prism-hero-logo"> <span class="prism-hero-brand">PRISM</span> </div> <h1 class="prism-hero-title"><span class="sr-only">PRISM: </span>A Multi-Dimensional Benchmark for Evaluating LLM Peer Reviewers</h1> <div class="flex flex-col items-center gap-6"> <div class="paper-authors"> <div class="paper-authors-list"> <span class="paper-author"> <span class="paper-author-name"> <a href="https://vinuni.edu.vn">Ngoc Phan Phuoc Loc</a> <sup class="paper-author-institution-indices"> 1 </sup> <sup class="paper-author-notes"> * </sup> </span> </span><span class="paper-author"> <span class="paper-author-name"> <a href="https://vinuni.edu.vn">Toan Huynh La Viet</a> <sup class="paper-author-institution-indices"> 1 </sup> <sup class="paper-author-notes"> * </sup> </span> </span><span class="paper-author"> <span class="paper-author-name"> <a href="https://vinuni.edu.vn">Thanh Tran Khanh</a> <sup class="paper-author-institution-indices"> 1 </sup> <sup class="paper-author-notes"> * </sup> </span> </span><span class="paper-author"> <span class="paper-author-name"> <a href="https://siebel.illinois.edu">Duy A Nguyen</a> <sup class="paper-author-institution-indices"> 1,2 </sup> </span> </span><span class="paper-author"> <span class="paper-author-name"> <a href="https://vinuni.edu.vn">Tuan Anh Nguyen Pham</a> <sup class="paper-author-institution-indices"> 1 </sup> </span> </span><span class="paper-author"> <span class="paper-author-name"> <a href="https://vinuni.edu.vn">Thanh Nguyen</a> <sup class="paper-author-institution-indices"> 1 </sup> </span> </span><span class="paper-author"> <span class="paper-author-name"> <a href="https://www.cse.nd.edu/people/faculty/nitesh-v-chawla/">Nitesh V. Chawla</a> <sup class="paper-author-institution-indices"> 3 </sup> </span> </span><span class="paper-author"> <span class="paper-author-name"> <a href="https://research.monash.edu/en/persons/wray-buntine">Wray Buntine</a> <sup class="paper-author-institution-indices"> 1,4 </sup> </span> </span><span class="paper-author"> <span class="paper-author-name"> <a href="https://vinuni.edu.vn">Kok-Seng Wong</a> <sup class="paper-author-institution-indices"> 1 </sup> </span> </span><span class="paper-author"> <span class="paper-author-name"> <a href="https://vinuni.edu.vn">Khoa D. Doan</a> <sup class="paper-author-institution-indices"> 1 </sup> <sup class="paper-author-notes"> † </sup> </span> </span><span class="paper-author"> <span class="paper-author-name"> <a href="https://vinuni.edu.vn">Binh T. Nguyen</a> <sup class="paper-author-institution-indices"> 1 </sup> <sup class="paper-author-notes"> † </sup> </span> </span> </div> <div class="paper-institutions"> <span class="paper-institution"> <sup>1</sup> <span>VinUniversity</span> <span class="paper-institution-separator">;</span> </span><span class="paper-institution"> <sup>2</sup> <span>University of Illinois, Urbana-Champaign</span> <span class="paper-institution-separator">;</span> </span><span class="paper-institution"> <sup>3</sup> <span>University of Notre Dame</span> <span class="paper-institution-separator">;</span> </span><span class="paper-institution"> <sup>4</sup> <span>Monash University</span> </span> </div> <div class="paper-institution-logos" aria-label="Institution logos"> <img class="paper-institution-logo" src="/vinuni.svg" alt="VinUniversity logo" width="590" height="112" loading="eager" decoding="async"><img class="paper-institution-logo" src="/uiuc-logo.webp" alt="University of Illinois, Urbana-Champaign logo" width="474" height="148" loading="eager" decoding="async"><img class="paper-institution-logo" src="/nd-logo.webp" alt="University of Notre Dame logo" width="160" height="160" loading="eager" decoding="async"><img class="paper-institution-logo" src="/monash-logo.webp" alt="Monash University logo" width="147" height="43" loading="eager" decoding="async"> </div> <div class="paper-notes"> <div class="paper-note"> <sup>*</sup> <span>Co-first Authors.</span> </div><div class="paper-note"> <sup>†</sup> <span>Co-corresponding authors. Correspondence to: khoa.dd@vinuni.edu.vn and binh.nt2@vinuni.edu.vn</span> </div> </div> </div> <div class="not-prose flex flex-row flex-wrap justify-center gap-2"> <a href="#" class="flex flex-row items-center gap-2 rounded-full bg-zinc-800 px-5 py-2 text-lg text-white hover:bg-black hover:no-underline dark:bg-zinc-200 dark:text-zinc-900 dark:hover:bg-zinc-50"> <svg width="1em" height="1em" data-icon="ri:github-line"> <symbol id="ai:ri:github-line" viewBox="0 0 24 24"><path fill="currentColor" d="M5.884 18.653c-.3-.2-.558-.455-.86-.816a51 51 0 0 1-.466-.579c-.463-.575-.755-.841-1.056-.95a1 1 0 1 1 .675-1.882c.752.27 1.261.735 1.947 1.588c-.094-.117.34.427.433.539c.19.227.33.365.44.438c.204.137.588.196 1.15.14c.024-.382.094-.753.202-1.095c-2.968-.726-4.648-2.64-4.648-6.396c0-1.24.37-2.356 1.058-3.292c-.218-.894-.185-1.975.302-3.192a1 1 0 0 1 .63-.582c.081-.024.127-.035.208-.047c.803-.124 1.937.17 3.415 1.096a11.7 11.7 0 0 1 2.687-.308c.912 0 1.819.104 2.684.308c1.477-.933 2.614-1.227 3.422-1.096q.128.02.218.05a1 1 0 0 1 .616.58c.487 1.216.52 2.296.302 3.19c.691.936 1.058 2.045 1.058 3.293c0 3.757-1.674 5.665-4.642 6.392c.125.415.19.878.19 1.38c0 .665-.002 1.299-.007 2.01c0 .19-.002.394-.005.706a1 1 0 0 1-.018 1.958c-1.14.227-1.984-.532-1.984-1.525l.002-.447l.005-.705c.005-.707.008-1.337.008-1.997c0-.697-.184-1.152-.426-1.361c-.661-.57-.326-1.654.541-1.751c2.966-.333 4.336-1.482 4.336-4.66c0-.955-.312-1.744-.913-2.404A1 1 0 0 1 17.2 6.19c.166-.414.236-.957.095-1.614l-.01.003c-.491.139-1.11.44-1.858.949a1 1 0 0 1-.833.135a9.6 9.6 0 0 0-2.592-.349c-.89 0-1.772.118-2.592.35a1 1 0 0 1-.829-.134c-.753-.507-1.374-.807-1.87-.947c-.143.653-.072 1.194.093 1.607a1 1 0 0 1-.189 1.045c-.597.655-.913 1.458-.913 2.404c0 3.172 1.371 4.328 4.322 4.66c.865.097 1.202 1.177.545 1.748c-.193.168-.43.732-.43 1.364v3.15c0 .985-.834 1.725-1.96 1.528a1 1 0 0 1-.04-1.962v-.99c-.91.061-1.661-.088-2.254-.485"/></symbol><use href="#ai:ri:github-line"></use> </svg> <span>Code</span> </a><a href="https://arxiv.org/abs/2605.26730" class="flex flex-row items-center gap-2 rounded-full bg-zinc-800 px-5 py-2 text-lg text-white hover:bg-black hover:no-underline dark:bg-zinc-200 dark:text-zinc-900 dark:hover:bg-zinc-50"> <svg width="0.88em" height="1em" data-icon="academicons:arxiv"> <symbol id="ai:academicons:arxiv" viewBox="0 0 448 512"><path fill="currentColor" d="M62.258 8.006a22.22 22.22 0 0 0-20.929 13.448c-3.404 8.169-.96 13.898 6.506 24.59c10.935 16.09 122.178 149.673 122.178 149.673l-24.619 23.038c-20.74 20.735-21.632 48.566-2.34 67.852l28.663 27.3l-79.976 98.235c-6.21 6.614-10.053 18.221-6.585 26.552a22.7 22.7 0 0 0 21.21 14.06a20.23 20.23 0 0 0 15.249-7.536l95.122-88.437L363.33 496.39a27.14 27.14 0 0 0 18.418 7.61a25.3 25.3 0 0 0 7.335-1.108a27.66 27.66 0 0 0 18.4-18.99a25.6 25.6 0 0 0-6.481-23.69L272.219 305.195l23.062-21.443c17.198-15.504 17.29-42.455.197-58.076l-25.257-24.228L357.417 98.46l.115-.133l.103-.14c7.793-10.123 11.52-17.92 7.502-27.806a36.17 36.17 0 0 0-23.647-18.37a24 24 0 0 0-3.166-.212l-.006.018a28.52 28.52 0 0 0-18.252 8.123l-.203.166l-.19.173L218.6 151.925L79.261 18.253S70.995 8.213 62.258 8.006m276.06 51.214q1.115.004 2.22.148a29.3 29.3 0 0 1 17.719 13.81c2.246 5.523 1.554 10.01-6.506 20.484L264.861 196.3l-40.882-39.22l100.68-91.304a21.77 21.77 0 0 1 13.66-6.536zM175.077 201.127L395.19 464.872c4.32 5.408 7.02 10.818 5.18 16.914a20.25 20.25 0 0 1-13.463 14.037a17.6 17.6 0 0 1-5.17.784a19.8 19.8 0 0 1-13.293-5.56l-220.15-209.694c-17.317-17.316-14.698-40.33 2.158-57.186z"/></symbol><use href="#ai:academicons:arxiv"></use> </svg> <span>arXiv</span> </a><a href="/demo" class="flex flex-row items-center gap-2 rounded-full bg-zinc-800 px-5 py-2 text-lg text-white hover:bg-black hover:no-underline dark:bg-zinc-200 dark:text-zinc-900 dark:hover:bg-zinc-50"> <svg width="1em" height="1em" data-icon="ri:play-circle-line"> <symbol id="ai:ri:play-circle-line" viewBox="0 0 24 24"><path fill="currentColor" d="M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10s-4.477 10-10 10m0-2a8 8 0 1 0 0-16a8 8 0 0 0 0 16M10.622 8.415l4.879 3.252a.4.4 0 0 1 0 .666l-4.88 3.252a.4.4 0 0 1-.621-.332V8.747a.4.4 0 0 1 .622-.332"/></symbol><use href="#ai:ri:play-circle-line"></use> </svg> <span>Demo</span> </a> </div> </div> </div> </header> <nav id="toc-sidebar" class="toc" aria-label="Page sections"> <span class="toc-brand">On this page</span> <ul id="toc-list" class="toc-list"></ul> </nav> <main> <div class="full-bleed bg-zinc-100 dark:bg-zinc-800/50 py-5"> <div class="mx-auto max-w-[65%] px-6 [&>:first-child]:mt-0 [&>:last-child]:mb-0"> <h2 id="abstract">Abstract</h2><p>Scientific peer review is under mounting strain as major machine learning venues face rapidly growing submission volumes, heavier reviewer workloads, and increasingly difficult paper-to-reviewer matching. At the same time, Large Language Models (LLMs) have moved from proofreading aids to automated reviewer agents capable of drafting full scientific critiques. This raises a central question: are LLMs sufficient reviewers for evaluating scientific work, especially when human reviewers themselves operate under severe time pressure?</p><p>We introduce <strong>PRISM</strong> (<strong>P</strong>eer <strong>R</strong>eview <strong>I</strong>ntelligence via <strong>S</strong>tructured <strong>M</strong>ulti-dimensional assessment), a benchmark for evaluating both LLM-generated and human reviews across four core duties: depth of analysis, novelty assessment, flaw identification and prioritization, and multi-dimensional constructiveness. Each duty is measured through a dedicated pipeline grounded in argument mining, retrieval-augmented verification, and consensus-based scoring. Across 1,000 papers from ICLR, ICML, and NeurIPS, PRISM shows that LLM reviewers can be strong task-matched specialists, but no single system matches the balanced performance of human reviewers. LLM reviewers are therefore best used as deliberate, human-assisted supplements rather than general-purpose replacements.</p> </div> </div> | |
| <figure> <div class="flex w-full justify-center **:my-0"> <object type="image/svg+xml" data="/prism-animation.svg" style="width:100%;max-width:1080px;margin:0 auto;display:block;" aria-label="PRISM animation showing the data source, evaluation pipelines, and output profile">Your browser does not support SVG</object> </div> <figcaption class="text-center"> <strong>PRISM animated overview.</strong> The framework processes 1,000 papers from five venue-years through four evaluation pipelines, producing a multi-dimensional review quality profile. </figcaption> </figure> | |
| <h2 id="insights">Insights</h2> | |
| <ol> | |
| <li>No single LLM reviewer is best at everything. Strong systems specialize in depth, novelty, flaw scanning, or constructiveness.</li> | |
| <li>LLMs are strong specialists, not full replacements for human reviewers. They excel at exhaustive scanning and systematic verification.</li> | |
| <li>The best workflow is human-led and LLM-assisted. Humans remain the most balanced and calibrated judges.</li> | |
| </ol> | |
| <h2 id="introducing-prism">Introducing PRISM</h2> | |
| <p>Major ML venues now receive tens of thousands of submissions, which makes reviewer assignment, workload, and review quality increasingly difficult to manage. LLM reviewers offer scale, but common evaluation methods often rely on surface similarity metrics or broad LLM-as-a-judge scores. These approaches can blur together fluency, factuality, and scientific rigor.</p> | |
| <p>PRISM asks a stricter question:</p> | |
| <blockquote> | |
| <p>Does a review provide grounded analysis, calibrated novelty judgment, valid flaw detection, and actionable feedback?</p> | |
| </blockquote> | |
| <p>To answer it, PRISM evaluates each manuscript-review pair through <strong>four independent and interpretable pipelines</strong>. Each pipeline extracts small review units, verifies them against the manuscript or prior literature, and computes metrics from those structured decisions instead of relying on a single opaque judge rating.</p> | |
| <figure> <div class="flex w-full justify-center **:my-0"> <picture> <source srcset="/_astro/PRISM_overview.pdf_Z1UCcav.avif 640w, /_astro/PRISM_overview.pdf_Z1aC8J1.avif 750w, /_astro/PRISM_overview.pdf_Z2v2Pdj.avif 828w, /_astro/PRISM_overview.pdf_2eWwOj.avif 1080w, /_astro/PRISM_overview.pdf_ehJQB.avif 1280w, /_astro/PRISM_overview.pdf_wjMGD.avif 1668w, /_astro/PRISM_overview.pdf_ZQS3qY.avif 2048w, /_astro/PRISM_overview.pdf_1EMsXd.avif 2560w, /_astro/PRISM_overview.pdf_Z2fOMvo.avif 2880w" type="image/avif"> <img src="/_astro/PRISM_overview.pdf_LYARL.png" srcset="/_astro/PRISM_overview.pdf_1hus4V.png 640w, /_astro/PRISM_overview.pdf_22uvvq.png 750w, /_astro/PRISM_overview.pdf_H4O28.png 828w, /_astro/PRISM_overview.pdf_cR6il.png 1080w, /_astro/PRISM_overview.pdf_Z1MMFEm.png 1280w, /_astro/PRISM_overview.pdf_Z1uKCOk.png 1668w, /_astro/PRISM_overview.pdf_2aVkWb.png 2048w, /_astro/PRISM_overview.pdf_Zmzgry.png 2560w, /_astro/PRISM_overview.pdf_LYARL.png 2880w" alt="PRISM Evaluation Pipeline: An LLM Reviewer Benchmark Overview" loading="lazy" decoding="async" sizes="(min-width: 2880px) 2880px, 100vw" data-astro-image="constrained" data-astro-image-fit="contain" data-astro-image-pos="center" width="2880" height="1619" class="rounded-lg max-h-[calc(100svh-3rem)] max-w-full w-min mx-auto "> </picture> </div> <figcaption class="text-center"> <strong>PRISM overview.</strong> Each review is decomposed into evidence units, novelty claims, flaw arguments, and atomic comments, then scored by modular evaluator pipelines. </figcaption> </figure> | |
| <h2 id="what-prism-measures">What PRISM Measures</h2> | |
| <table><thead><tr><th>Dimension</th><th>What it checks</th><th>Metric output</th></tr></thead><tbody><tr><td><strong>Depth of Analysis</strong></td><td>Whether reviews are detailed and grounded in manuscript or literature evidence</td><td>Premise Ratio, Grounding Score, DoA</td></tr><tr><td><strong>Novelty Assessment</strong></td><td>Whether novelty claims are supported by retrieved prior work</td><td>Novelty Score, Support Rate, Strict Support Rate</td></tr><tr><td><strong>Flaw Identification</strong></td><td>Whether reviews identify and prioritize critical vs. minor scientific issues</td><td>Critical Recall, Minor Recall, nCPS</td></tr><tr><td><strong>Constructiveness</strong></td><td>Whether feedback is actionable, specific, justified, solution-oriented, and professional</td><td>Mean Constructiveness Score</td></tr></tbody></table> | |
| <p>PRISM uses constrained LLM judging for extraction and labeling. The final scores are computed analytically from structured labels, retrieval evidence, and consensus verification — not from a single opaque judge rating.</p> | |
| <h2 id="methods">Methods</h2> | |
| <div class="carousel-wrapper" data-autoplay="true" data-interval-ms="4500" data-pause-on-hover="true" data-astro-cid-wfe7xcno> <div class="not-prose my-4 flex items-center justify-center" data-astro-cid-wfe7xcno> <button class="navigation-button prev-button h-[3rem] w-[3rem] cursor-pointer content-center disabled:cursor-not-allowed disabled:opacity-50 active:[&>svg]:scale-none!" type="button" data-astro-cid-wfe7xcno><svg width="1em" height="1em" class="mx-auto h-[2rem] w-[2rem] dark:text-zinc-200" data-astro-cid-wfe7xcno="true" data-icon="ri:arrow-drop-left-line"> <symbol id="ai:ri:arrow-drop-left-line" viewBox="0 0 24 24"><path fill="currentColor" d="m11.828 12l2.829 2.829l-1.414 1.414L9 12.001l4.243-4.243l1.414 1.414z"/></symbol><use href="#ai:ri:arrow-drop-left-line"></use> </svg></button> <div class="flex" data-astro-cid-wfe7xcno> <button class="page-button h-[3rem] w-[3rem] cursor-pointer content-center active:[&>div]:scale-none!" type="button" data-astro-cid-wfe7xcno> <div class="mx-auto h-[0.5rem] w-[0.5rem] rounded-full bg-black transition dark:bg-zinc-200" data-astro-cid-wfe7xcno></div> </button><button class="page-button h-[3rem] w-[3rem] cursor-pointer content-center active:[&>div]:scale-none!" type="button" data-astro-cid-wfe7xcno> <div class="mx-auto h-[0.5rem] w-[0.5rem] rounded-full bg-black transition dark:bg-zinc-200" data-astro-cid-wfe7xcno></div> </button><button class="page-button h-[3rem] w-[3rem] cursor-pointer content-center active:[&>div]:scale-none!" type="button" data-astro-cid-wfe7xcno> <div class="mx-auto h-[0.5rem] w-[0.5rem] rounded-full bg-black transition dark:bg-zinc-200" data-astro-cid-wfe7xcno></div> </button><button class="page-button h-[3rem] w-[3rem] cursor-pointer content-center active:[&>div]:scale-none!" type="button" data-astro-cid-wfe7xcno> <div class="mx-auto h-[0.5rem] w-[0.5rem] rounded-full bg-black transition dark:bg-zinc-200" data-astro-cid-wfe7xcno></div> </button> </div> <button class="navigation-button next-button h-[3rem] w-[3rem] cursor-pointer content-center disabled:cursor-not-allowed disabled:opacity-50 active:[&>svg]:scale-none!" type="button" data-astro-cid-wfe7xcno><svg width="1em" height="1em" class="mx-auto h-[2rem] w-[2rem] transition dark:text-zinc-200" data-astro-cid-wfe7xcno="true" data-icon="ri:arrow-drop-right-line"> <symbol id="ai:ri:arrow-drop-right-line" viewBox="0 0 24 24"><path fill="currentColor" d="M12.172 12L9.343 9.173l1.415-1.414L15 12l-4.242 4.242l-1.415-1.414z"/></symbol><use href="#ai:ri:arrow-drop-right-line"></use> </svg></button> </div> <div class="carousel-viewport w-[var(--actual-text-width)] overflow-hidden" data-astro-cid-wfe7xcno> <div class="carousel flex snap-x snap-mandatory overflow-x-auto scroll-smooth [scrollbar-width:none]" data-astro-cid-wfe7xcno> <div data-index="0" class="slide-viewport w-full shrink-0 snap-start"> <div class="slide w-full **:my-0"> <div class="method-slide method-slide-depth"><h3 id="method-1-depth-of-analysis-doa">Method 1: Depth of Analysis (DoA)</h3><p>A strong review does more than state opinions. It supports its judgments with evidence that can be traced to the manuscript or to the surrounding literature. PRISM splits each review into <strong>Argumentative Discourse Units (ADUs)</strong>, labels each unit as a claim or premise, and checks whether the premises are properly grounded.</p><p>The pipeline focuses on core review sections such as Summary, Strengths, and Weaknesses. Each ADU receives an argumentative role, an aspect label such as novelty, methodology, experiments, or clarity, and a grounding level:</p><p><table><thead><tr><th>Grounding Level</th><th>Meaning</th></tr></thead><tbody><tr><td><strong>0: Vague or generic</strong></td><td>The premise is not tied to specific evidence.</td></tr><tr><td><strong>1: Manuscript-grounded</strong></td><td>The premise refers to concrete paper content.</td></tr><tr><td><strong>2: Literature-grounded</strong></td><td>The premise uses external scientific context.</td></tr></tbody></table></p><p><strong>Main score:</strong> DoA combines evidence coverage (<strong>Premise Ratio</strong>) and evidence quality (<strong>Grounding Score</strong>) with a harmonic mean. A review must therefore be both detailed and well supported.</p></div> </div> </div><div data-index="1" class="slide-viewport w-full shrink-0 snap-start"> <div class="slide w-full **:my-0"> <div class="method-slide method-slide-novelty"><h3 id="method-2-novelty-assessment">Method 2: Novelty Assessment</h3><p>Novelty judgments are most useful when they are anchored in prior work rather than broad impressions. PRISM extracts verbatim novelty claims from the review, retrieves related papers from Semantic Scholar, and verifies whether the retrieved literature supports or contradicts each claim.</p><p>The pipeline has three stages: extract the paper’s core task, contribution anchors, and novelty claims; retrieve and diversify relevant prior work; then verify each claim-evidence pair as supporting, contradicting, or insufficiently evidenced.</p><p><strong>Main score:</strong> the pipeline reports a normalized novelty-support score together with support rates. These measures indicate whether the reviewer’s novelty statements are grounded in retrievable evidence. A high score means the claim is evidence grounded; it does not necessarily mean the reviewer reached the same judgment as a human expert.</p></div> </div> </div><div data-index="2" class="slide-viewport w-full shrink-0 snap-start"> <div class="slide w-full **:my-0"> <div class="method-slide method-slide-flaws"><h3 id="method-3-flaw-identification--major-issue-prioritization">Method 3: Flaw Identification & Major-Issue Prioritization</h3><p>Good reviewers need to identify real scientific flaws and give appropriate priority to the most serious ones. PRISM extracts flaw arguments from human and LLM reviews, verifies them against the paper, merges valid flaws into a consensus reference set, and labels them as <strong>Critical</strong> or <strong>Minor</strong>.</p><p>Because the true complete set of flaws in a manuscript is unobservable, PRISM builds a relative consensus ground truth from all human and LLM critiques. Candidate flaws are extracted, invalid or hallucinated critiques are removed, semantically equivalent flaws are merged, and valid issues are mapped back to their positions in the review.</p><p><strong>Main score:</strong> severity-stratified recall measures how many flaws a reviewer finds. The <strong>normalized Critique Prioritization Score (nCPS)</strong> measures whether critical flaws appear before minor issues.</p></div> </div> </div><div data-index="3" class="slide-viewport w-full shrink-0 snap-start"> <div class="slide w-full **:my-0"> <div class="method-slide method-slide-constructiveness"><h3 id="method-4-multi-dimensional-constructiveness">Method 4: Multi-dimensional Constructiveness</h3><p>A useful review should help authors improve the paper. PRISM breaks the review into <strong>Atomic Review Comments (ARCs)</strong> and scores each comment on five dimensions: actionability, specificity, justification, solution, and tone.</p><p>Each ARC is rated on a 0-2 scale for these five properties. The scoring is deliberately performed at the comment level. A review can be technically perceptive while still being unhelpful if it identifies a problem without explaining what the authors can do next.</p><p><strong>Main score:</strong> the <strong>Mean Constructiveness Score (MCS)</strong> averages these comment-level scores to quantify whether feedback is specific, justified, actionable, solution-oriented, and professional.</p><p><table><thead><tr><th>Constructiveness Dimension</th><th>What It Checks</th></tr></thead><tbody><tr><td><strong>Actionability</strong></td><td>Does the comment give implementable guidance?</td></tr><tr><td><strong>Specificity</strong></td><td>Does it point to concrete sections, equations, datasets, or claims?</td></tr><tr><td><strong>Justification</strong></td><td>Is the critique supported by reasoning or evidence?</td></tr><tr><td><strong>Solution</strong></td><td>Does it suggest a path to improve the paper?</td></tr><tr><td><strong>Tone</strong></td><td>Is the language professional and constructive?</td></tr></tbody></table></p></div> </div> </div> </div> </div> </div> <script type="module">const N=document.querySelectorAll("div.carousel-wrapper");N.forEach(e=>{const y=e.querySelector("div.carousel-viewport"),u=e.querySelector("div.carousel"),s=e.querySelectorAll("div.slide"),a=e.querySelectorAll("div.slide-viewport"),d=e.querySelector("button.prev-button"),f=e.querySelector("button.next-button"),E=e.querySelectorAll("button.page-button");if(!y||!u||!d||!f||s.length===0)return;let n=0,v,i=!1;const L=e.dataset.autoplay==="true",S=e.dataset.pauseOnHover!=="false",b=Number(e.dataset.intervalMs??"5000");function h(){d.disabled=n===0,f.disabled=n===s.length-1,E.forEach((t,o)=>{t.classList.toggle("active",o===n)})}function r(t,o=!1){const l=o?(t+s.length)%s.length:t;l<0||l>=s.length||(n=l,u.scrollTo({left:a[l].offsetLeft,behavior:"smooth"}),h())}function q(){!L||s.length<=1||!Number.isFinite(b)||b<=0||window.matchMedia("(prefers-reduced-motion: reduce)").matches||(v=window.setInterval(()=>{i||r(n+1,!0)},b))}if(d.addEventListener("click",()=>{r(n-1)}),f.addEventListener("click",()=>{r(n+1)}),E.forEach((t,o)=>{t.addEventListener("click",()=>{r(o)})}),S&&(e.addEventListener("pointerenter",()=>{i=!0}),e.addEventListener("pointerleave",()=>{i=!1}),e.addEventListener("focusin",()=>{i=!0}),e.addEventListener("focusout",()=>{i=!1})),h(),q(),window.addEventListener("beforeunload",()=>{v&&window.clearInterval(v)}),a.length===0)return;const x=new IntersectionObserver(t=>{const o=t.filter(c=>c.isIntersecting);if(o.length===0)return;const I=o.reduce((c,m)=>m.intersectionRatio>c.intersectionRatio?m:c).target,g=Number(I.dataset.index??"-1");Number.isNaN(g)||g===n||(n=g,h())},{root:u,threshold:.5});a.forEach(t=>x.observe(t))});</script> | |
| <h2 id="results-and-analysis">Results and Analysis</h2> | |
| <p>Across <strong>1,000 papers</strong> from ICLR, ICML, and NeurIPS, PRISM shows that LLM reviewers tend to specialize in different review responsibilities. No single system dominates all four dimensions.</p> | |
| <h3 id="headline-results">Headline Results</h3> | |
| <p><table><thead><tr><th>Evaluation Dimension</th><th>Best Automated System</th><th>Human Baseline</th></tr></thead><tbody><tr><td><strong>Depth of Analysis</strong></td><td>CycleReviewer: <strong>0.484</strong>; DeepReview: <strong>0.483</strong></td><td><strong>0.494</strong></td></tr><tr><td><strong>Novelty Assessment</strong></td><td>SEA: <strong>0.833</strong></td><td><strong>0.787</strong></td></tr><tr><td><strong>Critical Flaw Recall</strong></td><td>Reviewer2: <strong>0.591</strong></td><td><strong>0.343</strong></td></tr><tr><td><strong>Minor Flaw Recall</strong></td><td>Reviewer2: <strong>0.459</strong></td><td><strong>0.281</strong></td></tr><tr><td><strong>Prioritization</strong></td><td>SEA: <strong>0.977</strong></td><td><strong>0.973</strong></td></tr><tr><td><strong>Constructiveness</strong></td><td>DeepReview: <strong>0.634</strong></td><td><strong>0.566</strong></td></tr></tbody></table></p> | |
| <figure> <div class="flex w-full justify-center **:my-0"> <div class="spider-chart" role="img" aria-label="Spider chart comparing PRISM evaluation scores across human and automated reviewers" data-astro-cid-xsocqg6x> <svg class="spider-svg" viewBox="0 0 580 580" aria-hidden="true" data-astro-cid-xsocqg6x> <g class="spider-grid" data-astro-cid-xsocqg6x> <polygon points="290.0,253.0 322.0,271.5 322.0,308.5 290.0,327.0 258.0,308.5 258.0,271.5" data-astro-cid-xsocqg6x></polygon><polygon points="290.0,216.0 354.1,253.0 354.1,327.0 290.0,364.0 225.9,327.0 225.9,253.0" data-astro-cid-xsocqg6x></polygon><polygon points="290.0,179.0 386.1,234.5 386.1,345.5 290.0,401.0 193.9,345.5 193.9,234.5" data-astro-cid-xsocqg6x></polygon><polygon points="290.0,142.0 418.2,216.0 418.2,364.0 290.0,438.0 161.8,364.0 161.8,216.0" data-astro-cid-xsocqg6x></polygon><polygon points="290.0,105.0 450.2,197.5 450.2,382.5 290.0,475.0 129.8,382.5 129.8,197.5" data-astro-cid-xsocqg6x></polygon> <line x1="290" y1="290" x2="290" y2="105" data-astro-cid-xsocqg6x></line><line x1="290" y1="290" x2="450.2146997001212" y2="197.5" data-astro-cid-xsocqg6x></line><line x1="290" y1="290" x2="450.2146997001212" y2="382.5" data-astro-cid-xsocqg6x></line><line x1="290" y1="290" x2="290" y2="475" data-astro-cid-xsocqg6x></line><line x1="290" y1="290" x2="129.78530029987888" y2="382.50000000000006" data-astro-cid-xsocqg6x></line><line x1="290" y1="290" x2="129.78530029987886" y2="197.5" data-astro-cid-xsocqg6x></line> </g> <g class="spider-scale" data-astro-cid-xsocqg6x> <text x="298" y="257" data-astro-cid-xsocqg6x> 0.2 </text><text x="298" y="220" data-astro-cid-xsocqg6x> 0.4 </text><text x="298" y="183" data-astro-cid-xsocqg6x> 0.6 </text><text x="298" y="146" data-astro-cid-xsocqg6x> 0.8 </text><text x="298" y="109" data-astro-cid-xsocqg6x> 1.0 </text> </g> <g class="spider-labels" data-astro-cid-xsocqg6x> <text x="290" y="68" text-anchor="middle" data-astro-cid-xsocqg6x> <tspan x="290" dy="-0.55em" data-astro-cid-xsocqg6x> Depth of </tspan><tspan x="290" dy="1.15em" data-astro-cid-xsocqg6x> Analysis </tspan> </text><text x="482.2576396401454" y="179" text-anchor="start" data-astro-cid-xsocqg6x> <tspan x="482.2576396401454" dy="-0.55em" data-astro-cid-xsocqg6x> Novelty </tspan><tspan x="482.2576396401454" dy="1.15em" data-astro-cid-xsocqg6x> Assessment </tspan> </text><text x="482.2576396401454" y="400.99999999999994" text-anchor="start" data-astro-cid-xsocqg6x> <tspan x="482.2576396401454" dy="-0.55em" data-astro-cid-xsocqg6x> Critical Flaw </tspan><tspan x="482.2576396401454" dy="1.15em" data-astro-cid-xsocqg6x> Recall </tspan> </text><text x="290" y="512" text-anchor="middle" data-astro-cid-xsocqg6x> <tspan x="290" dy="-0.55em" data-astro-cid-xsocqg6x> Minor Flaw </tspan><tspan x="290" dy="1.15em" data-astro-cid-xsocqg6x> Recall </tspan> </text><text x="97.74236035985464" y="401.00000000000006" text-anchor="end" data-astro-cid-xsocqg6x> Prioritization </text><text x="97.74236035985464" y="178.99999999999997" text-anchor="end" data-astro-cid-xsocqg6x> Constructiveness </text> </g> <g class="spider-series" data-astro-cid-xsocqg6x> <g data-astro-cid-xsocqg6x> <polygon points="290.0,198.6 416.1,217.2 345.0,321.7 290.0,342.0 134.1,380.0 199.3,237.6" fill="#2563eb" stroke="#2563eb" data-astro-cid-xsocqg6x></polygon> <circle class="spider-dot" cx="290" cy="198.61" r="5" fill="#2563eb" data-system="Human" data-metric="Depth of Analysis" data-value="0.494" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="416.08896866399533" cy="217.2025" r="5" fill="#2563eb" data-system="Human" data-metric="Novelty Assessment" data-value="0.787" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="344.9536419971416" cy="321.72749999999996" r="5" fill="#2563eb" data-system="Human" data-metric="Critical Flaw Recall" data-value="0.343" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="290" cy="341.985" r="5" fill="#2563eb" data-system="Human" data-metric="Minor Flaw Recall" data-value="0.281" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="134.11109719178216" cy="380.00250000000005" r="5" fill="#2563eb" data-system="Human" data-metric="Prioritization" data-value="0.973" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="199.31847996973144" cy="237.64499999999998" r="5" fill="#2563eb" data-system="Human" data-metric="Constructiveness" data-value="0.566" data-astro-cid-xsocqg6x></circle> </g><g data-astro-cid-xsocqg6x> <polygon points="290.0,200.5 415.6,217.5 328.5,312.2 290.0,324.4 134.4,379.8 205.6,241.3" fill="#7c3aed" stroke="#7c3aed" data-astro-cid-xsocqg6x></polygon> <circle class="spider-dot" cx="290" cy="200.46" r="5" fill="#7c3aed" data-system="CycleReviewer" data-metric="Depth of Analysis" data-value="0.484" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="415.608324564895" cy="217.48000000000002" r="5" fill="#7c3aed" data-system="CycleReviewer" data-metric="Novelty Assessment" data-value="0.784" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="328.45152792802907" cy="312.2" r="5" fill="#7c3aed" data-system="CycleReviewer" data-metric="Critical Flaw Recall" data-value="0.240" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="290" cy="324.40999999999997" r="5" fill="#7c3aed" data-system="CycleReviewer" data-metric="Minor Flaw Recall" data-value="0.186" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="134.4315265911824" cy="379.81750000000005" r="5" fill="#7c3aed" data-system="CycleReviewer" data-metric="Prioritization" data-value="0.971" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="205.56685325803616" cy="241.2525" r="5" fill="#7c3aed" data-system="CycleReviewer" data-metric="Constructiveness" data-value="0.527" data-astro-cid-xsocqg6x></circle> </g><g data-astro-cid-xsocqg6x> <polygon points="290.0,200.6 411.6,219.8 343.2,320.7 290.0,332.2 135.1,379.4 188.4,231.4" fill="#059669" stroke="#059669" data-astro-cid-xsocqg6x></polygon> <circle class="spider-dot" cx="290" cy="200.64499999999998" r="5" fill="#059669" data-system="DeepReview" data-metric="Depth of Analysis" data-value="0.483" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="411.602957072392" cy="219.79250000000002" r="5" fill="#059669" data-system="DeepReview" data-metric="Novelty Assessment" data-value="0.759" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="343.1912803004402" cy="320.71" r="5" fill="#059669" data-system="DeepReview" data-metric="Critical Flaw Recall" data-value="0.332" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="290" cy="332.18" r="5" fill="#059669" data-system="DeepReview" data-metric="Minor Flaw Recall" data-value="0.228" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="135.07238538998288" cy="379.44750000000005" r="5" fill="#059669" data-system="DeepReview" data-metric="Prioritization" data-value="0.967" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="188.4238803901232" cy="231.355" r="5" fill="#059669" data-system="DeepReview" data-metric="Constructiveness" data-value="0.634" data-astro-cid-xsocqg6x></circle> </g><g data-astro-cid-xsocqg6x> <polygon points="290.0,220.3 416.1,217.2 384.7,344.7 290.0,374.9 133.8,380.2 197.9,236.8" fill="#dc2626" stroke="#dc2626" data-astro-cid-xsocqg6x></polygon> <circle class="spider-dot" cx="290" cy="220.255" r="5" fill="#dc2626" data-system="Reviewer2" data-metric="Depth of Analysis" data-value="0.377" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="416.08896866399533" cy="217.2025" r="5" fill="#dc2626" data-system="Reviewer2" data-metric="Novelty Assessment" data-value="0.787" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="384.6868875227716" cy="344.66749999999996" r="5" fill="#dc2626" data-system="Reviewer2" data-metric="Critical Flaw Recall" data-value="0.591" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="290" cy="374.915" r="5" fill="#dc2626" data-system="Reviewer2" data-metric="Minor Flaw Recall" data-value="0.459" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="133.7906677923819" cy="380.18750000000006" r="5" fill="#dc2626" data-system="Reviewer2" data-metric="Prioritization" data-value="0.975" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="197.87654767243038" cy="236.8125" r="5" fill="#dc2626" data-system="Reviewer2" data-metric="Constructiveness" data-value="0.575" data-astro-cid-xsocqg6x></circle> </g><g data-astro-cid-xsocqg6x> <polygon points="290.0,218.0 423.5,212.9 325.6,310.5 290.0,335.7 133.5,380.4 210.2,243.9" fill="#d97706" stroke="#d97706" data-astro-cid-xsocqg6x></polygon> <circle class="spider-dot" cx="290" cy="218.035" r="5" fill="#d97706" data-system="SEA" data-metric="Depth of Analysis" data-value="0.389" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="423.4588448502009" cy="212.9475" r="5" fill="#d97706" data-system="SEA" data-metric="Novelty Assessment" data-value="0.833" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="325.5676633334269" cy="310.53499999999997" r="5" fill="#d97706" data-system="SEA" data-metric="Critical Flaw Recall" data-value="0.222" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="290" cy="335.695" r="5" fill="#d97706" data-system="SEA" data-metric="Minor Flaw Recall" data-value="0.247" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="133.47023839298166" cy="380.37250000000006" r="5" fill="#d97706" data-system="SEA" data-metric="Prioritization" data-value="0.977" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="210.21307954933968" cy="243.935" r="5" fill="#d97706" data-system="SEA" data-metric="Constructiveness" data-value="0.498" data-astro-cid-xsocqg6x></circle> </g><g data-astro-cid-xsocqg6x> <polygon points="290.0,223.6 419.9,215.0 333.6,315.2 290.0,351.4 134.3,379.9 212.3,245.1" fill="#0f766e" stroke="#0f766e" data-astro-cid-xsocqg6x></polygon> <circle class="spider-dot" cx="290" cy="223.585" r="5" fill="#0f766e" data-system="TreeReview" data-metric="Depth of Analysis" data-value="0.359" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="419.93412145679827" cy="214.98250000000002" r="5" fill="#0f766e" data-system="TreeReview" data-metric="Novelty Assessment" data-value="0.811" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="333.57839831843296" cy="315.15999999999997" r="5" fill="#0f766e" data-system="TreeReview" data-metric="Critical Flaw Recall" data-value="0.272" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="290" cy="351.42" r="5" fill="#0f766e" data-system="TreeReview" data-metric="Minor Flaw Recall" data-value="0.332" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="134.27131189148227" cy="379.9100000000001" r="5" fill="#0f766e" data-system="TreeReview" data-metric="Prioritization" data-value="0.972" data-astro-cid-xsocqg6x></circle><circle class="spider-dot" cx="212.29587064544125" cy="245.1375" r="5" fill="#0f766e" data-system="TreeReview" data-metric="Constructiveness" data-value="0.485" data-astro-cid-xsocqg6x></circle> </g> </g> </svg> <div id="spider-tooltip" class="spider-tooltip" aria-hidden="true" data-astro-cid-xsocqg6x></div> <div class="spider-legend" aria-hidden="true" data-astro-cid-xsocqg6x> <div class="legend-row" data-astro-cid-xsocqg6x> <span class="legend-item" data-astro-cid-xsocqg6x> <i style="--series-color: #2563eb" data-astro-cid-xsocqg6x></i> <span class="legend-label" data-astro-cid-xsocqg6x> <strong data-astro-cid-xsocqg6x>Human</strong> <small data-astro-cid-xsocqg6x>Most balanced baseline</small> </span> </span><span class="legend-item" data-astro-cid-xsocqg6x> <i style="--series-color: #7c3aed" data-astro-cid-xsocqg6x></i> <span class="legend-label" data-astro-cid-xsocqg6x> <strong data-astro-cid-xsocqg6x>CycleReviewer</strong> <small data-astro-cid-xsocqg6x>Best depth (0.484)</small> </span> </span><span class="legend-item" data-astro-cid-xsocqg6x> <i style="--series-color: #059669" data-astro-cid-xsocqg6x></i> <span class="legend-label" data-astro-cid-xsocqg6x> <strong data-astro-cid-xsocqg6x>DeepReview</strong> <small data-astro-cid-xsocqg6x>Most constructive (0.634)</small> </span> </span> </div><div class="legend-row" data-astro-cid-xsocqg6x> <span class="legend-item" data-astro-cid-xsocqg6x> <i style="--series-color: #dc2626" data-astro-cid-xsocqg6x></i> <span class="legend-label" data-astro-cid-xsocqg6x> <strong data-astro-cid-xsocqg6x>Reviewer2</strong> <small data-astro-cid-xsocqg6x>Best flaw recall (0.591)</small> </span> </span><span class="legend-item" data-astro-cid-xsocqg6x> <i style="--series-color: #d97706" data-astro-cid-xsocqg6x></i> <span class="legend-label" data-astro-cid-xsocqg6x> <strong data-astro-cid-xsocqg6x>SEA</strong> <small data-astro-cid-xsocqg6x>Best novelty (0.833)</small> </span> </span><span class="legend-item" data-astro-cid-xsocqg6x> <i style="--series-color: #0f766e" data-astro-cid-xsocqg6x></i> <span class="legend-label" data-astro-cid-xsocqg6x> <strong data-astro-cid-xsocqg6x>TreeReview</strong> <small data-astro-cid-xsocqg6x>Best novelty recall (0.811)</small> </span> </span> </div> </div> </div> <script type="module">const e=document.getElementById("spider-tooltip"),r=document.querySelectorAll(".spider-dot");function a(t,s){const i=t.getAttribute("data-system"),n=t.getAttribute("data-metric"),o=t.getAttribute("data-value");e.innerHTML=`<div class="tt-system">${i}</div><div class="tt-metric">${n}: ${o}</div>`,e.classList.add("visible"),l(s)}function l(t){const s=t.clientX+14,i=t.clientY+10,{innerWidth:n,innerHeight:o}=window,c=e.offsetWidth,d=e.offsetHeight;e.style.left=(s+c>n?t.clientX-c-14:s)+"px",e.style.top=(i+d>o?t.clientY-d-10:i)+"px"}function v(){e.classList.remove("visible")}for(const t of r)t.addEventListener("mouseenter",s=>a(t,s)),t.addEventListener("mousemove",l),t.addEventListener("mouseleave",v);</script> </div> <figcaption class="text-center"> Spider chart of macro-averaged headline metrics, showing that automated reviewers have distinct strengths while humans remain the most balanced baseline. </figcaption> </figure> | |
| <h3 id="key-findings">Key Findings</h3> | |
| <ul> | |
| <li><strong>Humans remain the most balanced baseline:</strong> Human DoA = 0.494, leading through premise density and methodology focus.</li> | |
| <li><strong>DeepReview and CycleReviewer nearly match human depth:</strong> DoA = 0.483 and 0.484. Structured reasoning helps LLMs produce substantiated critiques.</li> | |
| <li><strong>SEA has the strongest novelty grounding:</strong> Novelty score = 0.833 vs. human = 0.787. Retrieval-oriented pipelines verify literature claims.</li> | |
| <li><strong>Reviewer2 is the best flaw scanner:</strong> Critical Recall = 0.591 vs. human = 0.343. LLMs surface missed issues for human reviewers.</li> | |
| <li><strong>DeepReview gives the most constructive feedback:</strong> MCS = 0.634 vs. human = 0.566. It closes the loop from “this is a problem” to “here is how to fix it.”</li> | |
| </ul> | |
| <p>Human and LLM reviewers are <strong>complementary</strong>. Humans excel at balanced judgment and calibration; LLMs excel at exhaustive scanning and systematic verification.</p> | |
| <h2 id="how-to-use-the-prism-benchmark">How to Use the PRISM Benchmark</h2> | |
| <p>Since no single system dominates all four dimensions, the evidence points toward targeted specialist use within a human-led pipeline.</p> | |
| <p><table><thead><tr><th>Need</th><th>Use</th></tr></thead><tbody><tr><td>Find more critical flaws</td><td>Reviewer2</td></tr><tr><td>Draft constructive feedback</td><td>DeepReview</td></tr><tr><td>Check novelty against literature</td><td>SEA</td></tr><tr><td>Make the final decision</td><td>Human reviewers</td></tr></tbody></table></p> | |
| <p>These systems are most effective as <strong>specialist assistants within a human-led pipeline</strong>, not as autonomous reviewers.</p> | |
| <h3 id="strengths-and-weaknesses-by-system">Strengths and weaknesses by system</h3> | |
| <p><table><thead><tr><th>System</th><th>Strength</th><th>Weakness</th></tr></thead><tbody><tr><td><strong>Reviewer2</strong></td><td>Exhaustive flaw scanning (highest recall)</td><td>Limited solution provision</td></tr><tr><td><strong>DeepReview</strong></td><td>Constructive feedback (actionable, professional)</td><td>Slightly lower flaw recall</td></tr><tr><td><strong>SEA</strong></td><td>Novelty verification (highest literature support)</td><td>Lower constructiveness</td></tr><tr><td><strong>CycleReviewer</strong></td><td>Strong analytical depth</td><td>High hallucination rate</td></tr><tr><td><strong>TreeReview</strong></td><td>Limited comparative advantage</td><td>Surface-level trap (24% effort on formatting)</td></tr></tbody></table></p> | |
| <h2 id="related-systems">Related Systems</h2> | |
| <p>PRISM belongs to public benchmarks and reviewer-assistance projects that emphasize inspectable evaluation. Related systems include <a href="https://arxiv.org/abs/2402.10886">Reviewer2</a>, <a href="https://aclanthology.org/2024.findings-emnlp.595/">SEA</a>, <a href="https://arxiv.org/abs/2503.08569">DeepReview</a>, <a href="https://aclanthology.org/2025.emnlp-main.790/">TreeReview</a>, and <a href="https://arxiv.org/abs/2411.00816">CycleReviewer</a>.</p> | |
| <h2 id="bibtex">BibTeX</h2> | |
| <div class="expressive-code"><link rel="stylesheet" href="/_astro/ec.0wf8d.css"/><script type="module" src="/_astro/ec.0vx5m.js"></script><figure class="frame"><figcaption class="header"></figcaption><pre data-language="bibtex"><code><div class="ec-line"><div class="code"><span style="--0:#F97583;--1:#BF3441">@article</span><span style="--0:#E1E4E8;--1:#24292E">{</span><span style="--0:#B392F0;--1:#6F42C1">prism2026</span><span style="--0:#E1E4E8;--1:#24292E">,</span></div></div><div class="ec-line"><div class="code"><span class="indent"> </span><span style="--0:#79B8FF;--1:#005CC5">title</span><span style="--0:#E1E4E8;--1:#24292E">=</span><span style="--0:#9ECBFF;--1:#032F62">{</span><span style="--0:#E1E4E8;--1:#24292E">PRISM: A Multi-Dimensional Benchmark for Evaluating LLM Peer Reviewers</span><span style="--0:#9ECBFF;--1:#032F62">}</span><span style="--0:#E1E4E8;--1:#24292E">,</span></div></div><div class="ec-line"><div class="code"><span class="indent"> </span><span style="--0:#79B8FF;--1:#005CC5">author</span><span style="--0:#E1E4E8;--1:#24292E">=</span><span style="--0:#9ECBFF;--1:#032F62">{</span><span style="--0:#E1E4E8;--1:#24292E">Ngoc Phan, Toan Huynh, Tran Khanh Thanh, Duy A. Nguyen, Nguyen Pham Tuan Anh, Thanh Nguyen, Nitesh V. Chawla, Wray Buntine, Kok-Seng Wong, Khoa D Doan, Binh Nguyen</span><span style="--0:#9ECBFF;--1:#032F62">}</span><span style="--0:#E1E4E8;--1:#24292E">,</span></div></div><div class="ec-line"><div class="code"><span class="indent"> </span><span style="--0:#79B8FF;--1:#005CC5">journal</span><span style="--0:#E1E4E8;--1:#24292E">=</span><span style="--0:#9ECBFF;--1:#032F62">{</span><span style="--0:#E1E4E8;--1:#24292E">arXiv preprint</span><span style="--0:#9ECBFF;--1:#032F62">}</span><span style="--0:#E1E4E8;--1:#24292E">,</span></div></div><div class="ec-line"><div class="code"><span class="indent"> </span><span style="--0:#79B8FF;--1:#005CC5">year</span><span style="--0:#E1E4E8;--1:#24292E">=</span><span style="--0:#9ECBFF;--1:#032F62">{</span><span style="--0:#E1E4E8;--1:#24292E">2026</span><span style="--0:#9ECBFF;--1:#032F62">}</span></div></div><div class="ec-line"><div class="code"><span style="--0:#E1E4E8;--1:#24292E">}</span></div></div></code></pre><div class="copy"><div aria-live="polite"></div><button title="Copy to clipboard" data-copied="Copied!" data-code="@article{prism2026, title={PRISM: A Multi-Dimensional Benchmark for Evaluating LLM Peer Reviewers}, author={Ngoc Phan, Toan Huynh, Tran Khanh Thanh, Duy A. Nguyen, Nguyen Pham Tuan Anh, Thanh Nguyen, Nitesh V. Chawla, Wray Buntine, Kok-Seng Wong, Khoa D Doan, Binh Nguyen}, journal={arXiv preprint}, year={2026}}"><div></div></button></div></figure></div> </main> <footer class="mx-auto max-w-[75%] px-6 text-center"> <p class="prose-sm text-center"> | |
| Built with <a href="https://research-template.roman.technology" rel="nofollow">Roman Hauksson-Neill's project page template</a> </p> </footer> <button id="back-to-top" aria-label="Back to top" class="fixed right-6 bottom-6 z-50 hidden h-12 w-12 items-center justify-center rounded-full bg-zinc-800 text-white shadow-lg transition-opacity duration-300 hover:bg-zinc-700 dark:bg-zinc-200 dark:text-zinc-900 dark:hover:bg-zinc-300"> <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"> <line x1="12" y1="19" x2="12" y2="5"></line> <polyline points="5 12 12 5 19 12"></polyline> </svg> </button> <script type="module">const m=document.getElementById("toc-list"),w=document.querySelector("main"),r=[],y=i=>i.toLowerCase().trim().replace(/[^\w\s-]/g,"").replace(/\s+/g,"-");if(m&&w){const i=Array.from(w.querySelectorAll("h2, h3"));let t=null,s=null,o=null;i.forEach(e=>{e.id||(e.id=y(e.textContent||""));const n=e.closest(".slide-viewport"),l=n?n.closest(".carousel-wrapper"):null,d=n?Number(n.dataset.index??"-1"):-1,u=!!l&&Number.isFinite(d)&&d>=0,p=document.createElement("a");p.href=`#${e.id}`,p.textContent=e.textContent||"",p.className="toc-link",p.addEventListener("click",h=>{if(h.preventDefault(),u&&l){l.scrollIntoView({behavior:"smooth",block:"start"});const g=l.querySelectorAll("button.page-button")[d];g&&g.click()}else{const f=document.getElementById(e.id);f&&f.scrollIntoView({behavior:"smooth",block:"start"})}history.replaceState(null,"",`#${e.id}`)});const c=document.createElement("li");c.className="toc-item",c.dataset.target=e.id,c.appendChild(p),e.tagName==="H2"?(m.appendChild(c),s=c,o=e.id,t=null,r.push({id:e.id,el:e,li:c})):s&&o?(t||(t=document.createElement("ul"),t.className="toc-sub",s.classList.add("has-sub"),s.appendChild(t)),t.appendChild(c),r.push({id:e.id,el:e,li:c,parentId:o,carousel:u?l:void 0,slideIndex:u?d:void 0})):(m.appendChild(c),r.push({id:e.id,el:e,li:c,carousel:u?l:void 0,slideIndex:u?d:void 0}))})}function C(i){document.querySelectorAll(".toc-item.active").forEach(s=>s.classList.remove("active"));const t=r.find(s=>s.id===i);if(t&&(t.li.classList.add("active"),t.parentId)){const s=r.find(o=>o.id===t.parentId);s&&s.li.classList.add("active")}}const E=r.filter(i=>i.carousel===void 0);let v=0;function I(){if(!r.length)return;const i=window.innerHeight*.3;let t=null,s=-1/0;for(const n of E){const l=n.el.getBoundingClientRect().top;l<=i&&l>s&&(t=n,s=l)}t||(t=E[0]??r[0]);let o=t.id;const e=r.find(n=>n.parentId===t.id&&n.carousel!==void 0&&n.carousel.querySelectorAll("button.page-button")[n.slideIndex??-1]?.classList.contains("active"));e&&(o=e.id),C(o)}function b(){v||(v=requestAnimationFrame(()=>{v=0,I()}))}window.addEventListener("scroll",b,{passive:!0});window.addEventListener("resize",b,{passive:!0});I();const L=new Set;r.forEach(i=>{i.carousel&&L.add(i.carousel)});L.forEach(i=>{const t=i.querySelectorAll("button.page-button"),s=new MutationObserver(()=>b());t.forEach(o=>s.observe(o,{attributes:!0,attributeFilter:["class"]}))});const a=document.getElementById("back-to-top");if(a){const i=()=>{window.scrollY>300?(a.classList.remove("hidden"),a.classList.add("flex")):(a.classList.add("hidden"),a.classList.remove("flex"))};window.addEventListener("scroll",i,{passive:!0}),i(),a.addEventListener("click",()=>{window.scrollTo({top:0,behavior:"smooth"})})}</script> </body> </html> |