Rebuild from 42a0074

assets/css/content.css

@@ -331,3 +331,16 @@
 	.content .pyramid-figure { grid-template-columns: 1fr; }
 	.content .pyramid-figure figure { position: static; order: -1; max-width: 320px; }
 }
+
+/* Dataset v1: a v0 crop beside the page image(s) that replace it. */
+.content .pairing {
+	display: grid;
+	grid-template-columns: repeat(auto-fit, minmax(160px, 220px));
+	gap: 16px;
+	align-items: end;
+	margin-bottom: 8px;
+}
+.content .pairing figure { margin: 0; }
+.content .pairing img { display: block; width: 100%; height: auto; }
+.content .pairing figcaption { font-size: 11.5px; text-align: left; }
+.content p.caption-text { font-size: 12.5px; color: var(--text-muted); margin-bottom: 8px; }

assets/js/nav.js

@@ -1,15 +1,25 @@
 // Tree utilities: flatten, lookup, sidebar rendering, breadcrumbs, pager, child cards.
 
+/**
+ * A node marked `"hidden": true` is still routable (a direct link opens it)
+ * but is left out of the sidebar, the child cards, the step numbering, the
+ * pager and search, and so are its descendants.
+ */
+export function visibleChildren(node) {
+	return (node.children || []).filter((child) => !child.hidden);
+}
+
 /**
  * Flatten the nav tree into a depth-first ordered list of nodes, each
- * annotated with its route path ("/", "/part-2", "/part-2/pool-creation")
- * and its parent chain (for breadcrumbs).
+ * annotated with its route path ("/", "/part-2", "/part-2/pool-creation"),
+ * its parent chain (for breadcrumbs) and whether it is hidden.
  */
 export function flattenTree(root) {
 	const flat = [];
 
 	function visit(node, path, ancestors) {
-		const entry = { node, path, ancestors };
+		const hidden = Boolean(node.hidden) || ancestors.some((a) => a.hidden);
+		const entry = { node, path, ancestors, hidden };
 		flat.push(entry);
 		const children = node.children || [];
 		children.forEach((child) => {
@@ -43,7 +53,7 @@ export function buildIndex(root) {
 function annotateSteps(node, label = null, color = null) {
 	let n = 0;
 	(node.children || []).forEach((child) => {
-		if (node.steps && child.step !== false) {
+		if (node.steps && child.step !== false && !child.hidden) {
 			n += 1;
 			child._step = label ? `${label}.${n}` : `${n}`;
 			child._color = color ?? n;
@@ -97,7 +107,7 @@ export function renderSidebar(container, root, index, activePath, onNavigate) {
 
 		// A part with sub-pages is reached from its label, so its long title
 		// row is dropped and its children sit directly under the label.
-		const children = part.children || [];
+		const children = visibleChildren(part);
 		const label = document.createElement(children.length ? "a" : "span");
 		label.className = "nav-group-label";
 		label.textContent = part.label || part.title;
@@ -138,7 +148,8 @@ function renderNode(node, path, index, activePath, onNavigate, isTop) {
 	link.href = `#${path}`;
 	link.dataset.path = path;
 
-	const hasChildren = node.children && node.children.length > 0;
+	const children = visibleChildren(node);
+	const hasChildren = children.length > 0;
 	const expanded = isAncestorOfActive(path, activePath);
 
 	if (hasChildren) {
@@ -170,7 +181,7 @@ function renderNode(node, path, index, activePath, onNavigate, isTop) {
 	if (hasChildren) {
 		const childrenEl = document.createElement("div");
 		childrenEl.className = "nav-children" + (expanded ? "" : " collapsed");
-		node.children.forEach((child) => {
+		children.forEach((child) => {
 			const childPath = `${path}/${child.slug}`;
 			childrenEl.appendChild(renderNode(child, childPath, index, activePath, onNavigate, false));
 		});
@@ -271,12 +282,13 @@ export function renderPageHead(container, entry, root) {
 export function renderChildren(container, entry) {
 	container.innerHTML = "";
 	const node = entry.node;
-	if (!node.children || node.children.length === 0) return;
+	const children = visibleChildren(node);
+	if (children.length === 0) return;
 
 	if (node.steps) {
 		const stepper = document.createElement("div");
 		stepper.className = "stepper";
-		node.children.forEach((child) => {
+		children.forEach((child) => {
 			const path = `${entry.path}/${child.slug}`;
 			const n = child._step;
 			const step = document.createElement("div");
@@ -297,7 +309,7 @@ export function renderChildren(container, entry) {
 
 	const grid = document.createElement("div");
 	grid.className = "card-grid";
-	node.children.forEach((child) => {
+	children.forEach((child) => {
 		const path = `${entry.path}/${child.slug}`;
 		const card = document.createElement("a");
 		card.className = "card";
@@ -314,8 +326,9 @@ export function renderChildren(container, entry) {
 
 /* ---------------- Pager (prev/next) ---------------- */
 
-export function renderPager(container, flat, activePath) {
+export function renderPager(container, allEntries, activePath) {
 	container.innerHTML = "";
+	const flat = allEntries.filter((e) => !e.hidden);
 	const idx = flat.findIndex((e) => e.path === activePath);
 	if (idx < 0) return;
 

assets/js/search.js

@@ -16,7 +16,7 @@ async function buildIndex(flat) {
 
 	const docs = await Promise.all(
 		flat
-			.filter((e) => e.node.file)
+			.filter((e) => e.node.file && !e.hidden)
 			.map(async (e, id) => {
 				let text = "";
 				try {

content/part-2/evaluation-framework/analysis/description-retrieval.html

@@ -0,0 +1,102 @@
+<p>
+	Can a local vision-language model's <em>description</em> of an image stand in for the image
+	in retrieval? Each of the four local models describes every image of a pool once, from the
+	image alone, and the queries are then matched against the descriptions, never the images
+	(condition B of <code>docs/retrieval-test-audit.md</code>). The reference row searches the
+	images directly with the strongest text-image encoder of the benchmark (condition A).
+</p>
+
+<h2>Setup</h2>
+<ul>
+	<li><strong>Queries:</strong> the 50 queries of the pyramid sample, one per paper.</li>
+	<li><strong>Pool:</strong> every image those queries are judged against: 732 images,
+	85 paper figures and 647 RKD distractors. Each query is ranked against
+	the whole pool; only its own targets count as hits.</li>
+	<li><strong>Describers:</strong> Qwen3.6, Qwen3.8, Gemma 4 and Mistral 3.2, served locally with
+	vLLM, temperature 0, thinking off (Mistral has none). One paragraph of 80&ndash;150 words per
+	image, with no caption, title or article. The prompt forbids naming an artist, collection,
+	date or specific work unless it is legible in the image.</li>
+	<li><strong>Matching:</strong> BM25 (k1 1.2, b 0.75) over the words of
+	the description, and a dense text embedder (<code>Qwen/Qwen3-Embedding-0.6B@97b0c61</code>, cosine). Nothing tuned.</li>
+	<li><strong>Statistics:</strong> mean over queries, 95% interval from 2,000 resamples of queries.
+	Bold marks the best description method.</li>
+</ul>
+
+<h2>Results</h2>
+<h3>Full pool (732 images)</h3>
+<div class="table-wrap"><table><thead><tr><th>Method</th><th class="num">MRR</th><th class="num">R@1</th><th class="num">R@5</th><th class="num">R@10</th><th class="num">nDCG@10</th><th class="num">Paper fig. in top 10</th></tr></thead><tbody><tr><td><strong>Image embedding</strong> (PE-Core bigG, no describer)</td><td class="num">0.70 <small>[0.58, 0.80]</small></td><td class="num">0.37 <small>[0.26, 0.46]</small></td><td class="num">0.75 <small>[0.64, 0.86]</small></td><td class="num">0.82 <small>[0.72, 0.91]</small></td><td class="num">0.68 <small>[0.58, 0.77]</small></td><td class="num">0.40 <small>[0.34, 0.46]</small></td></tr><tr><td>Qwen3.6-35B-A3B + BM25</td><td class="num">0.33 <small>[0.22, 0.44]</small></td><td class="num"><strong>0.16 <small>[0.08, 0.25]</small></strong></td><td class="num">0.31 <small>[0.21, 0.42]</small></td><td class="num">0.35 <small>[0.24, 0.47]</small></td><td class="num">0.28 <small>[0.19, 0.38]</small></td><td class="num">0.18 <small>[0.14, 0.22]</small></td></tr><tr><td>Qwen3.6-35B-A3B + dense</td><td class="num">0.29 <small>[0.19, 0.40]</small></td><td class="num">0.15 <small>[0.07, 0.23]</small></td><td class="num">0.27 <small>[0.17, 0.37]</small></td><td class="num">0.29 <small>[0.19, 0.40]</small></td><td class="num">0.24 <small>[0.16, 0.34]</small></td><td class="num">0.12 <small>[0.10, 0.15]</small></td></tr><tr><td>Qwen3.8-27B + BM25</td><td class="num">0.27 <small>[0.16, 0.38]</small></td><td class="num">0.12 <small>[0.05, 0.20]</small></td><td class="num">0.25 <small>[0.15, 0.36]</small></td><td class="num">0.35 <small>[0.24, 0.46]</small></td><td class="num">0.25 <small>[0.16, 0.34]</small></td><td class="num">0.21 <small>[0.17, 0.25]</small></td></tr><tr><td>Qwen3.8-27B + dense</td><td class="num"><strong>0.33 <small>[0.22, 0.44]</small></strong></td><td class="num">0.15 <small>[0.07, 0.24]</small></td><td class="num"><strong>0.32 <small>[0.22, 0.43]</small></strong></td><td class="num">0.35 <small>[0.25, 0.48]</small></td><td class="num">0.28 <small>[0.20, 0.38]</small></td><td class="num">0.20 <small>[0.17, 0.24]</small></td></tr><tr><td>Gemma 4 26B-A4B + BM25</td><td class="num">0.25 <small>[0.16, 0.35]</small></td><td class="num">0.13 <small>[0.05, 0.22]</small></td><td class="num">0.28 <small>[0.17, 0.40]</small></td><td class="num">0.34 <small>[0.22, 0.47]</small></td><td class="num">0.25 <small>[0.16, 0.35]</small></td><td class="num">0.23 <small>[0.18, 0.28]</small></td></tr><tr><td>Gemma 4 26B-A4B + dense</td><td class="num">0.30 <small>[0.20, 0.41]</small></td><td class="num">0.16 <small>[0.08, 0.25]</small></td><td class="num">0.26 <small>[0.16, 0.38]</small></td><td class="num"><strong>0.39 <small>[0.27, 0.51]</small></strong></td><td class="num"><strong>0.29 <small>[0.19, 0.40]</small></strong></td><td class="num">0.17 <small>[0.14, 0.20]</small></td></tr><tr><td>Mistral Small 3.2 24B + BM25</td><td class="num">0.17 <small>[0.10, 0.26]</small></td><td class="num">0.06 <small>[0.01, 0.12]</small></td><td class="num">0.16 <small>[0.08, 0.26]</small></td><td class="num">0.21 <small>[0.12, 0.32]</small></td><td class="num">0.15 <small>[0.08, 0.24]</small></td><td class="num">0.15 <small>[0.12, 0.18]</small></td></tr><tr><td>Mistral Small 3.2 24B + dense</td><td class="num">0.21 <small>[0.13, 0.32]</small></td><td class="num">0.09 <small>[0.03, 0.17]</small></td><td class="num">0.21 <small>[0.12, 0.31]</small></td><td class="num">0.31 <small>[0.20, 0.43]</small></td><td class="num">0.20 <small>[0.12, 0.29]</small></td><td class="num">0.14 <small>[0.11, 0.17]</small></td></tr></tbody></table></div>
+<p><em>Paper fig. in top 10</em> is the share of paper figures among the first ten results; the
+pool is 12% paper figures.</p>
+
+<h3>Paper figures only (85 images)</h3>
+<p>The same scores, ranked among the paper figures alone. Every target is a journal reproduction
+and every RKD record is not, so looking like a reproduction could be a cue by itself; here it is none.</p>
+<div class="table-wrap"><table><thead><tr><th>Method</th><th class="num">MRR</th><th class="num">R@1</th><th class="num">R@5</th><th class="num">R@10</th><th class="num">nDCG@10</th></tr></thead><tbody><tr><td><strong>Image embedding</strong> (PE-Core bigG, no describer)</td><td class="num">0.83 <small>[0.73, 0.91]</small></td><td class="num">0.50 <small>[0.39, 0.59]</small></td><td class="num">0.86 <small>[0.77, 0.94]</small></td><td class="num">0.91 <small>[0.83, 0.97]</small></td><td class="num">0.81 <small>[0.73, 0.88]</small></td></tr><tr><td>Qwen3.6-35B-A3B + BM25</td><td class="num">0.52 <small>[0.40, 0.65]</small></td><td class="num"><strong>0.29 <small>[0.19, 0.39]</small></strong></td><td class="num">0.45 <small>[0.33, 0.58]</small></td><td class="num">0.54 <small>[0.41, 0.66]</small></td><td class="num">0.46 <small>[0.35, 0.58]</small></td></tr><tr><td>Qwen3.6-35B-A3B + dense</td><td class="num">0.46 <small>[0.35, 0.59]</small></td><td class="num">0.24 <small>[0.15, 0.34]</small></td><td class="num">0.46 <small>[0.34, 0.58]</small></td><td class="num">0.56 <small>[0.43, 0.68]</small></td><td class="num">0.44 <small>[0.33, 0.54]</small></td></tr><tr><td>Qwen3.8-27B + BM25</td><td class="num">0.47 <small>[0.35, 0.59]</small></td><td class="num">0.24 <small>[0.15, 0.35]</small></td><td class="num">0.43 <small>[0.31, 0.55]</small></td><td class="num">0.48 <small>[0.36, 0.60]</small></td><td class="num">0.42 <small>[0.30, 0.53]</small></td></tr><tr><td>Qwen3.8-27B + dense</td><td class="num"><strong>0.52 <small>[0.41, 0.64]</small></strong></td><td class="num">0.27 <small>[0.18, 0.37]</small></td><td class="num"><strong>0.50 <small>[0.38, 0.61]</small></strong></td><td class="num">0.63 <small>[0.50, 0.75]</small></td><td class="num">0.50 <small>[0.39, 0.61]</small></td></tr><tr><td>Gemma 4 26B-A4B + BM25</td><td class="num">0.43 <small>[0.33, 0.55]</small></td><td class="num">0.24 <small>[0.14, 0.34]</small></td><td class="num">0.45 <small>[0.34, 0.57]</small></td><td class="num">0.54 <small>[0.42, 0.65]</small></td><td class="num">0.42 <small>[0.32, 0.53]</small></td></tr><tr><td>Gemma 4 26B-A4B + dense</td><td class="num">0.50 <small>[0.39, 0.62]</small></td><td class="num">0.27 <small>[0.17, 0.38]</small></td><td class="num">0.49 <small>[0.36, 0.61]</small></td><td class="num"><strong>0.65 <small>[0.53, 0.76]</small></strong></td><td class="num"><strong>0.50 <small>[0.40, 0.61]</small></strong></td></tr><tr><td>Mistral Small 3.2 24B + BM25</td><td class="num">0.35 <small>[0.26, 0.46]</small></td><td class="num">0.14 <small>[0.07, 0.23]</small></td><td class="num">0.41 <small>[0.29, 0.53]</small></td><td class="num">0.52 <small>[0.39, 0.65]</small></td><td class="num">0.36 <small>[0.27, 0.46]</small></td></tr><tr><td>Mistral Small 3.2 24B + dense</td><td class="num">0.47 <small>[0.35, 0.58]</small></td><td class="num">0.25 <small>[0.15, 0.36]</small></td><td class="num">0.46 <small>[0.34, 0.58]</small></td><td class="num">0.59 <small>[0.48, 0.70]</small></td><td class="num">0.45 <small>[0.35, 0.56]</small></td></tr></tbody></table></div>
+
+<h3>Descriptions</h3>
+<div class="table-wrap"><table><thead><tr><th>Describer</th><th class="num">Images described</th>
+<th class="num">Mean words</th></tr></thead><tbody><tr><td>Qwen3.6-35B-A3B</td><td class="num">732</td><td class="num">130.0</td></tr><tr><td>Qwen3.8-27B</td><td class="num">732</td><td class="num">124.3</td></tr><tr><td>Gemma 4 26B-A4B</td><td class="num">732</td><td class="num">140.9</td></tr><tr><td>Mistral Small 3.2 24B</td><td class="num">732</td><td class="num">124.8</td></tr></tbody></table></div>
+
+<h2>Findings</h2>
+<ul>
+	<li><strong>Descriptions lose to the images by a wide margin.</strong> The best description
+	setup reaches an MRR of about 0.33 against 0.70 for the image embedding, and finds a target in
+	the top ten for about 35&ndash;39% of queries against 82%.</li>
+	<li><strong>The gap is not the reproduction cue.</strong> Among paper figures only, the image
+	embedding still reaches MRR 0.83 against about 0.52 for the best descriptions.</li>
+	<li><strong>Qwen3.6, Qwen3.8 and Gemma 4 are indistinguishable</strong>: their intervals overlap
+	on every metric. <strong>Mistral is the weakest describer</strong>, clearly so with BM25.</li>
+	<li><strong>BM25 and the dense embedder are close</strong>; neither wins for every describer.</li>
+	<li><strong>Why the descriptions lose:</strong> the queries name artists and works (&ldquo;by
+	Bartholomeus Breenbergh or Rembrandt&rdquo;, &ldquo;of the painting Girl with a Flute&rdquo;),
+	and the describer was told never to name what it merely recognises. The image encoder has no
+	such rule and recognises styles and famous works. The descriptions are often accurate but
+	cautious: in the first example below, Gemma describes a preacher on a rock above a crowd and
+	never says &ldquo;John the Baptist&rdquo;.</li>
+</ul>
+
+<h2>Examples</h2>
+<p>The three queries where the image embedding ranks a target first and Gemma's descriptions
+rank it lowest.</p>
+<h3>Find paintings, drawings, or prints depicting &#x27;The Preaching of John the Baptist&#x27; executed by Bartholomeus Breenbergh or Rembrandt van Rijn during the 1630s.</h3>
+<p>First relevant image at rank <strong>1</strong> by image embedding,
+rank <strong>456</strong> of 732 by Gemma's descriptions (dense).</p>
+<figure><img src="https://huggingface.co/datasets/gspin/phd-report-media/resolve/main/figures/338c15ff709b/f001.jpg" alt="Target figure f001 of paper 338c15ff709b" style="max-height:320px" loading="lazy">
+<figcaption>Target: figure f001 of paper <code>338c15ff709b</code>. Gemma 4's description:</figcaption></figure>
+<blockquote><p>A landscape oil painting depicting a large group of figures gathered in a rugged, mountainous setting. In the foreground and middle ground, a crowd of people in classical or historical attire is clustered on a dirt path. Some figures appear to be in conversation, while others are engaged in more dramatic gestures; notably, a figure on the right stands atop a rock with an arm raised. To the left, large, crumbling stone ruins of a building or fortification rise up a hillside, partially overgrown with vegetation. The background features rolling green hills under a vast, cloudy sky. The composition uses a diagonal arrangement of figures and landscape elements to create depth. The color palette is dominated by earthy tones of brown, ochre, and green, contrasted with the soft blues and greys of the sky. The lighting is diffused, suggesting an overcast day, casting soft shadows across the scene.</p></blockquote><h3>Find technical images, such as X-radiographs, infrared reflectograms, or pigment analysis scans, of the painting Girl with a Flute.</h3>
+<p>First relevant image at rank <strong>1</strong> by image embedding,
+rank <strong>318</strong> of 732 by Gemma's descriptions (dense).</p>
+<figure><img src="https://huggingface.co/datasets/gspin/phd-report-media/resolve/main/figures/db9b246cf843/f005.jpg" alt="Target figure f005 of paper db9b246cf843" style="max-height:320px" loading="lazy">
+<figcaption>Target: figure f005 of paper <code>db9b246cf843</code>. Gemma 4's description:</figcaption></figure>
+<blockquote><p>This is a detail from an oil painting, focusing on the lower portion of a figure&#x27;s garments and a supporting object. The composition is dominated by heavy, textured drapery in dark, muted tones of charcoal and deep blue, contrasted against a central vertical band of lighter, silvery-grey fabric. A white arrow in the upper left points toward the dark folds, while a white rectangular box highlights the junction where the light fabric meets a horizontal, brownish-ochre element, possibly a wooden support or a limb. The brushwork is thick and impasto, creating a palpable sense of texture and weight. The lighting is soft and diffused, emphasizing the sculptural quality of the folds. The color palette is somber and limited, consisting primarily of dark greys, blues, and earthy browns, suggesting a serious or solemn subject matter.</p></blockquote><h3>Find paintings by Hendrick ter Brugghen depicting a single figure in a pensive, melancholic, or weeping pose, such as a philosopher or a penitent saint.</h3>
+<p>First relevant image at rank <strong>1</strong> by image embedding,
+rank <strong>128</strong> of 732 by Gemma's descriptions (dense).</p>
+<figure><img src="https://huggingface.co/datasets/gspin/phd-report-media/resolve/main/figures/8831c350c534/f002.jpg" alt="Target figure f002 of paper 8831c350c534" style="max-height:320px" loading="lazy">
+<figcaption>Target: figure f002 of paper <code>8831c350c534</code>. Gemma 4's description:</figcaption></figure>
+<blockquote><p>A vertical oil painting reproduction of a single figure in a dark, interior setting. The subject is a young woman shown from the waist up, leaning forward in a contemplative or weary pose. Her head is resting on her right hand, with her fingers pressed against her forehead, while her gaze is directed downward toward a small, lit candle on a table in the foreground. She wears a light-colored, loose-fitting garment with voluminous sleeves. The composition is dominated by dramatic chiaroscuro, with a single light source—the candle flame—casting intense highlights on her face, hands, and clothing, while leaving the background and much of the lower foreground in deep, heavy shadow. The color palette is limited to warm ochres, creams, and deep browns, emphasizing the intimate and somber mood of the scene.</p></blockquote>
+
+<h2>Prompt</h2>
+<pre><code>You describe one image for a search index of art-historical images. You see the image only: no caption, no title, no article.
+
+The image may show a whole work, a detail, a drawing, a print, a page of a book, a building, an object, a technical photograph (X-ray, infrared, verso) or a diagram. Describe what this image shows, so that a researcher&#x27;s search request for it would match your words.
+
+Write one paragraph of 80 to 150 words covering, as far as the image gives them:
+
+- what kind of image it is: reproduction type, medium and technique, whole work or detail;
+- the subject and its iconography: the scene, story or type, the figures and their attributes, actions and gestures;
+- objects, setting and landscape;
+- composition, colour and light;
+- any text legible in the image.
+
+Name a subject, scene, saint or style when the image itself supports it. Never name an artist, a collection, a date or a specific work unless it is legible in the image: say what you see, not what you recognise.
+</code></pre>
+
+<h2>Notes</h2>
+<ul>
+	<li>One description per image and model, served by <code>make serve-vllm
+	VLLM_MAX_NUM_SEQS=8</code>: 13&ndash;20 minutes per model on the L40S.</li>
+	<li>Gemma and Mistral run from RedHat's FP8 builds (the bf16 weights do not fit in 48&nbsp;GB).</li>
+	<li>50 queries make wide intervals: differences of less than about 0.1 are not resolved.</li>
+	<li>Reproduce with <code>taxonomy/scripts/describe_retrieval.sh</code>; numbers in
+	<code>taxonomy/results/describe_retrieval.md</code>.</li>
+</ul>

content/part-2/evaluation-framework/analysis/local-models.html

@@ -0,0 +1,104 @@
+<p>
+	A one-image smoke test of the four open-weight vision-language models that are
+	candidates to replace Gemini in the analysis layer, run locally on the L40S
+	(vLLM 0.31.0). Each model gets the same image, the same one-line prompt and
+	the same strict JSON schema, at temperature 0. It checks that the model loads,
+	reads an image and returns schema-valid JSON through the repository's own
+	<code>vllm</code> client. It is not a quality comparison: that is the pilot on
+	15 papers (<a href="https://github.com/llm-art/art-historical-papers-extraction/blob/main/docs/model-pilot.md"><code>docs/model-pilot.md</code></a>).
+</p>
+
+<h2>Input</h2>
+<figure>
+	<img src="https://huggingface.co/datasets/gspin/phd-report-media/resolve/main/figures/68590ebc5313/f001.jpg" alt="Hendrick ter Brugghen, Crucifixion with the Virgin and Saint John" loading="lazy">
+	<figcaption>Hendrick ter Brugghen, <em>Crucifixion with the Virgin and Saint John</em>,
+	ca. 1625, New York, The Metropolitan Museum of Art (public domain). Figure f001 of
+	paper <code>68590ebc5313</code> (JHNA), as extracted: 329 &times; 376 px.</figcaption>
+</figure>
+<p>Prompt: <code>Describe this image as JSON.</code> Schema: <code>medium</code> (string),
+<code>subject</code> (string), <code>people_count</code> (integer or null),
+<code>notes</code> (list of strings). No system prompt, no caption, no article text.</p>
+
+<h2>Results</h2>
+<div class="table-wrap"><table>
+	<thead><tr><th>Model</th><th>Checkpoint</th><th>Weights</th><th>Thinking</th>
+	<th>Valid JSON</th><th class="num">Prompt tokens</th><th class="num">Output tokens</th>
+	<th class="num">Seconds</th></tr></thead>
+	<tbody>
+<tr><td>Qwen3.6-35B-A3B</td><td><a href="https://huggingface.co/Qwen/Qwen3.6-35B-A3B-FP8/tree/95a723d"><code>Qwen/Qwen3.6-35B-A3B-FP8</code></a> <code>95a723d</code></td><td>official FP8</td><td>off</td><td>yes</td><td class="num">230</td><td class="num">157</td><td class="num">31.53</td></tr>
+<tr><td>Qwen3.8-27B</td><td><a href="https://huggingface.co/Qwen/Qwen3.8-27B-FP8/tree/017b9c7"><code>Qwen/Qwen3.8-27B-FP8</code></a> <code>017b9c7</code></td><td>official FP8</td><td>off</td><td>yes</td><td class="num">230</td><td class="num">177</td><td class="num">12.48</td></tr>
+<tr><td>Gemma 4 26B-A4B</td><td><a href="https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic/tree/ed35d7a"><code>RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic</code></a> <code>ed35d7a</code></td><td>third-party FP8</td><td>off</td><td>yes</td><td class="num">1106</td><td class="num">100</td><td class="num">16.95</td></tr>
+<tr><td>Mistral Small 3.2 24B</td><td><a href="https://huggingface.co/RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-FP8/tree/9bc507c"><code>RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-FP8</code></a> <code>9bc507c</code></td><td>third-party FP8</td><td>n/a (not a reasoning model)</td><td>yes</td><td class="num">192</td><td class="num">166</td><td class="num">9.82</td></tr>
+	</tbody>
+</table></div>
+
+<div class="callout warning">
+	<p>Seconds are wall-clock and include waiting behind the pilot's own requests on the
+	same server (one request at a time), so they are not a speed measurement. Prompt
+	tokens are mostly the image: models size images differently (Qwen by a pixel range,
+	Gemma by a token budget, Mistral by a 1,540 px cap), so equal images do not cost
+	equal tokens.</p>
+</div>
+
+<h2>Answers</h2>
+<h3>Qwen3.6-35B-A3B</h3>
+<pre><code class="language-json">{
+  &quot;medium&quot;: &quot;oil painting&quot;,
+  &quot;subject&quot;: &quot;Crucifixion of Jesus Christ&quot;,
+  &quot;people_count&quot;: 3,
+  &quot;notes&quot;: [
+    &quot;Jesus is depicted on the cross, with a white loincloth and dark hair.&quot;,
+    &quot;To the left, a woman in a grey hooded robe kneels in grief, head bowed.&quot;,
+    &quot;To the right, a man in red and dark robes looks up at Jesus with an expression of sorrow or devotion.&quot;,
+    &quot;The background is dark and atmospheric, with subtle vertical lines suggesting rain or light rays.&quot;,
+    &quot;The composition is vertical, emphasizing the height of the cross and the emotional weight of the scene.&quot;
+  ]
+}</code></pre>
+<h3>Qwen3.8-27B</h3>
+<pre><code class="language-json">{
+  &quot;medium&quot;: &quot;painting&quot;,
+  &quot;subject&quot;: &quot;The Crucifixion of Jesus Christ&quot;,
+  &quot;people_count&quot;: 3,
+  &quot;notes&quot;: [
+    &quot;The central figure is Jesus on the cross, depicted with a gaunt, emaciated body and a bowed head.&quot;,
+    &quot;To the left, a figure in a white hooded robe is shown in profile, looking down with hands clasped in prayer or grief.&quot;,
+    &quot;To the right, a man in a red robe looks up at Jesus, holding his foot.&quot;,
+    &quot;The overall color palette is dark and somber, dominated by browns, blacks, and muted earth tones, with the red robe providing a stark contrast.&quot;,
+    &quot;The style is dramatic and expressive, characteristic of the Baroque period.&quot;
+  ]
+}</code></pre>
+<h3>Gemma 4 26B-A4B</h3>
+<pre><code class="language-json">{
+  &quot;medium&quot;: &quot;oil on canvas&quot;,
+  &quot;subject&quot;: &quot;The Crucifixion of Jesus Christ&quot;,
+  &quot;people_count&quot;: 3,
+  &quot;notes&quot;: [
+    &quot;The central figure is Jesus Christ on the cross.&quot;,
+    &quot;To the left is a woman, likely the Virgin Mary, draped in white and blue robes.&quot;,
+    &quot;To the right is a man, likely Saint John the Apostle, wearing a red cloak.&quot;,
+    &quot;The background is dark and dramatic, typical of Baroque art style.&quot;
+  ]
+}</code></pre>
+<h3>Mistral Small 3.2 24B</h3>
+<pre><code class="language-json">{
+  &quot;medium&quot;: &quot;Painting&quot;,
+  &quot;subject&quot;: &quot;Religious&quot;,
+  &quot;people_count&quot;: 3,
+  &quot;notes&quot;: [
+    &quot;This image depicts a religious scene, specifically the crucifixion of Jesus Christ.&quot;,
+    &quot;The central figure is Jesus, nailed to a cross with his arms outstretched.&quot;,
+    &quot;Two other figures are present at the base of the cross, one on each side. The figure on the left appears to be a woman, possibly the Virgin Mary, and the figure on the right is a man, possibly John the Apostle.&quot;,
+    &quot;The painting uses a dark, dramatic color palette, emphasizing the somber and tragic nature of the scene.&quot;,
+    &quot;The background is dark and cloudy, adding to the overall mood of the painting.&quot;
+  ]
+}</code></pre>
+
+<h2>Notes</h2>
+<ul>
+	<li>Gemma and Mistral run from RedHat's FP8 builds because the official bf16 weights do
+	not fit in 48&nbsp;GB, so their output includes quantisation error.</li>
+	<li>Mistral's tokenizer rejects the <code>enable_thinking</code> switch the Qwen and Gemma
+	models need (HTTP 400), so it is called without it; it has no thinking mode.</li>
+	<li>Each server is started with <code>make serve-vllm VLLM_PRESET=&lt;preset&gt;</code>,
+	pinned to the sha above.</li>
+</ul>

content/part-2/evaluation-framework/index.html

@@ -20,7 +20,7 @@
 	<tbody>
 		<tr>
 			<td><a href="#/part-2/evaluation-framework/extraction">Extraction</a></td>
-			<td>Journal PDFs</td>
+			<td>Journal PDFs, plus article pages where the journal has them</td>
 			<td>Full text (<code>text.md</code>), one image per figure, captions</td>
 			<td>None</td>
 		</tr>
@@ -35,10 +35,16 @@
 
 <h2>Extraction</h2>
 <p>
-	Extraction is deterministic and uses no LLM. Docling converts each PDF to Markdown, every
-	figure is rendered as a standalone PNG, and each image is paired with its caption by a
-	journal-specific profile. Figures that the layout model merged together are split back
-	apart.
+	Extraction is deterministic and uses no LLM. Docling converts each PDF to Markdown. Where
+	the journal also publishes the article as a web page, the figures are the image files the
+	journal uploaded, with their captions read from the same page; otherwise each figure is
+	rendered from the PDF and paired with its caption by a journal-specific profile.
+</p>
+<p>
+	Over the 145 JHNA articles of the corpus this yields 3,069 figure images, every one with a
+	caption, at a median short side of 758 px (0.82 MP). The PDF crops of the same figures
+	have a median short side of 289 px (0.11 MP).
+	<a href="#/part-2/evaluation-framework/extraction">Details</a>.
 </p>
 
 <h2>Analysis</h2>

content/part-2/framework/applied.html

@@ -1,6 +1,6 @@
 <p>
 	All 300 labelled benchmark queries, each classified whole under the four frameworks
-	of the <a href="#/part-2/theoretical-framework">Theoretical framework</a>: the words that ask
+	of the <a href="#/part-2/irs-testing/theoretical-framework">Theoretical framework</a>: the words that ask
 	for each kind of information, with the framework&rsquo;s class, and what the framework has
 	no place for. Labels are <code>gemini/gemini-3.6-flash</code>&rsquo;s, unreviewed
 	(<code>taxonomy/results/query_levels.json</code>).

content/part-2/pool/{dataset.html → v0.html}

@@ -1,4 +1,4 @@
-<div class="callout"><p>Generated from <a href="https://github.com/llm-art/art-historical-papers-extraction/blob/main/dataset/coverage.md"><code>dataset/coverage.md</code></a> by <code>space/build.py</code>. Do not edit on the Space: edit the repository and republish.</p></div>
+<div class="callout"><p>Generated from <a href="https://github.com/llm-art/art-historical-papers-extraction/blob/main/dataset/coverage-v0.md"><code>dataset/coverage-v0.md</code></a> by <code>space/build.py</code>. Do not edit on the Space: edit the repository and republish.</p></div>
 <p><strong>Pool coverage — <code>gemini-gemini-3-5-flash-adcf1483</code></strong></p>
 <p>Generated 2026-10-01T19:33:01.462481+00:00.</p>
 <div class="table-wrap"><table>

content/part-2/pool/v1.html

@@ -0,0 +1,236 @@
+<p>
+	Dataset v1 rebuilds <a href="#/part-2/pool-creation/v0">v0</a> on better gold images and on
+	the annotators' judgements. The queries, distractors and fillers are carried over. What
+	changes is which images count as gold, what they look like, and how relevant each one is.
+</p>
+
+<div class="callout warning">
+	<p>
+		Work in progress (29 October 2026). All 145 papers are now extracted from their article
+		pages, and the image figures below cover all of them. Stage C has not yet been rerun on
+		the new images, so v1's qrels do not exist yet.
+	</p>
+</div>
+
+<h2>What changes from v0</h2>
+<div class="table-wrap"><table>
+	<thead><tr><th></th><th>v0</th><th>v1</th></tr></thead>
+	<tbody>
+		<tr><td>Gold images</td><td>Cropped from the PDF</td><td>The journal's own files, from the article page</td></tr>
+		<tr><td>Caption–image pairing</td><td>By position on the page; about half of the checked crops were mispaired</td><td>From the page, where the image link and its caption are one element</td></tr>
+		<tr><td>Stage C links</td><td>Gemini 3.5 Flash, on the PDF crops</td><td>Gemini 3.6 Flash, on the page images, with the same questions and subqueries</td></tr>
+		<tr><td>Relevance</td><td>Binary: every stage C link is a target</td><td>Graded: 2, 1 or 0, from annotators where they rated the link</td></tr>
+		<tr><td>Missed figures</td><td>Not used</td><td>Added as targets, flagged <code>added_by_annotator</code></td></tr>
+		<tr><td>Subqueries</td><td>All kept</td><td>Rated No dropped, Tangential and Partly flagged, unrated tagged unvalidated</td></tr>
+	</tbody>
+</table></div>
+
+<h2>Gold images from the article page</h2>
+<p>
+	Every paper in the corpus is from JHNA, which publishes each article as a web page as
+	well as a PDF. The page links each figure's image file next to its caption. Extraction
+	now takes the figures from there (<code>artpapers extract --figures auto</code>), and still
+	takes the text from the PDF. <code>text.md</code> is byte-identical to v0's for every paper
+	(145 of 145), so stage 0 and stage A carry over unchanged.
+</p>
+<p>The PDF crops had two problems, and the page fixes both:</p>
+<ul>
+	<li>
+		<strong>Pairing.</strong> On JHNA's side-by-side layout, matching a caption to the
+		picture above it often picks the neighbouring picture. Annotators noticed: "The
+		captions underneath the pictures were matched incorrectly, which sometimes confused me."
+	</li>
+	<li>
+		<strong>Resolution.</strong> JHNA's PDFs embed plates at about 100 dpi, so a crop is
+		small whatever scale it is rendered at.
+	</li>
+</ul>
+
+<h3>Pairing cases</h3>
+<p>
+	Each v0 crop was compared with every page image of its paper: the aspect ratios must agree
+	within 10%, and 32 × 32 grayscale thumbnails must correlate at 0.80 or more
+	(<code>artpapers dataset web-figures</code> applies the same test). Over the 2,736 captioned
+	crops of all 145 papers, a crop falls into one of four cases:
+</p>
+<div class="table-wrap"><table>
+	<thead><tr><th>Case</th><th>Meaning</th><th class="num">Captioned crops</th></tr></thead>
+	<tbody>
+		<tr><td>Accepted</td><td>The crop shows the figure its caption names.</td><td class="num">1,150 (42%)</td></tr>
+		<tr><td>Renumbered</td><td>The crop shows another figure of the same article: the caption went to the wrong picture.</td><td class="num">1,114 (41%)</td></tr>
+		<tr><td>Unmatched</td><td>The crop resembles no figure on the page closely enough.</td><td class="num">349 (13%)</td></tr>
+		<tr><td>Ambiguous</td><td>The crop matches several page figures equally well.</td><td class="num">123 (4%)</td></tr>
+	</tbody>
+</table></div>
+<p>
+	139 of the 145 papers have at least one captioned crop that is not accepted, and 702
+	crops are halves of a swapped pair (fig. 4 shows fig. 5's picture and the other way
+	round). Restricted to the 1,435 gold crops: 575 accepted (40%), 545 renumbered (38%), 226
+	unmatched (16%), 89 ambiguous (6%). Of v0's 1,692 query–figure links, 718 point at an
+	accepted crop; 245 of the 372 queries have at least one link to a crop that is not, and
+	124 have none to an accepted one.
+</p>
+<p>One example of each, with the v0 crop beside the v1 page image that now carries the caption.</p>
+
+<h4>Accepted</h4>
+<p class="caption-text"><code>067aab78bf2a</code> · Fig. 2 Jan Luyken, The Destruction of Jan de Baen’s Allegory on Cornelis de Witt (on May 13 or June 29, 1672)…</p>
+<div class="pairing"><figure><img src="https://huggingface.co/datasets/gspin/phd-report-media/resolve/main/figures/067aab78bf2a/f002.jpg" alt="067aab78bf2a:f002" loading="lazy"><figcaption><strong>v0: PDF crop for f002</strong><br>450 × 322 px</figcaption></figure><figure><img src="https://huggingface.co/datasets/gspin/phd-report-media/resolve/main/figures-html/067aab78bf2a/f002.jpg" alt="067aab78bf2a:f002" loading="lazy"><figcaption><strong>v1: page image for f002</strong><br>800 × 573 px</figcaption></figure></div>
+<p>Same etching in both. The page image is 800 px wide against the crop's 450.</p>
+
+<h4>Renumbered</h4>
+<p class="caption-text"><code>067aab78bf2a</code> · Fig. 8 Anonymous, Medal Commemorating the Death of Cornelis and Johan de Witt, Amsterdam, Rijksmuseum, inv. no…</p>
+<div class="pairing"><figure><img src="https://huggingface.co/datasets/gspin/phd-report-media/resolve/main/figures/067aab78bf2a/f008.jpg" alt="067aab78bf2a:f008" loading="lazy"><figcaption><strong>v0: PDF crop for f008</strong><br>469 × 233 px</figcaption></figure><figure><img src="https://huggingface.co/datasets/gspin/phd-report-media/resolve/main/figures-html/067aab78bf2a/f008.jpg" alt="067aab78bf2a:f008" loading="lazy"><figcaption><strong>v1: page image for f008</strong><br>800 × 411 px</figcaption></figure><figure><img src="https://huggingface.co/datasets/gspin/phd-report-media/resolve/main/figures-html/067aab78bf2a/f009.jpg" alt="067aab78bf2a:f009" loading="lazy"><figcaption><strong>v1: page image for f009, what the crop shows</strong><br>800 × 396 px</figcaption></figure></div>
+<p>The caption is for the medal on the death of the De Witt brothers, but the v0 crop shows the medal in their praise, which is fig. 9 on the page. In v0, a query about the commemorative medal had the wrong medal as its gold image.</p>
+
+<h4>Unmatched</h4>
+<p class="caption-text"><code>067aab78bf2a</code> · Fig. 3 Attributed to Jan de Baen, The Bodies of the Brothers Jan and Cornelis de Witt, Hanging from the Strappado (“wip”…</p>
+<div class="pairing"><figure><img src="https://huggingface.co/datasets/gspin/phd-report-media/resolve/main/figures/067aab78bf2a/f003.jpg" alt="067aab78bf2a:f003" loading="lazy"><figcaption><strong>v0: PDF crop for f003</strong><br>318 × 397 px</figcaption></figure><figure><img src="https://huggingface.co/datasets/gspin/phd-report-media/resolve/main/figures-html/067aab78bf2a/f003.jpg" alt="067aab78bf2a:f003" loading="lazy"><figcaption><strong>v1: page image for f003</strong><br>638 × 800 px</figcaption></figure></div>
+<p>By eye the crop and the page image are the same painting. The similarity (0.76) fell below the threshold because the crop is dark and small. So "unmatched" is not always a mismatch, and v1 does not carry a judgement over through an unmatched crop.</p>
+
+<h4>Ambiguous</h4>
+<p class="caption-text"><code>045ad8e9b924</code> · Fig. 6 Workshop or circle of Jacob de Wet, The Meeting of Abigail and David, oil on panel, 72.9 x 138.3 cm, Pr…</p>
+<div class="pairing"><figure><img src="https://huggingface.co/datasets/gspin/phd-report-media/resolve/main/figures/045ad8e9b924/f006.jpg" alt="045ad8e9b924:f006" loading="lazy"><figcaption><strong>v0: PDF crop for f006</strong><br>425 × 236 px</figcaption></figure><figure><img src="https://huggingface.co/datasets/gspin/phd-report-media/resolve/main/figures-html/045ad8e9b924/f006.jpg" alt="045ad8e9b924:f006" loading="lazy"><figcaption><strong>v1: page image for f006</strong><br>1600 × 925 px</figcaption></figure><figure><img src="https://huggingface.co/datasets/gspin/phd-report-media/resolve/main/figures-html/045ad8e9b924/f003.jpg" alt="045ad8e9b924:f003" loading="lazy"><figcaption><strong>v1: page image for f003, the map</strong><br>1600 × 896 px</figcaption></figure></div>
+<p>The caption is for a painting, but the crop is the article's map. The map appears three times on the page (figs. 3, 9 and 13), so the crop matches all three equally and cannot be assigned to one.</p>
+
+<h3>What the annotators saw</h3>
+<p>
+	Annotators judged v0's crops under v0's captions, so on mispaired figures their ratings
+	split: some rated the caption, others the picture. Three rated links:
+</p>
+
+<h4>Rated by the caption</h4>
+<p class="caption-text"><code>644d247f107d</code> · Fig. 13 Joachim von Sandrart, Company of Captain Cornelis Bicker and Lieutenant Frederick van Banchem, 1640. R…</p>
+<div class="pairing"><figure><img src="https://huggingface.co/datasets/gspin/phd-report-media/resolve/main/figures/644d247f107d/f013.jpg" alt="644d247f107d:f013" loading="lazy"><figcaption><strong>v0: PDF crop for f013</strong><br>351 × 420 px</figcaption></figure><figure><img src="https://huggingface.co/datasets/gspin/phd-report-media/resolve/main/figures-html/644d247f107d/f013.jpg" alt="644d247f107d:f013" loading="lazy"><figcaption><strong>v1: page image for f013</strong><br>595 × 800 px</figcaption></figure></div>
+<p>Subquery: <em>What is the composition and visual arrangement of Joachim von Sandrart's portrait of the Company of Captain Cornelis Bicker…?</em> The crop is a single portrait, not the militia company. One annotator rated it Fully, the other Partly: "The caption is right, but the image is wrong."</p>
+
+<h4>Rated by the picture</h4>
+<p class="caption-text"><code>f27edffd6057</code> · Fig. 9 Jean-François Millet, Young Mother Preparing the Evening Meal, 1848–50, pen and brown ink on paper, 20.…</p>
+<div class="pairing"><figure><img src="https://huggingface.co/datasets/gspin/phd-report-media/resolve/main/figures/f27edffd6057/f009.jpg" alt="f27edffd6057:f009" loading="lazy"><figcaption><strong>v0: PDF crop for f009</strong><br>326 × 358 px</figcaption></figure><figure><img src="https://huggingface.co/datasets/gspin/phd-report-media/resolve/main/figures-html/f27edffd6057/f009.jpg" alt="f27edffd6057:f009" loading="lazy"><figcaption><strong>v1: page image for f009</strong><br>650 × 800 px</figcaption></figure><figure><img src="https://huggingface.co/datasets/gspin/phd-report-media/resolve/main/figures-html/f27edffd6057/f010.jpg" alt="f27edffd6057:f010" loading="lazy"><figcaption><strong>v1: page image for f010, what the crop shows</strong><br>729 × 800 px</figcaption></figure></div>
+<p>Subquery: <em>What are the visual characteristics and compositions of Millet's depictions of peasant labor?</em> The caption names a Millet drawing, but the crop is Jan Luyken's print, fig. 10. Rated No: "image shown is figure 10, not figure 9".</p>
+
+<h4>Rated Fully despite the mismatch</h4>
+<p class="caption-text"><code>dbe3f50437b4</code> · Fig. 6 Simon de Vlieger, Calm Sea, oil on panel, 52 x 89 cm, Staatliches Museum Schwerin, inv. no. G 123 (artw…</p>
+<div class="pairing"><figure><img src="https://huggingface.co/datasets/gspin/phd-report-media/resolve/main/figures/dbe3f50437b4/f006.jpg" alt="dbe3f50437b4:f006" loading="lazy"><figcaption><strong>v0: PDF crop for f006</strong><br>363 × 287 px</figcaption></figure><figure><img src="https://huggingface.co/datasets/gspin/phd-report-media/resolve/main/figures-html/dbe3f50437b4/f006.jpg" alt="dbe3f50437b4:f006" loading="lazy"><figcaption><strong>v1: page image for f006</strong><br>600 × 350 px</figcaption></figure></div>
+<p>Subquery: <em>What characterized the final acquisitions of Duke Christian Ludwig of Mecklenburg-Schwerin in terms of style, subject, and origin?</em> The caption is De Vlieger's <em>Calm Sea</em>; the crop is a painting of poultry. Rated Fully, with the comment "The figure caption doesn't match the image": the annotator judged the caption.</p>
+
+<p>
+	This is why a rating is carried over only through an accepted crop, or a renumbered one
+	whose match is unambiguous, and is attached to the picture the annotator saw.
+</p>
+
+<h3>Resolution</h3>
+<p>
+	Short side in pixels. The 224 px column is the input size of the smallest encoders on
+	the leaderboard; an image below it is upscaled by the encoder.
+</p>
+<div class="table-wrap"><table>
+	<thead><tr><th>Set</th><th class="num">Images</th><th>Short side p10 / p50 / p90</th><th class="num">Median MP</th><th class="num">Under 224 px</th></tr></thead>
+	<tbody>
+		<tr><td>Gold, PDF crops (v0)</td><td class="num">1,420</td><td>197 / 289 / 376</td><td class="num">0.11</td><td class="num">260</td></tr>
+		<tr><td>Gold, article page (v1: the picture each v0 gold crop shows)</td><td class="num">1,120</td><td>402 / 670 / 1,536</td><td class="num">0.57</td><td class="num">6</td></tr>
+		<tr><td>All page figures (v1)</td><td class="num">3,069</td><td>415 / 758 / 1,583</td><td class="num">0.82</td><td class="num">13</td></tr>
+		<tr><td>Hard distractors (RKD)</td><td class="num">7,944</td><td>308 / 430 / 1,600</td><td class="num">0.22</td><td class="num">478</td></tr>
+		<tr><td>Fillers (RKD)</td><td class="num">24,804</td><td>344 / 560 / 1,600</td><td class="num">0.36</td><td class="num">362</td></tr>
+	</tbody>
+</table></div>
+<p>
+	For the 2,444 crops matched to a page image (accepted or renumbered), the page image's
+	short side is a median 2.5 times the crop's (p10 1.6, p90 5.7). Only 34 page images are
+	smaller than their crop. Restricted to the 1,120 gold crops, the median ratio is also 2.5.
+	v1 has 3,069 figures, every one with an image, and 311 of them are named parts
+	(<code>f012a</code>, <code>f007-left</code>). It has no uncaptioned figures: the page
+	carries only captioned ones, so v0's 469 uncaptioned crops (journal chrome, or pictures
+	the PDF pairing lost) drop out.
+</p>
+<p>
+	This matters for the benchmark's fairness. In v0 the gold images were the smallest in the
+	pool, at half the distractors' median area and a third of the fillers', so an encoder could tell them apart
+	by quality rather than content. In v1 they sit at or above the RKD images, and no gold
+	image is below 224 px. Small images are kept and flagged rather than dropped or upscaled.
+</p>
+
+<h2>Validation by the annotators</h2>
+<p>
+	The annotation study rated v0's queries and links. Its judgements are reused to validate
+	v1. They join directly: the export uses the same paper, question and subquery ids, and
+	the question and subquery text is identical for all 154 subqueries in both.
+</p>
+<div class="table-wrap"><table>
+	<thead><tr><th></th><th class="num">Count</th></tr></thead>
+	<tbody>
+		<tr><td>Papers annotated</td><td class="num">44 of 145</td></tr>
+		<tr><td>Annotators who left judgements</td><td class="num">69</td></tr>
+		<tr><td>Queries rated for correctness</td><td class="num">129 of 372</td></tr>
+		<tr><td>Query–figure links rated</td><td class="num">621 of 1,692</td></tr>
+		<tr><td>— rated by more than one annotator</td><td class="num">353</td></tr>
+		<tr><td>Missed figures on existing queries</td><td class="num">240</td></tr>
+		<tr><td>Missed figures on 25 subqueries stage C linked nothing to</td><td class="num">101</td></tr>
+		<tr><td>Figures seen and not picked (negatives)</td><td class="num">1,004</td></tr>
+	</tbody>
+</table></div>
+
+<h3>Ratings of v0's links</h3>
+<p>
+	Where two annotators rated a link, the lower rating is kept. Of the 621 rated links, 275
+	are Fully (grade 2), 110 Partly (grade 1) and 236 No or Tangential (grade 0). Annotators
+	rejected 38% of the links they rated.
+</p>
+
+<h3>Agreement between annotators</h3>
+<div class="table-wrap"><table>
+	<thead><tr><th>Scale</th><th class="num">Rater pairs</th><th class="num">Raw agreement</th><th class="num">Cohen's κ</th></tr></thead>
+	<tbody>
+		<tr><td>Grade 0 / 1 / 2</td><td class="num">353</td><td class="num">58%</td><td class="num">0.29</td></tr>
+		<tr><td>Relevant (Partly or Fully) vs not</td><td class="num">353</td><td class="num">67%</td><td class="num">0.29</td></tr>
+		<tr><td>Fully vs not</td><td class="num">353</td><td class="num">67%</td><td class="num">0.34</td></tr>
+	</tbody>
+</table></div>
+<p>
+	Agreement is low, and it sets the ceiling for scoring a model against the annotators:
+	Gemini 3.6's links cannot be expected to agree with them better than they agree with each
+	other. Part of the disagreement is the mispairing itself. Faced with a caption and the
+	wrong picture, some annotators rated the caption and others the picture.
+</p>
+
+<h3>Which judgements carry over</h3>
+<p>
+	A judgement is about the picture the annotator saw. Each v0 crop is matched to the v1
+	page image it shows, by image similarity. A judgement carries over to that page image when
+	the match is unambiguous, and is dropped when it is not. The recall step showed every
+	captioned figure but none of the uncaptioned ones, so "seen and not picked" is a negative
+	for captioned figures only.
+</p>
+
+<h2>Relevance in v1</h2>
+<ul>
+	<li><strong>Grade 2:</strong> rated Fully, or a stage C link no annotator rated.</li>
+	<li><strong>Grade 1:</strong> rated Partly.</li>
+	<li><strong>Grade 0:</strong> rated No or Tangential. The image stays in the pool if another query still uses it.</li>
+	<li><strong>Added:</strong> a figure an annotator marked as missed, flagged <code>added_by_annotator</code>.</li>
+</ul>
+<p>
+	Each pair also gets a link type (target, detail, technical or comparandum). Two
+	relations, <code>detail_of</code> and <code>same_work_as</code>, connect images of one
+	work, and are evaluated by a single documented rule.
+</p>
+
+<h2>Subqueries</h2>
+<p>
+	The prompts are unchanged for v1. Subqueries rated No are dropped, Tangential and Partly
+	are flagged, and the 243 queries no annotator rated are tagged unvalidated. The ratings
+	inform v2's prompts:
+</p>
+<ul>
+	<li>Comparanda are weak (8 of 30 rated Fully, against 25 of 50 primary subqueries). They should name the visual feature they share with the work.</li>
+	<li>"Not answerable from the paper as posed" is the most frequent reason for a low rating.</li>
+	<li>The second subquery is often vague. Several annotators said so in their exit comments.</li>
+	<li>Reproduction terms (high-resolution, detail photograph) belong in the image requirements, not in the query's content.</li>
+</ul>
+
+<h2>Still to do</h2>
+<ol>
+	<li>Rerun stages B and C with Gemini 3.6 Flash on the page images, inheriting v0's questions and subqueries.</li>
+	<li>Carry the annotators' judgements over to the page images, through the crop matches above.</li>
+	<li>Score the new links against the annotators, on papers held out from any rule-tuning.</li>
+	<li>Build the graded qrels, link types, relations and subquery flags, and rerun the leaderboard.</li>
+</ol>

content/part-2/repository.html

@@ -186,6 +186,16 @@ lettered sub-figure, sharing <code>number</code> with its sibling), and <code>u0
 a detected picture no caption claimed. A figure Docling finds but cannot
 rasterise, and a caption whose figure was never detected, are both still
 recorded with <code>image_path</code> absent — nothing is silently dropped.</p>
+<p><strong>JHNA figures come from the article page, not the PDF.</strong> When a PDF has a
+download sidecar (<code>&lt;slug&gt;.json</code>) whose URL the profile serves, <code>extract</code> reads
+the text from the PDF and the figures from the article's web page: each
+figure's caption and image link sit in one element there, so nothing is paired
+by position, and the linked image is the journal's upload (about 2x the PDF's
+embedded resolution). Such a figure has <code>caption_source: &quot;html&quot;</code>, an
+<code>image_url</code>, no <code>page</code>/<code>bbox</code>, and its <code>image_path</code> is the journal's JPEG or
+PNG byte for byte (<code>figures/f001.jpg</code>). Pages and images are cached under
+<code>data/cache/web/</code>. <code>--figures pdf</code> keeps the PDF path; <code>--figures html</code>
+refuses a paper with no page. See <code>src/artpapers/webextract.py</code>.</p>
 <h2>Reports</h2>
 <p>Two independent, read-only reports, split the same way the two layers are:
 <code>summary.md</code> is extraction (figures and captions, no LLM anywhere in it),
@@ -652,8 +662,9 @@ you the figures rather than handing you a plausible-looking wrong caption.
 explicitly fails the same way rather than silently doing something else.</p>
 <p>If you need a journal extracted, write it a profile — <code>burlington.py</code> is the
 template for sequence-based pairing, <code>gesta.py</code> for proximity-based pairing.
-<code>caption_source</code> has two values so far, <code>sequence</code> and <code>proximity</code>; a profile
-that finds captions a different way adds its own rather than reusing either.</p>
+<code>caption_source</code> has three values so far: <code>sequence</code> and <code>proximity</code> from the
+PDF, and <code>html</code> from an article page; a profile that finds captions a
+different way adds its own rather than reusing one.</p>
 <p><code>caption_number</code> is the number the caption text itself opens with (&quot;Fig. 3&quot;,
 &quot;3.&quot;, &quot;Abb. 3&quot;). Compare it against a figure's position to audit the pairing.</p>
 <h3>When a region is more than one figure</h3>

nav.json

@@ -16,7 +16,7 @@
       "slug": "part-2",
       "label": "Part II",
       "title": "How can we support Digital Art History with Multimodal LLMs",
-      "summary": "The IRS prototype: evaluation framework, theoretical framework, dataset and the IR system.",
+      "summary": "The IRS prototype: evaluation framework, dataset and retrieval.",
       "icon": "layers",
       "file": "content/part-2/index.html",
       "steps": true,
@@ -137,6 +137,22 @@
                       ]
                     }
                   ]
+                },
+                {
+                  "slug": "local-models",
+                  "title": "Local models: smoke test",
+                  "summary": "Qwen3.6, Qwen3.8, Gemma 4 and Mistral 3.2 on one image, served locally with vLLM.",
+                  "icon": "flask",
+                  "file": "content/part-2/evaluation-framework/analysis/local-models.html",
+                  "hidden": true
+                },
+                {
+                  "slug": "description-retrieval",
+                  "title": "Local models: retrieval by description",
+                  "summary": "Four local models describe 732 images; 50 queries are matched to the descriptions, against direct image search.",
+                  "icon": "flask",
+                  "file": "content/part-2/evaluation-framework/analysis/description-retrieval.html",
+                  "hidden": true
                 }
               ]
             },
@@ -151,43 +167,6 @@
           ],
           "steps": true
         },
-        {
-          "slug": "theoretical-framework",
-          "title": "Theoretical framework",
-          "icon": "layers",
-          "file": "content/part-2/framework/index.html",
-          "status": "Draft",
-          "children": [
-            {
-              "slug": "applied",
-              "title": "Frameworks applied to 300 queries",
-              "summary": "Every labelled query classified under Panofsky, J&C, Hollink and L1–L5; the first 20 in full.",
-              "icon": "list",
-              "file": "content/part-2/framework/applied.html"
-            },
-            {
-              "slug": "pyramid-queries",
-              "title": "Pyramid: 50 queries",
-              "summary": "50 queries, one per paper, split into constraints and classified under levels I–VII + N by Gemini 3.6 Flash and Qwen3-VL.",
-              "icon": "list",
-              "file": "content/part-2/framework/pyramid-queries.html"
-            },
-            {
-              "slug": "pyramid-images",
-              "title": "Pyramid: gold figures",
-              "summary": "The 85 gold figures of those queries, described at every level by both models, from the image alone.",
-              "icon": "layers",
-              "file": "content/part-2/framework/pyramid-images.html"
-            },
-            {
-              "slug": "slides",
-              "title": "Slides",
-              "summary": "The framework as a presentation, around the Van der Goes example.",
-              "icon": "book",
-              "file": "content/part-2/framework/slides.html"
-            }
-          ]
-        },
         {
           "slug": "pool-creation",
           "title": "Dataset",
@@ -197,22 +176,68 @@
           "status": "Draft",
           "children": [
             {
-              "slug": "dataset",
-              "title": "Dataset",
-              "summary": "Papers, queries and images in the latest build, the evaluation pools, RKD linkage and distractor judging.",
+              "slug": "v1",
+              "title": "Dataset v1",
+              "summary": "JHNA figures from the article pages, annotator validation and graded relevance: what changes from v0.",
+              "icon": "database",
+              "file": "content/part-2/pool/v1.html",
+              "status": "Draft"
+            },
+            {
+              "slug": "v0",
+              "title": "Dataset v0",
+              "summary": "The first build (run adcf1483): PDF figure crops, stage-C links as binary gold, 372 queries over 34,364 images.",
               "icon": "bar-chart",
-              "file": "content/part-2/pool/dataset.html"
+              "file": "content/part-2/pool/v0.html",
+              "hidden": true
             }
           ]
         },
         {
           "slug": "irs-testing",
-          "title": "IR system",
-          "summary": "Scoring zero-shot text-to-image retrieval models against the pool benchmark.",
+          "title": "Retrieval",
+          "summary": "What a retrieval system should find (the theoretical framework), and how zero-shot text-to-image models score on the dataset.",
           "icon": "flask",
           "file": "content/part-2/bench/index.html",
           "status": "Draft",
           "children": [
+            {
+              "slug": "theoretical-framework",
+              "title": "Theoretical framework",
+              "icon": "layers",
+              "file": "content/part-2/framework/index.html",
+              "status": "Draft",
+              "children": [
+                {
+                  "slug": "applied",
+                  "title": "Frameworks applied to 300 queries",
+                  "summary": "Every labelled query classified under Panofsky, J&C, Hollink and L1–L5; the first 20 in full.",
+                  "icon": "list",
+                  "file": "content/part-2/framework/applied.html"
+                },
+                {
+                  "slug": "pyramid-queries",
+                  "title": "Pyramid: 50 queries",
+                  "summary": "50 queries, one per paper, split into constraints and classified under levels I–VII + N by Gemini 3.6 Flash and Qwen3-VL.",
+                  "icon": "list",
+                  "file": "content/part-2/framework/pyramid-queries.html"
+                },
+                {
+                  "slug": "pyramid-images",
+                  "title": "Pyramid: gold figures",
+                  "summary": "The 85 gold figures of those queries, described at every level by both models, from the image alone.",
+                  "icon": "layers",
+                  "file": "content/part-2/framework/pyramid-images.html"
+                },
+                {
+                  "slug": "slides",
+                  "title": "Slides",
+                  "summary": "The framework as a presentation, around the Van der Goes example.",
+                  "icon": "book",
+                  "file": "content/part-2/framework/slides.html"
+                }
+              ]
+            },
             {
               "slug": "results",
               "title": "Results",