add compression analysis

app/src/content/chapters/analyses.mdx

@@ -100,6 +100,23 @@ Different prompt formats produce wildly different output lengths. <FigRef target
 </Wide>
 
 
+
+### Does Compression Predict Performance?
+
+Our prompts produce outputs ranging from 25% of the input length (Commentary) to 150% (Guided Rewrite at 12B). Does the degree of compression or expansion matter for downstream performance? <FigRef target="compression-performance" /> plots each experiment's compression ratio against its benchmark score.
+
+**There is no meaningful relationship between compression ratio and performance.** Highly compressive prompts (Commentary at 0.26x, Table at 0.25x) and expansive ones (Guided Rewrite at 1.5x) both appear across the full range of performance scores. The best-performing experiments cluster around 0.3x-0.8x compression, but this likely reflects the distribution of prompt types rather than any causal effect of compression itself. FAQ and Tutorial prompts, which happen to compress moderately, also happen to be the strongest prompts for other reasons (pedagogical restructuring, diverse output formats).
+
+This means you should not use compression ratio as a proxy for data quality. A prompt that produces concise outputs is not inherently better or worse than one that produces verbose outputs. What matters is the content and structure of the output, not its length relative to the input.
+
+<HtmlEmbed
+  id="compression-performance"
+  src="compression-performance.html"
+  data="rephrasing_metadata.json"
+  desc="Compression ratio (output/input tokens) vs downstream performance. The dashed line marks ratio = 1.0 (no compression). Hover over points for details."
+/>
+
+
 ### Is More Compute Worth It?
 
 GPU time across our 65 experiments varies by two orders of magnitude: the cheapest run (Table with SmolLM2) took 8 days, while the most expensive (Guided Rewrite with Gemma-3 27B) consumed over 15 months of GPU time. <FigRef target="cost-efficiency" /> plots each experiment's downstream performance against its GPU cost on a log scale, with a Pareto frontier connecting the most efficient configurations.

app/src/content/embeds/compression-performance.html

@@ -0,0 +1,375 @@
+<div class="d3-compression-perf" style="width:100%;margin:10px 0;min-height:400px;"></div>
+<style>
+  .d3-compression-perf { font-family: system-ui, -apple-system, sans-serif; position: relative; }
+  .d3-compression-perf .d3-tooltip {
+    position: absolute; top: 0; left: 0;
+    transform: translate(-9999px, -9999px);
+    pointer-events: none;
+    padding: 10px 14px; border-radius: 10px;
+    font-size: 13px; line-height: 1.4;
+    border: 1px solid var(--border-color);
+    background: var(--surface-bg); color: var(--text-color);
+    box-shadow: 0 6px 24px rgba(0,0,0,.22);
+    opacity: 0; transition: opacity .12s ease;
+    z-index: 20; max-width: 340px;
+  }
+  .d3-compression-perf .controls {
+    display: flex; gap: 16px; align-items: center; justify-content: flex-end; flex-wrap: wrap;
+    margin-top: 8px;
+  }
+  .d3-compression-perf .control-group {
+    display: flex; flex-direction: column; align-items: flex-start; gap: 4px;
+  }
+  .d3-compression-perf .controls label {
+    font-size: 13px; font-weight: 700; color: var(--text-color);
+  }
+  .d3-compression-perf .controls select {
+    font-size: 13px; padding: 6px 28px 6px 10px; border: 1px solid var(--border-color);
+    border-radius: 8px; background: var(--surface-bg); color: var(--text-color);
+    appearance: none; cursor: pointer;
+    background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' viewBox='0 0 12 12'%3E%3Cpath d='M3 5l3 3 3-3' stroke='%23888' stroke-width='1.5' fill='none'/%3E%3C/svg%3E");
+    background-repeat: no-repeat; background-position: right 8px center;
+  }
+  .d3-compression-perf .legend {
+    display: flex; flex-direction: column; align-items: flex-start; gap: 6px; margin-top: 8px;
+  }
+  .d3-compression-perf .legend-title { font-size: 13px; font-weight: 700; color: var(--text-color); }
+  .d3-compression-perf .legend .items { display: flex; flex-wrap: wrap; gap: 6px 14px; }
+  .d3-compression-perf .legend .item {
+    display: inline-flex; align-items: center; gap: 6px; white-space: nowrap;
+    font-size: 13px; color: var(--text-color); cursor: pointer;
+  }
+  .d3-compression-perf .legend .swatch {
+    width: 14px; height: 14px; border-radius: 3px; border: 1px solid var(--border-color);
+  }
+</style>
+<script>
+(() => {
+  const ensureD3 = (cb) => {
+    if (window.d3 && typeof window.d3.select === 'function') return cb();
+    let s = document.getElementById('d3-cdn-script');
+    if (!s) { s = document.createElement('script'); s.id = 'd3-cdn-script'; s.src = 'https://cdn.jsdelivr.net/npm/d3@7/dist/d3.min.js'; document.head.appendChild(s); }
+    const onReady = () => { if (window.d3 && typeof window.d3.select === 'function') cb(); };
+    s.addEventListener('load', onReady, { once: true });
+    if (window.d3) onReady();
+  };
+
+  const bootstrap = () => {
+    const scriptEl = document.currentScript;
+    let container = scriptEl ? scriptEl.previousElementSibling : null;
+    while (container && !(container.classList && container.classList.contains('d3-compression-perf'))) {
+      container = container.previousElementSibling;
+    }
+    if (!container) {
+      const cs = Array.from(document.querySelectorAll('.d3-compression-perf'))
+        .filter(el => !(el.dataset && el.dataset.mounted === 'true'));
+      container = cs[cs.length - 1] || null;
+    }
+    if (!container) return;
+    if (container.dataset.mounted === 'true') return;
+    container.dataset.mounted = 'true';
+
+    let mountEl = container;
+    while (mountEl && !mountEl.getAttribute?.('data-datafiles')) mountEl = mountEl.parentElement;
+    const dataAttr = mountEl?.getAttribute?.('data-datafiles');
+    const dataPaths = dataAttr
+      ? [dataAttr.includes('/') ? dataAttr : `/data/${dataAttr}`]
+      : ['/data/rephrasing_metadata.json', './assets/data/rephrasing_metadata.json'];
+
+    const fetchFirst = async (paths) => {
+      for (const p of paths) {
+        try { const r = await fetch(p, { cache: 'no-cache' }); if (r.ok) return r.json(); } catch(_) {}
+      }
+      throw new Error('Data not found');
+    };
+
+    fetchFirst(dataPaths).then(data => buildChart(data)).catch(err => {
+      container.innerHTML = `<pre style="color:red;padding:12px;">Error loading data: ${err.message}</pre>`;
+    });
+
+    function buildChart(rawData) {
+      const SOURCE_MAP = {
+        'fineweb-edu-hq-20BT': 'FW-Edu HQ', 'fineweb-edu-lq-20BT': 'FW-Edu LQ',
+        'dclm-37BT': 'DCLM', 'cosmopedia-25BT': 'Cosmopedia'
+      };
+      const PROMPT_LABELS = {
+        'article': 'Article', 'commentary': 'Commentary', 'discussion': 'Discussion',
+        'faq': 'FAQ', 'math': 'Math', 'table': 'Table', 'tutorial': 'Tutorial',
+        'distill': 'Distill', 'diverse_qa_pairs': 'Diverse QA',
+        'extract_knowledge': 'Extract Knowledge', 'knowledge_list': 'Knowledge List',
+        'wikipedia_style_rephrasing': 'Wikipedia Style',
+        'guided_rewrite_improved': 'Guided Rewrite+', 'guided_rewrite_original': 'Guided Rewrite'
+      };
+
+      const getFamily = (m) => {
+        const ml = m.toLowerCase();
+        if (ml.includes('smollm')) return 'SmolLM2';
+        if (ml.includes('gemma')) return 'Gemma';
+        if (ml.includes('qwen')) return 'Qwen';
+        if (ml.includes('falcon')) return 'Falcon';
+        if (ml.includes('granite')) return 'Granite';
+        if (ml.includes('llama')) return 'Llama';
+        return 'Other';
+      };
+
+      // Color by prompt type
+      const allPromptKeys = [...new Set(rawData.map(d => d.prompt.split('/')[1].replace('.md', '')))].sort();
+      const promptColors = {};
+      const cat = window.ColorPalettes ? window.ColorPalettes.getColors('categorical', allPromptKeys.length) : d3.schemeTableau10.concat(d3.schemePastel1);
+      allPromptKeys.forEach((k, i) => { promptColors[PROMPT_LABELS[k] || k] = cat[i % cat.length]; });
+
+      const METRIC_NAMES = {
+        'agg_score_macro': 'Aggregate Score (Macro)',
+        'agg_score_micro': 'Aggregate Score (Micro)',
+        'agg_score_RC': 'Reading Comprehension',
+        'agg_score_GK': 'General Knowledge',
+        'agg_score_NLU': 'Natural Language Understanding',
+        'agg_score_MATH': 'Math',
+        'agg_score_TABLE': 'Table Understanding',
+        'agg_score_RES': 'Reasoning',
+        'arc_cf:easy': 'ARC-Easy',
+        'drop': 'DROP',
+        'gsm8k': 'GSM8K',
+        'hellaswag_cf': 'HellaSwag',
+        'openbookqa_cf': 'OpenBookQA',
+        'piqa_cf': 'PIQA',
+        'squad_v2': 'SQuAD v2',
+        'treb_qa': 'TriviaQA',
+        'wikitablequestions': 'WikiTableQuestions',
+        'winogrande_cf': 'Winogrande',
+        'xcsqa_cf': 'XCSQA',
+        'mmlu_redux_cf:_average': 'MMLU Redux'
+      };
+      const AGG_ORDER = ['agg_score_macro', 'agg_score_micro', 'agg_score_RC', 'agg_score_GK', 'agg_score_NLU', 'agg_score_MATH', 'agg_score_TABLE', 'agg_score_RES'];
+      const metricName = (key) => METRIC_NAMES[key] || key;
+
+      const experiments = rawData.map(d => {
+        const [cat, promptFile] = d.prompt.split('/');
+        const promptKey = promptFile.replace('.md', '');
+        return {
+          run: d.run,
+          prompt: PROMPT_LABELS[promptKey] || promptKey,
+          model: d.model.split('/').pop(),
+          source: SOURCE_MAP[d.source_dataset] || d.source_dataset,
+          family: getFamily(d.model),
+          compressionRatio: d.compression_ratio,
+          inputPerDoc: d.input_token_count_mean,
+          outputPerDoc: d.output_token_count_mean,
+          tokenReduction: d.token_reduction_mean,
+          results: d.results
+        };
+      });
+
+      // Detect available metrics from first experiment's results
+      const allResultKeys = Object.keys(experiments[0].results);
+      const aggMetrics = AGG_ORDER.filter(k => allResultKeys.includes(k));
+      const indMetrics = allResultKeys.filter(k => !k.startsWith('agg_score'));
+
+      let currentMetric = aggMetrics[0] || allResultKeys[0];
+
+      const svg = d3.select(container).append('svg').attr('width', '100%').style('display', 'block');
+      const gGrid = svg.append('g');
+      const gRef = svg.append('g');
+      const gDots = svg.append('g');
+      const gAxes = svg.append('g');
+      const gAnnot = svg.append('g');
+
+      let tip = container.querySelector('.d3-tooltip');
+      let tipInner;
+      if (!tip) {
+        tip = document.createElement('div'); tip.className = 'd3-tooltip';
+        tipInner = document.createElement('div'); tipInner.className = 'd3-tooltip__inner';
+        tipInner.style.textAlign = 'left';
+        tip.appendChild(tipInner); container.appendChild(tip);
+      } else { tipInner = tip.querySelector('.d3-tooltip__inner') || tip; }
+
+      const margin = { top: 12, right: 16, bottom: 48, left: 56 };
+
+      function render() {
+        const width = container.clientWidth || 800;
+        const height = Math.max(360, Math.round(width / 2.2));
+        svg.attr('width', width).attr('height', height);
+        const iw = width - margin.left - margin.right;
+        const ih = height - margin.top - margin.bottom;
+        const metricLabel = metricName(currentMetric);
+
+        const xExtent = d3.extent(experiments, d => d.compressionRatio);
+        const xPad = (xExtent[1] - xExtent[0]) * 0.06;
+        const xScale = d3.scaleLinear()
+          .domain([xExtent[0] - xPad, xExtent[1] + xPad])
+          .range([margin.left, width - margin.right]);
+
+        const yVals = experiments.map(d => d.results[currentMetric]).filter(v => v != null);
+        const yPad = (d3.max(yVals) - d3.min(yVals)) * 0.08;
+        const yScale = d3.scaleLinear()
+          .domain([d3.min(yVals) - yPad, d3.max(yVals) + yPad])
+          .range([height - margin.bottom, margin.top]);
+
+        // Grid
+        const yTicks = yScale.ticks(6);
+        gGrid.selectAll('line').data(yTicks).join('line')
+          .attr('x1', margin.left).attr('x2', width - margin.right)
+          .attr('y1', d => yScale(d)).attr('y2', d => yScale(d))
+          .attr('stroke', 'var(--grid-color)').attr('stroke-width', 0.5);
+
+        // Reference line at compression_ratio = 1.0
+        gRef.selectAll('*').remove();
+        const x1 = xScale(1.0);
+        if (x1 > margin.left && x1 < width - margin.right) {
+          gRef.append('line')
+            .attr('x1', x1).attr('x2', x1)
+            .attr('y1', margin.top).attr('y2', height - margin.bottom)
+            .attr('stroke', 'var(--muted-color)').attr('stroke-width', 1)
+            .attr('stroke-dasharray', '6,4').attr('opacity', 0.5);
+        }
+
+        // Annotations
+        gAnnot.selectAll('*').remove();
+        const annotY = margin.top + 14;
+        if (x1 > margin.left + 60 && x1 < width - margin.right - 60) {
+          gAnnot.append('text')
+            .attr('x', x1 - 10).attr('y', annotY)
+            .attr('text-anchor', 'end').attr('fill', 'var(--muted-color)')
+            .attr('font-size', '12px').attr('font-style', 'italic')
+            .text('\u2190 compression');
+          gAnnot.append('text')
+            .attr('x', x1 + 10).attr('y', annotY)
+            .attr('text-anchor', 'start').attr('fill', 'var(--muted-color)')
+            .attr('font-size', '12px').attr('font-style', 'italic')
+            .text('expansion \u2192');
+        }
+
+        // Axes
+        gAxes.selectAll('*').remove();
+        gAxes.append('g')
+          .attr('transform', `translate(0,${height - margin.bottom})`)
+          .call(d3.axisBottom(xScale).ticks(8).tickFormat(d => d.toFixed(2)))
+          .call(g => g.select('.domain').attr('stroke', 'var(--axis-color)'))
+          .call(g => g.selectAll('.tick line').attr('stroke', 'var(--tick-color)'))
+          .call(g => g.selectAll('.tick text').attr('fill', 'var(--tick-color)').attr('font-size', '13px'));
+
+        gAxes.append('g')
+          .attr('transform', `translate(${margin.left},0)`)
+          .call(d3.axisLeft(yScale).ticks(6).tickFormat(d3.format('.2f')))
+          .call(g => g.select('.domain').attr('stroke', 'var(--axis-color)'))
+          .call(g => g.selectAll('.tick line').attr('stroke', 'var(--tick-color)'))
+          .call(g => g.selectAll('.tick text').attr('fill', 'var(--tick-color)').attr('font-size', '13px'));
+
+        gAxes.append('text')
+          .attr('x', margin.left + iw / 2).attr('y', height - 4)
+          .attr('text-anchor', 'middle').attr('fill', 'var(--text-color)')
+          .attr('font-size', '14px').attr('font-weight', '600')
+          .text('Compression ratio (output / input tokens)');
+
+        gAxes.append('text')
+          .attr('transform', 'rotate(-90)')
+          .attr('x', -(margin.top + ih / 2)).attr('y', 14)
+          .attr('text-anchor', 'middle').attr('fill', 'var(--text-color)')
+          .attr('font-size', '14px').attr('font-weight', '600')
+          .text(metricLabel);
+
+        // Dots
+        const rBase = Math.max(5, Math.min(9, width * 0.008));
+
+        gDots.selectAll('circle').data(experiments, d => d.run).join('circle')
+          .attr('cx', d => xScale(d.compressionRatio))
+          .attr('cy', d => yScale(d.results[currentMetric]))
+          .attr('r', rBase)
+          .attr('fill', d => promptColors[d.prompt] || '#999')
+          .attr('fill-opacity', 0.8)
+          .attr('stroke', d => promptColors[d.prompt] || '#999')
+          .attr('stroke-width', 1.5)
+          .attr('stroke-opacity', 0.3)
+          .attr('cursor', 'pointer')
+          .on('mouseenter', function(ev, d) {
+            d3.select(this).attr('r', rBase * 1.6).attr('fill-opacity', 1).attr('stroke-opacity', 0.8);
+            gDots.selectAll('circle').filter(c => c !== d)
+              .attr('fill-opacity', 0.15).attr('stroke-opacity', 0.08);
+            // Highlight same prompt
+            gDots.selectAll('circle').filter(c => c.prompt === d.prompt && c !== d)
+              .attr('fill-opacity', 0.5).attr('stroke-opacity', 0.4);
+            const score = d.results[currentMetric];
+            tipInner.innerHTML =
+              `<div style="font-weight:700;font-size:14px;margin-bottom:4px;">${d.prompt}</div>` +
+              `<div style="font-size:12px;color:var(--muted-color);margin-bottom:6px;">` +
+              `${d.model} · ${d.source}</div>` +
+              `<div style="display:grid;grid-template-columns:auto 1fr;gap:2px 10px;font-size:13px;">` +
+              `<span style="color:var(--muted-color);">Compression</span><span>${d.compressionRatio.toFixed(3)} (${(d.compressionRatio * 100).toFixed(0)}%)</span>` +
+              `<span style="color:var(--muted-color);">Input/doc</span><span>${Math.round(d.inputPerDoc)} tokens</span>` +
+              `<span style="color:var(--muted-color);">Output/doc</span><span>${Math.round(d.outputPerDoc)} tokens</span>` +
+              `<span style="color:var(--muted-color);">Reduction</span><span>${Math.round(d.tokenReduction)} tokens</span>` +
+              `<span style="color:var(--muted-color);">${metricLabel}</span><span style="font-weight:700;">${score != null ? score.toFixed(4) : 'N/A'}</span>` +
+              `</div>`;
+            tip.style.opacity = '1';
+          })
+          .on('mousemove', (ev) => {
+            const [mx, my] = d3.pointer(ev, container);
+            const bw = tip.offsetWidth || 280;
+            const bh = tip.offsetHeight || 160;
+            const ox = (mx + bw + 20 > width) ? -(bw + 12) : 14;
+            const oy = (my + bh + 20 > (height + 60)) ? -(bh + 12) : 14;
+            tip.style.transform = `translate(${Math.round(mx + ox)}px,${Math.round(my + oy)}px)`;
+          })
+          .on('mouseleave', function() {
+            gDots.selectAll('circle').attr('r', rBase).attr('fill-opacity', 0.8).attr('stroke-opacity', 0.3);
+            tip.style.opacity = '0';
+            tip.style.transform = 'translate(-9999px,-9999px)';
+          });
+      }
+
+      // Controls
+      const controls = document.createElement('div'); controls.className = 'controls';
+      const cg = document.createElement('div'); cg.className = 'control-group';
+      const lbl = document.createElement('label'); lbl.textContent = 'Metric'; lbl.setAttribute('for', 'cp-metric-select');
+      const sel = document.createElement('select'); sel.id = 'cp-metric-select';
+      const aggGroup = document.createElement('optgroup'); aggGroup.label = 'Aggregate Scores';
+      aggMetrics.forEach(k => {
+        const opt = document.createElement('option'); opt.value = k; opt.textContent = metricName(k);
+        if (k === currentMetric) opt.selected = true;
+        aggGroup.appendChild(opt);
+      });
+      const indGroup = document.createElement('optgroup'); indGroup.label = 'Individual Benchmarks';
+      indMetrics.forEach(k => {
+        const opt = document.createElement('option'); opt.value = k; opt.textContent = metricName(k);
+        if (k === currentMetric) opt.selected = true;
+        indGroup.appendChild(opt);
+      });
+      if (aggGroup.children.length) sel.appendChild(aggGroup);
+      if (indGroup.children.length) sel.appendChild(indGroup);
+      sel.addEventListener('change', () => { currentMetric = sel.value; render(); });
+      cg.appendChild(lbl); cg.appendChild(sel); controls.appendChild(cg); container.appendChild(controls);
+
+      // Legend
+      const legend = document.createElement('div'); legend.className = 'legend';
+      const ltitle = document.createElement('div'); ltitle.className = 'legend-title'; ltitle.textContent = 'Legend';
+      const items = document.createElement('div'); items.className = 'items';
+      const usedPrompts = [...new Set(experiments.map(d => d.prompt))].sort();
+      usedPrompts.forEach(p => {
+        const el = document.createElement('span'); el.className = 'item';
+        const sw = document.createElement('span'); sw.className = 'swatch'; sw.style.background = promptColors[p];
+        const txt = document.createElement('span'); txt.textContent = p;
+        el.appendChild(sw); el.appendChild(txt); items.appendChild(el);
+        el.addEventListener('mouseenter', () => {
+          svg.selectAll('circle').attr('fill-opacity', d => d.prompt === p ? 0.9 : 0.1)
+            .attr('stroke-opacity', d => d.prompt === p ? 0.6 : 0.05);
+        });
+        el.addEventListener('mouseleave', () => {
+          svg.selectAll('circle').attr('fill-opacity', 0.8).attr('stroke-opacity', 0.3);
+        });
+      });
+      // Reference line legend
+      const refItem = document.createElement('span'); refItem.className = 'item';
+      refItem.innerHTML = `<svg width="20" height="14" style="vertical-align:middle;"><line x1="0" y1="7" x2="20" y2="7" stroke="var(--muted-color)" stroke-width="1" stroke-dasharray="4,3" opacity="0.5"/></svg><span>ratio = 1.0</span>`;
+      items.appendChild(refItem);
+      legend.appendChild(ltitle); legend.appendChild(items); container.appendChild(legend);
+
+      render();
+      if (window.ResizeObserver) new ResizeObserver(() => render()).observe(container);
+      else window.addEventListener('resize', render);
+    }
+  };
+
+  if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', () => ensureD3(bootstrap), { once: true });
+  else ensureD3(bootstrap);
+})();
+</script>