Upload index.html

index.html

@@ -123,6 +123,20 @@
   </div>
 </div>
 
+<h2 class="section-title">Model Efficiency</h2>
+
+<div class="chart-grid">
+  <div class="chart-card full">
+    <h3>Parameters vs Avg Score — high efficiency zone (≥1σ above trend)</h3>
+    <canvas id="efficiencyChart" style="max-height:400px"></canvas>
+    <div style="display:flex;flex-direction:column;gap:2px;margin-top:.5rem;font-size:.75rem;color:#8b949e">
+      <span>Faint dashed line: average trend</span>
+      <span>Bold dashed line: high-efficiency threshold (trend + 1σ)</span>
+      <span>Yellow shaded area: models outperforming expectations for their size</span>
+    </div>
+  </div>
+</div>
+
 
 
 <div class="methodology" style="margin-top:1.5rem">
@@ -400,6 +414,16 @@ function getColor(value, min, max, lowerIsBetter, useLog = false) {
   return `color: rgb(${r}, ${g}, ${b})`;
 }
 
+function parseParams(str) {
+  if (!str || typeof str !== 'string') return NaN;
+  const s = str.toUpperCase().replace(/,/g, '');
+  if (s.endsWith('B')) return parseFloat(s) * 1e9;
+  if (s.endsWith('M')) return parseFloat(s) * 1e6;
+  if (s.endsWith('K')) return parseFloat(s) * 1e3;
+  const n = parseFloat(s);
+  return isNaN(n) ? NaN : n;
+}
+
 function renderTable() {
   const tbody = document.getElementById('leaderboard-body');
   const sortedModels = [...models].sort((a, b) => getScore(b) - getScore(a));
@@ -489,11 +513,166 @@ function buildChart(canvasId, metric, label, reverse) {
   });
 }
 
+function buildEfficiencyChart() {
+  const valid = models
+    .filter(d => d.blimp !== null && d.arc !== null && typeof d.blimp === 'number' && typeof d.arc === 'number')
+    .map(d => ({
+      ...d,
+      paramsNum: parseParams(d.params),
+      avgScore: (d.blimp + d.arc) / 2
+    }))
+    .filter(d => !isNaN(d.paramsNum) && d.paramsNum > 0);
+
+  if (valid.length < 2) return;
+
+  const logParams = valid.map(d => Math.log10(d.paramsNum));
+  const scores = valid.map(d => d.avgScore);
+
+  const n = valid.length;
+  const sumX = logParams.reduce((s, v) => s + v, 0);
+  const sumY = scores.reduce((s, v) => s + v, 0);
+  const sumXY = logParams.reduce((s, v, i) => s + v * scores[i], 0);
+  const sumX2 = logParams.reduce((s, v) => s + v * v, 0);
+  const slope = (n * sumXY - sumX * sumY) / (n * sumX2 - sumX * sumX);
+  const intercept = (sumY - slope * sumX) / n;
+
+  const residuals = valid.map(d => d.avgScore - (intercept + slope * Math.log10(d.paramsNum)));
+  const resMean = residuals.reduce((s, v) => s + v, 0) / n;
+  const resVariance = residuals.reduce((s, v) => s + (v - resMean) ** 2, 0) / n;
+  const resStddev = Math.sqrt(resVariance);
+  const thresholdShift = Math.max(resStddev, 3);
+
+  const sorted = [...valid].sort((a, b) => a.paramsNum - b.paramsNum);
+
+  const regData = [];
+  const thresholdData = [];
+  const steps = 100;
+  const logMin = 4;
+  const logMax = 9;
+  for (let i = 0; i <= steps; i++) {
+    const lx = logMin + (logMax - logMin) * (i / steps);
+    regData.push({ x: Math.pow(10, lx), y: intercept + slope * lx });
+    thresholdData.push({ x: Math.pow(10, lx), y: intercept + slope * lx + thresholdShift });
+  }
+
+  const zonePlugin = {
+    id: 'efficiencyZone',
+    beforeDraw(chart) {
+      const ctx = chart.ctx;
+      const xScale = chart.scales.x;
+      const yScale = chart.scales.y;
+      const { left, right, top, bottom } = chart.chartArea;
+
+      const leftX = xScale.min;
+      const rightX = xScale.max;
+      const leftY = intercept + slope * Math.log10(Math.max(leftX, 1)) + thresholdShift;
+      const rightY = intercept + slope * Math.log10(Math.max(rightX, 1)) + thresholdShift;
+      const leftYPix = yScale.getPixelForValue(leftY);
+      const rightYPix = yScale.getPixelForValue(rightY);
+
+      ctx.save();
+      ctx.beginPath();
+      ctx.rect(left, top, right - left, bottom - top);
+      ctx.clip();
+
+      ctx.beginPath();
+      ctx.moveTo(left, leftYPix);
+      ctx.lineTo(left, top);
+      ctx.lineTo(right, top);
+      ctx.lineTo(right, rightYPix);
+      ctx.closePath();
+      ctx.fillStyle = 'rgba(255, 230, 0, 0.12)';
+      ctx.fill();
+      ctx.restore();
+    }
+  };
+
+  new Chart(document.getElementById('efficiencyChart'), {
+    type: 'line',
+    data: {
+      datasets: [
+        {
+          label: 'Models',
+          data: sorted.map(d => ({ x: d.paramsNum, y: d.avgScore })),
+          showLine: false,
+          backgroundColor: sorted.map(d => colorMap[d.org]),
+          borderColor: sorted.map(d => colorMap[d.org]),
+          pointRadius: 6,
+          pointHoverRadius: 9,
+        },
+        {
+          label: 'Trend',
+          data: regData,
+          showLine: true,
+          borderColor: 'rgba(255, 200, 0, 0.5)',
+          borderWidth: 1.5,
+          borderDash: [4, 4],
+          pointRadius: 0,
+          fill: false,
+          tension: 0,
+        },
+        {
+          label: 'High Efficiency Threshold',
+          data: thresholdData,
+          showLine: true,
+          borderColor: 'rgba(255, 200, 0, 0.8)',
+          borderWidth: 2,
+          borderDash: [6, 4],
+          pointRadius: 0,
+          fill: false,
+          tension: 0,
+        }
+      ]
+    },
+    options: {
+      parsing: false,
+      responsive: true,
+      maintainAspectRatio: true,
+      scales: {
+        x: {
+          type: 'logarithmic',
+          title: { display: true, text: 'Parameters', color: '#8b949e' },
+          grid: { color: 'rgba(255,255,255,0.06)' },
+          ticks: {
+            color: '#8b949e',
+            callback: function(v) {
+              if (v >= 1000000) return (v / 1000000).toFixed(v >= 10000000 ? 0 : 1) + 'M';
+              if (v >= 1000) return (v / 1000).toFixed(v >= 10000 ? 0 : 1) + 'K';
+              return v.toString();
+            }
+          }
+        },
+        y: {
+          title: { display: true, text: 'Avg Score (BLiMP + ARC-Easy)', color: '#8b949e' },
+          min: 20,
+          max: 80,
+          grid: { color: 'rgba(255,255,255,0.06)' },
+          ticks: { color: '#8b949e', callback: v => v + '%' }
+        }
+      },
+      plugins: {
+        legend: { display: false },
+        tooltip: {
+          callbacks: {
+            label: ctx => {
+              if (ctx.dataset.label !== 'Models') return '';
+              const d = sorted[ctx.dataIndex];
+              return `${d.name}: ${d.params} params, ${d.avgScore.toFixed(1)}% avg`;
+            }
+          }
+        }
+      }
+    },
+    plugins: [zonePlugin]
+  });
+}
+
 window.addEventListener('DOMContentLoaded', () => {
   renderTable();
   buildChart('blimpChart', 'blimp', 'BLiMP Accuracy', false);
   buildChart('arcChart', 'arc', 'ARC-Easy Accuracy', false);
   buildChart('wikiChart', 'wiki', 'WikiText-2 Perplexity', true);
+  buildEfficiencyChart();
 });
 </script>
 </body>