Redesign CTR calibration for marketer workflow — metric input upfront, not buried

Move "Anchor to a real metric" from a hidden details section inside eval results
to a first-class input field next to the goal. Default to CTR with a dropdown
for other metrics (conversion rate, open rate, revenue, custom).

- Auto-apply calibration when eval completes if metric value is set
- Debounced re-apply on metric value/unit changes
- Table headers use actual metric name ("CTR Impact", not "Metric Impact")
- Keep Helps/Hurts columns alongside metric columns (was replacing them)
- Clear calibration re-renders gradient table back to score-only view
- Fix score:0 truthiness bug in frontend mean calculation
- Guard against division by zero in linear scaling (mean_score > 0)
- Tighten backend score filtering to isinstance check

web/app.py

@@ -346,6 +346,8 @@ async def set_calibration(sid: str, cal: CalibrationInput):
                for a in cal.anchors if a.metric_value > 0]
     if not anchors:
         raise HTTPException(400, "Need at least one anchor with metric_value > 0")
+    if any(a["mean_score"] <= 0 for a in anchors):
+        raise HTTPException(400, "Mean score must be positive")
 
     if len(anchors) == 1:
         # Single anchor: linear scaling. metric = k * mean_score
@@ -391,7 +393,7 @@ def _apply_calibration(session):
     if not cal or not ranked:
         return None
 
-    valid = [r for r in (session.get("eval_results") or []) if r and "score" in r]
+    valid = [r for r in (session.get("eval_results") or []) if r and isinstance(r.get("score"), (int, float))]
     if not valid:
         return None
     mean_score = sum(r["score"] for r in valid) / len(valid)

web/static/index.html

@@ -422,6 +422,28 @@
       <input type="text" id="cohortDesc" placeholder="e.g. 'Engineering managers at mid-stage startups' or 'US consumers aged 25-45'">
     </div>
 
+    <div class="field" id="metricAnchorField">
+      <label>Know the current performance?</label>
+      <div style="display:flex;gap:8px;align-items:center;flex-wrap:wrap">
+        <select id="calMetricName" style="width:auto;min-width:100px;padding:8px 10px">
+          <option value="CTR">CTR</option>
+          <option value="conversion rate">Conversion rate</option>
+          <option value="open rate">Open rate</option>
+          <option value="revenue">Revenue</option>
+          <option value="">Custom...</option>
+        </select>
+        <input type="text" id="calMetricNameCustom" class="hidden" placeholder="Metric name"
+               style="width:120px;padding:8px 10px">
+        <input type="number" id="calMetricValue" step="any" placeholder="e.g. 2.1"
+               style="width:100px;padding:8px 10px">
+        <input type="text" id="calMetricUnit" value="%" style="width:50px;padding:8px 10px;text-align:center">
+        <div id="calStatus" class="hidden" style="font-size:0.85rem;margin-left:4px"></div>
+      </div>
+      <p style="font-size:0.75rem;color:var(--text2);margin-top:4px">
+        Optional — if set, SGO translates score changes into predicted metric changes.
+      </p>
+    </div>
+
     <details class="mb-8">
       <summary style="cursor:pointer;color:var(--text2);font-size:0.85rem">Panel settings</summary>
       <div style="padding:12px 0">
@@ -456,44 +478,6 @@
         <summary style="cursor:pointer;color:var(--text2);font-size:0.9rem">Full analysis</summary>
         <div class="results-details" id="evalAnalysis"></div>
       </details>
-      <details class="mt-16">
-        <summary style="cursor:pointer;color:var(--text2);font-size:0.9rem">Anchor to a real metric (optional)</summary>
-        <div style="padding:12px 0">
-          <p style="font-size:0.8rem;color:var(--text2);margin-bottom:12px">
-            If you know the actual performance of this entity (e.g. CTR, conversion rate, revenue),
-            SGO can translate score changes into predicted metric changes.
-          </p>
-          <div style="display:flex;gap:10px;flex-wrap:wrap;align-items:flex-end">
-            <div class="field" style="flex:2;min-width:140px;margin-bottom:0">
-              <label>Metric name</label>
-              <input type="text" id="calMetricName" placeholder="e.g. CTR, conversion rate" value="CTR">
-            </div>
-            <div class="field" style="flex:1;min-width:80px;margin-bottom:0">
-              <label>Current value</label>
-              <input type="number" id="calMetricValue" step="any" placeholder="e.g. 2.1">
-            </div>
-            <div class="field" style="flex:1;min-width:60px;margin-bottom:0">
-              <label>Unit</label>
-              <input type="text" id="calMetricUnit" value="%" style="width:60px">
-            </div>
-            <button class="secondary" onclick="applyCalibration()" style="margin-bottom:0;white-space:nowrap">Apply</button>
-            <button class="secondary" onclick="clearCalibration()" id="calClearBtn" style="margin-bottom:0;display:none;padding:10px 12px;color:var(--red);border-color:var(--red)">Clear</button>
-          </div>
-          <div id="calStatus" class="hidden mt-12" style="font-size:0.85rem"></div>
-          <details id="calMultiAnchor" class="mt-12">
-            <summary style="cursor:pointer;color:var(--text2);font-size:0.8rem">Add more anchors for better calibration</summary>
-            <div style="padding:8px 0">
-              <p style="font-size:0.75rem;color:var(--text2);margin-bottom:8px">
-                With 2+ anchors (from other SGO runs with known metrics), calibration uses
-                Platt scaling instead of linear scaling for better accuracy.
-              </p>
-              <div id="extraAnchors"></div>
-              <button class="secondary" onclick="addAnchorRow()" style="padding:4px 12px;font-size:0.75rem">+ Add anchor</button>
-            </div>
-          </details>
-        </div>
-      </details>
-
       <div class="btn-row mt-16">
         <button onclick="runDirections()">Test what to change next</button>
         <button class="secondary" onclick="goToStep(3)">Check panel realism</button>
@@ -771,6 +755,7 @@ This only works because we have a strong product manager who keeps the backlog p
 
 let sessionId = null;
 let evalResultsData = null;
+let lastGradientData = null;
 
 // LLM credentials — stored only in browser JS memory, never persisted
 let llmApiKey = '';
@@ -1149,6 +1134,11 @@ async function runFullPipeline() {
         document.getElementById('negCount').textContent = d.negative;
         document.getElementById('evalAnalysis').textContent = d.analysis;
         document.getElementById('evalResults').classList.remove('hidden');
+
+        // Auto-apply calibration if user entered a metric value before running eval
+        const calVal = parseFloat(document.getElementById('calMetricValue').value);
+        if (calVal > 0) applyCalibration();
+
         resolve();
       });
 
@@ -1280,6 +1270,7 @@ async function runDirections() {
         }
 
         if (d.calibration) currentCalibration = d.calibration;
+        lastGradientData = {results: d.results, changes: suggestedChanges, ranked: d.ranked};
         renderGradientTable(d.results, suggestedChanges, d.ranked, d.calibrated);
         document.getElementById('gradientText').textContent = d.gradient;
         document.getElementById('changesTested').textContent =
@@ -1347,8 +1338,9 @@ function renderGradientTable(results, changes, ranked, calibrated) {
 
   // Update table header
   const thead = document.querySelector('#gradientTable thead tr');
+  const mn = (currentCalibration && currentCalibration.metric_name) || 'Metric';
   thead.innerHTML = hasCal
-    ? '<th>#</th><th>Change</th><th>Score Impact</th><th>Metric Impact</th><th>Predicted</th><th>Range</th>'
+    ? `<th>#</th><th>Change</th><th>Score</th><th>${esc(mn)} Impact</th><th>Predicted ${esc(mn)}</th><th>Helps</th><th>Hurts</th>`
     : '<th>#</th><th>Change</th><th>Avg Impact</th><th>Range</th><th>Helps</th><th>Hurts</th>';
 
   // Show calibration summary above table
@@ -1392,10 +1384,12 @@ function renderGradientTable(results, changes, ranked, calibrated) {
       calCols = '<td>—</td><td>—</td>';
     }
 
-    const rangeCols = hasCal ? '' : `
-        <td style="color:var(--text2)">${r.min_delta >= 0 ? '+' : ''}${r.min_delta} to +${r.max_delta}</td>
-        <td style="color:var(--green)">${r.positive}</td>
-        <td style="color:var(--red)">${r.negative}</td>`;
+    const rangeCols = hasCal
+      ? `<td style="color:var(--green)">${r.positive}</td>
+         <td style="color:var(--red)">${r.negative}</td>`
+      : `<td style="color:var(--text2)">${r.min_delta >= 0 ? '+' : ''}${r.min_delta} to +${r.max_delta}</td>
+         <td style="color:var(--green)">${r.positive}</td>
+         <td style="color:var(--red)">${r.negative}</td>`;
 
     // Summary row (clickable)
     tbody.innerHTML += `
@@ -1439,7 +1433,7 @@ function renderGradientTable(results, changes, ranked, calibrated) {
 
     tbody.innerHTML += `
       <tr id="${rowId}" class="hidden">
-        <td colspan="${hasCal ? 6 : 6}" style="padding:0;background:var(--bg);border-bottom:2px solid var(--border)">${detailHtml}</td>
+        <td colspan="${hasCal ? 7 : 6}" style="padding:0;background:var(--bg);border-bottom:2px solid var(--border)">${detailHtml}</td>
       </tr>
     `;
   });
@@ -1556,40 +1550,55 @@ function formatMetric(value, unit) {
   return value.toFixed(4) + (unit ? ' ' + unit : '');
 }
 
-function addAnchorRow() {
-  const container = document.getElementById('extraAnchors');
-  const idx = container.children.length;
-  const row = document.createElement('div');
-  row.style.cssText = 'display:flex;gap:8px;align-items:center;margin-bottom:6px';
-  row.innerHTML = `
-    <input type="number" step="any" placeholder="Mean score" style="flex:1;padding:6px;font-size:0.8rem" class="anchor-score">
-    <span style="font-size:0.8rem;color:var(--text2)">=</span>
-    <input type="number" step="any" placeholder="Metric value" style="flex:1;padding:6px;font-size:0.8rem" class="anchor-value">
-    <button class="secondary" onclick="this.parentElement.remove()" style="padding:4px 8px;font-size:0.75rem">x</button>
-  `;
-  container.appendChild(row);
+// Show/hide custom metric name input based on dropdown
+document.getElementById('calMetricName').addEventListener('change', function() {
+  const custom = document.getElementById('calMetricNameCustom');
+  if (this.value === '') {
+    custom.classList.remove('hidden');
+    custom.focus();
+  } else {
+    custom.classList.add('hidden');
+  }
+  applyCalibration();
+});
+
+// Re-apply calibration when value or unit changes (debounced)
+let _calDebounce = null;
+function debouncedApplyCalibration() {
+  clearTimeout(_calDebounce);
+  _calDebounce = setTimeout(() => {
+    const v = parseFloat(document.getElementById('calMetricValue').value);
+    if (v > 0) applyCalibration();
+    else if (currentCalibration) clearCalibration();
+  }, 600);
+}
+document.getElementById('calMetricValue').addEventListener('input', debouncedApplyCalibration);
+document.getElementById('calMetricUnit').addEventListener('input', debouncedApplyCalibration);
+
+function getMetricName() {
+  const sel = document.getElementById('calMetricName').value;
+  if (sel === '') return document.getElementById('calMetricNameCustom').value.trim() || 'metric';
+  return sel;
+}
+
+function getMeanScore() {
+  const valid = (evalResultsData || []).filter(r => r && typeof r.score === 'number');
+  if (!valid.length) return null;
+  return valid.reduce((s, r) => s + r.score, 0) / valid.length;
 }
 
 async function applyCalibration() {
-  if (!sessionId) return alert('Run evaluation first.');
-  const metricName = document.getElementById('calMetricName').value.trim() || 'metric';
+  if (!sessionId) return;
+  const metricName = getMetricName();
   const metricValue = parseFloat(document.getElementById('calMetricValue').value);
   const metricUnit = document.getElementById('calMetricUnit').value.trim() || '';
 
-  if (!metricValue || metricValue <= 0) return alert('Enter a positive metric value.');
+  if (!metricValue || metricValue <= 0) return;
 
-  // Get the current mean score from eval results
-  const valid = (evalResultsData || []).filter(r => r && r.score);
-  if (!valid.length) return alert('No evaluation data.');
-  const meanScore = valid.reduce((s, r) => s + r.score, 0) / valid.length;
+  const meanScore = getMeanScore();
+  if (!meanScore || meanScore <= 0) return;
 
-  // Build anchors: current entity + any extra
   const anchors = [{mean_score: meanScore, metric_value: metricValue}];
-  document.querySelectorAll('#extraAnchors > div').forEach(row => {
-    const score = parseFloat(row.querySelector('.anchor-score').value);
-    const value = parseFloat(row.querySelector('.anchor-value').value);
-    if (score > 0 && value > 0) anchors.push({mean_score: score, metric_value: value});
-  });
 
   try {
     const resp = await fetch(`/api/calibrate/${sessionId}`, {
@@ -1602,13 +1611,16 @@ async function applyCalibration() {
 
     currentCalibration = data.calibration;
     const status = document.getElementById('calStatus');
-    const method = anchors.length === 1 ? 'linear scaling' : 'Platt scaling';
-    status.innerHTML = `<span style="color:var(--green)">Calibrated (${esc(method)})</span> — gradient will show ${esc(metricName)} deltas`;
+    status.innerHTML = `<span style="color:var(--green)">Anchored: score ${meanScore.toFixed(1)} = ${metricValue}${esc(metricUnit)} ${esc(metricName)}</span>`;
     status.classList.remove('hidden');
-    document.getElementById('calClearBtn').style.display = '';
+
+    // Re-render gradient table with calibration if it exists
+    if (data.calibrated_gradient && lastGradientData) {
+      renderGradientTable(lastGradientData.results, lastGradientData.changes, lastGradientData.ranked, data.calibrated_gradient);
+    }
   } catch (e) {
     const status = document.getElementById('calStatus');
-    status.innerHTML = `<span style="color:var(--red)">Error: ${esc(e.message)}</span>`;
+    status.innerHTML = `<span style="color:var(--red)">${esc(e.message)}</span>`;
     status.classList.remove('hidden');
   }
 }
@@ -1618,7 +1630,15 @@ async function clearCalibration() {
   await fetch(`/api/calibrate/${sessionId}`, {method: 'DELETE', headers: llmHeaders()});
   currentCalibration = null;
   document.getElementById('calStatus').classList.add('hidden');
-  document.getElementById('calClearBtn').style.display = 'none';
+  document.getElementById('calMetricValue').value = '';
+
+  // Re-render gradient table without calibration
+  if (lastGradientData) {
+    renderGradientTable(lastGradientData.results, lastGradientData.changes, lastGradientData.ranked, null);
+  }
+  // Clear summary above gradient table
+  const calSummaryEl = document.getElementById('calSummary');
+  if (calSummaryEl) calSummaryEl.classList.add('hidden');
 }
 
 // ── Download report ──