Publish the undertow interface

README.md

@@ -1,10 +1,32 @@
 ---
-title: Undertow
-emoji: 👀
-colorFrom: green
-colorTo: indigo
+title: undertow
+emoji: 🌊
+colorFrom: blue
+colorTo: gray
 sdk: static
 pinned: false
+license: mit
+short_description: Four sentiment corpora, every model tested on all four
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# undertow
+
+A sentiment classifier reports one number: accuracy on a test set drawn from the same
+corpus it trained on. **undertow** is about the other twelve.
+
+Four corpora — IMDB, SST-2, Yelp, Amazon — two classifiers, and every model evaluated on
+every domain. A bag of words and a linear probe on frozen DistilBERT features are nearly
+tied on the test set everyone reports (86.0% vs 87.1%). On the three domains nobody
+reports, the probe wins by 3.5 points and carries 29% less domain-shift penalty. Picking
+the method on in-domain accuracy gets it wrong on two of the four corpora.
+
+This page runs the TF-IDF arm live in your browser: type a review and the four models,
+identical in every way except which 6,000 reviews they read, each give you a verdict.
+Nothing is sent anywhere and no answer is precomputed — the JavaScript is a tested port of
+the Python original, held to it at 1e-9.
+
+Corpora: [IMDB](https://ai.stanford.edu/~amaas/data/sentiment/) (Maas et al., 2011),
+[SST-2](https://gluebenchmark.com/tasks) (Socher et al., 2013), Yelp and Amazon polarity
+(Zhang et al., 2015).
+
+Method, controls and the full write-up: <https://github.com/UsmarHaider/undertow>

data.json

@@ -0,0 +1 @@
+{"domains":[{"key":"imdb","label":"IMDB","topic":"movies","length":"long","blurb":"Full-length movie reviews from IMDB (Maas et al., 2011)."},{"key":"sst2","label":"SST-2","topic":"movies","length":"short","blurb":"Single-sentence movie-review snippets from the Stanford Sentiment Treebank."},{"key":"yelp","label":"Yelp","topic":"commerce","length":"long","blurb":"Business and restaurant reviews from the Yelp polarity set (Zhang et al., 2015)."},{"key":"amazon","label":"Amazon","topic":"commerce","length":"long","blurb":"Product reviews from the Amazon polarity set (Zhang et al., 2015)."}],"methods":[{"key":"tfidf","label":"TF-IDF + LR","blurb":"No pretraining at all: word/char n-grams into logistic regression. The control."},{"key":"probe","label":"Linear probe","blurb":"Encoder frozen at its pretrained weights; only a linear head is trained."},{"key":"ft","label":"Fine-tune","blurb":"Everything trained end to end from a randomly initialised head."},{"key":"lpft","label":"LP-FT","blurb":"Probe first, then fine-tune everything starting from that head (Kumar et al., 2022)."}],"grid":[{"method":"tfidf","train":"imdb","eval":"imdb","accuracy":0.89,"auroc":0.95548,"oracle_accuracy":0.8925,"ece":0.04523,"positive_rate":0.5145,"n":4000},{"method":"tfidf","train":"imdb","eval":"sst2","accuracy":0.75584,"auroc":0.83595,"oracle_accuracy":0.76168,"ece":0.05019,"positive_rate":0.56425,"n":856},{"method":"tfidf","train":"imdb","eval":"yelp","accuracy":0.83225,"auroc":0.9174,"oracle_accuracy":0.83475,"ece":0.04099,"positive_rate":0.50475,"n":4000},{"method":"tfidf","train":"imdb","eval":"amazon","accuracy":0.815,"auroc":0.90786,"oracle_accuracy":0.8245,"ece":0.03108,"positive_rate":0.541,"n":4000},{"method":"probe","train":"imdb","eval":"imdb","accuracy":0.85325,"auroc":0.92651,"oracle_accuracy":0.854,"ece":0.01842,"positive_rate":0.49425,"n":4000},{"method":"probe","train":"imdb","eval":"sst2","accuracy":0.81659,"auroc":0.89228,"oracle_accuracy":0.82944,"ece":0.08696,"positive_rate":0.54556,"n":856},{"method":"probe","train":"imdb","eval":"yelp","accuracy":0.776,"auroc":0.92162,"oracle_accuracy":0.848,"ece":0.06786,"positive_rate":0.6575,"n":4000},{"method":"probe","train":"imdb","eval":"amazon","accuracy":0.8245,"auroc":0.92313,"oracle_accuracy":0.8445,"ece":0.03808,"positive_rate":0.572,"n":4000},{"method":"tfidf","train":"sst2","eval":"imdb","accuracy":0.74075,"auroc":0.87223,"oracle_accuracy":0.797,"ece":0.03908,"positive_rate":0.31825,"n":4000},{"method":"tfidf","train":"sst2","eval":"sst2","accuracy":0.75935,"auroc":0.85429,"oracle_accuracy":0.77453,"ece":0.04403,"positive_rate":0.51402,"n":856},{"method":"tfidf","train":"sst2","eval":"yelp","accuracy":0.744,"auroc":0.85866,"oracle_accuracy":0.77525,"ece":0.02788,"positive_rate":0.3205,"n":4000},{"method":"tfidf","train":"sst2","eval":"amazon","accuracy":0.7315,"auroc":0.84198,"oracle_accuracy":0.766,"ece":0.03088,"positive_rate":0.344,"n":4000},{"method":"probe","train":"sst2","eval":"imdb","accuracy":0.803,"auroc":0.89812,"oracle_accuracy":0.81725,"ece":0.04895,"positive_rate":0.3975,"n":4000},{"method":"probe","train":"sst2","eval":"sst2","accuracy":0.84112,"auroc":0.91053,"oracle_accuracy":0.84112,"ece":0.02411,"positive_rate":0.48364,"n":856},{"method":"probe","train":"sst2","eval":"yelp","accuracy":0.85325,"auroc":0.94071,"oracle_accuracy":0.868,"ece":0.05162,"positive_rate":0.42675,"n":4000},{"method":"probe","train":"sst2","eval":"amazon","accuracy":0.8165,"auroc":0.91471,"oracle_accuracy":0.83075,"ece":0.01821,"positive_rate":0.3815,"n":4000},{"method":"tfidf","train":"yelp","eval":"imdb","accuracy":0.77325,"auroc":0.88132,"oracle_accuracy":0.79675,"ece":0.05488,"positive_rate":0.38825,"n":4000},{"method":"tfidf","train":"yelp","eval":"sst2","accuracy":0.68575,"auroc":0.75413,"oracle_accuracy":0.69393,"ece":0.03664,"positive_rate":0.58294,"n":856},{"method":"tfidf","train":"yelp","eval":"yelp","accuracy":0.9145,"auroc":0.97352,"oracle_accuracy":0.9165,"ece":0.0887,"positive_rate":0.496,"n":4000},{"method":"tfidf","train":"yelp","eval":"amazon","accuracy":0.8345,"auroc":0.91429,"oracle_accuracy":0.83775,"ece":0.0922,"positive_rate":0.4695,"n":4000},{"method":"probe","train":"yelp","eval":"imdb","accuracy":0.775,"auroc":0.85999,"oracle_accuracy":0.779,"ece":0.0705,"positive_rate":0.4375,"n":4000},{"method":"probe","train":"yelp","eval":"sst2","accuracy":0.76519,"auroc":0.84705,"oracle_accuracy":0.77103,"ece":0.12793,"positive_rate":0.51986,"n":856},{"method":"probe","train":"yelp","eval":"yelp","accuracy":0.90625,"auroc":0.9663,"oracle_accuracy":0.90625,"ece":0.01732,"positive_rate":0.48925,"n":4000},{"method":"probe","train":"yelp","eval":"amazon","accuracy":0.84625,"auroc":0.93124,"oracle_accuracy":0.8565,"ece":0.03564,"positive_rate":0.52975,"n":4000},{"method":"tfidf","train":"amazon","eval":"imdb","accuracy":0.8315,"auroc":0.91761,"oracle_accuracy":0.836,"ece":0.08032,"positive_rate":0.466,"n":4000},{"method":"tfidf","train":"amazon","eval":"sst2","accuracy":0.71612,"auroc":0.79321,"oracle_accuracy":0.72196,"ece":0.04625,"positive_rate":0.57126,"n":856},{"method":"tfidf","train":"amazon","eval":"yelp","accuracy":0.865,"auroc":0.94176,"oracle_accuracy":0.86775,"ece":0.09825,"positive_rate":0.495,"n":4000},{"method":"tfidf","train":"amazon","eval":"amazon","accuracy":0.87675,"auroc":0.94913,"oracle_accuracy":0.88,"ece":0.09483,"positive_rate":0.49475,"n":4000},{"method":"probe","train":"amazon","eval":"imdb","accuracy":0.82,"auroc":0.90316,"oracle_accuracy":0.82275,"ece":0.01495,"positive_rate":0.47,"n":4000},{"method":"probe","train":"amazon","eval":"sst2","accuracy":0.79089,"auroc":0.88588,"oracle_accuracy":0.81659,"ece":0.08982,"positive_rate":0.55257,"n":856},{"method":"probe","train":"amazon","eval":"yelp","accuracy":0.86275,"auroc":0.94988,"oracle_accuracy":0.87675,"ece":0.01488,"positive_rate":0.55825,"n":4000},{"method":"probe","train":"amazon","eval":"amazon","accuracy":0.88425,"auroc":0.95476,"oracle_accuracy":0.88725,"ece":0.01302,"positive_rate":0.48875,"n":4000}],"report":{"methods":[{"method":"tfidf","id_accuracy":0.8601489485981308,"ood_accuracy":0.7771216900311527,"ood_gap":0.08302725856697812,"id_auroc":0.9331054482869683,"ood_auroc":0.8697007361605453,"ood_oracle_accuracy":0.7927766744548287,"id_ece":0.06819728512338892,"ood_ece":0.05238538153535782,"total_drop":0.08302725856697812,"ranking_drop":0.06737227414330216,"placement_drop":0.015654984423675966},{"method":"probe","id_accuracy":0.8712178738317757,"ood_accuracy":0.8124927959501558,"ood_gap":0.05872507788161996,"id_auroc":0.9395254573161411,"ood_auroc":0.9056478458908347,"ood_oracle_accuracy":0.830046339563863,"id_ece":0.018215950827522174,"ood_ece":0.0554498978555863,"total_drop":0.05872507788161996,"ranking_drop":0.04117153426791276,"placement_drop":0.0175535436137072}],"headline":{"primary_method":"probe","tfidf_id_accuracy":0.8601489485981308,"tfidf_ood_accuracy":0.7771216900311527,"tfidf_ood_gap":0.08302725856697812,"tfidf_ranking_drop":0.06737227414330216,"tfidf_placement_drop":0.015654984423675966,"tfidf_placement_share":0.1885523464688056,"probe_id_accuracy":0.8712178738317757,"probe_ood_accuracy":0.8124927959501558,"probe_ood_gap":0.05872507788161996,"probe_ranking_drop":0.04117153426791276,"probe_placement_drop":0.0175535436137072,"probe_placement_share":0.2989105208015601},"shift_axes":[{"method":"probe","shift":"topic-different/length-different","accuracy":0.806456191588785,"n":4},{"method":"probe","shift":"topic-different/length-same","accuracy":0.798875,"n":4},{"method":"probe","shift":"topic-same/length-different","accuracy":0.8097943925233645,"n":2},{"method":"probe","shift":"topic-same/length-same","accuracy":0.8545,"n":2},{"method":"tfidf","shift":"topic-different/length-different","accuracy":0.7193422897196262,"n":4},{"method":"tfidf","shift":"topic-different/length-same","accuracy":0.813,"n":4},{"method":"tfidf","shift":"topic-same/length-different","accuracy":0.7482955607476636,"n":2},{"method":"tfidf","shift":"topic-same/length-same","accuracy":0.84975,"n":2}],"mechanism":[{"method":"probe","scope":"id","cka_vs_pretrained":1.0,"posthoc_probe_accuracy":null,"pretrained_probe_accuracy":null,"probe_accuracy_delta":0.0,"n":4},{"method":"probe","scope":"ood","cka_vs_pretrained":1.0,"posthoc_probe_accuracy":null,"pretrained_probe_accuracy":null,"probe_accuracy_delta":0.0,"n":12}],"selection_regret":{"rows":[{"train_domain":"imdb","chosen_on_id":"tfidf","chosen_id_accuracy":0.89,"chosen_ood_accuracy":0.8010303738317757,"best_ood_method":"probe","best_ood_accuracy":0.8056962616822431,"regret":0.004665887850467398,"agrees":false},{"train_domain":"sst2","chosen_on_id":"probe","chosen_id_accuracy":0.8411214953271028,"chosen_ood_accuracy":0.82425,"best_ood_method":"probe","best_ood_accuracy":0.82425,"regret":0.0,"agrees":true},{"train_domain":"yelp","chosen_on_id":"tfidf","chosen_id_accuracy":0.9145,"chosen_ood_accuracy":0.7644992211838005,"best_ood_method":"probe","best_ood_accuracy":0.7954789719626169,"regret":0.030979750778816317,"agrees":false},{"train_domain":"amazon","chosen_on_id":"probe","chosen_id_accuracy":0.88425,"chosen_ood_accuracy":0.8245459501557632,"best_ood_method":"probe","best_ood_accuracy":0.8245459501557632,"regret":0.0,"agrees":true}],"disagreements":2,"mean_regret":0.008911409657320929,"max_regret":0.030979750778816317},"checkpoint_regret":{"rows":[],"mean_regret":null,"runs_where_earlier_checkpoint_was_better":0,"runs":0},"ft_vs_lpft":{"baseline":"ft","challenger":"lpft","id_mean_delta":null,"ood_mean_delta":null,"ood_cells":0,"ood_wins":0,"cells":[]},"probe_vs_ft":{"baseline":"probe","challenger":"ft","id_mean_delta":null,"ood_mean_delta":null,"ood_cells":0,"ood_wins":0,"cells":[]},"tfidf_vs_ft":{"baseline":"tfidf","challenger":"ft","id_mean_delta":null,"ood_mean_delta":null,"ood_cells":0,"ood_wins":0,"cells":[]}},"examples":[{"label":"Restaurant","text":"The service was slow but the pasta was worth the wait."},{"label":"Product","text":"Battery lasted about two days on a charge and the case feels solid after a month of being thrown in a bag. Setup took under five minutes."},{"label":"Film","text":"The premise is wasted on a script that explains every feeling out loud, and even the cast look bored by the third act. A long two hours."},{"label":"One-liner","text":"a quietly devastating performance that lingers long after the credits"}]}

index.html

@@ -1,19 +1,304 @@
 <!doctype html>
-<html>
-	<head>
-		<meta charset="utf-8" />
-		<meta name="viewport" content="width=device-width" />
-		<title>My static Space</title>
-		<link rel="stylesheet" href="style.css" />
-	</head>
-	<body>
-		<div class="card">
-			<h1>Welcome to your static Space!</h1>
-			<p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
-			<p>
-				Also don't forget to check the
-				<a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
-			</p>
-		</div>
-	</body>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>undertow — what fine-tuning costs you off-distribution</title>
+<meta name="description" content="Four sentiment models, four corpora, one architecture. Type a review and watch them disagree.">
+<style>
+:root {
+  color-scheme: light dark;
+  --blue: #2a78d6;
+  --orange: #eb6834;
+  --green: #2f9e6f;
+  --slate: #4a5568;
+  --grey: #9aa5b1;
+
+  --bg: #fbfcfd;
+  --panel: #ffffff;
+  --ink: #17212b;
+  --ink-soft: #55636f;
+  --ink-faint: #8593a0;
+  --line: #e2e7ec;
+  --shadow: 0 1px 2px rgba(16, 32, 48, .06), 0 8px 24px rgba(16, 32, 48, .05);
+  --heat: 42, 120, 214;
+}
+@media (prefers-color-scheme: dark) {
+  :root {
+    --bg: #10161c;
+    --panel: #171f27;
+    --ink: #e7edf3;
+    --ink-soft: #a3b1bf;
+    --ink-faint: #6f8090;
+    --line: #26313c;
+    --shadow: 0 1px 2px rgba(0, 0, 0, .3), 0 10px 30px rgba(0, 0, 0, .25);
+  }
+}
+* { box-sizing: border-box; }
+html { -webkit-text-size-adjust: 100%; }
+body {
+  margin: 0;
+  background: var(--bg);
+  color: var(--ink);
+  font: 15px/1.6 ui-sans-serif, -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+  font-variant-numeric: tabular-nums;
+}
+.wrap { max-width: 1080px; margin: 0 auto; padding: 0 20px 72px; }
+
+/* ------------------------------------------------------------------------- header */
+header.top { padding: 56px 0 28px; border-bottom: 1px solid var(--line); margin-bottom: 34px; }
+.eyebrow {
+  font-size: 12px; letter-spacing: .14em; text-transform: uppercase;
+  color: var(--ink-faint); margin: 0 0 10px;
+}
+h1 { font-size: clamp(34px, 6vw, 50px); line-height: 1.04; margin: 0 0 14px; letter-spacing: -.022em; }
+h1 .dim { color: var(--ink-faint); font-weight: 400; }
+.lede { font-size: 17px; color: var(--ink-soft); max-width: 62ch; margin: 0 0 18px; }
+.lede strong { color: var(--ink); font-weight: 600; }
+.top-links { display: flex; flex-wrap: wrap; gap: 8px; }
+.top-links a {
+  font-size: 13px; text-decoration: none; color: var(--ink-soft);
+  border: 1px solid var(--line); border-radius: 999px; padding: 5px 13px; background: var(--panel);
+}
+.top-links a:hover { color: var(--blue); border-color: var(--blue); }
+
+/* ------------------------------------------------------------------------ sections */
+section { margin: 0 0 52px; }
+h2 { font-size: 21px; margin: 0 0 6px; letter-spacing: -.01em; }
+.sub { color: var(--ink-soft); margin: 0 0 20px; max-width: 68ch; }
+.panel {
+  background: var(--panel); border: 1px solid var(--line);
+  border-radius: 14px; padding: 22px; box-shadow: var(--shadow);
+}
+
+/* -------------------------------------------------------------------- the scorer */
+#text {
+  width: 100%; min-height: 116px; resize: vertical; padding: 14px 15px;
+  border: 1px solid var(--line); border-radius: 10px; background: var(--bg);
+  color: var(--ink); font: inherit; line-height: 1.55;
+}
+#text:focus { outline: 2px solid var(--blue); outline-offset: -1px; border-color: transparent; }
+#examples { display: flex; flex-wrap: wrap; gap: 7px; margin: 12px 0 0; }
+.chip {
+  font: inherit; font-size: 13px; cursor: pointer; padding: 5px 12px;
+  border-radius: 999px; border: 1px solid var(--line);
+  background: var(--bg); color: var(--ink-soft);
+}
+.chip:hover { border-color: var(--blue); color: var(--blue); }
+.chip-ghost { color: var(--ink-faint); }
+
+.note { margin: 18px 0 0; padding: 12px 14px; border-radius: 10px; font-size: 14px; line-height: 1.55; }
+.note-idle { background: color-mix(in srgb, var(--grey) 12%, transparent); color: var(--ink-soft); }
+.note-split { background: color-mix(in srgb, var(--orange) 14%, transparent); color: var(--ink); }
+.note-agree { background: color-mix(in srgb, var(--green) 13%, transparent); color: var(--ink); }
+
+#verdicts {
+  display: grid; gap: 14px; margin-top: 18px;
+  grid-template-columns: repeat(auto-fit, minmax(216px, 1fr));
+}
+.verdict {
+  border: 1px solid var(--line); border-radius: 12px; padding: 14px 15px 12px;
+  background: var(--bg); border-top: 3px solid var(--grey);
+}
+.verdict.is-positive { border-top-color: var(--green); }
+.verdict.is-negative { border-top-color: var(--orange); }
+.verdict-head { display: flex; justify-content: space-between; align-items: baseline; gap: 8px; }
+.verdict-domain { font-weight: 650; font-size: 15px; }
+.verdict-meta { font-size: 11px; color: var(--ink-faint); letter-spacing: .03em; }
+.verdict-empty { color: var(--ink-faint); font-size: 13px; margin: 14px 0; }
+.verdict-label { margin: 10px 0 8px; font-size: 17px; font-weight: 650; letter-spacing: -.01em; }
+.is-positive .verdict-label { color: var(--green); }
+.is-negative .verdict-label { color: var(--orange); }
+.verdict-prob { font-size: 12px; font-weight: 450; color: var(--ink-faint); letter-spacing: 0; }
+.meter { position: relative; height: 7px; border-radius: 4px; background: color-mix(in srgb, var(--orange) 26%, transparent); overflow: hidden; }
+.meter-fill { height: 100%; background: var(--green); border-radius: 4px 0 0 4px; }
+/* The decision boundary. Without a visible mark at 0.5 the bar reads as a generic
+   progress meter rather than "which side of the threshold did this land on". */
+.meter-mid { position: absolute; left: 50%; top: -3px; bottom: -3px; width: 2px; margin-left: -1px; background: var(--ink); opacity: .55; border-radius: 1px; }
+.terms { list-style: none; margin: 12px 0 0; padding: 0; display: grid; gap: 3px; }
+.term { display: flex; justify-content: space-between; font-size: 12.5px; gap: 10px; }
+.term-text { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; color: var(--ink-soft); }
+.term-weight { font-variant-numeric: tabular-nums; }
+.term-pos .term-weight { color: var(--green); }
+.term-neg .term-weight { color: var(--orange); }
+.verdict-foot { margin: 11px 0 0; font-size: 11.5px; color: var(--ink-faint); }
+
+/* ------------------------------------------------------------------ transfer matrix */
+#method-tabs { display: flex; flex-wrap: wrap; gap: 6px; margin-bottom: 16px; }
+.tab {
+  font: inherit; font-size: 13.5px; cursor: pointer; padding: 6px 14px;
+  border-radius: 8px; border: 1px solid var(--line); background: var(--bg); color: var(--ink-soft);
+}
+.tab:hover { color: var(--ink); }
+.tab.is-active { background: var(--blue); border-color: var(--blue); color: #fff; }
+.matrix-scroll { overflow-x: auto; }
+#matrix {
+  display: grid; grid-template-columns: minmax(78px, auto) repeat(4, minmax(74px, 1fr));
+  gap: 4px; min-width: 440px;
+}
+.mcell { padding: 11px 8px; border-radius: 7px; text-align: center; font-size: 14px; }
+.mcell-corner { background: none; }
+.mcell-head { font-size: 12px; color: var(--ink-faint); font-weight: 600; letter-spacing: .02em; }
+.mcell-row { text-align: right; padding-right: 12px; display: flex; align-items: center; justify-content: flex-end; }
+.mcell-value { background: rgba(var(--heat), calc(.07 + var(--t) * .82)); color: var(--ink); font-variant-numeric: tabular-nums; }
+.mcell-value.is-dark { color: #fff; }
+.mcell-value.is-diagonal { font-weight: 700; box-shadow: inset 0 0 0 2px var(--ink); }
+.mcell-value.is-diagonal.is-dark { box-shadow: inset 0 0 0 2px #fff; }
+.axis-note { display: flex; justify-content: space-between; font-size: 12px; color: var(--ink-faint); margin-top: 10px; gap: 16px; }
+#matrix-blurb { margin: 14px 0 0; font-size: 13.5px; color: var(--ink-soft); }
+
+/* ---------------------------------------------------------------------- tables */
+table { width: 100%; border-collapse: collapse; font-size: 14px; }
+th, td { text-align: right; padding: 9px 10px; border-bottom: 1px solid var(--line); }
+thead th { font-size: 12px; color: var(--ink-faint); font-weight: 600; letter-spacing: .02em; }
+tbody th, thead th:first-child { text-align: left; font-weight: 600; }
+tbody tr:last-child th, tbody tr:last-child td { border-bottom: none; }
+.num-gap { color: var(--orange); font-weight: 600; }
+.table-scroll { overflow-x: auto; }
+
+/* ----------------------------------------------------------------- decomposition */
+#decomposition { display: grid; gap: 10px; }
+.decomp-row { display: grid; grid-template-columns: 96px 1fr 74px; align-items: center; gap: 12px; }
+.decomp-label { font-size: 13.5px; color: var(--ink-soft); }
+.decomp-bar { display: flex; height: 20px; border-radius: 5px; overflow: hidden; background: color-mix(in srgb, var(--grey) 14%, transparent); }
+.seg { height: 100%; }
+.seg-ranking { background: var(--orange); }
+.seg-placement { background: var(--blue); }
+.decomp-total { font-size: 13px; text-align: right; color: var(--ink-soft); font-variant-numeric: tabular-nums; }
+.legend { display: flex; gap: 18px; flex-wrap: wrap; margin-top: 16px; font-size: 12.5px; color: var(--ink-soft); }
+.legend span { display: inline-flex; align-items: center; gap: 7px; }
+.swatch { width: 11px; height: 11px; border-radius: 3px; }
+
+footer {
+  border-top: 1px solid var(--line); padding-top: 22px; margin-top: 8px;
+  font-size: 13px; color: var(--ink-faint); line-height: 1.7;
+}
+footer a { color: var(--ink-soft); }
+#loading { padding: 60px 0; color: var(--ink-faint); }
+</style>
+</head>
+<body>
+<div class="wrap">
+
+<header class="top">
+  <p class="eyebrow">Distribution shift · DistilBERT · four sentiment corpora</p>
+  <h1>undertow<span class="dim"> — the accuracy that isn't there</span></h1>
+  <p class="lede">
+    A sentiment classifier reports one number: accuracy on a test set drawn from the corpus
+    it trained on. This page is about the other twelve. Four corpora, two classifiers — a bag
+    of words, and a linear probe on frozen DistilBERT features — and every model evaluated on
+    every domain. Pretraining is worth <strong>+1.1 pp</strong> on the test set everyone
+    reports. On the three domains nobody reports it is worth <strong>+3.5 pp</strong>, and it
+    cuts the domain-shift penalty by <strong>29%</strong>.
+  </p>
+  <div class="top-links">
+    <a href="https://github.com/UsmarHaider/undertow">Method &amp; code on GitHub</a>
+  </div>
+</header>
+
+<p id="loading">Loading the models…</p>
+
+<main id="app" hidden>
+
+  <section>
+    <h2>Four models, one sentence</h2>
+    <p class="sub">
+      These are the real fitted TF-IDF models from the run below — one per corpus, identical
+      settings, running in your browser with nothing sent anywhere. Same architecture, same
+      hyper-parameters, same 6,000 training reviews each. The only thing that differs is
+      <em>which</em> 6,000.
+    </p>
+    <div class="panel">
+      <textarea id="text" placeholder="Paste or type a review…" spellcheck="false"></textarea>
+      <div id="examples"></div>
+      <p id="disagreement" class="note note-idle"></p>
+      <div id="verdicts"></div>
+    </div>
+  </section>
+
+  <section>
+    <h2>The transfer matrix</h2>
+    <p class="sub">
+      Row is what a model trained on; column is what it was tested on. The boxed diagonal is
+      the number that gets published. Everything off it is the number you get in production.
+      Hover any cell for its AUROC, its calibration, and how far a moved threshold would take it.
+    </p>
+    <div class="panel">
+      <div id="method-tabs"></div>
+      <div class="matrix-scroll"><div id="matrix"></div></div>
+      <div class="axis-note"><span>rows: trained on</span><span>columns: evaluated on</span></div>
+      <p id="matrix-blurb"></p>
+    </div>
+  </section>
+
+  <section>
+    <h2>In-domain versus everywhere else</h2>
+    <p class="sub">
+      Averaged over the four training domains — sixteen cells per method, four on the diagonal
+      and twelve off it. <em>Oracle</em> is the accuracy each model would reach out-of-domain if
+      someone re-placed its decision threshold and changed nothing else: the ceiling that costs
+      no retraining.
+    </p>
+    <div class="panel table-scroll">
+      <table>
+        <thead>
+          <tr>
+            <th>method</th><th>in-domain</th><th>out-of-domain</th><th>drop</th>
+            <th>OOD AUROC</th><th>OOD oracle</th><th>OOD ECE</th>
+          </tr>
+        </thead>
+        <tbody id="summary-body"></tbody>
+      </table>
+    </div>
+  </section>
+
+  <section>
+    <h2>What the drop is made of</h2>
+    <p class="sub">
+      The loss splits cleanly in two. <strong>Ranking</strong> is signal the representation
+      genuinely no longer has — no threshold recovers it. <strong>Placement</strong> is signal
+      that is still there, sitting on the wrong side of a boundary that was chosen in a
+      different domain.
+    </p>
+    <div class="panel">
+      <div id="decomposition"></div>
+      <div class="legend">
+        <span><i class="swatch" style="background: var(--orange)"></i> ranking — genuinely lost</span>
+        <span><i class="swatch" style="background: var(--blue)"></i> placement — a misplaced threshold</span>
+      </div>
+    </div>
+  </section>
+
+  <section>
+    <h2>Which shift hurts</h2>
+    <p class="sub">
+      The four corpora form a rough 2×2 — movies against commerce, full reviews against
+      one-liners — so the twelve out-of-domain cells can be grouped by what kind of distance
+      each one actually crosses.
+    </p>
+    <div class="panel table-scroll">
+      <table>
+        <thead><tr id="shift-head"></tr></thead>
+        <tbody id="shift-body"></tbody>
+      </table>
+    </div>
+  </section>
+
+  <footer>
+    <p>
+      Corpora: <a href="https://ai.stanford.edu/~amaas/data/sentiment/">IMDB</a> (Maas et al., 2011),
+      <a href="https://gluebenchmark.com/tasks">SST-2</a> (Socher et al., 2013),
+      Yelp and Amazon polarity (Zhang et al., 2015). Encoder: distilbert-base-uncased.
+      Every number on this page is regenerated by <code>make all</code>.
+    </p>
+    <p>
+      Built by <a href="https://github.com/UsmarHaider">Usmar Haider</a> ·
+      <a href="https://github.com/UsmarHaider/undertow">source, method and full write-up</a> · MIT
+    </p>
+  </footer>
+</main>
+
+</div>
+<script src="undertow.js"></script>
+</body>
 </html>

undertow.js

@@ -0,0 +1,374 @@
+/* undertow -- the browser half.
+ *
+ * Everything here runs on the visitor's machine. The four TF-IDF models are the real
+ * fitted models from `make grid`, shipped as JSON, and this file is a line-for-line port
+ * of `undertow/tfidf.py`: same tokeniser, same sublinear TF, same smooth IDF, same L2
+ * norm, same linear margin. A test in the suite runs this file under Node against the
+ * Python implementation on the same strings and requires agreement to 1e-9, so the four
+ * verdicts you see are the four verdicts the paper's tables were computed from.
+ *
+ * It is a port rather than an API call on purpose: the point of the page is that four
+ * models trained on four corpora disagree about one sentence, and that is only convincing
+ * if you can type your own sentence and get an answer with no round trip to anywhere.
+ */
+
+'use strict';
+
+/* sklearn's default analyzer is `(?u)\b\w\w+\b` over the lowercased string. JavaScript's
+ * `\w` is ASCII-only, so the Unicode property escapes are spelled out to match Python. */
+const TOKEN_RE = /(?<![\p{L}\p{N}_])[\p{L}\p{N}_]{2,}(?![\p{L}\p{N}_])/gu;
+
+function tokenizeWords(text) {
+  return String(text).toLowerCase().match(TOKEN_RE) || [];
+}
+
+function ngrams(tokens, lo, hi) {
+  const out = lo <= 1 ? tokens.slice() : [];
+  for (let n = Math.max(2, lo); n <= hi; n += 1) {
+    for (let i = 0; i + n <= tokens.length; i += 1) {
+      out.push(tokens.slice(i, i + n).join(' '));
+    }
+  }
+  return out;
+}
+
+class TfidfModel {
+  constructor(payload) {
+    this.terms = payload.terms;
+    this.idf = payload.idf;
+    this.coef = payload.coef;
+    this.intercept = payload.intercept;
+    this.ngramRange = payload.ngram_range || [1, 2];
+    this.vocabulary = new Map();
+    for (let i = 0; i < this.terms.length; i += 1) this.vocabulary.set(this.terms[i], i);
+  }
+
+  /* Sparse by construction: a review touches a few hundred of thirty thousand columns,
+   * so the dense 30k vector of the Python version is never materialised here. */
+  transformOne(text) {
+    const counts = new Map();
+    const grams = ngrams(tokenizeWords(text), this.ngramRange[0], this.ngramRange[1]);
+    for (const gram of grams) {
+      const idx = this.vocabulary.get(gram);
+      if (idx !== undefined) counts.set(idx, (counts.get(idx) || 0) + 1);
+    }
+    const entries = [];
+    let sumsq = 0;
+    for (const [idx, count] of counts) {
+      const value = (1 + Math.log(count)) * this.idf[idx];
+      sumsq += value * value;
+      entries.push([idx, value]);
+    }
+    const norm = Math.sqrt(sumsq);
+    if (norm > 0) for (const e of entries) e[1] /= norm;
+    return entries;
+  }
+
+  score(text) {
+    const entries = this.transformOne(text);
+    let margin = this.intercept;
+    const contributions = [];
+    for (const [idx, value] of entries) {
+      const contribution = value * this.coef[idx];
+      margin += contribution;
+      contributions.push({ term: this.terms[idx], weight: contribution });
+    }
+    contributions.sort((a, b) => Math.abs(b.weight) - Math.abs(a.weight));
+    return {
+      margin,
+      probability: 1 / (1 + Math.exp(-margin)),
+      label: margin > 0 ? 'positive' : 'negative',
+      matched: entries.length,
+      terms: contributions.slice(0, 12),
+    };
+  }
+}
+
+/* ------------------------------------------------------------------ Node parity hook */
+if (typeof module !== 'undefined' && module.exports) {
+  module.exports = { TfidfModel, tokenizeWords, ngrams };
+}
+
+/* --------------------------------------------------------------------------- the page */
+if (typeof document !== 'undefined') {
+  const state = { data: null, models: {}, method: 'ft' };
+  const $ = (sel) => document.querySelector(sel);
+  const el = (tag, cls, text) => {
+    const node = document.createElement(tag);
+    if (cls) node.className = cls;
+    if (text !== undefined) node.textContent = text;
+    return node;
+  };
+  const pct = (x) => `${(x * 100).toFixed(1)}%`;
+  const pp = (x) => `${x >= 0 ? '+' : ''}${(x * 100).toFixed(1)} pp`;
+
+  async function boot() {
+    const [data, models] = await Promise.all([
+      fetch('data.json').then((r) => r.json()),
+      fetch('models.json').then((r) => r.json()),
+    ]);
+    state.data = data;
+    for (const [key, payload] of Object.entries(models)) {
+      state.models[key] = new TfidfModel(payload);
+    }
+
+    /* ?text= and ?method= make a particular view of the page linkable, which is also how
+     * the README screenshots are captured reproducibly rather than by hand. */
+    const params = new URLSearchParams(window.location.search);
+    if (params.has('text')) $('#text').value = params.get('text');
+    if (params.has('example')) {
+      const wanted = params.get('example').toLowerCase();
+      const match = data.examples.find((e) => e.label.toLowerCase() === wanted);
+      if (match) $('#text').value = match.text;
+    }
+    if (params.has('method') && data.methods.some((m) => m.key === params.get('method'))) {
+      state.method = params.get('method');
+    }
+
+    renderExamples();
+    renderScorer();
+    renderMethodTabs();
+    renderMatrix();
+    renderSummary();
+    renderDecomposition();
+    renderShiftAxes();
+    $('#loading').remove();
+    $('#app').hidden = false;
+  }
+
+  /* --------------------------------------------------------------- the live scorer */
+
+  function renderExamples() {
+    const box = $('#examples');
+    for (const example of state.data.examples) {
+      const chip = el('button', 'chip', example.label);
+      chip.type = 'button';
+      chip.addEventListener('click', () => {
+        $('#text').value = example.text;
+        renderScorer();
+      });
+      box.append(chip);
+    }
+    const clear = el('button', 'chip chip-ghost', 'Clear');
+    clear.type = 'button';
+    clear.addEventListener('click', () => {
+      $('#text').value = '';
+      renderScorer();
+    });
+    box.append(clear);
+  }
+
+  function renderScorer() {
+    const text = $('#text').value.trim();
+    const box = $('#verdicts');
+    box.textContent = '';
+    const labels = {};
+    for (const domain of state.data.domains) {
+      const model = state.models[domain.key];
+      if (!model) continue;
+      const result = text ? model.score(text) : null;
+      labels[domain.key] = result ? result.label : null;
+      box.append(verdictCard(domain, result));
+    }
+    const seen = new Set(Object.values(labels).filter(Boolean));
+    const note = $('#disagreement');
+    if (!text) {
+      note.className = 'note note-idle';
+      note.textContent =
+        'Type a review, or pick one of the examples. Each card is a logistic model over '
+        + 'TF-IDF n-grams, trained on 6,000 reviews from one corpus and nothing else.';
+    } else if (seen.size > 1) {
+      const split = Object.entries(labels)
+        .filter(([, v]) => v)
+        .map(([k, v]) => `${state.data.domains.find((d) => d.key === k).label} says ${v}`)
+        .join(', ');
+      note.className = 'note note-split';
+      note.textContent = `The four models disagree. ${split}. Same sentence, same architecture — the only difference is which corpus each one read.`;
+    } else {
+      note.className = 'note note-agree';
+      note.textContent =
+        `All four agree this is ${[...seen][0]}. Agreement is the common case; the transfer `
+        + 'matrix below is where the disagreement shows up at scale.';
+    }
+  }
+
+  function verdictCard(domain, result) {
+    const card = el('article', 'verdict');
+    const head = el('header', 'verdict-head');
+    head.append(el('span', 'verdict-domain', domain.label));
+    head.append(el('span', 'verdict-meta', `${domain.topic} · ${domain.length}`));
+    card.append(head);
+
+    if (!result) {
+      card.append(el('p', 'verdict-empty', 'waiting for text'));
+      return card;
+    }
+
+    const positive = result.label === 'positive';
+    card.classList.add(positive ? 'is-positive' : 'is-negative');
+    const verdict = el('p', 'verdict-label', positive ? 'positive' : 'negative');
+    verdict.append(el('span', 'verdict-prob', ` ${pct(positive ? result.probability : 1 - result.probability)} confident`));
+    card.append(verdict);
+
+    const meter = el('div', 'meter');
+    const fill = el('div', 'meter-fill');
+    fill.style.width = `${Math.min(100, Math.max(0, result.probability * 100))}%`;
+    meter.append(fill);
+    const mid = el('div', 'meter-mid');
+    meter.append(mid);
+    card.append(meter);
+
+    const terms = el('ul', 'terms');
+    for (const t of result.terms.slice(0, 6)) {
+      const item = el('li', t.weight > 0 ? 'term term-pos' : 'term term-neg');
+      item.append(el('span', 'term-text', t.term));
+      item.append(el('span', 'term-weight', t.weight.toFixed(3)));
+      terms.append(item);
+    }
+    card.append(terms);
+    card.append(el('p', 'verdict-foot', `${result.matched} of its ${state.models[domain.key].terms.length.toLocaleString()} n-grams matched`));
+    return card;
+  }
+
+  /* ------------------------------------------------------------- the transfer matrix */
+
+  function renderMethodTabs() {
+    const box = $('#method-tabs');
+    for (const method of state.data.methods) {
+      if (!state.data.grid.some((r) => r.method === method.key)) continue;
+      const tab = el('button', 'tab', method.label);
+      tab.type = 'button';
+      tab.dataset.method = method.key;
+      tab.addEventListener('click', () => {
+        state.method = method.key;
+        renderMethodTabs();
+        renderMatrix();
+      });
+      if (method.key === state.method) tab.classList.add('is-active');
+      box.append(tab);
+    }
+    if (box.children.length > state.data.methods.length) return;
+  }
+
+  function renderMatrix() {
+    const box = $('#matrix');
+    box.textContent = '';
+    const domains = state.data.domains;
+    const cells = state.data.grid.filter((r) => r.method === state.method);
+    if (!cells.length) return;
+    const values = cells.map((c) => c.accuracy);
+    const lo = Math.min(...values);
+    const hi = Math.max(...values);
+
+    box.append(el('div', 'mcell mcell-corner', ''));
+    for (const d of domains) box.append(el('div', 'mcell mcell-head', d.label));
+
+    for (const train of domains) {
+      box.append(el('div', 'mcell mcell-head mcell-row', train.label));
+      for (const ev of domains) {
+        const cell = cells.find((c) => c.train === train.key && c.eval === ev.key);
+        const node = el('div', 'mcell mcell-value');
+        if (!cell) {
+          node.textContent = '–';
+          box.append(node);
+          continue;
+        }
+        const t = hi > lo ? (cell.accuracy - lo) / (hi - lo) : 0.5;
+        node.style.setProperty('--t', t.toFixed(3));
+        if (t > 0.62) node.classList.add('is-dark');
+        if (train.key === ev.key) node.classList.add('is-diagonal');
+        node.append(el('span', 'mcell-acc', pct(cell.accuracy)));
+        node.title =
+          `${state.method}: trained on ${train.label}, evaluated on ${ev.label}\n`
+          + `accuracy ${pct(cell.accuracy)} · AUROC ${pct(cell.auroc)}\n`
+          + `best reachable by moving the threshold ${pct(cell.oracle_accuracy)}\n`
+          + `predicts positive ${pct(cell.positive_rate)} of the time (truth is 50%)`;
+        box.append(node);
+      }
+    }
+    const blurb = state.data.methods.find((m) => m.key === state.method);
+    $('#matrix-blurb').textContent = blurb ? blurb.blurb : '';
+  }
+
+  /* -------------------------------------------------------------------- the summary */
+
+  function renderSummary() {
+    const body = $('#summary-body');
+    body.textContent = '';
+    for (const row of state.data.report.methods) {
+      const method = state.data.methods.find((m) => m.key === row.method);
+      const tr = el('tr');
+      tr.append(el('th', null, method ? method.label : row.method));
+      tr.append(el('td', null, pct(row.id_accuracy)));
+      tr.append(el('td', null, pct(row.ood_accuracy)));
+      const gap = el('td', 'num-gap', pp(-row.ood_gap));
+      tr.append(gap);
+      tr.append(el('td', null, pct(row.ood_auroc)));
+      tr.append(el('td', null, pct(row.ood_oracle_accuracy)));
+      tr.append(el('td', null, row.ood_ece.toFixed(3)));
+      body.append(tr);
+    }
+  }
+
+  /* ------------------------------------------------- what the gap is actually made of */
+
+  function renderDecomposition() {
+    const box = $('#decomposition');
+    box.textContent = '';
+    const rows = state.data.report.methods;
+    const worst = Math.max(...rows.map((r) => Math.max(0, r.ranking_drop) + Math.max(0, r.placement_drop)));
+    for (const row of rows) {
+      const method = state.data.methods.find((m) => m.key === row.method);
+      const item = el('div', 'decomp-row');
+      item.append(el('span', 'decomp-label', method ? method.label : row.method));
+      const bar = el('div', 'decomp-bar');
+      const ranking = Math.max(0, row.ranking_drop);
+      const placement = Math.max(0, row.placement_drop);
+      const scale = worst > 0 ? 100 / worst : 0;
+      const a = el('div', 'seg seg-ranking');
+      a.style.width = `${ranking * scale}%`;
+      a.title = `ranking: ${pp(ranking)} of accuracy the representation genuinely lost`;
+      const b = el('div', 'seg seg-placement');
+      b.style.width = `${placement * scale}%`;
+      b.title = `placement: ${pp(placement)} recoverable by moving the threshold alone`;
+      bar.append(a, b);
+      item.append(bar);
+      item.append(el('span', 'decomp-total', pp(-(ranking + placement))));
+      box.append(item);
+    }
+  }
+
+  /* ------------------------------------------------------- which shifts actually hurt */
+
+  function renderShiftAxes() {
+    const rows = state.data.report.shift_axes || [];
+    if (!rows.length) return;
+    const body = $('#shift-body');
+    const shifts = [...new Set(rows.map((r) => r.shift))].sort();
+    const head = $('#shift-head');
+    head.textContent = '';
+    head.append(el('th', null, 'method'));
+    for (const s of shifts) head.append(el('th', null, s.replace('topic-', 'topic ').replace('/length-', ' · length ')));
+    body.textContent = '';
+    for (const method of state.data.methods) {
+      if (!rows.some((r) => r.method === method.key)) continue;
+      const tr = el('tr');
+      tr.append(el('th', null, method.label));
+      for (const s of shifts) {
+        const row = rows.find((r) => r.method === method.key && r.shift === s);
+        tr.append(el('td', null, row ? pct(row.accuracy) : '–'));
+      }
+      body.append(tr);
+    }
+  }
+
+  document.addEventListener('DOMContentLoaded', () => {
+    $('#text').addEventListener('input', renderScorer);
+    boot().catch((err) => {
+      const box = $('#loading');
+      if (box) {
+        box.textContent = `Could not load the payload: ${err}. Run \`make web\` to build it.`;
+      }
+    });
+  });
+}