data/sample-attention.json

{
  "job_id": "sample",
  "title": "Attention Is All You Need",
  "subtitle": "the whole paper, animated \u2014 problem \u2192 prior work \u2192 method \u2192 results, and the same numbers flow through every card",
  "theme": {
    "bg": "#0B0E14",
    "ink": "#E8EDF3",
    "muted": "#8B95A1",
    "accent": "#58A6FF",
    "accent2": "#BC8CFF"
  },
  "concepts": [
    {
      "beat": "problem",
      "title": "The Problem",
      "primitive": "problem",
      "widget": "_generic",
      "label": "sequence modelling",
      "params": {
        "points": [
          "Long-range dependencies",
          "Sequential \u2014 no parallelism",
          "Slow, expensive training"
        ]
      },
      "explainer": "Imagine reading a 50-word sentence one word at a time, never allowed to skip ahead. That is an RNN \u2014 it cannot be parallelised, and long-range meaning bleeds away over each sequential step. Something had to change.",
      "clip": "clips/sample-attention/0.mp4"
    },
    {
      "beat": "prior",
      "title": "Existing Approaches",
      "primitive": "compare",
      "widget": "_generic",
      "label": "RNN / CNN seq2seq",
      "params": {
        "left": [
          "RNN / LSTM \u2014 sequential",
          "CNN seq2seq \u2014 limited range",
          "Hard to parallelise"
        ],
        "right": [
          "Self-attention \u2014 global",
          "O(1) path length",
          "Fully parallel"
        ]
      },
      "explainer": "LSTMs and GRUs tried gating to preserve memory, and convolutional models widened the receptive field \u2014 but all of them still bottleneck through sequence length. The shortest path between any two tokens is O(n) steps. This cost was considered unavoidable. Until it was not.",
      "clip": "clips/sample-attention/1.mp4"
    },
    {
      "beat": "method",
      "title": "Token Matrix X",
      "primitive": "matrix",
      "widget": "matrix",
      "label": "the input",
      "io": {
        "produces": "X"
      },
      "params": {
        "tokens": [
          "the",
          "cat",
          "sat"
        ],
        "X": [
          [
            1,
            0,
            1
          ],
          [
            0,
            2,
            0
          ],
          [
            1,
            1,
            0
          ]
        ]
      },
      "explainer": "The architecture starts here. Every token \u2014 the, cat, sat \u2014 is mapped to a row of numbers: a dense vector capturing meaning, position, and context. Stack those rows and you have matrix X, the input that feeds every subsequent computation.",
      "clip": "clips/sample-attention/2.mp4"
    },
    {
      "beat": "method",
      "title": "Scores  S = Q\u00b7K\u1d40",
      "primitive": "matmul",
      "widget": "matmul",
      "label": "QK\u1d40",
      "io": {
        "consumes": "X",
        "produces": "scores"
      },
      "params": {
        "A": [
          [
            1,
            0,
            1
          ],
          [
            0,
            2,
            0
          ],
          [
            1,
            1,
            0
          ]
        ],
        "B": [
          [
            1,
            0,
            1
          ],
          [
            0,
            2,
            1
          ],
          [
            1,
            0,
            0
          ]
        ]
      },
      "explainer": "How much should \"the\" attend to \"cat\"? Multiply their query and key vectors: one dot product, one scalar score. Do this for every pair simultaneously \u2014 that is Q\u00b7K\u1d40. The entire attention pattern emerges from a single matrix multiply.",
      "clip": "clips/sample-attention/3.mp4"
    },
    {
      "beat": "method",
      "title": "Softmax \u2192 weights",
      "primitive": "softmax",
      "widget": "softmax",
      "label": "softmax",
      "io": {
        "consumes": "scores",
        "produces": "weights"
      },
      "params": {
        "logits": [
          2.0,
          1.0,
          0.1
        ]
      },
      "explainer": "Raw scores vary wildly in magnitude \u2014 if you pass them straight to a weighted sum, the result explodes. First scale by 1/\u221ad to tame the variance, then exponentiate and normalise so each row sums to exactly 1. Now the weights read like a probability: how much does this token borrow from each other token?",
      "clip": "clips/sample-attention/4.mp4"
    },
    {
      "beat": "method",
      "title": "Self-Attention output",
      "primitive": "attention",
      "widget": "attention",
      "label": "weighted sum \u00b7V",
      "io": {
        "consumes": "X",
        "produces": "output"
      },
      "params": {
        "tokens": [
          "the",
          "cat",
          "sat"
        ],
        "X": [
          [
            1,
            0,
            1
          ],
          [
            0,
            2,
            0
          ],
          [
            1,
            1,
            0
          ]
        ]
      },
      "explainer": "The payoff: multiply the attention weights by the value vectors V. Each output token is a blend of everything in the sequence, weighted by relevance. \"the\" attends strongly to \"cat\", so its output carries cat information. This is why Transformers understand long-range dependencies in a single layer.",
      "clip": "clips/sample-attention/5.mp4"
    },
    {
      "beat": "method",
      "title": "Activation",
      "primitive": "plot",
      "widget": "plot",
      "label": "sigmoid",
      "params": {
        "shape": "sigmoid",
        "w": 1,
        "b": 0
      },
      "explainer": "After each attention block, a position-wise feed-forward network applies a nonlinearity. Without it, stacking attention layers adds nothing \u2014 they would all collapse to a single linear transform. The sigmoid, ReLU, or GELU bends the representation space, letting each layer learn genuinely new features.",
      "clip": "clips/sample-attention/6.mp4"
    },
    {
      "beat": "results",
      "title": "Results \u2014 BLEU",
      "primitive": "bar",
      "widget": "chart",
      "label": "WMT'14 EN\u2192DE",
      "params": {
        "kind": "bar",
        "title": "BLEU \u2014 WMT'14 English\u2192German",
        "y_label": "BLEU",
        "highlight": "Transformer",
        "categories": [
          "GNMT",
          "ConvS2S",
          "Transformer"
        ],
        "series": [
          {
            "name": "BLEU",
            "values": [
              24.6,
              25.2,
              28.4
            ]
          }
        ]
      },
      "explainer": "On the WMT 2014 English-to-German translation benchmark, the Transformer reaches 28.4 BLEU \u2014 surpassing every prior model while training in 3.5 days on 8 GPUs. The English-to-French result is even more striking: 41.0 BLEU, the best ever at the time, at a fraction of the compute cost.",
      "clip": "clips/sample-attention/7.mp4"
    },
    {
      "beat": "results",
      "title": "Training Curve",
      "primitive": "line",
      "widget": "chart",
      "label": "loss over steps",
      "params": {
        "kind": "line",
        "title": "Training loss over steps",
        "y_label": "loss",
        "series": [
          {
            "name": "loss",
            "points": [
              [
                0,
                5.2
              ],
              [
                1,
                3.1
              ],
              [
                2,
                2.2
              ],
              [
                3,
                1.7
              ],
              [
                4,
                1.4
              ],
              [
                5,
                1.25
              ],
              [
                6,
                1.15
              ],
              [
                7,
                1.1
              ]
            ]
          }
        ]
      },
      "explainer": "Because every position is computed in parallel, the Transformer converges far faster than sequential models. Loss drops steeply in the first thousand steps and plateaus cleanly \u2014 no gradient-vanishing stalls, no exploding signals. The architecture is not just more accurate; it is dramatically more efficient to train.",
      "clip": "clips/sample-attention/8.mp4"
    },
    {
      "beat": "closing",
      "title": "Takeaway",
      "primitive": "takeaway",
      "widget": "_generic",
      "label": "attention is all you need",
      "params": {
        "points": [
          "Pure attention \u2014 no recurrence",
          "New SOTA BLEU",
          "Far less compute, fully parallel"
        ]
      },
      "explainer": "Attention is all you need. No convolutions. No recurrence. Just the scaled dot-product of queries, keys, and values \u2014 applied in parallel across the whole sequence. This single architectural insight enabled GPT, BERT, and every large language model that followed. It changed the field entirely.",
      "clip": "clips/sample-attention/9.mp4"
    }
  ]
}