docs(plan): add Stage 0 eval-methodology gate to Option B plan

Full_Step-By-Step_Plan__Option_B_.txt

@@ -0,0 +1,323 @@
+Full Step-By-Step Plan (Option B)
+
+================================================================================
+STAGE 0 — EVALUATION-METHODOLOGY GATE  (run BEFORE Stage 1; may make Stage 1
+unnecessary)
+================================================================================
+
+Purpose: before spending 3.5 GPU-hours retraining, test whether the apparent
+gap to the IEEE BLEU-4 ~24 baseline is an evaluation-methodology artefact
+rather than a genuine quality deficit. Two pre-registered, falsifiable tests:
+
+  Part A (BLEU)         — rescore the EXISTING v2.0.0 predictions against all 5
+                          COCO references (the committed predictions.jsonl has
+                          only ~1.46 refs/image). Emits a pre-registered band:
+                          DOMINANT / MAJOR-BUT-PARTIAL / MINOR.
+  Part B (qualitative)  — blinded categorization of 30 predictions into
+                          SPECIFIC-CORRECT / GENERIC-CORRECT / PARTIALLY-CORRECT
+                          / INCORRECT, run WITHOUT seeing Part A's number.
+
+Combined verdict (see scripts/categorize_predictions.py docstring) decides
+whether Stage 1 is REQUIRED, OPTIONAL, or UNNECESSARY. If "don't retrain" or
+"ship without retraining", skip Stages 1-2 and go to Stage 7 (reframe).
+
+Pre-registration discipline: both scripts embed their predictions/rubric in
+their module docstrings and must be COMMITTED before being run. Part A is run
+and committed first; Part B runs afterward so the BLEU number cannot bias the
+qualitative read.
+
+--------------------------------------------------------------------------------
+Cell G0 — locate the COCO annotations file (path varies by dataset mount)
+
+ANN=$(find /kaggle/input -maxdepth 6 -name "captions_train2017.json" 2>/dev/null | head -1)
+echo "Annotations: $ANN"
+# If empty, the coco-2017-dataset is not attached — add it in the right sidebar.
+
+--------------------------------------------------------------------------------
+Cell G1 — PART A: 5-ref BLEU rescore + pre-registered band
+
+# Assumes the repo is already cloned + installed (see Stage 1 Cell 2/4). If
+# running Stage 0 standalone, clone first:
+#   !git clone https://github.com/apoorvrajdev/image-captioning-system.git
+#   %cd image-captioning-system
+!pip install -q nltk sacrebleu
+
+import subprocess
+ANN = subprocess.check_output(
+    "find /kaggle/input -maxdepth 6 -name captions_train2017.json 2>/dev/null | head -1",
+    shell=True).decode().strip()
+print("Annotations:", ANN)
+
+!python -m scripts.rescore_nltk_bleu \
+    --predictions-path results/stabilized-beam-w4-lp07-rp12/predictions.jsonl \
+    --coco-annotations "$ANN"
+
+# Watch the line: "PRE-REGISTERED BAND (...): <DOMINANT|MAJOR-BUT-PARTIAL|MINOR>".
+# The full table is also written to
+#   results/stabilized-beam-w4-lp07-rp12/metrics_5ref.json
+# Commit metrics_5ref.json before running Part B.
+
+--------------------------------------------------------------------------------
+Cell G2 — PART B: blinded qualitative worklist (run AFTER Part A is committed)
+
+# Blinding: do NOT open metrics_5ref.json before categorizing. This cell only
+# PREPARES the sample (prints 30 predictions + their 5 refs, no metrics). The
+# categorization itself is a judgment step done against the rubric in
+# scripts/categorize_predictions.py — either by you, or by handing the printed
+# worklist to a separate Claude Code turn. It never reads the BLEU output.
+
+import subprocess
+ANN = subprocess.check_output(
+    "find /kaggle/input -maxdepth 6 -name captions_train2017.json 2>/dev/null | head -1",
+    shell=True).decode().strip()
+
+!python -m scripts.categorize_predictions \
+    --predictions-path results/stabilized-beam-w4-lp07-rp12/predictions.jsonl \
+    --coco-annotations "$ANN"
+
+# After judging each sample, write a categories JSONL
+# ({sample_id, category, justification}) and finalize:
+#   !python -m scripts.categorize_predictions \
+#       --coco-annotations "$ANN" \
+#       --categories results/stabilized-beam-w4-lp07-rp12/categories.jsonl
+# Then apply the COMBINED DECISION RULE (categorize docstring) with the Part A
+# band to decide whether to proceed to Stage 1.
+
+--------------------------------------------------------------------------------
+Stage 0 decision gate:
+  - Combined verdict "don't retrain" / "ship without retraining" -> SKIP Stage 1
+    and Stage 2; go straight to Stage 7 (README reframe).
+  - Combined verdict "retrain" -> proceed to Stage 1 below.
+  - "Flag for human review" -> stop and decide manually before any GPU spend.
+
+================================================================================
+
+Stage 1 - Kaggle Training (you do this in a browser):-
+
+Step 1.1 - Create the Kaggle notebook
+Go to https://www.kaggle.com -> + Create -> New Notebook
+Right sidebar -> Settings:
+Accelerator: GPU T4 x2
+Internet: ON
+Persistence: Files only
+Add Data -> search awsaf49/coco-2017-dataset -> click Add
+Rename notebook to: image-captioning-baseline-recipe-v3
+
+Step 1.2 - Paste these cells in order
+Cell 1 - Confirm dataset path
+
+!ls /kaggle/input/datasets/awsaf49/
+!find /kaggle/input -maxdepth 6 -name "captions_train2017.json" 2>/dev/null
+
+Cell 2 - Clone repo (already pushed, current main has the right code)
+
+!git clone https://github.com/apoorvrajdev/image-captioning-system.git
+%cd image-captioning-system
+
+Cell 3 - Install tf-keras legacy shim
+
+!pip install -q tf-keras
+import os
+os.environ["TF_USE_LEGACY_KERAS"] = "1"
+
+Cell 4 - Install project deps
+
+import os
+os.environ["TF_USE_LEGACY_KERAS"] = "1"
+
+!sed -i '/^tensorflow/d' requirements.txt
+!pip install -q -r requirements.txt -r requirements-dev.txt -r requirements-eval.txt
+!pip install -q --no-deps -e .
+
+import tensorflow as tf
+import tf_keras
+print("TF:", tf.__version__, "| tf_keras:", tf_keras.__version__, "| GPUs:", tf.config.list_physical_devices("GPU"))
+
+# Expect: TF 2.19, tf_keras 2.x, 2 GPUs.
+
+Cell 5 - Patch base.yaml with Kaggle dataset path
+# NOTE: confirm the real mount from Cell 1's find output; the awsaf49 dataset
+# commonly mounts at /kaggle/input/coco-2017-dataset/coco2017 (no datasets/awsaf49 segment).
+
+!sed -i 's|base_path: data/coco2017|base_path: /kaggle/input/datasets/awsaf49/coco-2017-dataset/coco2017|' configs/base.yaml
+!grep base_path configs/base.yaml
+
+Cell 6 - TRAIN with base.yaml (the actual training, ~3.3h)
+
+import os
+os.environ["CAPTIONING__DATA__BASE_PATH"] = "/kaggle/input/datasets/awsaf49/coco-2017-dataset/coco2017"
+
+!python -m scripts.train \
+    --config configs/base.yaml \
+    --output-dir outputs/runs/baseline
+
+# Watch for: Epoch 1 loss ~3.6. EarlyStopping likely fires ~epoch 5-7 with restore_best_weights=True.
+
+Cell 7 - Copy artefacts to versioned dir
+
+!mkdir -p models/v3.0.0
+!cp outputs/runs/baseline/best.h5    models/v3.0.0/model.h5
+!cp outputs/runs/baseline/vocab.pkl  models/v3.0.0/vocab.pkl
+!cp outputs/runs/baseline/vocab.json models/v3.0.0/vocab.json
+!ls -la models/v3.0.0/
+
+Cell 8 - Greedy evaluation
+
+import os
+os.environ["CAPTIONING__DATA__BASE_PATH"] = "/kaggle/input/datasets/awsaf49/coco-2017-dataset/coco2017"
+
+!python -m scripts.evaluate \
+    --config configs/base.yaml \
+    --weights models/v3.0.0/model.h5 \
+    --tokenizer-dir models/v3.0.0 \
+    --results-root results \
+    --run-id baseline-greedy \
+    --model-id inceptionv3-transformer-baseline \
+    --decode-strategy greedy \
+    --max-samples 500
+
+Cell 9 - Beam evaluation (same params as v2.0.0 for apples-to-apples)
+
+import os
+os.environ["CAPTIONING__DATA__BASE_PATH"] = "/kaggle/input/datasets/awsaf49/coco-2017-dataset/coco2017"
+
+!python -m scripts.evaluate \
+    --config configs/base.yaml \
+    --weights models/v3.0.0/model.h5 \
+    --tokenizer-dir models/v3.0.0 \
+    --results-root results \
+    --run-id baseline-beam-w4-lp07-rp12 \
+    --model-id inceptionv3-transformer-baseline \
+    --decode-strategy beam \
+    --beam-width 4 \
+    --length-penalty 0.7 \
+    --repetition-penalty 1.2 \
+    --max-samples 500
+
+Cell 10 - Qualitative samples (30 random predictions)
+
+!python -m scripts.inspect_predictions \
+    --config configs/base.yaml \
+    --weights models/v3.0.0/model.h5 \
+    --tokenizer-dir models/v3.0.0 \
+    --decode-strategy beam \
+    --beam-width 4 \
+    --n-samples 30 \
+    --output results/baseline-beam-w4-lp07-rp12/qualitative.jsonl
+
+Cell 11 - Print metrics summary + side-by-side vs v2.0.0
+
+import json
+
+print("=" * 70)
+print("BASELINE RECIPE (v3.0.0) - configs/base.yaml")
+print("=" * 70)
+for run in ("baseline-greedy", "baseline-beam-w4-lp07-rp12"):
+    m = json.load(open(f"results/{run}/metrics.json"))
+    print(f"\n{run}:")
+    for k in ("bleu1", "bleu2", "bleu3", "bleu4", "rouge_l", "meteor", "cider"):
+        print(f"  {k:10s} = {m[k]:.4f}")
+
+print("\n" + "=" * 70)
+print("STABILIZED RECIPE (v2.0.0) - for comparison")
+print("=" * 70)
+print("""
+stabilized-greedy:
+  bleu1 = 42.20, bleu4 = 10.57, rouge_l = 37.57, meteor = 15.45, cider = 0.789
+stabilized-beam-w4-lp07-rp12:
+  bleu1 = 41.93, bleu4 = 10.39, rouge_l = 36.84, meteor = 15.56, cider = 0.826
+""")
+
+Cell 12 - Package handoff zip
+
+!mkdir -p /kaggle/working/handoff
+!cp -r results /kaggle/working/handoff/
+!cp -r models /kaggle/working/handoff/
+!cp outputs/runs/baseline/history.json /kaggle/working/handoff/ 2>/dev/null || true
+!cd /kaggle/working && zip -r /kaggle/working/handoff-baseline.zip handoff/
+!ls -la /kaggle/working/handoff-baseline.zip
+
+Step 1.3 - Run the notebook
+Hit Run All -> wait ~3.5 hours
+Save Version -> "Save & Run All (Commit)" so the outputs persist
+After completion, Output tab -> download handoff-baseline.zip
+
+Stage 2 - Decision Gate (1 minute, you do this looking at Cell 11 output)
+Look at the CIDEr numbers in Cell 11:
+
+Outcome / What to do
+base.yaml CIDEr >= 0.88 (significantly better than 0.826) -> Parity validated.
+    Proceed to Stage 3. Reframe as ablation in README.
+base.yaml CIDEr 0.83-0.87 (similar to stabilized) -> Mixed signal. Check Cell 10
+    qualitative captions -> if they look more specific, still ship. Otherwise
+    pause and investigate.
+base.yaml CIDEr < 0.82 (worse than stabilized) -> Surprise. Don't ship. There's
+    a hidden divergence we missed.
+Also check Cell 10 qualitative samples - look for confident, specific captions
+("a woman riding a brown horse on a beach") vs generic ones ("a person on a
+beach"). This matters more than the metric delta for the live demo.
+
+Stage 3 - Local Setup (5 min, you do this on your machine)
+
+Step 3.1 - Download + extract handoff zip
+
+mkdir -p "/d/PROJECT/New folder/handoff-baseline"
+cd "/d/PROJECT/New folder/handoff-baseline"
+unzip -o handoff-baseline.zip
+ls -la handoff/
+ls -la handoff/models/v3.0.0/
+ls -la handoff/results/
+
+# Expect: model.h5 (~227MB), vocab.json, vocab.pkl, two baseline-* results dirs, history.json.
+
+Step 3.2 - Sanity check the metrics
+
+cat handoff/results/baseline-greedy/metrics.json | python -m json.tool
+cat handoff/results/baseline-beam-w4-lp07-rp12/metrics.json | python -m json.tool
+
+Stage 4 - Upload to HF Hub (5 min)
+
+Step 4.1 - Upload v3.0.0 weights
+
+cd "/d/PROJECT/New folder/handoff-baseline/handoff"
+hf auth whoami || hf auth login
+hf upload apoorvrajdev/captioning-inceptionv3-transformer models/v3.0.0/model.h5 model.h5
+hf upload apoorvrajdev/captioning-inceptionv3-transformer models/v3.0.0/vocab.json vocab.json
+hf upload apoorvrajdev/captioning-inceptionv3-transformer models/v3.0.0/vocab.pkl vocab.pkl
+
+Step 4.2 - Tag as v3.0.0
+
+python -c "from huggingface_hub import HfApi; HfApi().create_tag('apoorvrajdev/captioning-inceptionv3-transformer', tag='v3.0.0')"
+
+# Verify at https://huggingface.co/apoorvrajdev/captioning-inceptionv3-transformer/tags
+
+Stage 5 - Flip Live Demo to v3.0.0 (2 min, browser)
+Go to https://huggingface.co/spaces/apoorvrajdev/image-captioning-api -> Settings
+Variables and secrets -> find BACKEND_WEIGHTS_HUB_REVISION
+Click edit -> change from v2.0.0 -> v3.0.0 -> Save
+Space auto-rebuilds (~2 min). Watch the Logs tab -> snapshot_download pulling
+v3.0.0 then model_loaded: true.
+Test: https://image-captioning-system.vercel.app -> drop an image -> check quality.
+
+Stage 6 - Commit Results to Repo (5 min)
+
+cd /d/PROJECT/image-captioning-system-main
+cp -r "/d/PROJECT/New folder/handoff-baseline/handoff/results/baseline-greedy" results/
+cp -r "/d/PROJECT/New folder/handoff-baseline/handoff/results/baseline-beam-w4-lp07-rp12" results/
+git add results/baseline-greedy results/baseline-beam-w4-lp07-rp12
+git status
+
+# Suggested commit (you run it):
+# git commit -m "feat(results): add baseline-recipe COCO eval (v3.0.0 checkpoint)"
+# git push origin main
+
+Stage 7 - Reframe README as Ablation (no Kaggle needed)
+Once Stage 6 is done (or once the Stage 0 gate says "don't retrain"), rewrite
+the README's "Model Quality" section as a recipe ablation:
+  Original recipe (v3.0.0) - constant Adam(1e-3), no label smoothing, EarlyStopping
+  Stabilized recipe (v2.0.0) - cosine LR + warmup + label smoothing 0.1 + dropout-off val
+  Side-by-side table of BLEU-1..4 / CIDEr / METEOR / ROUGE-L (greedy + beam) for both
+  Honest explanation: stabilization tricks designed for large-data regimes hurt
+  small-data captioning. The portfolio story becomes "I ran an ablation and
+  learned when modern recipes don't help" — and, from Stage 0, "I separated a
+  metric-methodology artefact from a genuine quality gap before spending GPU time."

Makefile

@@ -115,6 +115,27 @@ eval: ## Evaluate the latest model on COCO val (BLEU, CIDEr, METEOR, ROUGE)
 predict: ## CLI single-image inference (usage: make predict IMAGE=path/to/img.jpg)
 	$(PYTHON) -m scripts.predict --image $(IMAGE)
 
+# =============================================================================
+# Evaluation-methodology gate (Phase 1b — run BEFORE Kaggle Stage 1 retrain)
+# =============================================================================
+# Both targets need the official COCO 2017 captions file. Download + extract:
+#   curl -L -o ann.zip https://images.cocodataset.org/annotations/annotations_trainval2017.zip
+#   unzip ann.zip   # -> annotations/captions_train2017.json
+# Then pass its path via COCO_ANNOTATIONS, e.g.:
+#   make rescore-5ref COCO_ANNOTATIONS=annotations/captions_train2017.json
+
+.PHONY: rescore-5ref
+rescore-5ref: ## Part A: 5-ref BLEU rescore + pre-registered band (needs COCO_ANNOTATIONS=...)
+	$(PYTHON) -m scripts.rescore_nltk_bleu \
+		--predictions-path results/stabilized-beam-w4-lp07-rp12/predictions.jsonl \
+		--coco-annotations $(COCO_ANNOTATIONS)
+
+.PHONY: categorize-30
+categorize-30: ## Part B: blinded 30-sample categorization worklist (needs COCO_ANNOTATIONS=...)
+	$(PYTHON) -m scripts.categorize_predictions \
+		--predictions-path results/stabilized-beam-w4-lp07-rp12/predictions.jsonl \
+		--coco-annotations $(COCO_ANNOTATIONS)
+
 # =============================================================================
 # Backend (FastAPI)
 # =============================================================================

scripts/categorize_predictions.py

@@ -0,0 +1,278 @@
+"""Blinded qualitative categorization of caption predictions (Part B).
+
+This is the qualitative half of the two-part evaluation-methodology gate. It
+tests CIDEr-style image-specificity: does the checkpoint produce captions that
+meaningfully constrain the image, or generic category labels? It runs in a
+SEPARATE Claude Code turn from the BLEU rescore so the BLEU number cannot bias
+the categorization. By construction this script:
+
+  * does NOT import scripts/rescore_nltk_bleu.py,
+  * does NOT read results/.../metrics_5ref.json (or any BLEU output),
+  * shares no state with Part A.
+
+The blinding is structural: when the sample is drawn from qualitative.jsonl,
+the per-sample metric fields (sentence_bleu4, sentence_rouge_l, flags, ...) are
+dropped; only ``image`` and ``prediction`` are carried forward, then the full
+5-reference set is joined from the official COCO annotations.
+
+Workflow (two modes)
+--------------------
+  PREPARE  (default, no --categories):
+      Select the blinded sample, join 5 COCO refs, PRINT each sample as a
+      numbered block, and write a pending --output (category/justification
+      = null). The operator/agent then categorizes each sample BY JUDGMENT
+      using the rubric below.
+  FINALIZE (--categories PATH):
+      Read a JSONL of {sample_id, category, justification}, validate every
+      category against the four allowed values (invented categories raise),
+      enforce the <=25-word justification limit, merge, write the final
+      --output, and print category COUNTS (never naked percentages).
+
+CATEGORIZATION RUBRIC
+---------------------
+Each prediction is assigned EXACTLY ONE category:
+
+  SPECIFIC-CORRECT
+    Caption identifies the main subject correctly AND includes at least one
+    distinguishing attribute that meaningfully constrains the image — color,
+    count, named action, spatial relation, or a named secondary object.
+    Example: "a woman riding a brown horse on a beach."
+
+  GENERIC-CORRECT
+    Caption identifies the main subject correctly but lacks any distinguishing
+    attribute — the same caption could describe many photos of the category.
+    Example: "a person on a beach."
+
+  PARTIALLY-CORRECT
+    Identifies at least one element correctly and gets at least one other
+    element wrong (wrong color, wrong count, wrong action, wrong secondary
+    object). Example for an image of two skiers: "a man riding skis."
+
+  INCORRECT
+    Misidentifies the main subject, is incoherent, or describes a scene
+    clearly absent from all 5 references.
+
+COMBINED DECISION RULE (BLEU verdict from Part A + SPECIFIC-CORRECT count from
+Part B):
+
+  BLEU DOMINANT (>=18) AND SPECIFIC-CORRECT >= 12/30
+    -> Strong evidence for the metric-parity reframe. Don't retrain.
+       Pivot to Stage 7 reframe in the plan.
+
+  BLEU MAJOR-BUT-PARTIAL (14-18) AND SPECIFIC-CORRECT >= 15/30
+    -> Qualitative is strong enough to ship without retraining.
+       Reframe with a "BLEU underestimates this checkpoint" note.
+
+  BLEU MAJOR-BUT-PARTIAL (14-18) AND SPECIFIC-CORRECT < 10/30
+    -> Qualitative confirms BLEU. Retrain (Kaggle Stage 1).
+
+  BLEU MINOR (<=13)
+    -> Retrain regardless of qualitative. Metric gap is too large to argue
+       around with 30 samples.
+
+  Any combination not covered above (e.g. DOMINANT with SPECIFIC-CORRECT <12,
+  or MAJOR-BUT-PARTIAL with SPECIFIC-CORRECT 10-14)
+    -> Flag for human review. Do not auto-decide.
+
+Usage
+-----
+    # PREPARE the blinded worklist:
+    python -m scripts.categorize_predictions \
+        --coco-annotations /path/to/captions_train2017.json
+
+    # FINALIZE after judgment:
+    python -m scripts.categorize_predictions \
+        --coco-annotations /path/to/captions_train2017.json \
+        --categories results/stabilized-beam-w4-lp07-rp12/categories.jsonl
+"""
+
+from __future__ import annotations
+
+import json
+import random
+from collections import Counter
+from pathlib import Path
+
+import click
+
+# The four categories are fixed. Renaming or adding to this set (e.g.
+# "ALMOST-SPECIFIC") is forbidden — finalize mode raises on any other value.
+ALLOWED_CATEGORIES = ("SPECIFIC-CORRECT", "GENERIC-CORRECT", "PARTIALLY-CORRECT", "INCORRECT")
+MAX_JUSTIFICATION_WORDS = 25
+
+_RESULTS_DIR = Path("results/stabilized-beam-w4-lp07-rp12")
+
+
+def _image_id(image_path: str) -> int:
+    return int(Path(image_path).stem)
+
+
+def _load_coco_refs(path: Path) -> dict[int, list[str]]:
+    data = json.loads(path.read_text(encoding="utf-8"))
+    refs: dict[int, list[str]] = {}
+    for ann in data["annotations"]:
+        refs.setdefault(int(ann["image_id"]), []).append(ann["caption"])
+    return refs
+
+
+def _read_jsonl(path: Path) -> list[dict]:
+    rows: list[dict] = []
+    with path.open(encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                rows.append(json.loads(line))
+    return rows
+
+
+def _select_sample(predictions_path: Path, sample_size: int, seed: int) -> list[dict]:
+    """Return [{image, prediction}] for the blinded sample (metrics dropped)."""
+    qualitative = _RESULTS_DIR / "qualitative.jsonl"
+    if qualitative.exists():
+        rows = _read_jsonl(qualitative)
+        if len(rows) >= sample_size:
+            # Blinding: carry ONLY image + prediction; drop sentence_bleu4,
+            # sentence_rouge_l, flags, length_tokens, etc.
+            return [
+                {"image": r["image"], "prediction": r["prediction"]} for r in rows[:sample_size]
+            ]
+    rows = _read_jsonl(predictions_path)
+    picks = random.Random(seed).sample(rows, k=min(sample_size, len(rows)))
+    return [{"image": r["image"], "prediction": r["prediction"]} for r in picks]
+
+
+def _join_refs(sample: list[dict], coco: dict[int, list[str]]) -> list[dict]:
+    """Attach the full COCO ref list; raise (no fallback) on missing ids."""
+    missing = [_image_id(s["image"]) for s in sample if _image_id(s["image"]) not in coco]
+    if missing:
+        raise click.ClickException(
+            f"{len(missing)} sample image_id(s) absent from the annotations file. "
+            f"First 5: {missing[:5]}. Refusing to proceed with partial references."
+        )
+    out: list[dict] = []
+    for s in sample:
+        iid = _image_id(s["image"])
+        out.append({"sample_id": str(iid), "prediction": s["prediction"], "refs": coco[iid]})
+    return out
+
+
+@click.command()
+@click.option(
+    "--predictions-path",
+    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+    default=_RESULTS_DIR / "predictions.jsonl",
+    help="predictions.jsonl (fallback sample source if qualitative.jsonl is absent).",
+)
+@click.option(
+    "--coco-annotations",
+    required=True,
+    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+    help="captions_train2017.json — full 5-reference set per image.",
+)
+@click.option("--sample-size", type=int, default=30, help="Number of samples to categorize.")
+@click.option("--seed", type=int, default=42, help="RNG seed (only used for the fallback sampler).")
+@click.option(
+    "--output",
+    "output_path",
+    type=click.Path(path_type=Path),
+    default=_RESULTS_DIR / "qualitative_categorized.jsonl",
+    help="Where the categorized rows are written.",
+)
+@click.option(
+    "--categories",
+    "categories_path",
+    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+    default=None,
+    help="JSONL of {sample_id, category, justification}. Omit for PREPARE mode.",
+)
+def main(
+    predictions_path: Path,
+    coco_annotations: Path,
+    sample_size: int,
+    seed: int,
+    output_path: Path,
+    categories_path: Path | None,
+) -> None:
+    """Prepare a blinded sample (default) or finalize a categorized sample."""
+    coco = _load_coco_refs(coco_annotations)
+    sample = _select_sample(predictions_path, sample_size, seed)
+    joined = _join_refs(sample, coco)
+    n = len(joined)
+
+    if categories_path is None:
+        # ---- PREPARE: print blinded worklist, write pending output ---------
+        click.echo(f"Blinded worklist: {n} samples (predictions + 5 refs only; NO metrics shown).")
+        click.echo("Categorize each per the rubric in this script's docstring, then re-run")
+        click.echo("with --categories pointing at a JSONL of {sample_id, category, justification}.")
+        click.echo("=" * 80)
+        for i, row in enumerate(joined, start=1):
+            click.echo(f"[{i:>2}] sample_id={row['sample_id']}")
+            click.echo(f"     PRED: {row['prediction']}")
+            for j, ref in enumerate(row["refs"], start=1):
+                click.echo(f"     ref{j}: {ref}")
+            click.echo("-" * 80)
+        pending = [
+            {
+                "sample_id": r["sample_id"],
+                "prediction": r["prediction"],
+                "refs": r["refs"],
+                "category": None,
+                "justification": None,
+            }
+            for r in joined
+        ]
+        with output_path.open("w", encoding="utf-8") as f:
+            for row in pending:
+                f.write(json.dumps(row) + "\n")
+        click.echo(f"Wrote pending worklist ({n} rows, categories null): {output_path}")
+        return
+
+    # ---- FINALIZE: validate, merge, write, count ---------------------------
+    cat_rows = {r["sample_id"]: r for r in _read_jsonl(categories_path)}
+    final: list[dict] = []
+    for row in joined:
+        sid = row["sample_id"]
+        if sid not in cat_rows:
+            raise click.ClickException(f"sample_id {sid} missing from {categories_path}.")
+        category = cat_rows[sid]["category"]
+        justification = cat_rows[sid].get("justification", "")
+        if category not in ALLOWED_CATEGORIES:
+            raise click.ClickException(
+                f"sample_id {sid}: category {category!r} is not one of {ALLOWED_CATEGORIES}. "
+                "The rubric's four categories may not be renamed or expanded."
+            )
+        if len(str(justification).split()) > MAX_JUSTIFICATION_WORDS:
+            raise click.ClickException(
+                f"sample_id {sid}: justification exceeds {MAX_JUSTIFICATION_WORDS} words."
+            )
+        final.append(
+            {
+                "sample_id": sid,
+                "prediction": row["prediction"],
+                "refs": row["refs"],
+                "category": category,
+                "justification": justification,
+            }
+        )
+
+    with output_path.open("w", encoding="utf-8") as f:
+        for row in final:
+            f.write(json.dumps(row) + "\n")
+
+    counts = Counter(r["category"] for r in final)
+    click.echo(f"Categorized {n} samples -> {output_path}")
+    click.echo("Counts:")
+    click.echo("  " + ", ".join(f"{counts.get(c, 0)}/{n} {c}" for c in ALLOWED_CATEGORIES))
+    specific = counts.get("SPECIFIC-CORRECT", 0)
+    click.echo("")
+    click.echo(f"SPECIFIC-CORRECT = {specific}/{n}  (input to the COMBINED DECISION RULE)")
+    click.echo("Apply the COMBINED DECISION RULE in this script's docstring together with the")
+    click.echo("Part A band. This script does NOT read the BLEU output — do not auto-decide here.")
+    click.echo(
+        f"Note: with N={n}, a proportion carries roughly +/-18% sampling margin; "
+        "report counts, not point-estimate percentages."
+    )
+
+
+if __name__ == "__main__":
+    main()

scripts/rescore_nltk_bleu.py

@@ -0,0 +1,334 @@
+"""One-off diagnostic: rescore committed predictions under varying BLEU regimes.
+
+This script isolates the two evaluation-methodology axes that separate this
+repo's metrics from the IEEE notebook's reported BLEU-4 ~24:
+
+  * Axis A  — aggregation + smoothing (sacrebleu corpus vs NLTK smoothed
+              sentence-BLEU). A previous rescore showed this is a wash under
+              defensible smoothers when scored against the same references.
+  * Axis B  — reference count. The committed predictions.jsonl carries only
+              ~1.46 references/image (most have a single reference), not COCO's
+              canonical 5. This script joins the full 5-reference set from the
+              official annotations file and rescores against it.
+
+When ``--coco-annotations`` is omitted the script reproduces the original
+~1.46-ref behaviour, so the two reference-count regimes can be compared
+side-by-side in the same session.
+
+The scripts that gate the Kaggle retraining run are split in two — this one
+(BLEU) and ``scripts/categorize_predictions.py`` (blinded qualitative read) —
+and are run in SEPARATE turns so the BLEU number cannot bias the qualitative
+categorization. The two scripts share no code and no state.
+
+Usage
+-----
+    # 5-ref gating test (the real test):
+    python -m scripts.rescore_nltk_bleu \
+        --predictions-path results/stabilized-beam-w4-lp07-rp12/predictions.jsonl \
+        --coco-annotations /path/to/captions_train2017.json
+
+    # 1.46-ref reproduction (omit --coco-annotations):
+    python -m scripts.rescore_nltk_bleu \
+        --predictions-path results/stabilized-beam-w4-lp07-rp12/predictions.jsonl
+
+PRE-REGISTERED BLEU PREDICTION
+------------------------------
+HYPOTHESIS: Reference-count is the dominant remaining axis of the IEEE eval
+methodology gap (Axis A — aggregation+smoothing — was already shown to be a
+wash by the previous rescore under defensible smoothers).
+
+PREDICTION: 5-ref sacrebleu corpus BLEU-4 will land >= 18.
+
+DECISION RULE (BLEU-only):
+  >= 18  -> DOMINANT. Methodology (reference count) dominates the IEEE gap.
+  14-18  -> MAJOR-BUT-PARTIAL. Methodology is a major but partial factor.
+  <= 13  -> MINOR. Methodology contributes ~3 points at most. Checkpoint
+            genuinely underperforms IEEE.
+
+SECONDARY: 5-ref NLTK method1 should land within ~1 point of 5-ref sacrebleu
+corpus. If it diverges by >3 points, Axis A is not a wash under 5 refs after
+all and the multi-axis analysis above needs revision.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Sequence
+from pathlib import Path
+
+import click
+from nltk.translate.bleu_score import SmoothingFunction, sentence_bleu
+
+# Sentinels the training pipeline wraps captions in (mirrors
+# captioning.preprocessing.caption). Inlined so this one-off stays standalone
+# and TF-free (fast to run as a Kaggle cell).
+START_TOKEN = "[start]"
+END_TOKEN = "[end]"
+
+# Caption normalisation identical to captioning.preprocessing.caption
+# .preprocess_caption MINUS the sentinel wrap: lowercase, strip punctuation,
+# collapse whitespace. Applied to raw COCO captions so the 5-ref set lands in
+# the SAME token space as the predictions and the stored 1.46-ref set —
+# otherwise the reference-count axis would be contaminated by a tokenisation
+# mismatch.
+import re  # noqa: E402  (kept next to the patterns it owns)
+
+_PUNCTUATION_RE = re.compile(r"[^\w\s]")
+_WHITESPACE_RE = re.compile(r"\s+")
+
+# Cumulative BLEU-n weight vectors (uniform over the first n orders).
+_CUMULATIVE_WEIGHTS = {
+    1: (1.0, 0.0, 0.0, 0.0),
+    2: (0.5, 0.5, 0.0, 0.0),
+    3: (1 / 3, 1 / 3, 1 / 3, 0.0),
+    4: (0.25, 0.25, 0.25, 0.25),
+}
+
+_SMOOTHERS = {
+    "method0": SmoothingFunction().method0,
+    "method1": SmoothingFunction().method1,
+    "method4": SmoothingFunction().method4,
+    "method7": SmoothingFunction().method7,
+}
+
+
+def _normalize(text: str) -> str:
+    """Lowercase, strip punctuation, collapse whitespace (no sentinels)."""
+    if not text:
+        return ""
+    text = text.lower()
+    text = _PUNCTUATION_RE.sub("", text)
+    text = _WHITESPACE_RE.sub(" ", text)
+    return text.strip()
+
+
+def _strip_sentinels(caption: str) -> str:
+    """Remove [start]/[end], lowercase, collapse whitespace."""
+    if not caption:
+        return ""
+    cleaned = caption.replace(START_TOKEN, " ").replace(END_TOKEN, " ")
+    return _normalize(cleaned)
+
+
+def _image_id(image_path: str) -> int:
+    """COCO image_id from a .../train2017/000000530117.jpg path."""
+    stem = Path(image_path).stem
+    return int(stem)  # raises ValueError on a non-numeric stem (malformed path)
+
+
+def _load_predictions(path: Path) -> list[dict]:
+    rows: list[dict] = []
+    with path.open(encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                rows.append(json.loads(line))
+    return rows
+
+
+def _load_coco_refs(path: Path) -> dict[int, list[str]]:
+    """Build {image_id: [raw captions...]} from captions_train2017.json."""
+    data = json.loads(path.read_text(encoding="utf-8"))
+    refs: dict[int, list[str]] = {}
+    for ann in data["annotations"]:
+        refs.setdefault(int(ann["image_id"]), []).append(ann["caption"])
+    return refs
+
+
+def _refs_by_slot(references: Sequence[Sequence[str]]) -> list[list[str]]:
+    """Ragged per-example references -> sacrebleu per-slot layout."""
+    max_refs = max((len(r) for r in references), default=0)
+    return [[refs[i] if i < len(refs) else "" for refs in references] for i in range(max_refs)]
+
+
+def _sacrebleu_breakdown(preds: list[str], references: list[list[str]]) -> dict[int, float]:
+    """sacrebleu corpus BLEU-1..4, same config as captioning.evaluation.bleu."""
+    import sacrebleu
+
+    refs_by_slot = _refs_by_slot(references)
+    out: dict[int, float] = {}
+    for n in (1, 2, 3, 4):
+        scorer = sacrebleu.metrics.BLEU(max_ngram_order=n, effective_order=True)
+        out[n] = float(scorer.corpus_score(preds, refs_by_slot).score)
+    return out
+
+
+def _nltk_macro_breakdown(
+    hyps: list[list[str]], refs: list[list[list[str]]], smoother
+) -> dict[int, float]:
+    """Macro-averaged sentence BLEU-1..4 (0-100) for a given smoother."""
+    sums = {n: 0.0 for n in (1, 2, 3, 4)}
+    for hyp, ref_list in zip(hyps, refs, strict=True):
+        if not hyp:
+            continue
+        for n in (1, 2, 3, 4):
+            sums[n] += sentence_bleu(
+                ref_list, hyp, weights=_CUMULATIVE_WEIGHTS[n], smoothing_function=smoother
+            )
+    count = len(hyps) or 1
+    return {n: 100.0 * sums[n] / count for n in (1, 2, 3, 4)}
+
+
+def _band(bleu4: float) -> str:
+    """Map 5-ref sacrebleu corpus BLEU-4 to the pre-registered band."""
+    if bleu4 >= 18.0:
+        return "DOMINANT"
+    if bleu4 >= 14.0:
+        return "MAJOR-BUT-PARTIAL"
+    if bleu4 <= 13.0:
+        return "MINOR"
+    return "BOUNDARY-13-14-REVIEW"  # 13 < x < 14 — left undefined by the spec
+
+
+@click.command()
+@click.option(
+    "--predictions-path",
+    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+    default=Path("results/stabilized-beam-w4-lp07-rp12/predictions.jsonl"),
+    help="predictions.jsonl from a scripts.evaluate run.",
+)
+@click.option(
+    "--coco-annotations",
+    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+    default=None,
+    help="captions_train2017.json. When given, scores against all 5 COCO refs; "
+    "when omitted, reproduces the ~1.46-ref behaviour.",
+)
+@click.option(
+    "--smoother",
+    type=click.Choice(list(_SMOOTHERS)),
+    default="method1",
+    help="Primary NLTK smoother for the headline table (method4 always also shown).",
+)
+def main(predictions_path: Path, coco_annotations: Path | None, smoother: str) -> None:
+    """Rescore predictions; in 5-ref mode emit the pre-registered band."""
+    rows = _load_predictions(predictions_path)
+    five_ref = coco_annotations is not None
+
+    # ---- Build the active reference set ------------------------------------
+    preds: list[str] = []
+    refs_sacre: list[list[str]] = []  # normalised strings, per example
+    if coco_annotations is not None:
+        coco = _load_coco_refs(coco_annotations)
+        missing: list[int] = []
+        for rec in rows:
+            iid = _image_id(rec["image"])
+            if iid not in coco:
+                missing.append(iid)
+        if missing:
+            raise click.ClickException(
+                f"{len(missing)} prediction image_id(s) absent from "
+                f"{coco_annotations}. First 5: {missing[:5]}. "
+                "Refusing to fall back to single-ref scoring."
+            )
+        for rec in rows:
+            iid = _image_id(rec["image"])
+            preds.append(_normalize(rec["prediction"]))
+            refs_sacre.append([_normalize(c) for c in coco[iid]])
+    else:
+        for rec in rows:
+            preds.append(_normalize(rec["prediction"]))
+            refs_sacre.append([_strip_sentinels(r) for r in rec["references"]])
+
+    # NLTK works on token lists.
+    hyps_tok = [p.split() for p in preds]
+    refs_tok = [[r.split() for r in ref_list if r] for ref_list in refs_sacre]
+
+    # ---- Reference-count stats (A2) ----------------------------------------
+    counts = [len(r) for r in refs_sacre]
+    n = len(counts)
+    ref_stats = {
+        "n_examples": n,
+        "mean": round(sum(counts) / n, 4) if n else 0.0,
+        "min": min(counts) if counts else 0,
+        "max": max(counts) if counts else 0,
+        "n_lt_5": sum(1 for c in counts if c < 5),
+    }
+
+    # ---- Metrics -----------------------------------------------------------
+    new_sacre = _sacrebleu_breakdown(preds, refs_sacre)
+    new_primary = _nltk_macro_breakdown(hyps_tok, refs_tok, _SMOOTHERS[smoother])
+    new_method4 = _nltk_macro_breakdown(hyps_tok, refs_tok, _SMOOTHERS["method4"])
+
+    metrics_path = predictions_path.parent / "metrics.json"
+    committed = (
+        json.loads(metrics_path.read_text(encoding="utf-8")) if metrics_path.exists() else {}
+    )
+    committed_bleu = {n_: committed.get(f"bleu{n_}", float("nan")) for n_ in (1, 2, 3, 4)}
+
+    regime = "5-ref" if five_ref else "1.46-ref"
+
+    # ---- Headline four-column table (NO method7) ---------------------------
+    click.echo(f"Predictions   : {predictions_path}")
+    click.echo(
+        f"Reference set : {regime}  "
+        f"(mean {ref_stats['mean']}/image, min {ref_stats['min']}, "
+        f"max {ref_stats['max']}, {ref_stats['n_lt_5']}/{n} have <5 refs)"
+    )
+    click.echo("")
+    header = (
+        f"  {'metric':<8}{'committed sacre':>16}{f'new sacre ({regime})':>20}"
+        f"{f'NLTK {smoother}':>16}{'NLTK method4':>16}"
+    )
+    click.echo(header)
+    for n_ in (1, 2, 3, 4):
+        click.echo(
+            f"  BLEU-{n_:<3}{committed_bleu[n_]:>16.2f}{new_sacre[n_]:>20.2f}"
+            f"{new_primary[n_]:>16.2f}{new_method4[n_]:>16.2f}"
+        )
+    click.echo("")
+
+    # ---- Secondary check + band (5-ref only) -------------------------------
+    if five_ref:
+        delta = new_primary[4] - new_sacre[4]
+        axis_a_wash = abs(delta) <= 3.0
+        band = _band(new_sacre[4])
+        click.echo(f"SECONDARY CHECK: NLTK method1 BLEU-4 - sacrebleu corpus BLEU-4 = {delta:+.2f}")
+        click.echo(
+            "  -> Axis A is a wash under 5 refs (|delta| <= 3)."
+            if axis_a_wash
+            else "  -> Axis A is NOT a wash under 5 refs (|delta| > 3); revise the multi-axis analysis."
+        )
+        click.echo("")
+        click.echo(
+            f"PRE-REGISTERED BAND (5-ref sacrebleu corpus BLEU-4 = {new_sacre[4]:.2f}): {band}"
+        )
+        click.echo("")
+
+        out = {
+            "predictions_path": str(predictions_path),
+            "coco_annotations": str(coco_annotations),
+            "ref_stats": ref_stats,
+            "committed_1p46ref_sacrebleu": {f"bleu{k}": committed_bleu[k] for k in (1, 2, 3, 4)},
+            "new_5ref": {
+                "sacrebleu_corpus": {f"bleu{k}": new_sacre[k] for k in (1, 2, 3, 4)},
+                f"nltk_{smoother}": {f"bleu{k}": new_primary[k] for k in (1, 2, 3, 4)},
+                "nltk_method4": {f"bleu{k}": new_method4[k] for k in (1, 2, 3, 4)},
+            },
+            "band": band,
+            "band_basis": "5ref_sacrebleu_corpus_bleu4",
+            "secondary_check": {
+                "method1_vs_corpus_bleu4_delta": round(delta, 4),
+                "axis_a_wash_under_5ref": axis_a_wash,
+            },
+        }
+        out_path = predictions_path.parent / "metrics_5ref.json"
+        out_path.write_text(json.dumps(out, indent=2), encoding="utf-8")
+        click.echo(f"Wrote: {out_path}")
+    else:
+        click.echo(
+            "1.46-ref mode: band + metrics_5ref.json skipped "
+            "(pass --coco-annotations to run the gating test)."
+        )
+
+    # ---- Diagnostic: smoother sensitivity (NOT headline) -------------------
+    click.echo("")
+    click.echo("DIAGNOSTIC (smoother sensitivity — NOT the headline; method7 is known to inflate):")
+    click.echo(f"  {'smoother':<10}{'BLEU-4':>10}")
+    for name in ("method0", "method1", "method4", "method7"):
+        b4 = _nltk_macro_breakdown(hyps_tok, refs_tok, _SMOOTHERS[name])[4]
+        click.echo(f"  {name:<10}{b4:>10.2f}")
+
+
+if __name__ == "__main__":
+    main()