feat(dictation): word-level dictation scoring with tolerant normalization (#8)

## What & why
The ML core of M2 dictation: compares the learner's typed answer to the
reference and produces a word-level diagnosis. Pure and network-free,
reused
by M5 (pronunciation).

## Design
- Tolerance calibrated to what dictation tests — hearing/spelling the
right
word. Case, punctuation, spacing ignored; contractions ("don't" = "do
not")
and US/UK spelling ("colour" = "color") accepted as transcription
variants;
the word itself counts ("sheep" vs "ship") — the signal dictation must
catch.
- Word-aligned feedback via Levenshtein backtrace (jiwer): reference
shown with
substitutions/omissions/insertions marked, plus WER and an error
breakdown.

## Changes
- services/dictation.py: normalize_words, score_dictation →
DictationResult
(WER, S/D/I/H counts, per-word ops)
- main.py: render_dictation_feedback (marked reference + breakdown),
pure+tested
- Tests: +8 (82 total) — normalization, perfect-despite-variants,
phonetic
confusion caught, deletion/insertion alignment, empty
hypothesis/reference,
feedback rendering

## How to test
uv run pytest

## Out of scope
Listening tab wiring ASR → scoring → feedback (PR 4).

pyproject.toml

@@ -10,6 +10,7 @@ dependencies = [
     # Runtime deps only — keep the Docker image (HF Space, cpu-basic) lean.
     "gradio>=6.17,<7",
     "huggingface-hub>=1.18.0",
+    "jiwer>=4.0.0",
     "numpy>=2.4.6",
     "onnxruntime>=1.26.0",
     "openai>=2.41,<3",          # single OpenAI-compatible client: Gemini / Mistral / Ollama / OpenAI (ADR 0002)
@@ -41,7 +42,6 @@ train = [
 ]
 asr = [
     "faster-whisper>=1.0",
-    "jiwer>=3.0",
     "soundfile>=0.12",
 ]
 

src/tutor/app/main.py

@@ -18,6 +18,7 @@ from tutor.ml.cefr.inference import create_cefr_classifier
 from tutor.services.asr.base import ASRError
 from tutor.services.asr.factory import create_asr_client
 from tutor.services.cache import FileCache
+from tutor.services.dictation import DictationResult
 from tutor.services.llm.base import ChatMessage, LLMError
 from tutor.services.llm.factory import create_llm_client
 from tutor.services.reading import (
@@ -89,6 +90,33 @@ def render_cloze_text(text: str, blanks: list[dict]) -> str:
     return rendered
 
 
+def render_dictation_feedback(result: DictationResult) -> str:
+    """Reference sentence with errors marked, plus WER and an error breakdown."""
+    pieces: list[str] = []
+    for op in result.ops:
+        if op.op == "equal":
+            pieces.append(op.ref_word or "")
+        elif op.op == "substitute":
+            pieces.append(f"~~{op.hyp_word}~~ **{op.ref_word}**")
+        elif op.op == "delete":
+            pieces.append(f"**[missing: {op.ref_word}]**")
+        elif op.op == "insert":
+            pieces.append(f"~~{op.hyp_word}~~")
+    marked = " ".join(piece for piece in pieces if piece)
+
+    accuracy = round((1.0 - result.wer) * 100)
+    header = f"## {accuracy}% correct" + (" 🎉" if result.is_perfect else "")
+    breakdown = (
+        f"WER {result.wer:.0%} · {result.hits} correct, "
+        f"{result.substitutions} wrong, {result.deletions} missed, "
+        f"{result.insertions} extra"
+    )
+    if result.is_perfect:
+        return f"{header}\n\n{breakdown}"
+    legend = "_Marked below: **correct word**, ~~your version~~, **[missing: word]**._"
+    return f"{header}\n\n{breakdown}\n\n{legend}\n\n> {marked}"
+
+
 def build_app(settings: Settings | None = None) -> gr.Blocks:
     settings = settings or get_settings()
     llm = create_llm_client(settings)

src/tutor/services/dictation.py

@@ -0,0 +1,156 @@
+"""Dictation scoring (M2) — pure, network-free, reused by M5 pronunciation.
+
+Compares what the learner typed against the reference sentence and produces a
+word-level diagnosis. Design choices (ADR 0004 spirit):
+
+- Tolerance is calibrated to what dictation *tests*: hearing and spelling the
+  right word. Case, punctuation and spacing are ignored; contractions
+  ("don't" = "do not") and US/UK spelling ("colour" = "color") are accepted
+  (legitimate transcription variants, not listening errors); but the word
+  itself counts ("ship" vs "sheep", "there" vs "their") — that is exactly what
+  dictation must catch.
+- Feedback is word-aligned (Levenshtein backtrace via jiwer): the reference is
+  shown with substitutions, omissions and insertions marked, plus a WER and an
+  error breakdown. Aligning two sequences of different lengths is the core bit.
+"""
+
+import re
+from typing import Literal
+
+from pydantic import BaseModel
+
+# Minimal US/UK normalisation: fold common -our/-or, -ise/-ize, -re/-er, -ll-.
+# Not exhaustive — a pragmatic set that covers the bulk of dictation cases.
+_UK_US_SUFFIXES = (
+    ("our", "or"),  # colour -> color
+    ("ise", "ize"),  # realise -> realize
+    ("isation", "ization"),
+    ("yse", "yze"),  # analyse -> analyze
+    ("tre", "ter"),  # centre -> center
+)
+_CONTRACTIONS = {
+    "don't": "do not",
+    "doesn't": "does not",
+    "didn't": "did not",
+    "won't": "will not",
+    "can't": "cannot",
+    "couldn't": "could not",
+    "wouldn't": "would not",
+    "shouldn't": "should not",
+    "isn't": "is not",
+    "aren't": "are not",
+    "wasn't": "was not",
+    "weren't": "were not",
+    "haven't": "have not",
+    "hasn't": "has not",
+    "hadn't": "had not",
+    "i'm": "i am",
+    "you're": "you are",
+    "we're": "we are",
+    "they're": "they are",
+    "it's": "it is",
+    "that's": "that is",
+    "i've": "i have",
+    "i'll": "i will",
+    "let's": "let us",
+}
+
+OpType = Literal["equal", "substitute", "delete", "insert"]
+
+
+class WordOp(BaseModel):
+    """One alignment operation. ref_word/hyp_word are None where not applicable."""
+
+    op: OpType
+    ref_word: str | None = None
+    hyp_word: str | None = None
+
+
+class DictationResult(BaseModel):
+    wer: float
+    hits: int
+    substitutions: int
+    deletions: int
+    insertions: int
+    reference_words: list[str]
+    ops: list[WordOp]
+
+    @property
+    def is_perfect(self) -> bool:
+        return self.substitutions == 0 and self.deletions == 0 and self.insertions == 0
+
+
+def _expand_contractions(text: str) -> str:
+    return " ".join(_CONTRACTIONS.get(token, token) for token in text.split())
+
+
+def _fold_spelling(word: str) -> str:
+    for uk, us in _UK_US_SUFFIXES:
+        if word.endswith(uk):
+            return word[: -len(uk)] + us
+    return word
+
+
+def normalize_words(text: str) -> list[str]:
+    """Lowercase, drop punctuation, expand contractions, fold UK->US, split."""
+    text = text.lower()
+    text = re.sub(r"[^a-z0-9'\s]", " ", text)  # keep the apostrophe for contractions
+    text = _expand_contractions(text)
+    text = re.sub(r"[^a-z0-9\s]", " ", text)  # now drop stray apostrophes
+    return [_fold_spelling(word) for word in text.split()]
+
+
+def score_dictation(reference: str, hypothesis: str) -> DictationResult:
+    """Word-level scoring with a tolerant normalisation. Pure, no I/O."""
+    import jiwer
+
+    ref_words = normalize_words(reference)
+    hyp_words = normalize_words(hypothesis)
+
+    # jiwer needs non-empty input; handle the degenerate cases explicitly.
+    if not ref_words:
+        msg = "reference is empty after normalisation"
+        raise ValueError(msg)
+    if not hyp_words:
+        return DictationResult(
+            wer=1.0,
+            hits=0,
+            substitutions=0,
+            deletions=len(ref_words),
+            insertions=0,
+            reference_words=ref_words,
+            ops=[WordOp(op="delete", ref_word=word) for word in ref_words],
+        )
+
+    output = jiwer.process_words([" ".join(ref_words)], [" ".join(hyp_words)])
+    ops: list[WordOp] = []
+    for chunk in output.alignments[0]:
+        if chunk.type == "equal":
+            for offset in range(chunk.ref_end_idx - chunk.ref_start_idx):
+                word = ref_words[chunk.ref_start_idx + offset]
+                ops.append(WordOp(op="equal", ref_word=word, hyp_word=word))
+        elif chunk.type == "substitute":
+            for offset in range(chunk.ref_end_idx - chunk.ref_start_idx):
+                ops.append(
+                    WordOp(
+                        op="substitute",
+                        ref_word=ref_words[chunk.ref_start_idx + offset],
+                        hyp_word=hyp_words[chunk.hyp_start_idx + offset],
+                    )
+                )
+        elif chunk.type == "delete":
+            for offset in range(chunk.ref_end_idx - chunk.ref_start_idx):
+                ops.append(WordOp(op="delete", ref_word=ref_words[chunk.ref_start_idx + offset]))
+        elif chunk.type == "insert":
+            for offset in range(chunk.hyp_end_idx - chunk.hyp_start_idx):
+                ops.append(WordOp(op="insert", hyp_word=hyp_words[chunk.hyp_start_idx + offset]))
+
+    return DictationResult(
+        wer=output.wer,
+        hits=output.hits,
+        substitutions=output.substitutions,
+        deletions=output.deletions,
+        insertions=output.insertions,
+        reference_words=ref_words,
+        ops=ops,
+    )

tests/test_dictation.py

@@ -0,0 +1,60 @@
+import pytest
+
+from tutor.services.dictation import normalize_words, score_dictation
+
+
+def test_normalize_folds_case_punctuation_contractions_spelling() -> None:
+    assert normalize_words("Hello, World!") == ["hello", "world"]
+    assert normalize_words("I don't know") == ["i", "do", "not", "know"]
+    assert normalize_words("The colour of the centre") == ["the", "color", "of", "the", "center"]
+
+
+def test_perfect_despite_legitimate_variants() -> None:
+    result = score_dictation("I don't like the colour.", "I do not like the color")
+    assert result.is_perfect
+    assert result.wer == 0.0
+    assert result.substitutions == result.deletions == result.insertions == 0
+
+
+def test_catches_phonetic_confusion() -> None:
+    result = score_dictation("I saw a sheep on the hill", "I saw a ship on the hill")
+    assert result.substitutions == 1
+    subs = [(op.ref_word, op.hyp_word) for op in result.ops if op.op == "substitute"]
+    assert subs == [("sheep", "ship")]
+
+
+def test_deletion_and_alignment_order() -> None:
+    result = score_dictation("the quick brown fox jumps", "the quik brown fox")
+    assert result.substitutions == 1  # quick/quik
+    assert result.deletions == 1  # jumps missing
+    assert [op.op for op in result.ops] == ["equal", "substitute", "equal", "equal", "delete"]
+
+
+def test_insertion_is_flagged() -> None:
+    result = score_dictation("the cat sat", "the big cat sat")
+    assert result.insertions == 1
+    inserted = [op.hyp_word for op in result.ops if op.op == "insert"]
+    assert inserted == ["big"]
+
+
+def test_empty_hypothesis_is_all_deletions() -> None:
+    result = score_dictation("hello world", "   ")
+    assert result.wer == 1.0
+    assert result.deletions == 2
+    assert all(op.op == "delete" for op in result.ops)
+
+
+def test_empty_reference_raises() -> None:
+    with pytest.raises(ValueError, match="reference is empty"):
+        score_dictation("!!!", "anything")
+
+
+def test_render_feedback_marks_errors() -> None:
+    from tutor.app.main import render_dictation_feedback
+
+    perfect = render_dictation_feedback(score_dictation("the cat sat", "the cat sat"))
+    assert "100% correct 🎉" in perfect
+
+    marked = render_dictation_feedback(score_dictation("I saw a sheep", "I saw a ship"))
+    assert "**sheep**" in marked  # correct word in bold
+    assert "~~ship~~" in marked  # learner's wrong version struck through

uv.lock

@@ -2587,6 +2587,7 @@ source = { editable = "." }
 dependencies = [
     { name = "gradio" },
     { name = "huggingface-hub" },
+    { name = "jiwer" },
     { name = "numpy" },
     { name = "onnxruntime" },
     { name = "openai" },
@@ -2598,7 +2599,6 @@ dependencies = [
 [package.dev-dependencies]
 asr = [
     { name = "faster-whisper" },
-    { name = "jiwer" },
     { name = "soundfile" },
 ]
 data = [
@@ -2624,6 +2624,7 @@ train = [
 requires-dist = [
     { name = "gradio", specifier = ">=6.17,<7" },
     { name = "huggingface-hub", specifier = ">=1.18.0" },
+    { name = "jiwer", specifier = ">=4.0.0" },
     { name = "numpy", specifier = ">=2.4.6" },
     { name = "onnxruntime", specifier = ">=1.26.0" },
     { name = "openai", specifier = ">=2.41,<3" },
@@ -2635,7 +2636,6 @@ requires-dist = [
 [package.metadata.requires-dev]
 asr = [
     { name = "faster-whisper", specifier = ">=1.0" },
-    { name = "jiwer", specifier = ">=3.0" },
     { name = "soundfile", specifier = ">=0.12" },
 ]
 data = [