feat: add uncertainty handling in scoring and trace schema for improved feedback

app/scorer.py

@@ -198,6 +198,7 @@ class Reference:
 class ScorerConfig:
     flag_below: float = 50.0        # phoneme score under which a consonant gets a tip
     unrelated_ratio: float = 0.65   # normalised edit distance above which we say "different sentence"
+    feedback_uncertain_margin: float = 0.15  # below unrelated_ratio: score, but make no named sound claims
     issue_below: float = 90.0       # phoneme score under which it is listed as imprecise
     max_issues: int = 8
     max_tips: int | None = None     # None: as many as the rules document says (scoring.max_tips)
@@ -581,6 +582,8 @@ class Scorer:
         ar, ipa, toks, al, scores, unrelated = fit.ar, fit.ipa, fit.toks, fit.al, fit.scores, fit.unrelated
         i2t = self.vocab.i2t
         heard = " ".join(i2t[i] for i in fit.heard_ids)
+        fit_ratio = fit.distance / max(1, len(toks))
+        uncertain = bool(not unrelated and fit_ratio >= self.cfg.unrelated_ratio - self.cfg.feedback_uncertain_margin)
         # Measure against what the model gives a native on the same sound: a sound it misses in the
         # reference voice too (ʕ in على heard as dropped) is the model's weakness, not the learner's.
         heatmap = self._heatmap(fit.logp, toks, al, scores, duration_s) if diagnostics else None
@@ -617,7 +620,7 @@ class Scorer:
             heard_ph = "_" if sc.deleted else (i2t.get(sc.competitor) if sc.competitor is not None else None)
             check = self._check(tok, heard_ph)
             beyond.append(self._beyond_margin(tok, sc, heard_ph))
-            flagged = (bool(sc.score < self.cfg.flag_below) and beyond[-1]
+            flagged = (not uncertain and bool(sc.score < self.cfg.flag_below) and beyond[-1]
                        and (tok.phoneme not in VOWELS or self._flag_vowel(toks, idx)))
             if idx in fit.inserted:   # a vowel before this consonant (3.10): the reason its letter gives
                 spec = self.rules.insertion
@@ -644,7 +647,7 @@ class Scorer:
                     tip["idx"] = idx
                     tip["score"] = round(sc.score)
                     tips.append(tip)
-            if idx in fit.inserted and not unrelated and "cluster" not in seen:
+            if idx in fit.inserted and not unrelated and not uncertain and "cluster" not in seen:
                 seen.add("cluster")
                 spec = self.rules.insertion
                 tips.append({"title": self.rules.titles.get(spec["rule"], spec["rule"]), "text": str(spec["feedback"]),
@@ -665,7 +668,7 @@ class Scorer:
                 got = [i for j in at for i in carriers[w][j] if 0 <= i < len(phonemes)]
                 worst = min(got, key=lambda i: phonemes[i]["score"]) if got else None
                 letter = {"t": text, "score": phonemes[worst]["score"] if got else None}
-                if got and letter["score"] < LETTER_GOOD and not unrelated:
+                if got and letter["score"] < LETTER_GOOD and not unrelated and not uncertain:
                     letter.update(letter_reason(toks[worst].sound, heard_as[worst], checks[worst]))
                 letters.append(letter)
             if w in forgiven and min(ws) >= self.cfg.issue_below:
@@ -689,7 +692,7 @@ class Scorer:
         # Anything short of 100 must say why: every imprecise sound that did not get a full tip,
         # weakest first. If nothing is clearly off, name the weakest few.
         issues = []
-        if round(overall) < 100 and not unrelated:
+        if round(overall) < 100 and not unrelated and not uncertain:
             tipped = {t.pop("idx") for t in tips}
             rest = sorted((i for i in range(len(toks)) if i not in tipped and scores[i].score < 100),
                           key=lambda i: scores[i].score)
@@ -711,8 +714,11 @@ class Scorer:
         for t in tips:
             t.pop("idx", None)
         result = {"score": round(overall), "variant": {"ar": ar, "tr": " ".join(tr_words), "ipa": ipa}, "heard_ipa": heard, "heard_tr": " ".join(h for h in heard_words if h),
-                  "unrelated": unrelated, "words": words, "tips": tips, "issues": issues, "references": len(refs),
+                  "unrelated": unrelated, "uncertain": uncertain,
+                  "words": words, "tips": tips, "issues": issues, "references": len(refs),
                   **self._dialect(toks, scores)}
+        if uncertain:
+            result["uncertainty_message"] = "ההקלטה זוהתה רק באופן חלקי, ולכן אי אפשר לקבוע אילו הגאים נאמרו אחרת. נסו שוב."
         if calibrated:
             result["calibrated"] = calibrated
         if heatmap is not None:
@@ -785,6 +791,7 @@ class Scorer:
                 "deleted_comp": round(float(np.exp(g.deleted_comp)), 3),
                 "flag_below": cfg.flag_below, "issue_below": cfg.issue_below, "near_vowel_floor": cfg.near_vowel_floor,
                 "word_min_weight": cfg.word_min_weight, "vowel_weight": 0.5, "unrelated_ratio": cfg.unrelated_ratio,
+                "feedback_uncertain_ratio": round(cfg.unrelated_ratio - cfg.feedback_uncertain_margin, 3),
                 "unrelated_cap": 30.0, "letter_good": LETTER_GOOD, "max_tips": max_tips, "max_issues": cfg.max_issues,
                 "ref_min_mean": cfg.ref_min_mean, "margin_default": float(self.rules.scoring.get("margin_default", 0.0)),
                 "ayin_deleted_floor": cfg.ayin_deleted_floor,
@@ -804,6 +811,9 @@ class Scorer:
             "aipa": fit.aipa,
             "unrelated": {"heard": heard, "distance": fit.distance, "tokens": n,
                           "ratio": round(fit.distance / max(1, n), 3), "threshold": cfg.unrelated_ratio,
+                          "uncertain_threshold": round(cfg.unrelated_ratio - cfg.feedback_uncertain_margin, 3),
+                          "uncertain": (not fit.unrelated and fit.distance / max(1, n) >=
+                                        cfg.unrelated_ratio - cfg.feedback_uncertain_margin),
                           "unrelated": fit.unrelated},
             "references": [{"mean": round(float(np.mean(r.scores)), 1), "sounds": len(r.sounds)} for r in refs],
             "tokens": tokens,

docs/trace-schema.ts

@@ -22,6 +22,8 @@ export interface ScoreResult {
   heard_ipa: string;              // greedy decode: what the model heard, free of the expected text
   heard_tr: string;               // the same in Hebrew letters, cut at the aligned word starts
   unrelated: boolean;             // a different sentence (TraceUnrelated): score capped, no tips
+  uncertain: boolean;             // marginal target fit: score retained, categorical sound feedback suppressed
+  uncertainty_message?: string;
   words: Word[];
   tips: Tip[];                    // the most crucial mistakes, severity first (at most config.max_tips)
   issues: Issue[];                // the rest of the imprecise sounds
@@ -95,6 +97,7 @@ export interface TraceConfig {
   word_min_weight: number;
   vowel_weight: number;           // a vowel's weight in the sentence's phone mean
   unrelated_ratio: number;
+  feedback_uncertain_ratio: number;
   unrelated_cap: number;
   letter_good: number;
   max_tips: number;
@@ -128,6 +131,8 @@ export interface TraceUnrelated {
   tokens: number;
   ratio: number;                  // distance / tokens
   threshold: number;              // over this: a different sentence
+  uncertain_threshold: number;    // at/over this: suppress categorical phoneme feedback
+  uncertain: boolean;
   unrelated: boolean;
 }
 

static/index.html

@@ -3170,7 +3170,7 @@ function drawStory(){
     }else if(b!=null) tools.append(pill(b,"הציון הטוב ביותר"));
     if(b!=null||r){   // a line that was said shows its Arabic
       const tr=document.createElement("span"); tr.className="tr"; row.classList.add("said");
-      if(r&&!(r.score>STORY.pass&&!r.unrelated)&&r.words.length&&r.words.every(w=>w.letters)){
+      if(r&&!(r.score>STORY.pass&&!r.unrelated&&!r.uncertain)&&r.words.length&&r.words.every(w=>w.letters)){
         tr.classList.add("lit");
         r.words.forEach((w,k)=>{ if(k) tr.append(" "); litWord(tr,w); });
       }else setTr(tr,p.tr);
@@ -3758,7 +3758,7 @@ function render(r){
   $("score").textContent=r.score;
   const pill=$("pill"); const s=r.score;
   pill.className="pill "+(s>=85?"good":s>=60?"mid":"bad");
-  pill.textContent= r.unrelated ? "משפט אחר?" : s>=85?"מצוין": s>=60?"כמעט":"נסו שוב";
+  pill.textContent= r.unrelated ? "משפט אחר?" : r.uncertain ? "לא ברור" : s>=85?"מצוין": s>=60?"כמעט":"נסו שוב";
 
   $("words").hidden=true;
   drawAnalyzedAnswer(r);
@@ -3769,6 +3769,10 @@ function render(r){
     const li=document.createElement("li");
     li.innerHTML='<span class="what">ההקלטה לא דומה למשפט הצפוי</span><span>ייתכן שבחרתם ניסוח אחר. גלו את מילות התשובה ונסו את הנוסח המוצע.</span>';
     tips.appendChild(li);
+  }else if(r.uncertain){
+    const li=document.createElement("li");
+    li.innerHTML=`<span class="what">הזיהוי לא היה מספיק ברור</span><span>${r.uncertainty_message||"אי אפשר לקבוע אילו הגאים נאמרו אחרת. נסו שוב."}</span>`;
+    tips.appendChild(li);
   }
   for(const t of r.tips){
     const li=document.createElement("li");
@@ -3785,7 +3789,7 @@ function render(r){
     setTr(body,r.issues.map(i=>`${i.sound} ב״${i.word_tr}״ (${i.score})`+(i.missing?" – לא נשמעה":i.heard?` – נשמע כמו ${i.heard}`:"")).join(" · "));
     li.append(what,body); tips.appendChild(li);
   }
-  if(!r.unrelated && !r.tips.length && !r.issues.length){
+  if(!r.unrelated && !r.uncertain && !r.tips.length && !r.issues.length){
     const li=document.createElement("li"); li.innerHTML='<span class="what">כל העיצורים נשמעו נכון</span><span>נסו לומר את המשפט בקצב דיבור טבעי.</span>';
     tips.appendChild(li);
   }
@@ -3794,9 +3798,9 @@ function render(r){
   keepScore(cur.id,s); if(daily) dailySaid.add(cur.id);   // only an empty recording is not practice
   const story=!driving&&!$("story").hidden;
   if(story){ storyLast[cur.id]=r; drawStory(); }   // the line was said: its grade and its Arabic appear on it
-  if(story&&!(!r.unrelated&&s>STORY.pass)) storyRetry();
+  if(story&&!(!r.unrelated&&!r.uncertain&&s>STORY.pass)) storyRetry();
   if(bubbleCat()){ bubbleOutcome(r); return; }
-  if(!driving&&!r.unrelated&&s>=AUTO_NEXT.minimum){   // driving mode moves on by itself, in its own loop
+  if(!driving&&!r.unrelated&&!r.uncertain&&s>=AUTO_NEXT.minimum){   // driving mode moves on by itself, in its own loop
     $("status").textContent=`ציון ${s} ✓ עוברים למשפט הבא`;
     startAutoNext();
   }
@@ -3817,7 +3821,7 @@ function waText(r,passed){
   waMsg=el; $("bubbleFoot").prepend(el);
 }
 async function bubbleOutcome(r,expired=false){
-  const id=cur.id, correct=!r.unrelated&&r.score>=BUBBLE_PASS;
+  const id=cur.id, correct=!r.unrelated&&!r.uncertain&&r.score>=BUBBLE_PASS;
   if(bubbleResolving) return;
   bubbleResolving=true;
   if(!expired){
@@ -3959,7 +3963,8 @@ function drawAnalyzedAnswer(r){
     const pair=mark("analysis-word","","span"), expected=mark("expected","","span");
     if(word.letters&&word.letters.length) litWord(expected,word);
     else{ expected.classList.add(band(word.score)); setTr(expected,word.tr); }
-    const heard=mark(`heard-word${word.heard_tr?"":" empty"}`,"","span"); setTr(heard,word.heard_tr||"לא זוהה");
+    const heardText=r.uncertain?"לא זוהה בביטחון":(word.heard_tr||"לא זוהה");
+    const heard=mark(`heard-word${word.heard_tr&&!r.uncertain?"":" empty"}`,"","span"); setTr(heard,heardText);
     pair.append(expected,heard); main.append(pair);
   });
   card.replaceChildren(mark("analysis-key","למעלה: התשובה · למטה: המערכת שמעה","small"),main);

tests/test_scoring.py

@@ -311,6 +311,27 @@ def test_unrelated_sentence():
     assert r["unrelated"] and r["score"] <= 30
 
 
+def test_marginal_sentence_fit_suppresses_categorical_sound_feedback():
+    # Half of the target phones differ: still score the attempt, but do not tell the learner they said specific sounds.
+    phrase = {"variants": [("تتتتت", "t a t a t a t a t a")]}
+    phones = ["d", "a", "d", "a", "d", "a", "d", "a", "d", "a"]
+    r = custom_scorer(["t"], phones).score(None, phrase, diagnostics=True)
+    assert not r["unrelated"] and r["uncertain"]
+    assert r["tips"] == [] and r["issues"] == []
+    assert not any(p["flagged"] or p["heard"] for w in r["words"] for p in w["phonemes"])
+    assert r["uncertainty_message"]
+    assert r["trace"]["unrelated"]["uncertain"]
+    assert r["trace"]["unrelated"]["ratio"] == 0.5
+
+
+def test_clear_sentence_fit_still_reports_a_real_substitution():
+    phrase = {"variants": [("تتتتت", "t a t a t a t a t a")]}
+    phones = ["d", "a", "t", "a", "t", "a", "t", "a", "t", "a"]
+    r = custom_scorer(["t"], phones).score(None, phrase)
+    assert not r["unrelated"] and not r["uncertain"]
+    assert r["tips"][0]["key"] == "t>d"
+
+
 @pytest.mark.parametrize("p", PHRASES, ids=lambda p: p["he"])
 def test_phrase_bank_word_counts_match(p):
     for ar, ipa in p["variants"]: