3v324v23 commited on
Commit
0af696f
·
1 Parent(s): 6953a11

Add unit tests for pronunciation rules parsing and detection

Browse files

- Implement tests to ensure all YAML blocks in pronunciation-rules.md are parsed correctly into rules.
- Verify that every heard symbol has a corresponding Hebrew label.
- Check correct realizations per letter and dialect associations.
- Report undetectable rules and validate their detection status.

app/align.py CHANGED
@@ -71,8 +71,12 @@ class TokenScore:
71
  score: float # 0..100
72
  gop: float # log ratio target vs strongest competitor, <= 0
73
  peak: float # max log-posterior of the target inside its window
74
- competitor: int | None # strongest non-target, non-blank token in the window
75
  deleted: bool # target barely present and nothing else either
 
 
 
 
76
 
77
 
78
  @dataclass
@@ -86,14 +90,20 @@ class GopConfig:
86
 
87
 
88
  def score_tokens(logp: np.ndarray, align: Alignment, tokens: list[int], blank: int,
89
- cfg: GopConfig = GopConfig()) -> list[TokenScore]:
 
90
  """Per-token GOP.
91
 
92
  The window of token i runs from the end of token i-1 to the start of token i+1, so it covers
93
  the blank frames around the emission where a substituted phoneme typically peaks.
94
  The neighbours' own labels are excluded from the competitor set (coarticulation is not an error).
 
 
 
 
95
  """
96
  T = logp.shape[0]
 
97
  out: list[TokenScore] = []
98
  for i, tok in enumerate(tokens):
99
  a = align.spans[i - 1][1] if i > 0 else 0
@@ -101,13 +111,16 @@ def score_tokens(logp: np.ndarray, align: Alignment, tokens: list[int], blank: i
101
  a, b = min(a, align.spans[i][0]), max(b, align.spans[i][1])
102
  seg = logp[a:b]
103
  peak = float(seg[:, tok].max())
 
 
104
 
105
- masked = seg.max(0).copy()
106
  masked[blank] = NEG
107
- masked[tok] = NEG
108
- for j in (i - 1, i + 1):
109
  if 0 <= j < len(tokens):
110
- masked[tokens[j]] = NEG
 
 
111
  comp = int(masked.argmax())
112
  comp_peak = float(masked[comp])
113
 
@@ -115,8 +128,11 @@ def score_tokens(logp: np.ndarray, align: Alignment, tokens: list[int], blank: i
115
  deleted = bool(peak < cfg.deleted_peak and comp_peak < cfg.deleted_comp)
116
  confidence = min(1.0, float(np.exp(peak)) / cfg.confident_peak)
117
  score = 100.0 * float(np.exp(gop * cfg.leniency)) * float(np.sqrt(confidence))
 
 
118
  out.append(TokenScore(round(score, 1), round(gop, 3), round(peak, 3),
119
- comp if comp_peak > NEG / 2 else None, deleted))
 
120
  return out
121
 
122
 
@@ -131,10 +147,11 @@ def greedy_decode(logp: np.ndarray, blank: int) -> list[int]:
131
  return out
132
 
133
 
134
- def edit_distance(a: list, b: list) -> int:
 
135
  d = list(range(len(b) + 1))
136
  for i, x in enumerate(a, 1):
137
  prev, d[0] = d[0], i
138
  for j, y in enumerate(b, 1):
139
- prev, d[j] = d[j], min(d[j] + 1, d[j - 1] + 1, prev + (x != y))
140
  return d[len(b)]
 
71
  score: float # 0..100
72
  gop: float # log ratio target vs strongest competitor, <= 0
73
  peak: float # max log-posterior of the target inside its window
74
+ competitor: int | None # strongest non-target, non-blank token in the window (None: nothing, or a sound we do not name)
75
  deleted: bool # target barely present and nothing else either
76
+ realised: int | None = None # a class target (several correct realisations): the one said
77
+ other: bool = False # the strongest competitor is a sound outside the named set
78
+ comp_peak: float = NEG # max log-posterior of the strongest competitor (named or not) in the window
79
+ window: tuple[int, int] = (0, 0) # [start, end) frames the target and its competitors were read from
80
 
81
 
82
  @dataclass
 
90
 
91
 
92
  def score_tokens(logp: np.ndarray, align: Alignment, tokens: list[int], blank: int,
93
+ cfg: GopConfig = GopConfig(), accept: list[set[int] | None] | None = None,
94
+ n_real: int | None = None, named: set[int] | None = None) -> list[TokenScore]:
95
  """Per-token GOP.
96
 
97
  The window of token i runs from the end of token i-1 to the start of token i+1, so it covers
98
  the blank frames around the emission where a substituted phoneme typically peaks.
99
  The neighbours' own labels are excluded from the competitor set (coarticulation is not an error).
100
+
101
+ A class token (a virtual column pooling its correct realisations, `accept[i]` their ids) is scored on the
102
+ pooled column; its members are not competitors. Competitors come from the first `n_real` columns only, and
103
+ one outside `named` is reported as `other` rather than by name.
104
  """
105
  T = logp.shape[0]
106
+ accept = accept or [None] * len(tokens)
107
  out: list[TokenScore] = []
108
  for i, tok in enumerate(tokens):
109
  a = align.spans[i - 1][1] if i > 0 else 0
 
111
  a, b = min(a, align.spans[i][0]), max(b, align.spans[i][1])
112
  seg = logp[a:b]
113
  peak = float(seg[:, tok].max())
114
+ members = accept[i] or set()
115
+ realised = max(members, key=lambda m: seg[:, m].max()) if members else None
116
 
117
+ masked = (seg[:, :n_real] if n_real else seg).max(0).copy()
118
  masked[blank] = NEG
119
+ for j in (i - 1, i, i + 1):
 
120
  if 0 <= j < len(tokens):
121
+ for x in {tokens[j], *(accept[j] or ())}:
122
+ if x < len(masked):
123
+ masked[x] = NEG
124
  comp = int(masked.argmax())
125
  comp_peak = float(masked[comp])
126
 
 
128
  deleted = bool(peak < cfg.deleted_peak and comp_peak < cfg.deleted_comp)
129
  confidence = min(1.0, float(np.exp(peak)) / cfg.confident_peak)
130
  score = 100.0 * float(np.exp(gop * cfg.leniency)) * float(np.sqrt(confidence))
131
+ present = comp_peak > NEG / 2
132
+ other = bool(present and named is not None and comp not in named)
133
  out.append(TokenScore(round(score, 1), round(gop, 3), round(peak, 3),
134
+ comp if present and not other else None, deleted, realised, other,
135
+ round(comp_peak, 3), (int(a), int(b))))
136
  return out
137
 
138
 
 
147
  return out
148
 
149
 
150
+ def edit_distance(a: list, b: list, same=None) -> int:
151
+ same = same or (lambda x, y: x == y)
152
  d = list(range(len(b) + 1))
153
  for i, x in enumerate(a, 1):
154
  prev, d[0] = d[0], i
155
  for j, y in enumerate(b, 1):
156
+ prev, d[j] = d[j], min(d[j] + 1, d[j - 1] + 1, prev + (not same(x, y)))
157
  return d[len(b)]
app/archi.py ADDED
@@ -0,0 +1,181 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Abstract symbols for letters with more than one correct pronunciation (section 5 of pronunciation-rules.md).
2
+
3
+ phrases.py writes the realisation its author chose (ق as ʔ in قديش, q in القدس). The scorer must accept
4
+ the others too (kaddeesh is rural Palestinian), so before tokenising, archify() rewrites each phoneme that
5
+ comes from such a letter as "SYMBOL:written" (Q:ʔ), and tokenize() scores it against every correct
6
+ realisation at once. The written phoneme is kept for display, tips and calibration keys.
7
+
8
+ The letter behind a phoneme is found by aligning the Arabic letters of the word with its IPA consonants.
9
+ A phrase can also write a symbol itself: "Q", "Q:k", or with its own set of realisations, "TH{t,s}".
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import re
14
+ from functools import lru_cache
15
+
16
+ from .phones import base
17
+
18
+ # letter -> the symbol its phoneme is rewritten to
19
+ LETTER_SYMBOL = {"ق": "Q", "ك": "K", "ث": "TH", "ذ": "DH", "ظ": "ZH", "ج": "J"}
20
+
21
+ # letter -> base() names of the phonemes it is said as in the phrase bank (dialects, MSA, careless spelling)
22
+ _LETTERS = {
23
+ "ب": {"b"}, "ت": {"t"}, "ث": {"t", "s", "θ"}, "ج": {"dʒ", "g"}, "ح": {"ħ"}, "خ": {"x"}, "د": {"d"},
24
+ "ذ": {"d", "z", "ð"}, "ر": {"r"}, "ز": {"z"}, "س": {"s"}, "ش": {"ʃ"}, "ص": {"sˤ", "s"}, "ض": {"dˤ", "d"},
25
+ "ط": {"tˤ", "t"}, "ظ": {"ðˤ", "dˤ", "zˤ", "z", "ð"}, "ع": {"ʕ"}, "غ": {"ɣ"}, "ف": {"f"}, "ق": {"ʔ", "q", "k", "g"},
26
+ "ك": {"k", "tʃ"}, "ل": {"l"}, "م": {"m"}, "ن": {"n"}, "ه": {"h"}, "و": {"w"}, "ي": {"j"}, "ى": set(),
27
+ "ء": {"ʔ"}, "أ": {"ʔ"}, "إ": {"ʔ"}, "آ": {"ʔ"}, "ؤ": {"ʔ"}, "ئ": {"ʔ"}, "ا": {"ʔ"}, "ة": {"t"},
28
+ }
29
+ # letters that are often not said as a consonant at all (vowel letters, hamza seats, the article's ل): cheap to skip
30
+ _WEAK = set("اوىيةهءأإآؤئل")
31
+ _VOWELS = {"a", "i", "u", "e", "o", "ə"}
32
+
33
+ _TOKEN = re.compile(r"^([A-Z]+)(?:\{([^}]*)\})?(?::(.+))?$")
34
+
35
+
36
+ def split_token(tok: str) -> tuple[str | None, list[str] | None, str]:
37
+ """"Q:ʔ" -> ("Q", None, "ʔ"); "TH{t,s}" -> ("TH", ["t", "s"], "t"); "ʕ" -> (None, None, "ʕ").
38
+
39
+ The phoneme of a symbol written without one is its first listed realisation, or "" (the caller picks)."""
40
+ m = _TOKEN.match(tok)
41
+ if not m:
42
+ return None, None, tok
43
+ sym, members, written = m.group(1), m.group(2), m.group(3)
44
+ members = [p.strip() for p in members.split(",") if p.strip()] if members else None
45
+ return sym, members, written or (members[0] if members else "")
46
+
47
+
48
+ def written(tok: str) -> str:
49
+ """The phoneme a token is written as in the phrase (Q:ʔ -> ʔ)."""
50
+ return split_token(tok)[2] or tok
51
+
52
+
53
+ def _letters(ar_word: str) -> list[str]:
54
+ return [c for c in ar_word if c in _LETTERS]
55
+
56
+
57
+ def _units(phs: list[str]) -> list[tuple[str, list[int]]]:
58
+ """The consonants of an IPA word, a geminate run as one unit: (base name, its positions)."""
59
+ out: list[tuple[str, list[int]]] = []
60
+ for i, ph in enumerate(phs):
61
+ b = base(ph)
62
+ if b in _VOWELS:
63
+ continue
64
+ if out and out[-1][0] == b and out[-1][1][-1] == i - 1:
65
+ out[-1][1].append(i)
66
+ else:
67
+ out.append((b, [i]))
68
+ return out
69
+
70
+
71
+ _SKIP_LETTER, _SKIP_WEAK, _SKIP_UNIT, _MISMATCH = 1.0, 0.3, 1.0, 1.6
72
+
73
+
74
+ def _align(letters: list[str], units: list[str]) -> tuple[dict[int, set[int]], float]:
75
+ """For each letter, the units it matches on some cheapest alignment of letters to consonants.
76
+
77
+ A letter matched to different units on two cheapest alignments (or matched on one and skipped on
78
+ another) is ambiguous; the caller leaves it alone."""
79
+ n, m = len(letters), len(units)
80
+ inf = float("inf")
81
+
82
+ def cost(i, j):
83
+ return 0.0 if units[j] in _LETTERS[letters[i]] else _MISMATCH
84
+
85
+ def skip(i):
86
+ return _SKIP_WEAK if letters[i] in _WEAK else _SKIP_LETTER
87
+
88
+ f = [[inf] * (m + 1) for _ in range(n + 1)]
89
+ f[0][0] = 0.0
90
+ for i in range(n + 1):
91
+ for j in range(m + 1):
92
+ if i < n:
93
+ f[i + 1][j] = min(f[i + 1][j], f[i][j] + skip(i))
94
+ if j < m:
95
+ f[i][j + 1] = min(f[i][j + 1], f[i][j] + _SKIP_UNIT)
96
+ if i < n and j < m:
97
+ f[i + 1][j + 1] = min(f[i + 1][j + 1], f[i][j] + cost(i, j))
98
+ b = [[inf] * (m + 1) for _ in range(n + 1)]
99
+ b[n][m] = 0.0
100
+ for i in range(n, -1, -1):
101
+ for j in range(m, -1, -1):
102
+ if i < n:
103
+ b[i][j] = min(b[i][j], b[i + 1][j] + skip(i))
104
+ if j < m:
105
+ b[i][j] = min(b[i][j], b[i][j + 1] + _SKIP_UNIT)
106
+ if i < n and j < m:
107
+ b[i][j] = min(b[i][j], b[i + 1][j + 1] + cost(i, j))
108
+ best, eps = f[n][m], 1e-9
109
+ matched: dict[int, set[int]] = {}
110
+ for i in range(n):
111
+ for j in range(m):
112
+ if cost(i, j) == 0.0 and abs(f[i][j] + b[i + 1][j + 1] - best) < eps:
113
+ matched.setdefault(i, set()).add(j)
114
+ # skipping letter i on a cheapest alignment
115
+ if any(abs(f[i][j] + skip(i) + b[i + 1][j] - best) < eps for j in range(m + 1)):
116
+ matched.setdefault(i, set()).add(-1)
117
+ return matched, best
118
+
119
+
120
+ def archify_word(ipa_word: str, ar_word: str) -> tuple[str, list[str]]:
121
+ """One word: its IPA with symbol tokens, and the letters that could not be placed (ambiguous)."""
122
+ phs = ipa_word.split()
123
+ if any(split_token(p)[0] for p in phs):
124
+ return ipa_word, [] # the phrase writes its symbols itself
125
+ letters, units = _letters(ar_word), _units(phs)
126
+ matched, _ = _align(letters, [u for u, _ in units])
127
+ out, unresolved = list(phs), []
128
+ for i, letter in enumerate(letters):
129
+ sym = LETTER_SYMBOL.get(letter)
130
+ if sym is None:
131
+ continue
132
+ hits = matched.get(i, {-1})
133
+ if len(hits) != 1:
134
+ unresolved.append(letter)
135
+ continue
136
+ j = next(iter(hits))
137
+ if j < 0:
138
+ continue # not said at all (a dropped ق): nothing to rewrite
139
+ for k in units[j][1]:
140
+ out[k] = f"{sym}:{phs[k]}"
141
+ # the vowel of a final ة (and ه written for it): a, e and i are all correct (section 6.3 #5)
142
+ if (out and ar_word.endswith(("ة", "ه")) and "لله" not in ar_word
143
+ and base(out[-1]) in {"a", "e", "i"} and len(out) > 1):
144
+ out[-1] = f"E:{out[-1]}"
145
+ return " ".join(out), unresolved
146
+
147
+
148
+ @lru_cache(maxsize=32768) # the whole phrase bank (~12k variants) and the verb game's forms
149
+ def archify(ar: str, ipa: str) -> str:
150
+ """The variant's IPA with each phoneme of a many-ways letter rewritten as its symbol (see the module doc).
151
+
152
+ A variant whose Arabic and IPA word counts differ is returned as it is."""
153
+ ipa_words, ar_words = ipa.split("|"), ar.split()
154
+ if len(ipa_words) != len(ar_words):
155
+ return ipa
156
+ return " | ".join(archify_word(w.strip(), a)[0] for w, a in zip(ipa_words, ar_words))
157
+
158
+
159
+ def unresolved(ar: str, ipa: str) -> list[tuple[str, str]]:
160
+ """(Arabic word, letter) for every many-ways letter archify() could not place: the phrase must mark it."""
161
+ ipa_words, ar_words = ipa.split("|"), ar.split()
162
+ if len(ipa_words) != len(ar_words):
163
+ return []
164
+ return [(a, letter) for w, a in zip(ipa_words, ar_words) for letter in archify_word(w.strip(), a)[1]]
165
+
166
+
167
+ def diphthong_variants(ar: str, ipa: str) -> list[str]:
168
+ """The same variant with eː said as a j and oː as a w where the Arabic writes ي / و (section 6.3 #8).
169
+
170
+ A diphthong is two tokens against one, so it is a variant of its own rather than a class."""
171
+ ipa_words, ar_words = ipa.split("|"), ar.split()
172
+ if len(ipa_words) != len(ar_words):
173
+ return []
174
+ out = []
175
+ for long, glide, letter in (("eː", "j", "ي"), ("oː", "w", "و")):
176
+ words = [" ".join(f"a {glide}" if ph == long and letter in a else ph for ph in w.split())
177
+ for w, a in zip(ipa_words, ar_words)]
178
+ new = " | ".join(words)
179
+ if new != " | ".join(w.strip() for w in ipa_words):
180
+ out.append(new)
181
+ return out
app/main.py CHANGED
@@ -32,6 +32,7 @@ from .auth import AuthUnavailable, ClassConflict, SupabaseAuth, load_auth
32
  from .phrases import CATEGORIES, GAMES, LESSONS, LEVELS, PHRASES, public_phrases
33
  from .prompt_tts import ENGINES, PromptTTSUnavailable, hebrew_mp3
34
  from .reference import References
 
35
  from .scorer import Scorer, decode_audio, load_scorer
36
  from .speech_evaluation import SpeechEvaluationService
37
  from .texts import read_line, text_phrases
@@ -1169,7 +1170,10 @@ def create_app(scorer: Scorer | None = None, tts: SawtakTTS | None = None,
1169
  s: Scorer = app.state.scorer
1170
  return {"version": build_version(), "build": build_metadata(), "model": MODEL_ID, "tts": app.state.tts is not None,
1171
  "references": app.state.refs is not None, "sign_in": app.state.auth is not None,
1172
- "phoneme_fallbacks": s.vocab.downgraded}
 
 
 
1173
 
1174
  @app.get("/api/tts/{phrase_id}", dependencies=members)
1175
  def tts_phrase(phrase_id: int, voice: str = Query("", max_length=64)):
 
32
  from .phrases import CATEGORIES, GAMES, LESSONS, LEVELS, PHRASES, public_phrases
33
  from .prompt_tts import ENGINES, PromptTTSUnavailable, hebrew_mp3
34
  from .reference import References
35
+ from .rules import check_detectable
36
  from .scorer import Scorer, decode_audio, load_scorer
37
  from .speech_evaluation import SpeechEvaluationService
38
  from .texts import read_line, text_phrases
 
1170
  s: Scorer = app.state.scorer
1171
  return {"version": build_version(), "build": build_metadata(), "model": MODEL_ID, "tts": app.state.tts is not None,
1172
  "references": app.state.refs is not None, "sign_in": app.state.auth is not None,
1173
+ "phoneme_fallbacks": s.vocab.downgraded,
1174
+ # per rule of pronunciation-rules.md: can this model catch it at all (and if not, why)
1175
+ "rules": check_detectable(s.rules, lambda ph: s.vocab.sound_id(ph) is not None, s.vocab.model_id),
1176
+ "dialect_realisations_missing": {k: sorted(v) for k, v in s.vocab.class_missing.items()}}
1177
 
1178
  @app.get("/api/tts/{phrase_id}", dependencies=members)
1179
  def tts_phrase(phrase_id: int, voice: str = Query("", max_length=64)):
app/phones.py ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """The sound a model symbol stands for.
2
+
3
+ The model's vocab is multilingual (392 symbols) and spells one sound several ways: t / t̪ / tʰ / tː,
4
+ r / ɾ, a / æ / ɐ, x / χ. Left alone, a correctly said t loses to its own spelling variant t̪.
5
+ base() names the sound a symbol stands for; Vocab.merge() (scorer.py) pools the posteriors of each sound,
6
+ and rules.py writes its rules in the same names.
7
+ """
8
+ _STRIP = set("ːʰʲ.^[:\"̪̩̝̞̊̃") # length, aspiration, palatalisation, dental, syllabic, nasal ...
9
+ _SAME = {"æ": "a", "ɐ": "a", "ɑ": "a", "ä": "a", "ʌ": "a", "ɪ": "i", "ʊ": "u", "ɛ": "e", "ɔ": "o",
10
+ "ɾ": "r", "ɫ": "l", "ɡ": "g", "χ": "x", "ʁ": "ɣ", "ʒ": "dʒ", "dZ": "dʒ", "tS": "tʃ", "S": "ʃ",
11
+ "d͡ʒ": "dʒ", "t͡ʃ": "tʃ", "t͡s": "ts"}
12
+
13
+
14
+ def base(symbol: str) -> str:
15
+ """The sound a model symbol stands for, ignoring length and spelling variants. Emphatics stay distinct."""
16
+ s = "".join(c for c in symbol if c not in _STRIP)
17
+ return _SAME.get(s, s) or symbol
app/rules.py ADDED
@@ -0,0 +1,237 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Pronunciation rules for Hebrew-speaking learners, read from the yaml blocks of pronunciation-rules.md.
2
+
3
+ The document is written for two audiences: its student sections are served as a guide, and its scorer
4
+ sections (the ```yaml blocks) say what the scorer accepts and how it judges errors:
5
+
6
+ archiphonemes letters said differently in the urban and the rural dialect, and their correct realisations
7
+ checks per rule: (target, heard) -> severity and the Hebrew feedback
8
+ scoring margins per (target, heard) pair, max_tips, ...
9
+ heard_labels the Hebrew name of every heard symbol
10
+
11
+ Every symbol is stored by its base() name (phones.py), the names the scorer compares in, so a rule written
12
+ with ʁ matches a model that heard ɣ. check_detectable() tells which rules a model can catch at all.
13
+ """
14
+ from __future__ import annotations
15
+
16
+ import re
17
+ from dataclasses import dataclass, field
18
+ from functools import lru_cache
19
+ from pathlib import Path
20
+ from typing import Callable
21
+
22
+ import yaml
23
+
24
+ from .phones import base
25
+
26
+ RULES_PATH = Path(__file__).resolve().parent.parent / "static" / "assets" / "pronunciation-rules.md"
27
+
28
+ # Realisations correct only in learned words (ثقافة as saqaafe): accepted where the phrase writes them, not elsewhere.
29
+ LEARNED_ONLY = {"TH": {"s"}, "DH": {"z"}}
30
+ # Classes set by position, not by letter (section 6.3 #5 and #6): the vowel of a final ة, the vowel of a pronoun ending.
31
+ POSITIONAL = {"E": ["a", "e", "i"], "O": ["o", "u"]}
32
+
33
+
34
+ def norm(symbol: str) -> str:
35
+ """A rule symbol in the scorer's names. "_" (nothing heard) and the abstract symbols stay as they are."""
36
+ return symbol if symbol == "_" or symbol.isupper() else base(symbol)
37
+
38
+
39
+ @dataclass(frozen=True)
40
+ class Check:
41
+ rule: str
42
+ target: str
43
+ heard: str
44
+ severity: int
45
+ feedback: str
46
+
47
+
48
+ @dataclass
49
+ class Rules:
50
+ archiphonemes: dict[str, dict[str, list[str]]]
51
+ dialects: list[str]
52
+ optional: bool
53
+ scoring: dict
54
+ checks: dict[tuple[str, str], Check]
55
+ titles: dict[str, str] # rule id -> title
56
+ heard_labels: dict[str, str]
57
+ margins: dict[tuple[str, str], float] = field(default_factory=dict)
58
+
59
+ def members(self, symbol: str, written: str | None = None) -> list[str]:
60
+ """The correct realisations of an abstract symbol, in base() names, first the dialects' own.
61
+
62
+ A learned-only realisation (TH as s) counts only when the phrase writes it (`written`)."""
63
+ if symbol in POSITIONAL:
64
+ return list(POSITIONAL[symbol])
65
+ spec = self.archiphonemes.get(symbol)
66
+ if spec is None:
67
+ return []
68
+ out: list[str] = []
69
+ keys = [*self.dialects, "also_ok", *(["optional"] if self.optional else [])]
70
+ for key in keys:
71
+ for ph in spec.get(key, []):
72
+ ph = base(ph)
73
+ if ph not in out and (ph not in LEARNED_ONLY.get(symbol, ()) or ph == written):
74
+ out.append(ph)
75
+ return out
76
+
77
+ def dialects_of(self, symbol: str, phone: str) -> set[str]:
78
+ """The dialects in which `phone` is a correct realisation of `symbol` (empty: in none, or not dialectal)."""
79
+ spec = self.archiphonemes.get(symbol, {})
80
+ return {d for d in ("urban", "rural") if phone in {base(p) for p in spec.get(d, [])}}
81
+
82
+ def check(self, targets: str | list[str], heard: str | None) -> Check | None:
83
+ """The rule for hearing `heard` instead of the target: the first of `targets` that has one."""
84
+ if not heard:
85
+ return None
86
+ for t in [targets] if isinstance(targets, str) else targets:
87
+ c = self.checks.get((norm(t), norm(heard)))
88
+ if c is not None:
89
+ return c
90
+ return None
91
+
92
+ def margin(self, targets: str | list[str], heard: str | None) -> float:
93
+ """How much likelier than the target `heard` must be before it is a mistake (natural log). The first
94
+ of `targets` (the abstract symbol, then the written phoneme) with a margin of its own decides."""
95
+ default = float(self.scoring.get("margin_default", 0.0))
96
+ for t in [targets] if isinstance(targets, str) else targets:
97
+ m = self.margins.get((norm(t), norm(heard or "_")))
98
+ if m is not None:
99
+ return m
100
+ return default
101
+
102
+ def label(self, heard: str | None) -> str | None:
103
+ """The Hebrew name of a heard symbol, as section 8 gives it."""
104
+ if heard is None:
105
+ return None
106
+ return self.heard_labels.get(heard) or self.heard_labels.get(norm(heard))
107
+
108
+ def interference(self) -> set[str]:
109
+ """Every symbol some rule expects to hear instead of the target: the Hebrew sounds worth naming."""
110
+ return {h for (_, h) in self.checks if h != "_"}
111
+
112
+
113
+ def _blocks(text: str) -> list[dict]:
114
+ out = []
115
+ for block in re.findall(r"```yaml\n(.*?)```", text, re.S):
116
+ for doc in re.split(r"^---$", block, flags=re.M):
117
+ if doc.strip():
118
+ out.append(yaml.safe_load(doc))
119
+ return out
120
+
121
+
122
+ def parse(text: str) -> Rules:
123
+ archi, dialects, optional, scoring, labels = {}, ["urban", "rural"], False, {}, {}
124
+ checks: dict[tuple[str, str], Check] = {}
125
+ titles: dict[str, str] = {}
126
+ for doc in _blocks(text):
127
+ if "archiphonemes" in doc:
128
+ archi = {k: {d: list(v.get(d, [])) for d in ("urban", "rural", "also_ok", "optional")}
129
+ for k, v in doc["archiphonemes"].items()}
130
+ dialects = list(doc.get("dialects_enabled", dialects))
131
+ optional = bool(doc.get("optional_realisations_enabled", False))
132
+ if "scoring" in doc:
133
+ scoring = dict(doc["scoring"])
134
+ if "heard_labels" in doc:
135
+ labels = {str(k): str(v) for k, v in doc["heard_labels"].items()}
136
+ if "checks" in doc:
137
+ titles[doc["id"]] = doc.get("title", doc["id"])
138
+ for c in doc["checks"]:
139
+ for heard in c["heard"]:
140
+ key = (norm(str(c["target"])), norm(str(heard)))
141
+ # two spellings of one sound (ʁ and ɣ) meet here: the first rule written wins
142
+ checks.setdefault(key, Check(doc["id"], key[0], key[1], int(c["severity"]), str(c["feedback"])))
143
+ rules = Rules(archi, dialects, optional, scoring, checks, titles, labels)
144
+ for m in scoring.get("margins", []):
145
+ # base() drops vowel length, so a margin for long vowels would land on quality pairs (a / i) it was not meant for
146
+ for t in (t for t in m["target"] if "ː" not in str(t)):
147
+ for h in m["heard"]:
148
+ rules.margins[(norm(str(t)), norm(str(h)))] = float(m["margin"])
149
+ return rules
150
+
151
+
152
+ @lru_cache(maxsize=1)
153
+ def load(path: str = str(RULES_PATH)) -> Rules:
154
+ return parse(Path(path).read_text(encoding="utf-8"))
155
+
156
+
157
+ def undetectable(check: Check, rules: Rules, in_model: Callable[[str], bool],
158
+ heard_as: Callable[[str], object] | None = None) -> str | None:
159
+ """Why the model cannot catch this mistake, or None if it can.
160
+
161
+ `heard_as` names the model symbol a sound is heard as (stand-ins included: ðˤ as z); two sounds heard
162
+ as the same symbol cannot be told apart."""
163
+ same = (lambda a, b: a == b or (heard_as(a) is not None and heard_as(a) == heard_as(b))) if heard_as \
164
+ else (lambda a, b: a == b)
165
+ if same(check.target, check.heard):
166
+ return "same_sound" # the two collapse to one sound (aː / a: length is not modelled)
167
+ accepted = rules.members(check.target) if check.target.isupper() else [check.target]
168
+ if any(same(check.heard, p) for p in accepted):
169
+ return "accepted" # the 'mistake' is a correct realisation of the target
170
+ if not any(in_model(p) for p in accepted):
171
+ return "target_not_in_model"
172
+ if check.heard != "_" and not in_model(check.heard):
173
+ return "heard_not_in_model"
174
+ return None
175
+
176
+
177
+ def check_detectable(rules: Rules, in_model: Callable[[str], bool],
178
+ heard_as: Callable[[str], object] | None = None) -> dict[str, dict]:
179
+ """Per rule: whether at least one of its checks can fire, and why each of the others cannot."""
180
+ out: dict[str, dict] = {}
181
+ for c in rules.checks.values():
182
+ r = out.setdefault(c.rule, {"title": rules.titles.get(c.rule, c.rule), "detectable": False, "missing": []})
183
+ why = undetectable(c, rules, in_model, heard_as)
184
+ if why is None:
185
+ r["detectable"] = True
186
+ else:
187
+ r["missing"].append({"target": c.target, "heard": c.heard, "why": why})
188
+ return out
189
+
190
+
191
+ # ---------- exemptions (section 6.3): what the model hears as a mistake but is ordinary speech ----------
192
+
193
+ VOWELS = {"a", "i", "u", "e", "o", "aː", "iː", "uː", "eː", "oː"}
194
+ _VOWEL_GROUPS = [set("iɪeɛ"), set("uʊoɔ"), set("aæɐɑəɛe")] # a raised towards e is ordinary Palestinian speech
195
+
196
+
197
+ def near_vowels(expected: str, heard: str | None) -> bool:
198
+ """Same vowel with another length, or the neighbouring quality (i/e, u/o, a/ə, a/e)."""
199
+ if expected not in VOWELS or not heard:
200
+ return False
201
+ e, h = expected.rstrip("ː"), heard.rstrip("ː")
202
+ return any(e in g and h in g for g in _VOWEL_GROUPS)
203
+
204
+
205
+ def elided_article(toks, i: int, ar_words: list[str]) -> bool:
206
+ """Is token i the vowel of the article ال right after a word that ends in a vowel?
207
+
208
+ There it is not said (على الطّاولة is ʕ a l a | tˤ tˤ aː w l e), so its absence is not an error.
209
+ """
210
+ t = toks[i]
211
+ return (i > 0 and t.phoneme in VOWELS and toks[i - 1].word == t.word - 1 and toks[i - 1].phoneme in VOWELS
212
+ and t.word < len(ar_words) and ar_words[t.word].startswith("ال"))
213
+
214
+
215
+ # A glottal stop that is simply not there is how natives say it too (انا, اهلا; ق between vowels): #1, #2.
216
+ # A glottal stop replaced by another consonant still counts, unless the class accepts it (ق said as k).
217
+ def _glottal_dropped(toks, i, sc, heard, ar_words):
218
+ return sc.deleted and toks[i].sound in ("ʔ", "Q")
219
+
220
+
221
+ def _article_vowel_elided(toks, i, sc, heard, ar_words): # #3
222
+ return sc.deleted and elided_article(toks, i, ar_words)
223
+
224
+
225
+ # Neighbouring vowel qualities and vowel length vary between speakers and towns (il / el),
226
+ # and the model confuses them on its own: not worth marking a word down for (#7).
227
+ def _near_vowel(toks, i, sc, heard, ar_words):
228
+ return near_vowels(toks[i].phoneme, heard)
229
+
230
+
231
+ # (name, applies(tokens, index, TokenScore, heard symbol, Arabic words), score it gets: None = full marks,
232
+ # else the name of the ScorerConfig floor). The first that applies wins.
233
+ EXEMPTIONS = [
234
+ ("glottal_dropped", _glottal_dropped, None),
235
+ ("article_vowel_elided", _article_vowel_elided, None),
236
+ ("near_vowel", _near_vowel, "near_vowel_floor"),
237
+ ]
app/scorer.py CHANGED
@@ -11,11 +11,14 @@ from typing import Callable, Protocol
11
 
12
  import numpy as np
13
 
14
- from .align import GopConfig, ctc_viterbi, edit_distance, greedy_decode, score_tokens
15
- from .tips import NO_DELETION_TIP, VOWELS, letter_reason, sound_name, tip_for
16
- from .translit import CONSONANTS as _HEB_C, VOWELS as _HEB_V, to_hebrew, word_letters, word_to_hebrew
 
 
 
17
 
18
- HEBREW_SOUNDS = set(_HEB_C) | set(_HEB_V)
19
 
20
  log = logging.getLogger("ihki.scorer")
21
 
@@ -24,37 +27,24 @@ MAX_SECONDS = 32 # the recording includes the time to read the sentence and thi
24
 
25
  # Fallbacks when the model vocab lacks a phoneme. The first symbol found in the vocab wins.
26
  FALLBACKS = {
27
- "sˤ": ["sˤ", "sˠ", "s"], "tˤ": ["tˤ", "tˠ", "t"], "dˤ": ["dˤ", "dˠ", "d"],
28
  "ðˤ": ["ðˤ", "ðˠ", "zˤ", "z"], "dʒ": ["dʒ", "d͡ʒ", "ʒ"],
29
  "aː": ["aː", "a"], "iː": ["iː", "i"], "uː": ["uː", "u"], "eː": ["eː", "e"], "oː": ["oː", "o"],
30
  "x": ["x", "χ"], "ɣ": ["ɣ", "ʁ"], "q": ["q", "k"], "r": ["r", "ɾ"], "g": ["ɡ", "g"],
31
  }
32
 
33
 
34
- # The model's vocab is multilingual (392 symbols) and spells one sound several ways: t / t̪ / tʰ / tː,
35
- # r / ɾ, a / æ / ɐ, x / χ. Left alone, a correctly said t loses to its own spelling variant t̪.
36
- # base() names the sound a symbol stands for; Vocab.merge() pools the posteriors of each sound.
37
- _STRIP = set("ːʰʲ.^[:\"\u032a\u0329\u031d\u031e\u030a\u0303") # length, aspiration, palatalisation, dental, syllabic, nasal ...
38
- _SAME = {"æ": "a", "ɐ": "a", "ɑ": "a", "ä": "a", "ʌ": "a", "ɪ": "i", "ʊ": "u", "ɛ": "e", "ɔ": "o",
39
- "ɾ": "r", "ɫ": "l", "ɡ": "g", "χ": "x", "ʁ": "ɣ", "ʒ": "dʒ", "dZ": "dʒ", "tS": "tʃ", "S": "ʃ"}
40
-
41
-
42
- def base(symbol: str) -> str:
43
- """The sound a model symbol stands for, ignoring length and spelling variants. Emphatics stay distinct."""
44
- s = "".join(c for c in symbol if c not in _STRIP)
45
- return _SAME.get(s, s) or symbol
46
-
47
-
48
  class Emitter(Protocol):
49
  """Anything that turns 16 kHz mono float32 audio into (T, V) log-posteriors."""
50
  def __call__(self, wav: np.ndarray) -> np.ndarray: ...
51
 
52
 
53
  class Vocab:
54
- def __init__(self, token_to_id: dict[str, int], blank_token: str = "<pad>"):
55
  self.t2i = token_to_id
56
  self.i2t = {i: t for t, i in token_to_id.items()}
57
  self.blank = token_to_id[blank_token]
 
58
  self.downgraded: dict[str, str] = {}
59
  # sound classes: every id points at one representative id (the plain symbol when the vocab has it)
60
  groups: dict[str, list[int]] = {}
@@ -63,12 +53,20 @@ class Vocab:
63
  groups.setdefault(base(t), []).append(i)
64
  self.canon = {i: i for i in self.i2t}
65
  self._groups: list[tuple[int, list[int]]] = []
 
66
  for name, ids in groups.items():
67
  rep = token_to_id.get(name, ids[0]) if token_to_id.get(name) in ids else ids[0]
 
68
  for i in ids:
69
  self.canon[i] = rep
70
  if len(ids) > 1:
71
  self._groups.append((rep, ids))
 
 
 
 
 
 
72
 
73
  def merge(self, logp: np.ndarray) -> np.ndarray:
74
  """Pool each sound's spelling variants into its representative column; the other columns go silent."""
@@ -78,6 +76,53 @@ class Vocab:
78
  out[:, rep] = np.logaddexp.reduce(logp[:, ids], axis=1)
79
  return out
80
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
81
  def resolve(self, phoneme: str) -> list[int]:
82
  for cand in FALLBACKS.get(phoneme, [phoneme]):
83
  if cand in self.t2i:
@@ -97,54 +142,52 @@ class Vocab:
97
  self.downgraded[phoneme] = " ".join(self.i2t[i] for i in ids)
98
  return ids
99
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
100
 
101
  LETTER_GOOD = 75 # a letter scoring under this is coloured on the page (`band` in static/index.html)
102
 
103
 
104
  @dataclass
105
  class TargetToken:
106
- phoneme: str # phoneme as written in phrases.py
107
- id: int # model token id
108
- word: int # word index in the variant
 
 
 
 
109
 
110
 
111
  def tokenize(ipa: str, vocab: Vocab) -> list[TargetToken]:
112
  """IPA string -> model tokens. Geminates (and any repeated id) collapse to one CTC token."""
113
  out: list[TargetToken] = []
114
  for w, word in enumerate(ipa.split("|")):
115
- for ph in word.split():
116
- for tid in vocab.resolve(ph):
 
117
  if out and out[-1].id == tid:
118
  continue
119
- out.append(TargetToken(ph, tid, w))
120
  return out
121
 
122
 
123
- _VOWEL_GROUPS = [set("iɪeɛ"), set("uʊoɔ"), set("aæɐɑəɛe")] # a raised towards e is ordinary Palestinian speech
124
-
125
-
126
- def near_vowels(expected: str, heard: str | None) -> bool:
127
- """Same vowel with another length, or the neighbouring quality (i/e, u/o, a/ə, a/e)."""
128
- if expected not in VOWELS or not heard:
129
- return False
130
- e, h = expected.rstrip("ː"), heard.rstrip("ː")
131
- return any(e in g and h in g for g in _VOWEL_GROUPS)
132
-
133
-
134
- def elided_article(toks: list[TargetToken], i: int, ar_words: list[str]) -> bool:
135
- """Is token i the vowel of the article ال right after a word that ends in a vowel?
136
-
137
- There it is not said (على الطّاولة is ʕ a l a | tˤ tˤ aː w l e), so its absence is not an error.
138
- """
139
- t = toks[i]
140
- return (i > 0 and t.phoneme in VOWELS and toks[i - 1].word == t.word - 1 and toks[i - 1].phoneme in VOWELS
141
- and t.word < len(ar_words) and ar_words[t.word].startswith("ال"))
142
-
143
-
144
  @dataclass
145
  class Reference:
146
  """How the model scored a native voice saying a phrase: the baseline a learner is measured against."""
147
- sounds: list[tuple[int, str]] # (word, phoneme) of each sound of the variant the voice said
148
  scores: list[float] # their scores, same order
149
 
150
 
@@ -154,16 +197,42 @@ class ScorerConfig:
154
  unrelated_ratio: float = 0.65 # normalised edit distance above which we say "different sentence"
155
  issue_below: float = 90.0 # phoneme score under which it is listed as imprecise
156
  max_issues: int = 8
 
157
  near_vowel_floor: float = 90.0 # a vowel heard as its neighbour (i/e, u/o, a/ə, short/long) scores at least this
158
  word_min_weight: float = 0.5 # word score = this much weakest phoneme, the rest the mean of its phonemes
159
  ref_min_mean: float = 60.0 # a native reference whose mean phoneme score is under this is not trusted
160
  gop: GopConfig = field(default_factory=GopConfig)
161
 
162
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
163
  class Scorer:
164
  def __init__(self, emit: Emitter, vocab: Vocab, cfg: ScorerConfig | None = None):
165
  self.emit, self.vocab, self.cfg = emit, vocab, cfg or ScorerConfig()
 
166
  self.calibrations: dict[tuple[str, str, str], float] = {}
 
 
 
 
 
167
 
168
  def set_calibrations(self, rows: list[dict]) -> None:
169
  """Atomically replaces applied score floors. A blank phrase is a global rule."""
@@ -186,18 +255,18 @@ class Scorer:
186
  fit = self._fit(self.vocab.merge(self.emit(wav)), phrase)
187
  if fit is None:
188
  return None
189
- _, _, toks, _, scores, _, unrelated = fit
190
- if unrelated or float(np.mean([sc.score for sc in scores])) < self.cfg.ref_min_mean:
191
  return None
192
- return Reference([(t.word, t.phoneme) for t in toks], [sc.score for sc in scores])
193
 
194
  @staticmethod
195
  def _baseline(toks: list[TargetToken], refs: list[Reference]) -> list[float | None]:
196
  """Per target token, the lowest score a native reference got on the same sound (None: no reference has it).
197
 
198
  References may have said another accepted variant: only the sounds the two variants share are matched.
 
199
  """
200
- sounds = [(t.word, t.phoneme) for t in toks]
201
  out: list[float | None] = [None] * len(toks)
202
  for ref in refs:
203
  for a, b, n in SequenceMatcher(None, ref.sounds, sounds, autojunk=False).get_matching_blocks():
@@ -213,7 +282,7 @@ class Scorer:
213
  row = []
214
  for ph in word.split():
215
  idx = []
216
- for tid in self.vocab.resolve(ph):
217
  tid = self.vocab.canon[tid]
218
  if tid != last:
219
  n, last = n + 1, tid
@@ -236,7 +305,7 @@ class Scorer:
236
  """How each target token is written in the phrase bank, geminates included ("m m")."""
237
  runs: list[tuple[int, str, int]] = [] # (word, phoneme, repeat)
238
  for w, word in enumerate(ipa.split("|")):
239
- for ph in word.split():
240
  if runs and runs[-1][:2] == (w, ph):
241
  runs[-1] = (w, ph, runs[-1][2] + 1)
242
  else:
@@ -250,6 +319,7 @@ class Scorer:
250
 
251
  A sound heard as expected is written the way the answer writes it (length, gemination, emphasis:
252
  things the scorer does not tell apart), so the two lines differ only where the scorer heard a difference.
 
253
  """
254
  n_words = len(ipa.split("|"))
255
  starts = [None] * n_words # first frame of each expected word
@@ -261,57 +331,81 @@ class Scorer:
261
  expected[t] = (tok.id, spell)
262
  heard: list[list[str]] = [[] for _ in range(n_words)]
263
  prev, w = None, 0
264
- for t, i in enumerate(logp.argmax(1)):
265
  i = int(i)
266
  if i != prev and i != self.vocab.blank:
267
  while w + 1 < n_words and starts[w + 1] is not None and t >= starts[w + 1]:
268
  w += 1
269
  ph = base(self.vocab.i2t[i])
270
- if expected.get(t, (None,))[0] == i:
271
- heard[w].append(expected[t][1])
 
 
 
272
  elif ph in HEBREW_SOUNDS:
273
  heard[w].append(ph)
274
  prev = i
275
  # the Arabic spelling is passed so the heard line follows the answer's conventions (ק for ق, silent ל)
276
  return [word_to_hebrew(" ".join(phs), ar_words[w] if w < len(ar_words) else "") for w, phs in enumerate(heard)]
277
 
278
- def _fit(self, logp: np.ndarray, phrase: dict):
279
- """The accepted variant the audio fits best and the score of each of its sounds, from merged
280
- posteriors: (arabic, ipa, tokens, alignment, scores, greedy decode, unrelated), or None if too short."""
281
- blank = self.vocab.blank
282
- heard_ids = greedy_decode(logp, blank)
283
- best = None
284
  for ar, ipa in phrase["variants"]:
285
- toks = self._targets(ipa)
286
- al = ctc_viterbi(logp, [t.id for t in toks], blank)
287
- if al and (best is None or al.total_logp > best[3].total_logp):
288
- best = (ar, ipa, toks, al)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
289
  if best is None:
290
  return None
291
 
292
- ar, ipa, toks, al = best
293
  ids = [t.id for t in toks]
294
  ar_words = ar.split()
295
- scores = score_tokens(logp, al, ids, blank, self.cfg.gop)
296
- # A glottal stop that is simply not there is how natives say it too (انا, اهلا): not an error.
297
- # The same goes for the article's vowel after a vowel (على الطّاولة).
298
- # A glottal stop replaced by another consonant (ق said as k) still counts.
299
- scores = [replace(sc, score=100.0)
300
- if sc.deleted and (t.phoneme in NO_DELETION_TIP or elided_article(toks, i, ar_words)) else sc
301
- for i, (t, sc) in enumerate(zip(toks, scores))]
302
- # Neighbouring vowel qualities and vowel length vary between speakers and towns (il / el),
303
- # and the model confuses them on its own: not worth marking a word down for.
304
- scores = [replace(sc, score=max(sc.score, self.cfg.near_vowel_floor))
305
- if sc.competitor is not None and near_vowels(t.phoneme, self.vocab.i2t.get(sc.competitor)) else sc
306
- for t, sc in zip(toks, scores)]
307
- unrelated = bool(edit_distance(heard_ids, ids) / max(1, len(ids)) > self.cfg.unrelated_ratio)
308
- return ar, ipa, toks, al, scores, heard_ids, unrelated
309
 
310
  def _heatmap(self, logp: np.ndarray, toks: list[TargetToken], alignment, scores,
311
  duration_s: float | None) -> dict:
312
  """A compact posterior heatmap for teacher diagnostics: expected sounds, strongest competitors and blank.
313
 
314
  Max-pooling preserves short phoneme peaks while capping the response at 180 time columns.
 
315
  """
316
  expected_ids = list(dict.fromkeys(t.id for t in toks))
317
  competitor_ids = list(dict.fromkeys(sc.competitor for sc in scores
@@ -319,7 +413,8 @@ class Scorer:
319
  ids = expected_ids + competitor_ids + ([self.vocab.blank] if self.vocab.blank not in expected_ids else [])
320
  frames, bins = len(logp), min(len(logp), 180)
321
  edges = np.linspace(0, frames, bins + 1, dtype=int)
322
- labels = {t.id: t.phoneme for t in toks}
 
323
  rows = []
324
  for token_id in ids:
325
  probability = np.exp(logp[:, token_id])
@@ -327,7 +422,7 @@ class Scorer:
327
  rows.append({"phoneme": labels.get(token_id, "blank" if token_id == self.vocab.blank
328
  else self.vocab.i2t.get(token_id, "?")),
329
  "expected": token_id in expected_ids, "values": values})
330
- greedy = logp.argmax(1)
331
  heard = []
332
  start = 0
333
  for end in range(1, frames + 1):
@@ -343,26 +438,55 @@ class Scorer:
343
  "frames": frames, "frame_bins": bins, "rows": rows,
344
  "tracks": {"expected": expected, "heard": heard}}
345
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
346
  def score_logp(self, logp: np.ndarray, phrase: dict, refs: list[Reference] = (), diagnostics: bool = False,
347
  duration_s: float | None = None) -> dict:
348
  logp = self.vocab.merge(logp)
349
  fit = self._fit(logp, phrase)
350
  if fit is None:
351
  return {"error": "too_short", "message": "ההקלטה קצרה מדי. החזיקו את הכפתור ואמרו את כל המשפט."}
352
- ar, ipa, toks, al, scores, heard_ids, unrelated = fit
353
- heard = " ".join(self.vocab.i2t[i] for i in heard_ids)
 
354
  # Measure against what the model gives a native on the same sound: a sound it misses in the
355
  # reference voice too (ʕ in على heard as dropped) is the model's weakness, not the learner's.
356
- heatmap = self._heatmap(logp, toks, al, scores, duration_s) if diagnostics else None
357
- raw = scores
358
  scores = [sc if b is None or b >= 100 else replace(sc, score=round(min(100.0, 100.0 * sc.score / max(b, 1.0)), 1))
359
- for sc, b in zip(raw, self._baseline(toks, refs))]
 
 
360
  phrase_key, calibrated = str(phrase.get("key", phrase.get("he", ""))), []
361
- adjusted = []
362
  for tok, sc in zip(toks, scores):
363
- heard_ph = "_" if sc.deleted else (self.vocab.i2t.get(sc.competitor) if sc.competitor is not None else "")
364
- floor = self.calibrations.get((tok.phoneme, heard_ph, phrase_key),
365
- self.calibrations.get((tok.phoneme, heard_ph, "")))
 
 
366
  if floor is not None and sc.score < floor:
367
  calibrated.append({"expected": tok.phoneme, "heard": heard_ph, "from": round(sc.score), "to": round(floor)})
368
  sc = replace(sc, score=floor)
@@ -372,16 +496,26 @@ class Scorer:
372
 
373
  phonemes, tips, seen = [], [], set()
374
  heard_as = [] # per phoneme, what took its place ("_": nothing), flagged or not: the reason a letter gives
 
 
375
  ar_words = ar.split()
376
  tr_words = to_hebrew(ipa, ar)
377
  for idx, (tok, sc) in enumerate(zip(toks, scores)):
378
- heard_ph = "_" if sc.deleted else (self.vocab.i2t.get(sc.competitor) if sc.competitor is not None else None)
379
- flagged = bool(sc.score < self.cfg.flag_below) and tok.phoneme not in VOWELS
 
 
380
  heard_as.append(heard_ph)
381
- phonemes.append({"ipa": tok.phoneme, "word": tok.word, "score": sc.score,
382
- "flagged": flagged, "heard": heard_ph if flagged else None})
 
 
 
 
 
 
383
  if flagged and not unrelated:
384
- tip = tip_for(tok.phoneme, heard_ph)
385
  if tip and tip["key"] not in seen:
386
  seen.add(tip["key"])
387
  tip["word"] = ar_words[tok.word] if tok.word < len(ar_words) else ""
@@ -391,9 +525,9 @@ class Scorer:
391
  tips.append(tip)
392
 
393
  n_words = len(ipa.split("|"))
394
- heard_words = self._heard_hebrew(logp, toks, al, ipa, ar_words)
395
  words, k = [], self.cfg.word_min_weight
396
- carriers, ipa_words = self._carriers(ipa), ipa.split("|")
397
  for w in range(n_words):
398
  ws = [p["score"] for p in phonemes if p["word"] == w]
399
  # the transliteration letter by letter, each with the weakest score among its sounds; a letter the
@@ -404,7 +538,7 @@ class Scorer:
404
  worst = min(got, key=lambda i: phonemes[i]["score"]) if got else None
405
  letter = {"t": text, "score": phonemes[worst]["score"] if got else None}
406
  if got and letter["score"] < LETTER_GOOD and not unrelated:
407
- letter.update(letter_reason(phonemes[worst]["ipa"], heard_as[worst]))
408
  letters.append(letter)
409
  if w in forgiven and min(ws) >= self.cfg.issue_below:
410
  heard_words[w] = tr_words[w] # what differs from the answer here is what a native voice gets too
@@ -418,7 +552,10 @@ class Scorer:
418
  overall = 0.5 * phone_mean + 0.5 * word_mean # one wrong consonant must pull the sentence down
419
  if unrelated:
420
  overall = min(overall, 30.0)
421
- tips = sorted(tips, key=lambda t: t["score"])[:4] # most crucial first: the worst-scoring sounds
 
 
 
422
 
423
  # Anything short of 100 must say why: every imprecise sound that did not get a full tip,
424
  # weakest first. If nothing is clearly off, name the weakest few.
@@ -427,25 +564,105 @@ class Scorer:
427
  tipped = {t.pop("idx") for t in tips}
428
  rest = sorted((i for i in range(len(toks)) if i not in tipped and scores[i].score < 100),
429
  key=lambda i: scores[i].score)
430
- picked = [i for i in rest if scores[i].score < self.cfg.issue_below]
431
  if not picked and not tips:
432
  picked = rest[:3]
433
- for i in picked[:self.cfg.max_issues]:
 
434
  tok, sc = toks[i], scores[i]
435
- comp = self.vocab.i2t.get(sc.competitor) if sc.competitor is not None else None
436
- issues.append({"sound": sound_name(tok.phoneme) or tok.phoneme, "word_tr": tr_words[tok.word],
437
- "score": round(sc.score), "missing": bool(sc.deleted),
438
  "heard": None if sc.deleted else sound_name(comp)})
439
  for t in tips:
440
  t.pop("idx", None)
441
  result = {"score": round(overall), "variant": {"ar": ar, "tr": " ".join(tr_words), "ipa": ipa}, "heard_ipa": heard, "heard_tr": " ".join(h for h in heard_words if h),
442
- "unrelated": unrelated, "words": words, "tips": tips, "issues": issues, "references": len(refs)}
 
443
  if calibrated:
444
  result["calibrated"] = calibrated
445
  if heatmap is not None:
446
  result["phoneme_heatmap"] = heatmap
 
 
 
447
  return result
448
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
449
 
450
  # ---------- real model ----------
451
 
@@ -500,8 +717,10 @@ def load_scorer(model_id: str) -> Scorer:
500
  # resolve every phoneme once at startup so missing symbols show up in the logs and /api/health
501
  from .phrases import PHRASES
502
  for p in PHRASES:
503
- for _, ipa in p["variants"]:
504
- tokenize(ipa, vocab)
505
  if vocab.downgraded:
506
  log.warning("phonemes mapped to fallbacks: %s", vocab.downgraded)
 
 
507
  return scorer
 
11
 
12
  import numpy as np
13
 
14
+ from .align import Alignment, GopConfig, TokenScore, ctc_viterbi, edit_distance, greedy_decode, score_tokens
15
+ from .archi import archify, diphthong_variants, split_token, written
16
+ from .phones import base
17
+ from .rules import EXEMPTIONS, Rules, elided_article, near_vowels, load as load_rules # noqa: F401 (re-exported)
18
+ from .tips import VOWELS, letter_reason, sound_name, tip_for
19
+ from .translit import CONSONANTS as _HEB_C, HEARD_ONLY as _HEB_HEARD, VOWELS as _HEB_V, to_hebrew, word_letters, word_to_hebrew
20
 
21
+ HEBREW_SOUNDS = set(_HEB_C) | set(_HEB_V) | set(_HEB_HEARD)
22
 
23
  log = logging.getLogger("ihki.scorer")
24
 
 
27
 
28
  # Fallbacks when the model vocab lacks a phoneme. The first symbol found in the vocab wins.
29
  FALLBACKS = {
30
+ "sˤ": ["sˤ", "sˠ", "s"], "tˤ": ["tˤ", "tˠ", "t"], "dˤ": ["dˤ", "dˠ", "d"], "zˤ": ["zˤ", "zˠ", "z"],
31
  "ðˤ": ["ðˤ", "ðˠ", "zˤ", "z"], "dʒ": ["dʒ", "d͡ʒ", "ʒ"],
32
  "aː": ["aː", "a"], "iː": ["iː", "i"], "uː": ["uː", "u"], "eː": ["eː", "e"], "oː": ["oː", "o"],
33
  "x": ["x", "χ"], "ɣ": ["ɣ", "ʁ"], "q": ["q", "k"], "r": ["r", "ɾ"], "g": ["ɡ", "g"],
34
  }
35
 
36
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
37
  class Emitter(Protocol):
38
  """Anything that turns 16 kHz mono float32 audio into (T, V) log-posteriors."""
39
  def __call__(self, wav: np.ndarray) -> np.ndarray: ...
40
 
41
 
42
  class Vocab:
43
+ def __init__(self, token_to_id: dict[str, int], blank_token: str = "<pad>", rules: Rules | None = None):
44
  self.t2i = token_to_id
45
  self.i2t = {i: t for t, i in token_to_id.items()}
46
  self.blank = token_to_id[blank_token]
47
+ self.rules = rules or load_rules()
48
  self.downgraded: dict[str, str] = {}
49
  # sound classes: every id points at one representative id (the plain symbol when the vocab has it)
50
  groups: dict[str, list[int]] = {}
 
53
  groups.setdefault(base(t), []).append(i)
54
  self.canon = {i: i for i in self.i2t}
55
  self._groups: list[tuple[int, list[int]]] = []
56
+ self.rep_of: dict[str, int] = {} # base() name -> representative id
57
  for name, ids in groups.items():
58
  rep = token_to_id.get(name, ids[0]) if token_to_id.get(name) in ids else ids[0]
59
+ self.rep_of[name] = rep
60
  for i in ids:
61
  self.canon[i] = rep
62
  if len(ids) > 1:
63
  self._groups.append((rep, ids))
64
+ # Letters with several correct realisations (archi.py): one virtual id per (symbol, realisations),
65
+ # numbered after the real vocab, whose column extend() fills with the pooled posterior of its members.
66
+ self.n_real = max(self.i2t) + 1
67
+ self.members: dict[int, set[int]] = {}
68
+ self._classes: dict[tuple[str, frozenset[int]], int] = {}
69
+ self.class_missing: dict[str, set[str]] = {} # symbol -> realisations the model has no symbol for
70
 
71
  def merge(self, logp: np.ndarray) -> np.ndarray:
72
  """Pool each sound's spelling variants into its representative column; the other columns go silent."""
 
76
  out[:, rep] = np.logaddexp.reduce(logp[:, ids], axis=1)
77
  return out
78
 
79
+ def sound_id(self, phone: str) -> int | None:
80
+ """The representative id of a sound (any spelling of it), or None if the model has no symbol for it."""
81
+ return self.rep_of.get(base(phone))
82
+
83
+ def model_id(self, phone: str) -> int | None:
84
+ """The id the model hears `phone` as: its own, else its first FALLBACKS stand-in (ðˤ -> z), else None."""
85
+ for cand in [phone, *FALLBACKS.get(phone, [])]:
86
+ i = self.sound_id(cand)
87
+ if i is not None:
88
+ return i
89
+ return None
90
+
91
+ def add_class(self, symbol: str, phones: list[str]) -> int:
92
+ """The id that stands for 'any of `phones`': a virtual id, or the real one when only one is in the model.
93
+ A realisation the model lacks is heard as its stand-in, as a plain phoneme would be (zˤ as z)."""
94
+ ids: list[int] = []
95
+ for ph in phones:
96
+ i = self.model_id(ph)
97
+ if i is None:
98
+ self.class_missing.setdefault(symbol, set()).add(ph)
99
+ continue
100
+ if i != self.sound_id(ph):
101
+ self.downgraded[ph] = self.i2t[i]
102
+ if i not in ids:
103
+ ids.append(i)
104
+ if not ids:
105
+ raise KeyError(f"no realisation of {symbol} ({phones}) is in the model vocab")
106
+ if len(ids) == 1:
107
+ return ids[0]
108
+ key = (symbol, frozenset(ids))
109
+ if key not in self._classes:
110
+ vid = self.n_real + len(self._classes)
111
+ self._classes[key], self.members[vid] = vid, set(ids)
112
+ self.i2t[vid], self.canon[vid] = symbol, vid
113
+ return self._classes[key]
114
+
115
+ def extend(self, logp: np.ndarray) -> np.ndarray:
116
+ """Merged posteriors with one more column per class: the pooled posterior of its realisations."""
117
+ logp = logp[:, :self.n_real]
118
+ if not self.members:
119
+ return logp
120
+ out = np.full((logp.shape[0], self.n_real + len(self.members)), -1e9, dtype=logp.dtype)
121
+ out[:, :logp.shape[1]] = logp
122
+ for vid, ids in self.members.items():
123
+ out[:, vid] = np.logaddexp.reduce(logp[:, sorted(ids)], axis=1)
124
+ return out
125
+
126
  def resolve(self, phoneme: str) -> list[int]:
127
  for cand in FALLBACKS.get(phoneme, [phoneme]):
128
  if cand in self.t2i:
 
142
  self.downgraded[phoneme] = " ".join(self.i2t[i] for i in ids)
143
  return ids
144
 
145
+ def token(self, tok: str) -> tuple[list[int], str, str]:
146
+ """A token of an IPA string (a phoneme, or an archi.py symbol "Q:ʔ") -> (model ids, phoneme as written, sound).
147
+
148
+ The sound is the symbol for a many-ways letter, the phoneme otherwise."""
149
+ sym, members, ph = split_token(tok)
150
+ if sym is None:
151
+ return self.resolve(tok), tok, tok
152
+ phones = members or self.rules.members(sym, written=base(ph) if ph else None)
153
+ ph = ph or (phones[0] if phones else sym)
154
+ try:
155
+ return [self.add_class(sym, phones)], ph, sym
156
+ except KeyError:
157
+ return self.resolve(ph), ph, sym
158
+
159
 
160
  LETTER_GOOD = 75 # a letter scoring under this is coloured on the page (`band` in static/index.html)
161
 
162
 
163
  @dataclass
164
  class TargetToken:
165
+ phoneme: str # phoneme as written in phrases.py
166
+ id: int # model token id (a virtual id for a letter with several correct realisations)
167
+ word: int # word index in the variant
168
+ sound: str = "" # what the token stands for: the archi.py symbol (Q) or the phoneme itself
169
+
170
+ def __post_init__(self):
171
+ self.sound = self.sound or self.phoneme
172
 
173
 
174
  def tokenize(ipa: str, vocab: Vocab) -> list[TargetToken]:
175
  """IPA string -> model tokens. Geminates (and any repeated id) collapse to one CTC token."""
176
  out: list[TargetToken] = []
177
  for w, word in enumerate(ipa.split("|")):
178
+ for tok in word.split():
179
+ ids, ph, sound = vocab.token(tok)
180
+ for tid in ids:
181
  if out and out[-1].id == tid:
182
  continue
183
+ out.append(TargetToken(ph, tid, w, sound))
184
  return out
185
 
186
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
187
  @dataclass
188
  class Reference:
189
  """How the model scored a native voice saying a phrase: the baseline a learner is measured against."""
190
+ sounds: list[tuple[int, str]] # (word, sound) of each sound of the variant the voice said (Q, not ʔ / k)
191
  scores: list[float] # their scores, same order
192
 
193
 
 
197
  unrelated_ratio: float = 0.65 # normalised edit distance above which we say "different sentence"
198
  issue_below: float = 90.0 # phoneme score under which it is listed as imprecise
199
  max_issues: int = 8
200
+ max_tips: int | None = None # None: as many as the rules document says (scoring.max_tips)
201
  near_vowel_floor: float = 90.0 # a vowel heard as its neighbour (i/e, u/o, a/ə, short/long) scores at least this
202
  word_min_weight: float = 0.5 # word score = this much weakest phoneme, the rest the mean of its phonemes
203
  ref_min_mean: float = 60.0 # a native reference whose mean phoneme score is under this is not trusted
204
  gop: GopConfig = field(default_factory=GopConfig)
205
 
206
 
207
+ @dataclass
208
+ class Fit:
209
+ """The accepted variant the audio fits best, and how each of its sounds scored."""
210
+ ar: str
211
+ ipa: str # as written in the phrase bank (display, transliteration)
212
+ aipa: str # the same with archi.py symbols, one token per phoneme of `ipa`
213
+ toks: list[TargetToken]
214
+ al: Alignment
215
+ scores: list[TokenScore]
216
+ heard_ids: list[int] # greedy decode, real symbols only
217
+ unrelated: bool
218
+ logp: np.ndarray # merged posteriors with the class columns (Vocab.extend)
219
+ # for the teacher's stage-by-stage view (Scorer._trace)
220
+ candidates: list[tuple[str, str, float | None]] = field(default_factory=list) # (ar, ipa, path log-prob)
221
+ gop_scores: list[TokenScore] = field(default_factory=list) # before the exemptions
222
+ exempted: list[str | None] = field(default_factory=list) # per token, the exemption that applied
223
+ distance: int = 0 # edit distance of the greedy decode from the chosen variant
224
+
225
+
226
  class Scorer:
227
  def __init__(self, emit: Emitter, vocab: Vocab, cfg: ScorerConfig | None = None):
228
  self.emit, self.vocab, self.cfg = emit, vocab, cfg or ScorerConfig()
229
+ self.rules = vocab.rules
230
  self.calibrations: dict[tuple[str, str, str], float] = {}
231
+ # the sounds a competitor is named by: Arabic, the Hebrew sounds the rules expect instead, and their labels;
232
+ # anything else the multilingual model offers (clicks, retroflexes) is reported as "not clear"
233
+ names = (set(HEBREW_SOUNDS) | self.rules.interference() | set(self.rules.heard_labels)
234
+ | {p for s in self.rules.archiphonemes for p in self.rules.members(s)})
235
+ self.named = {i for i in (vocab.sound_id(n) for n in names) if i is not None}
236
 
237
  def set_calibrations(self, rows: list[dict]) -> None:
238
  """Atomically replaces applied score floors. A blank phrase is a global rule."""
 
255
  fit = self._fit(self.vocab.merge(self.emit(wav)), phrase)
256
  if fit is None:
257
  return None
258
+ if fit.unrelated or float(np.mean([sc.score for sc in fit.scores])) < self.cfg.ref_min_mean:
 
259
  return None
260
+ return Reference([(t.word, t.sound) for t in fit.toks], [sc.score for sc in fit.scores])
261
 
262
  @staticmethod
263
  def _baseline(toks: list[TargetToken], refs: list[Reference]) -> list[float | None]:
264
  """Per target token, the lowest score a native reference got on the same sound (None: no reference has it).
265
 
266
  References may have said another accepted variant: only the sounds the two variants share are matched.
267
+ Sounds are compared by symbol, so a voice that said ق as q is a baseline for a learner who says ʔ.
268
  """
269
+ sounds = [(t.word, t.sound) for t in toks]
270
  out: list[float | None] = [None] * len(toks)
271
  for ref in refs:
272
  for a, b, n in SequenceMatcher(None, ref.sounds, sounds, autojunk=False).get_matching_blocks():
 
282
  row = []
283
  for ph in word.split():
284
  idx = []
285
+ for tid in self.vocab.token(ph)[0]:
286
  tid = self.vocab.canon[tid]
287
  if tid != last:
288
  n, last = n + 1, tid
 
305
  """How each target token is written in the phrase bank, geminates included ("m m")."""
306
  runs: list[tuple[int, str, int]] = [] # (word, phoneme, repeat)
307
  for w, word in enumerate(ipa.split("|")):
308
+ for ph in map(written, word.split()):
309
  if runs and runs[-1][:2] == (w, ph):
310
  runs[-1] = (w, ph, runs[-1][2] + 1)
311
  else:
 
319
 
320
  A sound heard as expected is written the way the answer writes it (length, gemination, emphasis:
321
  things the scorer does not tell apart), so the two lines differ only where the scorer heard a difference.
322
+ A letter with several correct realisations is written as the one said (כּ for a rural ق), not coloured.
323
  """
324
  n_words = len(ipa.split("|"))
325
  starts = [None] * n_words # first frame of each expected word
 
331
  expected[t] = (tok.id, spell)
332
  heard: list[list[str]] = [[] for _ in range(n_words)]
333
  prev, w = None, 0
334
+ for t, i in enumerate(logp[:, :self.vocab.n_real].argmax(1)):
335
  i = int(i)
336
  if i != prev and i != self.vocab.blank:
337
  while w + 1 < n_words and starts[w + 1] is not None and t >= starts[w + 1]:
338
  w += 1
339
  ph = base(self.vocab.i2t[i])
340
+ tid, spell = expected.get(t, (None, ""))
341
+ if tid == i:
342
+ heard[w].append(spell)
343
+ elif i in self.vocab.members.get(tid, ()):
344
+ heard[w].append(spell if base(spell.split()[0]) == ph else ph)
345
  elif ph in HEBREW_SOUNDS:
346
  heard[w].append(ph)
347
  prev = i
348
  # the Arabic spelling is passed so the heard line follows the answer's conventions (ק for ق, silent ל)
349
  return [word_to_hebrew(" ".join(phs), ar_words[w] if w < len(ar_words) else "") for w, phs in enumerate(heard)]
350
 
351
+ def _variants(self, phrase: dict) -> list[tuple[str, str, str, list[TargetToken]]]:
352
+ """(arabic, ipa, ipa with symbols, tokens) for each accepted variant and its diphthong forms. Variants
353
+ that differ only in how a many-ways letter is said (ʔ / q / k for ق) become one: the first written.
354
+ A diphthong form the model has no symbols for is left out."""
355
+ out, seen = [], set()
 
356
  for ar, ipa in phrase["variants"]:
357
+ for n, form in enumerate((ipa, *diphthong_variants(ar, ipa))):
358
+ aipa = archify(ar, form)
359
+ if aipa in seen:
360
+ continue
361
+ seen.add(aipa)
362
+ try:
363
+ out.append((ar, form, aipa, self._targets(aipa)))
364
+ except KeyError:
365
+ if n == 0:
366
+ raise
367
+ return out
368
+
369
+ def _fit(self, logp: np.ndarray, phrase: dict) -> Fit | None:
370
+ """The accepted variant the audio fits best and the score of each of its sounds, from merged
371
+ posteriors, or None if the recording is too short for every variant."""
372
+ blank, vocab = self.vocab.blank, self.vocab
373
+ heard_ids = greedy_decode(logp[:, :vocab.n_real], blank)
374
+ cands = self._variants(phrase)
375
+ ext = vocab.extend(logp) # after tokenising: a variant may have brought a new class
376
+ best, tried = None, []
377
+ for ar, ipa, aipa, toks in cands:
378
+ al = ctc_viterbi(ext, [t.id for t in toks], blank)
379
+ tried.append((ar, ipa, round(al.total_logp, 2) if al else None))
380
+ if al and (best is None or al.total_logp > best[4].total_logp):
381
+ best = (ar, ipa, aipa, toks, al)
382
  if best is None:
383
  return None
384
 
385
+ ar, ipa, aipa, toks, al = best
386
  ids = [t.id for t in toks]
387
  ar_words = ar.split()
388
+ scores = score_tokens(ext, al, ids, blank, self.cfg.gop, [vocab.members.get(i) for i in ids],
389
+ vocab.n_real, self.named)
390
+ gop_scores, exempted = list(scores), [None] * len(scores)
391
+ for i, (t, sc) in enumerate(zip(toks, scores)):
392
+ heard = vocab.i2t.get(sc.competitor) if sc.competitor is not None else None
393
+ for name, applies, floor in EXEMPTIONS:
394
+ if applies(toks, i, sc, heard, ar_words):
395
+ scores[i] = replace(sc, score=max(sc.score, 100.0 if floor is None else getattr(self.cfg, floor)))
396
+ exempted[i] = name
397
+ break
398
+ same = lambda h, t: h == t or h in vocab.members.get(t, ())
399
+ distance = edit_distance(heard_ids, ids, same)
400
+ unrelated = bool(distance / max(1, len(ids)) > self.cfg.unrelated_ratio)
401
+ return Fit(ar, ipa, aipa, toks, al, scores, heard_ids, unrelated, ext, tried, gop_scores, exempted, distance)
402
 
403
  def _heatmap(self, logp: np.ndarray, toks: list[TargetToken], alignment, scores,
404
  duration_s: float | None) -> dict:
405
  """A compact posterior heatmap for teacher diagnostics: expected sounds, strongest competitors and blank.
406
 
407
  Max-pooling preserves short phoneme peaks while capping the response at 180 time columns.
408
+ A letter with several correct realisations is one row, the pooled posterior, labelled "ʔ/k/q".
409
  """
410
  expected_ids = list(dict.fromkeys(t.id for t in toks))
411
  competitor_ids = list(dict.fromkeys(sc.competitor for sc in scores
 
413
  ids = expected_ids + competitor_ids + ([self.vocab.blank] if self.vocab.blank not in expected_ids else [])
414
  frames, bins = len(logp), min(len(logp), 180)
415
  edges = np.linspace(0, frames, bins + 1, dtype=int)
416
+ labels = {t.id: ("/".join(self.vocab.i2t[m] for m in sorted(self.vocab.members[t.id]))
417
+ if t.id in self.vocab.members else t.phoneme) for t in toks}
418
  rows = []
419
  for token_id in ids:
420
  probability = np.exp(logp[:, token_id])
 
422
  rows.append({"phoneme": labels.get(token_id, "blank" if token_id == self.vocab.blank
423
  else self.vocab.i2t.get(token_id, "?")),
424
  "expected": token_id in expected_ids, "values": values})
425
+ greedy = logp[:, :self.vocab.n_real].argmax(1)
426
  heard = []
427
  start = 0
428
  for end in range(1, frames + 1):
 
438
  "frames": frames, "frame_bins": bins, "rows": rows,
439
  "tracks": {"expected": expected, "heard": heard}}
440
 
441
+ def _beyond_margin(self, tok: TargetToken, sc: TokenScore, heard: str | None) -> bool:
442
+ """Is the competitor likelier than the target by more than the pair's margin? Pairs the model confuses
443
+ on natives too (emphatics, a dropped h) need a clearer win before they count as a mistake."""
444
+ targets = [tok.sound, tok.phoneme]
445
+ if sc.deleted:
446
+ return sc.peak <= self.cfg.gop.deleted_peak - self.rules.margin(targets, "_")
447
+ if heard is None:
448
+ return True
449
+ return -sc.gop >= self.rules.margin(targets, heard)
450
+
451
+ def _dialect(self, toks: list[TargetToken], scores: list[TokenScore]) -> dict:
452
+ """Which dialect the accepted realisations point to: {"dialect": urban | rural} when they agree,
453
+ {"mixed_dialect": True} when one is urban-only and another rural-only. Never part of the score."""
454
+ seen: set[str] = set()
455
+ for t, sc in zip(toks, scores):
456
+ if sc.realised is None or sc.score < self.cfg.issue_below:
457
+ continue
458
+ ds = self.rules.dialects_of(t.sound, base(self.vocab.i2t[sc.realised]))
459
+ if len(ds) == 1:
460
+ seen |= ds
461
+ if len(seen) > 1:
462
+ return {"mixed_dialect": True}
463
+ return {"dialect": next(iter(seen))} if seen else {}
464
+
465
  def score_logp(self, logp: np.ndarray, phrase: dict, refs: list[Reference] = (), diagnostics: bool = False,
466
  duration_s: float | None = None) -> dict:
467
  logp = self.vocab.merge(logp)
468
  fit = self._fit(logp, phrase)
469
  if fit is None:
470
  return {"error": "too_short", "message": "ההקלטה קצרה מדי. החזיקו את הכפתור ואמרו את כל המשפט."}
471
+ ar, ipa, toks, al, scores, unrelated = fit.ar, fit.ipa, fit.toks, fit.al, fit.scores, fit.unrelated
472
+ i2t = self.vocab.i2t
473
+ heard = " ".join(i2t[i] for i in fit.heard_ids)
474
  # Measure against what the model gives a native on the same sound: a sound it misses in the
475
  # reference voice too (ʕ in على heard as dropped) is the model's weakness, not the learner's.
476
+ heatmap = self._heatmap(fit.logp, toks, al, scores, duration_s) if diagnostics else None
477
+ raw, bases = scores, self._baseline(toks, refs)
478
  scores = [sc if b is None or b >= 100 else replace(sc, score=round(min(100.0, 100.0 * sc.score / max(b, 1.0)), 1))
479
+ for sc, b in zip(raw, bases)]
480
+ after_baseline = scores
481
+ # teacher floors, by symbol first (Q), then by the phoneme written (rows made before symbols existed)
482
  phrase_key, calibrated = str(phrase.get("key", phrase.get("he", ""))), []
483
+ adjusted, floors = [], []
484
  for tok, sc in zip(toks, scores):
485
+ heard_ph = "_" if sc.deleted else (i2t.get(sc.competitor) if sc.competitor is not None else "")
486
+ floor = next((self.calibrations[k] for k in ((tok.sound, heard_ph, phrase_key), (tok.sound, heard_ph, ""),
487
+ (tok.phoneme, heard_ph, phrase_key), (tok.phoneme, heard_ph, ""))
488
+ if k in self.calibrations), None)
489
+ floors.append(floor)
490
  if floor is not None and sc.score < floor:
491
  calibrated.append({"expected": tok.phoneme, "heard": heard_ph, "from": round(sc.score), "to": round(floor)})
492
  sc = replace(sc, score=floor)
 
496
 
497
  phonemes, tips, seen = [], [], set()
498
  heard_as = [] # per phoneme, what took its place ("_": nothing), flagged or not: the reason a letter gives
499
+ checks = [] # per phoneme, the rule for what took its place (None: no rule)
500
+ beyond = [] # per phoneme, whether what took its place won by the pair's margin
501
  ar_words = ar.split()
502
  tr_words = to_hebrew(ipa, ar)
503
  for idx, (tok, sc) in enumerate(zip(toks, scores)):
504
+ heard_ph = "_" if sc.deleted else (i2t.get(sc.competitor) if sc.competitor is not None else None)
505
+ check = self.rules.check([tok.sound, tok.phoneme], heard_ph)
506
+ beyond.append(self._beyond_margin(tok, sc, heard_ph))
507
+ flagged = bool(sc.score < self.cfg.flag_below) and tok.phoneme not in VOWELS and beyond[-1]
508
  heard_as.append(heard_ph)
509
+ checks.append(check)
510
+ p = {"ipa": tok.phoneme, "word": tok.word, "score": sc.score,
511
+ "flagged": flagged, "heard": heard_ph if flagged else None}
512
+ if tok.sound != tok.phoneme:
513
+ p["sound"] = tok.sound
514
+ if sc.realised is not None:
515
+ p["realised"] = base(i2t[sc.realised])
516
+ phonemes.append(p)
517
  if flagged and not unrelated:
518
+ tip = tip_for(tok.sound, heard_ph, check)
519
  if tip and tip["key"] not in seen:
520
  seen.add(tip["key"])
521
  tip["word"] = ar_words[tok.word] if tok.word < len(ar_words) else ""
 
525
  tips.append(tip)
526
 
527
  n_words = len(ipa.split("|"))
528
+ heard_words = self._heard_hebrew(logp, toks, al, fit.aipa, ar_words)
529
  words, k = [], self.cfg.word_min_weight
530
+ carriers, ipa_words = self._carriers(fit.aipa), ipa.split("|")
531
  for w in range(n_words):
532
  ws = [p["score"] for p in phonemes if p["word"] == w]
533
  # the transliteration letter by letter, each with the weakest score among its sounds; a letter the
 
538
  worst = min(got, key=lambda i: phonemes[i]["score"]) if got else None
539
  letter = {"t": text, "score": phonemes[worst]["score"] if got else None}
540
  if got and letter["score"] < LETTER_GOOD and not unrelated:
541
+ letter.update(letter_reason(toks[worst].sound, heard_as[worst], checks[worst]))
542
  letters.append(letter)
543
  if w in forgiven and min(ws) >= self.cfg.issue_below:
544
  heard_words[w] = tr_words[w] # what differs from the answer here is what a native voice gets too
 
552
  overall = 0.5 * phone_mean + 0.5 * word_mean # one wrong consonant must pull the sentence down
553
  if unrelated:
554
  overall = min(overall, 30.0)
555
+ # most crucial first: the mistakes that change meaning (severity 3), then the worst-scoring sounds
556
+ max_tips = self.cfg.max_tips or int(self.rules.scoring.get("max_tips", 4))
557
+ tips = sorted(tips, key=lambda t: (-t["severity"], t["score"]))[:max_tips]
558
+ tip_idx, issue_idx = {t["idx"] for t in tips}, []
559
 
560
  # Anything short of 100 must say why: every imprecise sound that did not get a full tip,
561
  # weakest first. If nothing is clearly off, name the weakest few.
 
564
  tipped = {t.pop("idx") for t in tips}
565
  rest = sorted((i for i in range(len(toks)) if i not in tipped and scores[i].score < 100),
566
  key=lambda i: scores[i].score)
567
+ picked = [i for i in rest if scores[i].score < self.cfg.issue_below and beyond[i]]
568
  if not picked and not tips:
569
  picked = rest[:3]
570
+ issue_idx = picked[:self.cfg.max_issues]
571
+ for i in issue_idx:
572
  tok, sc = toks[i], scores[i]
573
+ comp = i2t.get(sc.competitor) if sc.competitor is not None else None
574
+ issues.append({"sound": sound_name(tok.sound) or sound_name(tok.phoneme) or tok.phoneme,
575
+ "word_tr": tr_words[tok.word], "score": round(sc.score), "missing": bool(sc.deleted),
576
  "heard": None if sc.deleted else sound_name(comp)})
577
  for t in tips:
578
  t.pop("idx", None)
579
  result = {"score": round(overall), "variant": {"ar": ar, "tr": " ".join(tr_words), "ipa": ipa}, "heard_ipa": heard, "heard_tr": " ".join(h for h in heard_words if h),
580
+ "unrelated": unrelated, "words": words, "tips": tips, "issues": issues, "references": len(refs),
581
+ **self._dialect(toks, scores)}
582
  if calibrated:
583
  result["calibrated"] = calibrated
584
  if heatmap is not None:
585
  result["phoneme_heatmap"] = heatmap
586
+ if diagnostics:
587
+ result["trace"] = self._trace(fit, refs, bases, after_baseline, floors, scores, phonemes, checks, beyond,
588
+ words, tips, tip_idx, set(issue_idx), phone_mean, word_mean, overall, heard)
589
  return result
590
 
591
+ def _trace(self, fit: Fit, refs, bases, after_baseline, floors, scores, phonemes, checks, beyond,
592
+ words, tips, tip_idx, issue_idx, phone_mean, word_mean, overall, heard) -> dict:
593
+ """Every stage of the classification with its numbers, weights and thresholds, for the teacher's view
594
+ (the 'stages' part of the phoneme analysis). Nothing here changes the result."""
595
+ cfg, g, i2t = self.cfg, self.cfg.gop, self.vocab.i2t
596
+ p = lambda lp: round(float(np.exp(lp)), 4) if lp > -1e20 else 0.0
597
+ max_tips = cfg.max_tips or int(self.rules.scoring.get("max_tips", 4))
598
+ n = len(fit.toks)
599
+ tokens = []
600
+ for i, (tok, gsc, sc) in enumerate(zip(fit.toks, fit.gop_scores, scores)):
601
+ heard_ph = "_" if gsc.deleted else (i2t.get(gsc.competitor) if gsc.competitor is not None else None)
602
+ members = self.vocab.members.get(tok.id)
603
+ targets = [tok.sound, tok.phoneme]
604
+ confidence = min(1.0, p(gsc.peak) / g.confident_peak)
605
+ check = checks[i]
606
+ b = bases[i]
607
+ stages = [
608
+ {"stage": "gop", "score": gsc.score},
609
+ {"stage": "exemption", "score": fit.scores[i].score, "applied": fit.exempted[i]},
610
+ {"stage": "baseline", "score": after_baseline[i].score, "ref": b,
611
+ "applied": b is not None and b < 100},
612
+ {"stage": "calibration", "score": sc.score, "floor": floors[i],
613
+ "applied": floors[i] is not None and after_baseline[i].score < floors[i]},
614
+ ]
615
+ tokens.append({
616
+ "i": i, "word": tok.word, "ipa": tok.phoneme, "sound": tok.sound,
617
+ "class": sorted(base(i2t[m]) for m in members) if members else None,
618
+ "realised": base(i2t[gsc.realised]) if gsc.realised is not None else None,
619
+ "span": list(fit.al.spans[i]), "window": list(gsc.window),
620
+ "p_target": p(gsc.peak), "competitor": heard_ph if not gsc.deleted else None,
621
+ "competitor_other": gsc.other, "p_competitor": p(gsc.comp_peak),
622
+ "gop": gsc.gop, "confidence": round(confidence, 3),
623
+ "gop_factor": round(float(np.exp(gsc.gop * g.leniency)), 3),
624
+ "confidence_factor": round(float(np.sqrt(confidence)), 3),
625
+ "deleted": gsc.deleted, "stages": stages, "final": sc.score,
626
+ "margin": {"needed": self.rules.margin(targets, "_" if gsc.deleted else heard_ph),
627
+ "got": (round(g.deleted_peak - gsc.peak, 3) if gsc.deleted else round(-gsc.gop, 3)),
628
+ "beyond": beyond[i]},
629
+ "vowel": tok.phoneme in VOWELS, "weight": 0.5 if tok.phoneme in VOWELS else 1.0,
630
+ "flagged": phonemes[i]["flagged"],
631
+ "rule": {"id": check.rule, "title": self.rules.titles.get(check.rule, check.rule),
632
+ "severity": check.severity} if check else None,
633
+ "tip": i in tip_idx, "issue": i in issue_idx,
634
+ })
635
+ return {
636
+ "config": {
637
+ "leniency": g.leniency, "confident_peak": g.confident_peak,
638
+ "deleted_peak": round(float(np.exp(g.deleted_peak)), 3),
639
+ "deleted_comp": round(float(np.exp(g.deleted_comp)), 3),
640
+ "flag_below": cfg.flag_below, "issue_below": cfg.issue_below, "near_vowel_floor": cfg.near_vowel_floor,
641
+ "word_min_weight": cfg.word_min_weight, "vowel_weight": 0.5, "unrelated_ratio": cfg.unrelated_ratio,
642
+ "unrelated_cap": 30.0, "letter_good": LETTER_GOOD, "max_tips": max_tips, "max_issues": cfg.max_issues,
643
+ "ref_min_mean": cfg.ref_min_mean, "margin_default": float(self.rules.scoring.get("margin_default", 0.0)),
644
+ "margins": [{"target": t, "heard": h, "margin": m} for (t, h), m in sorted(self.rules.margins.items())],
645
+ "exemptions": [name for name, _, _ in EXEMPTIONS],
646
+ },
647
+ "variants": [{"ar": ar, "ipa": ipa, "logp": lp, "chosen": ar == fit.ar and ipa == fit.ipa}
648
+ for ar, ipa, lp in fit.candidates],
649
+ "aipa": fit.aipa,
650
+ "unrelated": {"heard": heard, "distance": fit.distance, "tokens": n,
651
+ "ratio": round(fit.distance / max(1, n), 3), "threshold": cfg.unrelated_ratio,
652
+ "unrelated": fit.unrelated},
653
+ "references": [{"mean": round(float(np.mean(r.scores)), 1), "sounds": len(r.sounds)} for r in refs],
654
+ "tokens": tokens,
655
+ "words": [{"tr": w["tr"], "scores": [ph["score"] for ph in w["phonemes"]],
656
+ "min": min((ph["score"] for ph in w["phonemes"]), default=0.0),
657
+ "mean": round(float(np.mean([ph["score"] for ph in w["phonemes"]])), 1) if w["phonemes"] else 0.0,
658
+ "score": w["score"]} for w in words],
659
+ "sentence": {"phone_mean": round(phone_mean, 1), "word_mean": round(word_mean, 1),
660
+ "phone_weight": 0.5, "word_weight": 0.5,
661
+ "before_cap": round(0.5 * phone_mean + 0.5 * word_mean, 1), "final": round(overall),
662
+ "capped": fit.unrelated},
663
+ "tips": [{"key": t["key"], "severity": t["severity"], "score": t["score"]} for t in tips],
664
+ }
665
+
666
 
667
  # ---------- real model ----------
668
 
 
717
  # resolve every phoneme once at startup so missing symbols show up in the logs and /api/health
718
  from .phrases import PHRASES
719
  for p in PHRASES:
720
+ for ar, ipa in p["variants"]:
721
+ tokenize(archify(ar, ipa), vocab)
722
  if vocab.downgraded:
723
  log.warning("phonemes mapped to fallbacks: %s", vocab.downgraded)
724
+ if vocab.class_missing:
725
+ log.warning("realisations the model cannot hear, left out of their letter's class: %s", vocab.class_missing)
726
  return scorer
app/tips.py CHANGED
@@ -5,6 +5,8 @@ Phonemes are in the IPA used by phrases.py; HEARD_ALIASES maps model outputs ont
5
  """
6
  import re
7
 
 
 
8
  NAMES = {
9
  "ħ": "ח (ح)", "ʕ": "ע (ع)", "x": "ח' (خ)", "ɣ": "ע' (غ)", "r": "ר (ر)",
10
  "q": "ק (ق)", "ʔ": "עצירה סותמית (ء / ق)", "h": "ה (ه)",
@@ -12,6 +14,9 @@ NAMES = {
12
  "k": "כ (ك)", "s": "ס (س)", "t": "ת (ت)", "d": "ד (د)", "z": "ז (ز)",
13
  "ʃ": "ש (ش)", "dʒ": "ג' (ج)", "b": "ב (ب)", "f": "פ (ف)", "m": "מ (م)",
14
  "n": "נ (ن)", "l": "ל (ل)", "w": "ו (و)", "j": "י (ي)",
 
 
 
15
  }
16
 
17
  # model output symbol -> symbol used in TIPS keys
@@ -49,6 +54,7 @@ TIPS = {
49
 
50
  # bare Arabic letter in a tip text -> "Hebrew (Arabic)", so Arabic never appears without a Hebrew reading
51
  _PAIRED = {name[name.index("(") + 1]: name for name in NAMES.values() if "/" not in name}
 
52
  _BARE_AR = re.compile(r"(?<!\()[\u0621-\u064a](?![\u0621-\u064a])")
53
 
54
 
@@ -64,14 +70,17 @@ VOWEL_NAMES = {
64
 
65
 
66
  def sound_name(phoneme: str | None) -> str | None:
67
- """Hebrew name of a phoneme (expected or heard), or None if we have no name for it."""
 
 
 
68
  phoneme = HEARD_ALIASES.get(phoneme, phoneme)
69
- return NAMES.get(phoneme) or VOWEL_NAMES.get(phoneme)
70
 
71
 
72
- def letter_reason(expected: str, heard: str | None) -> dict:
73
  """Why a letter of the answer is coloured: {"why": the sound and what was heard in its place ("_": nothing was),
74
- "tip": how to say it, for the substitutions TIPS covers}."""
75
  name = sound_name(expected) or expected
76
  heard = HEARD_ALIASES.get(heard, heard)
77
  other = sound_name(heard)
@@ -81,27 +90,34 @@ def letter_reason(expected: str, heard: str | None) -> dict:
81
  why = f"{name} נשמעה כמו {other}"
82
  else:
83
  why = f"{name} לא נשמעה בבירור"
84
- text = TIPS.get((expected, heard))
85
  return {"why": why, **({"tip": with_hebrew(text)} if text else {})}
86
 
87
 
88
  VOWELS = {"a", "i", "u", "e", "o", "aː", "iː", "uː", "eː", "oː"}
89
- # the urban glottal stop is often dropped by natives too; don't nag about it
90
- NO_DELETION_TIP = {"ʔ"}
91
 
92
 
93
- def tip_for(expected: str, heard: str | None) -> dict | None:
94
- """Feedback for one flagged phoneme, or None if there is nothing useful to say."""
 
 
 
95
  if expected in VOWELS:
96
  return None
97
  heard = HEARD_ALIASES.get(heard, heard) if heard else "_"
98
  if heard == "_" and expected in NO_DELETION_TIP:
99
  return None
100
  name = NAMES.get(expected, expected)
 
 
 
101
  text = TIPS.get((expected, heard))
102
  if text:
103
- return {"title": name, "text": with_hebrew(text), "key": f"{expected}>{heard}"}
104
  if heard == "_":
105
- return {"title": name, "text": "לא נשמעה בבירור.", "key": f"{expected}>_"}
106
- text = f"נשמעה כמו {NAMES[heard]}." if heard in NAMES else "נשמעה אחרת מהצפוי."
107
- return {"title": name, "text": text, "key": f"{expected}>{heard}"}
 
 
5
  """
6
  import re
7
 
8
+ from .rules import load as load_rules
9
+
10
  NAMES = {
11
  "ħ": "ח (ح)", "ʕ": "ע (ع)", "x": "ח' (خ)", "ɣ": "ע' (غ)", "r": "ר (ر)",
12
  "q": "ק (ق)", "ʔ": "עצירה סותמית (ء / ق)", "h": "ה (ه)",
 
14
  "k": "כ (ك)", "s": "ס (س)", "t": "ת (ت)", "d": "ד (د)", "z": "ז (ز)",
15
  "ʃ": "ש (ش)", "dʒ": "ג' (ج)", "b": "ב (ب)", "f": "פ (ف)", "m": "מ (م)",
16
  "n": "נ (ن)", "l": "ל (ل)", "w": "ו (و)", "j": "י (ي)",
17
+ "tʃ": "צ' (tʃ)", "θ": "ת' (th)", "ð": "ד״ (th)",
18
+ # letters with several correct realisations (archi.py): named by the letter, not by one way of saying it
19
+ "Q": "ק (ق)", "K": "כ (ك)", "TH": "ת' (ث)", "DH": "ד״ (ذ)", "ZH": "ט' (ظ)", "J": "ג' (ج)",
20
  }
21
 
22
  # model output symbol -> symbol used in TIPS keys
 
54
 
55
  # bare Arabic letter in a tip text -> "Hebrew (Arabic)", so Arabic never appears without a Hebrew reading
56
  _PAIRED = {name[name.index("(") + 1]: name for name in NAMES.values() if "/" not in name}
57
+ DEFAULT_SEVERITY = 2 # a tip no rule in pronunciation-rules.md grades
58
  _BARE_AR = re.compile(r"(?<!\()[\u0621-\u064a](?![\u0621-\u064a])")
59
 
60
 
 
70
 
71
 
72
  def sound_name(phoneme: str | None) -> str | None:
73
+ """Hebrew name of a phoneme (expected or heard), or None if we have no name for it.
74
+ Sounds Arabic does not have (p, v, ts) are named by the labels of pronunciation-rules.md."""
75
+ if phoneme is None:
76
+ return None
77
  phoneme = HEARD_ALIASES.get(phoneme, phoneme)
78
+ return NAMES.get(phoneme) or VOWEL_NAMES.get(phoneme) or load_rules().label(phoneme)
79
 
80
 
81
+ def letter_reason(expected: str, heard: str | None, check=None) -> dict:
82
  """Why a letter of the answer is coloured: {"why": the sound and what was heard in its place ("_": nothing was),
83
+ "tip": how to say it, from the rule for that substitution (`check`, rules.Check) or TIPS}."""
84
  name = sound_name(expected) or expected
85
  heard = HEARD_ALIASES.get(heard, heard)
86
  other = sound_name(heard)
 
90
  why = f"{name} נשמעה כמו {other}"
91
  else:
92
  why = f"{name} לא נשמעה בבירור"
93
+ text = check.feedback if check else TIPS.get((expected, heard))
94
  return {"why": why, **({"tip": with_hebrew(text)} if text else {})}
95
 
96
 
97
  VOWELS = {"a", "i", "u", "e", "o", "aː", "iː", "uː", "eː", "oː"}
98
+ # the glottal stop (ء, and ق said the urban way) is often dropped by natives too; don't nag about it
99
+ NO_DELETION_TIP = {"ʔ", "Q"}
100
 
101
 
102
+ def tip_for(expected: str, heard: str | None, check=None) -> dict | None:
103
+ """Feedback for one flagged phoneme, or None if there is nothing useful to say.
104
+
105
+ `check` (rules.Check) is the rule of pronunciation-rules.md for this substitution: its feedback and
106
+ severity win over TIPS. The key stays "expected>heard" either way."""
107
  if expected in VOWELS:
108
  return None
109
  heard = HEARD_ALIASES.get(heard, heard) if heard else "_"
110
  if heard == "_" and expected in NO_DELETION_TIP:
111
  return None
112
  name = NAMES.get(expected, expected)
113
+ key = f"{expected}>{heard}"
114
+ if check:
115
+ return {"title": name, "text": with_hebrew(check.feedback), "key": key, "severity": check.severity}
116
  text = TIPS.get((expected, heard))
117
  if text:
118
+ return {"title": name, "text": with_hebrew(text), "key": key, "severity": DEFAULT_SEVERITY}
119
  if heard == "_":
120
+ return {"title": name, "text": "לא נשמעה בבירור.", "key": key, "severity": DEFAULT_SEVERITY}
121
+ other = sound_name(heard)
122
+ text = f"נשמעה כמו {other}." if other else "נשמעה אחרת מהצפוי."
123
+ return {"title": name, "text": text, "key": key, "severity": DEFAULT_SEVERITY}
app/translit.py CHANGED
@@ -10,7 +10,12 @@ CONSONANTS = {
10
  "ʔ": "א", "b": "בּ", "t": "ת", "dʒ": "ג'", "ħ": "ח", "x": "ח'", "d": "ד", "r": "ר", "z": "ז",
11
  "s": "ס", "ʃ": "ש", "sˤ": "צ", "dˤ": "ד'", "tˤ": "ט", "ðˤ": "ט'", "ʕ": "ע", "ɣ": "ע'", "f": "פ",
12
  "q": "ק", "k": "כּ", "g": "ג", "l": "ל", "m": "מ", "n": "נ", "h": "ה", "w": "ו", "j": "י",
 
 
13
  }
 
 
 
14
  # vowel -> ((mark on the consonant, letter after it) mid-word, (same) word-finally)
15
  VOWELS = {
16
  "a": (("ַ", ""), ("ַ", "א")), "i": (("ִ", ""), ("ִ", "י")), "u": (("ֻ", ""), ("", "וּ")),
@@ -23,8 +28,8 @@ SHADDA, SHVA = "\u0651", "\u05b0"
23
 
24
 
25
  def _letter(base: str, marks: str) -> str:
26
- """Points go on the letter itself, before a geresh."""
27
- return base[0] + marks + base[1:] if base.endswith("'") else base + marks
28
 
29
 
30
  def word_letters(ipa_word: str, ar_word: str = "") -> list[tuple[str, list[int]]]:
@@ -51,7 +56,7 @@ def word_letters(ipa_word: str, ar_word: str = "") -> list[tuple[str, list[int]]
51
  if ph in VOWELS: # vowel with no consonant before it (word-initial): carry it on an alef
52
  base, shadda = "א", ""
53
  else:
54
- base, shadda = CONSONANTS.get(ph, ph), ""
55
  i += 1
56
  if i < len(phs) and phs[i] == ph: # geminate
57
  shadda = SHADDA
 
10
  "ʔ": "א", "b": "בּ", "t": "ת", "dʒ": "ג'", "ħ": "ח", "x": "ח'", "d": "ד", "r": "ר", "z": "ז",
11
  "s": "ס", "ʃ": "ש", "sˤ": "צ", "dˤ": "ד'", "tˤ": "ט", "ðˤ": "ט'", "ʕ": "ע", "ɣ": "ע'", "f": "פ",
12
  "q": "ק", "k": "כּ", "g": "ג", "l": "ל", "m": "מ", "n": "נ", "h": "ה", "w": "ו", "j": "י",
13
+ # rural realisations (pronunciation-rules.md, section 1)
14
+ "tʃ": "צ'", "θ": "ת'", "ð": "ד״",
15
  }
16
+ # Sounds Arabic does not have, written only in the line of what was heard, never in an answer: a Hebrew
17
+ # speaker's p for ف, v for و / ب, and Hebrew צ (ts) for ص. Without them the heard line would drop the letter.
18
+ HEARD_ONLY = {"p": "פּ", "v": "ב", "β": "ב", "ʋ": "ב", "ts": "טס"}
19
  # vowel -> ((mark on the consonant, letter after it) mid-word, (same) word-finally)
20
  VOWELS = {
21
  "a": (("ַ", ""), ("ַ", "א")), "i": (("ִ", ""), ("ִ", "י")), "u": (("ֻ", ""), ("", "וּ")),
 
28
 
29
 
30
  def _letter(base: str, marks: str) -> str:
31
+ """Points go on the letter itself, before a geresh (or gershayim), and on the first letter of טס."""
32
+ return base[0] + marks + base[1:] if len(base) > 1 and base[1] != "\u05bc" else base + marks
33
 
34
 
35
  def word_letters(ipa_word: str, ar_word: str = "") -> list[tuple[str, list[int]]]:
 
56
  if ph in VOWELS: # vowel with no consonant before it (word-initial): carry it on an alef
57
  base, shadda = "א", ""
58
  else:
59
+ base, shadda = CONSONANTS.get(ph) or HEARD_ONLY.get(ph, ph), ""
60
  i += 1
61
  if i < len(phs) and phs[i] == ph: # geminate
62
  shadda = SHADDA
docs/scoring-dialects-plan.md ADDED
@@ -0,0 +1,172 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Plan: dialect acceptance and Hebrew-speaker rules in the scorer
2
+
3
+ Status: implemented 2026-10-15, except where "Implementation notes" at the end says otherwise. Source rules: [pronunciation-rules.md](../static/assets/pronunciation-rules.md).
4
+
5
+ Two layers on top of today's scoring path:
6
+
7
+ - **Layer A, dialect acceptance:** decides what counts as *correct*. Runs inside GOP: every correct realisation of a letter is pooled into one class, and members of the class never count as competitors.
8
+ - **Layer B, Hebrew-speaker rules:** decides how strictly to judge *real errors*. Runs after GOP: per-pair margins, severity, feedback, exemptions.
9
+
10
+ The native baseline and the teacher calibration floors stay where they are, but are keyed on the abstract symbol instead of the sound actually said.
11
+
12
+ Final order of the pipeline:
13
+
14
+ 1. `Vocab.merge` pools spelling variants (exists)
15
+ 2. Layer A: class columns → alignment → GOP against the class
16
+ 3. Layer B exemptions (6.3)
17
+ 4. Native baseline normalisation (exists, keyed on the abstract symbol)
18
+ 5. Layer B margins, severity, feedback
19
+ 6. Teacher DB floors (exists, last, manual override)
20
+
21
+ ## Phase 0: Rule loader (`app/rules.py`, new)
22
+
23
+ - Pull the YAML blocks out of `pronunciation-rules.md` (fenced ` ```yaml `, split on `---`). Load them into:
24
+ - `archiphonemes`, `dialects_enabled`, `optional_realisations_enabled`
25
+ - `scoring` (margins, floors)
26
+ - `checks` (target, heard, severity, feedback), collected from every rule block
27
+ - `heard_labels`
28
+ - Pass every symbol through `base()` when loading, so `ʁ` is stored as `ɣ`, `χ` as `x`, and so on. Warn about rules that become **undetectable**:
29
+ - target and heard end up the same sound after `base()`;
30
+ - a heard symbol is in the same accepted set as its target;
31
+ - a symbol is missing from the model vocab.
32
+ - Add `pyyaml` to `requirements.txt`. It's already installed through `huggingface_hub`, but it should be listed explicitly.
33
+ - Consider moving the rules file to `docs/` as the doc itself suggests. The student sections are served from `static/assets/`, so either keep it there or copy it at build time.
34
+ - **Tests:** every block parses; every `heard:` symbol has a label in section 8; the undetectable-rule check fires on a deliberately bad rule.
35
+
36
+ ## Phase 1: Layer A core, with `Q` only
37
+
38
+ ### 1a. Converting the phrases at load time, not in the source files
39
+
40
+ `app/workbook.py` has ~930 ق and `app/grammar.py` has more, so rewriting the IPA by hand isn't realistic.
41
+
42
+ - Add `archify(ar, ipa)` in a new `app/archi.py`. For each word whose Arabic contains ق, replace the matching `ʔ`/`q`/`k` with `Q`.
43
+ - Match by consonant order. `word_letters` in `app/translit.py` already lines up ق with its phoneme, so reuse that logic.
44
+ - **Ambiguous words** (both ء and ق, or ك next to ق) need an explicit marker written in the phrase. Add a test that fails on any ق word it can't resolve.
45
+ - After conversion, merge duplicate variants. Today's separate `q` and `k` variants (e.g. `i l q u d s`) become the same as the `ʔ` one.
46
+ - Run it inside `PHRASES` / `workbook_phrases()` so every consumer sees `Q`, including references and tokenisation at startup in `load_scorer`.
47
+
48
+ ### 1b. `Vocab` classes (`app/scorer.py`, `Vocab`)
49
+
50
+ - `Vocab(token_to_id, classes=rules.archiphonemes)` creates one **virtual id** per abstract symbol, numbered after the real vocabulary (V, V+1, …). It also keeps `members[vid]`, the real representative ids of the accepted realisations.
51
+ - `merge()` returns `logp` with extra columns: column `vid` = `logaddexp` over its members.
52
+ - `resolve("Q")` returns `[vid]`. `i2t[vid] = "Q"`.
53
+ - Anything that ranges over "real" symbols has to ignore virtual columns: `greedy_decode`, `_heard_hebrew`, the heatmap's "heard" track, and `edit_distance` for the unrelated check. Easiest is a `vocab.real(logp)` view of `logp[:, :V]`.
54
+
55
+ ### 1c. GOP (`app/align.py`, `score_tokens`)
56
+
57
+ - `score_tokens(..., accept: list[set[int]] | None)`. For a class token, mask all its members from the competitor search, and always mask every virtual column.
58
+ - Add `TokenScore.realised`: the member with the highest peak inside the window (`None` for plain tokens).
59
+ - The alignment itself doesn't change. It already runs on the virtual column.
60
+
61
+ ### 1d. Results and display
62
+
63
+ - Each phoneme entry gets `realised`. `heard` comes from the competitor as before, so an accepted realisation never shows up as "heard".
64
+ - `_heard_hebrew`: when the frame is inside a class token's span and the greedy symbol is one of its members, write the **realised** letter (כּ, א or ק) uncoloured.
65
+ - `to_hebrew` / `word_letters`: `Q` is written ק in the answer (today's ق rule already does this).
66
+ - `app/tips.py`: add `Q` to `NO_DELETION_TIP` (6.3 #2). Remove the `ʔ>k` qaf tip.
67
+
68
+ ### 1e. Baseline and calibrations
69
+
70
+ - References need no change. `Reference.sounds` now holds `(w, "Q")`, so the baseline is the same whatever realisation the TTS voice used. The cache is in memory and rebuilds on restart.
71
+ - Calibration lookup in `score_logp`: try `(archi, heard)` first, then fall back to `(realised target, heard)` so existing DB rows still apply. Add a teacher-panel note or a one-off SQL to re-key `ʔ` rows that came from ق phrases.
72
+
73
+ ### 1f. Tests (`tests/test_scoring.py`)
74
+
75
+ - Rewrite `test_k_for_urban_qaf_picks_closest_variant_and_tips`: `k` for `Q` scores ≥ 90, has no tip, and `realised == "k"`.
76
+ - New tests:
77
+ - `q` for `Q` is accepted;
78
+ - `Q` dropped between vowels is not flagged;
79
+ - `ʔ` from ء said as `k` **is** flagged (e.g. اهلا → kahla);
80
+ - a reference voiced with `q` doesn't penalise a learner who says `ʔ`;
81
+ - `archify` on a sample of workbook phrases, plus the ambiguity test.
82
+
83
+ ## Phase 2: The other five abstract symbols
84
+
85
+ - Extend `archify`: ك→`K`, ث→`TH`, ذ→`DH`, ظ→`ZH`, ج→`J`.
86
+ - **Per-word override syntax** for learned words, e.g. `TH{t,s}` (ثقافة) or a plain `t` to lock it. Parse it in `tokenize`, which creates a one-off class for that position.
87
+ - **Vocab gaps:** if `θ`, `ð`, `ðˤ` or `tʃ` aren't in the model, the class quietly shrinks. Log it, and expose it in `/api/health` under `rules` (section 7 of the rules doc).
88
+ - Add `tʃ` (צ'), `θ` (ת'), `ð` (ד״) to `CONSONANTS` in `app/translit.py` so rural realisations can be written in the heard line.
89
+ - Add `θ`, `ð`, `tʃ`, `ðˤ`, `dʒ` to the test vocab `SYMS`. Tests for each class, plus one mixed-dialect sentence that isn't penalised.
90
+
91
+ ## Phase 3: Positional classes and diphthongs
92
+
93
+ - **`E`** for a final ة/ه vowel → {a, e, i}. Derived automatically from the Arabic ending (`word_letters` already detects it).
94
+ - **`O`** for the vowel of a pronoun ending → {o, u}. This one needs an explicit marker in the phrase, because it can't be told reliably from the Arabic.
95
+ - **`EY` / `OW`**: expand into extra **variants** (`eː` | `a j`, `oː` | `a w`) inside `archify`, before variants are merged. `_fit` already picks the best variant. These can't be a per-token class, because one token stands against two.
96
+ - `near_vowels` stays for other positions until the vowel-length decision is made (see "Open decisions").
97
+ - Sounds that natives also drop (ʔ at the start or between vowels, il- after a vowel) are **not** Layer A: an optional token would need a separate alignment with and without it. They stay as Layer B exemptions.
98
+
99
+ ## Phase 4: Layer B, the rule engine
100
+
101
+ ### 4a. Competitor set
102
+
103
+ - Limit competitors to the Arabic sounds plus the Hebrew-interference sounds (every `heard:` symbol in `checks`). Report any other winner as `competitor=None, other=True` ("not clear").
104
+ - **Watch out:** this lowers `comp_peak` in the deletion test (`deleted_comp`). Re-check the deletion tests.
105
+
106
+ ### 4b. Flagging by margin
107
+
108
+ - In `score_logp`, flag a sound when `score < flag_below` **and** the competitor is outside the target's accepted set **and** `−gop ≥ margin(target, heard)`. `gop` is already `log P(target) − log P(competitor)`.
109
+ - The issues list keeps its threshold (`issue_below`) but also respects the margin.
110
+
111
+ ### 4c. Feedback
112
+
113
+ - `tip_for` looks up `checks[(target, heard)]` first, then falls back to `TIPS`. Keep the key format `expected>heard` so the front end doesn't change.
114
+ - Sort tips by severity, then by score. `max_tips` comes from the rules file.
115
+
116
+ ### 4d. Exemptions (6.3)
117
+
118
+ - Move `NO_DELETION_TIP`, `elided_article` and `near_vowels` into a predicate registry in `rules.py`: `(name, applies(toks, i, ar_words, sc) -> bool, action)`.
119
+ - Add the doc's cases 1–3, 4 (il- not assimilated) and 12 (`mixed_dialect` as info only).
120
+
121
+ ### 4e. Letters that can only be heard, never expected
122
+
123
+ - A `HEARD_ONLY` table in `translit.py`: `p`→פּ, `v`→ב/ו, `ts`→צ. Include it in `HEBREW_SOUNDS` so `_heard_hebrew` stops dropping these sounds. Today a student who says *pinjaan* sees the first letter missing instead of פּ.
124
+ - `sound_name` / `letter_reason` labels come from `heard_labels`.
125
+ - **Tests:** `f→p` (פִנְג'אן), `w→v`, `b→v`, `sˤ→ts`, `r→ʁ`. Emphatic just under the margin is not flagged, just over it is.
126
+
127
+ ## Phase 5: Teacher-facing parts
128
+
129
+ - `dialects_enabled` / `optional_realisations_enabled` as teacher settings, stored wherever teacher settings live today. Pass them into `Scorer.score` per request, so different classes can differ.
130
+ - The taught dialect only changes hints and info, never what's accepted.
131
+ - Teacher panel: show `realised` and `mixed_dialect` in the diagnostics view. Show abstract symbols by their Hebrew letter, never as "Q".
132
+ - `/api/health`: a `rules` field listing each rule and whether it can be detected.
133
+
134
+ ## Phase 6: Rollout check
135
+
136
+ - If recordings are stored, add a `scripts/rescore.py` that compares old and new scores per phrase. Review the biggest changes before switching over, especially ק phrases and emphatics.
137
+ - Ship Phase 1 on its own: it fixes ק/א and tests the class mechanism end to end.
138
+
139
+ ## Open decisions (outside this plan)
140
+
141
+ 1. **Vowel length:** the doc's `vowel_length` checks (severity 1, margin 1.1) vs. today's `near_vowel_floor`, which forgives length completely.
142
+ 2. **Sentence formula:** the doc's `mean(word_min)` and severity weights vs. today's 0.5/0.5 blend. Keep it behind a config flag until decided.
143
+ 3. **Bedouin `g` for `Q`:** on or off by default.
144
+ 4. **Extra sounds:** inserted sounds aren't detected, since forced alignment only scores expected sounds.
145
+
146
+ ## Main risks
147
+
148
+ - The virtual columns change the `logp` width. Anything that indexes `logp` by a real id is fine, but anything that iterates over all columns (heatmap, greedy) must use the real-only view.
149
+ - `archify` mismatches in the workbook. That's why the ambiguity test fails the build instead of guessing.
150
+ - Old calibration rows keyed on `ʔ` until they're re-keyed. That's why the lookup falls back to the old key.
151
+
152
+ ## Implementation notes
153
+
154
+ Code: `app/rules.py` (loader, detectability, exemptions), `app/archi.py` (symbols, letter alignment, diphthong variants), `app/phones.py` (`base()`, moved out of the scorer), `app/scorer.py` / `app/align.py` (classes, GOP, margins, tips), `scripts/rescore.py`. Tests: `tests/test_rules.py` and the "dialects and Hebrew-speaker rules" section of `tests/test_scoring.py`.
155
+
156
+ Where it differs from the plan above:
157
+
158
+ - **`archify` runs inside the scorer, not on `PHRASES`.** `Scorer._variants` converts each variant when scoring (cached). Display, TTS, transliteration and the verb game keep the plain IPA. Symbols replace phonemes one for one, so positions in the IPA still line up.
159
+ - **Token format** is `SYMBOL:written` (`Q:ʔ`), or `SYMBOL{a,b}` for a hand-written set. A phrase that writes a symbol itself is left alone.
160
+ - **Ambiguity:** across the bank (~11.6k variants) no ق is unresolved. 5 × ث merged with ت into one geminate (بعثت) and 1 × ج (جوز) stay plain phonemes; only an unresolved ق fails the test.
161
+ - **Learned-only realisations** (`TH` as s, `DH` as z) are accepted where the phrase writes them, with no override syntax needed.
162
+ - **Missing model symbols:** a class member the model lacks uses the same `FALLBACKS` stand-in as a plain phoneme. With the base model, `ZH` is `{dˤ, z}`, so "ظ heard as z" shows as `accepted` (undetectable) in `/api/health` → `rules`.
163
+ - **Competitor naming (4a)** keeps the GOP against the strongest symbol of any kind, so scores and deletions don't change. Only the *name* is withheld (`other`) when that symbol isn't Arabic or a Hebrew-interference sound.
164
+ - **Tips:** `max_tips` is now 3 (from the rules file), sorted by severity then score. Every tip carries `severity`.
165
+ - **Results** gain `phonemes[].sound` (`Q`), `phonemes[].realised` (`k`), and `dialect: urban|rural` or `mixed_dialect: true`. None of them affect the score.
166
+ - **Teacher view:** the expected/heard phone rows compare an accepted realisation with what was said, and show the dialect.
167
+
168
+ Not done:
169
+
170
+ - **Per-teacher dialect settings (Phase 5).** The accepted set comes from the rules file for everyone (`dialects_enabled`, `optional_realisations_enabled`). Since the taught dialect was only meant to change hints, nothing in scoring depends on it yet.
171
+ - **Re-keying old calibration rows.** The lookup falls back to the written phoneme, so old rows keep working. Rows proposed from the page are still keyed on `phonemes[].ipa`.
172
+ - **The open decisions** above (vowel length, sentence formula, Bedouin g, insertions) are unchanged. Vowel-length rules show as undetectable in `/api/health`.
requirements.txt CHANGED
@@ -2,6 +2,7 @@ fastapi>=0.110
2
  uvicorn[standard]>=0.29
3
  python-multipart>=0.0.9
4
  numpy>=1.26
 
5
  transformers>=4.41
6
  huggingface_hub>=0.23
7
  # torch: CPU build installed separately in the Dockerfile (see README for local installs)
 
2
  uvicorn[standard]>=0.29
3
  python-multipart>=0.0.9
4
  numpy>=1.26
5
+ pyyaml>=6.0 # the scorer reads its rules from the yaml blocks of static/assets/pronunciation-rules.md
6
  transformers>=4.41
7
  huggingface_hub>=0.23
8
  # torch: CPU build installed separately in the Dockerfile (see README for local installs)
scripts/rescore.py ADDED
@@ -0,0 +1,95 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Rescore the flagged recordings with the current scorer and list the biggest changes against the stored score.
3
+
4
+ Usage: python scripts/rescore.py # every flag that kept its recording
5
+ python scripts/rescore.py --status pending,fixed # only these review states
6
+ python scripts/rescore.py --limit 50 --top 20
7
+ python scripts/rescore.py --json out.json # every row, with the old and new sound-level results
8
+
9
+ Reads SUPABASE_URL, SUPABASE_PUBLISHABLE_KEY and SUPABASE_SECRET_KEY (as the server does) and MODEL_ID.
10
+ A flag stores the phrase it was said for, its old result and the recording, so the comparison needs no
11
+ other data. Run it before shipping a scoring change: the rows that moved most are the ones to listen to.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import argparse
16
+ import json
17
+ import os
18
+ import sys
19
+ from pathlib import Path
20
+
21
+ sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
22
+
23
+ from dotenv import load_dotenv # noqa: E402
24
+
25
+ from app.auth import load_auth # noqa: E402
26
+ from app.phrases import PHRASES # noqa: E402
27
+ from app.scorer import decode_audio, load_scorer # noqa: E402
28
+
29
+
30
+ def phrase_for(row: dict) -> dict:
31
+ """The bank phrase a flag was said for, or (a verb-game or text line) the variant it was scored on."""
32
+ key = row.get("phrase")
33
+ for p in PHRASES:
34
+ if p.get("key", p["he"]) == key:
35
+ return p
36
+ return {"variants": [(row["ar"], row["ipa"])], "key": key, "he": row.get("he", ""), "cat": row.get("cat", "")}
37
+
38
+
39
+ def sounds(result: dict) -> list[str]:
40
+ """The flagged sounds of a result, as 'ipa>heard'."""
41
+ return [f"{p.get('sound', p['ipa'])}>{p['heard']}" for w in result.get("words", []) for p in w.get("phonemes", [])
42
+ if p.get("flagged")]
43
+
44
+
45
+ def main() -> int:
46
+ ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
47
+ ap.add_argument("--status", default="pending,confirmed,rejected,fixed")
48
+ ap.add_argument("--limit", type=int, default=0, help="at most this many flags (0: all)")
49
+ ap.add_argument("--top", type=int, default=30, help="rows to print, biggest change first")
50
+ ap.add_argument("--json", type=Path, help="write every row here")
51
+ args = ap.parse_args()
52
+
53
+ load_dotenv()
54
+ auth = load_auth()
55
+ if auth is None or not auth.secret:
56
+ print("needs SUPABASE_URL, SUPABASE_PUBLISHABLE_KEY and SUPABASE_SECRET_KEY", file=sys.stderr)
57
+ return 2
58
+ scorer = load_scorer(os.environ.get("MODEL_ID", "facebook/wav2vec2-xlsr-53-espeak-cv-ft"))
59
+ flags = [f for s in args.status.split(",") for f in auth.list_flags(s.strip()) if f.get("audio")]
60
+ if args.limit:
61
+ flags = flags[:args.limit]
62
+
63
+ rows = []
64
+ for f in flags:
65
+ audio = auth.flag_audio(f["id"])
66
+ if audio is None:
67
+ continue
68
+ try:
69
+ new = scorer.score(decode_audio(audio[0]), phrase_for(f))
70
+ except (ValueError, KeyError) as e:
71
+ print(f"flag {f['id']}: skipped ({e})", file=sys.stderr)
72
+ continue
73
+ if new.get("error"):
74
+ continue
75
+ old = f.get("result") or {}
76
+ rows.append({"id": f["id"], "status": f.get("status"), "phrase": f.get("he") or f.get("phrase"),
77
+ "old": f.get("score"), "new": new["score"], "delta": new["score"] - (f.get("score") or 0),
78
+ "old_flagged": sounds(old), "new_flagged": sounds(new),
79
+ "dialect": "mixed" if new.get("mixed_dialect") else new.get("dialect", ""),
80
+ "old_result": old, "new_result": new})
81
+
82
+ rows.sort(key=lambda r: -abs(r["delta"]))
83
+ print(f"{len(rows)} flags rescored; mean change {sum(r['delta'] for r in rows) / max(1, len(rows)):+.1f}")
84
+ for r in rows[:args.top]:
85
+ print(f"{r['id']:>6} {r['status'] or '':<9} {r['old']!s:>4} -> {r['new']:>3} ({r['delta']:+4}) "
86
+ f"{r['dialect']:<6} {r['phrase']}")
87
+ if r["old_flagged"] != r["new_flagged"]:
88
+ print(f" flagged: {' '.join(r['old_flagged']) or '-'} -> {' '.join(r['new_flagged']) or '-'}")
89
+ if args.json:
90
+ args.json.write_text(json.dumps(rows, ensure_ascii=False, indent=1), encoding="utf-8")
91
+ return 0
92
+
93
+
94
+ if __name__ == "__main__":
95
+ sys.exit(main())
static/assets/arabic_verbs.json ADDED
@@ -0,0 +1,451 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alif": {
3
+ "source": {
4
+ "title": "פעלים בערבית מדוברת המתחילים באליף",
5
+ "url": "https://www.youtube.com/watch?v=c95B23kxOws",
6
+ "dialect": "Palestinian spoken Arabic",
7
+ "notes": "One entry per unique verb, in video order. The video repeats أبطل and أخذ. Phonemes use the scorer alphabet and are space-separated."
8
+ },
9
+ "verbs": [
10
+ {"hebrew": "בא", "timestamp": "01:03", "past": {"arabic": "اجا", "phonemes": "ʔ a dʒ a"}, "future": {"arabic": "يجي", "phonemes": "j i dʒ i"}},
11
+ {"hebrew": "לקח", "timestamp": "01:12", "past": {"arabic": "اخد", "phonemes": "ʔ a x a d"}, "future": {"arabic": "ياخد", "phonemes": "j oː x i d", "accepted_phonemes": ["j aː x u d", "j oː x u d"]}},
12
+ {"hebrew": "חייך", "timestamp": "01:21", "past": {"arabic": "ابتسم", "phonemes": "ʔ i b t a s a m"}, "future": {"arabic": "يبتسم", "phonemes": "j i b t a s i m"}},
13
+ {"hebrew": "דחה", "timestamp": "01:29", "past": {"arabic": "أجّل", "phonemes": "ʔ a dʒ dʒ a l"}, "future": {"arabic": "يأجّل", "phonemes": "j ʔ a dʒ dʒ i l"}},
14
+ {"hebrew": "השכיר", "timestamp": "01:39", "past": {"arabic": "أجّر", "phonemes": "ʔ a dʒ dʒ a r"}, "future": {"arabic": "يأجّر", "phonemes": "j ʔ a dʒ dʒ i r"}},
15
+ {"hebrew": "הדהים", "timestamp": "01:55", "past": {"arabic": "أدهش", "phonemes": "ʔ a d h a ʃ"}, "future": {"arabic": "يدهش", "phonemes": "j i d h a ʃ"}},
16
+ {"hebrew": "ירד", "timestamp": "02:03", "past": {"arabic": "انخفض", "phonemes": "ʔ i n x a f a dˤ"}, "future": {"arabic": "ينخفض", "phonemes": "j i n x a f i dˤ"}},
17
+ {"hebrew": "הדגיש", "timestamp": "02:13", "past": {"arabic": "أبرز", "phonemes": "ʔ a b r a z"}, "future": {"arabic": "يبرز", "phonemes": "j i b r u z"}},
18
+ {"hebrew": "נטרל, פירק", "timestamp": "02:21", "past": {"arabic": "أبطل", "phonemes": "ʔ a b tˤ a l"}, "future": {"arabic": "يبطل", "phonemes": "j i b tˤ a l"}},
19
+ {"hebrew": "אירח לחברה, ישב עם אורח כדי שלא יישאר לבד", "timestamp": "02:30", "past": {"arabic": "آنس", "phonemes": "ʔ aː n a s"}, "future": {"arabic": "يآنس", "phonemes": "j ʔ aː n i s"}},
20
+ {"hebrew": "התעניין, החשיב", "timestamp": "02:46", "past": {"arabic": "اهتمّ", "phonemes": "ʔ i h t a m m"}, "future": {"arabic": "يهتمّ", "phonemes": "j i h t a m m"}},
21
+ {"hebrew": "היה חלוק בדעתו", "timestamp": "02:52", "past": {"arabic": "اختلف", "phonemes": "ʔ i x t a l a f"}, "future": {"arabic": "يختلف", "phonemes": "j i x t a l i f"}},
22
+ {"hebrew": "בחר", "timestamp": "03:19", "past": {"arabic": "اختار", "phonemes": "ʔ i x t aː r"}, "future": {"arabic": "يختار", "phonemes": "j i x t aː r"}},
23
+ {"hebrew": "המציא", "timestamp": "03:28", "past": {"arabic": "اخترع", "phonemes": "ʔ i x t a r a ʕ"}, "future": {"arabic": "يخترع", "phonemes": "j i x t a r i ʕ"}},
24
+ {"hebrew": "גילה", "timestamp": "03:39", "past": {"arabic": "اكتشف", "phonemes": "ʔ i k t a ʃ a f"}, "future": {"arabic": "يكتشف", "phonemes": "j i k t a ʃ i f"}},
25
+ {"hebrew": "התחזה", "timestamp": "03:51", "past": {"arabic": "انتحل", "phonemes": "ʔ i n t a ħ a l"}, "future": {"arabic": "ينتحل", "phonemes": "j i n t a ħ i l"}},
26
+ {"hebrew": "נפצע", "timestamp": "04:00", "past": {"arabic": "انجرح", "phonemes": "ʔ i n dʒ a r a ħ"}, "future": {"arabic": "ينجرح", "phonemes": "j i n dʒ a r i ħ"}},
27
+ {"hebrew": "נדרס", "timestamp": "04:09", "past": {"arabic": "اندهس", "phonemes": "ʔ i n d a h a s"}, "future": {"arabic": "يندهس", "phonemes": "j i n d a h i s"}},
28
+ {"hebrew": "התהפך", "timestamp": "04:27", "past": {"arabic": "انقلب", "phonemes": "ʔ i n Q a l a b"}, "future": {"arabic": "ينقلب", "phonemes": "j i n Q a l i b"}},
29
+ {"hebrew": "עבר ממקום למקום", "timestamp": "04:35", "past": {"arabic": "انتقل", "phonemes": "ʔ i n t a Q a l"}, "future": {"arabic": "ينتقل", "phonemes": "j i n t a Q i l"}},
30
+ {"hebrew": "הבשיל", "timestamp": "04:45", "past": {"arabic": "استوى", "phonemes": "ʔ i s t a w a"}, "future": {"arabic": "يستوي", "phonemes": "j i s t a w i"}},
31
+ {"hebrew": "ייבא", "timestamp": "04:54", "past": {"arabic": "استورد", "phonemes": "ʔ i s t a w r a d"}, "future": {"arabic": "يستورد", "phonemes": "j i s t a w r i d"}},
32
+ {"hebrew": "חשב", "timestamp": "05:04", "past": {"arabic": "افتكر", "phonemes": "ʔ i f t a k a r"}, "future": {"arabic": "يفتكر", "phonemes": "j i f t a k i r"}},
33
+ {"hebrew": "עבד", "timestamp": "05:14", "past": {"arabic": "اشتغل", "phonemes": "ʔ i ʃ t a ɣ a l", "accepted_phonemes": ["ʔ a ʃ t ɣ i l", "ʔ a ʃ t i ɣ i l"]}, "future": {"arabic": "يشتغل", "phonemes": "j i ʃ t a ɣ i l"}},
34
+ {"hebrew": "ניצת, נדלק", "timestamp": "05:23", "past": {"arabic": "اشتعل", "phonemes": "ʔ i ʃ t a ʕ a l"}, "future": {"arabic": "يشتعل", "phonemes": "j i ʃ t a ʕ i l"}},
35
+ {"hebrew": "השתתף", "timestamp": "05:33", "past": {"arabic": "اشترك", "phonemes": "ʔ i ʃ t a r a k"}, "future": {"arabic": "يشترك", "phonemes": "j i ʃ t a r i k"}},
36
+ {"hebrew": "אימץ פתרון או קיבל החלטה", "timestamp": "05:42", "past": {"arabic": "اتّخذ", "phonemes": "ʔ i t t a x a d"}, "future": {"arabic": "يتّخذ", "phonemes": "j i t t a x i d"}},
37
+ {"hebrew": "נשען על חפץ", "timestamp": "05:54", "past": {"arabic": "اتّكأ", "phonemes": "ʔ i t t a k a ʔ"}, "future": {"arabic": "يتّكئ", "phonemes": "j i t t a k i ʔ"}},
38
+ {"hebrew": "הסכים", "timestamp": "06:07", "past": {"arabic": "اتّفق", "phonemes": "ʔ i t t a f a Q"}, "future": {"arabic": "يتّفق", "phonemes": "j i t t a f i Q"}},
39
+ {"hebrew": "התקשר", "timestamp": "06:15", "past": {"arabic": "اتّصل", "phonemes": "ʔ i t t a sˤ a l"}, "future": {"arabic": "يتّصل", "phonemes": "j i t t a sˤ i l"}},
40
+ {"hebrew": "התעייף", "timestamp": "06:24", "past": {"arabic": "اتعب", "phonemes": "ʔ i t ʕ a b"}, "future": {"arabic": "يتعب", "phonemes": "j i t ʕ a b"}},
41
+ {"hebrew": "עלה", "timestamp": "06:34", "past": {"arabic": "ارتفع", "phonemes": "ʔ i r t a f a ʕ"}, "future": {"arabic": "يرتفع", "phonemes": "j i r t a f i ʕ"}},
42
+ {"hebrew": "שיתף", "timestamp": "06:43", "past": {"arabic": "أشرك", "phonemes": "ʔ a ʃ r a k"}, "future": {"arabic": "يشرك", "phonemes": "j i ʃ r i k"}},
43
+ {"hebrew": "אלתר", "timestamp": "06:51", "past": {"arabic": "ارتجل", "phonemes": "ʔ i r t a dʒ a l"}, "future": {"arabic": "يرتجل", "phonemes": "j i r t a dʒ i l"}},
44
+ {"hebrew": "הרחיק", "timestamp": "07:08", "past": {"arabic": "أبعد", "phonemes": "ʔ a b ʕ a d"}, "future": {"arabic": "يبعد", "phonemes": "j i b ʕ i d"}},
45
+ {"hebrew": "הזניח", "timestamp": "07:19", "past": {"arabic": "أهمل", "phonemes": "ʔ a h m a l"}, "future": {"arabic": "يهمل", "phonemes": "j i h m i l"}},
46
+ {"hebrew": "איחר", "timestamp": "07:53", "past": {"arabic": "اتأخّر", "phonemes": "ʔ i t ʔ a x x a r"}, "future": {"arabic": "يتأخّر", "phonemes": "j i t ʔ a x x a r"}},
47
+ {"hebrew": "תמך ב־, אישר", "timestamp": "08:02", "past": {"arabic": "أكّد", "phonemes": "ʔ a k k a d"}, "future": {"arabic": "يأكّد", "phonemes": "j ʔ a k k i d"}},
48
+ {"hebrew": "אכל", "timestamp": "08:08", "past": {"arabic": "اكل", "phonemes": "ʔ a k a l"}, "future": {"arabic": "ياكل", "phonemes": "j oː k u l", "accepted_phonemes": ["j oː k i l", "j aː k u l"]}},
49
+ {"hebrew": "ציווה", "timestamp": "08:36", "past": {"arabic": "أمر", "phonemes": "ʔ a m a r"}, "future": {"arabic": "يأمر", "phonemes": "j ʔ a m u r"}},
50
+ {"hebrew": "נתן שהות", "timestamp": "08:40", "past": {"arabic": "أمهل", "phonemes": "ʔ a m h a l"}, "future": {"arabic": "يمهل", "phonemes": "j i m h i l"}},
51
+ {"hebrew": "סיים", "timestamp": "08:54", "past": {"arabic": "أنهى", "phonemes": "ʔ a n h a"}, "future": {"arabic": "ينهي", "phonemes": "j i n h i"}},
52
+ {"hebrew": "ייסד", "timestamp": "09:01", "past": {"arabic": "أسّس", "phonemes": "ʔ a s s a s"}, "future": {"arabic": "يأسّس", "phonemes": "j ʔ a s s i s"}},
53
+ {"hebrew": "שיכר", "timestamp": "09:10", "past": {"arabic": "أسكر", "phonemes": "ʔ a s k a r"}, "future": {"arabic": "يسكر", "phonemes": "j i s k i r"}},
54
+ {"hebrew": "מיהר", "timestamp": "09:24", "past": {"arabic": "أسرع", "phonemes": "ʔ a s r a ʕ"}, "future": {"arabic": "يسرع", "phonemes": "j i s r a ʕ"}},
55
+ {"hebrew": "נתן", "timestamp": "09:33", "past": {"arabic": "أعطى", "phonemes": "ʔ a ʕ tˤ a"}, "future": {"arabic": "يعطي", "phonemes": "j i ʕ tˤ i"}},
56
+ {"hebrew": "הדביק במחלה", "timestamp": "09:40", "past": {"arabic": "أعدى", "phonemes": "ʔ a ʕ d a"}, "future": {"arabic": "يعدي", "phonemes": "j i ʕ d i"}},
57
+ {"hebrew": "הכריז", "timestamp": "09:48", "past": {"arabic": "أعلن", "phonemes": "ʔ a ʕ l a n"}, "future": {"arabic": "يعلن", "phonemes": "j i ʕ l i n"}},
58
+ {"hebrew": "פטר", "timestamp": "09:56", "past": {"arabic": "أعفى", "phonemes": "ʔ a ʕ f a"}, "future": {"arabic": "يعفي", "phonemes": "j i ʕ f i"}},
59
+ {"hebrew": "עיוור, סינוור", "timestamp": "10:05", "past": {"arabic": "أعمى", "phonemes": "ʔ a ʕ m a"}, "future": {"arabic": "يعمي", "phonemes": "j i ʕ m i"}},
60
+ {"hebrew": "השביע את רצונו של", "timestamp": "10:12", "past": {"arabic": "أرضى", "phonemes": "ʔ a r dˤ a"}, "future": {"arabic": "يرضي", "phonemes": "j i r dˤ i"}},
61
+ {"hebrew": "סימן, אותת", "timestamp": "10:23", "past": {"arabic": "أشّر", "phonemes": "ʔ a ʃ ʃ a r"}, "future": {"arabic": "يأشّر", "phonemes": "j ʔ a ʃ ʃ i r"}},
62
+ {"hebrew": "השפיע", "timestamp": "10:32", "past": {"arabic": "أثّر", "phonemes": "ʔ a TH TH a r"}, "future": {"arabic": "يأثّر", "phonemes": "j ʔ a TH TH i r"}},
63
+ {"hebrew": "השלים בין, תיקן", "timestamp": "10:41", "past": {"arabic": "أصلح", "phonemes": "ʔ a sˤ l a ħ"}, "future": {"arabic": "يصلح", "phonemes": "j i sˤ l i ħ"}}
64
+ ]
65
+ },
66
+ "ba": {
67
+ "source": {
68
+ "title": "פעלים שימושיים בערבית מדוברת — אות בא",
69
+ "url": "https://www.youtube.com/watch?v=w19b7mJ-j4U",
70
+ "dialect": "Palestinian spoken Arabic",
71
+ "notes": "One entry per verb, in video order. Phonemes use the scorer alphabet and are space-separated."
72
+ },
73
+ "verbs": [
74
+ {"hebrew": "התחיל", "timestamp": "01:39", "past": {"arabic": "بدأ", "phonemes": "b a d a ʔ"}, "future": {"arabic": "يبدأ", "phonemes": "j i b d a ʔ"}},
75
+ {"hebrew": "בכה", "timestamp": "01:48", "past": {"arabic": "بكى", "phonemes": "b a k a"}, "future": {"arabic": "يبكي", "phonemes": "j i b k i"}},
76
+ {"hebrew": "נשאר", "timestamp": "01:57", "past": {"arabic": "بقي", "phonemes": "b a Q i"}, "future": {"arabic": "يبقى", "phonemes": "j i b Q a"}},
77
+ {"hebrew": "מכר", "timestamp": "02:06", "past": {"arabic": "باع", "phonemes": "b aː ʕ"}, "future": {"arabic": "يبيع", "phonemes": "j i b iː ʕ"}},
78
+ {"hebrew": "אירח מישהו כל הלילה", "timestamp": "02:17", "past": {"arabic": "بات", "phonemes": "b aː t"}, "future": {"arabic": "يبيت", "phonemes": "j i b iː t"}},
79
+ {"hebrew": "נזף, דיבר בצורה קשה", "timestamp": "02:26", "past": {"arabic": "بهدل", "phonemes": "b a h d a l"}, "future": {"arabic": "يبهدل", "phonemes": "j i b a h d i l"}},
80
+ {"hebrew": "החליף, שינה", "timestamp": "02:36", "past": {"arabic": "بدّل", "phonemes": "b a d d a l"}, "future": {"arabic": "يبدّل", "phonemes": "j i b a d d i l"}},
81
+ {"hebrew": "חפר באדמה", "timestamp": "02:46", "past": {"arabic": "بحش", "phonemes": "b a ħ a ʃ"}, "future": {"arabic": "يبحش", "phonemes": "j i b ħ a ʃ"}},
82
+ {"hebrew": "סיווג", "timestamp": "02:56", "past": {"arabic": "بوّب", "phonemes": "b a w w a b"}, "future": {"arabic": "يبوّب", "phonemes": "j i b a w w i b"}},
83
+ {"hebrew": "הפסיק, חזר בו", "timestamp": "03:06", "past": {"arabic": "بطّل", "phonemes": "b a tˤ tˤ a l"}, "future": {"arabic": "يبطّل", "phonemes": "j i b a tˤ tˤ i l"}},
84
+ {"hebrew": "הופיע, נמצא", "timestamp": "03:15", "past": {"arabic": "بيّن", "phonemes": "b a j j a n"}, "future": {"arabic": "يبيّن", "phonemes": "j i b a j j i n"}},
85
+ {"hebrew": "הודיע", "timestamp": "03:25", "past": {"arabic": "بلّغ", "phonemes": "b a l l a ɣ"}, "future": {"arabic": "يبلّغ", "phonemes": "j i b a l l i ɣ"}},
86
+ {"hebrew": "בלע", "timestamp": "03:38", "past": {"arabic": "بلع", "phonemes": "b a l a ʕ"}, "future": {"arabic": "يبلع", "phonemes": "j i b l a ʕ"}},
87
+ {"hebrew": "התחיל", "timestamp": "03:44", "past": {"arabic": "بلّش", "phonemes": "b a l l a ʃ"}, "future": {"arabic": "يبلّش", "phonemes": "j i b a l l i ʃ"}},
88
+ {"hebrew": "בכה בקול, יילל", "timestamp": "04:05", "past": {"arabic": "بعق", "phonemes": "b a ʕ a Q"}, "future": {"arabic": "يبعق", "phonemes": "j i b ʕ a Q"}},
89
+ {"hebrew": "פיזר וגם בזבז", "timestamp": "04:14", "past": {"arabic": "بعتر", "phonemes": "b a ʕ t a r"}, "future": {"arabic": "يبعتر", "phonemes": "j i b a ʕ t i r"}},
90
+ {"hebrew": "בנה", "timestamp": "04:23", "past": {"arabic": "بنى", "phonemes": "b a n a"}, "future": {"arabic": "يبني", "phonemes": "j i b n i"}},
91
+ {"hebrew": "הצדיק", "timestamp": "04:30", "past": {"arabic": "برّر", "phonemes": "b a r r a r"}, "future": {"arabic": "يبرّر", "phonemes": "j i b a r r i r"}},
92
+ {"hebrew": "כיער", "timestamp": "04:37", "past": {"arabic": "بشّع", "phonemes": "b a ʃ ʃ a ʕ"}, "future": {"arabic": "يبشّع", "phonemes": "j i b a ʃ ʃ i ʕ"}},
93
+ {"hebrew": "תכנת", "timestamp": "04:47", "past": {"arabic": "برمج", "phonemes": "b a r m a dʒ"}, "future": {"arabic": "يبرمج", "phonemes": "j i b a r m i dʒ"}},
94
+ {"hebrew": "שיחד", "timestamp": "05:11", "past": {"arabic": "برطل", "phonemes": "b a r tˤ a l"}, "future": {"arabic": "يبرطل", "phonemes": "j i b a r tˤ i l"}}
95
+ ]
96
+ },
97
+ "jim": {
98
+ "source": {"title": "פעלים שימושיים בערבית מדוברת — אות ג'ים", "url": "https://www.youtube.com/watch?v=A90bTU8w3BY", "dialect": "Palestinian spoken Arabic"},
99
+ "verbs": [
100
+ {"hebrew":"הביא","past":{"arabic":"جاب","phonemes":"J aː b"},"future":{"arabic":"يجيب","phonemes":"j i J iː b"}},
101
+ {"hebrew":"ענה","past":{"arabic":"جاوب","phonemes":"J aː w a b"},"future":{"arabic":"يجاوب","phonemes":"j i J aː w i b"}},
102
+ {"hebrew":"גרר, סחב","past":{"arabic":"جرجر","phonemes":"J a r J a r"},"future":{"arabic":"يجرجر","phonemes":"j i J a r J i r"}},
103
+ {"hebrew":"ניסה","past":{"arabic":"جرّب","phonemes":"J a r r a b"},"future":{"arabic":"يجرّب","phonemes":"j i J a r r i b"}},
104
+ {"hebrew":"הקפיא","past":{"arabic":"جمّد","phonemes":"J a m m a d"},"future":{"arabic":"يجمّد","phonemes":"j i J a m m i d"}},
105
+ {"hebrew":"ייבש","past":{"arabic":"جفّف","phonemes":"J a f f a f"},"future":{"arabic":"يجفّف","phonemes":"j i J a f f i f"}},
106
+ {"hebrew":"חידש","past":{"arabic":"جدّد","phonemes":"J a d d a d"},"future":{"arabic":"يجدّد","phonemes":"j i J a d d i d"}},
107
+ {"hebrew":"רץ, קרה, זרם","past":{"arabic":"جرى","phonemes":"J a r a"},"future":{"arabic":"يجري","phonemes":"j i J r i"}},
108
+ {"hebrew":"גמל, העניש","past":{"arabic":"جازى","phonemes":"J aː z a"},"future":{"arabic":"يجازي","phonemes":"j i J aː z i"}},
109
+ {"hebrew":"גרר, סחב","past":{"arabic":"جرّ","phonemes":"J a r r"},"future":{"arabic":"يجرّ","phonemes":"j i J u r r"}}
110
+ ]
111
+ },
112
+ "dal": {
113
+ "source": {"title":"פעלים שימושיים בערבית מדוברת — אות דאל","url":"https://www.youtube.com/watch?v=Wu2oeHyKujo","dialect":"Palestinian spoken Arabic"},
114
+ "verbs": [
115
+ {"hebrew":"אבד","past":{"arabic":"ضاع","phonemes":"dˤ aː ʕ"},"future":{"arabic":"يضيع","phonemes":"j i dˤ iː ʕ"}},
116
+ {"hebrew":"סידר, הסדיר","past":{"arabic":"دبّر","phonemes":"d a b b a r"},"future":{"arabic":"يدبّر","phonemes":"j i d a b b i r"}},
117
+ {"hebrew":"צבע","past":{"arabic":"دهن","phonemes":"d a h a n"},"future":{"arabic":"يدهن","phonemes":"j i d h a n"}},
118
+ {"hebrew":"דרס","past":{"arabic":"دهس","phonemes":"d a h a s"},"future":{"arabic":"يدهس","phonemes":"j i d h a s"}},
119
+ {"hebrew":"חיפש","past":{"arabic":"دوّر على","phonemes":"d a w w a r ʕ a l a"},"future":{"arabic":"يدوّر على","phonemes":"j i d a w w i r ʕ a l a"}},
120
+ {"hebrew":"עישן","past":{"arabic":"دخّن","phonemes":"d a x x a n"},"future":{"arabic":"يدخّن","phonemes":"j i d a x x i n"}},
121
+ {"hebrew":"הצביע על, הראה","past":{"arabic":"دلّ على","phonemes":"d a l l ʕ a l a"},"future":{"arabic":"يدلّ على","phonemes":"j i d u l l ʕ a l a"}},
122
+ {"hebrew":"שילם, דחף","past":{"arabic":"دفع","phonemes":"d a f a ʕ"},"future":{"arabic":"يدفع","phonemes":"j i d f a ʕ"}},
123
+ {"hebrew":"דפק","past":{"arabic":"دقّ","phonemes":"d a Q Q"},"future":{"arabic":"يدقّ","phonemes":"j i d u Q Q"}},
124
+ {"hebrew":"עזב, נטש","past":{"arabic":"دشّر","phonemes":"d a ʃ ʃ a r"},"future":{"arabic":"يدشّر","phonemes":"j i d a ʃ ʃ i r"}},
125
+ {"hebrew":"סובב, הפנה","past":{"arabic":"دار","phonemes":"d aː r"},"future":{"arabic":"يدير","phonemes":"j i d iː r"}}
126
+ ]
127
+ },
128
+ "ha": {
129
+ "source": {"title":"פעלים שימושיים בערבית מדוברת — אות ה","url":"https://www.youtube.com/watch?v=le--lfAYjgE","dialect":"Palestinian spoken Arabic"},
130
+ "verbs": [
131
+ {"hebrew":"ברח","past":{"arabic":"هرب","phonemes":"h a r a b"},"future":{"arabic":"يهرب","phonemes":"j i h r a b"}},
132
+ {"hebrew":"היגר","past":{"arabic":"هاجر","phonemes":"h aː J a r"},"future":{"arabic":"يهاجر","phonemes":"j i h aː J i r"}},
133
+ {"hebrew":"הגלה, גרם להגר","past":{"arabic":"هجّر","phonemes":"h a J J a r"},"future":{"arabic":"يهجّر","phonemes":"j i h a J J i r"}},
134
+ {"hebrew":"נפל, ירד","past":{"arabic":"هبط","phonemes":"h a b a tˤ"},"future":{"arabic":"يهبط","phonemes":"j i h b a tˤ"}},
135
+ {"hebrew":"הוריד, החליש","past":{"arabic":"هبّط","phonemes":"h a b b a tˤ"},"future":{"arabic":"يهبّط","phonemes":"j i h a b b i tˤ"}},
136
+ {"hebrew":"התכוון, שאף","past":{"arabic":"هدف","phonemes":"h a d a f"},"future":{"arabic":"يهدف","phonemes":"j i h d a f"}},
137
+ {"hebrew":"בירך","past":{"arabic":"هنّى","phonemes":"h a n n a"},"future":{"arabic":"يهنّي","phonemes":"j i h a n n i"}},
138
+ {"hebrew":"הריע, קרא","past":{"arabic":"هتف","phonemes":"h a t a f"},"future":{"arabic":"يهتف","phonemes":"j i h t i f"}}
139
+ ]
140
+ },
141
+ "waw": {
142
+ "source": {"title":"פעלים שימושיים בערבית מדוברת — אות ואו","url":"https://www.youtube.com/watch?v=r5nOxemdyMI","dialect":"Palestinian spoken Arabic"},
143
+ "verbs": [
144
+ {"hebrew":"מצא","past":{"arabic":"وجد","phonemes":"w a J a d"},"future":{"arabic":"يجد","phonemes":"j i J i d"}},
145
+ {"hebrew":"הביא למקום","past":{"arabic":"ودّى","phonemes":"w a d d a"},"future":{"arabic":"يودّي","phonemes":"j i w a d d i"}},
146
+ {"hebrew":"נפרד","past":{"arabic":"ودّع","phonemes":"w a d d a ʕ"},"future":{"arabic":"يودّع","phonemes":"j i w a d d i ʕ"}},
147
+ {"hebrew":"מינה, הסמיך","past":{"arabic":"وكّل","phonemes":"w a k k a l"},"future":{"arabic":"يوكّل","phonemes":"j i w a k k i l"}},
148
+ {"hebrew":"עצר, עמד","past":{"arabic":"وقف","phonemes":"w a Q a f"},"future":{"arabic":"يوقف","phonemes":"j i w Q a f"}},
149
+ {"hebrew":"ייבא","past":{"arabic":"ورّد","phonemes":"w a r r a d"},"future":{"arabic":"يورّد","phonemes":"j i w a r r i d"}},
150
+ {"hebrew":"הנמיך","past":{"arabic":"وطّى","phonemes":"w a tˤ tˤ a"},"future":{"arabic":"يوطّي","phonemes":"j i w a tˤ tˤ i"}},
151
+ {"hebrew":"חיבר, הוביל, הביא","past":{"arabic":"وصّل","phonemes":"w a sˤ sˤ a l"},"future":{"arabic":"يوصّل","phonemes":"j i w a sˤ sˤ i l"}}
152
+ ]
153
+ },
154
+ "zay": {
155
+ "source": {"title":"פעלים שימושיים בערבית מדוברת — אות זאי","url":"https://www.youtube.com/watch?v=Qgal8RB1N00","dialect":"Palestinian spoken Arabic"},
156
+ "verbs": [
157
+ {"hebrew":"כעס, התרגז","past":{"arabic":"زعل","phonemes":"z a ʕ a l"},"future":{"arabic":"يزعل","phonemes":"j i z ʕ a l"}},
158
+ {"hebrew":"זז, הסיר","past":{"arabic":"زاح","phonemes":"z aː ħ"},"future":{"arabic":"يزيح","phonemes":"j i z iː ħ"}},
159
+ {"hebrew":"סיפק, צייד","past":{"arabic":"زوّد","phonemes":"z a w w a d"},"future":{"arabic":"يزوّد","phonemes":"j i z a w w i d"}},
160
+ {"hebrew":"צפר","past":{"arabic":"زمّر","phonemes":"z a m m a r"},"future":{"arabic":"يزمّر","phonemes":"j i z a m m i r"}},
161
+ {"hebrew":"הוסיף, גדל","past":{"arabic":"زاد","phonemes":"z aː d"},"future":{"arabic":"يزيد","phonemes":"j i z iː d"}},
162
+ {"hebrew":"ביקר","past":{"arabic":"زار","phonemes":"z aː r"},"future":{"arabic":"يزور","phonemes":"j i z uː r"}},
163
+ {"hebrew":"הזכיר, ציטט","past":{"arabic":"ذكر","phonemes":"DH a k a r"},"future":{"arabic":"يذكر","phonemes":"j i DH k u r"}},
164
+ {"hebrew":"הזכיר למישהו","past":{"arabic":"ذكّر","phonemes":"DH a k k a r"},"future":{"arabic":"يذكّر","phonemes":"j i DH a k k i r"}},
165
+ {"hebrew":"דחף","past":{"arabic":"زقّ","phonemes":"z a Q Q"},"future":{"arabic":"يزقّ","phonemes":"j i z u Q Q"}},
166
+ {"hebrew":"זייף","past":{"arabic":"زيّف","phonemes":"z a j j a f"},"future":{"arabic":"يزيّف","phonemes":"j i z a j j i f"}},
167
+ {"hebrew":"שחט","past":{"arabic":"ذبح","phonemes":"DH a b a ħ"},"future":{"arabic":"يذبح","phonemes":"j i DH b a ħ"}},
168
+ {"hebrew":"נמאס לו","past":{"arabic":"زهق","phonemes":"z a h a Q"},"future":{"arabic":"يزهق","phonemes":"j i z h a Q"}},
169
+ {"hebrew":"החליק, גרם להחליק","past":{"arabic":"زحلق","phonemes":"z a ħ l a Q"},"future":{"arabic":"يزحلق","phonemes":"j i z a ħ l i Q"}}
170
+ ]
171
+ },
172
+ "kha": {
173
+ "source": {"title":"פעלים שימושיים בערבית מדוברת — אות ח'א","url":"https://www.youtube.com/watch?v=RJnw8Bjtvlg","dialect":"Palestinian spoken Arabic"},
174
+ "verbs": [
175
+ {"hebrew":"בגד","past":{"arabic":"خان","phonemes":"x aː n"},"future":{"arabic":"يخون","phonemes":"j i x uː n"}},
176
+ {"hebrew":"נגמר, הסתיים","past":{"arabic":"خلص","phonemes":"x a l a sˤ"},"future":{"arabic":"يخلص","phonemes":"j i x l a sˤ"}},
177
+ {"hebrew":"הפסיד","past":{"arabic":"خسر","phonemes":"x a s a r"},"future":{"arabic":"يخسر","phonemes":"j i x s a r"}},
178
+ {"hebrew":"התקלקל","past":{"arabic":"خرب","phonemes":"x a r a b"},"future":{"arabic":"يخرب","phonemes":"j i x r a b"}},
179
+ {"hebrew":"פחד","past":{"arabic":"خاف","phonemes":"x aː f"},"future":{"arabic":"يخاف","phonemes":"j i x aː f"}},
180
+ {"hebrew":"הודיע, סיפר","past":{"arabic":"خبّر","phonemes":"x a b b a r"},"future":{"arabic":"يخبّر","phonemes":"j i x a b b i r"}},
181
+ {"hebrew":"אחסן","past":{"arabic":"خزّن","phonemes":"x a z z a n"},"future":{"arabic":"يخزّن","phonemes":"j i x a z z i n"}},
182
+ {"hebrew":"התבייש","past":{"arabic":"خجل","phonemes":"x a J a l"},"future":{"arabic":"يخجل","phonemes":"j i x J a l"}},
183
+ {"hebrew":"השאיר, נטש","past":{"arabic":"خلّى","phonemes":"x a l l a"},"future":{"arabic":"يخلّي","phonemes":"j i x a l l i"}},
184
+ {"hebrew":"השאיר אחריו, הוליד","past":{"arabic":"خلّف","phonemes":"x a l l a f"},"future":{"arabic":"يخلّف","phonemes":"j i x a l l i f"}},
185
+ {"hebrew":"יצר, ברא","past":{"arabic":"خلق","phonemes":"x a l a Q"},"future":{"arabic":"يخلق","phonemes":"j i x l a Q"}},
186
+ {"hebrew":"הפחית","past":{"arabic":"خفّف","phonemes":"x a f f a f"},"future":{"arabic":"يخفّف","phonemes":"j i x a f f i f"}},
187
+ {"hebrew":"התערבב, התחבר","past":{"arabic":"خالط","phonemes":"x aː l a tˤ"},"future":{"arabic":"يخالط","phonemes":"j i x aː l i tˤ"}},
188
+ {"hebrew":"אכזב","past":{"arabic":"خيّب","phonemes":"x a j j a b"},"future":{"arabic":"يخيّب","phonemes":"j i x a j j i b"}},
189
+ {"hebrew":"חתם","past":{"arabic":"ختم","phonemes":"x a t a m"},"future":{"arabic":"يختم","phonemes":"j i x t i m"}},
190
+ {"hebrew":"קלקל","past":{"arabic":"خرّب","phonemes":"x a r r a b"},"future":{"arabic":"يخرّب","phonemes":"j i x a r r i b"}},
191
+ {"hebrew":"יצא","past":{"arabic":"خرج","phonemes":"x a r a J"},"future":{"arabic":"يخرج","phonemes":"j i x r a J"}},
192
+ {"hebrew":"נשא נאום","past":{"arabic":"خطب","phonemes":"x a tˤ a b"},"future":{"arabic":"يخطب","phonemes":"j i x tˤ a b"}}
193
+ ]
194
+ },
195
+ "ta_emphatic": {
196
+ "source":{"title":"פעלים שימושיים בערבית מדוברת — אות טא","url":"https://www.youtube.com/watch?v=qYeEWydByZU","dialect":"Palestinian spoken Arabic"},
197
+ "verbs":[
198
+ {"hebrew":"נרפא, היה טוב","past":{"arabic":"طاب","phonemes":"tˤ aː b"},"future":{"arabic":"يطيب","phonemes":"j i tˤ iː b"}},
199
+ {"hebrew":"הדפיס","past":{"arabic":"طبع","phonemes":"tˤ a b a ʕ"},"future":{"arabic":"يطبع","phonemes":"j i tˤ b a ʕ"}},
200
+ {"hebrew":"האריך, התעכב","past":{"arabic":"طوّل","phonemes":"tˤ a w w a l"},"future":{"arabic":"يطوّل","phonemes":"j i tˤ a w w i l"}},
201
+ {"hebrew":"דרש, ביקש","past":{"arabic":"طلب","phonemes":"tˤ a l a b"},"future":{"arabic":"يطلب","phonemes":"j i tˤ l a b"}},
202
+ {"hebrew":"האכיל","past":{"arabic":"طعّم","phonemes":"tˤ a ʕ ʕ a m"},"future":{"arabic":"يطعّم","phonemes":"j i tˤ a ʕ ʕ i m"}},
203
+ {"hebrew":"היה מסוגל לסבול","past":{"arabic":"طاق","phonemes":"tˤ aː Q"},"future":{"arabic":"يطيق","phonemes":"j i tˤ iː Q"}},
204
+ {"hebrew":"הרגיע","past":{"arabic":"طمّن","phonemes":"tˤ a m m a n"},"future":{"arabic":"يطمّن","phonemes":"j i tˤ a m m i n"}},
205
+ {"hebrew":"פיצח","past":{"arabic":"طقش","phonemes":"tˤ a Q a ʃ"},"future":{"arabic":"يطقش","phonemes":"j i tˤ Q a ʃ"}},
206
+ {"hebrew":"התעלם","past":{"arabic":"طنّش","phonemes":"tˤ a n n a ʃ"},"future":{"arabic":"يطنّش","phonemes":"j i tˤ a n n i ʃ"}},
207
+ {"hebrew":"עף","past":{"arabic":"طار","phonemes":"tˤ aː r"},"future":{"arabic":"يطير","phonemes":"j i tˤ iː r"}},
208
+ {"hebrew":"רדף","past":{"arabic":"طارد","phonemes":"tˤ aː r a d"},"future":{"arabic":"يطارد","phonemes":"j i tˤ aː r i d"}},
209
+ {"hebrew":"כיבה","past":{"arabic":"طفّى","phonemes":"tˤ a f f a"},"future":{"arabic":"يطفّي","phonemes":"j i tˤ a f f i"}},
210
+ {"hebrew":"יישם","past":{"arabic":"طبّق","phonemes":"tˤ a b b a Q"},"future":{"arabic":"يطبّق","phonemes":"j i tˤ a b b i Q"}},
211
+ {"hebrew":"הציץ, ביקר","past":{"arabic":"طلّ","phonemes":"tˤ a l l"},"future":{"arabic":"يطلّ","phonemes":"j i tˤ u l l"}},
212
+ {"hebrew":"שאף","past":{"arabic":"طمح","phonemes":"tˤ a m a ħ"},"future":{"arabic":"يطمح","phonemes":"j i tˤ m a ħ"}},
213
+ {"hebrew":"התיז","past":{"arabic":"طرّش","phonemes":"tˤ a r r a ʃ"},"future":{"arabic":"يطرّش","phonemes":"j i tˤ a r r i ʃ"}}
214
+ ]
215
+ },
216
+ "kaf": {
217
+ "source":{"title":"פעלים שימושיים בערבית מדוברת — אות כאף","url":"https://www.youtube.com/watch?v=wSoGsCXTyig","dialect":"Palestinian spoken Arabic"},
218
+ "verbs":[
219
+ {"hebrew":"יצר, הרכיב","past":{"arabic":"كوّن","phonemes":"k a w w a n"},"future":{"arabic":"يكوّن","phonemes":"j i k a w w i n"}},
220
+ {"hebrew":"המשיך, השלים","past":{"arabic":"كمّل","phonemes":"k a m m a l"},"future":{"arabic":"يكمّل","phonemes":"j i k a m m i l"}},
221
+ {"hebrew":"טאטא","past":{"arabic":"كنس","phonemes":"k a n a s"},"future":{"arabic":"يكنس","phonemes":"j i k n u s"}},
222
+ {"hebrew":"שפך","past":{"arabic":"كبّ","phonemes":"k a b b"},"future":{"arabic":"يكبّ","phonemes":"j i k u b b"}},
223
+ {"hebrew":"חרק שיניים","past":{"arabic":"كدّ","phonemes":"k a d d"},"future":{"arabic":"يكدّ","phonemes":"j i k u d d"}},
224
+ {"hebrew":"עלה במחיר","past":{"arabic":"كلّف","phonemes":"k a l l a f"},"future":{"arabic":"يكلّف","phonemes":"j i k a l l i f"}},
225
+ {"hebrew":"גדל","past":{"arabic":"كبر","phonemes":"k a b a r"},"future":{"arabic":"يكبر","phonemes":"j i k b a r"}},
226
+ {"hebrew":"קימט את המצח","past":{"arabic":"كشّر","phonemes":"k a ʃ ʃ a r"},"future":{"arabic":"يكشّر","phonemes":"j i k a ʃ ʃ i r"}},
227
+ {"hebrew":"שבר","past":{"arabic":"كسر","phonemes":"k a s a r"},"future":{"arabic":"يكسر","phonemes":"j i k s a r"}},
228
+ {"hebrew":"כתב","past":{"arabic":"كتب","phonemes":"k a t a b"},"future":{"arabic":"يكتب","phonemes":"j i k t u b"}},
229
+ {"hebrew":"צבר","past":{"arabic":"كدّس","phonemes":"k a d d a s"},"future":{"arabic":"يكدّس","phonemes":"j i k a d d i s"}},
230
+ {"hebrew":"עשה בלגן","past":{"arabic":"كركب","phonemes":"k a r k a b"},"future":{"arabic":"يكركب","phonemes":"j i k a r k i b"}},
231
+ {"hebrew":"חזר על","past":{"arabic":"كرّر","phonemes":"k a r r a r"},"future":{"arabic":"يكرّر","phonemes":"j i k a r r i r"}},
232
+ {"hebrew":"גירש","past":{"arabic":"كشّ","phonemes":"k a ʃ ʃ"},"future":{"arabic":"يكشّ","phonemes":"j i k u ʃ ʃ"}}
233
+ ]
234
+ },
235
+ "lam": {
236
+ "source":{"title":"פעלים שימושיים בערבית מדוברת — אות לאם","url":"https://www.youtube.com/watch?v=-Jd2w7EEs90","dialect":"Palestinian spoken Arabic"},
237
+ "verbs":[
238
+ {"hebrew":"צופף","past":{"arabic":"لزّ","phonemes":"l a z z"},"future":{"arabic":"يلزّ","phonemes":"j i l u z z"}},
239
+ {"hebrew":"עקץ, הכיש, צרב","past":{"arabic":"لسع","phonemes":"l a s a ʕ"},"future":{"arabic":"يلسع","phonemes":"j i l s a ʕ"}},
240
+ {"hebrew":"הפציר","past":{"arabic":"لحّ","phonemes":"l a ħ ħ"},"future":{"arabic":"يلحّ","phonemes":"j i l i ħ ħ"}},
241
+ {"hebrew":"אסף","past":{"arabic":"لمّ","phonemes":"l a m m"},"future":{"arabic":"يلمّ","phonemes":"j i l u m m"}},
242
+ {"hebrew":"האשים, גינה","past":{"arabic":"لام","phonemes":"l aː m"},"future":{"arabic":"يلوم","phonemes":"j i l uː m"}},
243
+ {"hebrew":"לימד בשינון","past":{"arabic":"لقّن","phonemes":"l a Q Q a n"},"future":{"arabic":"يلقّن","phonemes":"j i l a Q Q i n"}},
244
+ {"hebrew":"מילא בקשה","past":{"arabic":"لبّى","phonemes":"l a b b a"},"future":{"arabic":"يلبّي","phonemes":"j i l a b b i"}},
245
+ {"hebrew":"לבש","past":{"arabic":"لبس","phonemes":"l a b a s"},"future":{"arabic":"يلبس","phonemes":"j i l b a s"}},
246
+ {"hebrew":"הלביש","past":{"arabic":"لبّس","phonemes":"l a b b a s"},"future":{"arabic":"يلبّس","phonemes":"j i l a b b i s"}},
247
+ {"hebrew":"טייח, העלים","past":{"arabic":"لفلف","phonemes":"l a f l a f"},"future":{"arabic":"يلفلف","phonemes":"j i l a f l i f"}},
248
+ {"hebrew":"סטר","past":{"arabic":"لطش","phonemes":"l a tˤ a ʃ"},"future":{"arabic":"يلطش","phonemes":"j i l tˤ a ʃ"}},
249
+ {"hebrew":"רמז","past":{"arabic":"لمّح","phonemes":"l a m m a ħ"},"future":{"arabic":"يلمّح","phonemes":"j i l a m m i ħ"}}
250
+ ]
251
+ },
252
+ "mim": {
253
+ "source":{"title":"פעלים שימושיים בערבית מדוברת — אות מים","url":"https://www.youtube.com/watch?v=yKUcU45Ngpg","dialect":"Palestinian spoken Arabic"},
254
+ "verbs":[
255
+ {"hebrew":"דחה ועיכב","past":{"arabic":"ماطل","phonemes":"m aː tˤ a l"},"future":{"arabic":"يماطل","phonemes":"j i m aː tˤ i l"}},
256
+ {"hebrew":"עסק ב־, הפעיל","past":{"arabic":"مارس","phonemes":"m aː r a s"},"future":{"arabic":"يمارس","phonemes":"j i m aː r i s"}},
257
+ {"hebrew":"מת","past":{"arabic":"مات","phonemes":"m aː t"},"future":{"arabic":"يموت","phonemes":"j i m uː t"}},
258
+ {"hebrew":"מימן","past":{"arabic":"موّل","phonemes":"m a w w a l"},"future":{"arabic":"يموّل","phonemes":"j i m a w w i l"}},
259
+ {"hebrew":"קרע לגזרים","past":{"arabic":"مزّع","phonemes":"m a z z a ʕ"},"future":{"arabic":"يمزّع","phonemes":"j i m a z z i ʕ"}},
260
+ {"hebrew":"הבחין בין","past":{"arabic":"ميّز","phonemes":"m a j j a z"},"future":{"arabic":"يميّز","phonemes":"j i m a j j i z"}},
261
+ {"hebrew":"מילא","past":{"arabic":"ملا","phonemes":"m a l a"},"future":{"arabic":"يملي","phonemes":"j i m l i"}},
262
+ {"hebrew":"שיבח","past":{"arabic":"مدح","phonemes":"m a d a ħ"},"future":{"arabic":"يمدح","phonemes":"j i m d a ħ"}},
263
+ {"hebrew":"סיפק","past":{"arabic":"موّن","phonemes":"m a w w a n"},"future":{"arabic":"يموّن","phonemes":"j i m a w w i n"}},
264
+ {"hebrew":"שטף, היה בעלים","past":{"arabic":"ملك","phonemes":"m a l a k"},"future":{"arabic":"يملك","phonemes":"j i m l u k"}},
265
+ {"hebrew":"ליטף, החליק","past":{"arabic":"ملّس","phonemes":"m a l l a s"},"future":{"arabic":"يملّس","phonemes":"j i m a l l i s"}},
266
+ {"hebrew":"העניק","past":{"arabic":"منح","phonemes":"m a n a ħ"},"future":{"arabic":"يمنح","phonemes":"j i m n a ħ"}},
267
+ {"hebrew":"מנע, אסר","past":{"arabic":"منع","phonemes":"m a n a ʕ"},"future":{"arabic":"يمنع","phonemes":"j i m n a ʕ"}},
268
+ {"hebrew":"מצץ","past":{"arabic":"مصّ","phonemes":"m a sˤ sˤ"},"future":{"arabic":"يمصّ","phonemes":"j i m u sˤ sˤ"}},
269
+ {"hebrew":"העמיד פנים","past":{"arabic":"مثّل","phonemes":"m a TH TH a l"},"future":{"arabic":"يمثّل","phonemes":"j i m a TH TH i l"}},
270
+ {"hebrew":"ערבב","past":{"arabic":"مزج","phonemes":"m a z a J"},"future":{"arabic":"يمزج","phonemes":"j i m z a J"}},
271
+ {"hebrew":"התנדנד","past":{"arabic":"مرجح","phonemes":"m a r J a ħ"},"future":{"arabic":"يمرجح","phonemes":"j i m a r J i ħ"}},
272
+ {"hebrew":"הוליך, הפעיל","past":{"arabic":"مشّ��","phonemes":"m a ʃ ʃ a"},"future":{"arabic":"يمشّي","phonemes":"j i m a ʃ ʃ i"}},
273
+ {"hebrew":"עבר","past":{"arabic":"مرق","phonemes":"m a r a Q"},"future":{"arabic":"يمرق","phonemes":"j i m r u Q"}}
274
+ ]
275
+ },
276
+ "nun": {
277
+ "source":{"title":"פעלים שימושיים בערבית מדוברת — אות נון","url":"https://www.youtube.com/watch?v=KdipB6ik22Y","dialect":"Palestinian spoken Arabic"},
278
+ "verbs":[
279
+ {"hebrew":"ישן, שכב","past":{"arabic":"نام","phonemes":"n aː m"},"future":{"arabic":"ينام","phonemes":"j i n aː m"}},
280
+ {"hebrew":"הרדים","past":{"arabic":"نوّم","phonemes":"n a w w a m"},"future":{"arabic":"ينوّم","phonemes":"j i n a w w i m"}},
281
+ {"hebrew":"שכח","past":{"arabic":"نسي","phonemes":"n a s i"},"future":{"arabic":"ينسى","phonemes":"j i n s a"}},
282
+ {"hebrew":"השכיח","past":{"arabic":"نسّى","phonemes":"n a s s a"},"future":{"arabic":"ينسّي","phonemes":"j i n a s s i"}},
283
+ {"hebrew":"התנמנם","past":{"arabic":"نعس","phonemes":"n i ʕ i s"},"future":{"arabic":"ينعس","phonemes":"j i n ʕ a s"}},
284
+ {"hebrew":"נשף, נשם","past":{"arabic":"نفش","phonemes":"n a f a ʃ"},"future":{"arabic":"ينفش","phonemes":"j i n f u ʃ"}},
285
+ {"hebrew":"ניתר","past":{"arabic":"نطّ","phonemes":"n a tˤ tˤ"},"future":{"arabic":"ينطّ","phonemes":"j i n u tˤ tˤ"}},
286
+ {"hebrew":"העביר, העתיק","past":{"arabic":"نقل","phonemes":"n a Q a l"},"future":{"arabic":"ينقل","phonemes":"j i n Q u l"}},
287
+ {"hebrew":"התחרה","past":{"arabic":"نافس","phonemes":"n aː f a s"},"future":{"arabic":"ينافس","phonemes":"j i n aː f i s"}},
288
+ {"hebrew":"ניקה","past":{"arabic":"نضّف","phonemes":"n a dˤ dˤ a f"},"future":{"arabic":"ينضّف","phonemes":"j i n a dˤ dˤ i f"}},
289
+ {"hebrew":"בחר","past":{"arabic":"نقّى","phonemes":"n a Q Q a"},"future":{"arabic":"ينقّي","phonemes":"j i n a Q Q i"}},
290
+ {"hebrew":"סגר","past":{"arabic":"نتّ","phonemes":"n a t t"},"future":{"arabic":"ينتّ","phonemes":"j i n u t t"}}
291
+ ]
292
+ },
293
+ "sin": {
294
+ "source":{"title":"פעלים שימושיים בערבית מדוברת — אות סין","url":"https://www.youtube.com/watch?v=DlM_AscrScY","dialect":"Palestinian spoken Arabic"},
295
+ "verbs":[
296
+ {"hebrew":"החביא, שמר","past":{"arabic":"ستر","phonemes":"s a t a r"},"future":{"arabic":"يستر","phonemes":"j i s t u r"}},
297
+ {"hebrew":"שמע","past":{"arabic":"سمع","phonemes":"s a m i ʕ"},"future":{"arabic":"يسمع","phonemes":"j i s m a ʕ"}},
298
+ {"hebrew":"עזר","past":{"arabic":"ساعد","phonemes":"s aː ʕ a d"},"future":{"arabic":"يساعد","phonemes":"j i s aː ʕ i d"}},
299
+ {"hebrew":"כישף","past":{"arabic":"سحر","phonemes":"s a ħ a r"},"future":{"arabic":"يسحر","phonemes":"j i s ħ a r"}},
300
+ {"hebrew":"שאל","past":{"arabic":"سأل","phonemes":"s a ʔ a l"},"future":{"arabic":"يسأل","phonemes":"j i s ʔ a l"}},
301
+ {"hebrew":"גר, שכן","past":{"arabic":"سكن","phonemes":"s a k a n"},"future":{"arabic":"يسكن","phonemes":"j i s k u n"}},
302
+ {"hebrew":"השתלט","past":{"arabic":"سيطر","phonemes":"s a j tˤ a r"},"future":{"arabic":"يسيطر","phonemes":"j i s a j tˤ i r"}},
303
+ {"hebrew":"נסע","past":{"arabic":"سافر","phonemes":"s aː f a r"},"future":{"arabic":"يسافر","phonemes":"j i s aː f i r"}},
304
+ {"hebrew":"הרגיז","past":{"arabic":"سفّن","phonemes":"s a f f a n"},"future":{"arabic":"يسفّن","phonemes":"j i s a f f i n"}},
305
+ {"hebrew":"סלח","past":{"arabic":"سامح","phonemes":"s aː m a ħ"},"future":{"arabic":"يسامح","phonemes":"j i s aː m i ħ"}},
306
+ {"hebrew":"גנב","past":{"arabic":"سرق","phonemes":"s a r a Q"},"future":{"arabic":"يسرق","phonemes":"j i s r u Q"}},
307
+ {"hebrew":"שתק","past":{"arabic":"سكت","phonemes":"s a k a t"},"future":{"arabic":"يسكت","phonemes":"j i s k u t"}},
308
+ {"hebrew":"צחק","past":{"arabic":"سخف","phonemes":"s a x a f"},"future":{"arabic":"يسخف","phonemes":"j i s x a f"}},
309
+ {"hebrew":"קילל","past":{"arabic":"سبّ","phonemes":"s a b b"},"future":{"arabic":"يسبّ","phonemes":"j i s u b b"}},
310
+ {"hebrew":"עזב","past":{"arabic":"ساب","phonemes":"s aː b"},"future":{"arabic":"يسيب","phonemes":"j i s iː b"}},
311
+ {"hebrew":"שיבח, עזר לשחות","past":{"arabic":"سبّح","phonemes":"s a b b a ħ"},"future":{"arabic":"يسبّح","phonemes":"j i s a b b i ħ"}},
312
+ {"hebrew":"הקל","past":{"arabic":"سهّل","phonemes":"s a h h a l"},"future":{"arabic":"يسهّل","phonemes":"j i s a h h i l"}},
313
+ {"hebrew":"סידר, עשה","past":{"arabic":"سوّى","phonemes":"s a w w a"},"future":{"arabic":"يسوّي","phonemes":"j i s a w w i"}}
314
+ ]
315
+ },
316
+ "ayn": {
317
+ "source":{"title":"פעלים שימושיים בערבית מדוברת — אות עין","url":"https://www.youtube.com/watch?v=BZU4jitxcuU","dialect":"Palestinian spoken Arabic"},
318
+ "verbs":[
319
+ {"hebrew":"גילה עירנות","past":{"arabic":"وعى","phonemes":"w a ʕ a"},"future":{"arabic":"يوعى","phonemes":"j i w ʕ a"}},
320
+ {"hebrew":"לימד","past":{"arabic":"علّم","phonemes":"ʕ a l l a m"},"future":{"arabic":"يعلّم","phonemes":"j i ʕ a l l i m"}},
321
+ {"hebrew":"כעס","past":{"arabic":"عكّر","phonemes":"ʕ a k k a r"},"future":{"arabic":"يعكّر","phonemes":"j i ʕ a k k i r"}},
322
+ {"hebrew":"הזיע","past":{"arabic":"عرق","phonemes":"ʕ a r i Q"},"future":{"arabic":"يعرق","phonemes":"j i ʕ r a Q"}},
323
+ {"hebrew":"איפס, כיוון","past":{"arabic":"عيّر","phonemes":"ʕ a j j a r"},"future":{"arabic":"يعيّر","phonemes":"j i ʕ a j j i r"}},
324
+ {"hebrew":"הכיר, ידע","past":{"arabic":"عرف","phonemes":"ʕ i r i f"},"future":{"arabic":"يعرف","phonemes":"j i ʕ r a f"}},
325
+ {"hebrew":"בירך לכבוד חג","past":{"arabic":"عايد","phonemes":"ʕ aː j a d"},"future":{"arabic":"يعايد","phonemes":"j i ʕ aː j i d"}},
326
+ {"hebrew":"ספר","past":{"arabic":"عدّ","phonemes":"ʕ a d d"},"future":{"arabic":"يعدّ","phonemes":"j i ʕ u d d"}},
327
+ {"hebrew":"חזר על","past":{"arabic":"عاد","phonemes":"ʕ aː d"},"future":{"arabic":"يعيد","phonemes":"j i ʕ iː d"}},
328
+ {"hebrew":"שם מכשול","past":{"arabic":"عرقل","phonemes":"ʕ a r Q a l"},"future":{"arabic":"يعرقل","phonemes":"j i ʕ a r Q i l"}},
329
+ {"hebrew":"חי","past":{"arabic":"عاش","phonemes":"ʕ aː ʃ"},"future":{"arabic":"يعيش","phonemes":"j i ʕ iː ʃ"}},
330
+ {"hebrew":"עבר, נכנס","past":{"arabic":"عبر","phonemes":"ʕ a b a r"},"future":{"arabic":"يعبر","phonemes":"j i ʕ b u r"}},
331
+ {"hebrew":"התעקש","past":{"arabic":"عاند","phonemes":"ʕ aː n a d"},"future":{"arabic":"يعاند","phonemes":"j i ʕ aː n i d"}},
332
+ {"hebrew":"נתן פיצוי","past":{"arabic":"عوّض","phonemes":"ʕ a w w a dˤ"},"future":{"arabic":"يعوّض","phonemes":"j i ʕ a w w i dˤ"}},
333
+ {"hebrew":"אהב, חיבב","past":{"arabic":"عزّ","phonemes":"ʕ a z z"},"future":{"arabic":"يعزّ","phonemes":"j i ʕ i z z"}},
334
+ {"hebrew":"הדיח, מיהר","past":{"arabic":"عجّل","phonemes":"ʕ a J J a l"},"future":{"arabic":"يعجّل","phonemes":"j i ʕ a J J i l"}},
335
+ {"hebrew":"בנה","past":{"arabic":"عمّر","phonemes":"ʕ a m m a r"},"future":{"arabic":"يعمّر","phonemes":"j i ʕ a m m i r"}}
336
+ ]
337
+ },
338
+ "ghayn": {
339
+ "source":{"title":"פעלים שימושיים בערבית מדוברת — אות ע'ין","url":"https://www.youtube.com/watch?v=kqtBFuKjx8o","dialect":"Palestinian spoken Arabic"},
340
+ "verbs":[
341
+ {"hebrew":"עזב, נעלם","past":{"arabic":"غاب","phonemes":"ɣ aː b"},"future":{"arabic":"يغيب","phonemes":"j i ɣ iː b"}},
342
+ {"hebrew":"שקע","past":{"arabic":"غرق","phonemes":"ɣ i r i Q"},"future":{"arabic":"يغرق","phonemes":"j i ɣ r a Q"}},
343
+ {"hebrew":"שטף","past":{"arabic":"غسل","phonemes":"ɣ a s a l"},"future":{"arabic":"يغسل","phonemes":"j i ɣ s i l"}},
344
+ {"hebrew":"כיסה","past":{"arabic":"غطّى","phonemes":"ɣ a tˤ tˤ a"},"future":{"arabic":"يغطّي","phonemes":"j i ɣ a tˤ tˤ i"}},
345
+ {"hebrew":"שינה","past":{"arabic":"غيّر","phonemes":"ɣ a j j a r"},"future":{"arabic":"يغيّر","phonemes":"j i ɣ a j j i r"}},
346
+ {"hebrew":"רתח","past":{"arabic":"غلى","phonemes":"ɣ a l a"},"future":{"arabic":"يغلي","phonemes":"j i ɣ l i"}},
347
+ {"hebrew":"ניצח","past":{"arabic":"غلب","phonemes":"ɣ a l a b"},"future":{"arabic":"يغلب","phonemes":"j i ɣ l a b"}},
348
+ {"hebrew":"סלח","past":{"arabic":"غفر","phonemes":"ɣ a f a r"},"future":{"arabic":"يغفر","phonemes":"j i ɣ f i r"}}
349
+ ]
350
+ },
351
+ "fa": {
352
+ "source":{"title":"פעלים שימושיים בערבית מדוברת — אות פא","url":"https://www.youtube.com/watch?v=pCn0vhBVR5Q","dialect":"Palestinian spoken Arabic"},
353
+ "verbs":[
354
+ {"hebrew":"נכשל","past":{"arabic":"فشل","phonemes":"f a ʃ a l"},"future":{"arabic":"يفشل","phonemes":"j i f ʃ a l"}},
355
+ {"hebrew":"הבין","past":{"arabic":"فهم","phonemes":"f i h i m"},"future":{"arabic":"يفهم","phonemes":"j i f h a m"}},
356
+ {"hebrew":"פתח","past":{"arabic":"فتح","phonemes":"f a t a ħ"},"future":{"arabic":"يفتح","phonemes":"j i f t a ħ"}},
357
+ {"hebrew":"נכנס, עבר","past":{"arabic":"فات","phonemes":"f aː t"},"future":{"arabic":"يفوت","phonemes":"j i f uː t"}},
358
+ {"hebrew":"הפריד","past":{"arabic":"فرّق","phonemes":"f a r r a Q"},"future":{"arabic":"يفرّق","phonemes":"j i f a r r i Q"}},
359
+ {"hebrew":"ברח","past":{"arabic":"فلّ","phonemes":"f a l l"},"future":{"arabic":"يفلّ","phonemes":"j i f u l l"}},
360
+ {"hebrew":"חשב, הניח","past":{"arabic":"فكّر","phonemes":"f a k k a r"},"future":{"arabic":"يفكّر","phonemes":"j i f a k k i r"}},
361
+ {"hebrew":"שחרר, התיר","past":{"arabic":"فكّ","phonemes":"f a k k"},"future":{"arabic":"يفكّ","phonemes":"j i f u k k"}},
362
+ {"hebrew":"התעורר","past":{"arabic":"فاق","phonemes":"f aː Q"},"future":{"arabic":"يفيق","phonemes":"j i f iː Q"}},
363
+ {"hebrew":"הפסיק, ניתק","past":{"arabic":"فصل","phonemes":"f a sˤ a l"},"future":{"arabic":"يفصل","phonemes":"j i f sˤ a l"}}
364
+ ]
365
+ },
366
+ "dad": {
367
+ "source":{"title":"פעלים שימושיים בערבית מדוברת — אות דאד","url":"https://www.youtube.com/watch?v=ULZkU-MoU6w","dialect":"Palestinian spoken Arabic"},
368
+ "verbs":[
369
+ {"hebrew":"היכה","past":{"arabic":"ضرب","phonemes":"dˤ a r a b"},"future":{"arabic":"يضرب","phonemes":"j i dˤ r u b"}},
370
+ {"hebrew":"צחק","past":{"arabic":"ضحك","phonemes":"dˤ i ħ i k"},"future":{"arabic":"يضحك","phonemes":"j i dˤ ħ a k"}},
371
+ {"hebrew":"נשאר","past":{"arabic":"ضلّ","phonemes":"dˤ a l l"},"future":{"arabic":"يضلّ","phonemes":"j i dˤ a l l"}},
372
+ {"hebrew":"הלך לאיבוד","past":{"arabic":"ضاع","phonemes":"dˤ aː ʕ"},"future":{"arabic":"يضيع","phonemes":"j i dˤ iː ʕ"}},
373
+ {"hebrew":"הבטיח","past":{"arabic":"ضمن","phonemes":"dˤ a m a n"},"future":{"arabic":"يضمن","phonemes":"j i dˤ m a n"}},
374
+ {"hebrew":"לחץ","past":{"arabic":"ضغط","phonemes":"dˤ a ɣ a tˤ"},"future":{"arabic":"يضغط","phonemes":"j i dˤ ɣ a tˤ"}}
375
+ ]
376
+ },
377
+ "sad": {
378
+ "source":{"title":"פעלים שימושיים בערבית מדוברת — אות צאד","url":"https://www.youtube.com/watch?v=jCpab4z3dXY","dialect":"Palestinian spoken Arabic"},
379
+ "verbs":[
380
+ {"hebrew":"קרה","past":{"arabic":"صار","phonemes":"sˤ aː r"},"future":{"arabic":"يصير","phonemes":"j i sˤ iː r"}},
381
+ {"hebrew":"צעק","past":{"arabic":"صرخ","phonemes":"sˤ a r a x"},"future":{"arabic":"يصرخ","phonemes":"j i sˤ r a x"}},
382
+ {"hebrew":"תיקן","past":{"arabic":"صلّح","phonemes":"sˤ a l l a ħ"},"future":{"arabic":"يصلّح","phonemes":"j i sˤ a l l i ħ"}},
383
+ {"hebrew":"צם","past":{"arabic":"صام","phonemes":"sˤ aː m"},"future":{"arabic":"يصوم","phonemes":"j i sˤ uː m"}},
384
+ {"hebrew":"צילם","past":{"arabic":"صوّر","phonemes":"sˤ a w w a r"},"future":{"arabic":"يصوّر","phonemes":"j i sˤ a w w i r"}},
385
+ {"hebrew":"סגר","past":{"arabic":"سكّر","phonemes":"s a k k a r"},"future":{"arabic":"يسكّر","phonemes":"j i s a k k i r"}},
386
+ {"hebrew":"התעורר","past":{"arabic":"صحي","phonemes":"sˤ i ħ i"},"future":{"arabic":"يصحى","phonemes":"j i sˤ ħ a"}}
387
+ ]
388
+ },
389
+ "qaf": {
390
+ "source":{"title":"פעלים שימושיים בערבית מדוברת — אות קאף","url":"https://www.youtube.com/watch?v=zx4scJYgYAM","dialect":"Palestinian spoken Arabic"},
391
+ "verbs":[
392
+ {"hebrew":"אמר","past":{"arabic":"قال","phonemes":"Q aː l"},"future":{"arabic":"يقول","phonemes":"j i Q uː l"}},
393
+ {"hebrew":"קרא","past":{"arabic":"قرأ","phonemes":"Q a r a ʔ"},"future":{"arabic":"يقرأ","phonemes":"j i Q r a ʔ"}},
394
+ {"hebrew":"ישב","past":{"arabic":"قعد","phonemes":"Q a ʕ a d"},"future":{"arabic":"يقعد","phonemes":"j i Q ʕ a d"}},
395
+ {"hebrew":"קם","past":{"arabic":"قام","phonemes":"Q aː m"},"future":{"arabic":"يقوم","phonemes":"j i Q uː m"}},
396
+ {"hebrew":"החליט","past":{"arabic":"قرّر","phonemes":"Q a r r a r"},"future":{"arabic":"يقرّر","phonemes":"j i Q a r r i r"}},
397
+ {"hebrew":"חתך","past":{"arabic":"قطع","phonemes":"Q a tˤ a ʕ"},"future":{"arabic":"يقطع","phonemes":"j i Q tˤ a ʕ"}},
398
+ {"hebrew":"סגר","past":{"arabic":"قفل","phonemes":"Q a f a l"},"future":{"arabic":"يقفل","phonemes":"j i Q f u l"}},
399
+ {"hebrew":"הסכים","past":{"arabic":"قبل","phonemes":"Q i b i l"},"future":{"arabic":"يقبل","phonemes":"j i Q b a l"}},
400
+ {"hebrew":"מדד","past":{"arabic":"قاس","phonemes":"Q aː s"},"future":{"arabic":"يقيس","phonemes":"j i Q iː s"}},
401
+ {"hebrew":"פגש","past":{"arabic":"قابل","phonemes":"Q aː b a l"},"future":{"arabic":"يقابل","phonemes":"j i Q aː b i l"}}
402
+ ]
403
+ },
404
+ "ra": {
405
+ "source":{"title":"פעלים שימושיים בערבית מדוברת — אות רא","url":"https://www.youtube.com/watch?v=fMIJT4HowVg","dialect":"Palestinian spoken Arabic"},
406
+ "verbs":[
407
+ {"hebrew":"הלך","past":{"arabic":"راح","phonemes":"r aː ħ"},"future":{"arabic":"يروح","phonemes":"j i r uː ħ"}},
408
+ {"hebrew":"ראה","past":{"arabic":"رأى","phonemes":"r a ʔ a"},"future":{"arabic":"يرى","phonemes":"j i r a"}},
409
+ {"hebrew":"חזר","past":{"arabic":"رجع","phonemes":"r a J a ʕ"},"future":{"arabic":"يرجع","phonemes":"j i r J a ʕ"}},
410
+ {"hebrew":"ענה","past":{"arabic":"ردّ","phonemes":"r a d d"},"future":{"arabic":"يردّ","phonemes":"j i r u d d"}},
411
+ {"hebrew":"רכב","past":{"arabic":"ركب","phonemes":"r i k i b"},"future":{"arabic":"يركب","phonemes":"j i r k a b"}},
412
+ {"hebrew":"רץ","past":{"arabic":"ركض","phonemes":"r a k a dˤ"},"future":{"arabic":"يركض","phonemes":"j i r k u dˤ"}},
413
+ {"hebrew":"סידר","past":{"arabic":"رتّب","phonemes":"r a t t a b"},"future":{"arabic":"يرتّب","phonemes":"j i r a t t i b"}},
414
+ {"hebrew":"זרק","past":{"arabic":"رمى","phonemes":"r a m a"},"future":{"arabic":"يرمي","phonemes":"j i r m i"}},
415
+ {"hebrew":"ניצח","past":{"arabic":"ربح","phonemes":"r i b i ħ"},"future":{"arabic":"يربح","phonemes":"j i r b a ħ"}},
416
+ {"hebrew":"גידל","past":{"arabic":"ربّى","phonemes":"r a b b a"},"future":{"arabic":"يربّي","phonemes":"j i r a b b i"}}
417
+ ]
418
+ },
419
+ "shin": {
420
+ "source":{"title":"פעלים שימושיים בערבית מדוברת — אות שין","url":"https://www.youtube.com/watch?v=yrL4wM1-jH8","dialect":"Palestinian spoken Arabic"},
421
+ "verbs":[
422
+ {"hebrew":"ראה","past":{"arabic":"شاف","phonemes":"ʃ aː f"},"future":{"arabic":"يشوف","phonemes":"j i ʃ uː f"}},
423
+ {"hebrew":"שתה","past":{"arabic":"شرب","phonemes":"ʃ i r i b"},"future":{"arabic":"يشرب","phonemes":"j i ʃ r a b"}},
424
+ {"hebrew":"עבד","past":{"arabic":"اشتغل","phonemes":"ʔ i ʃ t a ɣ a l"},"future":{"arabic":"يشتغل","phonemes":"j i ʃ t a ɣ i l"}},
425
+ {"hebrew":"קנה","past":{"arabic":"شرى","phonemes":"ʃ a r a"},"future":{"arabic":"يشري","phonemes":"j i ʃ r i"}},
426
+ {"hebrew":"הסביר","past":{"arabic":"شرح","phonemes":"ʃ a r a ħ"},"future":{"arabic":"يشرح","phonemes":"j i ʃ r a ħ"}},
427
+ {"hebrew":"הדליק","past":{"arabic":"شغّل","phonemes":"ʃ a ɣ ɣ a l"},"future":{"arabic":"يشغّل","phonemes":"j i ʃ a ɣ ɣ i l"}},
428
+ {"hebrew":"סחב","past":{"arabic":"شال","phonemes":"ʃ aː l"},"future":{"arabic":"يشيل","phonemes":"j i ʃ iː l"}},
429
+ {"hebrew":"השתתף","past":{"arabic":"شارك","phonemes":"ʃ aː r a k"},"future":{"arabic":"يشارك","phonemes":"j i ʃ aː r i k"}},
430
+ {"hebrew":"קילל","past":{"arabic":"شتم","phonemes":"ʃ a t a m"},"future":{"arabic":"يشتم","phonemes":"j i ʃ t u m"}},
431
+ {"hebrew":"משך","past":{"arabic":"شدّ","phonemes":"ʃ a d d"},"future":{"arabic":"يشدّ","phonemes":"j i ʃ u d d"}}
432
+ ]
433
+ },
434
+ "ta": {
435
+ "source":{"title":"פעלים שימושיים בערבית מדוברת — אות תא, חלק 1","url":"https://www.youtube.com/watch?v=z41lWU0o54M","dialect":"Palestinian spoken Arabic"},
436
+ "verbs":[
437
+ {"hebrew":"עזב","past":{"arabic":"ترك","phonemes":"t a r a k"},"future":{"arabic":"يترك","phonemes":"j i t r u k"}},
438
+ {"hebrew":"עייף","past":{"arabic":"تعب","phonemes":"t i ʕ i b"},"future":{"arabic":"يتعب","phonemes":"j i t ʕ a b"}},
439
+ {"hebrew":"למד","past":{"arabic":"تعلّم","phonemes":"t a ʕ a l l a m"},"future":{"arabic":"يتعلّم","phonemes":"j i t a ʕ a l l a m"}},
440
+ {"hebrew":"דיבר","past":{"arabic":"تكلّم","phonemes":"t k a l l a m"},"future":{"arabic":"يتكلّم","phonemes":"j i t k a l l a m"}},
441
+ {"hebrew":"התחתן","past":{"arabic":"تجوّز","phonemes":"t J a w w a z"},"future":{"arabic":"يتجوّز","phonemes":"j i t J a w w a z"}},
442
+ {"hebrew":"איחר","past":{"arabic":"تأخّر","phonemes":"t ʔ a x x a r"},"future":{"arabic":"يتأخّر","phonemes":"j i t ʔ a x x a r"}},
443
+ {"hebrew":"התקשר","past":{"arabic":"اتّصل","phonemes":"ʔ i t t a sˤ a l"},"future":{"arabic":"يتّصل","phonemes":"j i t t a sˤ i l"}},
444
+ {"hebrew":"השתנה","past":{"arabic":"تغيّر","phonemes":"t ɣ a j j a r"},"future":{"arabic":"يتغيّر","phonemes":"j i t ɣ a j j a r"}},
445
+ {"hebrew":"הסכים","past":{"arabic":"اتّفق","phonemes":"ʔ i t t a f a Q"},"future":{"arabic":"يتّفق","phonemes":"j i t t a f i Q"}},
446
+ {"hebrew":"התקדם","past":{"arabic":"تقدّم","phonemes":"t Q a d d a m"},"future":{"arabic":"يتقدّم","phonemes":"j i t Q a d d a m"}},
447
+ {"hebrew":"תרגם","past":{"arabic":"ترجم","phonemes":"t a r J a m"},"future":{"arabic":"يترجم","phonemes":"j i t a r J i m"}},
448
+ {"hebrew":"נזכר","past":{"arabic":"تذكّر","phonemes":"t DH a k k a r"},"future":{"arabic":"يتذكّر","phonemes":"j i t DH a k k a r"}}
449
+ ]
450
+ }
451
+ }
static/index.html CHANGED
@@ -310,6 +310,7 @@ body.driving .mic::before{display:none} body.driving .mic-zone.two{display:flex}
310
  .flag-facts{display:grid;grid-template-columns:auto 1fr;gap:6px 12px;margin:8px 0 0}
311
  .flag-facts dt{color:var(--muted);font-size:13px;white-space:nowrap}
312
  .flag-facts dd{margin:0;min-width:0;overflow-wrap:anywhere}
 
313
  .flag-facts .ipa{direction:ltr;text-align:right;font:13px/1.5 ui-monospace,monospace;unicode-bidi:isolate}
314
  .flag-facts ul{margin:0;padding:0;list-style:none;display:flex;flex-direction:column;gap:3px}
315
  .flag-facts b.good{color:var(--olive)} .flag-facts b.mid{color:#a86a00} .flag-facts b.bad{color:#9f2f27}
@@ -571,6 +572,55 @@ dialog::backdrop{background:rgba(0,0,0,.45)}
571
  .dlg-head h2{font:400 22px/1 var(--f-display);margin:0}
572
  .dlg-head h2 small{display:block;margin-top:4px;font:600 13px/1.3 var(--f-body);color:var(--muted)}
573
  .phone-note{margin:4px 0 12px;color:var(--muted);font-size:13px}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
574
  .phone-rows{direction:ltr;overflow-x:auto;padding:3px 0 8px}
575
  .phone-row{display:grid;grid-template-columns:72px repeat(var(--phones),minmax(34px,1fr));gap:4px;min-width:max-content;margin-bottom:5px;align-items:stretch}
576
  .phone-row>span{display:grid;place-items:center;min-height:34px;padding:3px 5px;border-radius:7px;background:var(--bg);font:600 16px/1.2 sans-serif}
@@ -976,6 +1026,15 @@ body.quiet :is(#mic,#playMine,#driveBtn,.phone-btn),body.driving :is(#quietBtn,#
976
  <p class="phone-note">זהו פירוש המודל להקלטה, ולא תעתיק פונטי מוסמך. תנועות קרובות ואורך תנועה מסומנים כהתאמה מקורבת.</p>
977
  <div class="phone-rows" id="phoneRows"></div>
978
  <div class="phone-legend"><span><i class="good"></i>התאמה</span><span><i style="background:var(--olive-soft)"></i>התאמה מקורבת</span><span><i class="bad"></i>החלפה או השמטה</span><span><i class="mid"></i>תוספת</span></div>
 
 
 
 
 
 
 
 
 
979
  <section id="trackBox"><h3 class="heatmap-title">מיקום הפונמות בזמן</h3><div class="phone-track-wrap"><canvas id="phoneTrack" role="img" aria-label="רצף הפונמות הצפוי והמזוהה לאורך זמן"></canvas></div></section>
980
  <section id="heatmapBox"><h3 class="heatmap-title">הסתברות פונמות לאורך ההקלטה</h3><div class="heatmap-wrap"><canvas id="phoneHeatmap" role="img" aria-label="מפת חום של הסתברויות הפונמות לאורך ההקלטה"></canvas></div><div class="heatmap-scale"><span>0%</span><i></i><span>100%</span><span>· ★ פונמה צפויה</span></div></section>
981
  </dialog>
@@ -2170,7 +2229,8 @@ function alignPhones(expected,heard){
2170
  return out.reverse();
2171
  }
2172
  function drawPhones(r){
2173
- const expected=r.words.flatMap(w=>(w.phonemes||[]).map(p=>p.ipa)), heard=(r.heard_ipa||"").trim().split(/\s+/).filter(Boolean);
 
2174
  const aligned=alignPhones(expected,heard), rows=$("phoneRows"); rows.innerHTML=""; rows.style.setProperty("--phones",aligned.length);
2175
  const make=(name,which)=>{
2176
  const row=mark("phone-row","","div"); row.append(mark("phone-label",name));
@@ -2182,8 +2242,144 @@ function drawPhones(r){
2182
  rows.append(row);
2183
  };
2184
  make("צפוי",0); make("המודל שמע",1);
 
 
2185
  drawPhoneTrack(r.phoneme_heatmap);
2186
  drawHeatmap(r.phoneme_heatmap);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2187
  }
2188
  function heatColor(value){
2189
  const stops=[[245,247,245],[159,196,172],[40,116,93],[240,180,60],[159,47,39]], x=Math.max(0,Math.min(100,value))/25;
 
310
  .flag-facts{display:grid;grid-template-columns:auto 1fr;gap:6px 12px;margin:8px 0 0}
311
  .flag-facts dt{color:var(--muted);font-size:13px;white-space:nowrap}
312
  .flag-facts dd{margin:0;min-width:0;overflow-wrap:anywhere}
313
+ .phone-dialect{direction:rtl;text-align:right;margin-top:6px;font-size:13px;color:var(--muted)}
314
  .flag-facts .ipa{direction:ltr;text-align:right;font:13px/1.5 ui-monospace,monospace;unicode-bidi:isolate}
315
  .flag-facts ul{margin:0;padding:0;list-style:none;display:flex;flex-direction:column;gap:3px}
316
  .flag-facts b.good{color:var(--olive)} .flag-facts b.mid{color:#a86a00} .flag-facts b.bad{color:#9f2f27}
 
572
  .dlg-head h2{font:400 22px/1 var(--f-display);margin:0}
573
  .dlg-head h2 small{display:block;margin-top:4px;font:600 13px/1.3 var(--f-body);color:var(--muted)}
574
  .phone-note{margin:4px 0 12px;color:var(--muted);font-size:13px}
575
+ /* the phoneme analysis is a teacher's desktop tool: it takes the whole screen */
576
+ #phoneDlg{width:min(98vw,1880px);max-height:95vh}
577
+ .stage-flow{display:grid;grid-template-columns:repeat(10,minmax(0,1fr));gap:10px;margin-bottom:14px}
578
+ .stage-card{position:relative;display:flex;flex-direction:column;gap:6px;padding:10px 10px 9px;border:1px solid var(--line);border-radius:12px;background:var(--bg);font-size:12px;line-height:1.35;cursor:default}
579
+ .stage-card:not(:last-child)::after{content:"←";position:absolute;left:-10px;top:14px;color:var(--muted);font-size:13px}
580
+ .stage-card:hover,.stage-card.on{border-color:var(--olive);box-shadow:0 0 0 2px var(--olive-soft)}
581
+ .stage-card h4{margin:0;font:700 13px/1.25 var(--f-body)}
582
+ .stage-card h4 b{display:inline-grid;place-items:center;width:19px;height:19px;margin-inline-end:5px;border-radius:50%;background:var(--olive);color:#fff;font-size:11px}
583
+ .stage-card .what{color:var(--muted)}
584
+ .stage-card dl{display:grid;grid-template-columns:1fr auto;gap:2px 6px;margin:0}
585
+ .stage-card dt{color:var(--muted)}
586
+ .stage-card dd{margin:0;font-weight:700;font-variant-numeric:tabular-nums;direction:ltr;text-align:left}
587
+ .stage-card .formula{direction:ltr;text-align:left;overflow-wrap:anywhere;padding:4px 6px;border-radius:6px;background:var(--surface);font:12px/1.4 ui-monospace,monospace}
588
+ .stage-card .got{margin-top:auto;padding-top:6px;border-top:1px dashed var(--line);font-weight:700}
589
+ .stage-grid{display:grid;grid-template-columns:1fr 1fr;gap:12px}
590
+ .stage-panel{border:1px solid var(--line);border-radius:12px;padding:10px 12px;font-size:13px}
591
+ .stage-panel h4{margin:0 0 6px;font:700 13px/1.3 var(--f-body)}
592
+ .stage-panel table{width:100%;border-collapse:collapse}
593
+ .stage-panel td,.stage-panel th{padding:3px 6px;border-bottom:1px solid var(--line);text-align:start;font-variant-numeric:tabular-nums}
594
+ .stage-panel tr.chosen{background:var(--ok-bg)}
595
+ .stage-panel .eq{direction:ltr;text-align:left;font:13px/1.7 ui-monospace,monospace}
596
+ .ltr{direction:ltr;unicode-bidi:isolate;font-family:ui-monospace,monospace}
597
+ .stage-bar{position:relative;height:10px;margin:6px 0 2px;border-radius:999px;background:var(--line);direction:ltr}
598
+ .stage-bar i{position:absolute;inset:0 auto 0 0;border-radius:999px;background:var(--olive)}
599
+ .stage-bar b{position:absolute;top:-3px;bottom:-3px;width:2px;background:var(--sumac)}
600
+ .stage-table-wrap{overflow:auto;max-height:62vh;border:1px solid var(--line);border-radius:10px}
601
+ .stage-table{width:100%;border-collapse:separate;border-spacing:0;font-size:13px;font-variant-numeric:tabular-nums}
602
+ .stage-table th{position:sticky;top:0;z-index:1;background:var(--surface);padding:6px 7px;border-bottom:1px solid var(--line);font:700 12px/1.25 var(--f-body);text-align:center;vertical-align:bottom}
603
+ .stage-table th small{display:block;font-weight:500;color:var(--muted)}
604
+ .stage-table td{padding:5px 7px;border-bottom:1px solid var(--line);text-align:center;white-space:nowrap}
605
+ .stage-table tr.stage-word td{background:var(--olive-soft);font-weight:700;text-align:start}
606
+ .stage-table tr.stage-word td .eq{display:inline-block;font:12px ui-monospace,monospace;direction:ltr;unicode-bidi:isolate;margin-inline-start:28px;font-weight:500;color:var(--muted)}
607
+ .stage-table td.ok{background:var(--ok-bg);color:var(--ok)}
608
+ .stage-table td.mid{background:var(--amber-bg);color:var(--amber)}
609
+ .stage-table td.bad{background:var(--sumac-bg);color:var(--sumac)}
610
+ .stage-table td.chg{box-shadow:inset 0 0 0 2px var(--olive)}
611
+ .stage-table td.skip{color:var(--muted)}
612
+ .stage-table td small{display:block;font-size:11px;color:var(--muted);font-weight:500}
613
+ .stage-table .dec-flag{color:var(--sumac);font-weight:700}
614
+ .stage-table .dec-issue{color:var(--amber);font-weight:700}
615
+ .stage-table .dec-ok{color:var(--ok)}
616
+ #stagesBox[data-hl="gop"] .st-gop,#stagesBox[data-hl="exemption"] .st-exemption,#stagesBox[data-hl="baseline"] .st-baseline,
617
+ #stagesBox[data-hl="calibration"] .st-calibration,#stagesBox[data-hl="decision"] .st-decision,#stagesBox[data-hl="variant"] .st-variant,
618
+ #stagesBox[data-hl="tips"] .st-tips{outline:2px solid var(--olive);outline-offset:-2px}
619
+ #stagesBox[data-hl="word"] tr.stage-word td,#stagesBox[data-hl="sentence"] #stageSentence,#stagesBox[data-hl="unrelated"] #stageVariants{outline:2px solid var(--olive);outline-offset:-2px}
620
+ .stage-legend{display:flex;gap:8px 16px;flex-wrap:wrap;margin-top:6px;font-size:12px;color:var(--muted)}
621
+ .stage-legend i{display:inline-block;width:10px;height:10px;border-radius:3px;margin-inline-end:4px}
622
+ .stage-legend .ok{background:var(--ok-bg)}.stage-legend .mid{background:var(--amber-bg)}.stage-legend .bad{background:var(--sumac-bg)}
623
+ .stage-legend .chg{box-shadow:inset 0 0 0 2px var(--olive)}
624
  .phone-rows{direction:ltr;overflow-x:auto;padding:3px 0 8px}
625
  .phone-row{display:grid;grid-template-columns:72px repeat(var(--phones),minmax(34px,1fr));gap:4px;min-width:max-content;margin-bottom:5px;align-items:stretch}
626
  .phone-row>span{display:grid;place-items:center;min-height:34px;padding:3px 5px;border-radius:7px;background:var(--bg);font:600 16px/1.2 sans-serif}
 
1026
  <p class="phone-note">זהו פירוש המודל להקלטה, ולא תעתיק פונטי מוסמך. תנועות קרובות ואורך תנועה מסומנים כהתאמה מקורבת.</p>
1027
  <div class="phone-rows" id="phoneRows"></div>
1028
  <div class="phone-legend"><span><i class="good"></i>התאמה</span><span><i style="background:var(--olive-soft)"></i>התאמה מקורבת</span><span><i class="bad"></i>החלפה או השמטה</span><span><i class="mid"></i>תוספת</span></div>
1029
+ <section id="stagesBox" hidden>
1030
+ <h3 class="heatmap-title">שלבי הסיווג</h3>
1031
+ <p class="phone-note">כל הגה עובר את השלבים לפי הסדר. הכרטיסים מציגים את הספים והמשקלים של כל שלב ומה קרה בהקלטה הזו. מעבר עם העכבר על כרטיס מדגיש את העמודות שלו בטבלה.</p>
1032
+ <div class="stage-flow" id="stageFlow"></div>
1033
+ <div class="stage-grid"><div class="stage-panel" id="stageVariants"></div><div class="stage-panel" id="stageSentence"></div></div>
1034
+ <h3 class="heatmap-title">הגה אחר הגה</h3>
1035
+ <div class="stage-table-wrap"><table class="stage-table" id="stageTable"></table></div>
1036
+ <div class="stage-legend"><span><i class="ok"></i>מעל סף הרשימה</span><span><i class="mid"></i>בין סף הסימון לסף הרשימה</span><span><i class="bad"></i>מתחת לסף הסימון</span><span><i class="chg"></i>השלב שינה את הציון</span></div>
1037
+ </section>
1038
  <section id="trackBox"><h3 class="heatmap-title">מיקום הפונמות בזמן</h3><div class="phone-track-wrap"><canvas id="phoneTrack" role="img" aria-label="רצף הפונמות הצפוי והמזוהה לאורך זמן"></canvas></div></section>
1039
  <section id="heatmapBox"><h3 class="heatmap-title">הסתברות פונמות לאורך ההקלטה</h3><div class="heatmap-wrap"><canvas id="phoneHeatmap" role="img" aria-label="מפת חום של הסתברויות הפונמות לאורך ההקלטה"></canvas></div><div class="heatmap-scale"><span>0%</span><i></i><span>100%</span><span>· ★ פונמה צפויה</span></div></section>
1040
  </dialog>
 
2229
  return out.reverse();
2230
  }
2231
  function drawPhones(r){
2232
+ // a letter with several correct realisations (ق: ʔ / k / q) is compared with the one said, so an accepted one is not a mismatch
2233
+ const expected=r.words.flatMap(w=>(w.phonemes||[]).map(p=>p.realised&&!p.flagged?p.realised:p.ipa)), heard=(r.heard_ipa||"").trim().split(/\s+/).filter(Boolean);
2234
  const aligned=alignPhones(expected,heard), rows=$("phoneRows"); rows.innerHTML=""; rows.style.setProperty("--phones",aligned.length);
2235
  const make=(name,which)=>{
2236
  const row=mark("phone-row","","div"); row.append(mark("phone-label",name));
 
2242
  rows.append(row);
2243
  };
2244
  make("צפוי",0); make("המודל שמע",1);
2245
+ const dialect=r.mixed_dialect?"ניב: מעורב (עירוני וכפרי באותו משפט, לא הוריד ציון)":r.dialect==="urban"?"ניב: עירוני":r.dialect==="rural"?"ניב: כפרי":"";
2246
+ if(dialect) rows.append(mark("phone-dialect",dialect,"div"));
2247
  drawPhoneTrack(r.phoneme_heatmap);
2248
  drawHeatmap(r.phoneme_heatmap);
2249
+ drawStages(r);
2250
+ }
2251
+ // The classification stage by stage (r.trace, teachers only): the flow with each stage's thresholds and weights,
2252
+ // then every sound through the stages. Scores are 0..100; probabilities are the model's posteriors in the sound's window.
2253
+ const EXEMPTION_NAMES={glottal_dropped:"עצירה סותמית שהושמטה (ء / ق)",article_vowel_elided:"תנועת ה־ال אחרי תנועה",near_vowel:"תנועה קרובה"};
2254
+ function drawStages(r){
2255
+ const box=$("stagesBox"), t=r.trace;
2256
+ if(!t){ box.hidden=true; return; }
2257
+ box.hidden=false;
2258
+ const c=t.config, toks=t.tokens;
2259
+ const E=(tag,cls,...kids)=>{ const e=document.createElement(tag); if(cls) e.className=cls; for(const k of kids) if(k!=null&&k!=="") e.append(k); return e; };
2260
+ const num=(x,d=1)=>x==null?"—":String(Math.round(x*10**d)/10**d);
2261
+ const pct=x=>x==null?"—":x>0&&x<0.001?"<0.1%":num(100*x,1)+"%";
2262
+ const ltr=x=>E("span","ltr",String(x));
2263
+ const band=s=>s>=c.issue_below?"ok":s>=c.flag_below?"mid":"bad";
2264
+ const count=f=>toks.filter(f).length;
2265
+ const stageOf=(x,name)=>x.stages.find(s=>s.stage===name);
2266
+
2267
+ // the flow: one card per stage, in order
2268
+ const flow=$("stageFlow"); flow.innerHTML="";
2269
+ const card=(id,n,title,what,params,formula,got)=>{
2270
+ const el=E("div","stage-card",E("h4","",E("b","",String(n)),title),E("div","what",what));
2271
+ el.dataset.stage=id;
2272
+ if(params.length){ const dl=E("dl"); for(const [k,v] of params) dl.append(E("dt","",k),E("dd","",String(v))); el.append(dl); }
2273
+ if(formula) el.append(E("div","formula",formula));
2274
+ el.append(E("div","got",got));
2275
+ el.onmouseenter=()=>{ box.dataset.hl=id; };
2276
+ el.onmouseleave=()=>{ delete box.dataset.hl; };
2277
+ flow.append(el);
2278
+ };
2279
+ const chosen=t.variants.find(v=>v.chosen)||{};
2280
+ const u=t.unrelated, sen=t.sentence;
2281
+ const flaggedN=count(x=>x.flagged), issueN=count(x=>x.issue), underFlag=count(x=>x.final<c.flag_below&&!x.flagged);
2282
+ card("variant",1,"בחירת וריאנט","יישור כפוי (Viterbi) של כל וריאנט מקובל, כולל צורות דיפתונג; הסימנים המופשטים (Q, K, TH...) מאחדים מימושים.",
2283
+ [["וריאנטים שנבדקו",t.variants.length],["קריטריון","log P מסלול מרבי"]],null,
2284
+ `נבחר: log P ${num(chosen.logp,2)}`);
2285
+ card("unrelated",2,"משפט אחר?","מרחק עריכה בין הפענוח החופשי לווריאנט, יחסית למספר ההגאים.",
2286
+ [["סף יחסי",c.unrelated_ratio],["תקרת ציון",c.unrelated_cap]],`${u.distance} / ${u.tokens} = ${u.ratio}`,
2287
+ u.unrelated?"משפט אחר: הציון מוגבל, בלי טיפים":"אותו משפט");
2288
+ card("gop",3,"GOP לכל הגה","יחס בין ההגה הצפוי למתחרה החזק בחלון שלו, מוכפל בביטחון.",
2289
+ [["λ סלחנות",c.leniency],["p לביטחון מלא",c.confident_peak],["השמטה: p צפוי <",c.deleted_peak],["… וגם p מתחרה <",c.deleted_comp]],
2290
+ `100·e^(GOP·${c.leniency})·√min(1,p/${c.confident_peak})`,
2291
+ `${count(x=>stageOf(x,"gop").score<c.flag_below)} הגאים מתחת ל־${c.flag_below} · ${count(x=>x.deleted)} הושמטו`);
2292
+ card("exemption",4,"פטורים (6.3)","מה שהמודל שומע כטעות אבל הוא דיבור רגיל.",
2293
+ c.exemptions.map(e=>[EXEMPTION_NAMES[e]||e,e==="near_vowel"?"≥ "+c.near_vowel_floor:"100"]),null,
2294
+ `${count(x=>stageOf(x,"exemption").applied)} פטורים הוחלו`);
2295
+ card("baseline",5,"קו בסיס ילידי","ציון ההגה מחולק בציון שהמודל נותן לקול הילידי על אותו הגה.",
2296
+ [["הקלטות ילידיות",t.references.length],["ממוצע מינימלי לאמון",c.ref_min_mean]],"min(100, 100·score / ref)",
2297
+ `${count(x=>stageOf(x,"baseline").applied)} הגאים הותאמו`);
2298
+ card("calibration",6,"כיול מורה","רצפות שהמורה אישר לבלבול מסוים (צפוי → נשמע).",
2299
+ [["מפתח","סמל, ואז הגה כתוב"]],"max(score, floor)",
2300
+ `${count(x=>stageOf(x,"calibration").applied)} רצפות הוחלו`);
2301
+ card("decision",7,"החלטת סימון","עיצור מסומן אם הציון נמוך והמתחרה ניצח במרווח של הזוג.",
2302
+ [["סימון: ציון <",c.flag_below],["מרווח ברירת מחדל",c.margin_default],["זוגות עם מרווח",c.margins.length],["רשימת אי־דיוק: ציון <",c.issue_below]],
2303
+ "−GOP ≥ margin(target, heard)",
2304
+ `${flaggedN} מסומנים · ${issueN} ברשימה${underFlag?` · ${underFlag} נמוכים אך לא סומנו`:""}`);
2305
+ card("word",8,"ציון מילה","שילוב של ההגה החלש במילה וממוצע ההגאים שלה.",
2306
+ [["k משקל החלש",c.word_min_weight]],`k·min + (1−k)·mean`,
2307
+ t.words.map(w=>Math.round(w.score)).join(" · "));
2308
+ card("sentence",9,"ציון משפט","ממוצע הגאים משוקלל (תנועה במשקל נמוך) וממוצע המילים.",
2309
+ [["משקל תנועה",c.vowel_weight],["ממוצע הגאים",sen.phone_weight],["ממוצע מילים",sen.word_weight]],
2310
+ `${sen.phone_weight}·${sen.phone_mean} + ${sen.word_weight}·${sen.word_mean}`,
2311
+ `${sen.before_cap} → ${sen.final}${sen.capped?" (מוגבל)":""}`);
2312
+ card("tips",10,"טיפים","מהמסומנים: חומרה גבוהה קודם, ואז הציון הנמוך.",
2313
+ [["מקסימום",c.max_tips],["מקסימום ברשימה",c.max_issues]],null,
2314
+ t.tips.length?t.tips.map(x=>`${x.key} (${x.severity})`).join(" · "):"אין טיפים");
2315
+
2316
+ // variants and the "different sentence" check
2317
+ const vp=$("stageVariants"); vp.innerHTML="";
2318
+ vp.append(E("h4","","1–2 · הווריאנטים והבדיקה למשפט אחר"));
2319
+ const vt=E("table"); vt.append(E("tr","",E("th","","ערבית"),E("th","","IPA"),E("th","","log P")));
2320
+ for(const v of [...t.variants].sort((a,b)=>(b.logp??-1e9)-(a.logp??-1e9))){
2321
+ const row=E("tr",v.chosen?"chosen":"",E("td","",v.ar),E("td","",ltr(v.ipa)),E("td","",ltr(v.logp==null?"קצר מדי":num(v.logp,2))));
2322
+ vt.append(row);
2323
+ }
2324
+ vp.append(vt,E("div","",E("small","","עם סמלים: "),ltr(t.aipa)),E("div","",E("small","","הפענוח החופשי: "),ltr(u.heard||"—")));
2325
+ const bar=E("div","stage-bar"), fill=E("i"), mk=E("b");
2326
+ fill.style.width=Math.min(100,100*u.ratio)+"%"; mk.style.left=100*u.threshold+"%"; bar.append(fill,mk);
2327
+ vp.append(bar,E("small","",`מרחק יחסי ${u.ratio} · הסף ${u.threshold} (הקו האדום)`));
2328
+
2329
+ // word and sentence formulas with this attempt's numbers
2330
+ const sp=$("stageSentence"); sp.innerHTML="";
2331
+ sp.append(E("h4","","8–9 · מילים ומשפט"));
2332
+ const wt=E("table"); wt.append(E("tr","",E("th","","מילה"),E("th","","min"),E("th","","mean"),E("th","","k·min+(1−k)·mean")));
2333
+ for(const w of t.words) wt.append(E("tr","",E("td","",w.tr),E("td","",ltr(num(w.min))),E("td","",ltr(num(w.mean))),
2334
+ E("td","",ltr(`${c.word_min_weight}·${num(w.min)} + ${num(1-c.word_min_weight,2)}·${num(w.mean)} = ${num(w.score)}`))));
2335
+ const vowels=toks.filter(x=>x.vowel).length;
2336
+ sp.append(wt,E("div","eq",`phone_mean = Σ(w·score)/Σw (w=${c.vowel_weight} ×${vowels} vowels, 1 ×${toks.length-vowels} consonants) = ${sen.phone_mean}`),
2337
+ E("div","eq",`word_mean = ${sen.word_mean}`),
2338
+ E("div","eq",`score = ${sen.phone_weight}·${sen.phone_mean} + ${sen.word_weight}·${sen.word_mean} = ${sen.before_cap}${sen.capped?` → min(·, ${c.unrelated_cap})`:""} → ${sen.final}`));
2339
+
2340
+ // every sound through the stages
2341
+ const tb=$("stageTable"); tb.innerHTML="";
2342
+ const head=[["#",""],["הגה","כתוב · סמל · מחלקה"],["חלון","פריימים",""],["P צפוי","","gop"],["מתחרה","P","gop"],["GOP","log יחס","gop"],
2343
+ ["e^(GOP·λ)",`λ=${c.leniency}`,"gop"],["√ביטחון",`p/${c.confident_peak}`,"gop"],["ציון GOP","","gop"],["פטור","","exemption"],
2344
+ ["קו בסיס","ref","baseline"],["כיול","floor","calibration"],["סופי","",""],["מרווח","בפועל / נדרש","decision"],
2345
+ ["החלטה",`סימון <${c.flag_below} · רשימה <${c.issue_below}`,"decision"],["כלל","חומרה","tips"],["משקל","","sentence"]];
2346
+ const hr=E("tr"); for(const [a,b,st] of head){ const th=E("th",st?"st-"+st:"",a,b?E("small","",b):null); hr.append(th); } tb.append(E("thead","",hr));
2347
+ const body=E("tbody");
2348
+ t.words.forEach((w,wi)=>{
2349
+ body.append(E("tr","stage-word",(()=>{ const td=E("td","",`מילה ${wi+1}: ${w.tr} · ${num(w.score)}`,E("span","eq",`k·min + (1−k)·mean = ${c.word_min_weight}·${num(w.min)} + ${num(1-c.word_min_weight,2)}·${num(w.mean)} = ${num(w.score)}`)); td.colSpan=head.length; return td; })()));
2350
+ for(const x of toks.filter(x=>x.word===wi)){
2351
+ const gop=stageOf(x,"gop"), ex=stageOf(x,"exemption"), bl=stageOf(x,"baseline"), cal=stageOf(x,"calibration");
2352
+ const who=E("td","",ltr(x.ipa));
2353
+ if(x.sound!==x.ipa) who.append(E("small","",ltr(x.sound+(x.class?" {"+x.class.join(",")+"}":"")+(x.realised?" → "+x.realised:""))));
2354
+ const comp=E("td","st-gop",x.deleted?"הושמט":x.competitor_other?"צליל לא מוכר":x.competitor?ltr(x.competitor):"—",E("small","",pct(x.p_competitor)));
2355
+ const step=(st,stage,label)=>{ const td=E("td","st-"+stage+(st.applied?" chg "+band(st.score):" skip"),st.applied?num(st.score):"·");
2356
+ if(label) td.append(E("small","",label)); return td; };
2357
+ const m=x.margin, beyondTxt=m.beyond?"✓":"✗";
2358
+ let dec, decCls;
2359
+ if(x.flagged){ dec="מסומן"; decCls="dec-flag"; }
2360
+ else if(x.issue){ dec="ברשימה"; decCls="dec-issue"; }
2361
+ else if(x.final<c.flag_below&&x.vowel){ dec="תנועה: לא מסומנת"; decCls="dec-issue"; }
2362
+ else if(x.final<c.flag_below&&!m.beyond){ dec="מרווח לא הושג"; decCls="dec-issue"; }
2363
+ else if(x.final<c.issue_below){ dec="נמוך, לא ברשימה"; decCls="dec-issue"; }
2364
+ else { dec="תקין"; decCls="dec-ok"; }
2365
+ const decTd=E("td","st-decision",E("span",decCls,dec)); if(x.tip) decTd.append(E("small","","+ טיפ"));
2366
+ body.append(E("tr","",
2367
+ E("td","",String(x.i+1)), who,
2368
+ E("td","",ltr(`${x.window[0]}–${x.window[1]}`),E("small","",ltr(`span ${x.span[0]}–${x.span[1]}`))),
2369
+ E("td","st-gop",pct(x.p_target)), comp,
2370
+ E("td","st-gop",ltr(num(x.gop,3))), E("td","st-gop",ltr(num(x.gop_factor,3))), E("td","st-gop",ltr(num(x.confidence_factor,3))),
2371
+ E("td","st-gop "+band(gop.score),num(gop.score)),
2372
+ step(ex,"exemption",ex.applied?(EXEMPTION_NAMES[ex.applied]||ex.applied):""),
2373
+ step(bl,"baseline",bl.ref!=null?"ref "+num(bl.ref):""),
2374
+ step(cal,"calibration",cal.floor!=null?"floor "+num(cal.floor):""),
2375
+ E("td",band(x.final),num(x.final)),
2376
+ E("td","st-decision",ltr(`${num(m.got,2)} / ${num(m.needed,2)} ${beyondTxt}`)),
2377
+ decTd,
2378
+ E("td","st-tips",x.rule?x.rule.title:"—",x.rule?E("small","","חומרה "+x.rule.severity):null),
2379
+ E("td","st-sentence",String(x.weight))));
2380
+ }
2381
+ });
2382
+ tb.append(body);
2383
  }
2384
  function heatColor(value){
2385
  const stops=[[245,247,245],[159,196,172],[40,116,93],[240,180,60],[159,47,39]], x=Math.max(0,Math.min(100,value))/25;
tests/test_rules.py ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """The yaml blocks of pronunciation-rules.md, as the scorer reads them."""
2
+ import re
3
+
4
+ import yaml
5
+
6
+ from app.rules import RULES_PATH, Check, check_detectable, load, parse, undetectable
7
+
8
+
9
+ def test_every_block_parses_into_rules():
10
+ r = load()
11
+ assert {"Q", "K", "TH", "DH", "ZH", "J"} <= set(r.archiphonemes)
12
+ assert r.dialects == ["urban", "rural"] and len(r.checks) >= 30
13
+ assert r.check("ħ", "χ").severity == 3 # written χ, stored by its base() name x
14
+ assert r.margin(["ZH", "ðˤ"], "z") == 0.7 # ZH has no margin of its own: the written phoneme's applies
15
+
16
+
17
+ def test_every_heard_symbol_has_a_hebrew_label():
18
+ text = RULES_PATH.read_text(encoding="utf-8")
19
+ heard = {str(h) for block in re.findall(r"```yaml\n(.*?)```", text, re.S)
20
+ for doc in re.split(r"^---$", block, flags=re.M) if doc.strip()
21
+ for c in (yaml.safe_load(doc) or {}).get("checks", []) for h in c["heard"]}
22
+ assert heard - set(load().heard_labels) == set()
23
+
24
+
25
+ def test_correct_realisations_per_letter():
26
+ r = load()
27
+ assert r.members("Q") == ["ʔ", "k", "q"] # Bedouin g is off by default
28
+ assert r.members("TH") == ["t", "θ"] and "s" in r.members("TH", written="s") # s only in learned words
29
+ assert r.dialects_of("Q", "k") == {"rural"} and r.dialects_of("K", "k") == {"urban", "rural"}
30
+
31
+
32
+ def test_undetectable_rules_are_reported():
33
+ r = load()
34
+ everything = lambda p: True
35
+ assert undetectable(Check("x", "ħ", "ħ", 1, ""), r, everything) == "same_sound"
36
+ assert undetectable(Check("x", "Q", "k", 1, ""), r, everything) == "accepted"
37
+ assert undetectable(Check("x", "ħ", "ǀ", 1, ""), r, lambda p: p != "ǀ") == "heard_not_in_model"
38
+ report = check_detectable(r, everything)
39
+ assert report["pharyngeal_h"]["detectable"]
40
+ # vowel length is not modelled (base() drops ː): the long/short checks are listed as undetectable
41
+ assert {m["why"] for m in report["vowel_length"]["missing"]} == {"same_sound"}
42
+ bad = parse("```yaml\nid: bad\ntitle: bad\nchecks:\n - {target: aː, heard: [a], severity: 1, feedback: x}\n```\n")
43
+ assert not check_detectable(bad, everything)["bad"]["detectable"]
tests/test_scoring.py CHANGED
@@ -120,13 +120,172 @@ def test_h_instead_of_pharyngeal():
120
  assert r["tips"][0]["key"] == "ħ>h"
121
 
122
 
123
- def test_k_for_urban_qaf_picks_closest_variant_and_tips():
124
- # learner says "kaddeesh" for قديش: closest variant is the glottal-stop one, ʔ flagged as k
125
  logp = emissions(["k", "a", "d", "eː", "ʃ", "i", "s", "aː", "ʕ", "a"])
126
  r = scorer_for(logp).score(None, TIME)
127
- keys = [t["key"] for t in r["tips"]]
128
- assert r["variant"]["ipa"].startswith("ʔ")
129
- assert "ʔ>k" in keys
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
130
 
131
 
132
  def test_unrelated_sentence():
@@ -210,7 +369,8 @@ def test_tts_endpoint_serves_phrase_audio_and_503_without_key():
210
  assert r.headers["x-tts-voice-id"] == voices[1]["id"] and unquote(r.headers["x-tts-voice-name"]) == voices[1]["name"]
211
  assert c.get("/api/tts/0?voice=not-in-the-list").status_code == 400
212
  assert c.get("/api/tts/9999").status_code == 404
213
- assert c.get("/api/health").json()["tts"] is True
 
214
 
215
 
216
  def test_spelling_variants_of_a_sound_are_not_errors():
@@ -1546,3 +1706,18 @@ def test_a_tutor_flags_a_recording_scored_wrong(monkeypatch):
1546
  assert "create table public.tutors" in c.get("/api/class", headers=tok("t")).json()["setup"][-1]["sql"]
1547
  r = rank("t", email="s@b.c", tutor=True)
1548
  assert r.status_code == 503 and "create table public.tutors" in r.json()["detail"]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
120
  assert r["tips"][0]["key"] == "ħ>h"
121
 
122
 
123
+ def test_k_for_qaf_is_rural_and_correct():
124
+ # learner says "kaddeesh" for قديش: rural Palestinian, so ق is accepted as k; the answer stays the urban one
125
  logp = emissions(["k", "a", "d", "eː", "ʃ", "i", "s", "aː", "ʕ", "a"])
126
  r = scorer_for(logp).score(None, TIME)
127
+ q = r["words"][0]["phonemes"][0]
128
+ assert r["variant"]["ipa"].startswith("ʔ") and q["sound"] == "Q" and q["realised"] == "k"
129
+ assert q["score"] >= 90 and not q["flagged"] and not r["tips"] and r["score"] >= 90
130
+ assert r["words"][0]["heard_tr"].startswith("כּ") # written as said, not coloured as a mistake
131
+
132
+
133
+ # ---------- dialects and Hebrew-speaker rules (docs/scoring-dialects-plan.md) ----------
134
+
135
+ def custom_scorer(extra, phones, frames=3, gap=2, peak=0.9):
136
+ """A scorer over SYMS plus `extra`, hearing `phones` cleanly."""
137
+ syms = SYMS + [s for s in extra if s not in SYMS]
138
+ vocab = {s: i for i, s in enumerate(syms)}
139
+
140
+ def row(sym, p):
141
+ r = np.full(len(syms), (1 - p) / (len(syms) - 1))
142
+ r[vocab[sym]] = p
143
+ return r
144
+
145
+ rows = [row("<pad>", 0.95)] * gap
146
+ for ph in phones:
147
+ rows += [row(ph, peak)] * frames + [row("<pad>", 0.95)] * gap
148
+ return Scorer(lambda wav: np.log(np.array(rows)), Vocab(vocab))
149
+
150
+
151
+ COFFEE = {"variants": [("قهوة", "ʔ a h w e")]}
152
+
153
+
154
+ def test_q_for_qaf_is_accepted():
155
+ r = scorer_for(emissions(["q", "a", "d", "eː", "ʃ", "i", "s", "aː", "ʕ", "a"])).score(None, TIME)
156
+ assert r["words"][0]["phonemes"][0]["realised"] == "q" and not r["tips"] and r["score"] >= 90
157
+
158
+
159
+ def test_dropped_qaf_between_vowels_is_not_an_error():
160
+ r = scorer_for(emissions(["a", "h", "w", "e"])).score(None, COFFEE)
161
+ assert r["score"] >= 90 and not r["tips"] and not any(i["missing"] for i in r["issues"])
162
+
163
+
164
+ def test_hamza_is_not_a_qaf():
165
+ # اهلا is written with ء: kahla is not a dialect form, it is a mistake
166
+ phrase = {"variants": [("اهلا", "ʔ a h l a")]}
167
+ r = scorer_for(emissions(["k", "a", "h", "l", "a"])).score(None, phrase)
168
+ p = r["words"][0]["phonemes"][0]
169
+ assert "sound" not in p and p["flagged"] and p["heard"] == "k"
170
+ assert [t["key"] for t in r["tips"]] == ["ʔ>k"]
171
+
172
+
173
+ def test_reference_is_matched_by_letter_not_by_realisation():
174
+ ref = scorer_for(emissions(["q", "a", "h", "w", "e"])).reference(None, COFFEE) # the voice says ق as q
175
+ assert ref.sounds[0] == (0, "Q")
176
+ r = scorer_for(emissions(["ʔ", "a", "h", "w", "e"])).score(None, COFFEE, [ref])
177
+ assert r["score"] >= 95 and not r["tips"] and r["references"] == 1
178
+
179
+
180
+ def test_mixed_dialect_is_noted_but_not_marked_down():
181
+ phrase = {"variants": [("قديش قهوة", "ʔ a d d eː ʃ | ʔ a h w e")]}
182
+ mixed = scorer_for(emissions(["ʔ", "a", "d", "eː", "ʃ", "k", "a", "h", "w", "e"])).score(None, phrase)
183
+ assert mixed["mixed_dialect"] and mixed["score"] >= 90 and not mixed["tips"]
184
+ rural = scorer_for(emissions(["k", "a", "d", "eː", "ʃ", "k", "a", "h", "w", "e"])).score(None, phrase)
185
+ assert rural["dialect"] == "rural" and "mixed_dialect" not in rural
186
+
187
+
188
+ def test_rural_kaf_and_tha():
189
+ phrase = {"variants": [("كثير", "k t iː r")]}
190
+ r = custom_scorer(["tʃ", "θ", "r", "iː", "t"], ["tʃ", "θ", "iː", "r"]).score(None, phrase)
191
+ assert [p.get("realised") for p in r["words"][0]["phonemes"][:2]] == ["tʃ", "θ"]
192
+ assert r["score"] >= 90 and not r["tips"] and r["dialect"] == "rural"
193
+ r = custom_scorer(["tʃ", "θ", "r", "iː", "t"], ["tʃ", "t", "iː", "r"]).score(None, phrase)
194
+ assert r["mixed_dialect"] and r["score"] >= 90 # rural ك with urban ث
195
+
196
+
197
+ def test_final_ta_marbuta_vowel_may_be_a_e_or_i():
198
+ for vowel in ("a", "e", "i"):
199
+ r = scorer_for(emissions(["ʔ", "a", "h", "w", vowel])).score(None, COFFEE)
200
+ assert r["words"][0]["phonemes"][-1]["score"] >= 95, vowel
201
+
202
+
203
+ def test_diphthong_for_long_vowel():
204
+ # بيت said as bayt (rural / older speech) for beet
205
+ phrase = {"variants": [("بيت", "b eː t")]}
206
+ r = custom_scorer(["b", "t", "j"], ["b", "a", "j", "t"]).score(None, phrase)
207
+ assert r["score"] >= 90 and not r["tips"] and r["variant"]["ipa"] == "b a j t"
208
+
209
+
210
+ def test_archify_places_each_letter():
211
+ from app.archi import archify, unresolved
212
+ assert archify("قديش الساعة", "ʔ a d d eː ʃ | i s s aː ʕ a") == "Q:ʔ a d d eː ʃ | i s s aː ʕ E:a"
213
+ assert archify("انا من القدس", "ʔ a n a | m i n | i l q u d s") == "ʔ a n a | m i n | i l Q:q u d s"
214
+ assert archify("شكرا كثير", "ʃ u k r a n | k t iː r") == "ʃ u K:k r a n | K:k TH:t iː r"
215
+ assert archify("قهوة", "Q:k a h w e") == "Q:k a h w e" # written by hand: left alone
216
+ assert unresolved("بعثت", "b a ʕ a t t") == [("بعثت", "ث")] # ث and ت said as one geminate
217
+
218
+
219
+ def test_every_qaf_in_the_bank_is_placed():
220
+ from app.archi import unresolved
221
+ bad = {(w, ipa) for p in PHRASES for ar, ipa in p["variants"] for w, letter in unresolved(ar, ipa) if letter == "ق"}
222
+ assert not bad, f"mark these ق by hand in the phrase (Q:ʔ): {sorted(bad)[:10]}"
223
+
224
+
225
+ def test_emphatic_heard_plain_is_a_mistake_with_the_rules_feedback():
226
+ from app.rules import load
227
+ phrase = {"variants": [("ظهر", "ðˤ u h r")]}
228
+ r = custom_scorer(["ðˤ", "dˤ", "zˤ", "z", "u", "r"], ["z", "u", "h", "r"]).score(None, phrase)
229
+ assert [t["key"] for t in r["tips"]] == ["ZH>z"]
230
+ assert r["tips"][0]["text"] == load().check("ZH", "z").feedback and r["tips"][0]["severity"] == 2
231
+
232
+
233
+ def margin_scorer(competitor_ratio):
234
+ """طين where tˤ is weak (p 0.07) and plain t is `competitor_ratio` times likelier in the same frames."""
235
+ syms = SYMS + ["tˤ", "t"]
236
+ vocab = {s: i for i, s in enumerate(syms)}
237
+ V = len(syms)
238
+
239
+ def row(probs):
240
+ r = np.full(V, 1e-4)
241
+ for sym, p in probs.items():
242
+ r[vocab[sym]] = p
243
+ return r / r.sum()
244
+
245
+ rows = [row({"<pad>": 0.95})] * 2
246
+ rows += [row({"tˤ": 0.07, "t": 0.07 * competitor_ratio, "<pad>": 0.8})] * 3 + [row({"<pad>": 0.95})] * 2
247
+ for ph in ("i", "n"):
248
+ rows += [row({ph: 0.9})] * 3 + [row({"<pad>": 0.95})] * 2
249
+ return Scorer(lambda wav: np.log(np.array(rows)), Vocab(vocab))
250
+
251
+
252
+ def test_emphatic_needs_a_clear_win_before_it_is_flagged():
253
+ phrase = {"variants": [("طين", "tˤ i n")]}
254
+ under = margin_scorer(1.8).score(None, phrase) # plain t under twice as likely: the model's doubt
255
+ p = under["words"][0]["phonemes"][0]
256
+ assert p["score"] < 50 and not p["flagged"] and not under["tips"]
257
+ over = margin_scorer(2.5).score(None, phrase)
258
+ assert over["words"][0]["phonemes"][0]["flagged"] and [t["key"] for t in over["tips"]] == ["tˤ>t"]
259
+
260
+
261
+ @pytest.mark.parametrize("ar,ipa,extra,said,key,heard_letter", [
262
+ ("فنجان", "f i n dʒ aː n", ["f", "p", "dʒ"], ["p", "i", "n", "dʒ", "aː", "n"], "f>p", "פּ"),
263
+ ("وين", "w eː n", ["v"], ["v", "eː", "n"], "w>v", "ב"),
264
+ ("بس", "b a s", ["b", "v"], ["v", "a", "s"], "b>v", "ב"),
265
+ ("صباح", "sˤ a b aː ħ", ["sˤ", "ts", "b"], ["ts", "a", "b", "aː", "ħ"], "sˤ>ts", "טס"),
266
+ ("مرة", "m a r r a", ["r", "ʁ"], ["m", "a", "ʁ", "a"], "r>ʁ", None),
267
+ ])
268
+ def test_hebrew_speaker_substitutions(ar, ipa, extra, said, key, heard_letter):
269
+ from app.rules import load
270
+ r = custom_scorer(extra, said).score(None, {"variants": [(ar, ipa)]})
271
+ tip = next(t for t in r["tips"] if t["key"] == key)
272
+ expected, heard = key.split(">")
273
+ assert tip["text"] == load().check(expected, heard).feedback
274
+ if heard_letter: # a sound Arabic does not have is still written in the heard line
275
+ assert heard_letter in r["heard_tr"].replace("\u05b0", "").replace("\u05b7", "").replace("\u05b4", "")
276
+
277
+
278
+ def test_a_sound_outside_arabic_and_hebrew_is_not_named():
279
+ # the multilingual model offers a click for ħ: reported as unclear, not as "heard as ǀ"
280
+ r = custom_scorer(["ǀ"], ["w", "eː", "n", "i", "l", "ǀ", "a", "m", "aː", "m"]).score(None, TOILET)
281
+ h = next(p for p in r["words"][1]["phonemes"] if p["ipa"] == "ħ")
282
+ assert h["flagged"] and h["heard"] is None
283
+
284
+
285
+ def test_meaning_changing_mistakes_lead_the_tips():
286
+ # w said as v (severity 2) and ħ as Hebrew ח (severity 3): ħ leads even though v scored lower
287
+ r = custom_scorer(["v"], ["v", "eː", "n", "i", "l", "χ", "a", "m", "aː", "m"]).score(None, TOILET)
288
+ assert [t["key"] for t in r["tips"]][:2] == ["ħ>x", "w>v"]
289
 
290
 
291
  def test_unrelated_sentence():
 
369
  assert r.headers["x-tts-voice-id"] == voices[1]["id"] and unquote(r.headers["x-tts-voice-name"]) == voices[1]["name"]
370
  assert c.get("/api/tts/0?voice=not-in-the-list").status_code == 400
371
  assert c.get("/api/tts/9999").status_code == 404
372
+ health = c.get("/api/health").json()
373
+ assert health["tts"] is True and health["rules"]["pharyngeal_h"]["detectable"]
374
 
375
 
376
  def test_spelling_variants_of_a_sound_are_not_errors():
 
1706
  assert "create table public.tutors" in c.get("/api/class", headers=tok("t")).json()["setup"][-1]["sql"]
1707
  r = rank("t", email="s@b.c", tutor=True)
1708
  assert r.status_code == 503 and "create table public.tutors" in r.json()["detail"]
1709
+
1710
+
1711
+ def test_teacher_trace_follows_every_stage():
1712
+ # ħ said as χ, then forgiven by a teacher floor: each stage of the trace shows where the score came from
1713
+ logp = emissions(["w", "eː", "n", "i", "l", "χ", "a", "m", "aː", "m"])
1714
+ scorer = scorer_for(logp)
1715
+ assert "trace" not in scorer.score(None, TOILET) # students do not get it
1716
+ scorer.set_calibrations([{"expected": "ħ", "heard": "χ", "floor": 60}])
1717
+ t = scorer.score_logp(logp, TOILET, diagnostics=True)["trace"]
1718
+ assert t["config"]["flag_below"] == 50 and t["config"]["leniency"] == 0.5 and t["variants"][0]["chosen"]
1719
+ h = next(x for x in t["tokens"] if x["ipa"] == "ħ")
1720
+ assert [s["stage"] for s in h["stages"]] == ["gop", "exemption", "baseline", "calibration"]
1721
+ assert h["stages"][0]["score"] < 50 and h["stages"][3]["applied"] and h["final"] == 60
1722
+ assert h["competitor"] == "χ" and h["rule"]["severity"] == 3 and not h["flagged"]
1723
+ assert t["sentence"]["final"] == round(0.5 * t["sentence"]["phone_mean"] + 0.5 * t["sentence"]["word_mean"])