Add unit tests for pronunciation rules parsing and detection
Browse files- Implement tests to ensure all YAML blocks in pronunciation-rules.md are parsed correctly into rules.
- Verify that every heard symbol has a corresponding Hebrew label.
- Check correct realizations per letter and dialect associations.
- Report undetectable rules and validate their detection status.
- app/align.py +26 -9
- app/archi.py +181 -0
- app/main.py +5 -1
- app/phones.py +17 -0
- app/rules.py +237 -0
- app/scorer.py +329 -110
- app/tips.py +29 -13
- app/translit.py +8 -3
- docs/scoring-dialects-plan.md +172 -0
- requirements.txt +1 -0
- scripts/rescore.py +95 -0
- static/assets/arabic_verbs.json +451 -0
- static/index.html +197 -1
- tests/test_rules.py +43 -0
- tests/test_scoring.py +181 -6
app/align.py
CHANGED
|
@@ -71,8 +71,12 @@ class TokenScore:
|
|
| 71 |
score: float # 0..100
|
| 72 |
gop: float # log ratio target vs strongest competitor, <= 0
|
| 73 |
peak: float # max log-posterior of the target inside its window
|
| 74 |
-
competitor: int | None # strongest non-target, non-blank token in the window
|
| 75 |
deleted: bool # target barely present and nothing else either
|
|
|
|
|
|
|
|
|
|
|
|
|
| 76 |
|
| 77 |
|
| 78 |
@dataclass
|
|
@@ -86,14 +90,20 @@ class GopConfig:
|
|
| 86 |
|
| 87 |
|
| 88 |
def score_tokens(logp: np.ndarray, align: Alignment, tokens: list[int], blank: int,
|
| 89 |
-
cfg: GopConfig = GopConfig()
|
|
|
|
| 90 |
"""Per-token GOP.
|
| 91 |
|
| 92 |
The window of token i runs from the end of token i-1 to the start of token i+1, so it covers
|
| 93 |
the blank frames around the emission where a substituted phoneme typically peaks.
|
| 94 |
The neighbours' own labels are excluded from the competitor set (coarticulation is not an error).
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
"""
|
| 96 |
T = logp.shape[0]
|
|
|
|
| 97 |
out: list[TokenScore] = []
|
| 98 |
for i, tok in enumerate(tokens):
|
| 99 |
a = align.spans[i - 1][1] if i > 0 else 0
|
|
@@ -101,13 +111,16 @@ def score_tokens(logp: np.ndarray, align: Alignment, tokens: list[int], blank: i
|
|
| 101 |
a, b = min(a, align.spans[i][0]), max(b, align.spans[i][1])
|
| 102 |
seg = logp[a:b]
|
| 103 |
peak = float(seg[:, tok].max())
|
|
|
|
|
|
|
| 104 |
|
| 105 |
-
masked = seg.max(0).copy()
|
| 106 |
masked[blank] = NEG
|
| 107 |
-
|
| 108 |
-
for j in (i - 1, i + 1):
|
| 109 |
if 0 <= j < len(tokens):
|
| 110 |
-
|
|
|
|
|
|
|
| 111 |
comp = int(masked.argmax())
|
| 112 |
comp_peak = float(masked[comp])
|
| 113 |
|
|
@@ -115,8 +128,11 @@ def score_tokens(logp: np.ndarray, align: Alignment, tokens: list[int], blank: i
|
|
| 115 |
deleted = bool(peak < cfg.deleted_peak and comp_peak < cfg.deleted_comp)
|
| 116 |
confidence = min(1.0, float(np.exp(peak)) / cfg.confident_peak)
|
| 117 |
score = 100.0 * float(np.exp(gop * cfg.leniency)) * float(np.sqrt(confidence))
|
|
|
|
|
|
|
| 118 |
out.append(TokenScore(round(score, 1), round(gop, 3), round(peak, 3),
|
| 119 |
-
comp if
|
|
|
|
| 120 |
return out
|
| 121 |
|
| 122 |
|
|
@@ -131,10 +147,11 @@ def greedy_decode(logp: np.ndarray, blank: int) -> list[int]:
|
|
| 131 |
return out
|
| 132 |
|
| 133 |
|
| 134 |
-
def edit_distance(a: list, b: list) -> int:
|
|
|
|
| 135 |
d = list(range(len(b) + 1))
|
| 136 |
for i, x in enumerate(a, 1):
|
| 137 |
prev, d[0] = d[0], i
|
| 138 |
for j, y in enumerate(b, 1):
|
| 139 |
-
prev, d[j] = d[j], min(d[j] + 1, d[j - 1] + 1, prev + (x
|
| 140 |
return d[len(b)]
|
|
|
|
| 71 |
score: float # 0..100
|
| 72 |
gop: float # log ratio target vs strongest competitor, <= 0
|
| 73 |
peak: float # max log-posterior of the target inside its window
|
| 74 |
+
competitor: int | None # strongest non-target, non-blank token in the window (None: nothing, or a sound we do not name)
|
| 75 |
deleted: bool # target barely present and nothing else either
|
| 76 |
+
realised: int | None = None # a class target (several correct realisations): the one said
|
| 77 |
+
other: bool = False # the strongest competitor is a sound outside the named set
|
| 78 |
+
comp_peak: float = NEG # max log-posterior of the strongest competitor (named or not) in the window
|
| 79 |
+
window: tuple[int, int] = (0, 0) # [start, end) frames the target and its competitors were read from
|
| 80 |
|
| 81 |
|
| 82 |
@dataclass
|
|
|
|
| 90 |
|
| 91 |
|
| 92 |
def score_tokens(logp: np.ndarray, align: Alignment, tokens: list[int], blank: int,
|
| 93 |
+
cfg: GopConfig = GopConfig(), accept: list[set[int] | None] | None = None,
|
| 94 |
+
n_real: int | None = None, named: set[int] | None = None) -> list[TokenScore]:
|
| 95 |
"""Per-token GOP.
|
| 96 |
|
| 97 |
The window of token i runs from the end of token i-1 to the start of token i+1, so it covers
|
| 98 |
the blank frames around the emission where a substituted phoneme typically peaks.
|
| 99 |
The neighbours' own labels are excluded from the competitor set (coarticulation is not an error).
|
| 100 |
+
|
| 101 |
+
A class token (a virtual column pooling its correct realisations, `accept[i]` their ids) is scored on the
|
| 102 |
+
pooled column; its members are not competitors. Competitors come from the first `n_real` columns only, and
|
| 103 |
+
one outside `named` is reported as `other` rather than by name.
|
| 104 |
"""
|
| 105 |
T = logp.shape[0]
|
| 106 |
+
accept = accept or [None] * len(tokens)
|
| 107 |
out: list[TokenScore] = []
|
| 108 |
for i, tok in enumerate(tokens):
|
| 109 |
a = align.spans[i - 1][1] if i > 0 else 0
|
|
|
|
| 111 |
a, b = min(a, align.spans[i][0]), max(b, align.spans[i][1])
|
| 112 |
seg = logp[a:b]
|
| 113 |
peak = float(seg[:, tok].max())
|
| 114 |
+
members = accept[i] or set()
|
| 115 |
+
realised = max(members, key=lambda m: seg[:, m].max()) if members else None
|
| 116 |
|
| 117 |
+
masked = (seg[:, :n_real] if n_real else seg).max(0).copy()
|
| 118 |
masked[blank] = NEG
|
| 119 |
+
for j in (i - 1, i, i + 1):
|
|
|
|
| 120 |
if 0 <= j < len(tokens):
|
| 121 |
+
for x in {tokens[j], *(accept[j] or ())}:
|
| 122 |
+
if x < len(masked):
|
| 123 |
+
masked[x] = NEG
|
| 124 |
comp = int(masked.argmax())
|
| 125 |
comp_peak = float(masked[comp])
|
| 126 |
|
|
|
|
| 128 |
deleted = bool(peak < cfg.deleted_peak and comp_peak < cfg.deleted_comp)
|
| 129 |
confidence = min(1.0, float(np.exp(peak)) / cfg.confident_peak)
|
| 130 |
score = 100.0 * float(np.exp(gop * cfg.leniency)) * float(np.sqrt(confidence))
|
| 131 |
+
present = comp_peak > NEG / 2
|
| 132 |
+
other = bool(present and named is not None and comp not in named)
|
| 133 |
out.append(TokenScore(round(score, 1), round(gop, 3), round(peak, 3),
|
| 134 |
+
comp if present and not other else None, deleted, realised, other,
|
| 135 |
+
round(comp_peak, 3), (int(a), int(b))))
|
| 136 |
return out
|
| 137 |
|
| 138 |
|
|
|
|
| 147 |
return out
|
| 148 |
|
| 149 |
|
| 150 |
+
def edit_distance(a: list, b: list, same=None) -> int:
|
| 151 |
+
same = same or (lambda x, y: x == y)
|
| 152 |
d = list(range(len(b) + 1))
|
| 153 |
for i, x in enumerate(a, 1):
|
| 154 |
prev, d[0] = d[0], i
|
| 155 |
for j, y in enumerate(b, 1):
|
| 156 |
+
prev, d[j] = d[j], min(d[j] + 1, d[j - 1] + 1, prev + (not same(x, y)))
|
| 157 |
return d[len(b)]
|
app/archi.py
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Abstract symbols for letters with more than one correct pronunciation (section 5 of pronunciation-rules.md).
|
| 2 |
+
|
| 3 |
+
phrases.py writes the realisation its author chose (ق as ʔ in قديش, q in القدس). The scorer must accept
|
| 4 |
+
the others too (kaddeesh is rural Palestinian), so before tokenising, archify() rewrites each phoneme that
|
| 5 |
+
comes from such a letter as "SYMBOL:written" (Q:ʔ), and tokenize() scores it against every correct
|
| 6 |
+
realisation at once. The written phoneme is kept for display, tips and calibration keys.
|
| 7 |
+
|
| 8 |
+
The letter behind a phoneme is found by aligning the Arabic letters of the word with its IPA consonants.
|
| 9 |
+
A phrase can also write a symbol itself: "Q", "Q:k", or with its own set of realisations, "TH{t,s}".
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import re
|
| 14 |
+
from functools import lru_cache
|
| 15 |
+
|
| 16 |
+
from .phones import base
|
| 17 |
+
|
| 18 |
+
# letter -> the symbol its phoneme is rewritten to
|
| 19 |
+
LETTER_SYMBOL = {"ق": "Q", "ك": "K", "ث": "TH", "ذ": "DH", "ظ": "ZH", "ج": "J"}
|
| 20 |
+
|
| 21 |
+
# letter -> base() names of the phonemes it is said as in the phrase bank (dialects, MSA, careless spelling)
|
| 22 |
+
_LETTERS = {
|
| 23 |
+
"ب": {"b"}, "ت": {"t"}, "ث": {"t", "s", "θ"}, "ج": {"dʒ", "g"}, "ح": {"ħ"}, "خ": {"x"}, "د": {"d"},
|
| 24 |
+
"ذ": {"d", "z", "ð"}, "ر": {"r"}, "ز": {"z"}, "س": {"s"}, "ش": {"ʃ"}, "ص": {"sˤ", "s"}, "ض": {"dˤ", "d"},
|
| 25 |
+
"ط": {"tˤ", "t"}, "ظ": {"ðˤ", "dˤ", "zˤ", "z", "ð"}, "ع": {"ʕ"}, "غ": {"ɣ"}, "ف": {"f"}, "ق": {"ʔ", "q", "k", "g"},
|
| 26 |
+
"ك": {"k", "tʃ"}, "ل": {"l"}, "م": {"m"}, "ن": {"n"}, "ه": {"h"}, "و": {"w"}, "ي": {"j"}, "ى": set(),
|
| 27 |
+
"ء": {"ʔ"}, "أ": {"ʔ"}, "إ": {"ʔ"}, "آ": {"ʔ"}, "ؤ": {"ʔ"}, "ئ": {"ʔ"}, "ا": {"ʔ"}, "ة": {"t"},
|
| 28 |
+
}
|
| 29 |
+
# letters that are often not said as a consonant at all (vowel letters, hamza seats, the article's ل): cheap to skip
|
| 30 |
+
_WEAK = set("اوىيةهءأإآؤئل")
|
| 31 |
+
_VOWELS = {"a", "i", "u", "e", "o", "ə"}
|
| 32 |
+
|
| 33 |
+
_TOKEN = re.compile(r"^([A-Z]+)(?:\{([^}]*)\})?(?::(.+))?$")
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def split_token(tok: str) -> tuple[str | None, list[str] | None, str]:
|
| 37 |
+
""""Q:ʔ" -> ("Q", None, "ʔ"); "TH{t,s}" -> ("TH", ["t", "s"], "t"); "ʕ" -> (None, None, "ʕ").
|
| 38 |
+
|
| 39 |
+
The phoneme of a symbol written without one is its first listed realisation, or "" (the caller picks)."""
|
| 40 |
+
m = _TOKEN.match(tok)
|
| 41 |
+
if not m:
|
| 42 |
+
return None, None, tok
|
| 43 |
+
sym, members, written = m.group(1), m.group(2), m.group(3)
|
| 44 |
+
members = [p.strip() for p in members.split(",") if p.strip()] if members else None
|
| 45 |
+
return sym, members, written or (members[0] if members else "")
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def written(tok: str) -> str:
|
| 49 |
+
"""The phoneme a token is written as in the phrase (Q:ʔ -> ʔ)."""
|
| 50 |
+
return split_token(tok)[2] or tok
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _letters(ar_word: str) -> list[str]:
|
| 54 |
+
return [c for c in ar_word if c in _LETTERS]
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def _units(phs: list[str]) -> list[tuple[str, list[int]]]:
|
| 58 |
+
"""The consonants of an IPA word, a geminate run as one unit: (base name, its positions)."""
|
| 59 |
+
out: list[tuple[str, list[int]]] = []
|
| 60 |
+
for i, ph in enumerate(phs):
|
| 61 |
+
b = base(ph)
|
| 62 |
+
if b in _VOWELS:
|
| 63 |
+
continue
|
| 64 |
+
if out and out[-1][0] == b and out[-1][1][-1] == i - 1:
|
| 65 |
+
out[-1][1].append(i)
|
| 66 |
+
else:
|
| 67 |
+
out.append((b, [i]))
|
| 68 |
+
return out
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
_SKIP_LETTER, _SKIP_WEAK, _SKIP_UNIT, _MISMATCH = 1.0, 0.3, 1.0, 1.6
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def _align(letters: list[str], units: list[str]) -> tuple[dict[int, set[int]], float]:
|
| 75 |
+
"""For each letter, the units it matches on some cheapest alignment of letters to consonants.
|
| 76 |
+
|
| 77 |
+
A letter matched to different units on two cheapest alignments (or matched on one and skipped on
|
| 78 |
+
another) is ambiguous; the caller leaves it alone."""
|
| 79 |
+
n, m = len(letters), len(units)
|
| 80 |
+
inf = float("inf")
|
| 81 |
+
|
| 82 |
+
def cost(i, j):
|
| 83 |
+
return 0.0 if units[j] in _LETTERS[letters[i]] else _MISMATCH
|
| 84 |
+
|
| 85 |
+
def skip(i):
|
| 86 |
+
return _SKIP_WEAK if letters[i] in _WEAK else _SKIP_LETTER
|
| 87 |
+
|
| 88 |
+
f = [[inf] * (m + 1) for _ in range(n + 1)]
|
| 89 |
+
f[0][0] = 0.0
|
| 90 |
+
for i in range(n + 1):
|
| 91 |
+
for j in range(m + 1):
|
| 92 |
+
if i < n:
|
| 93 |
+
f[i + 1][j] = min(f[i + 1][j], f[i][j] + skip(i))
|
| 94 |
+
if j < m:
|
| 95 |
+
f[i][j + 1] = min(f[i][j + 1], f[i][j] + _SKIP_UNIT)
|
| 96 |
+
if i < n and j < m:
|
| 97 |
+
f[i + 1][j + 1] = min(f[i + 1][j + 1], f[i][j] + cost(i, j))
|
| 98 |
+
b = [[inf] * (m + 1) for _ in range(n + 1)]
|
| 99 |
+
b[n][m] = 0.0
|
| 100 |
+
for i in range(n, -1, -1):
|
| 101 |
+
for j in range(m, -1, -1):
|
| 102 |
+
if i < n:
|
| 103 |
+
b[i][j] = min(b[i][j], b[i + 1][j] + skip(i))
|
| 104 |
+
if j < m:
|
| 105 |
+
b[i][j] = min(b[i][j], b[i][j + 1] + _SKIP_UNIT)
|
| 106 |
+
if i < n and j < m:
|
| 107 |
+
b[i][j] = min(b[i][j], b[i + 1][j + 1] + cost(i, j))
|
| 108 |
+
best, eps = f[n][m], 1e-9
|
| 109 |
+
matched: dict[int, set[int]] = {}
|
| 110 |
+
for i in range(n):
|
| 111 |
+
for j in range(m):
|
| 112 |
+
if cost(i, j) == 0.0 and abs(f[i][j] + b[i + 1][j + 1] - best) < eps:
|
| 113 |
+
matched.setdefault(i, set()).add(j)
|
| 114 |
+
# skipping letter i on a cheapest alignment
|
| 115 |
+
if any(abs(f[i][j] + skip(i) + b[i + 1][j] - best) < eps for j in range(m + 1)):
|
| 116 |
+
matched.setdefault(i, set()).add(-1)
|
| 117 |
+
return matched, best
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def archify_word(ipa_word: str, ar_word: str) -> tuple[str, list[str]]:
|
| 121 |
+
"""One word: its IPA with symbol tokens, and the letters that could not be placed (ambiguous)."""
|
| 122 |
+
phs = ipa_word.split()
|
| 123 |
+
if any(split_token(p)[0] for p in phs):
|
| 124 |
+
return ipa_word, [] # the phrase writes its symbols itself
|
| 125 |
+
letters, units = _letters(ar_word), _units(phs)
|
| 126 |
+
matched, _ = _align(letters, [u for u, _ in units])
|
| 127 |
+
out, unresolved = list(phs), []
|
| 128 |
+
for i, letter in enumerate(letters):
|
| 129 |
+
sym = LETTER_SYMBOL.get(letter)
|
| 130 |
+
if sym is None:
|
| 131 |
+
continue
|
| 132 |
+
hits = matched.get(i, {-1})
|
| 133 |
+
if len(hits) != 1:
|
| 134 |
+
unresolved.append(letter)
|
| 135 |
+
continue
|
| 136 |
+
j = next(iter(hits))
|
| 137 |
+
if j < 0:
|
| 138 |
+
continue # not said at all (a dropped ق): nothing to rewrite
|
| 139 |
+
for k in units[j][1]:
|
| 140 |
+
out[k] = f"{sym}:{phs[k]}"
|
| 141 |
+
# the vowel of a final ة (and ه written for it): a, e and i are all correct (section 6.3 #5)
|
| 142 |
+
if (out and ar_word.endswith(("ة", "ه")) and "لله" not in ar_word
|
| 143 |
+
and base(out[-1]) in {"a", "e", "i"} and len(out) > 1):
|
| 144 |
+
out[-1] = f"E:{out[-1]}"
|
| 145 |
+
return " ".join(out), unresolved
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
@lru_cache(maxsize=32768) # the whole phrase bank (~12k variants) and the verb game's forms
|
| 149 |
+
def archify(ar: str, ipa: str) -> str:
|
| 150 |
+
"""The variant's IPA with each phoneme of a many-ways letter rewritten as its symbol (see the module doc).
|
| 151 |
+
|
| 152 |
+
A variant whose Arabic and IPA word counts differ is returned as it is."""
|
| 153 |
+
ipa_words, ar_words = ipa.split("|"), ar.split()
|
| 154 |
+
if len(ipa_words) != len(ar_words):
|
| 155 |
+
return ipa
|
| 156 |
+
return " | ".join(archify_word(w.strip(), a)[0] for w, a in zip(ipa_words, ar_words))
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def unresolved(ar: str, ipa: str) -> list[tuple[str, str]]:
|
| 160 |
+
"""(Arabic word, letter) for every many-ways letter archify() could not place: the phrase must mark it."""
|
| 161 |
+
ipa_words, ar_words = ipa.split("|"), ar.split()
|
| 162 |
+
if len(ipa_words) != len(ar_words):
|
| 163 |
+
return []
|
| 164 |
+
return [(a, letter) for w, a in zip(ipa_words, ar_words) for letter in archify_word(w.strip(), a)[1]]
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def diphthong_variants(ar: str, ipa: str) -> list[str]:
|
| 168 |
+
"""The same variant with eː said as a j and oː as a w where the Arabic writes ي / و (section 6.3 #8).
|
| 169 |
+
|
| 170 |
+
A diphthong is two tokens against one, so it is a variant of its own rather than a class."""
|
| 171 |
+
ipa_words, ar_words = ipa.split("|"), ar.split()
|
| 172 |
+
if len(ipa_words) != len(ar_words):
|
| 173 |
+
return []
|
| 174 |
+
out = []
|
| 175 |
+
for long, glide, letter in (("eː", "j", "ي"), ("oː", "w", "و")):
|
| 176 |
+
words = [" ".join(f"a {glide}" if ph == long and letter in a else ph for ph in w.split())
|
| 177 |
+
for w, a in zip(ipa_words, ar_words)]
|
| 178 |
+
new = " | ".join(words)
|
| 179 |
+
if new != " | ".join(w.strip() for w in ipa_words):
|
| 180 |
+
out.append(new)
|
| 181 |
+
return out
|
app/main.py
CHANGED
|
@@ -32,6 +32,7 @@ from .auth import AuthUnavailable, ClassConflict, SupabaseAuth, load_auth
|
|
| 32 |
from .phrases import CATEGORIES, GAMES, LESSONS, LEVELS, PHRASES, public_phrases
|
| 33 |
from .prompt_tts import ENGINES, PromptTTSUnavailable, hebrew_mp3
|
| 34 |
from .reference import References
|
|
|
|
| 35 |
from .scorer import Scorer, decode_audio, load_scorer
|
| 36 |
from .speech_evaluation import SpeechEvaluationService
|
| 37 |
from .texts import read_line, text_phrases
|
|
@@ -1169,7 +1170,10 @@ def create_app(scorer: Scorer | None = None, tts: SawtakTTS | None = None,
|
|
| 1169 |
s: Scorer = app.state.scorer
|
| 1170 |
return {"version": build_version(), "build": build_metadata(), "model": MODEL_ID, "tts": app.state.tts is not None,
|
| 1171 |
"references": app.state.refs is not None, "sign_in": app.state.auth is not None,
|
| 1172 |
-
"phoneme_fallbacks": s.vocab.downgraded
|
|
|
|
|
|
|
|
|
|
| 1173 |
|
| 1174 |
@app.get("/api/tts/{phrase_id}", dependencies=members)
|
| 1175 |
def tts_phrase(phrase_id: int, voice: str = Query("", max_length=64)):
|
|
|
|
| 32 |
from .phrases import CATEGORIES, GAMES, LESSONS, LEVELS, PHRASES, public_phrases
|
| 33 |
from .prompt_tts import ENGINES, PromptTTSUnavailable, hebrew_mp3
|
| 34 |
from .reference import References
|
| 35 |
+
from .rules import check_detectable
|
| 36 |
from .scorer import Scorer, decode_audio, load_scorer
|
| 37 |
from .speech_evaluation import SpeechEvaluationService
|
| 38 |
from .texts import read_line, text_phrases
|
|
|
|
| 1170 |
s: Scorer = app.state.scorer
|
| 1171 |
return {"version": build_version(), "build": build_metadata(), "model": MODEL_ID, "tts": app.state.tts is not None,
|
| 1172 |
"references": app.state.refs is not None, "sign_in": app.state.auth is not None,
|
| 1173 |
+
"phoneme_fallbacks": s.vocab.downgraded,
|
| 1174 |
+
# per rule of pronunciation-rules.md: can this model catch it at all (and if not, why)
|
| 1175 |
+
"rules": check_detectable(s.rules, lambda ph: s.vocab.sound_id(ph) is not None, s.vocab.model_id),
|
| 1176 |
+
"dialect_realisations_missing": {k: sorted(v) for k, v in s.vocab.class_missing.items()}}
|
| 1177 |
|
| 1178 |
@app.get("/api/tts/{phrase_id}", dependencies=members)
|
| 1179 |
def tts_phrase(phrase_id: int, voice: str = Query("", max_length=64)):
|
app/phones.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The sound a model symbol stands for.
|
| 2 |
+
|
| 3 |
+
The model's vocab is multilingual (392 symbols) and spells one sound several ways: t / t̪ / tʰ / tː,
|
| 4 |
+
r / ɾ, a / æ / ɐ, x / χ. Left alone, a correctly said t loses to its own spelling variant t̪.
|
| 5 |
+
base() names the sound a symbol stands for; Vocab.merge() (scorer.py) pools the posteriors of each sound,
|
| 6 |
+
and rules.py writes its rules in the same names.
|
| 7 |
+
"""
|
| 8 |
+
_STRIP = set("ːʰʲ.^[:\"̪̩̝̞̊̃") # length, aspiration, palatalisation, dental, syllabic, nasal ...
|
| 9 |
+
_SAME = {"æ": "a", "ɐ": "a", "ɑ": "a", "ä": "a", "ʌ": "a", "ɪ": "i", "ʊ": "u", "ɛ": "e", "ɔ": "o",
|
| 10 |
+
"ɾ": "r", "ɫ": "l", "ɡ": "g", "χ": "x", "ʁ": "ɣ", "ʒ": "dʒ", "dZ": "dʒ", "tS": "tʃ", "S": "ʃ",
|
| 11 |
+
"d͡ʒ": "dʒ", "t͡ʃ": "tʃ", "t͡s": "ts"}
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def base(symbol: str) -> str:
|
| 15 |
+
"""The sound a model symbol stands for, ignoring length and spelling variants. Emphatics stay distinct."""
|
| 16 |
+
s = "".join(c for c in symbol if c not in _STRIP)
|
| 17 |
+
return _SAME.get(s, s) or symbol
|
app/rules.py
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Pronunciation rules for Hebrew-speaking learners, read from the yaml blocks of pronunciation-rules.md.
|
| 2 |
+
|
| 3 |
+
The document is written for two audiences: its student sections are served as a guide, and its scorer
|
| 4 |
+
sections (the ```yaml blocks) say what the scorer accepts and how it judges errors:
|
| 5 |
+
|
| 6 |
+
archiphonemes letters said differently in the urban and the rural dialect, and their correct realisations
|
| 7 |
+
checks per rule: (target, heard) -> severity and the Hebrew feedback
|
| 8 |
+
scoring margins per (target, heard) pair, max_tips, ...
|
| 9 |
+
heard_labels the Hebrew name of every heard symbol
|
| 10 |
+
|
| 11 |
+
Every symbol is stored by its base() name (phones.py), the names the scorer compares in, so a rule written
|
| 12 |
+
with ʁ matches a model that heard ɣ. check_detectable() tells which rules a model can catch at all.
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import re
|
| 17 |
+
from dataclasses import dataclass, field
|
| 18 |
+
from functools import lru_cache
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
from typing import Callable
|
| 21 |
+
|
| 22 |
+
import yaml
|
| 23 |
+
|
| 24 |
+
from .phones import base
|
| 25 |
+
|
| 26 |
+
RULES_PATH = Path(__file__).resolve().parent.parent / "static" / "assets" / "pronunciation-rules.md"
|
| 27 |
+
|
| 28 |
+
# Realisations correct only in learned words (ثقافة as saqaafe): accepted where the phrase writes them, not elsewhere.
|
| 29 |
+
LEARNED_ONLY = {"TH": {"s"}, "DH": {"z"}}
|
| 30 |
+
# Classes set by position, not by letter (section 6.3 #5 and #6): the vowel of a final ة, the vowel of a pronoun ending.
|
| 31 |
+
POSITIONAL = {"E": ["a", "e", "i"], "O": ["o", "u"]}
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def norm(symbol: str) -> str:
|
| 35 |
+
"""A rule symbol in the scorer's names. "_" (nothing heard) and the abstract symbols stay as they are."""
|
| 36 |
+
return symbol if symbol == "_" or symbol.isupper() else base(symbol)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
@dataclass(frozen=True)
|
| 40 |
+
class Check:
|
| 41 |
+
rule: str
|
| 42 |
+
target: str
|
| 43 |
+
heard: str
|
| 44 |
+
severity: int
|
| 45 |
+
feedback: str
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
@dataclass
|
| 49 |
+
class Rules:
|
| 50 |
+
archiphonemes: dict[str, dict[str, list[str]]]
|
| 51 |
+
dialects: list[str]
|
| 52 |
+
optional: bool
|
| 53 |
+
scoring: dict
|
| 54 |
+
checks: dict[tuple[str, str], Check]
|
| 55 |
+
titles: dict[str, str] # rule id -> title
|
| 56 |
+
heard_labels: dict[str, str]
|
| 57 |
+
margins: dict[tuple[str, str], float] = field(default_factory=dict)
|
| 58 |
+
|
| 59 |
+
def members(self, symbol: str, written: str | None = None) -> list[str]:
|
| 60 |
+
"""The correct realisations of an abstract symbol, in base() names, first the dialects' own.
|
| 61 |
+
|
| 62 |
+
A learned-only realisation (TH as s) counts only when the phrase writes it (`written`)."""
|
| 63 |
+
if symbol in POSITIONAL:
|
| 64 |
+
return list(POSITIONAL[symbol])
|
| 65 |
+
spec = self.archiphonemes.get(symbol)
|
| 66 |
+
if spec is None:
|
| 67 |
+
return []
|
| 68 |
+
out: list[str] = []
|
| 69 |
+
keys = [*self.dialects, "also_ok", *(["optional"] if self.optional else [])]
|
| 70 |
+
for key in keys:
|
| 71 |
+
for ph in spec.get(key, []):
|
| 72 |
+
ph = base(ph)
|
| 73 |
+
if ph not in out and (ph not in LEARNED_ONLY.get(symbol, ()) or ph == written):
|
| 74 |
+
out.append(ph)
|
| 75 |
+
return out
|
| 76 |
+
|
| 77 |
+
def dialects_of(self, symbol: str, phone: str) -> set[str]:
|
| 78 |
+
"""The dialects in which `phone` is a correct realisation of `symbol` (empty: in none, or not dialectal)."""
|
| 79 |
+
spec = self.archiphonemes.get(symbol, {})
|
| 80 |
+
return {d for d in ("urban", "rural") if phone in {base(p) for p in spec.get(d, [])}}
|
| 81 |
+
|
| 82 |
+
def check(self, targets: str | list[str], heard: str | None) -> Check | None:
|
| 83 |
+
"""The rule for hearing `heard` instead of the target: the first of `targets` that has one."""
|
| 84 |
+
if not heard:
|
| 85 |
+
return None
|
| 86 |
+
for t in [targets] if isinstance(targets, str) else targets:
|
| 87 |
+
c = self.checks.get((norm(t), norm(heard)))
|
| 88 |
+
if c is not None:
|
| 89 |
+
return c
|
| 90 |
+
return None
|
| 91 |
+
|
| 92 |
+
def margin(self, targets: str | list[str], heard: str | None) -> float:
|
| 93 |
+
"""How much likelier than the target `heard` must be before it is a mistake (natural log). The first
|
| 94 |
+
of `targets` (the abstract symbol, then the written phoneme) with a margin of its own decides."""
|
| 95 |
+
default = float(self.scoring.get("margin_default", 0.0))
|
| 96 |
+
for t in [targets] if isinstance(targets, str) else targets:
|
| 97 |
+
m = self.margins.get((norm(t), norm(heard or "_")))
|
| 98 |
+
if m is not None:
|
| 99 |
+
return m
|
| 100 |
+
return default
|
| 101 |
+
|
| 102 |
+
def label(self, heard: str | None) -> str | None:
|
| 103 |
+
"""The Hebrew name of a heard symbol, as section 8 gives it."""
|
| 104 |
+
if heard is None:
|
| 105 |
+
return None
|
| 106 |
+
return self.heard_labels.get(heard) or self.heard_labels.get(norm(heard))
|
| 107 |
+
|
| 108 |
+
def interference(self) -> set[str]:
|
| 109 |
+
"""Every symbol some rule expects to hear instead of the target: the Hebrew sounds worth naming."""
|
| 110 |
+
return {h for (_, h) in self.checks if h != "_"}
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def _blocks(text: str) -> list[dict]:
|
| 114 |
+
out = []
|
| 115 |
+
for block in re.findall(r"```yaml\n(.*?)```", text, re.S):
|
| 116 |
+
for doc in re.split(r"^---$", block, flags=re.M):
|
| 117 |
+
if doc.strip():
|
| 118 |
+
out.append(yaml.safe_load(doc))
|
| 119 |
+
return out
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def parse(text: str) -> Rules:
|
| 123 |
+
archi, dialects, optional, scoring, labels = {}, ["urban", "rural"], False, {}, {}
|
| 124 |
+
checks: dict[tuple[str, str], Check] = {}
|
| 125 |
+
titles: dict[str, str] = {}
|
| 126 |
+
for doc in _blocks(text):
|
| 127 |
+
if "archiphonemes" in doc:
|
| 128 |
+
archi = {k: {d: list(v.get(d, [])) for d in ("urban", "rural", "also_ok", "optional")}
|
| 129 |
+
for k, v in doc["archiphonemes"].items()}
|
| 130 |
+
dialects = list(doc.get("dialects_enabled", dialects))
|
| 131 |
+
optional = bool(doc.get("optional_realisations_enabled", False))
|
| 132 |
+
if "scoring" in doc:
|
| 133 |
+
scoring = dict(doc["scoring"])
|
| 134 |
+
if "heard_labels" in doc:
|
| 135 |
+
labels = {str(k): str(v) for k, v in doc["heard_labels"].items()}
|
| 136 |
+
if "checks" in doc:
|
| 137 |
+
titles[doc["id"]] = doc.get("title", doc["id"])
|
| 138 |
+
for c in doc["checks"]:
|
| 139 |
+
for heard in c["heard"]:
|
| 140 |
+
key = (norm(str(c["target"])), norm(str(heard)))
|
| 141 |
+
# two spellings of one sound (ʁ and ɣ) meet here: the first rule written wins
|
| 142 |
+
checks.setdefault(key, Check(doc["id"], key[0], key[1], int(c["severity"]), str(c["feedback"])))
|
| 143 |
+
rules = Rules(archi, dialects, optional, scoring, checks, titles, labels)
|
| 144 |
+
for m in scoring.get("margins", []):
|
| 145 |
+
# base() drops vowel length, so a margin for long vowels would land on quality pairs (a / i) it was not meant for
|
| 146 |
+
for t in (t for t in m["target"] if "ː" not in str(t)):
|
| 147 |
+
for h in m["heard"]:
|
| 148 |
+
rules.margins[(norm(str(t)), norm(str(h)))] = float(m["margin"])
|
| 149 |
+
return rules
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
@lru_cache(maxsize=1)
|
| 153 |
+
def load(path: str = str(RULES_PATH)) -> Rules:
|
| 154 |
+
return parse(Path(path).read_text(encoding="utf-8"))
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def undetectable(check: Check, rules: Rules, in_model: Callable[[str], bool],
|
| 158 |
+
heard_as: Callable[[str], object] | None = None) -> str | None:
|
| 159 |
+
"""Why the model cannot catch this mistake, or None if it can.
|
| 160 |
+
|
| 161 |
+
`heard_as` names the model symbol a sound is heard as (stand-ins included: ðˤ as z); two sounds heard
|
| 162 |
+
as the same symbol cannot be told apart."""
|
| 163 |
+
same = (lambda a, b: a == b or (heard_as(a) is not None and heard_as(a) == heard_as(b))) if heard_as \
|
| 164 |
+
else (lambda a, b: a == b)
|
| 165 |
+
if same(check.target, check.heard):
|
| 166 |
+
return "same_sound" # the two collapse to one sound (aː / a: length is not modelled)
|
| 167 |
+
accepted = rules.members(check.target) if check.target.isupper() else [check.target]
|
| 168 |
+
if any(same(check.heard, p) for p in accepted):
|
| 169 |
+
return "accepted" # the 'mistake' is a correct realisation of the target
|
| 170 |
+
if not any(in_model(p) for p in accepted):
|
| 171 |
+
return "target_not_in_model"
|
| 172 |
+
if check.heard != "_" and not in_model(check.heard):
|
| 173 |
+
return "heard_not_in_model"
|
| 174 |
+
return None
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def check_detectable(rules: Rules, in_model: Callable[[str], bool],
|
| 178 |
+
heard_as: Callable[[str], object] | None = None) -> dict[str, dict]:
|
| 179 |
+
"""Per rule: whether at least one of its checks can fire, and why each of the others cannot."""
|
| 180 |
+
out: dict[str, dict] = {}
|
| 181 |
+
for c in rules.checks.values():
|
| 182 |
+
r = out.setdefault(c.rule, {"title": rules.titles.get(c.rule, c.rule), "detectable": False, "missing": []})
|
| 183 |
+
why = undetectable(c, rules, in_model, heard_as)
|
| 184 |
+
if why is None:
|
| 185 |
+
r["detectable"] = True
|
| 186 |
+
else:
|
| 187 |
+
r["missing"].append({"target": c.target, "heard": c.heard, "why": why})
|
| 188 |
+
return out
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
# ---------- exemptions (section 6.3): what the model hears as a mistake but is ordinary speech ----------
|
| 192 |
+
|
| 193 |
+
VOWELS = {"a", "i", "u", "e", "o", "aː", "iː", "uː", "eː", "oː"}
|
| 194 |
+
_VOWEL_GROUPS = [set("iɪeɛ"), set("uʊoɔ"), set("aæɐɑəɛe")] # a raised towards e is ordinary Palestinian speech
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def near_vowels(expected: str, heard: str | None) -> bool:
|
| 198 |
+
"""Same vowel with another length, or the neighbouring quality (i/e, u/o, a/ə, a/e)."""
|
| 199 |
+
if expected not in VOWELS or not heard:
|
| 200 |
+
return False
|
| 201 |
+
e, h = expected.rstrip("ː"), heard.rstrip("ː")
|
| 202 |
+
return any(e in g and h in g for g in _VOWEL_GROUPS)
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
def elided_article(toks, i: int, ar_words: list[str]) -> bool:
|
| 206 |
+
"""Is token i the vowel of the article ال right after a word that ends in a vowel?
|
| 207 |
+
|
| 208 |
+
There it is not said (على الطّاولة is ʕ a l a | tˤ tˤ aː w l e), so its absence is not an error.
|
| 209 |
+
"""
|
| 210 |
+
t = toks[i]
|
| 211 |
+
return (i > 0 and t.phoneme in VOWELS and toks[i - 1].word == t.word - 1 and toks[i - 1].phoneme in VOWELS
|
| 212 |
+
and t.word < len(ar_words) and ar_words[t.word].startswith("ال"))
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
# A glottal stop that is simply not there is how natives say it too (انا, اهلا; ق between vowels): #1, #2.
|
| 216 |
+
# A glottal stop replaced by another consonant still counts, unless the class accepts it (ق said as k).
|
| 217 |
+
def _glottal_dropped(toks, i, sc, heard, ar_words):
|
| 218 |
+
return sc.deleted and toks[i].sound in ("ʔ", "Q")
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def _article_vowel_elided(toks, i, sc, heard, ar_words): # #3
|
| 222 |
+
return sc.deleted and elided_article(toks, i, ar_words)
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
# Neighbouring vowel qualities and vowel length vary between speakers and towns (il / el),
|
| 226 |
+
# and the model confuses them on its own: not worth marking a word down for (#7).
|
| 227 |
+
def _near_vowel(toks, i, sc, heard, ar_words):
|
| 228 |
+
return near_vowels(toks[i].phoneme, heard)
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
# (name, applies(tokens, index, TokenScore, heard symbol, Arabic words), score it gets: None = full marks,
|
| 232 |
+
# else the name of the ScorerConfig floor). The first that applies wins.
|
| 233 |
+
EXEMPTIONS = [
|
| 234 |
+
("glottal_dropped", _glottal_dropped, None),
|
| 235 |
+
("article_vowel_elided", _article_vowel_elided, None),
|
| 236 |
+
("near_vowel", _near_vowel, "near_vowel_floor"),
|
| 237 |
+
]
|
app/scorer.py
CHANGED
|
@@ -11,11 +11,14 @@ from typing import Callable, Protocol
|
|
| 11 |
|
| 12 |
import numpy as np
|
| 13 |
|
| 14 |
-
from .align import GopConfig, ctc_viterbi, edit_distance, greedy_decode, score_tokens
|
| 15 |
-
from .
|
| 16 |
-
from .
|
|
|
|
|
|
|
|
|
|
| 17 |
|
| 18 |
-
HEBREW_SOUNDS = set(_HEB_C) | set(_HEB_V)
|
| 19 |
|
| 20 |
log = logging.getLogger("ihki.scorer")
|
| 21 |
|
|
@@ -24,37 +27,24 @@ MAX_SECONDS = 32 # the recording includes the time to read the sentence and thi
|
|
| 24 |
|
| 25 |
# Fallbacks when the model vocab lacks a phoneme. The first symbol found in the vocab wins.
|
| 26 |
FALLBACKS = {
|
| 27 |
-
"sˤ": ["sˤ", "sˠ", "s"], "tˤ": ["tˤ", "tˠ", "t"], "dˤ": ["dˤ", "dˠ", "d"],
|
| 28 |
"ðˤ": ["ðˤ", "ðˠ", "zˤ", "z"], "dʒ": ["dʒ", "d͡ʒ", "ʒ"],
|
| 29 |
"aː": ["aː", "a"], "iː": ["iː", "i"], "uː": ["uː", "u"], "eː": ["eː", "e"], "oː": ["oː", "o"],
|
| 30 |
"x": ["x", "χ"], "ɣ": ["ɣ", "ʁ"], "q": ["q", "k"], "r": ["r", "ɾ"], "g": ["ɡ", "g"],
|
| 31 |
}
|
| 32 |
|
| 33 |
|
| 34 |
-
# The model's vocab is multilingual (392 symbols) and spells one sound several ways: t / t̪ / tʰ / tː,
|
| 35 |
-
# r / ɾ, a / æ / ɐ, x / χ. Left alone, a correctly said t loses to its own spelling variant t̪.
|
| 36 |
-
# base() names the sound a symbol stands for; Vocab.merge() pools the posteriors of each sound.
|
| 37 |
-
_STRIP = set("ːʰʲ.^[:\"\u032a\u0329\u031d\u031e\u030a\u0303") # length, aspiration, palatalisation, dental, syllabic, nasal ...
|
| 38 |
-
_SAME = {"æ": "a", "ɐ": "a", "ɑ": "a", "ä": "a", "ʌ": "a", "ɪ": "i", "ʊ": "u", "ɛ": "e", "ɔ": "o",
|
| 39 |
-
"ɾ": "r", "ɫ": "l", "ɡ": "g", "χ": "x", "ʁ": "ɣ", "ʒ": "dʒ", "dZ": "dʒ", "tS": "tʃ", "S": "ʃ"}
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
def base(symbol: str) -> str:
|
| 43 |
-
"""The sound a model symbol stands for, ignoring length and spelling variants. Emphatics stay distinct."""
|
| 44 |
-
s = "".join(c for c in symbol if c not in _STRIP)
|
| 45 |
-
return _SAME.get(s, s) or symbol
|
| 46 |
-
|
| 47 |
-
|
| 48 |
class Emitter(Protocol):
|
| 49 |
"""Anything that turns 16 kHz mono float32 audio into (T, V) log-posteriors."""
|
| 50 |
def __call__(self, wav: np.ndarray) -> np.ndarray: ...
|
| 51 |
|
| 52 |
|
| 53 |
class Vocab:
|
| 54 |
-
def __init__(self, token_to_id: dict[str, int], blank_token: str = "<pad>"):
|
| 55 |
self.t2i = token_to_id
|
| 56 |
self.i2t = {i: t for t, i in token_to_id.items()}
|
| 57 |
self.blank = token_to_id[blank_token]
|
|
|
|
| 58 |
self.downgraded: dict[str, str] = {}
|
| 59 |
# sound classes: every id points at one representative id (the plain symbol when the vocab has it)
|
| 60 |
groups: dict[str, list[int]] = {}
|
|
@@ -63,12 +53,20 @@ class Vocab:
|
|
| 63 |
groups.setdefault(base(t), []).append(i)
|
| 64 |
self.canon = {i: i for i in self.i2t}
|
| 65 |
self._groups: list[tuple[int, list[int]]] = []
|
|
|
|
| 66 |
for name, ids in groups.items():
|
| 67 |
rep = token_to_id.get(name, ids[0]) if token_to_id.get(name) in ids else ids[0]
|
|
|
|
| 68 |
for i in ids:
|
| 69 |
self.canon[i] = rep
|
| 70 |
if len(ids) > 1:
|
| 71 |
self._groups.append((rep, ids))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
|
| 73 |
def merge(self, logp: np.ndarray) -> np.ndarray:
|
| 74 |
"""Pool each sound's spelling variants into its representative column; the other columns go silent."""
|
|
@@ -78,6 +76,53 @@ class Vocab:
|
|
| 78 |
out[:, rep] = np.logaddexp.reduce(logp[:, ids], axis=1)
|
| 79 |
return out
|
| 80 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
def resolve(self, phoneme: str) -> list[int]:
|
| 82 |
for cand in FALLBACKS.get(phoneme, [phoneme]):
|
| 83 |
if cand in self.t2i:
|
|
@@ -97,54 +142,52 @@ class Vocab:
|
|
| 97 |
self.downgraded[phoneme] = " ".join(self.i2t[i] for i in ids)
|
| 98 |
return ids
|
| 99 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 100 |
|
| 101 |
LETTER_GOOD = 75 # a letter scoring under this is coloured on the page (`band` in static/index.html)
|
| 102 |
|
| 103 |
|
| 104 |
@dataclass
|
| 105 |
class TargetToken:
|
| 106 |
-
phoneme: str
|
| 107 |
-
id: int
|
| 108 |
-
word: int
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
|
| 110 |
|
| 111 |
def tokenize(ipa: str, vocab: Vocab) -> list[TargetToken]:
|
| 112 |
"""IPA string -> model tokens. Geminates (and any repeated id) collapse to one CTC token."""
|
| 113 |
out: list[TargetToken] = []
|
| 114 |
for w, word in enumerate(ipa.split("|")):
|
| 115 |
-
for
|
| 116 |
-
|
|
|
|
| 117 |
if out and out[-1].id == tid:
|
| 118 |
continue
|
| 119 |
-
out.append(TargetToken(ph, tid, w))
|
| 120 |
return out
|
| 121 |
|
| 122 |
|
| 123 |
-
_VOWEL_GROUPS = [set("iɪeɛ"), set("uʊoɔ"), set("aæɐɑəɛe")] # a raised towards e is ordinary Palestinian speech
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
def near_vowels(expected: str, heard: str | None) -> bool:
|
| 127 |
-
"""Same vowel with another length, or the neighbouring quality (i/e, u/o, a/ə, a/e)."""
|
| 128 |
-
if expected not in VOWELS or not heard:
|
| 129 |
-
return False
|
| 130 |
-
e, h = expected.rstrip("ː"), heard.rstrip("ː")
|
| 131 |
-
return any(e in g and h in g for g in _VOWEL_GROUPS)
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
def elided_article(toks: list[TargetToken], i: int, ar_words: list[str]) -> bool:
|
| 135 |
-
"""Is token i the vowel of the article ال right after a word that ends in a vowel?
|
| 136 |
-
|
| 137 |
-
There it is not said (على الطّاولة is ʕ a l a | tˤ tˤ aː w l e), so its absence is not an error.
|
| 138 |
-
"""
|
| 139 |
-
t = toks[i]
|
| 140 |
-
return (i > 0 and t.phoneme in VOWELS and toks[i - 1].word == t.word - 1 and toks[i - 1].phoneme in VOWELS
|
| 141 |
-
and t.word < len(ar_words) and ar_words[t.word].startswith("ال"))
|
| 142 |
-
|
| 143 |
-
|
| 144 |
@dataclass
|
| 145 |
class Reference:
|
| 146 |
"""How the model scored a native voice saying a phrase: the baseline a learner is measured against."""
|
| 147 |
-
sounds: list[tuple[int, str]] # (word,
|
| 148 |
scores: list[float] # their scores, same order
|
| 149 |
|
| 150 |
|
|
@@ -154,16 +197,42 @@ class ScorerConfig:
|
|
| 154 |
unrelated_ratio: float = 0.65 # normalised edit distance above which we say "different sentence"
|
| 155 |
issue_below: float = 90.0 # phoneme score under which it is listed as imprecise
|
| 156 |
max_issues: int = 8
|
|
|
|
| 157 |
near_vowel_floor: float = 90.0 # a vowel heard as its neighbour (i/e, u/o, a/ə, short/long) scores at least this
|
| 158 |
word_min_weight: float = 0.5 # word score = this much weakest phoneme, the rest the mean of its phonemes
|
| 159 |
ref_min_mean: float = 60.0 # a native reference whose mean phoneme score is under this is not trusted
|
| 160 |
gop: GopConfig = field(default_factory=GopConfig)
|
| 161 |
|
| 162 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 163 |
class Scorer:
|
| 164 |
def __init__(self, emit: Emitter, vocab: Vocab, cfg: ScorerConfig | None = None):
|
| 165 |
self.emit, self.vocab, self.cfg = emit, vocab, cfg or ScorerConfig()
|
|
|
|
| 166 |
self.calibrations: dict[tuple[str, str, str], float] = {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 167 |
|
| 168 |
def set_calibrations(self, rows: list[dict]) -> None:
|
| 169 |
"""Atomically replaces applied score floors. A blank phrase is a global rule."""
|
|
@@ -186,18 +255,18 @@ class Scorer:
|
|
| 186 |
fit = self._fit(self.vocab.merge(self.emit(wav)), phrase)
|
| 187 |
if fit is None:
|
| 188 |
return None
|
| 189 |
-
|
| 190 |
-
if unrelated or float(np.mean([sc.score for sc in scores])) < self.cfg.ref_min_mean:
|
| 191 |
return None
|
| 192 |
-
return Reference([(t.word, t.
|
| 193 |
|
| 194 |
@staticmethod
|
| 195 |
def _baseline(toks: list[TargetToken], refs: list[Reference]) -> list[float | None]:
|
| 196 |
"""Per target token, the lowest score a native reference got on the same sound (None: no reference has it).
|
| 197 |
|
| 198 |
References may have said another accepted variant: only the sounds the two variants share are matched.
|
|
|
|
| 199 |
"""
|
| 200 |
-
sounds = [(t.word, t.
|
| 201 |
out: list[float | None] = [None] * len(toks)
|
| 202 |
for ref in refs:
|
| 203 |
for a, b, n in SequenceMatcher(None, ref.sounds, sounds, autojunk=False).get_matching_blocks():
|
|
@@ -213,7 +282,7 @@ class Scorer:
|
|
| 213 |
row = []
|
| 214 |
for ph in word.split():
|
| 215 |
idx = []
|
| 216 |
-
for tid in self.vocab.
|
| 217 |
tid = self.vocab.canon[tid]
|
| 218 |
if tid != last:
|
| 219 |
n, last = n + 1, tid
|
|
@@ -236,7 +305,7 @@ class Scorer:
|
|
| 236 |
"""How each target token is written in the phrase bank, geminates included ("m m")."""
|
| 237 |
runs: list[tuple[int, str, int]] = [] # (word, phoneme, repeat)
|
| 238 |
for w, word in enumerate(ipa.split("|")):
|
| 239 |
-
for ph in word.split():
|
| 240 |
if runs and runs[-1][:2] == (w, ph):
|
| 241 |
runs[-1] = (w, ph, runs[-1][2] + 1)
|
| 242 |
else:
|
|
@@ -250,6 +319,7 @@ class Scorer:
|
|
| 250 |
|
| 251 |
A sound heard as expected is written the way the answer writes it (length, gemination, emphasis:
|
| 252 |
things the scorer does not tell apart), so the two lines differ only where the scorer heard a difference.
|
|
|
|
| 253 |
"""
|
| 254 |
n_words = len(ipa.split("|"))
|
| 255 |
starts = [None] * n_words # first frame of each expected word
|
|
@@ -261,57 +331,81 @@ class Scorer:
|
|
| 261 |
expected[t] = (tok.id, spell)
|
| 262 |
heard: list[list[str]] = [[] for _ in range(n_words)]
|
| 263 |
prev, w = None, 0
|
| 264 |
-
for t, i in enumerate(logp.argmax(1)):
|
| 265 |
i = int(i)
|
| 266 |
if i != prev and i != self.vocab.blank:
|
| 267 |
while w + 1 < n_words and starts[w + 1] is not None and t >= starts[w + 1]:
|
| 268 |
w += 1
|
| 269 |
ph = base(self.vocab.i2t[i])
|
| 270 |
-
|
| 271 |
-
|
|
|
|
|
|
|
|
|
|
| 272 |
elif ph in HEBREW_SOUNDS:
|
| 273 |
heard[w].append(ph)
|
| 274 |
prev = i
|
| 275 |
# the Arabic spelling is passed so the heard line follows the answer's conventions (ק for ق, silent ל)
|
| 276 |
return [word_to_hebrew(" ".join(phs), ar_words[w] if w < len(ar_words) else "") for w, phs in enumerate(heard)]
|
| 277 |
|
| 278 |
-
def
|
| 279 |
-
"""
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
best = None
|
| 284 |
for ar, ipa in phrase["variants"]:
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 289 |
if best is None:
|
| 290 |
return None
|
| 291 |
|
| 292 |
-
ar, ipa, toks, al = best
|
| 293 |
ids = [t.id for t in toks]
|
| 294 |
ar_words = ar.split()
|
| 295 |
-
scores = score_tokens(
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
unrelated = bool(
|
| 308 |
-
return ar, ipa, toks, al, scores, heard_ids, unrelated
|
| 309 |
|
| 310 |
def _heatmap(self, logp: np.ndarray, toks: list[TargetToken], alignment, scores,
|
| 311 |
duration_s: float | None) -> dict:
|
| 312 |
"""A compact posterior heatmap for teacher diagnostics: expected sounds, strongest competitors and blank.
|
| 313 |
|
| 314 |
Max-pooling preserves short phoneme peaks while capping the response at 180 time columns.
|
|
|
|
| 315 |
"""
|
| 316 |
expected_ids = list(dict.fromkeys(t.id for t in toks))
|
| 317 |
competitor_ids = list(dict.fromkeys(sc.competitor for sc in scores
|
|
@@ -319,7 +413,8 @@ class Scorer:
|
|
| 319 |
ids = expected_ids + competitor_ids + ([self.vocab.blank] if self.vocab.blank not in expected_ids else [])
|
| 320 |
frames, bins = len(logp), min(len(logp), 180)
|
| 321 |
edges = np.linspace(0, frames, bins + 1, dtype=int)
|
| 322 |
-
labels = {t.id:
|
|
|
|
| 323 |
rows = []
|
| 324 |
for token_id in ids:
|
| 325 |
probability = np.exp(logp[:, token_id])
|
|
@@ -327,7 +422,7 @@ class Scorer:
|
|
| 327 |
rows.append({"phoneme": labels.get(token_id, "blank" if token_id == self.vocab.blank
|
| 328 |
else self.vocab.i2t.get(token_id, "?")),
|
| 329 |
"expected": token_id in expected_ids, "values": values})
|
| 330 |
-
greedy = logp.argmax(1)
|
| 331 |
heard = []
|
| 332 |
start = 0
|
| 333 |
for end in range(1, frames + 1):
|
|
@@ -343,26 +438,55 @@ class Scorer:
|
|
| 343 |
"frames": frames, "frame_bins": bins, "rows": rows,
|
| 344 |
"tracks": {"expected": expected, "heard": heard}}
|
| 345 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 346 |
def score_logp(self, logp: np.ndarray, phrase: dict, refs: list[Reference] = (), diagnostics: bool = False,
|
| 347 |
duration_s: float | None = None) -> dict:
|
| 348 |
logp = self.vocab.merge(logp)
|
| 349 |
fit = self._fit(logp, phrase)
|
| 350 |
if fit is None:
|
| 351 |
return {"error": "too_short", "message": "ההקלטה קצרה מדי. החזיקו את הכפתור ואמרו את כל המשפט."}
|
| 352 |
-
ar, ipa, toks, al, scores,
|
| 353 |
-
|
|
|
|
| 354 |
# Measure against what the model gives a native on the same sound: a sound it misses in the
|
| 355 |
# reference voice too (ʕ in على heard as dropped) is the model's weakness, not the learner's.
|
| 356 |
-
heatmap = self._heatmap(logp, toks, al, scores, duration_s) if diagnostics else None
|
| 357 |
-
raw = scores
|
| 358 |
scores = [sc if b is None or b >= 100 else replace(sc, score=round(min(100.0, 100.0 * sc.score / max(b, 1.0)), 1))
|
| 359 |
-
for sc, b in zip(raw,
|
|
|
|
|
|
|
| 360 |
phrase_key, calibrated = str(phrase.get("key", phrase.get("he", ""))), []
|
| 361 |
-
adjusted = []
|
| 362 |
for tok, sc in zip(toks, scores):
|
| 363 |
-
heard_ph = "_" if sc.deleted else (
|
| 364 |
-
floor = self.calibrations
|
| 365 |
-
|
|
|
|
|
|
|
| 366 |
if floor is not None and sc.score < floor:
|
| 367 |
calibrated.append({"expected": tok.phoneme, "heard": heard_ph, "from": round(sc.score), "to": round(floor)})
|
| 368 |
sc = replace(sc, score=floor)
|
|
@@ -372,16 +496,26 @@ class Scorer:
|
|
| 372 |
|
| 373 |
phonemes, tips, seen = [], [], set()
|
| 374 |
heard_as = [] # per phoneme, what took its place ("_": nothing), flagged or not: the reason a letter gives
|
|
|
|
|
|
|
| 375 |
ar_words = ar.split()
|
| 376 |
tr_words = to_hebrew(ipa, ar)
|
| 377 |
for idx, (tok, sc) in enumerate(zip(toks, scores)):
|
| 378 |
-
heard_ph = "_" if sc.deleted else (
|
| 379 |
-
|
|
|
|
|
|
|
| 380 |
heard_as.append(heard_ph)
|
| 381 |
-
|
| 382 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 383 |
if flagged and not unrelated:
|
| 384 |
-
tip = tip_for(tok.
|
| 385 |
if tip and tip["key"] not in seen:
|
| 386 |
seen.add(tip["key"])
|
| 387 |
tip["word"] = ar_words[tok.word] if tok.word < len(ar_words) else ""
|
|
@@ -391,9 +525,9 @@ class Scorer:
|
|
| 391 |
tips.append(tip)
|
| 392 |
|
| 393 |
n_words = len(ipa.split("|"))
|
| 394 |
-
heard_words = self._heard_hebrew(logp, toks, al,
|
| 395 |
words, k = [], self.cfg.word_min_weight
|
| 396 |
-
carriers, ipa_words = self._carriers(
|
| 397 |
for w in range(n_words):
|
| 398 |
ws = [p["score"] for p in phonemes if p["word"] == w]
|
| 399 |
# the transliteration letter by letter, each with the weakest score among its sounds; a letter the
|
|
@@ -404,7 +538,7 @@ class Scorer:
|
|
| 404 |
worst = min(got, key=lambda i: phonemes[i]["score"]) if got else None
|
| 405 |
letter = {"t": text, "score": phonemes[worst]["score"] if got else None}
|
| 406 |
if got and letter["score"] < LETTER_GOOD and not unrelated:
|
| 407 |
-
letter.update(letter_reason(
|
| 408 |
letters.append(letter)
|
| 409 |
if w in forgiven and min(ws) >= self.cfg.issue_below:
|
| 410 |
heard_words[w] = tr_words[w] # what differs from the answer here is what a native voice gets too
|
|
@@ -418,7 +552,10 @@ class Scorer:
|
|
| 418 |
overall = 0.5 * phone_mean + 0.5 * word_mean # one wrong consonant must pull the sentence down
|
| 419 |
if unrelated:
|
| 420 |
overall = min(overall, 30.0)
|
| 421 |
-
|
|
|
|
|
|
|
|
|
|
| 422 |
|
| 423 |
# Anything short of 100 must say why: every imprecise sound that did not get a full tip,
|
| 424 |
# weakest first. If nothing is clearly off, name the weakest few.
|
|
@@ -427,25 +564,105 @@ class Scorer:
|
|
| 427 |
tipped = {t.pop("idx") for t in tips}
|
| 428 |
rest = sorted((i for i in range(len(toks)) if i not in tipped and scores[i].score < 100),
|
| 429 |
key=lambda i: scores[i].score)
|
| 430 |
-
picked = [i for i in rest if scores[i].score < self.cfg.issue_below]
|
| 431 |
if not picked and not tips:
|
| 432 |
picked = rest[:3]
|
| 433 |
-
|
|
|
|
| 434 |
tok, sc = toks[i], scores[i]
|
| 435 |
-
comp =
|
| 436 |
-
issues.append({"sound": sound_name(tok.
|
| 437 |
-
"score": round(sc.score), "missing": bool(sc.deleted),
|
| 438 |
"heard": None if sc.deleted else sound_name(comp)})
|
| 439 |
for t in tips:
|
| 440 |
t.pop("idx", None)
|
| 441 |
result = {"score": round(overall), "variant": {"ar": ar, "tr": " ".join(tr_words), "ipa": ipa}, "heard_ipa": heard, "heard_tr": " ".join(h for h in heard_words if h),
|
| 442 |
-
"unrelated": unrelated, "words": words, "tips": tips, "issues": issues, "references": len(refs)
|
|
|
|
| 443 |
if calibrated:
|
| 444 |
result["calibrated"] = calibrated
|
| 445 |
if heatmap is not None:
|
| 446 |
result["phoneme_heatmap"] = heatmap
|
|
|
|
|
|
|
|
|
|
| 447 |
return result
|
| 448 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 449 |
|
| 450 |
# ---------- real model ----------
|
| 451 |
|
|
@@ -500,8 +717,10 @@ def load_scorer(model_id: str) -> Scorer:
|
|
| 500 |
# resolve every phoneme once at startup so missing symbols show up in the logs and /api/health
|
| 501 |
from .phrases import PHRASES
|
| 502 |
for p in PHRASES:
|
| 503 |
-
for
|
| 504 |
-
tokenize(ipa, vocab)
|
| 505 |
if vocab.downgraded:
|
| 506 |
log.warning("phonemes mapped to fallbacks: %s", vocab.downgraded)
|
|
|
|
|
|
|
| 507 |
return scorer
|
|
|
|
| 11 |
|
| 12 |
import numpy as np
|
| 13 |
|
| 14 |
+
from .align import Alignment, GopConfig, TokenScore, ctc_viterbi, edit_distance, greedy_decode, score_tokens
|
| 15 |
+
from .archi import archify, diphthong_variants, split_token, written
|
| 16 |
+
from .phones import base
|
| 17 |
+
from .rules import EXEMPTIONS, Rules, elided_article, near_vowels, load as load_rules # noqa: F401 (re-exported)
|
| 18 |
+
from .tips import VOWELS, letter_reason, sound_name, tip_for
|
| 19 |
+
from .translit import CONSONANTS as _HEB_C, HEARD_ONLY as _HEB_HEARD, VOWELS as _HEB_V, to_hebrew, word_letters, word_to_hebrew
|
| 20 |
|
| 21 |
+
HEBREW_SOUNDS = set(_HEB_C) | set(_HEB_V) | set(_HEB_HEARD)
|
| 22 |
|
| 23 |
log = logging.getLogger("ihki.scorer")
|
| 24 |
|
|
|
|
| 27 |
|
| 28 |
# Fallbacks when the model vocab lacks a phoneme. The first symbol found in the vocab wins.
|
| 29 |
FALLBACKS = {
|
| 30 |
+
"sˤ": ["sˤ", "sˠ", "s"], "tˤ": ["tˤ", "tˠ", "t"], "dˤ": ["dˤ", "dˠ", "d"], "zˤ": ["zˤ", "zˠ", "z"],
|
| 31 |
"ðˤ": ["ðˤ", "ðˠ", "zˤ", "z"], "dʒ": ["dʒ", "d͡ʒ", "ʒ"],
|
| 32 |
"aː": ["aː", "a"], "iː": ["iː", "i"], "uː": ["uː", "u"], "eː": ["eː", "e"], "oː": ["oː", "o"],
|
| 33 |
"x": ["x", "χ"], "ɣ": ["ɣ", "ʁ"], "q": ["q", "k"], "r": ["r", "ɾ"], "g": ["ɡ", "g"],
|
| 34 |
}
|
| 35 |
|
| 36 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
class Emitter(Protocol):
|
| 38 |
"""Anything that turns 16 kHz mono float32 audio into (T, V) log-posteriors."""
|
| 39 |
def __call__(self, wav: np.ndarray) -> np.ndarray: ...
|
| 40 |
|
| 41 |
|
| 42 |
class Vocab:
|
| 43 |
+
def __init__(self, token_to_id: dict[str, int], blank_token: str = "<pad>", rules: Rules | None = None):
|
| 44 |
self.t2i = token_to_id
|
| 45 |
self.i2t = {i: t for t, i in token_to_id.items()}
|
| 46 |
self.blank = token_to_id[blank_token]
|
| 47 |
+
self.rules = rules or load_rules()
|
| 48 |
self.downgraded: dict[str, str] = {}
|
| 49 |
# sound classes: every id points at one representative id (the plain symbol when the vocab has it)
|
| 50 |
groups: dict[str, list[int]] = {}
|
|
|
|
| 53 |
groups.setdefault(base(t), []).append(i)
|
| 54 |
self.canon = {i: i for i in self.i2t}
|
| 55 |
self._groups: list[tuple[int, list[int]]] = []
|
| 56 |
+
self.rep_of: dict[str, int] = {} # base() name -> representative id
|
| 57 |
for name, ids in groups.items():
|
| 58 |
rep = token_to_id.get(name, ids[0]) if token_to_id.get(name) in ids else ids[0]
|
| 59 |
+
self.rep_of[name] = rep
|
| 60 |
for i in ids:
|
| 61 |
self.canon[i] = rep
|
| 62 |
if len(ids) > 1:
|
| 63 |
self._groups.append((rep, ids))
|
| 64 |
+
# Letters with several correct realisations (archi.py): one virtual id per (symbol, realisations),
|
| 65 |
+
# numbered after the real vocab, whose column extend() fills with the pooled posterior of its members.
|
| 66 |
+
self.n_real = max(self.i2t) + 1
|
| 67 |
+
self.members: dict[int, set[int]] = {}
|
| 68 |
+
self._classes: dict[tuple[str, frozenset[int]], int] = {}
|
| 69 |
+
self.class_missing: dict[str, set[str]] = {} # symbol -> realisations the model has no symbol for
|
| 70 |
|
| 71 |
def merge(self, logp: np.ndarray) -> np.ndarray:
|
| 72 |
"""Pool each sound's spelling variants into its representative column; the other columns go silent."""
|
|
|
|
| 76 |
out[:, rep] = np.logaddexp.reduce(logp[:, ids], axis=1)
|
| 77 |
return out
|
| 78 |
|
| 79 |
+
def sound_id(self, phone: str) -> int | None:
|
| 80 |
+
"""The representative id of a sound (any spelling of it), or None if the model has no symbol for it."""
|
| 81 |
+
return self.rep_of.get(base(phone))
|
| 82 |
+
|
| 83 |
+
def model_id(self, phone: str) -> int | None:
|
| 84 |
+
"""The id the model hears `phone` as: its own, else its first FALLBACKS stand-in (ðˤ -> z), else None."""
|
| 85 |
+
for cand in [phone, *FALLBACKS.get(phone, [])]:
|
| 86 |
+
i = self.sound_id(cand)
|
| 87 |
+
if i is not None:
|
| 88 |
+
return i
|
| 89 |
+
return None
|
| 90 |
+
|
| 91 |
+
def add_class(self, symbol: str, phones: list[str]) -> int:
|
| 92 |
+
"""The id that stands for 'any of `phones`': a virtual id, or the real one when only one is in the model.
|
| 93 |
+
A realisation the model lacks is heard as its stand-in, as a plain phoneme would be (zˤ as z)."""
|
| 94 |
+
ids: list[int] = []
|
| 95 |
+
for ph in phones:
|
| 96 |
+
i = self.model_id(ph)
|
| 97 |
+
if i is None:
|
| 98 |
+
self.class_missing.setdefault(symbol, set()).add(ph)
|
| 99 |
+
continue
|
| 100 |
+
if i != self.sound_id(ph):
|
| 101 |
+
self.downgraded[ph] = self.i2t[i]
|
| 102 |
+
if i not in ids:
|
| 103 |
+
ids.append(i)
|
| 104 |
+
if not ids:
|
| 105 |
+
raise KeyError(f"no realisation of {symbol} ({phones}) is in the model vocab")
|
| 106 |
+
if len(ids) == 1:
|
| 107 |
+
return ids[0]
|
| 108 |
+
key = (symbol, frozenset(ids))
|
| 109 |
+
if key not in self._classes:
|
| 110 |
+
vid = self.n_real + len(self._classes)
|
| 111 |
+
self._classes[key], self.members[vid] = vid, set(ids)
|
| 112 |
+
self.i2t[vid], self.canon[vid] = symbol, vid
|
| 113 |
+
return self._classes[key]
|
| 114 |
+
|
| 115 |
+
def extend(self, logp: np.ndarray) -> np.ndarray:
|
| 116 |
+
"""Merged posteriors with one more column per class: the pooled posterior of its realisations."""
|
| 117 |
+
logp = logp[:, :self.n_real]
|
| 118 |
+
if not self.members:
|
| 119 |
+
return logp
|
| 120 |
+
out = np.full((logp.shape[0], self.n_real + len(self.members)), -1e9, dtype=logp.dtype)
|
| 121 |
+
out[:, :logp.shape[1]] = logp
|
| 122 |
+
for vid, ids in self.members.items():
|
| 123 |
+
out[:, vid] = np.logaddexp.reduce(logp[:, sorted(ids)], axis=1)
|
| 124 |
+
return out
|
| 125 |
+
|
| 126 |
def resolve(self, phoneme: str) -> list[int]:
|
| 127 |
for cand in FALLBACKS.get(phoneme, [phoneme]):
|
| 128 |
if cand in self.t2i:
|
|
|
|
| 142 |
self.downgraded[phoneme] = " ".join(self.i2t[i] for i in ids)
|
| 143 |
return ids
|
| 144 |
|
| 145 |
+
def token(self, tok: str) -> tuple[list[int], str, str]:
|
| 146 |
+
"""A token of an IPA string (a phoneme, or an archi.py symbol "Q:ʔ") -> (model ids, phoneme as written, sound).
|
| 147 |
+
|
| 148 |
+
The sound is the symbol for a many-ways letter, the phoneme otherwise."""
|
| 149 |
+
sym, members, ph = split_token(tok)
|
| 150 |
+
if sym is None:
|
| 151 |
+
return self.resolve(tok), tok, tok
|
| 152 |
+
phones = members or self.rules.members(sym, written=base(ph) if ph else None)
|
| 153 |
+
ph = ph or (phones[0] if phones else sym)
|
| 154 |
+
try:
|
| 155 |
+
return [self.add_class(sym, phones)], ph, sym
|
| 156 |
+
except KeyError:
|
| 157 |
+
return self.resolve(ph), ph, sym
|
| 158 |
+
|
| 159 |
|
| 160 |
LETTER_GOOD = 75 # a letter scoring under this is coloured on the page (`band` in static/index.html)
|
| 161 |
|
| 162 |
|
| 163 |
@dataclass
|
| 164 |
class TargetToken:
|
| 165 |
+
phoneme: str # phoneme as written in phrases.py
|
| 166 |
+
id: int # model token id (a virtual id for a letter with several correct realisations)
|
| 167 |
+
word: int # word index in the variant
|
| 168 |
+
sound: str = "" # what the token stands for: the archi.py symbol (Q) or the phoneme itself
|
| 169 |
+
|
| 170 |
+
def __post_init__(self):
|
| 171 |
+
self.sound = self.sound or self.phoneme
|
| 172 |
|
| 173 |
|
| 174 |
def tokenize(ipa: str, vocab: Vocab) -> list[TargetToken]:
|
| 175 |
"""IPA string -> model tokens. Geminates (and any repeated id) collapse to one CTC token."""
|
| 176 |
out: list[TargetToken] = []
|
| 177 |
for w, word in enumerate(ipa.split("|")):
|
| 178 |
+
for tok in word.split():
|
| 179 |
+
ids, ph, sound = vocab.token(tok)
|
| 180 |
+
for tid in ids:
|
| 181 |
if out and out[-1].id == tid:
|
| 182 |
continue
|
| 183 |
+
out.append(TargetToken(ph, tid, w, sound))
|
| 184 |
return out
|
| 185 |
|
| 186 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 187 |
@dataclass
|
| 188 |
class Reference:
|
| 189 |
"""How the model scored a native voice saying a phrase: the baseline a learner is measured against."""
|
| 190 |
+
sounds: list[tuple[int, str]] # (word, sound) of each sound of the variant the voice said (Q, not ʔ / k)
|
| 191 |
scores: list[float] # their scores, same order
|
| 192 |
|
| 193 |
|
|
|
|
| 197 |
unrelated_ratio: float = 0.65 # normalised edit distance above which we say "different sentence"
|
| 198 |
issue_below: float = 90.0 # phoneme score under which it is listed as imprecise
|
| 199 |
max_issues: int = 8
|
| 200 |
+
max_tips: int | None = None # None: as many as the rules document says (scoring.max_tips)
|
| 201 |
near_vowel_floor: float = 90.0 # a vowel heard as its neighbour (i/e, u/o, a/ə, short/long) scores at least this
|
| 202 |
word_min_weight: float = 0.5 # word score = this much weakest phoneme, the rest the mean of its phonemes
|
| 203 |
ref_min_mean: float = 60.0 # a native reference whose mean phoneme score is under this is not trusted
|
| 204 |
gop: GopConfig = field(default_factory=GopConfig)
|
| 205 |
|
| 206 |
|
| 207 |
+
@dataclass
|
| 208 |
+
class Fit:
|
| 209 |
+
"""The accepted variant the audio fits best, and how each of its sounds scored."""
|
| 210 |
+
ar: str
|
| 211 |
+
ipa: str # as written in the phrase bank (display, transliteration)
|
| 212 |
+
aipa: str # the same with archi.py symbols, one token per phoneme of `ipa`
|
| 213 |
+
toks: list[TargetToken]
|
| 214 |
+
al: Alignment
|
| 215 |
+
scores: list[TokenScore]
|
| 216 |
+
heard_ids: list[int] # greedy decode, real symbols only
|
| 217 |
+
unrelated: bool
|
| 218 |
+
logp: np.ndarray # merged posteriors with the class columns (Vocab.extend)
|
| 219 |
+
# for the teacher's stage-by-stage view (Scorer._trace)
|
| 220 |
+
candidates: list[tuple[str, str, float | None]] = field(default_factory=list) # (ar, ipa, path log-prob)
|
| 221 |
+
gop_scores: list[TokenScore] = field(default_factory=list) # before the exemptions
|
| 222 |
+
exempted: list[str | None] = field(default_factory=list) # per token, the exemption that applied
|
| 223 |
+
distance: int = 0 # edit distance of the greedy decode from the chosen variant
|
| 224 |
+
|
| 225 |
+
|
| 226 |
class Scorer:
|
| 227 |
def __init__(self, emit: Emitter, vocab: Vocab, cfg: ScorerConfig | None = None):
|
| 228 |
self.emit, self.vocab, self.cfg = emit, vocab, cfg or ScorerConfig()
|
| 229 |
+
self.rules = vocab.rules
|
| 230 |
self.calibrations: dict[tuple[str, str, str], float] = {}
|
| 231 |
+
# the sounds a competitor is named by: Arabic, the Hebrew sounds the rules expect instead, and their labels;
|
| 232 |
+
# anything else the multilingual model offers (clicks, retroflexes) is reported as "not clear"
|
| 233 |
+
names = (set(HEBREW_SOUNDS) | self.rules.interference() | set(self.rules.heard_labels)
|
| 234 |
+
| {p for s in self.rules.archiphonemes for p in self.rules.members(s)})
|
| 235 |
+
self.named = {i for i in (vocab.sound_id(n) for n in names) if i is not None}
|
| 236 |
|
| 237 |
def set_calibrations(self, rows: list[dict]) -> None:
|
| 238 |
"""Atomically replaces applied score floors. A blank phrase is a global rule."""
|
|
|
|
| 255 |
fit = self._fit(self.vocab.merge(self.emit(wav)), phrase)
|
| 256 |
if fit is None:
|
| 257 |
return None
|
| 258 |
+
if fit.unrelated or float(np.mean([sc.score for sc in fit.scores])) < self.cfg.ref_min_mean:
|
|
|
|
| 259 |
return None
|
| 260 |
+
return Reference([(t.word, t.sound) for t in fit.toks], [sc.score for sc in fit.scores])
|
| 261 |
|
| 262 |
@staticmethod
|
| 263 |
def _baseline(toks: list[TargetToken], refs: list[Reference]) -> list[float | None]:
|
| 264 |
"""Per target token, the lowest score a native reference got on the same sound (None: no reference has it).
|
| 265 |
|
| 266 |
References may have said another accepted variant: only the sounds the two variants share are matched.
|
| 267 |
+
Sounds are compared by symbol, so a voice that said ق as q is a baseline for a learner who says ʔ.
|
| 268 |
"""
|
| 269 |
+
sounds = [(t.word, t.sound) for t in toks]
|
| 270 |
out: list[float | None] = [None] * len(toks)
|
| 271 |
for ref in refs:
|
| 272 |
for a, b, n in SequenceMatcher(None, ref.sounds, sounds, autojunk=False).get_matching_blocks():
|
|
|
|
| 282 |
row = []
|
| 283 |
for ph in word.split():
|
| 284 |
idx = []
|
| 285 |
+
for tid in self.vocab.token(ph)[0]:
|
| 286 |
tid = self.vocab.canon[tid]
|
| 287 |
if tid != last:
|
| 288 |
n, last = n + 1, tid
|
|
|
|
| 305 |
"""How each target token is written in the phrase bank, geminates included ("m m")."""
|
| 306 |
runs: list[tuple[int, str, int]] = [] # (word, phoneme, repeat)
|
| 307 |
for w, word in enumerate(ipa.split("|")):
|
| 308 |
+
for ph in map(written, word.split()):
|
| 309 |
if runs and runs[-1][:2] == (w, ph):
|
| 310 |
runs[-1] = (w, ph, runs[-1][2] + 1)
|
| 311 |
else:
|
|
|
|
| 319 |
|
| 320 |
A sound heard as expected is written the way the answer writes it (length, gemination, emphasis:
|
| 321 |
things the scorer does not tell apart), so the two lines differ only where the scorer heard a difference.
|
| 322 |
+
A letter with several correct realisations is written as the one said (כּ for a rural ق), not coloured.
|
| 323 |
"""
|
| 324 |
n_words = len(ipa.split("|"))
|
| 325 |
starts = [None] * n_words # first frame of each expected word
|
|
|
|
| 331 |
expected[t] = (tok.id, spell)
|
| 332 |
heard: list[list[str]] = [[] for _ in range(n_words)]
|
| 333 |
prev, w = None, 0
|
| 334 |
+
for t, i in enumerate(logp[:, :self.vocab.n_real].argmax(1)):
|
| 335 |
i = int(i)
|
| 336 |
if i != prev and i != self.vocab.blank:
|
| 337 |
while w + 1 < n_words and starts[w + 1] is not None and t >= starts[w + 1]:
|
| 338 |
w += 1
|
| 339 |
ph = base(self.vocab.i2t[i])
|
| 340 |
+
tid, spell = expected.get(t, (None, ""))
|
| 341 |
+
if tid == i:
|
| 342 |
+
heard[w].append(spell)
|
| 343 |
+
elif i in self.vocab.members.get(tid, ()):
|
| 344 |
+
heard[w].append(spell if base(spell.split()[0]) == ph else ph)
|
| 345 |
elif ph in HEBREW_SOUNDS:
|
| 346 |
heard[w].append(ph)
|
| 347 |
prev = i
|
| 348 |
# the Arabic spelling is passed so the heard line follows the answer's conventions (ק for ق, silent ל)
|
| 349 |
return [word_to_hebrew(" ".join(phs), ar_words[w] if w < len(ar_words) else "") for w, phs in enumerate(heard)]
|
| 350 |
|
| 351 |
+
def _variants(self, phrase: dict) -> list[tuple[str, str, str, list[TargetToken]]]:
|
| 352 |
+
"""(arabic, ipa, ipa with symbols, tokens) for each accepted variant and its diphthong forms. Variants
|
| 353 |
+
that differ only in how a many-ways letter is said (ʔ / q / k for ق) become one: the first written.
|
| 354 |
+
A diphthong form the model has no symbols for is left out."""
|
| 355 |
+
out, seen = [], set()
|
|
|
|
| 356 |
for ar, ipa in phrase["variants"]:
|
| 357 |
+
for n, form in enumerate((ipa, *diphthong_variants(ar, ipa))):
|
| 358 |
+
aipa = archify(ar, form)
|
| 359 |
+
if aipa in seen:
|
| 360 |
+
continue
|
| 361 |
+
seen.add(aipa)
|
| 362 |
+
try:
|
| 363 |
+
out.append((ar, form, aipa, self._targets(aipa)))
|
| 364 |
+
except KeyError:
|
| 365 |
+
if n == 0:
|
| 366 |
+
raise
|
| 367 |
+
return out
|
| 368 |
+
|
| 369 |
+
def _fit(self, logp: np.ndarray, phrase: dict) -> Fit | None:
|
| 370 |
+
"""The accepted variant the audio fits best and the score of each of its sounds, from merged
|
| 371 |
+
posteriors, or None if the recording is too short for every variant."""
|
| 372 |
+
blank, vocab = self.vocab.blank, self.vocab
|
| 373 |
+
heard_ids = greedy_decode(logp[:, :vocab.n_real], blank)
|
| 374 |
+
cands = self._variants(phrase)
|
| 375 |
+
ext = vocab.extend(logp) # after tokenising: a variant may have brought a new class
|
| 376 |
+
best, tried = None, []
|
| 377 |
+
for ar, ipa, aipa, toks in cands:
|
| 378 |
+
al = ctc_viterbi(ext, [t.id for t in toks], blank)
|
| 379 |
+
tried.append((ar, ipa, round(al.total_logp, 2) if al else None))
|
| 380 |
+
if al and (best is None or al.total_logp > best[4].total_logp):
|
| 381 |
+
best = (ar, ipa, aipa, toks, al)
|
| 382 |
if best is None:
|
| 383 |
return None
|
| 384 |
|
| 385 |
+
ar, ipa, aipa, toks, al = best
|
| 386 |
ids = [t.id for t in toks]
|
| 387 |
ar_words = ar.split()
|
| 388 |
+
scores = score_tokens(ext, al, ids, blank, self.cfg.gop, [vocab.members.get(i) for i in ids],
|
| 389 |
+
vocab.n_real, self.named)
|
| 390 |
+
gop_scores, exempted = list(scores), [None] * len(scores)
|
| 391 |
+
for i, (t, sc) in enumerate(zip(toks, scores)):
|
| 392 |
+
heard = vocab.i2t.get(sc.competitor) if sc.competitor is not None else None
|
| 393 |
+
for name, applies, floor in EXEMPTIONS:
|
| 394 |
+
if applies(toks, i, sc, heard, ar_words):
|
| 395 |
+
scores[i] = replace(sc, score=max(sc.score, 100.0 if floor is None else getattr(self.cfg, floor)))
|
| 396 |
+
exempted[i] = name
|
| 397 |
+
break
|
| 398 |
+
same = lambda h, t: h == t or h in vocab.members.get(t, ())
|
| 399 |
+
distance = edit_distance(heard_ids, ids, same)
|
| 400 |
+
unrelated = bool(distance / max(1, len(ids)) > self.cfg.unrelated_ratio)
|
| 401 |
+
return Fit(ar, ipa, aipa, toks, al, scores, heard_ids, unrelated, ext, tried, gop_scores, exempted, distance)
|
| 402 |
|
| 403 |
def _heatmap(self, logp: np.ndarray, toks: list[TargetToken], alignment, scores,
|
| 404 |
duration_s: float | None) -> dict:
|
| 405 |
"""A compact posterior heatmap for teacher diagnostics: expected sounds, strongest competitors and blank.
|
| 406 |
|
| 407 |
Max-pooling preserves short phoneme peaks while capping the response at 180 time columns.
|
| 408 |
+
A letter with several correct realisations is one row, the pooled posterior, labelled "ʔ/k/q".
|
| 409 |
"""
|
| 410 |
expected_ids = list(dict.fromkeys(t.id for t in toks))
|
| 411 |
competitor_ids = list(dict.fromkeys(sc.competitor for sc in scores
|
|
|
|
| 413 |
ids = expected_ids + competitor_ids + ([self.vocab.blank] if self.vocab.blank not in expected_ids else [])
|
| 414 |
frames, bins = len(logp), min(len(logp), 180)
|
| 415 |
edges = np.linspace(0, frames, bins + 1, dtype=int)
|
| 416 |
+
labels = {t.id: ("/".join(self.vocab.i2t[m] for m in sorted(self.vocab.members[t.id]))
|
| 417 |
+
if t.id in self.vocab.members else t.phoneme) for t in toks}
|
| 418 |
rows = []
|
| 419 |
for token_id in ids:
|
| 420 |
probability = np.exp(logp[:, token_id])
|
|
|
|
| 422 |
rows.append({"phoneme": labels.get(token_id, "blank" if token_id == self.vocab.blank
|
| 423 |
else self.vocab.i2t.get(token_id, "?")),
|
| 424 |
"expected": token_id in expected_ids, "values": values})
|
| 425 |
+
greedy = logp[:, :self.vocab.n_real].argmax(1)
|
| 426 |
heard = []
|
| 427 |
start = 0
|
| 428 |
for end in range(1, frames + 1):
|
|
|
|
| 438 |
"frames": frames, "frame_bins": bins, "rows": rows,
|
| 439 |
"tracks": {"expected": expected, "heard": heard}}
|
| 440 |
|
| 441 |
+
def _beyond_margin(self, tok: TargetToken, sc: TokenScore, heard: str | None) -> bool:
|
| 442 |
+
"""Is the competitor likelier than the target by more than the pair's margin? Pairs the model confuses
|
| 443 |
+
on natives too (emphatics, a dropped h) need a clearer win before they count as a mistake."""
|
| 444 |
+
targets = [tok.sound, tok.phoneme]
|
| 445 |
+
if sc.deleted:
|
| 446 |
+
return sc.peak <= self.cfg.gop.deleted_peak - self.rules.margin(targets, "_")
|
| 447 |
+
if heard is None:
|
| 448 |
+
return True
|
| 449 |
+
return -sc.gop >= self.rules.margin(targets, heard)
|
| 450 |
+
|
| 451 |
+
def _dialect(self, toks: list[TargetToken], scores: list[TokenScore]) -> dict:
|
| 452 |
+
"""Which dialect the accepted realisations point to: {"dialect": urban | rural} when they agree,
|
| 453 |
+
{"mixed_dialect": True} when one is urban-only and another rural-only. Never part of the score."""
|
| 454 |
+
seen: set[str] = set()
|
| 455 |
+
for t, sc in zip(toks, scores):
|
| 456 |
+
if sc.realised is None or sc.score < self.cfg.issue_below:
|
| 457 |
+
continue
|
| 458 |
+
ds = self.rules.dialects_of(t.sound, base(self.vocab.i2t[sc.realised]))
|
| 459 |
+
if len(ds) == 1:
|
| 460 |
+
seen |= ds
|
| 461 |
+
if len(seen) > 1:
|
| 462 |
+
return {"mixed_dialect": True}
|
| 463 |
+
return {"dialect": next(iter(seen))} if seen else {}
|
| 464 |
+
|
| 465 |
def score_logp(self, logp: np.ndarray, phrase: dict, refs: list[Reference] = (), diagnostics: bool = False,
|
| 466 |
duration_s: float | None = None) -> dict:
|
| 467 |
logp = self.vocab.merge(logp)
|
| 468 |
fit = self._fit(logp, phrase)
|
| 469 |
if fit is None:
|
| 470 |
return {"error": "too_short", "message": "ההקלטה קצרה מדי. החזיקו את הכפתור ואמרו את כל המשפט."}
|
| 471 |
+
ar, ipa, toks, al, scores, unrelated = fit.ar, fit.ipa, fit.toks, fit.al, fit.scores, fit.unrelated
|
| 472 |
+
i2t = self.vocab.i2t
|
| 473 |
+
heard = " ".join(i2t[i] for i in fit.heard_ids)
|
| 474 |
# Measure against what the model gives a native on the same sound: a sound it misses in the
|
| 475 |
# reference voice too (ʕ in على heard as dropped) is the model's weakness, not the learner's.
|
| 476 |
+
heatmap = self._heatmap(fit.logp, toks, al, scores, duration_s) if diagnostics else None
|
| 477 |
+
raw, bases = scores, self._baseline(toks, refs)
|
| 478 |
scores = [sc if b is None or b >= 100 else replace(sc, score=round(min(100.0, 100.0 * sc.score / max(b, 1.0)), 1))
|
| 479 |
+
for sc, b in zip(raw, bases)]
|
| 480 |
+
after_baseline = scores
|
| 481 |
+
# teacher floors, by symbol first (Q), then by the phoneme written (rows made before symbols existed)
|
| 482 |
phrase_key, calibrated = str(phrase.get("key", phrase.get("he", ""))), []
|
| 483 |
+
adjusted, floors = [], []
|
| 484 |
for tok, sc in zip(toks, scores):
|
| 485 |
+
heard_ph = "_" if sc.deleted else (i2t.get(sc.competitor) if sc.competitor is not None else "")
|
| 486 |
+
floor = next((self.calibrations[k] for k in ((tok.sound, heard_ph, phrase_key), (tok.sound, heard_ph, ""),
|
| 487 |
+
(tok.phoneme, heard_ph, phrase_key), (tok.phoneme, heard_ph, ""))
|
| 488 |
+
if k in self.calibrations), None)
|
| 489 |
+
floors.append(floor)
|
| 490 |
if floor is not None and sc.score < floor:
|
| 491 |
calibrated.append({"expected": tok.phoneme, "heard": heard_ph, "from": round(sc.score), "to": round(floor)})
|
| 492 |
sc = replace(sc, score=floor)
|
|
|
|
| 496 |
|
| 497 |
phonemes, tips, seen = [], [], set()
|
| 498 |
heard_as = [] # per phoneme, what took its place ("_": nothing), flagged or not: the reason a letter gives
|
| 499 |
+
checks = [] # per phoneme, the rule for what took its place (None: no rule)
|
| 500 |
+
beyond = [] # per phoneme, whether what took its place won by the pair's margin
|
| 501 |
ar_words = ar.split()
|
| 502 |
tr_words = to_hebrew(ipa, ar)
|
| 503 |
for idx, (tok, sc) in enumerate(zip(toks, scores)):
|
| 504 |
+
heard_ph = "_" if sc.deleted else (i2t.get(sc.competitor) if sc.competitor is not None else None)
|
| 505 |
+
check = self.rules.check([tok.sound, tok.phoneme], heard_ph)
|
| 506 |
+
beyond.append(self._beyond_margin(tok, sc, heard_ph))
|
| 507 |
+
flagged = bool(sc.score < self.cfg.flag_below) and tok.phoneme not in VOWELS and beyond[-1]
|
| 508 |
heard_as.append(heard_ph)
|
| 509 |
+
checks.append(check)
|
| 510 |
+
p = {"ipa": tok.phoneme, "word": tok.word, "score": sc.score,
|
| 511 |
+
"flagged": flagged, "heard": heard_ph if flagged else None}
|
| 512 |
+
if tok.sound != tok.phoneme:
|
| 513 |
+
p["sound"] = tok.sound
|
| 514 |
+
if sc.realised is not None:
|
| 515 |
+
p["realised"] = base(i2t[sc.realised])
|
| 516 |
+
phonemes.append(p)
|
| 517 |
if flagged and not unrelated:
|
| 518 |
+
tip = tip_for(tok.sound, heard_ph, check)
|
| 519 |
if tip and tip["key"] not in seen:
|
| 520 |
seen.add(tip["key"])
|
| 521 |
tip["word"] = ar_words[tok.word] if tok.word < len(ar_words) else ""
|
|
|
|
| 525 |
tips.append(tip)
|
| 526 |
|
| 527 |
n_words = len(ipa.split("|"))
|
| 528 |
+
heard_words = self._heard_hebrew(logp, toks, al, fit.aipa, ar_words)
|
| 529 |
words, k = [], self.cfg.word_min_weight
|
| 530 |
+
carriers, ipa_words = self._carriers(fit.aipa), ipa.split("|")
|
| 531 |
for w in range(n_words):
|
| 532 |
ws = [p["score"] for p in phonemes if p["word"] == w]
|
| 533 |
# the transliteration letter by letter, each with the weakest score among its sounds; a letter the
|
|
|
|
| 538 |
worst = min(got, key=lambda i: phonemes[i]["score"]) if got else None
|
| 539 |
letter = {"t": text, "score": phonemes[worst]["score"] if got else None}
|
| 540 |
if got and letter["score"] < LETTER_GOOD and not unrelated:
|
| 541 |
+
letter.update(letter_reason(toks[worst].sound, heard_as[worst], checks[worst]))
|
| 542 |
letters.append(letter)
|
| 543 |
if w in forgiven and min(ws) >= self.cfg.issue_below:
|
| 544 |
heard_words[w] = tr_words[w] # what differs from the answer here is what a native voice gets too
|
|
|
|
| 552 |
overall = 0.5 * phone_mean + 0.5 * word_mean # one wrong consonant must pull the sentence down
|
| 553 |
if unrelated:
|
| 554 |
overall = min(overall, 30.0)
|
| 555 |
+
# most crucial first: the mistakes that change meaning (severity 3), then the worst-scoring sounds
|
| 556 |
+
max_tips = self.cfg.max_tips or int(self.rules.scoring.get("max_tips", 4))
|
| 557 |
+
tips = sorted(tips, key=lambda t: (-t["severity"], t["score"]))[:max_tips]
|
| 558 |
+
tip_idx, issue_idx = {t["idx"] for t in tips}, []
|
| 559 |
|
| 560 |
# Anything short of 100 must say why: every imprecise sound that did not get a full tip,
|
| 561 |
# weakest first. If nothing is clearly off, name the weakest few.
|
|
|
|
| 564 |
tipped = {t.pop("idx") for t in tips}
|
| 565 |
rest = sorted((i for i in range(len(toks)) if i not in tipped and scores[i].score < 100),
|
| 566 |
key=lambda i: scores[i].score)
|
| 567 |
+
picked = [i for i in rest if scores[i].score < self.cfg.issue_below and beyond[i]]
|
| 568 |
if not picked and not tips:
|
| 569 |
picked = rest[:3]
|
| 570 |
+
issue_idx = picked[:self.cfg.max_issues]
|
| 571 |
+
for i in issue_idx:
|
| 572 |
tok, sc = toks[i], scores[i]
|
| 573 |
+
comp = i2t.get(sc.competitor) if sc.competitor is not None else None
|
| 574 |
+
issues.append({"sound": sound_name(tok.sound) or sound_name(tok.phoneme) or tok.phoneme,
|
| 575 |
+
"word_tr": tr_words[tok.word], "score": round(sc.score), "missing": bool(sc.deleted),
|
| 576 |
"heard": None if sc.deleted else sound_name(comp)})
|
| 577 |
for t in tips:
|
| 578 |
t.pop("idx", None)
|
| 579 |
result = {"score": round(overall), "variant": {"ar": ar, "tr": " ".join(tr_words), "ipa": ipa}, "heard_ipa": heard, "heard_tr": " ".join(h for h in heard_words if h),
|
| 580 |
+
"unrelated": unrelated, "words": words, "tips": tips, "issues": issues, "references": len(refs),
|
| 581 |
+
**self._dialect(toks, scores)}
|
| 582 |
if calibrated:
|
| 583 |
result["calibrated"] = calibrated
|
| 584 |
if heatmap is not None:
|
| 585 |
result["phoneme_heatmap"] = heatmap
|
| 586 |
+
if diagnostics:
|
| 587 |
+
result["trace"] = self._trace(fit, refs, bases, after_baseline, floors, scores, phonemes, checks, beyond,
|
| 588 |
+
words, tips, tip_idx, set(issue_idx), phone_mean, word_mean, overall, heard)
|
| 589 |
return result
|
| 590 |
|
| 591 |
+
def _trace(self, fit: Fit, refs, bases, after_baseline, floors, scores, phonemes, checks, beyond,
|
| 592 |
+
words, tips, tip_idx, issue_idx, phone_mean, word_mean, overall, heard) -> dict:
|
| 593 |
+
"""Every stage of the classification with its numbers, weights and thresholds, for the teacher's view
|
| 594 |
+
(the 'stages' part of the phoneme analysis). Nothing here changes the result."""
|
| 595 |
+
cfg, g, i2t = self.cfg, self.cfg.gop, self.vocab.i2t
|
| 596 |
+
p = lambda lp: round(float(np.exp(lp)), 4) if lp > -1e20 else 0.0
|
| 597 |
+
max_tips = cfg.max_tips or int(self.rules.scoring.get("max_tips", 4))
|
| 598 |
+
n = len(fit.toks)
|
| 599 |
+
tokens = []
|
| 600 |
+
for i, (tok, gsc, sc) in enumerate(zip(fit.toks, fit.gop_scores, scores)):
|
| 601 |
+
heard_ph = "_" if gsc.deleted else (i2t.get(gsc.competitor) if gsc.competitor is not None else None)
|
| 602 |
+
members = self.vocab.members.get(tok.id)
|
| 603 |
+
targets = [tok.sound, tok.phoneme]
|
| 604 |
+
confidence = min(1.0, p(gsc.peak) / g.confident_peak)
|
| 605 |
+
check = checks[i]
|
| 606 |
+
b = bases[i]
|
| 607 |
+
stages = [
|
| 608 |
+
{"stage": "gop", "score": gsc.score},
|
| 609 |
+
{"stage": "exemption", "score": fit.scores[i].score, "applied": fit.exempted[i]},
|
| 610 |
+
{"stage": "baseline", "score": after_baseline[i].score, "ref": b,
|
| 611 |
+
"applied": b is not None and b < 100},
|
| 612 |
+
{"stage": "calibration", "score": sc.score, "floor": floors[i],
|
| 613 |
+
"applied": floors[i] is not None and after_baseline[i].score < floors[i]},
|
| 614 |
+
]
|
| 615 |
+
tokens.append({
|
| 616 |
+
"i": i, "word": tok.word, "ipa": tok.phoneme, "sound": tok.sound,
|
| 617 |
+
"class": sorted(base(i2t[m]) for m in members) if members else None,
|
| 618 |
+
"realised": base(i2t[gsc.realised]) if gsc.realised is not None else None,
|
| 619 |
+
"span": list(fit.al.spans[i]), "window": list(gsc.window),
|
| 620 |
+
"p_target": p(gsc.peak), "competitor": heard_ph if not gsc.deleted else None,
|
| 621 |
+
"competitor_other": gsc.other, "p_competitor": p(gsc.comp_peak),
|
| 622 |
+
"gop": gsc.gop, "confidence": round(confidence, 3),
|
| 623 |
+
"gop_factor": round(float(np.exp(gsc.gop * g.leniency)), 3),
|
| 624 |
+
"confidence_factor": round(float(np.sqrt(confidence)), 3),
|
| 625 |
+
"deleted": gsc.deleted, "stages": stages, "final": sc.score,
|
| 626 |
+
"margin": {"needed": self.rules.margin(targets, "_" if gsc.deleted else heard_ph),
|
| 627 |
+
"got": (round(g.deleted_peak - gsc.peak, 3) if gsc.deleted else round(-gsc.gop, 3)),
|
| 628 |
+
"beyond": beyond[i]},
|
| 629 |
+
"vowel": tok.phoneme in VOWELS, "weight": 0.5 if tok.phoneme in VOWELS else 1.0,
|
| 630 |
+
"flagged": phonemes[i]["flagged"],
|
| 631 |
+
"rule": {"id": check.rule, "title": self.rules.titles.get(check.rule, check.rule),
|
| 632 |
+
"severity": check.severity} if check else None,
|
| 633 |
+
"tip": i in tip_idx, "issue": i in issue_idx,
|
| 634 |
+
})
|
| 635 |
+
return {
|
| 636 |
+
"config": {
|
| 637 |
+
"leniency": g.leniency, "confident_peak": g.confident_peak,
|
| 638 |
+
"deleted_peak": round(float(np.exp(g.deleted_peak)), 3),
|
| 639 |
+
"deleted_comp": round(float(np.exp(g.deleted_comp)), 3),
|
| 640 |
+
"flag_below": cfg.flag_below, "issue_below": cfg.issue_below, "near_vowel_floor": cfg.near_vowel_floor,
|
| 641 |
+
"word_min_weight": cfg.word_min_weight, "vowel_weight": 0.5, "unrelated_ratio": cfg.unrelated_ratio,
|
| 642 |
+
"unrelated_cap": 30.0, "letter_good": LETTER_GOOD, "max_tips": max_tips, "max_issues": cfg.max_issues,
|
| 643 |
+
"ref_min_mean": cfg.ref_min_mean, "margin_default": float(self.rules.scoring.get("margin_default", 0.0)),
|
| 644 |
+
"margins": [{"target": t, "heard": h, "margin": m} for (t, h), m in sorted(self.rules.margins.items())],
|
| 645 |
+
"exemptions": [name for name, _, _ in EXEMPTIONS],
|
| 646 |
+
},
|
| 647 |
+
"variants": [{"ar": ar, "ipa": ipa, "logp": lp, "chosen": ar == fit.ar and ipa == fit.ipa}
|
| 648 |
+
for ar, ipa, lp in fit.candidates],
|
| 649 |
+
"aipa": fit.aipa,
|
| 650 |
+
"unrelated": {"heard": heard, "distance": fit.distance, "tokens": n,
|
| 651 |
+
"ratio": round(fit.distance / max(1, n), 3), "threshold": cfg.unrelated_ratio,
|
| 652 |
+
"unrelated": fit.unrelated},
|
| 653 |
+
"references": [{"mean": round(float(np.mean(r.scores)), 1), "sounds": len(r.sounds)} for r in refs],
|
| 654 |
+
"tokens": tokens,
|
| 655 |
+
"words": [{"tr": w["tr"], "scores": [ph["score"] for ph in w["phonemes"]],
|
| 656 |
+
"min": min((ph["score"] for ph in w["phonemes"]), default=0.0),
|
| 657 |
+
"mean": round(float(np.mean([ph["score"] for ph in w["phonemes"]])), 1) if w["phonemes"] else 0.0,
|
| 658 |
+
"score": w["score"]} for w in words],
|
| 659 |
+
"sentence": {"phone_mean": round(phone_mean, 1), "word_mean": round(word_mean, 1),
|
| 660 |
+
"phone_weight": 0.5, "word_weight": 0.5,
|
| 661 |
+
"before_cap": round(0.5 * phone_mean + 0.5 * word_mean, 1), "final": round(overall),
|
| 662 |
+
"capped": fit.unrelated},
|
| 663 |
+
"tips": [{"key": t["key"], "severity": t["severity"], "score": t["score"]} for t in tips],
|
| 664 |
+
}
|
| 665 |
+
|
| 666 |
|
| 667 |
# ---------- real model ----------
|
| 668 |
|
|
|
|
| 717 |
# resolve every phoneme once at startup so missing symbols show up in the logs and /api/health
|
| 718 |
from .phrases import PHRASES
|
| 719 |
for p in PHRASES:
|
| 720 |
+
for ar, ipa in p["variants"]:
|
| 721 |
+
tokenize(archify(ar, ipa), vocab)
|
| 722 |
if vocab.downgraded:
|
| 723 |
log.warning("phonemes mapped to fallbacks: %s", vocab.downgraded)
|
| 724 |
+
if vocab.class_missing:
|
| 725 |
+
log.warning("realisations the model cannot hear, left out of their letter's class: %s", vocab.class_missing)
|
| 726 |
return scorer
|
app/tips.py
CHANGED
|
@@ -5,6 +5,8 @@ Phonemes are in the IPA used by phrases.py; HEARD_ALIASES maps model outputs ont
|
|
| 5 |
"""
|
| 6 |
import re
|
| 7 |
|
|
|
|
|
|
|
| 8 |
NAMES = {
|
| 9 |
"ħ": "ח (ح)", "ʕ": "ע (ع)", "x": "ח' (خ)", "ɣ": "ע' (غ)", "r": "ר (ر)",
|
| 10 |
"q": "ק (ق)", "ʔ": "עצירה סותמית (ء / ق)", "h": "ה (ه)",
|
|
@@ -12,6 +14,9 @@ NAMES = {
|
|
| 12 |
"k": "כ (ك)", "s": "ס (س)", "t": "ת (ت)", "d": "ד (د)", "z": "ז (ز)",
|
| 13 |
"ʃ": "ש (ش)", "dʒ": "ג' (ج)", "b": "ב (ب)", "f": "פ (ف)", "m": "מ (م)",
|
| 14 |
"n": "נ (ن)", "l": "ל (ل)", "w": "ו (و)", "j": "י (ي)",
|
|
|
|
|
|
|
|
|
|
| 15 |
}
|
| 16 |
|
| 17 |
# model output symbol -> symbol used in TIPS keys
|
|
@@ -49,6 +54,7 @@ TIPS = {
|
|
| 49 |
|
| 50 |
# bare Arabic letter in a tip text -> "Hebrew (Arabic)", so Arabic never appears without a Hebrew reading
|
| 51 |
_PAIRED = {name[name.index("(") + 1]: name for name in NAMES.values() if "/" not in name}
|
|
|
|
| 52 |
_BARE_AR = re.compile(r"(?<!\()[\u0621-\u064a](?![\u0621-\u064a])")
|
| 53 |
|
| 54 |
|
|
@@ -64,14 +70,17 @@ VOWEL_NAMES = {
|
|
| 64 |
|
| 65 |
|
| 66 |
def sound_name(phoneme: str | None) -> str | None:
|
| 67 |
-
"""Hebrew name of a phoneme (expected or heard), or None if we have no name for it.
|
|
|
|
|
|
|
|
|
|
| 68 |
phoneme = HEARD_ALIASES.get(phoneme, phoneme)
|
| 69 |
-
return NAMES.get(phoneme) or VOWEL_NAMES.get(phoneme)
|
| 70 |
|
| 71 |
|
| 72 |
-
def letter_reason(expected: str, heard: str | None) -> dict:
|
| 73 |
"""Why a letter of the answer is coloured: {"why": the sound and what was heard in its place ("_": nothing was),
|
| 74 |
-
"tip": how to say it,
|
| 75 |
name = sound_name(expected) or expected
|
| 76 |
heard = HEARD_ALIASES.get(heard, heard)
|
| 77 |
other = sound_name(heard)
|
|
@@ -81,27 +90,34 @@ def letter_reason(expected: str, heard: str | None) -> dict:
|
|
| 81 |
why = f"{name} נשמעה כמו {other}"
|
| 82 |
else:
|
| 83 |
why = f"{name} לא נשמעה בבירור"
|
| 84 |
-
text = TIPS.get((expected, heard))
|
| 85 |
return {"why": why, **({"tip": with_hebrew(text)} if text else {})}
|
| 86 |
|
| 87 |
|
| 88 |
VOWELS = {"a", "i", "u", "e", "o", "aː", "iː", "uː", "eː", "oː"}
|
| 89 |
-
# the
|
| 90 |
-
NO_DELETION_TIP = {"ʔ"}
|
| 91 |
|
| 92 |
|
| 93 |
-
def tip_for(expected: str, heard: str | None) -> dict | None:
|
| 94 |
-
"""Feedback for one flagged phoneme, or None if there is nothing useful to say.
|
|
|
|
|
|
|
|
|
|
| 95 |
if expected in VOWELS:
|
| 96 |
return None
|
| 97 |
heard = HEARD_ALIASES.get(heard, heard) if heard else "_"
|
| 98 |
if heard == "_" and expected in NO_DELETION_TIP:
|
| 99 |
return None
|
| 100 |
name = NAMES.get(expected, expected)
|
|
|
|
|
|
|
|
|
|
| 101 |
text = TIPS.get((expected, heard))
|
| 102 |
if text:
|
| 103 |
-
return {"title": name, "text": with_hebrew(text), "key":
|
| 104 |
if heard == "_":
|
| 105 |
-
return {"title": name, "text": "לא נשמעה בבירור.", "key":
|
| 106 |
-
|
| 107 |
-
|
|
|
|
|
|
| 5 |
"""
|
| 6 |
import re
|
| 7 |
|
| 8 |
+
from .rules import load as load_rules
|
| 9 |
+
|
| 10 |
NAMES = {
|
| 11 |
"ħ": "ח (ح)", "ʕ": "ע (ع)", "x": "ח' (خ)", "ɣ": "ע' (غ)", "r": "ר (ر)",
|
| 12 |
"q": "ק (ق)", "ʔ": "עצירה סותמית (ء / ق)", "h": "ה (ه)",
|
|
|
|
| 14 |
"k": "כ (ك)", "s": "ס (س)", "t": "ת (ت)", "d": "ד (د)", "z": "ז (ز)",
|
| 15 |
"ʃ": "ש (ش)", "dʒ": "ג' (ج)", "b": "ב (ب)", "f": "פ (ف)", "m": "מ (م)",
|
| 16 |
"n": "נ (ن)", "l": "ל (ل)", "w": "ו (و)", "j": "י (ي)",
|
| 17 |
+
"tʃ": "צ' (tʃ)", "θ": "ת' (th)", "ð": "ד״ (th)",
|
| 18 |
+
# letters with several correct realisations (archi.py): named by the letter, not by one way of saying it
|
| 19 |
+
"Q": "ק (ق)", "K": "כ (ك)", "TH": "ת' (ث)", "DH": "ד״ (ذ)", "ZH": "ט' (ظ)", "J": "ג' (ج)",
|
| 20 |
}
|
| 21 |
|
| 22 |
# model output symbol -> symbol used in TIPS keys
|
|
|
|
| 54 |
|
| 55 |
# bare Arabic letter in a tip text -> "Hebrew (Arabic)", so Arabic never appears without a Hebrew reading
|
| 56 |
_PAIRED = {name[name.index("(") + 1]: name for name in NAMES.values() if "/" not in name}
|
| 57 |
+
DEFAULT_SEVERITY = 2 # a tip no rule in pronunciation-rules.md grades
|
| 58 |
_BARE_AR = re.compile(r"(?<!\()[\u0621-\u064a](?![\u0621-\u064a])")
|
| 59 |
|
| 60 |
|
|
|
|
| 70 |
|
| 71 |
|
| 72 |
def sound_name(phoneme: str | None) -> str | None:
|
| 73 |
+
"""Hebrew name of a phoneme (expected or heard), or None if we have no name for it.
|
| 74 |
+
Sounds Arabic does not have (p, v, ts) are named by the labels of pronunciation-rules.md."""
|
| 75 |
+
if phoneme is None:
|
| 76 |
+
return None
|
| 77 |
phoneme = HEARD_ALIASES.get(phoneme, phoneme)
|
| 78 |
+
return NAMES.get(phoneme) or VOWEL_NAMES.get(phoneme) or load_rules().label(phoneme)
|
| 79 |
|
| 80 |
|
| 81 |
+
def letter_reason(expected: str, heard: str | None, check=None) -> dict:
|
| 82 |
"""Why a letter of the answer is coloured: {"why": the sound and what was heard in its place ("_": nothing was),
|
| 83 |
+
"tip": how to say it, from the rule for that substitution (`check`, rules.Check) or TIPS}."""
|
| 84 |
name = sound_name(expected) or expected
|
| 85 |
heard = HEARD_ALIASES.get(heard, heard)
|
| 86 |
other = sound_name(heard)
|
|
|
|
| 90 |
why = f"{name} נשמעה כמו {other}"
|
| 91 |
else:
|
| 92 |
why = f"{name} לא נשמעה בבירור"
|
| 93 |
+
text = check.feedback if check else TIPS.get((expected, heard))
|
| 94 |
return {"why": why, **({"tip": with_hebrew(text)} if text else {})}
|
| 95 |
|
| 96 |
|
| 97 |
VOWELS = {"a", "i", "u", "e", "o", "aː", "iː", "uː", "eː", "oː"}
|
| 98 |
+
# the glottal stop (ء, and ق said the urban way) is often dropped by natives too; don't nag about it
|
| 99 |
+
NO_DELETION_TIP = {"ʔ", "Q"}
|
| 100 |
|
| 101 |
|
| 102 |
+
def tip_for(expected: str, heard: str | None, check=None) -> dict | None:
|
| 103 |
+
"""Feedback for one flagged phoneme, or None if there is nothing useful to say.
|
| 104 |
+
|
| 105 |
+
`check` (rules.Check) is the rule of pronunciation-rules.md for this substitution: its feedback and
|
| 106 |
+
severity win over TIPS. The key stays "expected>heard" either way."""
|
| 107 |
if expected in VOWELS:
|
| 108 |
return None
|
| 109 |
heard = HEARD_ALIASES.get(heard, heard) if heard else "_"
|
| 110 |
if heard == "_" and expected in NO_DELETION_TIP:
|
| 111 |
return None
|
| 112 |
name = NAMES.get(expected, expected)
|
| 113 |
+
key = f"{expected}>{heard}"
|
| 114 |
+
if check:
|
| 115 |
+
return {"title": name, "text": with_hebrew(check.feedback), "key": key, "severity": check.severity}
|
| 116 |
text = TIPS.get((expected, heard))
|
| 117 |
if text:
|
| 118 |
+
return {"title": name, "text": with_hebrew(text), "key": key, "severity": DEFAULT_SEVERITY}
|
| 119 |
if heard == "_":
|
| 120 |
+
return {"title": name, "text": "לא נשמעה בבירור.", "key": key, "severity": DEFAULT_SEVERITY}
|
| 121 |
+
other = sound_name(heard)
|
| 122 |
+
text = f"נשמעה כמו {other}." if other else "נשמעה אחרת מהצפוי."
|
| 123 |
+
return {"title": name, "text": text, "key": key, "severity": DEFAULT_SEVERITY}
|
app/translit.py
CHANGED
|
@@ -10,7 +10,12 @@ CONSONANTS = {
|
|
| 10 |
"ʔ": "א", "b": "בּ", "t": "ת", "dʒ": "ג'", "ħ": "ח", "x": "ח'", "d": "ד", "r": "ר", "z": "ז",
|
| 11 |
"s": "ס", "ʃ": "ש", "sˤ": "צ", "dˤ": "ד'", "tˤ": "ט", "ðˤ": "ט'", "ʕ": "ע", "ɣ": "ע'", "f": "פ",
|
| 12 |
"q": "ק", "k": "כּ", "g": "ג", "l": "ל", "m": "מ", "n": "נ", "h": "ה", "w": "ו", "j": "י",
|
|
|
|
|
|
|
| 13 |
}
|
|
|
|
|
|
|
|
|
|
| 14 |
# vowel -> ((mark on the consonant, letter after it) mid-word, (same) word-finally)
|
| 15 |
VOWELS = {
|
| 16 |
"a": (("ַ", ""), ("ַ", "א")), "i": (("ִ", ""), ("ִ", "י")), "u": (("ֻ", ""), ("", "וּ")),
|
|
@@ -23,8 +28,8 @@ SHADDA, SHVA = "\u0651", "\u05b0"
|
|
| 23 |
|
| 24 |
|
| 25 |
def _letter(base: str, marks: str) -> str:
|
| 26 |
-
"""Points go on the letter itself, before a geresh."""
|
| 27 |
-
return base[0] + marks + base[1:] if
|
| 28 |
|
| 29 |
|
| 30 |
def word_letters(ipa_word: str, ar_word: str = "") -> list[tuple[str, list[int]]]:
|
|
@@ -51,7 +56,7 @@ def word_letters(ipa_word: str, ar_word: str = "") -> list[tuple[str, list[int]]
|
|
| 51 |
if ph in VOWELS: # vowel with no consonant before it (word-initial): carry it on an alef
|
| 52 |
base, shadda = "א", ""
|
| 53 |
else:
|
| 54 |
-
base, shadda = CONSONANTS.get(ph, ph), ""
|
| 55 |
i += 1
|
| 56 |
if i < len(phs) and phs[i] == ph: # geminate
|
| 57 |
shadda = SHADDA
|
|
|
|
| 10 |
"ʔ": "א", "b": "בּ", "t": "ת", "dʒ": "ג'", "ħ": "ח", "x": "ח'", "d": "ד", "r": "ר", "z": "ז",
|
| 11 |
"s": "ס", "ʃ": "ש", "sˤ": "צ", "dˤ": "ד'", "tˤ": "ט", "ðˤ": "ט'", "ʕ": "ע", "ɣ": "ע'", "f": "פ",
|
| 12 |
"q": "ק", "k": "כּ", "g": "ג", "l": "ל", "m": "מ", "n": "נ", "h": "ה", "w": "ו", "j": "י",
|
| 13 |
+
# rural realisations (pronunciation-rules.md, section 1)
|
| 14 |
+
"tʃ": "צ'", "θ": "ת'", "ð": "ד״",
|
| 15 |
}
|
| 16 |
+
# Sounds Arabic does not have, written only in the line of what was heard, never in an answer: a Hebrew
|
| 17 |
+
# speaker's p for ف, v for و / ب, and Hebrew צ (ts) for ص. Without them the heard line would drop the letter.
|
| 18 |
+
HEARD_ONLY = {"p": "פּ", "v": "ב", "β": "ב", "ʋ": "ב", "ts": "טס"}
|
| 19 |
# vowel -> ((mark on the consonant, letter after it) mid-word, (same) word-finally)
|
| 20 |
VOWELS = {
|
| 21 |
"a": (("ַ", ""), ("ַ", "א")), "i": (("ִ", ""), ("ִ", "י")), "u": (("ֻ", ""), ("", "וּ")),
|
|
|
|
| 28 |
|
| 29 |
|
| 30 |
def _letter(base: str, marks: str) -> str:
|
| 31 |
+
"""Points go on the letter itself, before a geresh (or gershayim), and on the first letter of טס."""
|
| 32 |
+
return base[0] + marks + base[1:] if len(base) > 1 and base[1] != "\u05bc" else base + marks
|
| 33 |
|
| 34 |
|
| 35 |
def word_letters(ipa_word: str, ar_word: str = "") -> list[tuple[str, list[int]]]:
|
|
|
|
| 56 |
if ph in VOWELS: # vowel with no consonant before it (word-initial): carry it on an alef
|
| 57 |
base, shadda = "א", ""
|
| 58 |
else:
|
| 59 |
+
base, shadda = CONSONANTS.get(ph) or HEARD_ONLY.get(ph, ph), ""
|
| 60 |
i += 1
|
| 61 |
if i < len(phs) and phs[i] == ph: # geminate
|
| 62 |
shadda = SHADDA
|
docs/scoring-dialects-plan.md
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Plan: dialect acceptance and Hebrew-speaker rules in the scorer
|
| 2 |
+
|
| 3 |
+
Status: implemented 2026-10-15, except where "Implementation notes" at the end says otherwise. Source rules: [pronunciation-rules.md](../static/assets/pronunciation-rules.md).
|
| 4 |
+
|
| 5 |
+
Two layers on top of today's scoring path:
|
| 6 |
+
|
| 7 |
+
- **Layer A, dialect acceptance:** decides what counts as *correct*. Runs inside GOP: every correct realisation of a letter is pooled into one class, and members of the class never count as competitors.
|
| 8 |
+
- **Layer B, Hebrew-speaker rules:** decides how strictly to judge *real errors*. Runs after GOP: per-pair margins, severity, feedback, exemptions.
|
| 9 |
+
|
| 10 |
+
The native baseline and the teacher calibration floors stay where they are, but are keyed on the abstract symbol instead of the sound actually said.
|
| 11 |
+
|
| 12 |
+
Final order of the pipeline:
|
| 13 |
+
|
| 14 |
+
1. `Vocab.merge` pools spelling variants (exists)
|
| 15 |
+
2. Layer A: class columns → alignment → GOP against the class
|
| 16 |
+
3. Layer B exemptions (6.3)
|
| 17 |
+
4. Native baseline normalisation (exists, keyed on the abstract symbol)
|
| 18 |
+
5. Layer B margins, severity, feedback
|
| 19 |
+
6. Teacher DB floors (exists, last, manual override)
|
| 20 |
+
|
| 21 |
+
## Phase 0: Rule loader (`app/rules.py`, new)
|
| 22 |
+
|
| 23 |
+
- Pull the YAML blocks out of `pronunciation-rules.md` (fenced ` ```yaml `, split on `---`). Load them into:
|
| 24 |
+
- `archiphonemes`, `dialects_enabled`, `optional_realisations_enabled`
|
| 25 |
+
- `scoring` (margins, floors)
|
| 26 |
+
- `checks` (target, heard, severity, feedback), collected from every rule block
|
| 27 |
+
- `heard_labels`
|
| 28 |
+
- Pass every symbol through `base()` when loading, so `ʁ` is stored as `ɣ`, `χ` as `x`, and so on. Warn about rules that become **undetectable**:
|
| 29 |
+
- target and heard end up the same sound after `base()`;
|
| 30 |
+
- a heard symbol is in the same accepted set as its target;
|
| 31 |
+
- a symbol is missing from the model vocab.
|
| 32 |
+
- Add `pyyaml` to `requirements.txt`. It's already installed through `huggingface_hub`, but it should be listed explicitly.
|
| 33 |
+
- Consider moving the rules file to `docs/` as the doc itself suggests. The student sections are served from `static/assets/`, so either keep it there or copy it at build time.
|
| 34 |
+
- **Tests:** every block parses; every `heard:` symbol has a label in section 8; the undetectable-rule check fires on a deliberately bad rule.
|
| 35 |
+
|
| 36 |
+
## Phase 1: Layer A core, with `Q` only
|
| 37 |
+
|
| 38 |
+
### 1a. Converting the phrases at load time, not in the source files
|
| 39 |
+
|
| 40 |
+
`app/workbook.py` has ~930 ق and `app/grammar.py` has more, so rewriting the IPA by hand isn't realistic.
|
| 41 |
+
|
| 42 |
+
- Add `archify(ar, ipa)` in a new `app/archi.py`. For each word whose Arabic contains ق, replace the matching `ʔ`/`q`/`k` with `Q`.
|
| 43 |
+
- Match by consonant order. `word_letters` in `app/translit.py` already lines up ق with its phoneme, so reuse that logic.
|
| 44 |
+
- **Ambiguous words** (both ء and ق, or ك next to ق) need an explicit marker written in the phrase. Add a test that fails on any ق word it can't resolve.
|
| 45 |
+
- After conversion, merge duplicate variants. Today's separate `q` and `k` variants (e.g. `i l q u d s`) become the same as the `ʔ` one.
|
| 46 |
+
- Run it inside `PHRASES` / `workbook_phrases()` so every consumer sees `Q`, including references and tokenisation at startup in `load_scorer`.
|
| 47 |
+
|
| 48 |
+
### 1b. `Vocab` classes (`app/scorer.py`, `Vocab`)
|
| 49 |
+
|
| 50 |
+
- `Vocab(token_to_id, classes=rules.archiphonemes)` creates one **virtual id** per abstract symbol, numbered after the real vocabulary (V, V+1, …). It also keeps `members[vid]`, the real representative ids of the accepted realisations.
|
| 51 |
+
- `merge()` returns `logp` with extra columns: column `vid` = `logaddexp` over its members.
|
| 52 |
+
- `resolve("Q")` returns `[vid]`. `i2t[vid] = "Q"`.
|
| 53 |
+
- Anything that ranges over "real" symbols has to ignore virtual columns: `greedy_decode`, `_heard_hebrew`, the heatmap's "heard" track, and `edit_distance` for the unrelated check. Easiest is a `vocab.real(logp)` view of `logp[:, :V]`.
|
| 54 |
+
|
| 55 |
+
### 1c. GOP (`app/align.py`, `score_tokens`)
|
| 56 |
+
|
| 57 |
+
- `score_tokens(..., accept: list[set[int]] | None)`. For a class token, mask all its members from the competitor search, and always mask every virtual column.
|
| 58 |
+
- Add `TokenScore.realised`: the member with the highest peak inside the window (`None` for plain tokens).
|
| 59 |
+
- The alignment itself doesn't change. It already runs on the virtual column.
|
| 60 |
+
|
| 61 |
+
### 1d. Results and display
|
| 62 |
+
|
| 63 |
+
- Each phoneme entry gets `realised`. `heard` comes from the competitor as before, so an accepted realisation never shows up as "heard".
|
| 64 |
+
- `_heard_hebrew`: when the frame is inside a class token's span and the greedy symbol is one of its members, write the **realised** letter (כּ, א or ק) uncoloured.
|
| 65 |
+
- `to_hebrew` / `word_letters`: `Q` is written ק in the answer (today's ق rule already does this).
|
| 66 |
+
- `app/tips.py`: add `Q` to `NO_DELETION_TIP` (6.3 #2). Remove the `ʔ>k` qaf tip.
|
| 67 |
+
|
| 68 |
+
### 1e. Baseline and calibrations
|
| 69 |
+
|
| 70 |
+
- References need no change. `Reference.sounds` now holds `(w, "Q")`, so the baseline is the same whatever realisation the TTS voice used. The cache is in memory and rebuilds on restart.
|
| 71 |
+
- Calibration lookup in `score_logp`: try `(archi, heard)` first, then fall back to `(realised target, heard)` so existing DB rows still apply. Add a teacher-panel note or a one-off SQL to re-key `ʔ` rows that came from ق phrases.
|
| 72 |
+
|
| 73 |
+
### 1f. Tests (`tests/test_scoring.py`)
|
| 74 |
+
|
| 75 |
+
- Rewrite `test_k_for_urban_qaf_picks_closest_variant_and_tips`: `k` for `Q` scores ≥ 90, has no tip, and `realised == "k"`.
|
| 76 |
+
- New tests:
|
| 77 |
+
- `q` for `Q` is accepted;
|
| 78 |
+
- `Q` dropped between vowels is not flagged;
|
| 79 |
+
- `ʔ` from ء said as `k` **is** flagged (e.g. اهلا → kahla);
|
| 80 |
+
- a reference voiced with `q` doesn't penalise a learner who says `ʔ`;
|
| 81 |
+
- `archify` on a sample of workbook phrases, plus the ambiguity test.
|
| 82 |
+
|
| 83 |
+
## Phase 2: The other five abstract symbols
|
| 84 |
+
|
| 85 |
+
- Extend `archify`: ك→`K`, ث→`TH`, ذ→`DH`, ظ→`ZH`, ج→`J`.
|
| 86 |
+
- **Per-word override syntax** for learned words, e.g. `TH{t,s}` (ثقافة) or a plain `t` to lock it. Parse it in `tokenize`, which creates a one-off class for that position.
|
| 87 |
+
- **Vocab gaps:** if `θ`, `ð`, `ðˤ` or `tʃ` aren't in the model, the class quietly shrinks. Log it, and expose it in `/api/health` under `rules` (section 7 of the rules doc).
|
| 88 |
+
- Add `tʃ` (צ'), `θ` (ת'), `ð` (ד״) to `CONSONANTS` in `app/translit.py` so rural realisations can be written in the heard line.
|
| 89 |
+
- Add `θ`, `ð`, `tʃ`, `ðˤ`, `dʒ` to the test vocab `SYMS`. Tests for each class, plus one mixed-dialect sentence that isn't penalised.
|
| 90 |
+
|
| 91 |
+
## Phase 3: Positional classes and diphthongs
|
| 92 |
+
|
| 93 |
+
- **`E`** for a final ة/ه vowel → {a, e, i}. Derived automatically from the Arabic ending (`word_letters` already detects it).
|
| 94 |
+
- **`O`** for the vowel of a pronoun ending → {o, u}. This one needs an explicit marker in the phrase, because it can't be told reliably from the Arabic.
|
| 95 |
+
- **`EY` / `OW`**: expand into extra **variants** (`eː` | `a j`, `oː` | `a w`) inside `archify`, before variants are merged. `_fit` already picks the best variant. These can't be a per-token class, because one token stands against two.
|
| 96 |
+
- `near_vowels` stays for other positions until the vowel-length decision is made (see "Open decisions").
|
| 97 |
+
- Sounds that natives also drop (ʔ at the start or between vowels, il- after a vowel) are **not** Layer A: an optional token would need a separate alignment with and without it. They stay as Layer B exemptions.
|
| 98 |
+
|
| 99 |
+
## Phase 4: Layer B, the rule engine
|
| 100 |
+
|
| 101 |
+
### 4a. Competitor set
|
| 102 |
+
|
| 103 |
+
- Limit competitors to the Arabic sounds plus the Hebrew-interference sounds (every `heard:` symbol in `checks`). Report any other winner as `competitor=None, other=True` ("not clear").
|
| 104 |
+
- **Watch out:** this lowers `comp_peak` in the deletion test (`deleted_comp`). Re-check the deletion tests.
|
| 105 |
+
|
| 106 |
+
### 4b. Flagging by margin
|
| 107 |
+
|
| 108 |
+
- In `score_logp`, flag a sound when `score < flag_below` **and** the competitor is outside the target's accepted set **and** `−gop ≥ margin(target, heard)`. `gop` is already `log P(target) − log P(competitor)`.
|
| 109 |
+
- The issues list keeps its threshold (`issue_below`) but also respects the margin.
|
| 110 |
+
|
| 111 |
+
### 4c. Feedback
|
| 112 |
+
|
| 113 |
+
- `tip_for` looks up `checks[(target, heard)]` first, then falls back to `TIPS`. Keep the key format `expected>heard` so the front end doesn't change.
|
| 114 |
+
- Sort tips by severity, then by score. `max_tips` comes from the rules file.
|
| 115 |
+
|
| 116 |
+
### 4d. Exemptions (6.3)
|
| 117 |
+
|
| 118 |
+
- Move `NO_DELETION_TIP`, `elided_article` and `near_vowels` into a predicate registry in `rules.py`: `(name, applies(toks, i, ar_words, sc) -> bool, action)`.
|
| 119 |
+
- Add the doc's cases 1–3, 4 (il- not assimilated) and 12 (`mixed_dialect` as info only).
|
| 120 |
+
|
| 121 |
+
### 4e. Letters that can only be heard, never expected
|
| 122 |
+
|
| 123 |
+
- A `HEARD_ONLY` table in `translit.py`: `p`→פּ, `v`→ב/ו, `ts`→צ. Include it in `HEBREW_SOUNDS` so `_heard_hebrew` stops dropping these sounds. Today a student who says *pinjaan* sees the first letter missing instead of פּ.
|
| 124 |
+
- `sound_name` / `letter_reason` labels come from `heard_labels`.
|
| 125 |
+
- **Tests:** `f→p` (פִנְג'אן), `w→v`, `b→v`, `sˤ→ts`, `r→ʁ`. Emphatic just under the margin is not flagged, just over it is.
|
| 126 |
+
|
| 127 |
+
## Phase 5: Teacher-facing parts
|
| 128 |
+
|
| 129 |
+
- `dialects_enabled` / `optional_realisations_enabled` as teacher settings, stored wherever teacher settings live today. Pass them into `Scorer.score` per request, so different classes can differ.
|
| 130 |
+
- The taught dialect only changes hints and info, never what's accepted.
|
| 131 |
+
- Teacher panel: show `realised` and `mixed_dialect` in the diagnostics view. Show abstract symbols by their Hebrew letter, never as "Q".
|
| 132 |
+
- `/api/health`: a `rules` field listing each rule and whether it can be detected.
|
| 133 |
+
|
| 134 |
+
## Phase 6: Rollout check
|
| 135 |
+
|
| 136 |
+
- If recordings are stored, add a `scripts/rescore.py` that compares old and new scores per phrase. Review the biggest changes before switching over, especially ק phrases and emphatics.
|
| 137 |
+
- Ship Phase 1 on its own: it fixes ק/א and tests the class mechanism end to end.
|
| 138 |
+
|
| 139 |
+
## Open decisions (outside this plan)
|
| 140 |
+
|
| 141 |
+
1. **Vowel length:** the doc's `vowel_length` checks (severity 1, margin 1.1) vs. today's `near_vowel_floor`, which forgives length completely.
|
| 142 |
+
2. **Sentence formula:** the doc's `mean(word_min)` and severity weights vs. today's 0.5/0.5 blend. Keep it behind a config flag until decided.
|
| 143 |
+
3. **Bedouin `g` for `Q`:** on or off by default.
|
| 144 |
+
4. **Extra sounds:** inserted sounds aren't detected, since forced alignment only scores expected sounds.
|
| 145 |
+
|
| 146 |
+
## Main risks
|
| 147 |
+
|
| 148 |
+
- The virtual columns change the `logp` width. Anything that indexes `logp` by a real id is fine, but anything that iterates over all columns (heatmap, greedy) must use the real-only view.
|
| 149 |
+
- `archify` mismatches in the workbook. That's why the ambiguity test fails the build instead of guessing.
|
| 150 |
+
- Old calibration rows keyed on `ʔ` until they're re-keyed. That's why the lookup falls back to the old key.
|
| 151 |
+
|
| 152 |
+
## Implementation notes
|
| 153 |
+
|
| 154 |
+
Code: `app/rules.py` (loader, detectability, exemptions), `app/archi.py` (symbols, letter alignment, diphthong variants), `app/phones.py` (`base()`, moved out of the scorer), `app/scorer.py` / `app/align.py` (classes, GOP, margins, tips), `scripts/rescore.py`. Tests: `tests/test_rules.py` and the "dialects and Hebrew-speaker rules" section of `tests/test_scoring.py`.
|
| 155 |
+
|
| 156 |
+
Where it differs from the plan above:
|
| 157 |
+
|
| 158 |
+
- **`archify` runs inside the scorer, not on `PHRASES`.** `Scorer._variants` converts each variant when scoring (cached). Display, TTS, transliteration and the verb game keep the plain IPA. Symbols replace phonemes one for one, so positions in the IPA still line up.
|
| 159 |
+
- **Token format** is `SYMBOL:written` (`Q:ʔ`), or `SYMBOL{a,b}` for a hand-written set. A phrase that writes a symbol itself is left alone.
|
| 160 |
+
- **Ambiguity:** across the bank (~11.6k variants) no ق is unresolved. 5 × ث merged with ت into one geminate (بعثت) and 1 × ج (جوز) stay plain phonemes; only an unresolved ق fails the test.
|
| 161 |
+
- **Learned-only realisations** (`TH` as s, `DH` as z) are accepted where the phrase writes them, with no override syntax needed.
|
| 162 |
+
- **Missing model symbols:** a class member the model lacks uses the same `FALLBACKS` stand-in as a plain phoneme. With the base model, `ZH` is `{dˤ, z}`, so "ظ heard as z" shows as `accepted` (undetectable) in `/api/health` → `rules`.
|
| 163 |
+
- **Competitor naming (4a)** keeps the GOP against the strongest symbol of any kind, so scores and deletions don't change. Only the *name* is withheld (`other`) when that symbol isn't Arabic or a Hebrew-interference sound.
|
| 164 |
+
- **Tips:** `max_tips` is now 3 (from the rules file), sorted by severity then score. Every tip carries `severity`.
|
| 165 |
+
- **Results** gain `phonemes[].sound` (`Q`), `phonemes[].realised` (`k`), and `dialect: urban|rural` or `mixed_dialect: true`. None of them affect the score.
|
| 166 |
+
- **Teacher view:** the expected/heard phone rows compare an accepted realisation with what was said, and show the dialect.
|
| 167 |
+
|
| 168 |
+
Not done:
|
| 169 |
+
|
| 170 |
+
- **Per-teacher dialect settings (Phase 5).** The accepted set comes from the rules file for everyone (`dialects_enabled`, `optional_realisations_enabled`). Since the taught dialect was only meant to change hints, nothing in scoring depends on it yet.
|
| 171 |
+
- **Re-keying old calibration rows.** The lookup falls back to the written phoneme, so old rows keep working. Rows proposed from the page are still keyed on `phonemes[].ipa`.
|
| 172 |
+
- **The open decisions** above (vowel length, sentence formula, Bedouin g, insertions) are unchanged. Vowel-length rules show as undetectable in `/api/health`.
|
requirements.txt
CHANGED
|
@@ -2,6 +2,7 @@ fastapi>=0.110
|
|
| 2 |
uvicorn[standard]>=0.29
|
| 3 |
python-multipart>=0.0.9
|
| 4 |
numpy>=1.26
|
|
|
|
| 5 |
transformers>=4.41
|
| 6 |
huggingface_hub>=0.23
|
| 7 |
# torch: CPU build installed separately in the Dockerfile (see README for local installs)
|
|
|
|
| 2 |
uvicorn[standard]>=0.29
|
| 3 |
python-multipart>=0.0.9
|
| 4 |
numpy>=1.26
|
| 5 |
+
pyyaml>=6.0 # the scorer reads its rules from the yaml blocks of static/assets/pronunciation-rules.md
|
| 6 |
transformers>=4.41
|
| 7 |
huggingface_hub>=0.23
|
| 8 |
# torch: CPU build installed separately in the Dockerfile (see README for local installs)
|
scripts/rescore.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Rescore the flagged recordings with the current scorer and list the biggest changes against the stored score.
|
| 3 |
+
|
| 4 |
+
Usage: python scripts/rescore.py # every flag that kept its recording
|
| 5 |
+
python scripts/rescore.py --status pending,fixed # only these review states
|
| 6 |
+
python scripts/rescore.py --limit 50 --top 20
|
| 7 |
+
python scripts/rescore.py --json out.json # every row, with the old and new sound-level results
|
| 8 |
+
|
| 9 |
+
Reads SUPABASE_URL, SUPABASE_PUBLISHABLE_KEY and SUPABASE_SECRET_KEY (as the server does) and MODEL_ID.
|
| 10 |
+
A flag stores the phrase it was said for, its old result and the recording, so the comparison needs no
|
| 11 |
+
other data. Run it before shipping a scoring change: the rows that moved most are the ones to listen to.
|
| 12 |
+
"""
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import argparse
|
| 16 |
+
import json
|
| 17 |
+
import os
|
| 18 |
+
import sys
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
|
| 21 |
+
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
| 22 |
+
|
| 23 |
+
from dotenv import load_dotenv # noqa: E402
|
| 24 |
+
|
| 25 |
+
from app.auth import load_auth # noqa: E402
|
| 26 |
+
from app.phrases import PHRASES # noqa: E402
|
| 27 |
+
from app.scorer import decode_audio, load_scorer # noqa: E402
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def phrase_for(row: dict) -> dict:
|
| 31 |
+
"""The bank phrase a flag was said for, or (a verb-game or text line) the variant it was scored on."""
|
| 32 |
+
key = row.get("phrase")
|
| 33 |
+
for p in PHRASES:
|
| 34 |
+
if p.get("key", p["he"]) == key:
|
| 35 |
+
return p
|
| 36 |
+
return {"variants": [(row["ar"], row["ipa"])], "key": key, "he": row.get("he", ""), "cat": row.get("cat", "")}
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def sounds(result: dict) -> list[str]:
|
| 40 |
+
"""The flagged sounds of a result, as 'ipa>heard'."""
|
| 41 |
+
return [f"{p.get('sound', p['ipa'])}>{p['heard']}" for w in result.get("words", []) for p in w.get("phonemes", [])
|
| 42 |
+
if p.get("flagged")]
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def main() -> int:
|
| 46 |
+
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
| 47 |
+
ap.add_argument("--status", default="pending,confirmed,rejected,fixed")
|
| 48 |
+
ap.add_argument("--limit", type=int, default=0, help="at most this many flags (0: all)")
|
| 49 |
+
ap.add_argument("--top", type=int, default=30, help="rows to print, biggest change first")
|
| 50 |
+
ap.add_argument("--json", type=Path, help="write every row here")
|
| 51 |
+
args = ap.parse_args()
|
| 52 |
+
|
| 53 |
+
load_dotenv()
|
| 54 |
+
auth = load_auth()
|
| 55 |
+
if auth is None or not auth.secret:
|
| 56 |
+
print("needs SUPABASE_URL, SUPABASE_PUBLISHABLE_KEY and SUPABASE_SECRET_KEY", file=sys.stderr)
|
| 57 |
+
return 2
|
| 58 |
+
scorer = load_scorer(os.environ.get("MODEL_ID", "facebook/wav2vec2-xlsr-53-espeak-cv-ft"))
|
| 59 |
+
flags = [f for s in args.status.split(",") for f in auth.list_flags(s.strip()) if f.get("audio")]
|
| 60 |
+
if args.limit:
|
| 61 |
+
flags = flags[:args.limit]
|
| 62 |
+
|
| 63 |
+
rows = []
|
| 64 |
+
for f in flags:
|
| 65 |
+
audio = auth.flag_audio(f["id"])
|
| 66 |
+
if audio is None:
|
| 67 |
+
continue
|
| 68 |
+
try:
|
| 69 |
+
new = scorer.score(decode_audio(audio[0]), phrase_for(f))
|
| 70 |
+
except (ValueError, KeyError) as e:
|
| 71 |
+
print(f"flag {f['id']}: skipped ({e})", file=sys.stderr)
|
| 72 |
+
continue
|
| 73 |
+
if new.get("error"):
|
| 74 |
+
continue
|
| 75 |
+
old = f.get("result") or {}
|
| 76 |
+
rows.append({"id": f["id"], "status": f.get("status"), "phrase": f.get("he") or f.get("phrase"),
|
| 77 |
+
"old": f.get("score"), "new": new["score"], "delta": new["score"] - (f.get("score") or 0),
|
| 78 |
+
"old_flagged": sounds(old), "new_flagged": sounds(new),
|
| 79 |
+
"dialect": "mixed" if new.get("mixed_dialect") else new.get("dialect", ""),
|
| 80 |
+
"old_result": old, "new_result": new})
|
| 81 |
+
|
| 82 |
+
rows.sort(key=lambda r: -abs(r["delta"]))
|
| 83 |
+
print(f"{len(rows)} flags rescored; mean change {sum(r['delta'] for r in rows) / max(1, len(rows)):+.1f}")
|
| 84 |
+
for r in rows[:args.top]:
|
| 85 |
+
print(f"{r['id']:>6} {r['status'] or '':<9} {r['old']!s:>4} -> {r['new']:>3} ({r['delta']:+4}) "
|
| 86 |
+
f"{r['dialect']:<6} {r['phrase']}")
|
| 87 |
+
if r["old_flagged"] != r["new_flagged"]:
|
| 88 |
+
print(f" flagged: {' '.join(r['old_flagged']) or '-'} -> {' '.join(r['new_flagged']) or '-'}")
|
| 89 |
+
if args.json:
|
| 90 |
+
args.json.write_text(json.dumps(rows, ensure_ascii=False, indent=1), encoding="utf-8")
|
| 91 |
+
return 0
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
if __name__ == "__main__":
|
| 95 |
+
sys.exit(main())
|
static/assets/arabic_verbs.json
ADDED
|
@@ -0,0 +1,451 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"alif": {
|
| 3 |
+
"source": {
|
| 4 |
+
"title": "פעלים בערבית מדוברת המתחילים באליף",
|
| 5 |
+
"url": "https://www.youtube.com/watch?v=c95B23kxOws",
|
| 6 |
+
"dialect": "Palestinian spoken Arabic",
|
| 7 |
+
"notes": "One entry per unique verb, in video order. The video repeats أبطل and أخذ. Phonemes use the scorer alphabet and are space-separated."
|
| 8 |
+
},
|
| 9 |
+
"verbs": [
|
| 10 |
+
{"hebrew": "בא", "timestamp": "01:03", "past": {"arabic": "اجا", "phonemes": "ʔ a dʒ a"}, "future": {"arabic": "يجي", "phonemes": "j i dʒ i"}},
|
| 11 |
+
{"hebrew": "לקח", "timestamp": "01:12", "past": {"arabic": "اخد", "phonemes": "ʔ a x a d"}, "future": {"arabic": "ياخد", "phonemes": "j oː x i d", "accepted_phonemes": ["j aː x u d", "j oː x u d"]}},
|
| 12 |
+
{"hebrew": "חייך", "timestamp": "01:21", "past": {"arabic": "ابتسم", "phonemes": "ʔ i b t a s a m"}, "future": {"arabic": "يبتسم", "phonemes": "j i b t a s i m"}},
|
| 13 |
+
{"hebrew": "דחה", "timestamp": "01:29", "past": {"arabic": "أجّل", "phonemes": "ʔ a dʒ dʒ a l"}, "future": {"arabic": "يأجّل", "phonemes": "j ʔ a dʒ dʒ i l"}},
|
| 14 |
+
{"hebrew": "השכיר", "timestamp": "01:39", "past": {"arabic": "أجّر", "phonemes": "ʔ a dʒ dʒ a r"}, "future": {"arabic": "يأجّر", "phonemes": "j ʔ a dʒ dʒ i r"}},
|
| 15 |
+
{"hebrew": "הדהים", "timestamp": "01:55", "past": {"arabic": "أدهش", "phonemes": "ʔ a d h a ʃ"}, "future": {"arabic": "يدهش", "phonemes": "j i d h a ʃ"}},
|
| 16 |
+
{"hebrew": "ירד", "timestamp": "02:03", "past": {"arabic": "انخفض", "phonemes": "ʔ i n x a f a dˤ"}, "future": {"arabic": "ينخفض", "phonemes": "j i n x a f i dˤ"}},
|
| 17 |
+
{"hebrew": "הדגיש", "timestamp": "02:13", "past": {"arabic": "أبرز", "phonemes": "ʔ a b r a z"}, "future": {"arabic": "يبرز", "phonemes": "j i b r u z"}},
|
| 18 |
+
{"hebrew": "נטרל, פירק", "timestamp": "02:21", "past": {"arabic": "أبطل", "phonemes": "ʔ a b tˤ a l"}, "future": {"arabic": "يبطل", "phonemes": "j i b tˤ a l"}},
|
| 19 |
+
{"hebrew": "אירח לחברה, ישב עם אורח כדי שלא יישאר לבד", "timestamp": "02:30", "past": {"arabic": "آنس", "phonemes": "ʔ aː n a s"}, "future": {"arabic": "يآنس", "phonemes": "j ʔ aː n i s"}},
|
| 20 |
+
{"hebrew": "התעניין, החשיב", "timestamp": "02:46", "past": {"arabic": "اهتمّ", "phonemes": "ʔ i h t a m m"}, "future": {"arabic": "يهتمّ", "phonemes": "j i h t a m m"}},
|
| 21 |
+
{"hebrew": "היה חלוק בדעתו", "timestamp": "02:52", "past": {"arabic": "اختلف", "phonemes": "ʔ i x t a l a f"}, "future": {"arabic": "يختلف", "phonemes": "j i x t a l i f"}},
|
| 22 |
+
{"hebrew": "בחר", "timestamp": "03:19", "past": {"arabic": "اختار", "phonemes": "ʔ i x t aː r"}, "future": {"arabic": "يختار", "phonemes": "j i x t aː r"}},
|
| 23 |
+
{"hebrew": "המציא", "timestamp": "03:28", "past": {"arabic": "اخترع", "phonemes": "ʔ i x t a r a ʕ"}, "future": {"arabic": "يخترع", "phonemes": "j i x t a r i ʕ"}},
|
| 24 |
+
{"hebrew": "גילה", "timestamp": "03:39", "past": {"arabic": "اكتشف", "phonemes": "ʔ i k t a ʃ a f"}, "future": {"arabic": "يكتشف", "phonemes": "j i k t a ʃ i f"}},
|
| 25 |
+
{"hebrew": "התחזה", "timestamp": "03:51", "past": {"arabic": "انتحل", "phonemes": "ʔ i n t a ħ a l"}, "future": {"arabic": "ينتحل", "phonemes": "j i n t a ħ i l"}},
|
| 26 |
+
{"hebrew": "נפצע", "timestamp": "04:00", "past": {"arabic": "انجرح", "phonemes": "ʔ i n dʒ a r a ħ"}, "future": {"arabic": "ينجرح", "phonemes": "j i n dʒ a r i ħ"}},
|
| 27 |
+
{"hebrew": "נדרס", "timestamp": "04:09", "past": {"arabic": "اندهس", "phonemes": "ʔ i n d a h a s"}, "future": {"arabic": "يندهس", "phonemes": "j i n d a h i s"}},
|
| 28 |
+
{"hebrew": "התהפך", "timestamp": "04:27", "past": {"arabic": "انقلب", "phonemes": "ʔ i n Q a l a b"}, "future": {"arabic": "ينقلب", "phonemes": "j i n Q a l i b"}},
|
| 29 |
+
{"hebrew": "עבר ממקום למקום", "timestamp": "04:35", "past": {"arabic": "انتقل", "phonemes": "ʔ i n t a Q a l"}, "future": {"arabic": "ينتقل", "phonemes": "j i n t a Q i l"}},
|
| 30 |
+
{"hebrew": "הבשיל", "timestamp": "04:45", "past": {"arabic": "استوى", "phonemes": "ʔ i s t a w a"}, "future": {"arabic": "يستوي", "phonemes": "j i s t a w i"}},
|
| 31 |
+
{"hebrew": "ייבא", "timestamp": "04:54", "past": {"arabic": "استورد", "phonemes": "ʔ i s t a w r a d"}, "future": {"arabic": "يستورد", "phonemes": "j i s t a w r i d"}},
|
| 32 |
+
{"hebrew": "חשב", "timestamp": "05:04", "past": {"arabic": "افتكر", "phonemes": "ʔ i f t a k a r"}, "future": {"arabic": "يفتكر", "phonemes": "j i f t a k i r"}},
|
| 33 |
+
{"hebrew": "עבד", "timestamp": "05:14", "past": {"arabic": "اشتغل", "phonemes": "ʔ i ʃ t a ɣ a l", "accepted_phonemes": ["ʔ a ʃ t ɣ i l", "ʔ a ʃ t i ɣ i l"]}, "future": {"arabic": "يشتغل", "phonemes": "j i ʃ t a ɣ i l"}},
|
| 34 |
+
{"hebrew": "ניצת, נדלק", "timestamp": "05:23", "past": {"arabic": "اشتعل", "phonemes": "ʔ i ʃ t a ʕ a l"}, "future": {"arabic": "يشتعل", "phonemes": "j i ʃ t a ʕ i l"}},
|
| 35 |
+
{"hebrew": "השתתף", "timestamp": "05:33", "past": {"arabic": "اشترك", "phonemes": "ʔ i ʃ t a r a k"}, "future": {"arabic": "يشترك", "phonemes": "j i ʃ t a r i k"}},
|
| 36 |
+
{"hebrew": "אימץ פתרון או קיבל החלטה", "timestamp": "05:42", "past": {"arabic": "اتّخذ", "phonemes": "ʔ i t t a x a d"}, "future": {"arabic": "يتّخذ", "phonemes": "j i t t a x i d"}},
|
| 37 |
+
{"hebrew": "נשען על חפץ", "timestamp": "05:54", "past": {"arabic": "اتّكأ", "phonemes": "ʔ i t t a k a ʔ"}, "future": {"arabic": "يتّكئ", "phonemes": "j i t t a k i ʔ"}},
|
| 38 |
+
{"hebrew": "הסכים", "timestamp": "06:07", "past": {"arabic": "اتّفق", "phonemes": "ʔ i t t a f a Q"}, "future": {"arabic": "يتّفق", "phonemes": "j i t t a f i Q"}},
|
| 39 |
+
{"hebrew": "התקשר", "timestamp": "06:15", "past": {"arabic": "اتّصل", "phonemes": "ʔ i t t a sˤ a l"}, "future": {"arabic": "يتّصل", "phonemes": "j i t t a sˤ i l"}},
|
| 40 |
+
{"hebrew": "התעייף", "timestamp": "06:24", "past": {"arabic": "اتعب", "phonemes": "ʔ i t ʕ a b"}, "future": {"arabic": "يتعب", "phonemes": "j i t ʕ a b"}},
|
| 41 |
+
{"hebrew": "עלה", "timestamp": "06:34", "past": {"arabic": "ارتفع", "phonemes": "ʔ i r t a f a ʕ"}, "future": {"arabic": "يرتفع", "phonemes": "j i r t a f i ʕ"}},
|
| 42 |
+
{"hebrew": "שיתף", "timestamp": "06:43", "past": {"arabic": "أشرك", "phonemes": "ʔ a ʃ r a k"}, "future": {"arabic": "يشرك", "phonemes": "j i ʃ r i k"}},
|
| 43 |
+
{"hebrew": "אלתר", "timestamp": "06:51", "past": {"arabic": "ارتجل", "phonemes": "ʔ i r t a dʒ a l"}, "future": {"arabic": "يرتجل", "phonemes": "j i r t a dʒ i l"}},
|
| 44 |
+
{"hebrew": "הרחיק", "timestamp": "07:08", "past": {"arabic": "أبعد", "phonemes": "ʔ a b ʕ a d"}, "future": {"arabic": "يبعد", "phonemes": "j i b ʕ i d"}},
|
| 45 |
+
{"hebrew": "הזניח", "timestamp": "07:19", "past": {"arabic": "أهمل", "phonemes": "ʔ a h m a l"}, "future": {"arabic": "يهمل", "phonemes": "j i h m i l"}},
|
| 46 |
+
{"hebrew": "איחר", "timestamp": "07:53", "past": {"arabic": "اتأخّر", "phonemes": "ʔ i t ʔ a x x a r"}, "future": {"arabic": "يتأخّر", "phonemes": "j i t ʔ a x x a r"}},
|
| 47 |
+
{"hebrew": "תמך ב־, אישר", "timestamp": "08:02", "past": {"arabic": "أكّد", "phonemes": "ʔ a k k a d"}, "future": {"arabic": "يأكّد", "phonemes": "j ʔ a k k i d"}},
|
| 48 |
+
{"hebrew": "אכל", "timestamp": "08:08", "past": {"arabic": "اكل", "phonemes": "ʔ a k a l"}, "future": {"arabic": "ياكل", "phonemes": "j oː k u l", "accepted_phonemes": ["j oː k i l", "j aː k u l"]}},
|
| 49 |
+
{"hebrew": "ציווה", "timestamp": "08:36", "past": {"arabic": "أمر", "phonemes": "ʔ a m a r"}, "future": {"arabic": "يأمر", "phonemes": "j ʔ a m u r"}},
|
| 50 |
+
{"hebrew": "נתן שהות", "timestamp": "08:40", "past": {"arabic": "أمهل", "phonemes": "ʔ a m h a l"}, "future": {"arabic": "يمهل", "phonemes": "j i m h i l"}},
|
| 51 |
+
{"hebrew": "סיים", "timestamp": "08:54", "past": {"arabic": "أنهى", "phonemes": "ʔ a n h a"}, "future": {"arabic": "ينهي", "phonemes": "j i n h i"}},
|
| 52 |
+
{"hebrew": "ייסד", "timestamp": "09:01", "past": {"arabic": "أسّس", "phonemes": "ʔ a s s a s"}, "future": {"arabic": "يأسّس", "phonemes": "j ʔ a s s i s"}},
|
| 53 |
+
{"hebrew": "שיכר", "timestamp": "09:10", "past": {"arabic": "أسكر", "phonemes": "ʔ a s k a r"}, "future": {"arabic": "يسكر", "phonemes": "j i s k i r"}},
|
| 54 |
+
{"hebrew": "מיהר", "timestamp": "09:24", "past": {"arabic": "أسرع", "phonemes": "ʔ a s r a ʕ"}, "future": {"arabic": "يسرع", "phonemes": "j i s r a ʕ"}},
|
| 55 |
+
{"hebrew": "נתן", "timestamp": "09:33", "past": {"arabic": "أعطى", "phonemes": "ʔ a ʕ tˤ a"}, "future": {"arabic": "يعطي", "phonemes": "j i ʕ tˤ i"}},
|
| 56 |
+
{"hebrew": "הדביק במחלה", "timestamp": "09:40", "past": {"arabic": "أعدى", "phonemes": "ʔ a ʕ d a"}, "future": {"arabic": "يعدي", "phonemes": "j i ʕ d i"}},
|
| 57 |
+
{"hebrew": "הכריז", "timestamp": "09:48", "past": {"arabic": "أعلن", "phonemes": "ʔ a ʕ l a n"}, "future": {"arabic": "يعلن", "phonemes": "j i ʕ l i n"}},
|
| 58 |
+
{"hebrew": "פטר", "timestamp": "09:56", "past": {"arabic": "أعفى", "phonemes": "ʔ a ʕ f a"}, "future": {"arabic": "يعفي", "phonemes": "j i ʕ f i"}},
|
| 59 |
+
{"hebrew": "עיוור, סינוור", "timestamp": "10:05", "past": {"arabic": "أعمى", "phonemes": "ʔ a ʕ m a"}, "future": {"arabic": "يعمي", "phonemes": "j i ʕ m i"}},
|
| 60 |
+
{"hebrew": "השביע את רצונו של", "timestamp": "10:12", "past": {"arabic": "أرضى", "phonemes": "ʔ a r dˤ a"}, "future": {"arabic": "يرضي", "phonemes": "j i r dˤ i"}},
|
| 61 |
+
{"hebrew": "סימן, אותת", "timestamp": "10:23", "past": {"arabic": "أشّر", "phonemes": "ʔ a ʃ ʃ a r"}, "future": {"arabic": "يأشّر", "phonemes": "j ʔ a ʃ ʃ i r"}},
|
| 62 |
+
{"hebrew": "השפיע", "timestamp": "10:32", "past": {"arabic": "أثّر", "phonemes": "ʔ a TH TH a r"}, "future": {"arabic": "يأثّر", "phonemes": "j ʔ a TH TH i r"}},
|
| 63 |
+
{"hebrew": "השלים בין, תיקן", "timestamp": "10:41", "past": {"arabic": "أصلح", "phonemes": "ʔ a sˤ l a ħ"}, "future": {"arabic": "يصلح", "phonemes": "j i sˤ l i ħ"}}
|
| 64 |
+
]
|
| 65 |
+
},
|
| 66 |
+
"ba": {
|
| 67 |
+
"source": {
|
| 68 |
+
"title": "פעלים שימושיים בערבית מדוברת — אות בא",
|
| 69 |
+
"url": "https://www.youtube.com/watch?v=w19b7mJ-j4U",
|
| 70 |
+
"dialect": "Palestinian spoken Arabic",
|
| 71 |
+
"notes": "One entry per verb, in video order. Phonemes use the scorer alphabet and are space-separated."
|
| 72 |
+
},
|
| 73 |
+
"verbs": [
|
| 74 |
+
{"hebrew": "התחיל", "timestamp": "01:39", "past": {"arabic": "بدأ", "phonemes": "b a d a ʔ"}, "future": {"arabic": "يبدأ", "phonemes": "j i b d a ʔ"}},
|
| 75 |
+
{"hebrew": "בכה", "timestamp": "01:48", "past": {"arabic": "بكى", "phonemes": "b a k a"}, "future": {"arabic": "يبكي", "phonemes": "j i b k i"}},
|
| 76 |
+
{"hebrew": "נשאר", "timestamp": "01:57", "past": {"arabic": "بقي", "phonemes": "b a Q i"}, "future": {"arabic": "يبقى", "phonemes": "j i b Q a"}},
|
| 77 |
+
{"hebrew": "מכר", "timestamp": "02:06", "past": {"arabic": "باع", "phonemes": "b aː ʕ"}, "future": {"arabic": "يبيع", "phonemes": "j i b iː ʕ"}},
|
| 78 |
+
{"hebrew": "אירח מישהו כל הלילה", "timestamp": "02:17", "past": {"arabic": "بات", "phonemes": "b aː t"}, "future": {"arabic": "يبيت", "phonemes": "j i b iː t"}},
|
| 79 |
+
{"hebrew": "נזף, דיבר בצורה קשה", "timestamp": "02:26", "past": {"arabic": "بهدل", "phonemes": "b a h d a l"}, "future": {"arabic": "يبهدل", "phonemes": "j i b a h d i l"}},
|
| 80 |
+
{"hebrew": "החליף, שינה", "timestamp": "02:36", "past": {"arabic": "بدّل", "phonemes": "b a d d a l"}, "future": {"arabic": "يبدّل", "phonemes": "j i b a d d i l"}},
|
| 81 |
+
{"hebrew": "חפר באדמה", "timestamp": "02:46", "past": {"arabic": "بحش", "phonemes": "b a ħ a ʃ"}, "future": {"arabic": "يبحش", "phonemes": "j i b ħ a ʃ"}},
|
| 82 |
+
{"hebrew": "סיווג", "timestamp": "02:56", "past": {"arabic": "بوّب", "phonemes": "b a w w a b"}, "future": {"arabic": "يبوّب", "phonemes": "j i b a w w i b"}},
|
| 83 |
+
{"hebrew": "הפסיק, חזר בו", "timestamp": "03:06", "past": {"arabic": "بطّل", "phonemes": "b a tˤ tˤ a l"}, "future": {"arabic": "يبطّل", "phonemes": "j i b a tˤ tˤ i l"}},
|
| 84 |
+
{"hebrew": "הופיע, נמצא", "timestamp": "03:15", "past": {"arabic": "بيّن", "phonemes": "b a j j a n"}, "future": {"arabic": "يبيّن", "phonemes": "j i b a j j i n"}},
|
| 85 |
+
{"hebrew": "הודיע", "timestamp": "03:25", "past": {"arabic": "بلّغ", "phonemes": "b a l l a ɣ"}, "future": {"arabic": "يبلّغ", "phonemes": "j i b a l l i ɣ"}},
|
| 86 |
+
{"hebrew": "בלע", "timestamp": "03:38", "past": {"arabic": "بلع", "phonemes": "b a l a ʕ"}, "future": {"arabic": "يبلع", "phonemes": "j i b l a ʕ"}},
|
| 87 |
+
{"hebrew": "התחיל", "timestamp": "03:44", "past": {"arabic": "بلّش", "phonemes": "b a l l a ʃ"}, "future": {"arabic": "يبلّش", "phonemes": "j i b a l l i ʃ"}},
|
| 88 |
+
{"hebrew": "בכה בקול, יילל", "timestamp": "04:05", "past": {"arabic": "بعق", "phonemes": "b a ʕ a Q"}, "future": {"arabic": "يبعق", "phonemes": "j i b ʕ a Q"}},
|
| 89 |
+
{"hebrew": "פיזר וגם בזבז", "timestamp": "04:14", "past": {"arabic": "بعتر", "phonemes": "b a ʕ t a r"}, "future": {"arabic": "يبعتر", "phonemes": "j i b a ʕ t i r"}},
|
| 90 |
+
{"hebrew": "בנה", "timestamp": "04:23", "past": {"arabic": "بنى", "phonemes": "b a n a"}, "future": {"arabic": "يبني", "phonemes": "j i b n i"}},
|
| 91 |
+
{"hebrew": "הצדיק", "timestamp": "04:30", "past": {"arabic": "برّر", "phonemes": "b a r r a r"}, "future": {"arabic": "يبرّر", "phonemes": "j i b a r r i r"}},
|
| 92 |
+
{"hebrew": "כיער", "timestamp": "04:37", "past": {"arabic": "بشّع", "phonemes": "b a ʃ ʃ a ʕ"}, "future": {"arabic": "يبشّع", "phonemes": "j i b a ʃ ʃ i ʕ"}},
|
| 93 |
+
{"hebrew": "תכנת", "timestamp": "04:47", "past": {"arabic": "برمج", "phonemes": "b a r m a dʒ"}, "future": {"arabic": "يبرمج", "phonemes": "j i b a r m i dʒ"}},
|
| 94 |
+
{"hebrew": "שיחד", "timestamp": "05:11", "past": {"arabic": "برطل", "phonemes": "b a r tˤ a l"}, "future": {"arabic": "يبرطل", "phonemes": "j i b a r tˤ i l"}}
|
| 95 |
+
]
|
| 96 |
+
},
|
| 97 |
+
"jim": {
|
| 98 |
+
"source": {"title": "פעלים שימושיים בערבית מדוברת — אות ג'ים", "url": "https://www.youtube.com/watch?v=A90bTU8w3BY", "dialect": "Palestinian spoken Arabic"},
|
| 99 |
+
"verbs": [
|
| 100 |
+
{"hebrew":"הביא","past":{"arabic":"جاب","phonemes":"J aː b"},"future":{"arabic":"يجيب","phonemes":"j i J iː b"}},
|
| 101 |
+
{"hebrew":"ענה","past":{"arabic":"جاوب","phonemes":"J aː w a b"},"future":{"arabic":"يجاوب","phonemes":"j i J aː w i b"}},
|
| 102 |
+
{"hebrew":"גרר, סחב","past":{"arabic":"جرجر","phonemes":"J a r J a r"},"future":{"arabic":"يجرجر","phonemes":"j i J a r J i r"}},
|
| 103 |
+
{"hebrew":"ניסה","past":{"arabic":"جرّب","phonemes":"J a r r a b"},"future":{"arabic":"يجرّب","phonemes":"j i J a r r i b"}},
|
| 104 |
+
{"hebrew":"הקפיא","past":{"arabic":"جمّد","phonemes":"J a m m a d"},"future":{"arabic":"يجمّد","phonemes":"j i J a m m i d"}},
|
| 105 |
+
{"hebrew":"ייבש","past":{"arabic":"جفّف","phonemes":"J a f f a f"},"future":{"arabic":"يجفّف","phonemes":"j i J a f f i f"}},
|
| 106 |
+
{"hebrew":"חידש","past":{"arabic":"جدّد","phonemes":"J a d d a d"},"future":{"arabic":"يجدّد","phonemes":"j i J a d d i d"}},
|
| 107 |
+
{"hebrew":"רץ, קרה, זרם","past":{"arabic":"جرى","phonemes":"J a r a"},"future":{"arabic":"يجري","phonemes":"j i J r i"}},
|
| 108 |
+
{"hebrew":"גמל, העניש","past":{"arabic":"جازى","phonemes":"J aː z a"},"future":{"arabic":"يجازي","phonemes":"j i J aː z i"}},
|
| 109 |
+
{"hebrew":"גרר, סחב","past":{"arabic":"جرّ","phonemes":"J a r r"},"future":{"arabic":"يجرّ","phonemes":"j i J u r r"}}
|
| 110 |
+
]
|
| 111 |
+
},
|
| 112 |
+
"dal": {
|
| 113 |
+
"source": {"title":"פעלים שימושיים בערבית מדוברת — אות דאל","url":"https://www.youtube.com/watch?v=Wu2oeHyKujo","dialect":"Palestinian spoken Arabic"},
|
| 114 |
+
"verbs": [
|
| 115 |
+
{"hebrew":"אבד","past":{"arabic":"ضاع","phonemes":"dˤ aː ʕ"},"future":{"arabic":"يضيع","phonemes":"j i dˤ iː ʕ"}},
|
| 116 |
+
{"hebrew":"סידר, הסדיר","past":{"arabic":"دبّر","phonemes":"d a b b a r"},"future":{"arabic":"يدبّر","phonemes":"j i d a b b i r"}},
|
| 117 |
+
{"hebrew":"צבע","past":{"arabic":"دهن","phonemes":"d a h a n"},"future":{"arabic":"يدهن","phonemes":"j i d h a n"}},
|
| 118 |
+
{"hebrew":"דרס","past":{"arabic":"دهس","phonemes":"d a h a s"},"future":{"arabic":"يدهس","phonemes":"j i d h a s"}},
|
| 119 |
+
{"hebrew":"חיפש","past":{"arabic":"دوّر على","phonemes":"d a w w a r ʕ a l a"},"future":{"arabic":"يدوّر على","phonemes":"j i d a w w i r ʕ a l a"}},
|
| 120 |
+
{"hebrew":"עישן","past":{"arabic":"دخّن","phonemes":"d a x x a n"},"future":{"arabic":"يدخّن","phonemes":"j i d a x x i n"}},
|
| 121 |
+
{"hebrew":"הצביע על, הראה","past":{"arabic":"دلّ على","phonemes":"d a l l ʕ a l a"},"future":{"arabic":"يدلّ على","phonemes":"j i d u l l ʕ a l a"}},
|
| 122 |
+
{"hebrew":"שילם, דחף","past":{"arabic":"دفع","phonemes":"d a f a ʕ"},"future":{"arabic":"يدفع","phonemes":"j i d f a ʕ"}},
|
| 123 |
+
{"hebrew":"דפק","past":{"arabic":"دقّ","phonemes":"d a Q Q"},"future":{"arabic":"يدقّ","phonemes":"j i d u Q Q"}},
|
| 124 |
+
{"hebrew":"עזב, נטש","past":{"arabic":"دشّر","phonemes":"d a ʃ ʃ a r"},"future":{"arabic":"يدشّر","phonemes":"j i d a ʃ ʃ i r"}},
|
| 125 |
+
{"hebrew":"סובב, הפנה","past":{"arabic":"دار","phonemes":"d aː r"},"future":{"arabic":"يدير","phonemes":"j i d iː r"}}
|
| 126 |
+
]
|
| 127 |
+
},
|
| 128 |
+
"ha": {
|
| 129 |
+
"source": {"title":"פעלים שימושיים בערבית מדוברת — אות ה","url":"https://www.youtube.com/watch?v=le--lfAYjgE","dialect":"Palestinian spoken Arabic"},
|
| 130 |
+
"verbs": [
|
| 131 |
+
{"hebrew":"ברח","past":{"arabic":"هرب","phonemes":"h a r a b"},"future":{"arabic":"يهرب","phonemes":"j i h r a b"}},
|
| 132 |
+
{"hebrew":"היגר","past":{"arabic":"هاجر","phonemes":"h aː J a r"},"future":{"arabic":"يهاجر","phonemes":"j i h aː J i r"}},
|
| 133 |
+
{"hebrew":"הגלה, גרם להגר","past":{"arabic":"هجّر","phonemes":"h a J J a r"},"future":{"arabic":"يهجّر","phonemes":"j i h a J J i r"}},
|
| 134 |
+
{"hebrew":"נפל, ירד","past":{"arabic":"هبط","phonemes":"h a b a tˤ"},"future":{"arabic":"يهبط","phonemes":"j i h b a tˤ"}},
|
| 135 |
+
{"hebrew":"הוריד, החליש","past":{"arabic":"هبّط","phonemes":"h a b b a tˤ"},"future":{"arabic":"يهبّط","phonemes":"j i h a b b i tˤ"}},
|
| 136 |
+
{"hebrew":"התכוון, שאף","past":{"arabic":"هدف","phonemes":"h a d a f"},"future":{"arabic":"يهدف","phonemes":"j i h d a f"}},
|
| 137 |
+
{"hebrew":"בירך","past":{"arabic":"هنّى","phonemes":"h a n n a"},"future":{"arabic":"يهنّي","phonemes":"j i h a n n i"}},
|
| 138 |
+
{"hebrew":"הריע, קרא","past":{"arabic":"هتف","phonemes":"h a t a f"},"future":{"arabic":"يهتف","phonemes":"j i h t i f"}}
|
| 139 |
+
]
|
| 140 |
+
},
|
| 141 |
+
"waw": {
|
| 142 |
+
"source": {"title":"פעלים שימושיים בערבית מדוברת — אות ואו","url":"https://www.youtube.com/watch?v=r5nOxemdyMI","dialect":"Palestinian spoken Arabic"},
|
| 143 |
+
"verbs": [
|
| 144 |
+
{"hebrew":"מצא","past":{"arabic":"وجد","phonemes":"w a J a d"},"future":{"arabic":"يجد","phonemes":"j i J i d"}},
|
| 145 |
+
{"hebrew":"הביא למקום","past":{"arabic":"ودّى","phonemes":"w a d d a"},"future":{"arabic":"يودّي","phonemes":"j i w a d d i"}},
|
| 146 |
+
{"hebrew":"נפרד","past":{"arabic":"ودّع","phonemes":"w a d d a ʕ"},"future":{"arabic":"يودّع","phonemes":"j i w a d d i ʕ"}},
|
| 147 |
+
{"hebrew":"מינה, הסמיך","past":{"arabic":"وكّل","phonemes":"w a k k a l"},"future":{"arabic":"يوكّل","phonemes":"j i w a k k i l"}},
|
| 148 |
+
{"hebrew":"עצר, עמד","past":{"arabic":"وقف","phonemes":"w a Q a f"},"future":{"arabic":"يوقف","phonemes":"j i w Q a f"}},
|
| 149 |
+
{"hebrew":"ייבא","past":{"arabic":"ورّد","phonemes":"w a r r a d"},"future":{"arabic":"يورّد","phonemes":"j i w a r r i d"}},
|
| 150 |
+
{"hebrew":"הנמיך","past":{"arabic":"وطّى","phonemes":"w a tˤ tˤ a"},"future":{"arabic":"يوطّي","phonemes":"j i w a tˤ tˤ i"}},
|
| 151 |
+
{"hebrew":"חיבר, הוביל, הביא","past":{"arabic":"وصّل","phonemes":"w a sˤ sˤ a l"},"future":{"arabic":"يوصّل","phonemes":"j i w a sˤ sˤ i l"}}
|
| 152 |
+
]
|
| 153 |
+
},
|
| 154 |
+
"zay": {
|
| 155 |
+
"source": {"title":"פעלים שימושיים בערבית מדוברת — אות זאי","url":"https://www.youtube.com/watch?v=Qgal8RB1N00","dialect":"Palestinian spoken Arabic"},
|
| 156 |
+
"verbs": [
|
| 157 |
+
{"hebrew":"כעס, התרגז","past":{"arabic":"زعل","phonemes":"z a ʕ a l"},"future":{"arabic":"يزعل","phonemes":"j i z ʕ a l"}},
|
| 158 |
+
{"hebrew":"זז, הסיר","past":{"arabic":"زاح","phonemes":"z aː ħ"},"future":{"arabic":"يزيح","phonemes":"j i z iː ħ"}},
|
| 159 |
+
{"hebrew":"סיפק, צייד","past":{"arabic":"زوّد","phonemes":"z a w w a d"},"future":{"arabic":"يزوّد","phonemes":"j i z a w w i d"}},
|
| 160 |
+
{"hebrew":"צפר","past":{"arabic":"زمّر","phonemes":"z a m m a r"},"future":{"arabic":"يزمّر","phonemes":"j i z a m m i r"}},
|
| 161 |
+
{"hebrew":"הוסיף, גדל","past":{"arabic":"زاد","phonemes":"z aː d"},"future":{"arabic":"يزيد","phonemes":"j i z iː d"}},
|
| 162 |
+
{"hebrew":"ביקר","past":{"arabic":"زار","phonemes":"z aː r"},"future":{"arabic":"يزور","phonemes":"j i z uː r"}},
|
| 163 |
+
{"hebrew":"הזכיר, ציטט","past":{"arabic":"ذكر","phonemes":"DH a k a r"},"future":{"arabic":"يذكر","phonemes":"j i DH k u r"}},
|
| 164 |
+
{"hebrew":"הזכיר למישהו","past":{"arabic":"ذكّر","phonemes":"DH a k k a r"},"future":{"arabic":"يذكّر","phonemes":"j i DH a k k i r"}},
|
| 165 |
+
{"hebrew":"דחף","past":{"arabic":"زقّ","phonemes":"z a Q Q"},"future":{"arabic":"يزقّ","phonemes":"j i z u Q Q"}},
|
| 166 |
+
{"hebrew":"זייף","past":{"arabic":"زيّف","phonemes":"z a j j a f"},"future":{"arabic":"يزيّف","phonemes":"j i z a j j i f"}},
|
| 167 |
+
{"hebrew":"שחט","past":{"arabic":"ذبح","phonemes":"DH a b a ħ"},"future":{"arabic":"يذبح","phonemes":"j i DH b a ħ"}},
|
| 168 |
+
{"hebrew":"נמאס לו","past":{"arabic":"زهق","phonemes":"z a h a Q"},"future":{"arabic":"يزهق","phonemes":"j i z h a Q"}},
|
| 169 |
+
{"hebrew":"החליק, גרם להחליק","past":{"arabic":"زحلق","phonemes":"z a ħ l a Q"},"future":{"arabic":"يزحلق","phonemes":"j i z a ħ l i Q"}}
|
| 170 |
+
]
|
| 171 |
+
},
|
| 172 |
+
"kha": {
|
| 173 |
+
"source": {"title":"פעלים שימושיים בערבית מדוברת — אות ח'א","url":"https://www.youtube.com/watch?v=RJnw8Bjtvlg","dialect":"Palestinian spoken Arabic"},
|
| 174 |
+
"verbs": [
|
| 175 |
+
{"hebrew":"בגד","past":{"arabic":"خان","phonemes":"x aː n"},"future":{"arabic":"يخون","phonemes":"j i x uː n"}},
|
| 176 |
+
{"hebrew":"נגמר, הסתיים","past":{"arabic":"خلص","phonemes":"x a l a sˤ"},"future":{"arabic":"يخلص","phonemes":"j i x l a sˤ"}},
|
| 177 |
+
{"hebrew":"הפסיד","past":{"arabic":"خسر","phonemes":"x a s a r"},"future":{"arabic":"يخسر","phonemes":"j i x s a r"}},
|
| 178 |
+
{"hebrew":"התקלקל","past":{"arabic":"خرب","phonemes":"x a r a b"},"future":{"arabic":"يخرب","phonemes":"j i x r a b"}},
|
| 179 |
+
{"hebrew":"פחד","past":{"arabic":"خاف","phonemes":"x aː f"},"future":{"arabic":"يخاف","phonemes":"j i x aː f"}},
|
| 180 |
+
{"hebrew":"הודיע, סיפר","past":{"arabic":"خبّر","phonemes":"x a b b a r"},"future":{"arabic":"يخبّر","phonemes":"j i x a b b i r"}},
|
| 181 |
+
{"hebrew":"אחסן","past":{"arabic":"خزّن","phonemes":"x a z z a n"},"future":{"arabic":"يخزّن","phonemes":"j i x a z z i n"}},
|
| 182 |
+
{"hebrew":"התבייש","past":{"arabic":"خجل","phonemes":"x a J a l"},"future":{"arabic":"يخجل","phonemes":"j i x J a l"}},
|
| 183 |
+
{"hebrew":"השאיר, נטש","past":{"arabic":"خلّى","phonemes":"x a l l a"},"future":{"arabic":"يخلّي","phonemes":"j i x a l l i"}},
|
| 184 |
+
{"hebrew":"השאיר אחריו, הוליד","past":{"arabic":"خلّف","phonemes":"x a l l a f"},"future":{"arabic":"يخلّف","phonemes":"j i x a l l i f"}},
|
| 185 |
+
{"hebrew":"יצר, ברא","past":{"arabic":"خلق","phonemes":"x a l a Q"},"future":{"arabic":"يخلق","phonemes":"j i x l a Q"}},
|
| 186 |
+
{"hebrew":"הפחית","past":{"arabic":"خفّف","phonemes":"x a f f a f"},"future":{"arabic":"يخفّف","phonemes":"j i x a f f i f"}},
|
| 187 |
+
{"hebrew":"התערבב, התחבר","past":{"arabic":"خالط","phonemes":"x aː l a tˤ"},"future":{"arabic":"يخالط","phonemes":"j i x aː l i tˤ"}},
|
| 188 |
+
{"hebrew":"אכזב","past":{"arabic":"خيّب","phonemes":"x a j j a b"},"future":{"arabic":"يخيّب","phonemes":"j i x a j j i b"}},
|
| 189 |
+
{"hebrew":"חתם","past":{"arabic":"ختم","phonemes":"x a t a m"},"future":{"arabic":"يختم","phonemes":"j i x t i m"}},
|
| 190 |
+
{"hebrew":"קלקל","past":{"arabic":"خرّب","phonemes":"x a r r a b"},"future":{"arabic":"يخرّب","phonemes":"j i x a r r i b"}},
|
| 191 |
+
{"hebrew":"יצא","past":{"arabic":"خرج","phonemes":"x a r a J"},"future":{"arabic":"يخرج","phonemes":"j i x r a J"}},
|
| 192 |
+
{"hebrew":"נשא נאום","past":{"arabic":"خطب","phonemes":"x a tˤ a b"},"future":{"arabic":"يخطب","phonemes":"j i x tˤ a b"}}
|
| 193 |
+
]
|
| 194 |
+
},
|
| 195 |
+
"ta_emphatic": {
|
| 196 |
+
"source":{"title":"פעלים שימושיים בערבית מדוברת — אות טא","url":"https://www.youtube.com/watch?v=qYeEWydByZU","dialect":"Palestinian spoken Arabic"},
|
| 197 |
+
"verbs":[
|
| 198 |
+
{"hebrew":"נרפא, היה טוב","past":{"arabic":"طاب","phonemes":"tˤ aː b"},"future":{"arabic":"يطيب","phonemes":"j i tˤ iː b"}},
|
| 199 |
+
{"hebrew":"הדפיס","past":{"arabic":"طبع","phonemes":"tˤ a b a ʕ"},"future":{"arabic":"يطبع","phonemes":"j i tˤ b a ʕ"}},
|
| 200 |
+
{"hebrew":"האריך, התעכב","past":{"arabic":"طوّل","phonemes":"tˤ a w w a l"},"future":{"arabic":"يطوّل","phonemes":"j i tˤ a w w i l"}},
|
| 201 |
+
{"hebrew":"דרש, ביקש","past":{"arabic":"طلب","phonemes":"tˤ a l a b"},"future":{"arabic":"يطلب","phonemes":"j i tˤ l a b"}},
|
| 202 |
+
{"hebrew":"האכיל","past":{"arabic":"طعّم","phonemes":"tˤ a ʕ ʕ a m"},"future":{"arabic":"يطعّم","phonemes":"j i tˤ a ʕ ʕ i m"}},
|
| 203 |
+
{"hebrew":"היה מסוגל לסבול","past":{"arabic":"طاق","phonemes":"tˤ aː Q"},"future":{"arabic":"يطيق","phonemes":"j i tˤ iː Q"}},
|
| 204 |
+
{"hebrew":"הרגיע","past":{"arabic":"طمّن","phonemes":"tˤ a m m a n"},"future":{"arabic":"يطمّن","phonemes":"j i tˤ a m m i n"}},
|
| 205 |
+
{"hebrew":"פיצח","past":{"arabic":"طقش","phonemes":"tˤ a Q a ʃ"},"future":{"arabic":"يطقش","phonemes":"j i tˤ Q a ʃ"}},
|
| 206 |
+
{"hebrew":"התעלם","past":{"arabic":"طنّش","phonemes":"tˤ a n n a ʃ"},"future":{"arabic":"يطنّش","phonemes":"j i tˤ a n n i ʃ"}},
|
| 207 |
+
{"hebrew":"עף","past":{"arabic":"طار","phonemes":"tˤ aː r"},"future":{"arabic":"يطير","phonemes":"j i tˤ iː r"}},
|
| 208 |
+
{"hebrew":"רדף","past":{"arabic":"طارد","phonemes":"tˤ aː r a d"},"future":{"arabic":"يطارد","phonemes":"j i tˤ aː r i d"}},
|
| 209 |
+
{"hebrew":"כיבה","past":{"arabic":"طفّى","phonemes":"tˤ a f f a"},"future":{"arabic":"يطفّي","phonemes":"j i tˤ a f f i"}},
|
| 210 |
+
{"hebrew":"יישם","past":{"arabic":"طبّق","phonemes":"tˤ a b b a Q"},"future":{"arabic":"يطبّق","phonemes":"j i tˤ a b b i Q"}},
|
| 211 |
+
{"hebrew":"הציץ, ביקר","past":{"arabic":"طلّ","phonemes":"tˤ a l l"},"future":{"arabic":"يطلّ","phonemes":"j i tˤ u l l"}},
|
| 212 |
+
{"hebrew":"שאף","past":{"arabic":"طمح","phonemes":"tˤ a m a ħ"},"future":{"arabic":"يطمح","phonemes":"j i tˤ m a ħ"}},
|
| 213 |
+
{"hebrew":"התיז","past":{"arabic":"طرّش","phonemes":"tˤ a r r a ʃ"},"future":{"arabic":"يطرّش","phonemes":"j i tˤ a r r i ʃ"}}
|
| 214 |
+
]
|
| 215 |
+
},
|
| 216 |
+
"kaf": {
|
| 217 |
+
"source":{"title":"פעלים שימושיים בערבית מדוברת — אות כאף","url":"https://www.youtube.com/watch?v=wSoGsCXTyig","dialect":"Palestinian spoken Arabic"},
|
| 218 |
+
"verbs":[
|
| 219 |
+
{"hebrew":"יצר, הרכיב","past":{"arabic":"كوّن","phonemes":"k a w w a n"},"future":{"arabic":"يكوّن","phonemes":"j i k a w w i n"}},
|
| 220 |
+
{"hebrew":"המשיך, השלים","past":{"arabic":"كمّل","phonemes":"k a m m a l"},"future":{"arabic":"يكمّل","phonemes":"j i k a m m i l"}},
|
| 221 |
+
{"hebrew":"טאטא","past":{"arabic":"كنس","phonemes":"k a n a s"},"future":{"arabic":"يكنس","phonemes":"j i k n u s"}},
|
| 222 |
+
{"hebrew":"שפך","past":{"arabic":"كبّ","phonemes":"k a b b"},"future":{"arabic":"يكبّ","phonemes":"j i k u b b"}},
|
| 223 |
+
{"hebrew":"חרק שיניים","past":{"arabic":"كدّ","phonemes":"k a d d"},"future":{"arabic":"يكدّ","phonemes":"j i k u d d"}},
|
| 224 |
+
{"hebrew":"עלה במחיר","past":{"arabic":"كلّف","phonemes":"k a l l a f"},"future":{"arabic":"يكلّف","phonemes":"j i k a l l i f"}},
|
| 225 |
+
{"hebrew":"גדל","past":{"arabic":"كبر","phonemes":"k a b a r"},"future":{"arabic":"يكبر","phonemes":"j i k b a r"}},
|
| 226 |
+
{"hebrew":"קימט את המצח","past":{"arabic":"كشّر","phonemes":"k a ʃ ʃ a r"},"future":{"arabic":"يكشّر","phonemes":"j i k a ʃ ʃ i r"}},
|
| 227 |
+
{"hebrew":"שבר","past":{"arabic":"كسر","phonemes":"k a s a r"},"future":{"arabic":"يكسر","phonemes":"j i k s a r"}},
|
| 228 |
+
{"hebrew":"כתב","past":{"arabic":"كتب","phonemes":"k a t a b"},"future":{"arabic":"يكتب","phonemes":"j i k t u b"}},
|
| 229 |
+
{"hebrew":"צבר","past":{"arabic":"كدّس","phonemes":"k a d d a s"},"future":{"arabic":"يكدّس","phonemes":"j i k a d d i s"}},
|
| 230 |
+
{"hebrew":"עשה בלגן","past":{"arabic":"كركب","phonemes":"k a r k a b"},"future":{"arabic":"يكركب","phonemes":"j i k a r k i b"}},
|
| 231 |
+
{"hebrew":"חזר על","past":{"arabic":"كرّر","phonemes":"k a r r a r"},"future":{"arabic":"يكرّر","phonemes":"j i k a r r i r"}},
|
| 232 |
+
{"hebrew":"גירש","past":{"arabic":"كشّ","phonemes":"k a ʃ ʃ"},"future":{"arabic":"يكشّ","phonemes":"j i k u ʃ ʃ"}}
|
| 233 |
+
]
|
| 234 |
+
},
|
| 235 |
+
"lam": {
|
| 236 |
+
"source":{"title":"פעלים שימושיים בערבית מדוברת — אות לאם","url":"https://www.youtube.com/watch?v=-Jd2w7EEs90","dialect":"Palestinian spoken Arabic"},
|
| 237 |
+
"verbs":[
|
| 238 |
+
{"hebrew":"צופף","past":{"arabic":"لزّ","phonemes":"l a z z"},"future":{"arabic":"يلزّ","phonemes":"j i l u z z"}},
|
| 239 |
+
{"hebrew":"עקץ, הכיש, צרב","past":{"arabic":"لسع","phonemes":"l a s a ʕ"},"future":{"arabic":"يلسع","phonemes":"j i l s a ʕ"}},
|
| 240 |
+
{"hebrew":"הפציר","past":{"arabic":"لحّ","phonemes":"l a ħ ħ"},"future":{"arabic":"يلحّ","phonemes":"j i l i ħ ħ"}},
|
| 241 |
+
{"hebrew":"אסף","past":{"arabic":"لمّ","phonemes":"l a m m"},"future":{"arabic":"يلمّ","phonemes":"j i l u m m"}},
|
| 242 |
+
{"hebrew":"האשים, גינה","past":{"arabic":"لام","phonemes":"l aː m"},"future":{"arabic":"يلوم","phonemes":"j i l uː m"}},
|
| 243 |
+
{"hebrew":"לימד בשינון","past":{"arabic":"لقّن","phonemes":"l a Q Q a n"},"future":{"arabic":"يلقّن","phonemes":"j i l a Q Q i n"}},
|
| 244 |
+
{"hebrew":"מילא בקשה","past":{"arabic":"لبّى","phonemes":"l a b b a"},"future":{"arabic":"يلبّي","phonemes":"j i l a b b i"}},
|
| 245 |
+
{"hebrew":"לבש","past":{"arabic":"لبس","phonemes":"l a b a s"},"future":{"arabic":"يلبس","phonemes":"j i l b a s"}},
|
| 246 |
+
{"hebrew":"הלביש","past":{"arabic":"لبّس","phonemes":"l a b b a s"},"future":{"arabic":"يلبّس","phonemes":"j i l a b b i s"}},
|
| 247 |
+
{"hebrew":"טייח, העלים","past":{"arabic":"لفلف","phonemes":"l a f l a f"},"future":{"arabic":"يلفلف","phonemes":"j i l a f l i f"}},
|
| 248 |
+
{"hebrew":"סטר","past":{"arabic":"لطش","phonemes":"l a tˤ a ʃ"},"future":{"arabic":"يلطش","phonemes":"j i l tˤ a ʃ"}},
|
| 249 |
+
{"hebrew":"רמז","past":{"arabic":"لمّح","phonemes":"l a m m a ħ"},"future":{"arabic":"يلمّح","phonemes":"j i l a m m i ħ"}}
|
| 250 |
+
]
|
| 251 |
+
},
|
| 252 |
+
"mim": {
|
| 253 |
+
"source":{"title":"פעלים שימושיים בערבית מדוברת — אות מים","url":"https://www.youtube.com/watch?v=yKUcU45Ngpg","dialect":"Palestinian spoken Arabic"},
|
| 254 |
+
"verbs":[
|
| 255 |
+
{"hebrew":"דחה ועיכב","past":{"arabic":"ماطل","phonemes":"m aː tˤ a l"},"future":{"arabic":"يماطل","phonemes":"j i m aː tˤ i l"}},
|
| 256 |
+
{"hebrew":"עסק ב־, הפעיל","past":{"arabic":"مارس","phonemes":"m aː r a s"},"future":{"arabic":"يمارس","phonemes":"j i m aː r i s"}},
|
| 257 |
+
{"hebrew":"מת","past":{"arabic":"مات","phonemes":"m aː t"},"future":{"arabic":"يموت","phonemes":"j i m uː t"}},
|
| 258 |
+
{"hebrew":"מימן","past":{"arabic":"موّل","phonemes":"m a w w a l"},"future":{"arabic":"يموّل","phonemes":"j i m a w w i l"}},
|
| 259 |
+
{"hebrew":"קרע לגזרים","past":{"arabic":"مزّع","phonemes":"m a z z a ʕ"},"future":{"arabic":"يمزّع","phonemes":"j i m a z z i ʕ"}},
|
| 260 |
+
{"hebrew":"הבחין בין","past":{"arabic":"ميّز","phonemes":"m a j j a z"},"future":{"arabic":"يميّز","phonemes":"j i m a j j i z"}},
|
| 261 |
+
{"hebrew":"מילא","past":{"arabic":"ملا","phonemes":"m a l a"},"future":{"arabic":"يملي","phonemes":"j i m l i"}},
|
| 262 |
+
{"hebrew":"שיבח","past":{"arabic":"مدح","phonemes":"m a d a ħ"},"future":{"arabic":"يمدح","phonemes":"j i m d a ħ"}},
|
| 263 |
+
{"hebrew":"סיפק","past":{"arabic":"موّن","phonemes":"m a w w a n"},"future":{"arabic":"يموّن","phonemes":"j i m a w w i n"}},
|
| 264 |
+
{"hebrew":"שטף, היה בעלים","past":{"arabic":"ملك","phonemes":"m a l a k"},"future":{"arabic":"يملك","phonemes":"j i m l u k"}},
|
| 265 |
+
{"hebrew":"ליטף, החליק","past":{"arabic":"ملّس","phonemes":"m a l l a s"},"future":{"arabic":"يملّس","phonemes":"j i m a l l i s"}},
|
| 266 |
+
{"hebrew":"העניק","past":{"arabic":"منح","phonemes":"m a n a ħ"},"future":{"arabic":"يمنح","phonemes":"j i m n a ħ"}},
|
| 267 |
+
{"hebrew":"מנע, אסר","past":{"arabic":"منع","phonemes":"m a n a ʕ"},"future":{"arabic":"يمنع","phonemes":"j i m n a ʕ"}},
|
| 268 |
+
{"hebrew":"מצץ","past":{"arabic":"مصّ","phonemes":"m a sˤ sˤ"},"future":{"arabic":"يمصّ","phonemes":"j i m u sˤ sˤ"}},
|
| 269 |
+
{"hebrew":"העמיד פנים","past":{"arabic":"مثّل","phonemes":"m a TH TH a l"},"future":{"arabic":"يمثّل","phonemes":"j i m a TH TH i l"}},
|
| 270 |
+
{"hebrew":"ערבב","past":{"arabic":"مزج","phonemes":"m a z a J"},"future":{"arabic":"يمزج","phonemes":"j i m z a J"}},
|
| 271 |
+
{"hebrew":"התנדנד","past":{"arabic":"مرجح","phonemes":"m a r J a ħ"},"future":{"arabic":"يمرجح","phonemes":"j i m a r J i ħ"}},
|
| 272 |
+
{"hebrew":"הוליך, הפעיל","past":{"arabic":"مشّ��","phonemes":"m a ʃ ʃ a"},"future":{"arabic":"يمشّي","phonemes":"j i m a ʃ ʃ i"}},
|
| 273 |
+
{"hebrew":"עבר","past":{"arabic":"مرق","phonemes":"m a r a Q"},"future":{"arabic":"يمرق","phonemes":"j i m r u Q"}}
|
| 274 |
+
]
|
| 275 |
+
},
|
| 276 |
+
"nun": {
|
| 277 |
+
"source":{"title":"פעלים שימושיים בערבית מדוברת — אות נון","url":"https://www.youtube.com/watch?v=KdipB6ik22Y","dialect":"Palestinian spoken Arabic"},
|
| 278 |
+
"verbs":[
|
| 279 |
+
{"hebrew":"ישן, שכב","past":{"arabic":"نام","phonemes":"n aː m"},"future":{"arabic":"ينام","phonemes":"j i n aː m"}},
|
| 280 |
+
{"hebrew":"הרדים","past":{"arabic":"نوّم","phonemes":"n a w w a m"},"future":{"arabic":"ينوّم","phonemes":"j i n a w w i m"}},
|
| 281 |
+
{"hebrew":"שכח","past":{"arabic":"نسي","phonemes":"n a s i"},"future":{"arabic":"ينسى","phonemes":"j i n s a"}},
|
| 282 |
+
{"hebrew":"השכיח","past":{"arabic":"نسّى","phonemes":"n a s s a"},"future":{"arabic":"ينسّي","phonemes":"j i n a s s i"}},
|
| 283 |
+
{"hebrew":"התנמנם","past":{"arabic":"نعس","phonemes":"n i ʕ i s"},"future":{"arabic":"ينعس","phonemes":"j i n ʕ a s"}},
|
| 284 |
+
{"hebrew":"נשף, נשם","past":{"arabic":"نفش","phonemes":"n a f a ʃ"},"future":{"arabic":"ينفش","phonemes":"j i n f u ʃ"}},
|
| 285 |
+
{"hebrew":"ניתר","past":{"arabic":"نطّ","phonemes":"n a tˤ tˤ"},"future":{"arabic":"ينطّ","phonemes":"j i n u tˤ tˤ"}},
|
| 286 |
+
{"hebrew":"העביר, העתיק","past":{"arabic":"نقل","phonemes":"n a Q a l"},"future":{"arabic":"ينقل","phonemes":"j i n Q u l"}},
|
| 287 |
+
{"hebrew":"התחרה","past":{"arabic":"نافس","phonemes":"n aː f a s"},"future":{"arabic":"ينافس","phonemes":"j i n aː f i s"}},
|
| 288 |
+
{"hebrew":"ניקה","past":{"arabic":"نضّف","phonemes":"n a dˤ dˤ a f"},"future":{"arabic":"ينضّف","phonemes":"j i n a dˤ dˤ i f"}},
|
| 289 |
+
{"hebrew":"בחר","past":{"arabic":"نقّى","phonemes":"n a Q Q a"},"future":{"arabic":"ينقّي","phonemes":"j i n a Q Q i"}},
|
| 290 |
+
{"hebrew":"סגר","past":{"arabic":"نتّ","phonemes":"n a t t"},"future":{"arabic":"ينتّ","phonemes":"j i n u t t"}}
|
| 291 |
+
]
|
| 292 |
+
},
|
| 293 |
+
"sin": {
|
| 294 |
+
"source":{"title":"פעלים שימושיים בערבית מדוברת — אות סין","url":"https://www.youtube.com/watch?v=DlM_AscrScY","dialect":"Palestinian spoken Arabic"},
|
| 295 |
+
"verbs":[
|
| 296 |
+
{"hebrew":"החביא, שמר","past":{"arabic":"ستر","phonemes":"s a t a r"},"future":{"arabic":"يستر","phonemes":"j i s t u r"}},
|
| 297 |
+
{"hebrew":"שמע","past":{"arabic":"سمع","phonemes":"s a m i ʕ"},"future":{"arabic":"يسمع","phonemes":"j i s m a ʕ"}},
|
| 298 |
+
{"hebrew":"עזר","past":{"arabic":"ساعد","phonemes":"s aː ʕ a d"},"future":{"arabic":"يساعد","phonemes":"j i s aː ʕ i d"}},
|
| 299 |
+
{"hebrew":"כישף","past":{"arabic":"سحر","phonemes":"s a ħ a r"},"future":{"arabic":"يسحر","phonemes":"j i s ħ a r"}},
|
| 300 |
+
{"hebrew":"שאל","past":{"arabic":"سأل","phonemes":"s a ʔ a l"},"future":{"arabic":"يسأل","phonemes":"j i s ʔ a l"}},
|
| 301 |
+
{"hebrew":"גר, שכן","past":{"arabic":"سكن","phonemes":"s a k a n"},"future":{"arabic":"يسكن","phonemes":"j i s k u n"}},
|
| 302 |
+
{"hebrew":"השתלט","past":{"arabic":"سيطر","phonemes":"s a j tˤ a r"},"future":{"arabic":"يسيطر","phonemes":"j i s a j tˤ i r"}},
|
| 303 |
+
{"hebrew":"נסע","past":{"arabic":"سافر","phonemes":"s aː f a r"},"future":{"arabic":"يسافر","phonemes":"j i s aː f i r"}},
|
| 304 |
+
{"hebrew":"הרגיז","past":{"arabic":"سفّن","phonemes":"s a f f a n"},"future":{"arabic":"يسفّن","phonemes":"j i s a f f i n"}},
|
| 305 |
+
{"hebrew":"סלח","past":{"arabic":"سامح","phonemes":"s aː m a ħ"},"future":{"arabic":"يسامح","phonemes":"j i s aː m i ħ"}},
|
| 306 |
+
{"hebrew":"גנב","past":{"arabic":"سرق","phonemes":"s a r a Q"},"future":{"arabic":"يسرق","phonemes":"j i s r u Q"}},
|
| 307 |
+
{"hebrew":"שתק","past":{"arabic":"سكت","phonemes":"s a k a t"},"future":{"arabic":"يسكت","phonemes":"j i s k u t"}},
|
| 308 |
+
{"hebrew":"צחק","past":{"arabic":"سخف","phonemes":"s a x a f"},"future":{"arabic":"يسخف","phonemes":"j i s x a f"}},
|
| 309 |
+
{"hebrew":"קילל","past":{"arabic":"سبّ","phonemes":"s a b b"},"future":{"arabic":"يسبّ","phonemes":"j i s u b b"}},
|
| 310 |
+
{"hebrew":"עזב","past":{"arabic":"ساب","phonemes":"s aː b"},"future":{"arabic":"يسيب","phonemes":"j i s iː b"}},
|
| 311 |
+
{"hebrew":"שיבח, עזר לשחות","past":{"arabic":"سبّح","phonemes":"s a b b a ħ"},"future":{"arabic":"يسبّح","phonemes":"j i s a b b i ħ"}},
|
| 312 |
+
{"hebrew":"הקל","past":{"arabic":"سهّل","phonemes":"s a h h a l"},"future":{"arabic":"يسهّل","phonemes":"j i s a h h i l"}},
|
| 313 |
+
{"hebrew":"סידר, עשה","past":{"arabic":"سوّى","phonemes":"s a w w a"},"future":{"arabic":"يسوّي","phonemes":"j i s a w w i"}}
|
| 314 |
+
]
|
| 315 |
+
},
|
| 316 |
+
"ayn": {
|
| 317 |
+
"source":{"title":"פעלים שימושיים בערבית מדוברת — אות עין","url":"https://www.youtube.com/watch?v=BZU4jitxcuU","dialect":"Palestinian spoken Arabic"},
|
| 318 |
+
"verbs":[
|
| 319 |
+
{"hebrew":"גילה עירנות","past":{"arabic":"وعى","phonemes":"w a ʕ a"},"future":{"arabic":"يوعى","phonemes":"j i w ʕ a"}},
|
| 320 |
+
{"hebrew":"לימד","past":{"arabic":"علّم","phonemes":"ʕ a l l a m"},"future":{"arabic":"يعلّم","phonemes":"j i ʕ a l l i m"}},
|
| 321 |
+
{"hebrew":"כעס","past":{"arabic":"عكّر","phonemes":"ʕ a k k a r"},"future":{"arabic":"يعكّر","phonemes":"j i ʕ a k k i r"}},
|
| 322 |
+
{"hebrew":"הזיע","past":{"arabic":"عرق","phonemes":"ʕ a r i Q"},"future":{"arabic":"يعرق","phonemes":"j i ʕ r a Q"}},
|
| 323 |
+
{"hebrew":"איפס, כיוון","past":{"arabic":"عيّر","phonemes":"ʕ a j j a r"},"future":{"arabic":"يعيّر","phonemes":"j i ʕ a j j i r"}},
|
| 324 |
+
{"hebrew":"הכיר, ידע","past":{"arabic":"عرف","phonemes":"ʕ i r i f"},"future":{"arabic":"يعرف","phonemes":"j i ʕ r a f"}},
|
| 325 |
+
{"hebrew":"בירך לכבוד חג","past":{"arabic":"عايد","phonemes":"ʕ aː j a d"},"future":{"arabic":"يعايد","phonemes":"j i ʕ aː j i d"}},
|
| 326 |
+
{"hebrew":"ספר","past":{"arabic":"عدّ","phonemes":"ʕ a d d"},"future":{"arabic":"يعدّ","phonemes":"j i ʕ u d d"}},
|
| 327 |
+
{"hebrew":"חזר על","past":{"arabic":"عاد","phonemes":"ʕ aː d"},"future":{"arabic":"يعيد","phonemes":"j i ʕ iː d"}},
|
| 328 |
+
{"hebrew":"שם מכשול","past":{"arabic":"عرقل","phonemes":"ʕ a r Q a l"},"future":{"arabic":"يعرقل","phonemes":"j i ʕ a r Q i l"}},
|
| 329 |
+
{"hebrew":"חי","past":{"arabic":"عاش","phonemes":"ʕ aː ʃ"},"future":{"arabic":"يعيش","phonemes":"j i ʕ iː ʃ"}},
|
| 330 |
+
{"hebrew":"עבר, נכנס","past":{"arabic":"عبر","phonemes":"ʕ a b a r"},"future":{"arabic":"يعبر","phonemes":"j i ʕ b u r"}},
|
| 331 |
+
{"hebrew":"התעקש","past":{"arabic":"عاند","phonemes":"ʕ aː n a d"},"future":{"arabic":"يعاند","phonemes":"j i ʕ aː n i d"}},
|
| 332 |
+
{"hebrew":"נתן פיצוי","past":{"arabic":"عوّض","phonemes":"ʕ a w w a dˤ"},"future":{"arabic":"يعوّض","phonemes":"j i ʕ a w w i dˤ"}},
|
| 333 |
+
{"hebrew":"אהב, חיבב","past":{"arabic":"عزّ","phonemes":"ʕ a z z"},"future":{"arabic":"يعزّ","phonemes":"j i ʕ i z z"}},
|
| 334 |
+
{"hebrew":"הדיח, מיהר","past":{"arabic":"عجّل","phonemes":"ʕ a J J a l"},"future":{"arabic":"يعجّل","phonemes":"j i ʕ a J J i l"}},
|
| 335 |
+
{"hebrew":"בנה","past":{"arabic":"عمّر","phonemes":"ʕ a m m a r"},"future":{"arabic":"يعمّر","phonemes":"j i ʕ a m m i r"}}
|
| 336 |
+
]
|
| 337 |
+
},
|
| 338 |
+
"ghayn": {
|
| 339 |
+
"source":{"title":"פעלים שימושיים בערבית מדוברת — אות ע'ין","url":"https://www.youtube.com/watch?v=kqtBFuKjx8o","dialect":"Palestinian spoken Arabic"},
|
| 340 |
+
"verbs":[
|
| 341 |
+
{"hebrew":"עזב, נעלם","past":{"arabic":"غاب","phonemes":"ɣ aː b"},"future":{"arabic":"يغيب","phonemes":"j i ɣ iː b"}},
|
| 342 |
+
{"hebrew":"שקע","past":{"arabic":"غرق","phonemes":"ɣ i r i Q"},"future":{"arabic":"يغرق","phonemes":"j i ɣ r a Q"}},
|
| 343 |
+
{"hebrew":"שטף","past":{"arabic":"غسل","phonemes":"ɣ a s a l"},"future":{"arabic":"يغسل","phonemes":"j i ɣ s i l"}},
|
| 344 |
+
{"hebrew":"כיסה","past":{"arabic":"غطّى","phonemes":"ɣ a tˤ tˤ a"},"future":{"arabic":"يغطّي","phonemes":"j i ɣ a tˤ tˤ i"}},
|
| 345 |
+
{"hebrew":"שינה","past":{"arabic":"غيّر","phonemes":"ɣ a j j a r"},"future":{"arabic":"يغيّر","phonemes":"j i ɣ a j j i r"}},
|
| 346 |
+
{"hebrew":"רתח","past":{"arabic":"غلى","phonemes":"ɣ a l a"},"future":{"arabic":"يغلي","phonemes":"j i ɣ l i"}},
|
| 347 |
+
{"hebrew":"ניצח","past":{"arabic":"غلب","phonemes":"ɣ a l a b"},"future":{"arabic":"يغلب","phonemes":"j i ɣ l a b"}},
|
| 348 |
+
{"hebrew":"סלח","past":{"arabic":"غفر","phonemes":"ɣ a f a r"},"future":{"arabic":"يغفر","phonemes":"j i ɣ f i r"}}
|
| 349 |
+
]
|
| 350 |
+
},
|
| 351 |
+
"fa": {
|
| 352 |
+
"source":{"title":"פעלים שימושיים בערבית מדוברת — אות פא","url":"https://www.youtube.com/watch?v=pCn0vhBVR5Q","dialect":"Palestinian spoken Arabic"},
|
| 353 |
+
"verbs":[
|
| 354 |
+
{"hebrew":"נכשל","past":{"arabic":"فشل","phonemes":"f a ʃ a l"},"future":{"arabic":"يفشل","phonemes":"j i f ʃ a l"}},
|
| 355 |
+
{"hebrew":"הבין","past":{"arabic":"فهم","phonemes":"f i h i m"},"future":{"arabic":"يفهم","phonemes":"j i f h a m"}},
|
| 356 |
+
{"hebrew":"פתח","past":{"arabic":"فتح","phonemes":"f a t a ħ"},"future":{"arabic":"يفتح","phonemes":"j i f t a ħ"}},
|
| 357 |
+
{"hebrew":"נכנס, עבר","past":{"arabic":"فات","phonemes":"f aː t"},"future":{"arabic":"يفوت","phonemes":"j i f uː t"}},
|
| 358 |
+
{"hebrew":"הפריד","past":{"arabic":"فرّق","phonemes":"f a r r a Q"},"future":{"arabic":"يفرّق","phonemes":"j i f a r r i Q"}},
|
| 359 |
+
{"hebrew":"ברח","past":{"arabic":"فلّ","phonemes":"f a l l"},"future":{"arabic":"يفلّ","phonemes":"j i f u l l"}},
|
| 360 |
+
{"hebrew":"חשב, הניח","past":{"arabic":"فكّر","phonemes":"f a k k a r"},"future":{"arabic":"يفكّر","phonemes":"j i f a k k i r"}},
|
| 361 |
+
{"hebrew":"שחרר, התיר","past":{"arabic":"فكّ","phonemes":"f a k k"},"future":{"arabic":"يفكّ","phonemes":"j i f u k k"}},
|
| 362 |
+
{"hebrew":"התעורר","past":{"arabic":"فاق","phonemes":"f aː Q"},"future":{"arabic":"يفيق","phonemes":"j i f iː Q"}},
|
| 363 |
+
{"hebrew":"הפסיק, ניתק","past":{"arabic":"فصل","phonemes":"f a sˤ a l"},"future":{"arabic":"يفصل","phonemes":"j i f sˤ a l"}}
|
| 364 |
+
]
|
| 365 |
+
},
|
| 366 |
+
"dad": {
|
| 367 |
+
"source":{"title":"פעלים שימושיים בערבית מדוברת — אות דאד","url":"https://www.youtube.com/watch?v=ULZkU-MoU6w","dialect":"Palestinian spoken Arabic"},
|
| 368 |
+
"verbs":[
|
| 369 |
+
{"hebrew":"היכה","past":{"arabic":"ضرب","phonemes":"dˤ a r a b"},"future":{"arabic":"يضرب","phonemes":"j i dˤ r u b"}},
|
| 370 |
+
{"hebrew":"צחק","past":{"arabic":"ضحك","phonemes":"dˤ i ħ i k"},"future":{"arabic":"يضحك","phonemes":"j i dˤ ħ a k"}},
|
| 371 |
+
{"hebrew":"נשאר","past":{"arabic":"ضلّ","phonemes":"dˤ a l l"},"future":{"arabic":"يضلّ","phonemes":"j i dˤ a l l"}},
|
| 372 |
+
{"hebrew":"הלך לאיבוד","past":{"arabic":"ضاع","phonemes":"dˤ aː ʕ"},"future":{"arabic":"يضيع","phonemes":"j i dˤ iː ʕ"}},
|
| 373 |
+
{"hebrew":"הבטיח","past":{"arabic":"ضمن","phonemes":"dˤ a m a n"},"future":{"arabic":"يضمن","phonemes":"j i dˤ m a n"}},
|
| 374 |
+
{"hebrew":"לחץ","past":{"arabic":"ضغط","phonemes":"dˤ a ɣ a tˤ"},"future":{"arabic":"يضغط","phonemes":"j i dˤ ɣ a tˤ"}}
|
| 375 |
+
]
|
| 376 |
+
},
|
| 377 |
+
"sad": {
|
| 378 |
+
"source":{"title":"פעלים שימושיים בערבית מדוברת — אות צאד","url":"https://www.youtube.com/watch?v=jCpab4z3dXY","dialect":"Palestinian spoken Arabic"},
|
| 379 |
+
"verbs":[
|
| 380 |
+
{"hebrew":"קרה","past":{"arabic":"صار","phonemes":"sˤ aː r"},"future":{"arabic":"يصير","phonemes":"j i sˤ iː r"}},
|
| 381 |
+
{"hebrew":"צעק","past":{"arabic":"صرخ","phonemes":"sˤ a r a x"},"future":{"arabic":"يصرخ","phonemes":"j i sˤ r a x"}},
|
| 382 |
+
{"hebrew":"תיקן","past":{"arabic":"صلّح","phonemes":"sˤ a l l a ħ"},"future":{"arabic":"يصلّح","phonemes":"j i sˤ a l l i ħ"}},
|
| 383 |
+
{"hebrew":"צם","past":{"arabic":"صام","phonemes":"sˤ aː m"},"future":{"arabic":"يصوم","phonemes":"j i sˤ uː m"}},
|
| 384 |
+
{"hebrew":"צילם","past":{"arabic":"صوّر","phonemes":"sˤ a w w a r"},"future":{"arabic":"يصوّر","phonemes":"j i sˤ a w w i r"}},
|
| 385 |
+
{"hebrew":"סגר","past":{"arabic":"سكّر","phonemes":"s a k k a r"},"future":{"arabic":"يسكّر","phonemes":"j i s a k k i r"}},
|
| 386 |
+
{"hebrew":"התעורר","past":{"arabic":"صحي","phonemes":"sˤ i ħ i"},"future":{"arabic":"يصحى","phonemes":"j i sˤ ħ a"}}
|
| 387 |
+
]
|
| 388 |
+
},
|
| 389 |
+
"qaf": {
|
| 390 |
+
"source":{"title":"פעלים שימושיים בערבית מדוברת — אות קאף","url":"https://www.youtube.com/watch?v=zx4scJYgYAM","dialect":"Palestinian spoken Arabic"},
|
| 391 |
+
"verbs":[
|
| 392 |
+
{"hebrew":"אמר","past":{"arabic":"قال","phonemes":"Q aː l"},"future":{"arabic":"يقول","phonemes":"j i Q uː l"}},
|
| 393 |
+
{"hebrew":"קרא","past":{"arabic":"قرأ","phonemes":"Q a r a ʔ"},"future":{"arabic":"يقرأ","phonemes":"j i Q r a ʔ"}},
|
| 394 |
+
{"hebrew":"ישב","past":{"arabic":"قعد","phonemes":"Q a ʕ a d"},"future":{"arabic":"يقعد","phonemes":"j i Q ʕ a d"}},
|
| 395 |
+
{"hebrew":"קם","past":{"arabic":"قام","phonemes":"Q aː m"},"future":{"arabic":"يقوم","phonemes":"j i Q uː m"}},
|
| 396 |
+
{"hebrew":"החליט","past":{"arabic":"قرّر","phonemes":"Q a r r a r"},"future":{"arabic":"يقرّر","phonemes":"j i Q a r r i r"}},
|
| 397 |
+
{"hebrew":"חתך","past":{"arabic":"قطع","phonemes":"Q a tˤ a ʕ"},"future":{"arabic":"يقطع","phonemes":"j i Q tˤ a ʕ"}},
|
| 398 |
+
{"hebrew":"סגר","past":{"arabic":"قفل","phonemes":"Q a f a l"},"future":{"arabic":"يقفل","phonemes":"j i Q f u l"}},
|
| 399 |
+
{"hebrew":"הסכים","past":{"arabic":"قبل","phonemes":"Q i b i l"},"future":{"arabic":"يقبل","phonemes":"j i Q b a l"}},
|
| 400 |
+
{"hebrew":"מדד","past":{"arabic":"قاس","phonemes":"Q aː s"},"future":{"arabic":"يقيس","phonemes":"j i Q iː s"}},
|
| 401 |
+
{"hebrew":"פגש","past":{"arabic":"قابل","phonemes":"Q aː b a l"},"future":{"arabic":"يقابل","phonemes":"j i Q aː b i l"}}
|
| 402 |
+
]
|
| 403 |
+
},
|
| 404 |
+
"ra": {
|
| 405 |
+
"source":{"title":"פעלים שימושיים בערבית מדוברת — אות רא","url":"https://www.youtube.com/watch?v=fMIJT4HowVg","dialect":"Palestinian spoken Arabic"},
|
| 406 |
+
"verbs":[
|
| 407 |
+
{"hebrew":"הלך","past":{"arabic":"راح","phonemes":"r aː ħ"},"future":{"arabic":"يروح","phonemes":"j i r uː ħ"}},
|
| 408 |
+
{"hebrew":"ראה","past":{"arabic":"رأى","phonemes":"r a ʔ a"},"future":{"arabic":"يرى","phonemes":"j i r a"}},
|
| 409 |
+
{"hebrew":"חזר","past":{"arabic":"رجع","phonemes":"r a J a ʕ"},"future":{"arabic":"يرجع","phonemes":"j i r J a ʕ"}},
|
| 410 |
+
{"hebrew":"ענה","past":{"arabic":"ردّ","phonemes":"r a d d"},"future":{"arabic":"يردّ","phonemes":"j i r u d d"}},
|
| 411 |
+
{"hebrew":"רכב","past":{"arabic":"ركب","phonemes":"r i k i b"},"future":{"arabic":"يركب","phonemes":"j i r k a b"}},
|
| 412 |
+
{"hebrew":"רץ","past":{"arabic":"ركض","phonemes":"r a k a dˤ"},"future":{"arabic":"يركض","phonemes":"j i r k u dˤ"}},
|
| 413 |
+
{"hebrew":"סידר","past":{"arabic":"رتّب","phonemes":"r a t t a b"},"future":{"arabic":"يرتّب","phonemes":"j i r a t t i b"}},
|
| 414 |
+
{"hebrew":"זרק","past":{"arabic":"رمى","phonemes":"r a m a"},"future":{"arabic":"يرمي","phonemes":"j i r m i"}},
|
| 415 |
+
{"hebrew":"ניצח","past":{"arabic":"ربح","phonemes":"r i b i ħ"},"future":{"arabic":"يربح","phonemes":"j i r b a ħ"}},
|
| 416 |
+
{"hebrew":"גידל","past":{"arabic":"ربّى","phonemes":"r a b b a"},"future":{"arabic":"يربّي","phonemes":"j i r a b b i"}}
|
| 417 |
+
]
|
| 418 |
+
},
|
| 419 |
+
"shin": {
|
| 420 |
+
"source":{"title":"פעלים שימושיים בערבית מדוברת — אות שין","url":"https://www.youtube.com/watch?v=yrL4wM1-jH8","dialect":"Palestinian spoken Arabic"},
|
| 421 |
+
"verbs":[
|
| 422 |
+
{"hebrew":"ראה","past":{"arabic":"شاف","phonemes":"ʃ aː f"},"future":{"arabic":"يشوف","phonemes":"j i ʃ uː f"}},
|
| 423 |
+
{"hebrew":"שתה","past":{"arabic":"شرب","phonemes":"ʃ i r i b"},"future":{"arabic":"يشرب","phonemes":"j i ʃ r a b"}},
|
| 424 |
+
{"hebrew":"עבד","past":{"arabic":"اشتغل","phonemes":"ʔ i ʃ t a ɣ a l"},"future":{"arabic":"يشتغل","phonemes":"j i ʃ t a ɣ i l"}},
|
| 425 |
+
{"hebrew":"קנה","past":{"arabic":"شرى","phonemes":"ʃ a r a"},"future":{"arabic":"يشري","phonemes":"j i ʃ r i"}},
|
| 426 |
+
{"hebrew":"הסביר","past":{"arabic":"شرح","phonemes":"ʃ a r a ħ"},"future":{"arabic":"يشرح","phonemes":"j i ʃ r a ħ"}},
|
| 427 |
+
{"hebrew":"הדליק","past":{"arabic":"شغّل","phonemes":"ʃ a ɣ ɣ a l"},"future":{"arabic":"يشغّل","phonemes":"j i ʃ a ɣ ɣ i l"}},
|
| 428 |
+
{"hebrew":"סחב","past":{"arabic":"شال","phonemes":"ʃ aː l"},"future":{"arabic":"يشيل","phonemes":"j i ʃ iː l"}},
|
| 429 |
+
{"hebrew":"השתתף","past":{"arabic":"شارك","phonemes":"ʃ aː r a k"},"future":{"arabic":"يشارك","phonemes":"j i ʃ aː r i k"}},
|
| 430 |
+
{"hebrew":"קילל","past":{"arabic":"شتم","phonemes":"ʃ a t a m"},"future":{"arabic":"يشتم","phonemes":"j i ʃ t u m"}},
|
| 431 |
+
{"hebrew":"משך","past":{"arabic":"شدّ","phonemes":"ʃ a d d"},"future":{"arabic":"يشدّ","phonemes":"j i ʃ u d d"}}
|
| 432 |
+
]
|
| 433 |
+
},
|
| 434 |
+
"ta": {
|
| 435 |
+
"source":{"title":"פעלים שימושיים בערבית מדוברת — אות תא, חלק 1","url":"https://www.youtube.com/watch?v=z41lWU0o54M","dialect":"Palestinian spoken Arabic"},
|
| 436 |
+
"verbs":[
|
| 437 |
+
{"hebrew":"עזב","past":{"arabic":"ترك","phonemes":"t a r a k"},"future":{"arabic":"يترك","phonemes":"j i t r u k"}},
|
| 438 |
+
{"hebrew":"עייף","past":{"arabic":"تعب","phonemes":"t i ʕ i b"},"future":{"arabic":"يتعب","phonemes":"j i t ʕ a b"}},
|
| 439 |
+
{"hebrew":"למד","past":{"arabic":"تعلّم","phonemes":"t a ʕ a l l a m"},"future":{"arabic":"يتعلّم","phonemes":"j i t a ʕ a l l a m"}},
|
| 440 |
+
{"hebrew":"דיבר","past":{"arabic":"تكلّم","phonemes":"t k a l l a m"},"future":{"arabic":"يتكلّم","phonemes":"j i t k a l l a m"}},
|
| 441 |
+
{"hebrew":"התחתן","past":{"arabic":"تجوّز","phonemes":"t J a w w a z"},"future":{"arabic":"يتجوّز","phonemes":"j i t J a w w a z"}},
|
| 442 |
+
{"hebrew":"איחר","past":{"arabic":"تأخّر","phonemes":"t ʔ a x x a r"},"future":{"arabic":"يتأخّر","phonemes":"j i t ʔ a x x a r"}},
|
| 443 |
+
{"hebrew":"התקשר","past":{"arabic":"اتّصل","phonemes":"ʔ i t t a sˤ a l"},"future":{"arabic":"يتّصل","phonemes":"j i t t a sˤ i l"}},
|
| 444 |
+
{"hebrew":"השתנה","past":{"arabic":"تغيّر","phonemes":"t ɣ a j j a r"},"future":{"arabic":"يتغيّر","phonemes":"j i t ɣ a j j a r"}},
|
| 445 |
+
{"hebrew":"הסכים","past":{"arabic":"اتّفق","phonemes":"ʔ i t t a f a Q"},"future":{"arabic":"يتّفق","phonemes":"j i t t a f i Q"}},
|
| 446 |
+
{"hebrew":"התקדם","past":{"arabic":"تقدّم","phonemes":"t Q a d d a m"},"future":{"arabic":"يتقدّم","phonemes":"j i t Q a d d a m"}},
|
| 447 |
+
{"hebrew":"תרגם","past":{"arabic":"ترجم","phonemes":"t a r J a m"},"future":{"arabic":"يترجم","phonemes":"j i t a r J i m"}},
|
| 448 |
+
{"hebrew":"נזכר","past":{"arabic":"تذكّر","phonemes":"t DH a k k a r"},"future":{"arabic":"يتذكّر","phonemes":"j i t DH a k k a r"}}
|
| 449 |
+
]
|
| 450 |
+
}
|
| 451 |
+
}
|
static/index.html
CHANGED
|
@@ -310,6 +310,7 @@ body.driving .mic::before{display:none} body.driving .mic-zone.two{display:flex}
|
|
| 310 |
.flag-facts{display:grid;grid-template-columns:auto 1fr;gap:6px 12px;margin:8px 0 0}
|
| 311 |
.flag-facts dt{color:var(--muted);font-size:13px;white-space:nowrap}
|
| 312 |
.flag-facts dd{margin:0;min-width:0;overflow-wrap:anywhere}
|
|
|
|
| 313 |
.flag-facts .ipa{direction:ltr;text-align:right;font:13px/1.5 ui-monospace,monospace;unicode-bidi:isolate}
|
| 314 |
.flag-facts ul{margin:0;padding:0;list-style:none;display:flex;flex-direction:column;gap:3px}
|
| 315 |
.flag-facts b.good{color:var(--olive)} .flag-facts b.mid{color:#a86a00} .flag-facts b.bad{color:#9f2f27}
|
|
@@ -571,6 +572,55 @@ dialog::backdrop{background:rgba(0,0,0,.45)}
|
|
| 571 |
.dlg-head h2{font:400 22px/1 var(--f-display);margin:0}
|
| 572 |
.dlg-head h2 small{display:block;margin-top:4px;font:600 13px/1.3 var(--f-body);color:var(--muted)}
|
| 573 |
.phone-note{margin:4px 0 12px;color:var(--muted);font-size:13px}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 574 |
.phone-rows{direction:ltr;overflow-x:auto;padding:3px 0 8px}
|
| 575 |
.phone-row{display:grid;grid-template-columns:72px repeat(var(--phones),minmax(34px,1fr));gap:4px;min-width:max-content;margin-bottom:5px;align-items:stretch}
|
| 576 |
.phone-row>span{display:grid;place-items:center;min-height:34px;padding:3px 5px;border-radius:7px;background:var(--bg);font:600 16px/1.2 sans-serif}
|
|
@@ -976,6 +1026,15 @@ body.quiet :is(#mic,#playMine,#driveBtn,.phone-btn),body.driving :is(#quietBtn,#
|
|
| 976 |
<p class="phone-note">זהו פירוש המודל להקלטה, ולא תעתיק פונטי מוסמך. תנועות קרובות ואורך תנועה מסומנים כהתאמה מקורבת.</p>
|
| 977 |
<div class="phone-rows" id="phoneRows"></div>
|
| 978 |
<div class="phone-legend"><span><i class="good"></i>התאמה</span><span><i style="background:var(--olive-soft)"></i>התאמה מקורבת</span><span><i class="bad"></i>החלפה או השמטה</span><span><i class="mid"></i>תוספת</span></div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 979 |
<section id="trackBox"><h3 class="heatmap-title">מיקום הפונמות בזמן</h3><div class="phone-track-wrap"><canvas id="phoneTrack" role="img" aria-label="רצף הפונמות הצפוי והמזוהה לאורך זמן"></canvas></div></section>
|
| 980 |
<section id="heatmapBox"><h3 class="heatmap-title">הסתברות פונמות לאורך ההקלטה</h3><div class="heatmap-wrap"><canvas id="phoneHeatmap" role="img" aria-label="מפת חום של הסתברויות הפונמות לאורך ההקלטה"></canvas></div><div class="heatmap-scale"><span>0%</span><i></i><span>100%</span><span>· ★ פונמה צפויה</span></div></section>
|
| 981 |
</dialog>
|
|
@@ -2170,7 +2229,8 @@ function alignPhones(expected,heard){
|
|
| 2170 |
return out.reverse();
|
| 2171 |
}
|
| 2172 |
function drawPhones(r){
|
| 2173 |
-
|
|
|
|
| 2174 |
const aligned=alignPhones(expected,heard), rows=$("phoneRows"); rows.innerHTML=""; rows.style.setProperty("--phones",aligned.length);
|
| 2175 |
const make=(name,which)=>{
|
| 2176 |
const row=mark("phone-row","","div"); row.append(mark("phone-label",name));
|
|
@@ -2182,8 +2242,144 @@ function drawPhones(r){
|
|
| 2182 |
rows.append(row);
|
| 2183 |
};
|
| 2184 |
make("צפוי",0); make("המודל שמע",1);
|
|
|
|
|
|
|
| 2185 |
drawPhoneTrack(r.phoneme_heatmap);
|
| 2186 |
drawHeatmap(r.phoneme_heatmap);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2187 |
}
|
| 2188 |
function heatColor(value){
|
| 2189 |
const stops=[[245,247,245],[159,196,172],[40,116,93],[240,180,60],[159,47,39]], x=Math.max(0,Math.min(100,value))/25;
|
|
|
|
| 310 |
.flag-facts{display:grid;grid-template-columns:auto 1fr;gap:6px 12px;margin:8px 0 0}
|
| 311 |
.flag-facts dt{color:var(--muted);font-size:13px;white-space:nowrap}
|
| 312 |
.flag-facts dd{margin:0;min-width:0;overflow-wrap:anywhere}
|
| 313 |
+
.phone-dialect{direction:rtl;text-align:right;margin-top:6px;font-size:13px;color:var(--muted)}
|
| 314 |
.flag-facts .ipa{direction:ltr;text-align:right;font:13px/1.5 ui-monospace,monospace;unicode-bidi:isolate}
|
| 315 |
.flag-facts ul{margin:0;padding:0;list-style:none;display:flex;flex-direction:column;gap:3px}
|
| 316 |
.flag-facts b.good{color:var(--olive)} .flag-facts b.mid{color:#a86a00} .flag-facts b.bad{color:#9f2f27}
|
|
|
|
| 572 |
.dlg-head h2{font:400 22px/1 var(--f-display);margin:0}
|
| 573 |
.dlg-head h2 small{display:block;margin-top:4px;font:600 13px/1.3 var(--f-body);color:var(--muted)}
|
| 574 |
.phone-note{margin:4px 0 12px;color:var(--muted);font-size:13px}
|
| 575 |
+
/* the phoneme analysis is a teacher's desktop tool: it takes the whole screen */
|
| 576 |
+
#phoneDlg{width:min(98vw,1880px);max-height:95vh}
|
| 577 |
+
.stage-flow{display:grid;grid-template-columns:repeat(10,minmax(0,1fr));gap:10px;margin-bottom:14px}
|
| 578 |
+
.stage-card{position:relative;display:flex;flex-direction:column;gap:6px;padding:10px 10px 9px;border:1px solid var(--line);border-radius:12px;background:var(--bg);font-size:12px;line-height:1.35;cursor:default}
|
| 579 |
+
.stage-card:not(:last-child)::after{content:"←";position:absolute;left:-10px;top:14px;color:var(--muted);font-size:13px}
|
| 580 |
+
.stage-card:hover,.stage-card.on{border-color:var(--olive);box-shadow:0 0 0 2px var(--olive-soft)}
|
| 581 |
+
.stage-card h4{margin:0;font:700 13px/1.25 var(--f-body)}
|
| 582 |
+
.stage-card h4 b{display:inline-grid;place-items:center;width:19px;height:19px;margin-inline-end:5px;border-radius:50%;background:var(--olive);color:#fff;font-size:11px}
|
| 583 |
+
.stage-card .what{color:var(--muted)}
|
| 584 |
+
.stage-card dl{display:grid;grid-template-columns:1fr auto;gap:2px 6px;margin:0}
|
| 585 |
+
.stage-card dt{color:var(--muted)}
|
| 586 |
+
.stage-card dd{margin:0;font-weight:700;font-variant-numeric:tabular-nums;direction:ltr;text-align:left}
|
| 587 |
+
.stage-card .formula{direction:ltr;text-align:left;overflow-wrap:anywhere;padding:4px 6px;border-radius:6px;background:var(--surface);font:12px/1.4 ui-monospace,monospace}
|
| 588 |
+
.stage-card .got{margin-top:auto;padding-top:6px;border-top:1px dashed var(--line);font-weight:700}
|
| 589 |
+
.stage-grid{display:grid;grid-template-columns:1fr 1fr;gap:12px}
|
| 590 |
+
.stage-panel{border:1px solid var(--line);border-radius:12px;padding:10px 12px;font-size:13px}
|
| 591 |
+
.stage-panel h4{margin:0 0 6px;font:700 13px/1.3 var(--f-body)}
|
| 592 |
+
.stage-panel table{width:100%;border-collapse:collapse}
|
| 593 |
+
.stage-panel td,.stage-panel th{padding:3px 6px;border-bottom:1px solid var(--line);text-align:start;font-variant-numeric:tabular-nums}
|
| 594 |
+
.stage-panel tr.chosen{background:var(--ok-bg)}
|
| 595 |
+
.stage-panel .eq{direction:ltr;text-align:left;font:13px/1.7 ui-monospace,monospace}
|
| 596 |
+
.ltr{direction:ltr;unicode-bidi:isolate;font-family:ui-monospace,monospace}
|
| 597 |
+
.stage-bar{position:relative;height:10px;margin:6px 0 2px;border-radius:999px;background:var(--line);direction:ltr}
|
| 598 |
+
.stage-bar i{position:absolute;inset:0 auto 0 0;border-radius:999px;background:var(--olive)}
|
| 599 |
+
.stage-bar b{position:absolute;top:-3px;bottom:-3px;width:2px;background:var(--sumac)}
|
| 600 |
+
.stage-table-wrap{overflow:auto;max-height:62vh;border:1px solid var(--line);border-radius:10px}
|
| 601 |
+
.stage-table{width:100%;border-collapse:separate;border-spacing:0;font-size:13px;font-variant-numeric:tabular-nums}
|
| 602 |
+
.stage-table th{position:sticky;top:0;z-index:1;background:var(--surface);padding:6px 7px;border-bottom:1px solid var(--line);font:700 12px/1.25 var(--f-body);text-align:center;vertical-align:bottom}
|
| 603 |
+
.stage-table th small{display:block;font-weight:500;color:var(--muted)}
|
| 604 |
+
.stage-table td{padding:5px 7px;border-bottom:1px solid var(--line);text-align:center;white-space:nowrap}
|
| 605 |
+
.stage-table tr.stage-word td{background:var(--olive-soft);font-weight:700;text-align:start}
|
| 606 |
+
.stage-table tr.stage-word td .eq{display:inline-block;font:12px ui-monospace,monospace;direction:ltr;unicode-bidi:isolate;margin-inline-start:28px;font-weight:500;color:var(--muted)}
|
| 607 |
+
.stage-table td.ok{background:var(--ok-bg);color:var(--ok)}
|
| 608 |
+
.stage-table td.mid{background:var(--amber-bg);color:var(--amber)}
|
| 609 |
+
.stage-table td.bad{background:var(--sumac-bg);color:var(--sumac)}
|
| 610 |
+
.stage-table td.chg{box-shadow:inset 0 0 0 2px var(--olive)}
|
| 611 |
+
.stage-table td.skip{color:var(--muted)}
|
| 612 |
+
.stage-table td small{display:block;font-size:11px;color:var(--muted);font-weight:500}
|
| 613 |
+
.stage-table .dec-flag{color:var(--sumac);font-weight:700}
|
| 614 |
+
.stage-table .dec-issue{color:var(--amber);font-weight:700}
|
| 615 |
+
.stage-table .dec-ok{color:var(--ok)}
|
| 616 |
+
#stagesBox[data-hl="gop"] .st-gop,#stagesBox[data-hl="exemption"] .st-exemption,#stagesBox[data-hl="baseline"] .st-baseline,
|
| 617 |
+
#stagesBox[data-hl="calibration"] .st-calibration,#stagesBox[data-hl="decision"] .st-decision,#stagesBox[data-hl="variant"] .st-variant,
|
| 618 |
+
#stagesBox[data-hl="tips"] .st-tips{outline:2px solid var(--olive);outline-offset:-2px}
|
| 619 |
+
#stagesBox[data-hl="word"] tr.stage-word td,#stagesBox[data-hl="sentence"] #stageSentence,#stagesBox[data-hl="unrelated"] #stageVariants{outline:2px solid var(--olive);outline-offset:-2px}
|
| 620 |
+
.stage-legend{display:flex;gap:8px 16px;flex-wrap:wrap;margin-top:6px;font-size:12px;color:var(--muted)}
|
| 621 |
+
.stage-legend i{display:inline-block;width:10px;height:10px;border-radius:3px;margin-inline-end:4px}
|
| 622 |
+
.stage-legend .ok{background:var(--ok-bg)}.stage-legend .mid{background:var(--amber-bg)}.stage-legend .bad{background:var(--sumac-bg)}
|
| 623 |
+
.stage-legend .chg{box-shadow:inset 0 0 0 2px var(--olive)}
|
| 624 |
.phone-rows{direction:ltr;overflow-x:auto;padding:3px 0 8px}
|
| 625 |
.phone-row{display:grid;grid-template-columns:72px repeat(var(--phones),minmax(34px,1fr));gap:4px;min-width:max-content;margin-bottom:5px;align-items:stretch}
|
| 626 |
.phone-row>span{display:grid;place-items:center;min-height:34px;padding:3px 5px;border-radius:7px;background:var(--bg);font:600 16px/1.2 sans-serif}
|
|
|
|
| 1026 |
<p class="phone-note">זהו פירוש המודל להקלטה, ולא תעתיק פונטי מוסמך. תנועות קרובות ואורך תנועה מסומנים כהתאמה מקורבת.</p>
|
| 1027 |
<div class="phone-rows" id="phoneRows"></div>
|
| 1028 |
<div class="phone-legend"><span><i class="good"></i>התאמה</span><span><i style="background:var(--olive-soft)"></i>התאמה מקורבת</span><span><i class="bad"></i>החלפה או השמטה</span><span><i class="mid"></i>תוספת</span></div>
|
| 1029 |
+
<section id="stagesBox" hidden>
|
| 1030 |
+
<h3 class="heatmap-title">שלבי הסיווג</h3>
|
| 1031 |
+
<p class="phone-note">כל הגה עובר את השלבים לפי הסדר. הכרטיסים מציגים את הספים והמשקלים של כל שלב ומה קרה בהקלטה הזו. מעבר עם העכבר על כרטיס מדגיש את העמודות שלו בטבלה.</p>
|
| 1032 |
+
<div class="stage-flow" id="stageFlow"></div>
|
| 1033 |
+
<div class="stage-grid"><div class="stage-panel" id="stageVariants"></div><div class="stage-panel" id="stageSentence"></div></div>
|
| 1034 |
+
<h3 class="heatmap-title">הגה אחר הגה</h3>
|
| 1035 |
+
<div class="stage-table-wrap"><table class="stage-table" id="stageTable"></table></div>
|
| 1036 |
+
<div class="stage-legend"><span><i class="ok"></i>מעל סף הרשימה</span><span><i class="mid"></i>בין סף הסימון לסף הרשימה</span><span><i class="bad"></i>מתחת לסף הסימון</span><span><i class="chg"></i>השלב שינה את הציון</span></div>
|
| 1037 |
+
</section>
|
| 1038 |
<section id="trackBox"><h3 class="heatmap-title">מיקום הפונמות בזמן</h3><div class="phone-track-wrap"><canvas id="phoneTrack" role="img" aria-label="רצף הפונמות הצפוי והמזוהה לאורך זמן"></canvas></div></section>
|
| 1039 |
<section id="heatmapBox"><h3 class="heatmap-title">הסתברות פונמות לאורך ההקלטה</h3><div class="heatmap-wrap"><canvas id="phoneHeatmap" role="img" aria-label="מפת חום של הסתברויות הפונמות לאורך ההקלטה"></canvas></div><div class="heatmap-scale"><span>0%</span><i></i><span>100%</span><span>· ★ פונמה צפויה</span></div></section>
|
| 1040 |
</dialog>
|
|
|
|
| 2229 |
return out.reverse();
|
| 2230 |
}
|
| 2231 |
function drawPhones(r){
|
| 2232 |
+
// a letter with several correct realisations (ق: ʔ / k / q) is compared with the one said, so an accepted one is not a mismatch
|
| 2233 |
+
const expected=r.words.flatMap(w=>(w.phonemes||[]).map(p=>p.realised&&!p.flagged?p.realised:p.ipa)), heard=(r.heard_ipa||"").trim().split(/\s+/).filter(Boolean);
|
| 2234 |
const aligned=alignPhones(expected,heard), rows=$("phoneRows"); rows.innerHTML=""; rows.style.setProperty("--phones",aligned.length);
|
| 2235 |
const make=(name,which)=>{
|
| 2236 |
const row=mark("phone-row","","div"); row.append(mark("phone-label",name));
|
|
|
|
| 2242 |
rows.append(row);
|
| 2243 |
};
|
| 2244 |
make("צפוי",0); make("המודל שמע",1);
|
| 2245 |
+
const dialect=r.mixed_dialect?"ניב: מעורב (עירוני וכפרי באותו משפט, לא הוריד ציון)":r.dialect==="urban"?"ניב: עירוני":r.dialect==="rural"?"ניב: כפרי":"";
|
| 2246 |
+
if(dialect) rows.append(mark("phone-dialect",dialect,"div"));
|
| 2247 |
drawPhoneTrack(r.phoneme_heatmap);
|
| 2248 |
drawHeatmap(r.phoneme_heatmap);
|
| 2249 |
+
drawStages(r);
|
| 2250 |
+
}
|
| 2251 |
+
// The classification stage by stage (r.trace, teachers only): the flow with each stage's thresholds and weights,
|
| 2252 |
+
// then every sound through the stages. Scores are 0..100; probabilities are the model's posteriors in the sound's window.
|
| 2253 |
+
const EXEMPTION_NAMES={glottal_dropped:"עצירה סותמית שהושמטה (ء / ق)",article_vowel_elided:"תנועת ה־ال אחרי תנועה",near_vowel:"תנועה קרובה"};
|
| 2254 |
+
function drawStages(r){
|
| 2255 |
+
const box=$("stagesBox"), t=r.trace;
|
| 2256 |
+
if(!t){ box.hidden=true; return; }
|
| 2257 |
+
box.hidden=false;
|
| 2258 |
+
const c=t.config, toks=t.tokens;
|
| 2259 |
+
const E=(tag,cls,...kids)=>{ const e=document.createElement(tag); if(cls) e.className=cls; for(const k of kids) if(k!=null&&k!=="") e.append(k); return e; };
|
| 2260 |
+
const num=(x,d=1)=>x==null?"—":String(Math.round(x*10**d)/10**d);
|
| 2261 |
+
const pct=x=>x==null?"—":x>0&&x<0.001?"<0.1%":num(100*x,1)+"%";
|
| 2262 |
+
const ltr=x=>E("span","ltr",String(x));
|
| 2263 |
+
const band=s=>s>=c.issue_below?"ok":s>=c.flag_below?"mid":"bad";
|
| 2264 |
+
const count=f=>toks.filter(f).length;
|
| 2265 |
+
const stageOf=(x,name)=>x.stages.find(s=>s.stage===name);
|
| 2266 |
+
|
| 2267 |
+
// the flow: one card per stage, in order
|
| 2268 |
+
const flow=$("stageFlow"); flow.innerHTML="";
|
| 2269 |
+
const card=(id,n,title,what,params,formula,got)=>{
|
| 2270 |
+
const el=E("div","stage-card",E("h4","",E("b","",String(n)),title),E("div","what",what));
|
| 2271 |
+
el.dataset.stage=id;
|
| 2272 |
+
if(params.length){ const dl=E("dl"); for(const [k,v] of params) dl.append(E("dt","",k),E("dd","",String(v))); el.append(dl); }
|
| 2273 |
+
if(formula) el.append(E("div","formula",formula));
|
| 2274 |
+
el.append(E("div","got",got));
|
| 2275 |
+
el.onmouseenter=()=>{ box.dataset.hl=id; };
|
| 2276 |
+
el.onmouseleave=()=>{ delete box.dataset.hl; };
|
| 2277 |
+
flow.append(el);
|
| 2278 |
+
};
|
| 2279 |
+
const chosen=t.variants.find(v=>v.chosen)||{};
|
| 2280 |
+
const u=t.unrelated, sen=t.sentence;
|
| 2281 |
+
const flaggedN=count(x=>x.flagged), issueN=count(x=>x.issue), underFlag=count(x=>x.final<c.flag_below&&!x.flagged);
|
| 2282 |
+
card("variant",1,"בחירת וריאנט","יישור כפוי (Viterbi) של כל וריאנט מקובל, כולל צורות דיפתונג; הסימנים המופשטים (Q, K, TH...) מאחדים מימושים.",
|
| 2283 |
+
[["וריאנטים שנבדקו",t.variants.length],["קריטריון","log P מסלול מרבי"]],null,
|
| 2284 |
+
`נבחר: log P ${num(chosen.logp,2)}`);
|
| 2285 |
+
card("unrelated",2,"משפט אחר?","מרחק עריכה בין הפענוח החופשי לווריאנט, יחסית למספר ההגאים.",
|
| 2286 |
+
[["סף יחסי",c.unrelated_ratio],["תקרת ציון",c.unrelated_cap]],`${u.distance} / ${u.tokens} = ${u.ratio}`,
|
| 2287 |
+
u.unrelated?"משפט אחר: הציון מוגבל, בלי טיפים":"אותו משפט");
|
| 2288 |
+
card("gop",3,"GOP לכל הגה","יחס בין ההגה הצפוי למתחרה החזק בחלון שלו, מוכפל בביטחון.",
|
| 2289 |
+
[["λ סלחנות",c.leniency],["p לביטחון מלא",c.confident_peak],["השמטה: p צפוי <",c.deleted_peak],["… וגם p מתחרה <",c.deleted_comp]],
|
| 2290 |
+
`100·e^(GOP·${c.leniency})·√min(1,p/${c.confident_peak})`,
|
| 2291 |
+
`${count(x=>stageOf(x,"gop").score<c.flag_below)} הגאים מתחת ל־${c.flag_below} · ${count(x=>x.deleted)} הושמטו`);
|
| 2292 |
+
card("exemption",4,"פטורים (6.3)","מה שהמודל שומע כטעות אבל הוא דיבור רגיל.",
|
| 2293 |
+
c.exemptions.map(e=>[EXEMPTION_NAMES[e]||e,e==="near_vowel"?"≥ "+c.near_vowel_floor:"100"]),null,
|
| 2294 |
+
`${count(x=>stageOf(x,"exemption").applied)} פטורים הוחלו`);
|
| 2295 |
+
card("baseline",5,"קו בסיס ילידי","ציון ההגה מחולק בציון שהמודל נותן לקול הילידי על אותו הגה.",
|
| 2296 |
+
[["הקלטות ילידיות",t.references.length],["ממוצע מינימלי לאמון",c.ref_min_mean]],"min(100, 100·score / ref)",
|
| 2297 |
+
`${count(x=>stageOf(x,"baseline").applied)} הגאים הותאמו`);
|
| 2298 |
+
card("calibration",6,"כיול מורה","רצפות שהמורה אישר לבלבול מסוים (צפוי → נשמע).",
|
| 2299 |
+
[["מפתח","סמל, ואז הגה כתוב"]],"max(score, floor)",
|
| 2300 |
+
`${count(x=>stageOf(x,"calibration").applied)} רצפות הוחלו`);
|
| 2301 |
+
card("decision",7,"החלטת סימון","עיצור מסומן אם הציון נמוך והמתחרה ניצח במרווח של הזוג.",
|
| 2302 |
+
[["סימון: ציון <",c.flag_below],["מרווח ברירת מחדל",c.margin_default],["זוגות עם מרווח",c.margins.length],["רשימת אי־דיוק: ציון <",c.issue_below]],
|
| 2303 |
+
"−GOP ≥ margin(target, heard)",
|
| 2304 |
+
`${flaggedN} מסומנים · ${issueN} ברשימה${underFlag?` · ${underFlag} נמוכים אך לא סומנו`:""}`);
|
| 2305 |
+
card("word",8,"ציון מילה","שילוב של ההגה החלש במילה וממוצע ההגאים שלה.",
|
| 2306 |
+
[["k משקל החלש",c.word_min_weight]],`k·min + (1−k)·mean`,
|
| 2307 |
+
t.words.map(w=>Math.round(w.score)).join(" · "));
|
| 2308 |
+
card("sentence",9,"ציון משפט","ממוצע הגאים משוקלל (תנועה במשקל נמוך) וממוצע המילים.",
|
| 2309 |
+
[["משקל תנועה",c.vowel_weight],["ממוצע הגאים",sen.phone_weight],["ממוצע מילים",sen.word_weight]],
|
| 2310 |
+
`${sen.phone_weight}·${sen.phone_mean} + ${sen.word_weight}·${sen.word_mean}`,
|
| 2311 |
+
`${sen.before_cap} → ${sen.final}${sen.capped?" (מוגבל)":""}`);
|
| 2312 |
+
card("tips",10,"טיפים","מהמסומנים: חומרה גבוהה קודם, ואז הציון הנמוך.",
|
| 2313 |
+
[["מקסימום",c.max_tips],["מקסימום ברשימה",c.max_issues]],null,
|
| 2314 |
+
t.tips.length?t.tips.map(x=>`${x.key} (${x.severity})`).join(" · "):"אין טיפים");
|
| 2315 |
+
|
| 2316 |
+
// variants and the "different sentence" check
|
| 2317 |
+
const vp=$("stageVariants"); vp.innerHTML="";
|
| 2318 |
+
vp.append(E("h4","","1–2 · הווריאנטים והבדיקה למשפט אחר"));
|
| 2319 |
+
const vt=E("table"); vt.append(E("tr","",E("th","","ערבית"),E("th","","IPA"),E("th","","log P")));
|
| 2320 |
+
for(const v of [...t.variants].sort((a,b)=>(b.logp??-1e9)-(a.logp??-1e9))){
|
| 2321 |
+
const row=E("tr",v.chosen?"chosen":"",E("td","",v.ar),E("td","",ltr(v.ipa)),E("td","",ltr(v.logp==null?"קצר מדי":num(v.logp,2))));
|
| 2322 |
+
vt.append(row);
|
| 2323 |
+
}
|
| 2324 |
+
vp.append(vt,E("div","",E("small","","עם סמלים: "),ltr(t.aipa)),E("div","",E("small","","הפענוח החופשי: "),ltr(u.heard||"—")));
|
| 2325 |
+
const bar=E("div","stage-bar"), fill=E("i"), mk=E("b");
|
| 2326 |
+
fill.style.width=Math.min(100,100*u.ratio)+"%"; mk.style.left=100*u.threshold+"%"; bar.append(fill,mk);
|
| 2327 |
+
vp.append(bar,E("small","",`מרחק יחסי ${u.ratio} · הסף ${u.threshold} (הקו האדום)`));
|
| 2328 |
+
|
| 2329 |
+
// word and sentence formulas with this attempt's numbers
|
| 2330 |
+
const sp=$("stageSentence"); sp.innerHTML="";
|
| 2331 |
+
sp.append(E("h4","","8–9 · מילים ומשפט"));
|
| 2332 |
+
const wt=E("table"); wt.append(E("tr","",E("th","","מילה"),E("th","","min"),E("th","","mean"),E("th","","k·min+(1−k)·mean")));
|
| 2333 |
+
for(const w of t.words) wt.append(E("tr","",E("td","",w.tr),E("td","",ltr(num(w.min))),E("td","",ltr(num(w.mean))),
|
| 2334 |
+
E("td","",ltr(`${c.word_min_weight}·${num(w.min)} + ${num(1-c.word_min_weight,2)}·${num(w.mean)} = ${num(w.score)}`))));
|
| 2335 |
+
const vowels=toks.filter(x=>x.vowel).length;
|
| 2336 |
+
sp.append(wt,E("div","eq",`phone_mean = Σ(w·score)/Σw (w=${c.vowel_weight} ×${vowels} vowels, 1 ×${toks.length-vowels} consonants) = ${sen.phone_mean}`),
|
| 2337 |
+
E("div","eq",`word_mean = ${sen.word_mean}`),
|
| 2338 |
+
E("div","eq",`score = ${sen.phone_weight}·${sen.phone_mean} + ${sen.word_weight}·${sen.word_mean} = ${sen.before_cap}${sen.capped?` → min(·, ${c.unrelated_cap})`:""} → ${sen.final}`));
|
| 2339 |
+
|
| 2340 |
+
// every sound through the stages
|
| 2341 |
+
const tb=$("stageTable"); tb.innerHTML="";
|
| 2342 |
+
const head=[["#",""],["הגה","כתוב · סמל · מחלקה"],["חלון","פריימים",""],["P צפוי","","gop"],["מתחרה","P","gop"],["GOP","log יחס","gop"],
|
| 2343 |
+
["e^(GOP·λ)",`λ=${c.leniency}`,"gop"],["√ביטחון",`p/${c.confident_peak}`,"gop"],["ציון GOP","","gop"],["פטור","","exemption"],
|
| 2344 |
+
["קו בסיס","ref","baseline"],["כיול","floor","calibration"],["סופי","",""],["מרווח","בפועל / נדרש","decision"],
|
| 2345 |
+
["החלטה",`סימון <${c.flag_below} · רשימה <${c.issue_below}`,"decision"],["כלל","חומרה","tips"],["משקל","","sentence"]];
|
| 2346 |
+
const hr=E("tr"); for(const [a,b,st] of head){ const th=E("th",st?"st-"+st:"",a,b?E("small","",b):null); hr.append(th); } tb.append(E("thead","",hr));
|
| 2347 |
+
const body=E("tbody");
|
| 2348 |
+
t.words.forEach((w,wi)=>{
|
| 2349 |
+
body.append(E("tr","stage-word",(()=>{ const td=E("td","",`מילה ${wi+1}: ${w.tr} · ${num(w.score)}`,E("span","eq",`k·min + (1−k)·mean = ${c.word_min_weight}·${num(w.min)} + ${num(1-c.word_min_weight,2)}·${num(w.mean)} = ${num(w.score)}`)); td.colSpan=head.length; return td; })()));
|
| 2350 |
+
for(const x of toks.filter(x=>x.word===wi)){
|
| 2351 |
+
const gop=stageOf(x,"gop"), ex=stageOf(x,"exemption"), bl=stageOf(x,"baseline"), cal=stageOf(x,"calibration");
|
| 2352 |
+
const who=E("td","",ltr(x.ipa));
|
| 2353 |
+
if(x.sound!==x.ipa) who.append(E("small","",ltr(x.sound+(x.class?" {"+x.class.join(",")+"}":"")+(x.realised?" → "+x.realised:""))));
|
| 2354 |
+
const comp=E("td","st-gop",x.deleted?"הושמט":x.competitor_other?"צליל לא מוכר":x.competitor?ltr(x.competitor):"—",E("small","",pct(x.p_competitor)));
|
| 2355 |
+
const step=(st,stage,label)=>{ const td=E("td","st-"+stage+(st.applied?" chg "+band(st.score):" skip"),st.applied?num(st.score):"·");
|
| 2356 |
+
if(label) td.append(E("small","",label)); return td; };
|
| 2357 |
+
const m=x.margin, beyondTxt=m.beyond?"✓":"✗";
|
| 2358 |
+
let dec, decCls;
|
| 2359 |
+
if(x.flagged){ dec="מסומן"; decCls="dec-flag"; }
|
| 2360 |
+
else if(x.issue){ dec="ברשימה"; decCls="dec-issue"; }
|
| 2361 |
+
else if(x.final<c.flag_below&&x.vowel){ dec="תנועה: לא מסומנת"; decCls="dec-issue"; }
|
| 2362 |
+
else if(x.final<c.flag_below&&!m.beyond){ dec="מרווח לא הושג"; decCls="dec-issue"; }
|
| 2363 |
+
else if(x.final<c.issue_below){ dec="נמוך, לא ברשימה"; decCls="dec-issue"; }
|
| 2364 |
+
else { dec="תקין"; decCls="dec-ok"; }
|
| 2365 |
+
const decTd=E("td","st-decision",E("span",decCls,dec)); if(x.tip) decTd.append(E("small","","+ טיפ"));
|
| 2366 |
+
body.append(E("tr","",
|
| 2367 |
+
E("td","",String(x.i+1)), who,
|
| 2368 |
+
E("td","",ltr(`${x.window[0]}–${x.window[1]}`),E("small","",ltr(`span ${x.span[0]}–${x.span[1]}`))),
|
| 2369 |
+
E("td","st-gop",pct(x.p_target)), comp,
|
| 2370 |
+
E("td","st-gop",ltr(num(x.gop,3))), E("td","st-gop",ltr(num(x.gop_factor,3))), E("td","st-gop",ltr(num(x.confidence_factor,3))),
|
| 2371 |
+
E("td","st-gop "+band(gop.score),num(gop.score)),
|
| 2372 |
+
step(ex,"exemption",ex.applied?(EXEMPTION_NAMES[ex.applied]||ex.applied):""),
|
| 2373 |
+
step(bl,"baseline",bl.ref!=null?"ref "+num(bl.ref):""),
|
| 2374 |
+
step(cal,"calibration",cal.floor!=null?"floor "+num(cal.floor):""),
|
| 2375 |
+
E("td",band(x.final),num(x.final)),
|
| 2376 |
+
E("td","st-decision",ltr(`${num(m.got,2)} / ${num(m.needed,2)} ${beyondTxt}`)),
|
| 2377 |
+
decTd,
|
| 2378 |
+
E("td","st-tips",x.rule?x.rule.title:"—",x.rule?E("small","","חומרה "+x.rule.severity):null),
|
| 2379 |
+
E("td","st-sentence",String(x.weight))));
|
| 2380 |
+
}
|
| 2381 |
+
});
|
| 2382 |
+
tb.append(body);
|
| 2383 |
}
|
| 2384 |
function heatColor(value){
|
| 2385 |
const stops=[[245,247,245],[159,196,172],[40,116,93],[240,180,60],[159,47,39]], x=Math.max(0,Math.min(100,value))/25;
|
tests/test_rules.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The yaml blocks of pronunciation-rules.md, as the scorer reads them."""
|
| 2 |
+
import re
|
| 3 |
+
|
| 4 |
+
import yaml
|
| 5 |
+
|
| 6 |
+
from app.rules import RULES_PATH, Check, check_detectable, load, parse, undetectable
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def test_every_block_parses_into_rules():
|
| 10 |
+
r = load()
|
| 11 |
+
assert {"Q", "K", "TH", "DH", "ZH", "J"} <= set(r.archiphonemes)
|
| 12 |
+
assert r.dialects == ["urban", "rural"] and len(r.checks) >= 30
|
| 13 |
+
assert r.check("ħ", "χ").severity == 3 # written χ, stored by its base() name x
|
| 14 |
+
assert r.margin(["ZH", "ðˤ"], "z") == 0.7 # ZH has no margin of its own: the written phoneme's applies
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def test_every_heard_symbol_has_a_hebrew_label():
|
| 18 |
+
text = RULES_PATH.read_text(encoding="utf-8")
|
| 19 |
+
heard = {str(h) for block in re.findall(r"```yaml\n(.*?)```", text, re.S)
|
| 20 |
+
for doc in re.split(r"^---$", block, flags=re.M) if doc.strip()
|
| 21 |
+
for c in (yaml.safe_load(doc) or {}).get("checks", []) for h in c["heard"]}
|
| 22 |
+
assert heard - set(load().heard_labels) == set()
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def test_correct_realisations_per_letter():
|
| 26 |
+
r = load()
|
| 27 |
+
assert r.members("Q") == ["ʔ", "k", "q"] # Bedouin g is off by default
|
| 28 |
+
assert r.members("TH") == ["t", "θ"] and "s" in r.members("TH", written="s") # s only in learned words
|
| 29 |
+
assert r.dialects_of("Q", "k") == {"rural"} and r.dialects_of("K", "k") == {"urban", "rural"}
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def test_undetectable_rules_are_reported():
|
| 33 |
+
r = load()
|
| 34 |
+
everything = lambda p: True
|
| 35 |
+
assert undetectable(Check("x", "ħ", "ħ", 1, ""), r, everything) == "same_sound"
|
| 36 |
+
assert undetectable(Check("x", "Q", "k", 1, ""), r, everything) == "accepted"
|
| 37 |
+
assert undetectable(Check("x", "ħ", "ǀ", 1, ""), r, lambda p: p != "ǀ") == "heard_not_in_model"
|
| 38 |
+
report = check_detectable(r, everything)
|
| 39 |
+
assert report["pharyngeal_h"]["detectable"]
|
| 40 |
+
# vowel length is not modelled (base() drops ː): the long/short checks are listed as undetectable
|
| 41 |
+
assert {m["why"] for m in report["vowel_length"]["missing"]} == {"same_sound"}
|
| 42 |
+
bad = parse("```yaml\nid: bad\ntitle: bad\nchecks:\n - {target: aː, heard: [a], severity: 1, feedback: x}\n```\n")
|
| 43 |
+
assert not check_detectable(bad, everything)["bad"]["detectable"]
|
tests/test_scoring.py
CHANGED
|
@@ -120,13 +120,172 @@ def test_h_instead_of_pharyngeal():
|
|
| 120 |
assert r["tips"][0]["key"] == "ħ>h"
|
| 121 |
|
| 122 |
|
| 123 |
-
def
|
| 124 |
-
# learner says "kaddeesh" for قديش:
|
| 125 |
logp = emissions(["k", "a", "d", "eː", "ʃ", "i", "s", "aː", "ʕ", "a"])
|
| 126 |
r = scorer_for(logp).score(None, TIME)
|
| 127 |
-
|
| 128 |
-
assert r["variant"]["ipa"].startswith("ʔ")
|
| 129 |
-
assert "
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 130 |
|
| 131 |
|
| 132 |
def test_unrelated_sentence():
|
|
@@ -210,7 +369,8 @@ def test_tts_endpoint_serves_phrase_audio_and_503_without_key():
|
|
| 210 |
assert r.headers["x-tts-voice-id"] == voices[1]["id"] and unquote(r.headers["x-tts-voice-name"]) == voices[1]["name"]
|
| 211 |
assert c.get("/api/tts/0?voice=not-in-the-list").status_code == 400
|
| 212 |
assert c.get("/api/tts/9999").status_code == 404
|
| 213 |
-
|
|
|
|
| 214 |
|
| 215 |
|
| 216 |
def test_spelling_variants_of_a_sound_are_not_errors():
|
|
@@ -1546,3 +1706,18 @@ def test_a_tutor_flags_a_recording_scored_wrong(monkeypatch):
|
|
| 1546 |
assert "create table public.tutors" in c.get("/api/class", headers=tok("t")).json()["setup"][-1]["sql"]
|
| 1547 |
r = rank("t", email="s@b.c", tutor=True)
|
| 1548 |
assert r.status_code == 503 and "create table public.tutors" in r.json()["detail"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
assert r["tips"][0]["key"] == "ħ>h"
|
| 121 |
|
| 122 |
|
| 123 |
+
def test_k_for_qaf_is_rural_and_correct():
|
| 124 |
+
# learner says "kaddeesh" for قديش: rural Palestinian, so ق is accepted as k; the answer stays the urban one
|
| 125 |
logp = emissions(["k", "a", "d", "eː", "ʃ", "i", "s", "aː", "ʕ", "a"])
|
| 126 |
r = scorer_for(logp).score(None, TIME)
|
| 127 |
+
q = r["words"][0]["phonemes"][0]
|
| 128 |
+
assert r["variant"]["ipa"].startswith("ʔ") and q["sound"] == "Q" and q["realised"] == "k"
|
| 129 |
+
assert q["score"] >= 90 and not q["flagged"] and not r["tips"] and r["score"] >= 90
|
| 130 |
+
assert r["words"][0]["heard_tr"].startswith("כּ") # written as said, not coloured as a mistake
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
# ---------- dialects and Hebrew-speaker rules (docs/scoring-dialects-plan.md) ----------
|
| 134 |
+
|
| 135 |
+
def custom_scorer(extra, phones, frames=3, gap=2, peak=0.9):
|
| 136 |
+
"""A scorer over SYMS plus `extra`, hearing `phones` cleanly."""
|
| 137 |
+
syms = SYMS + [s for s in extra if s not in SYMS]
|
| 138 |
+
vocab = {s: i for i, s in enumerate(syms)}
|
| 139 |
+
|
| 140 |
+
def row(sym, p):
|
| 141 |
+
r = np.full(len(syms), (1 - p) / (len(syms) - 1))
|
| 142 |
+
r[vocab[sym]] = p
|
| 143 |
+
return r
|
| 144 |
+
|
| 145 |
+
rows = [row("<pad>", 0.95)] * gap
|
| 146 |
+
for ph in phones:
|
| 147 |
+
rows += [row(ph, peak)] * frames + [row("<pad>", 0.95)] * gap
|
| 148 |
+
return Scorer(lambda wav: np.log(np.array(rows)), Vocab(vocab))
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
COFFEE = {"variants": [("قهوة", "ʔ a h w e")]}
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def test_q_for_qaf_is_accepted():
|
| 155 |
+
r = scorer_for(emissions(["q", "a", "d", "eː", "ʃ", "i", "s", "aː", "ʕ", "a"])).score(None, TIME)
|
| 156 |
+
assert r["words"][0]["phonemes"][0]["realised"] == "q" and not r["tips"] and r["score"] >= 90
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def test_dropped_qaf_between_vowels_is_not_an_error():
|
| 160 |
+
r = scorer_for(emissions(["a", "h", "w", "e"])).score(None, COFFEE)
|
| 161 |
+
assert r["score"] >= 90 and not r["tips"] and not any(i["missing"] for i in r["issues"])
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def test_hamza_is_not_a_qaf():
|
| 165 |
+
# اهلا is written with ء: kahla is not a dialect form, it is a mistake
|
| 166 |
+
phrase = {"variants": [("اهلا", "ʔ a h l a")]}
|
| 167 |
+
r = scorer_for(emissions(["k", "a", "h", "l", "a"])).score(None, phrase)
|
| 168 |
+
p = r["words"][0]["phonemes"][0]
|
| 169 |
+
assert "sound" not in p and p["flagged"] and p["heard"] == "k"
|
| 170 |
+
assert [t["key"] for t in r["tips"]] == ["ʔ>k"]
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def test_reference_is_matched_by_letter_not_by_realisation():
|
| 174 |
+
ref = scorer_for(emissions(["q", "a", "h", "w", "e"])).reference(None, COFFEE) # the voice says ق as q
|
| 175 |
+
assert ref.sounds[0] == (0, "Q")
|
| 176 |
+
r = scorer_for(emissions(["ʔ", "a", "h", "w", "e"])).score(None, COFFEE, [ref])
|
| 177 |
+
assert r["score"] >= 95 and not r["tips"] and r["references"] == 1
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
def test_mixed_dialect_is_noted_but_not_marked_down():
|
| 181 |
+
phrase = {"variants": [("قديش قهوة", "ʔ a d d eː ʃ | ʔ a h w e")]}
|
| 182 |
+
mixed = scorer_for(emissions(["ʔ", "a", "d", "eː", "ʃ", "k", "a", "h", "w", "e"])).score(None, phrase)
|
| 183 |
+
assert mixed["mixed_dialect"] and mixed["score"] >= 90 and not mixed["tips"]
|
| 184 |
+
rural = scorer_for(emissions(["k", "a", "d", "eː", "ʃ", "k", "a", "h", "w", "e"])).score(None, phrase)
|
| 185 |
+
assert rural["dialect"] == "rural" and "mixed_dialect" not in rural
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
def test_rural_kaf_and_tha():
|
| 189 |
+
phrase = {"variants": [("كثير", "k t iː r")]}
|
| 190 |
+
r = custom_scorer(["tʃ", "θ", "r", "iː", "t"], ["tʃ", "θ", "iː", "r"]).score(None, phrase)
|
| 191 |
+
assert [p.get("realised") for p in r["words"][0]["phonemes"][:2]] == ["tʃ", "θ"]
|
| 192 |
+
assert r["score"] >= 90 and not r["tips"] and r["dialect"] == "rural"
|
| 193 |
+
r = custom_scorer(["tʃ", "θ", "r", "iː", "t"], ["tʃ", "t", "iː", "r"]).score(None, phrase)
|
| 194 |
+
assert r["mixed_dialect"] and r["score"] >= 90 # rural ك with urban ث
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def test_final_ta_marbuta_vowel_may_be_a_e_or_i():
|
| 198 |
+
for vowel in ("a", "e", "i"):
|
| 199 |
+
r = scorer_for(emissions(["ʔ", "a", "h", "w", vowel])).score(None, COFFEE)
|
| 200 |
+
assert r["words"][0]["phonemes"][-1]["score"] >= 95, vowel
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
def test_diphthong_for_long_vowel():
|
| 204 |
+
# بيت said as bayt (rural / older speech) for beet
|
| 205 |
+
phrase = {"variants": [("بيت", "b eː t")]}
|
| 206 |
+
r = custom_scorer(["b", "t", "j"], ["b", "a", "j", "t"]).score(None, phrase)
|
| 207 |
+
assert r["score"] >= 90 and not r["tips"] and r["variant"]["ipa"] == "b a j t"
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def test_archify_places_each_letter():
|
| 211 |
+
from app.archi import archify, unresolved
|
| 212 |
+
assert archify("قديش الساعة", "ʔ a d d eː ʃ | i s s aː ʕ a") == "Q:ʔ a d d eː ʃ | i s s aː ʕ E:a"
|
| 213 |
+
assert archify("انا من القدس", "ʔ a n a | m i n | i l q u d s") == "ʔ a n a | m i n | i l Q:q u d s"
|
| 214 |
+
assert archify("شكرا كثير", "ʃ u k r a n | k t iː r") == "ʃ u K:k r a n | K:k TH:t iː r"
|
| 215 |
+
assert archify("قهوة", "Q:k a h w e") == "Q:k a h w e" # written by hand: left alone
|
| 216 |
+
assert unresolved("بعثت", "b a ʕ a t t") == [("بعثت", "ث")] # ث and ت said as one geminate
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
def test_every_qaf_in_the_bank_is_placed():
|
| 220 |
+
from app.archi import unresolved
|
| 221 |
+
bad = {(w, ipa) for p in PHRASES for ar, ipa in p["variants"] for w, letter in unresolved(ar, ipa) if letter == "ق"}
|
| 222 |
+
assert not bad, f"mark these ق by hand in the phrase (Q:ʔ): {sorted(bad)[:10]}"
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
def test_emphatic_heard_plain_is_a_mistake_with_the_rules_feedback():
|
| 226 |
+
from app.rules import load
|
| 227 |
+
phrase = {"variants": [("ظهر", "ðˤ u h r")]}
|
| 228 |
+
r = custom_scorer(["ðˤ", "dˤ", "zˤ", "z", "u", "r"], ["z", "u", "h", "r"]).score(None, phrase)
|
| 229 |
+
assert [t["key"] for t in r["tips"]] == ["ZH>z"]
|
| 230 |
+
assert r["tips"][0]["text"] == load().check("ZH", "z").feedback and r["tips"][0]["severity"] == 2
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
def margin_scorer(competitor_ratio):
|
| 234 |
+
"""طين where tˤ is weak (p 0.07) and plain t is `competitor_ratio` times likelier in the same frames."""
|
| 235 |
+
syms = SYMS + ["tˤ", "t"]
|
| 236 |
+
vocab = {s: i for i, s in enumerate(syms)}
|
| 237 |
+
V = len(syms)
|
| 238 |
+
|
| 239 |
+
def row(probs):
|
| 240 |
+
r = np.full(V, 1e-4)
|
| 241 |
+
for sym, p in probs.items():
|
| 242 |
+
r[vocab[sym]] = p
|
| 243 |
+
return r / r.sum()
|
| 244 |
+
|
| 245 |
+
rows = [row({"<pad>": 0.95})] * 2
|
| 246 |
+
rows += [row({"tˤ": 0.07, "t": 0.07 * competitor_ratio, "<pad>": 0.8})] * 3 + [row({"<pad>": 0.95})] * 2
|
| 247 |
+
for ph in ("i", "n"):
|
| 248 |
+
rows += [row({ph: 0.9})] * 3 + [row({"<pad>": 0.95})] * 2
|
| 249 |
+
return Scorer(lambda wav: np.log(np.array(rows)), Vocab(vocab))
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
def test_emphatic_needs_a_clear_win_before_it_is_flagged():
|
| 253 |
+
phrase = {"variants": [("طين", "tˤ i n")]}
|
| 254 |
+
under = margin_scorer(1.8).score(None, phrase) # plain t under twice as likely: the model's doubt
|
| 255 |
+
p = under["words"][0]["phonemes"][0]
|
| 256 |
+
assert p["score"] < 50 and not p["flagged"] and not under["tips"]
|
| 257 |
+
over = margin_scorer(2.5).score(None, phrase)
|
| 258 |
+
assert over["words"][0]["phonemes"][0]["flagged"] and [t["key"] for t in over["tips"]] == ["tˤ>t"]
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
@pytest.mark.parametrize("ar,ipa,extra,said,key,heard_letter", [
|
| 262 |
+
("فنجان", "f i n dʒ aː n", ["f", "p", "dʒ"], ["p", "i", "n", "dʒ", "aː", "n"], "f>p", "פּ"),
|
| 263 |
+
("وين", "w eː n", ["v"], ["v", "eː", "n"], "w>v", "ב"),
|
| 264 |
+
("بس", "b a s", ["b", "v"], ["v", "a", "s"], "b>v", "ב"),
|
| 265 |
+
("صباح", "sˤ a b aː ħ", ["sˤ", "ts", "b"], ["ts", "a", "b", "aː", "ħ"], "sˤ>ts", "טס"),
|
| 266 |
+
("مرة", "m a r r a", ["r", "ʁ"], ["m", "a", "ʁ", "a"], "r>ʁ", None),
|
| 267 |
+
])
|
| 268 |
+
def test_hebrew_speaker_substitutions(ar, ipa, extra, said, key, heard_letter):
|
| 269 |
+
from app.rules import load
|
| 270 |
+
r = custom_scorer(extra, said).score(None, {"variants": [(ar, ipa)]})
|
| 271 |
+
tip = next(t for t in r["tips"] if t["key"] == key)
|
| 272 |
+
expected, heard = key.split(">")
|
| 273 |
+
assert tip["text"] == load().check(expected, heard).feedback
|
| 274 |
+
if heard_letter: # a sound Arabic does not have is still written in the heard line
|
| 275 |
+
assert heard_letter in r["heard_tr"].replace("\u05b0", "").replace("\u05b7", "").replace("\u05b4", "")
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
def test_a_sound_outside_arabic_and_hebrew_is_not_named():
|
| 279 |
+
# the multilingual model offers a click for ħ: reported as unclear, not as "heard as ǀ"
|
| 280 |
+
r = custom_scorer(["ǀ"], ["w", "eː", "n", "i", "l", "ǀ", "a", "m", "aː", "m"]).score(None, TOILET)
|
| 281 |
+
h = next(p for p in r["words"][1]["phonemes"] if p["ipa"] == "ħ")
|
| 282 |
+
assert h["flagged"] and h["heard"] is None
|
| 283 |
+
|
| 284 |
+
|
| 285 |
+
def test_meaning_changing_mistakes_lead_the_tips():
|
| 286 |
+
# w said as v (severity 2) and ħ as Hebrew ח (severity 3): ħ leads even though v scored lower
|
| 287 |
+
r = custom_scorer(["v"], ["v", "eː", "n", "i", "l", "χ", "a", "m", "aː", "m"]).score(None, TOILET)
|
| 288 |
+
assert [t["key"] for t in r["tips"]][:2] == ["ħ>x", "w>v"]
|
| 289 |
|
| 290 |
|
| 291 |
def test_unrelated_sentence():
|
|
|
|
| 369 |
assert r.headers["x-tts-voice-id"] == voices[1]["id"] and unquote(r.headers["x-tts-voice-name"]) == voices[1]["name"]
|
| 370 |
assert c.get("/api/tts/0?voice=not-in-the-list").status_code == 400
|
| 371 |
assert c.get("/api/tts/9999").status_code == 404
|
| 372 |
+
health = c.get("/api/health").json()
|
| 373 |
+
assert health["tts"] is True and health["rules"]["pharyngeal_h"]["detectable"]
|
| 374 |
|
| 375 |
|
| 376 |
def test_spelling_variants_of_a_sound_are_not_errors():
|
|
|
|
| 1706 |
assert "create table public.tutors" in c.get("/api/class", headers=tok("t")).json()["setup"][-1]["sql"]
|
| 1707 |
r = rank("t", email="s@b.c", tutor=True)
|
| 1708 |
assert r.status_code == 503 and "create table public.tutors" in r.json()["detail"]
|
| 1709 |
+
|
| 1710 |
+
|
| 1711 |
+
def test_teacher_trace_follows_every_stage():
|
| 1712 |
+
# ħ said as χ, then forgiven by a teacher floor: each stage of the trace shows where the score came from
|
| 1713 |
+
logp = emissions(["w", "eː", "n", "i", "l", "χ", "a", "m", "aː", "m"])
|
| 1714 |
+
scorer = scorer_for(logp)
|
| 1715 |
+
assert "trace" not in scorer.score(None, TOILET) # students do not get it
|
| 1716 |
+
scorer.set_calibrations([{"expected": "ħ", "heard": "χ", "floor": 60}])
|
| 1717 |
+
t = scorer.score_logp(logp, TOILET, diagnostics=True)["trace"]
|
| 1718 |
+
assert t["config"]["flag_below"] == 50 and t["config"]["leniency"] == 0.5 and t["variants"][0]["chosen"]
|
| 1719 |
+
h = next(x for x in t["tokens"] if x["ipa"] == "ħ")
|
| 1720 |
+
assert [s["stage"] for s in h["stages"]] == ["gop", "exemption", "baseline", "calibration"]
|
| 1721 |
+
assert h["stages"][0]["score"] < 50 and h["stages"][3]["applied"] and h["final"] == 60
|
| 1722 |
+
assert h["competitor"] == "χ" and h["rule"]["severity"] == 3 and not h["flagged"]
|
| 1723 |
+
assert t["sentence"]["final"] == round(0.5 * t["sentence"]["phone_mean"] + 0.5 * t["sentence"]["word_mean"])
|