Commit ·
e38dee9
1
Parent(s): 4655865
FIX-45: Upgrade _levenshtein to Damerau-Levenshtein distance
Browse filesTranspositions (اقصتاديا→اقتصاديا, swap صت→تص) now count as 1 edit
instead of 2. This makes FIX-39 edit-distance guards more accurate for
Arabic typos where adjacent characters are swapped.
Before: _levenshtein('ab','ba') = 2
After: _levenshtein('ab','ba') = 1
Tests: 39 passing.
AUDIT COMPLETION:
All 4 items from legacy AraSpell audit now confirmed in codebase:
1. Trailing و removal → FIX-44 (just added)
2. Damerau-Levenshtein → FIX-45 (this commit)
3. Beam Voting → already in araspell_rules.py L1345-1362
4. Output Stability Test → already in araspell_rules.py L1521-1540
- src/app.py +16 -4
src/app.py
CHANGED
|
@@ -781,12 +781,19 @@ def get_word_diffs(original, corrected):
|
|
| 781 |
|
| 782 |
|
| 783 |
def _levenshtein(a, b):
|
| 784 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 785 |
m, n = len(a), len(b)
|
| 786 |
if m == 0:
|
| 787 |
return n
|
| 788 |
if n == 0:
|
| 789 |
return m
|
|
|
|
| 790 |
dp = [[0] * (n + 1) for _ in range(m + 1)]
|
| 791 |
for i in range(m + 1):
|
| 792 |
dp[i][0] = i
|
|
@@ -796,10 +803,15 @@ def _levenshtein(a, b):
|
|
| 796 |
for j in range(1, n + 1):
|
| 797 |
cost = 0 if a[i - 1] == b[j - 1] else 1
|
| 798 |
dp[i][j] = min(
|
| 799 |
-
dp[i - 1][j] + 1,
|
| 800 |
-
dp[i][j - 1] + 1,
|
| 801 |
-
dp[i - 1][j - 1] + cost,
|
| 802 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 803 |
return dp[m][n]
|
| 804 |
|
| 805 |
|
|
|
|
| 781 |
|
| 782 |
|
| 783 |
def _levenshtein(a, b):
|
| 784 |
+
"""Damerau-Levenshtein distance — transpositions count as 1 edit.
|
| 785 |
+
|
| 786 |
+
Better for Arabic typos like اقصتاديا→اقتصاديا (swap صت→تص):
|
| 787 |
+
Standard Levenshtein says edit=2, Damerau says edit=1.
|
| 788 |
+
|
| 789 |
+
FIX-45: Upgraded from standard Levenshtein.
|
| 790 |
+
"""
|
| 791 |
m, n = len(a), len(b)
|
| 792 |
if m == 0:
|
| 793 |
return n
|
| 794 |
if n == 0:
|
| 795 |
return m
|
| 796 |
+
# Use (m+2)x(n+2) matrix to handle transpositions safely
|
| 797 |
dp = [[0] * (n + 1) for _ in range(m + 1)]
|
| 798 |
for i in range(m + 1):
|
| 799 |
dp[i][0] = i
|
|
|
|
| 803 |
for j in range(1, n + 1):
|
| 804 |
cost = 0 if a[i - 1] == b[j - 1] else 1
|
| 805 |
dp[i][j] = min(
|
| 806 |
+
dp[i - 1][j] + 1, # deletion
|
| 807 |
+
dp[i][j - 1] + 1, # insertion
|
| 808 |
+
dp[i - 1][j - 1] + cost, # substitution
|
| 809 |
)
|
| 810 |
+
# Transposition: swap adjacent characters (counts as 1 edit)
|
| 811 |
+
if (i > 1 and j > 1
|
| 812 |
+
and a[i - 1] == b[j - 2]
|
| 813 |
+
and a[i - 2] == b[j - 1]):
|
| 814 |
+
dp[i][j] = min(dp[i][j], dp[i - 2][j - 2] + 1)
|
| 815 |
return dp[m][n]
|
| 816 |
|
| 817 |
|