Upload aztext/alphabet.py with huggingface_hub

aztext/alphabet.py

@@ -0,0 +1,66 @@
+"""Canonical Azerbaijani (Latin script) alphabet constants and case helpers.
+
+Modern North Azerbaijani (Republic of Azerbaijan) uses a 32-letter Latin
+alphabet. Beyond ASCII it adds:  ə Ə, ğ Ğ, ı I (dotless i), i İ (dotted i),
+ö Ö, ş Ş, ç Ç, ü Ü.
+
+The dotted/dotless ``i`` distinction is *semantically meaningful* and must
+never be collapsed:  i (U+0069) ≠ ı (U+0131)  and  İ (U+0130) ≠ I (U+0049).
+
+Python's default ``str.lower``/``str.upper`` implement the *non-Turkic* rules
+(``'i'.upper() == 'I'``), which is wrong for Azerbaijani.  Use :func:`az_lower`
+and :func:`az_upper` instead — they honour the Turkic i mappings:
+
+    upper('i') = 'İ'   lower('I') = 'ı'
+    upper('ı') = 'I'   lower('İ') = 'i'
+"""
+
+# --- Vowels and consonants (lowercase) ---------------------------------------
+# 9 vowels
+AZ_VOWELS = "aeəıioöuü"
+# 23 consonants (dotless/ordinary letters only; İ/I handled by the vowel i/ı)
+AZ_CONSONANTS = "bcçdfgğhxjklmnpqrsştvyz"
+
+# --- Full lowercase / uppercase alphabets (in canonical alphabet order) -------
+# Alphabet order: a b c ç d e ə f g ğ h x ı i j k q l m n o ö p r s ş t u ü v y z
+AZ_LOWER = "abcçdeəfgğhxıijkqlmnoöprsştuüvyz"
+AZ_UPPER = "ABCÇDEƏFGĞHXIİJKQLMNOÖPRSŞTUÜVYZ"
+
+# All Azerbaijani letters (both cases), for membership tests / regex classes.
+AZ_LETTERS = AZ_LOWER + AZ_UPPER
+
+# --- Turkic-aware case mappings for the i-family -----------------------------
+# These four characters are the ones Python gets wrong for Azerbaijani.
+#   i  U+0069  LATIN SMALL LETTER I            -> upper İ  U+0130
+#   İ  U+0130  LATIN CAPITAL LETTER I WITH DOT -> lower i  U+0069
+#   ı  U+0131  LATIN SMALL LETTER DOTLESS I    -> upper I  U+0049
+#   I  U+0049  LATIN CAPITAL LETTER I          -> lower ı  U+0131
+_UPPER_I_MAP = {"i": "İ", "ı": "I"}
+_LOWER_I_MAP = {"İ": "i", "I": "ı"}
+
+
+def az_upper(text: str) -> str:
+    """Uppercase ``text`` with correct Azerbaijani i-mapping.
+
+    ``i`` -> ``İ`` and ``ı`` -> ``I`` are applied first so that the subsequent
+    plain ``str.upper()`` (which correctly handles ə→Ə, ç→Ç, ş→Ş, ğ→Ğ, ö→Ö,
+    ü→Ü, and all ASCII) never touches the dotted/dotless ``i`` incorrectly.
+    """
+    text = text.replace("i", "İ").replace("ı", "I")
+    return text.upper()
+
+
+def az_lower(text: str) -> str:
+    """Lowercase ``text`` with correct Azerbaijani i-mapping.
+
+    ``İ`` -> ``i`` and ``I`` -> ``ı`` are applied first.  This also avoids
+    Python's quirk where ``'İ'.lower()`` yields the two-character sequence
+    ``'i' + COMBINING DOT ABOVE`` instead of a plain ``'i'``.
+    """
+    text = text.replace("İ", "i").replace("I", "ı")
+    return text.lower()
+
+
+def is_az_letter(ch: str) -> bool:
+    """True if ``ch`` is a single Azerbaijani alphabet letter."""
+    return len(ch) == 1 and ch in AZ_LETTERS