markup bug fix

=0.25.1

@@ -0,0 +1,20 @@
+Requirement already satisfied: transformers in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (4.45.2)
+Requirement already satisfied: huggingface-hub in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (0.24.7)
+Requirement already satisfied: filelock in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from transformers) (3.16.1)
+Requirement already satisfied: numpy>=1.17 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from transformers) (1.26.4)
+Requirement already satisfied: packaging>=20.0 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from transformers) (24.0)
+Requirement already satisfied: pyyaml>=5.1 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from transformers) (6.0.1)
+Requirement already satisfied: regex!=2019.12.17 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from transformers) (2026.5.9)
+Requirement already satisfied: requests in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from transformers) (2.32.3)
+Requirement already satisfied: safetensors>=0.4.1 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from transformers) (0.4.5)
+Requirement already satisfied: tokenizers<0.21,>=0.20 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from transformers) (0.20.3)
+Requirement already satisfied: tqdm>=4.27 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from transformers) (4.66.5)
+Requirement already satisfied: fsspec>=2023.5.0 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from huggingface-hub) (2024.6.1)
+Requirement already satisfied: typing-extensions>=3.7.4.3 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from huggingface-hub) (4.11.0)
+Requirement already satisfied: charset-normalizer<4,>=2 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from requests->transformers) (3.3.2)
+Requirement already satisfied: idna<4,>=2.5 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from requests->transformers) (3.6)
+Requirement already satisfied: urllib3<3,>=1.21.1 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from requests->transformers) (2.2.1)
+Requirement already satisfied: certifi>=2017.4.17 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from requests->transformers) (2024.2.2)
+
+[notice] A new release of pip is available: 24.3.1 -> 26.1.2
+[notice] To update, run: pip3 install --upgrade pip

app.py

@@ -151,10 +151,32 @@ def _mark_syllable_plain(syllable: str, label_id: int) -> str:
     marker = "_" if label_id == 1 else "^"
     chars = list(syllable)
 
+    # Find the last vowel, skipping trailing non-letter characters (like punctuation)
+    vowel_idx = -1
     for i in range(len(chars) - 1, -1, -1):
         if vowel(chars[i]):
-            return "".join(chars[: i + 1]) + marker + "".join(chars[i + 1 :])
-
+            vowel_idx = i
+            break
+        # Skip markup characters and non-letter characters
+        ch = chars[i]
+        if ch not in "^_" and ("\u0370" <= ch <= "\u03ff" or "\u1f00" <= ch <= "\u1fff"):
+            # It's a Greek letter but not a vowel
+            continue
+        if not ("\u0370" <= ch <= "\u03ff" or "\u1f00" <= ch <= "\u1fff") and ch not in "^_":
+            # It's not a Greek letter or markup - it's punctuation or other, skip
+            continue
+    
+    if vowel_idx >= 0:
+        # Found a vowel, insert marker after it
+        return "".join(chars[:vowel_idx + 1]) + marker + "".join(chars[vowel_idx + 1:])
+    
+    # No vowel found, find the last Greek letter
+    for i in range(len(chars) - 1, -1, -1):
+        if "\u0370" <= chars[i] <= "\u03ff" or "\u1f00" <= chars[i] <= "\u1fff":
+            # Insert marker after the last Greek letter
+            return "".join(chars[:i + 1]) + marker + "".join(chars[i + 1:])
+    
+    # No Greek letters found, append marker at the end
     return syllable + marker
 
 

pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-testpaths = test_markup.py
+testpaths = tests
 python_files = test_*.py
 python_classes = Test*
 python_functions = test_*

tests/test_bug_neaniasfix.py

@@ -0,0 +1,79 @@
+"""
+Test for the νεανίας bug fix.
+
+The bug is that the final alpha in νεανίας should be marked as long,
+but it wasn't being marked correctly.
+
+Additionally, punctuation handling was fixed so that marks are not placed
+after punctuation characters.
+
+Expected: νεα_νί^α_ς
+Current (buggy): νεα_νί^ας
+"""
+
+import pytest
+import sys
+from pathlib import Path
+
+# Add parent directory to path to import app module
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from app import _render_plain_line_per_word
+
+
+def test_neaniaste_final_alpha_marked():
+    """Test that the final alpha in νεανίας is correctly marked as long."""
+    # Use the mini model since it works correctly
+    model_id = "Ericu950/macronizer_mini"
+    
+    input_text = "νεανίας"
+    output = _render_plain_line_per_word(input_text, model_id)
+    
+    # The expected output should have the long mark on the final alpha before sigma
+    expected = "νεα_νί^α_ς"
+    
+    assert output == expected, (
+        f"Output mismatch for '{input_text}'\n"
+        f"Expected: {repr(expected)}\n"
+        f"Got:      {repr(output)}"
+    )
+
+
+def test_neaniaste_in_full_sentence():
+    """Test νεανίας in the full sentence provided by user."""
+    model_id = "Ericu950/macronizer_mini"
+    
+    input_text = "νεανίας ἀάατός ἐστιν καὶ καλός. τὰ παῖδες τὰ καλά"
+    output = _render_plain_line_per_word(input_text, model_id)
+    
+    # The first word should be νεα_νί^α_ς
+    first_word = output.split()[0]
+    expected_first = "νεα_νί^α_ς"
+    
+    assert first_word == expected_first, (
+        f"First word mismatch\n"
+        f"Expected: {repr(expected_first)}\n"
+        f"Got:      {repr(first_word)}"
+    )
+
+
+def test_punctuation_not_marked():
+    """Test that punctuation is preserved and marks are placed before punctuation."""
+    model_id = "Ericu950/macronizer_mini"
+    
+    # Word with punctuation
+    input_text = "καλός."
+    output = _render_plain_line_per_word(input_text, model_id)
+    
+    # The period should be preserved
+    assert "." in output, f"Period not preserved in output: {repr(output)}"
+    
+    # The mark should be placed before the period, not forming "σ._"
+    # The output should have the period at the very end
+    assert output.endswith("."), f"Period should be at the end: {repr(output)}"
+
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v", "-s"])
+

test_markup.py → tests/test_markup.py

@@ -4,6 +4,12 @@ Output should equal: input.replace("^", "").replace("_", "")
 """
 
 import pytest
+import sys
+from pathlib import Path
+
+# Add parent directory to path to import app module
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
 from app import _render_plain_line_per_word, DEFAULT_MODEL_ID