add option for preserving english text

app.py

@@ -1,10 +1,10 @@
 import gradio as gr
 
-from segmentation import classifier, has_myanmar, preprocess, segment
+from segmentation import has_myanmar, preprocess, segment
 
 
-def segment_text_heuristics_supplement_way(text: str) -> str:
-    """Segment Myanmar text."""
+def segment_myanmar_only(text: str) -> str:
+    """Segment only Myanmar text, preserving English/Latin characters as-is."""
     tokens = preprocess(text)
 
     segmented_texts = []
@@ -19,21 +19,11 @@ def segment_text_heuristics_supplement_way(text: str) -> str:
     return result
 
 
-def segment_text(text: str) -> str:
-    """Segment Myanmar text."""
-    tokens = classifier(text)
-
-    segmented_words = []
-
-    for item in tokens:
-        if item["entity_group"] == "B":
-            segmented_words.append(item["word"])
-        else:  # 'I' - append to previous word
-            segmented_words[-1] += item["word"]
-
-    result = " ".join(segmented_words)
-
-    return result
+def process_text(text: str, should_preserve_english: bool) -> str:
+    """Process text based on the selected mode."""
+    if should_preserve_english:
+        return segment_myanmar_only(text)
+    return segment(text)
 
 
 css = """
@@ -69,6 +59,12 @@ with gr.Blocks(css=css) as demo:
             input_text = gr.Textbox(label="Input Text", placeholder="Enter Myanmar text here...", lines=8)
             output_text = gr.Textbox(label="Segmented Text", lines=8)
 
+        preserve_english = gr.Checkbox(
+            label="Preserve English text",
+            value=False,
+            info="Only segment Myanmar text and add spaces between English/Myanmar boundaries",
+        )
+
         run_button = gr.Button("Segment", variant="primary")
 
         gr.Examples(
@@ -92,8 +88,8 @@ with gr.Blocks(css=css) as demo:
             inputs=input_text,
         )
 
-    run_button.click(fn=segment_text, inputs=input_text, outputs=output_text)
-    input_text.submit(fn=segment_text, inputs=input_text, outputs=output_text)
+    run_button.click(fn=process_text, inputs=[input_text, preserve_english], outputs=output_text)
+    input_text.submit(fn=process_text, inputs=[input_text, preserve_english], outputs=output_text)
 
 if __name__ == "__main__":
     demo.launch()

segmentation.py

@@ -114,11 +114,15 @@ def preprocess(text):
 
 
 def segment(text):
-    classifier_result = classifier(text)
-    words = [r["word"] for r in classifier_result]
-    entities = [r["entity"] for r in classifier_result]
-    result = reconstruct(words, entities)
-    result = result.replace("▁", " ")
-    result = re.sub(r"\s+", " ", result).strip()
+    tokens = classifier(text)
+    segmented_text = []
 
-    return result
+    for item in tokens:
+        if item["entity_group"] == "B":
+            segmented_text.append(item["word"])
+        else:  # 'I' - append to previous word
+            segmented_text[-1] += item["word"]
+
+    segmented_text = " ".join(segmented_text)
+
+    return segmented_text