Update app.py

app.py

@@ -17,12 +17,8 @@ def calculate_bertscore(translations, references, lang):
     P, R, F1 = bert_score(translations, references, lang=lang)
     return F1.mean().item()
 
-def tokenize_text(text, lang_code):
-    if lang_code == "zh":
-        return ' '.join(jieba.cut(text))
-    # For other languages, we'll use a simple space-based tokenization
-    # This might not be ideal for all languages, but it's a start
-    return ' '.join(text.split())
+def tokenize_chinese(text):
+    return ' '.join(jieba.cut(text))
 
 st.title("Machine Translation Quality Evaluation")
 st.write("Input the translated text and the reference translation to compute BLEU, TER, CHRF, and BERTScore metrics.")
@@ -48,10 +44,19 @@ reference_input = st.text_area("Reference Translation", height=200)
 if st.button("Evaluate"):
     if translation_input and reference_input:
         try:
-            translations = [tokenize_text(translation_input.strip(), target_lang_code)]
-            references = [tokenize_text(reference_input.strip(), target_lang_code)]
-
-            st.write("Debug: Inputs received and tokenized")
+            # Process translation input
+            if target_lang_code == "zh":
+                translations = [tokenize_chinese(translation_input.strip())]
+            else:
+                translations = [translation_input.strip()]
+
+            # Process reference input
+            if target_lang_code == "zh":
+                references = [tokenize_chinese(reference_input.strip())]
+            else:
+                references = [reference_input.strip()]
+
+            st.write("Debug: Inputs processed")
             st.write(f"Translation: {translations}")
             st.write(f"Reference: {references}")