change tokenization strategy to PyCantonese

chinesemeteor.py

@@ -43,6 +43,7 @@ from nltk import word_tokenize
 import nltk
 import evaluate
 import re
+import pycantonese
 
 # Download once
 nltk.download("wordnet", quiet=True)
@@ -151,6 +152,10 @@ class ChineseMETEOR(evaluate.Metric):
         nltk.download('punkt_tab', quiet=True)
         # CwnGraph auto-downloads on first use
 
+    def _tokenize_chinese(self, sentence):
+        """Tokenize Chinese sentence using PyCantonese"""
+        return pycantonese.segment(sentence)
+        
     def _compute(self, predictions: List[str], references: List[str]) -> Dict[str, float]:
         pred_seg = [" ".join(jieba.cut(p.strip())) for p in predictions]
         ref_seg  = [" ".join(jieba.cut(r.strip())) for r in references]
@@ -218,8 +223,10 @@ class ChineseMETEOR(evaluate.Metric):
 
         scores = [
             meteor_score.single_meteor_score(
-                word_tokenize(ref),
-                word_tokenize(hyp),
+                #word_tokenize(ref),
+                self._tokenize_chinese(ref),
+                #word_tokenize(hyp),
+                self._tokenize_chinese(hyp),
                 wordnet=chinese_wn
             )
             for ref, hyp in zip(ref_seg, pred_seg)

requirements.txt

@@ -2,4 +2,5 @@ evaluate>=0.4.1
 jieba_fast
 CwnGraph>=0.3.0
 nltk>=3.8
-numpy
+numpy
+pycantonese