simplify codes and bug fix

chinesemeteor.py

@@ -139,18 +139,15 @@ class ChineseMETEOR(evaluate.Metric):
         nltk.download("punkt", quiet=True)
         nltk.download('punkt_tab', quiet=True)
         # CwnGraph auto-downloads on first use
-
-    def _tokenize_chinese(self, sentence):
-        """Tokenize Chinese sentence using PyCantonese"""
-        return pycantonese.segment(sentence)
         
     def _compute(self, predictions: List[str], references: List[str]) -> Dict[str, float]:
         original_stdout = sys.stdout # store original output
         sys.stdout = open(os.devnull, 'w')
 
         try:
-            pred_seg = [" ".join(pycantonese.segment(p.strip())) for p in predictions]
-            ref_seg  = [" ".join(pycantonese.segment(r.strip())) for r in references]
+            # Tokenize using PyCantonese
+            pred_seg = [pycantonese.segment(p.strip()) for p in predictions]
+            ref_seg  = [pycantonese.segment(r.strip()) for r in references]
     
             # --- Apply Real Chinese WordNet into METEOR algorithm ---
             def _cwn_synsets(self, word, pos=None):  # Matches NLTK method call
@@ -215,10 +212,6 @@ class ChineseMETEOR(evaluate.Metric):
     
             scores = [
                 meteor_score.single_meteor_score(
-                    #word_tokenize(ref),
-                    #self._tokenize_chinese(ref),
-                    #word_tokenize(hyp),
-                    #self._tokenize_chinese(hyp),
                     ref,
                     hyp,
                     wordnet=chinese_wn