diable lfs for 00b notebook

.gitattributes

@@ -56,7 +56,6 @@ results/mac-results_few_shots.csv filter=lfs diff=lfs merge=lfs -text
 results/mac-results_metrics.csv filter=lfs diff=lfs merge=lfs -text
 notebooks/00_Data[[:space:]]Analysis.ipynb filter=lfs diff=lfs merge=lfs -text
 notebooks/00a_Data[[:space:]]Analysis_greedy_decoding.ipynb filter=lfs diff=lfs merge=lfs -text
-notebooks/00b_Data[[:space:]]Analysis_Few_Shots.ipynb filter=lfs diff=lfs merge=lfs -text
 notebooks/01_Few-shot_Prompting.ipynb filter=lfs diff=lfs merge=lfs -text
 notebooks/01a_Few-shot_Prompting.ipynb filter=lfs diff=lfs merge=lfs -text
 logs/l40-1gpu.txt filter=lfs diff=lfs merge=lfs -text

llm_toolkit/translation_utils.py

@@ -19,6 +19,7 @@ rouge = evaluate.load("rouge")
 meteor = evaluate.load("meteor")
 accuracy = evaluate.load("accuracy")
 sacrebleu = evaluate.load("sacrebleu")
+comet = evaluate.load("comet")
 
 
 def extract_answer(text, debug=False):
@@ -43,7 +44,7 @@ def extract_answer(text, debug=False):
     return text
 
 
-def calc_metrics(references, predictions, debug=False):
+def calc_metrics(references, predictions, sources=None, debug=False):
     assert len(references) == len(
         predictions
     ), f"lengths are difference: {len(references)} != {len(predictions)}"
@@ -51,6 +52,10 @@ def calc_metrics(references, predictions, debug=False):
     predictions = [extract_answer(text) for text in predictions]
     results = {}
 
+    results["comet"] = comet.compute(
+        predictions=predictions, references=references, sources=sources
+    )["mean_score"]
+
     results["meteor"] = meteor.compute(predictions=predictions, references=references)[
         "meteor"
     ]
@@ -233,7 +238,18 @@ def detect_repetition_scores(row, col, debug=False):
     )
 
 
-def contains_chinese(text):
+def count_chinese_characters(text):
+    # Define a regular expression pattern for Chinese characters
+    chinese_char_pattern = r"[\u4e00-\u9fff]"
+
+    # Use re.findall to find all Chinese characters in the text
+    chinese_chars = re.findall(chinese_char_pattern, text)
+
+    # Return the count of Chinese characters
+    return len(chinese_chars)
+
+
+def count_chinese_characters(text):
     chinese_char_pattern = re.compile(r"[\u4e00-\u9fff]")
     return 1 if chinese_char_pattern.search(text) else 0
 
@@ -264,7 +280,7 @@ def get_metrics(df, max_output_tokens=2048, variant="rpp"):
     repetition_score = []
     total_repetitions = []
     num_max_output_tokens = []
-    num_incomplete_translations = []
+    translation_completeness = []
     columns = df.columns[2:]
 
     df[
@@ -275,8 +291,13 @@ def get_metrics(df, max_output_tokens=2048, variant="rpp"):
         ]
     ] = df["english"].apply(detect_scores)
 
+    new_col = f"count_chinese_characters-ground_truth"
+    df[new_col] = df["chinese"].apply(count_chinese_characters)
+
     for col in columns:
-        metrics = calc_metrics(df["english"], df[col], debug=True)
+        metrics = calc_metrics(
+            df["english"], df[col], sources=df["chinese"], debug=True
+        )
         print(f"{col}: {metrics}")
 
         meteor.append(metrics["meteor"])
@@ -298,9 +319,11 @@ def get_metrics(df, max_output_tokens=2048, variant="rpp"):
             lambda x: len(tokenizers[model](x)["input_ids"])
         )
 
-        new_col = f"contains_chinese-{col}"
-        df[new_col] = df[col].apply(contains_chinese)
-        num_incomplete_translations.append(df[new_col].sum())
+        new_col = f"count_chinese_characters-{col}"
+        df[new_col] = df[col].apply(count_chinese_characters)
+        translation_completeness.append(
+            1 - df[new_col].sum() / df["count_chinese_characters-ground_truth"].sum()
+        )
 
         new_col = f"output_tokens-{col}"
         df[new_col] = df[col].apply(lambda x: len(tokenizers[model](x)["input_ids"]))
@@ -320,7 +343,7 @@ def get_metrics(df, max_output_tokens=2048, variant="rpp"):
         lambda x: x["meteor"] / math.log10(10 + x["total_repetitions"]), axis=1
     )
 
-    metrics_df["num_incomplete_translations"] = num_incomplete_translations
+    metrics_df["translation_completeness"] = translation_completeness
     metrics_df["num_max_output_tokens"] = num_max_output_tokens
 
     if variant != "rpp":
@@ -359,8 +382,10 @@ def analyze_translation_results(df, col, max_new_tokens=300, repetition_threshol
     )
     print_row_details(df2, range(len(df2)))
 
-    contains_chinese = f"contains_chinese-{col}"
-    df3 = df[df[contains_chinese] > 0][["chinese", "english", col, contains_chinese]]
+    count_chinese_characters = f"count_chinese_characters-{col}"
+    df3 = df[df[count_chinese_characters] > 0][
+        ["chinese", "english", col, count_chinese_characters]
+    ]
 
     print(f"\n*** Found {len(df3)} rows with incomplete translations for {col}")
     print_row_details(df3, range(len(df3)))

requirements.txt

@@ -19,3 +19,4 @@ einops==0.8.0
 accelerate==0.32.0
 peft==0.11.1
 sacrebleu==2.4.2
+unbabel-comet==2.2.2