完成训练

requirements-colab.txt

@@ -8,13 +8,14 @@
 transformers>=4.36.0,<4.45.0
 datasets>=2.16.0,<3.0.0
 tokenizers>=0.15.0,<0.20.0
-sentencepiece>=0.1.99,<0.2.0
+sentencepiece>=0.2.0,<0.3.0
 accelerate>=0.25.0,<0.35.0
 peft>=0.7.0,<0.12.0
 sacrebleu>=2.4.0,<3.0.0
-unbabel-comet>=2.2.0,<2.3.0
 omegaconf>=2.3.0,<3.0.0
 rich>=13.0.0,<14.0.0
 tqdm>=4.66.0,<5.0.0
 wandb>=0.16.0,<0.18.0
-tensorboard>=2.15.0,<2.17.0
+tensorboard>=2.15.0,<2.17.0
+protobuf>=5.28.3,<7.0.0
+fsspec==2025.3.0

requirements.txt

@@ -7,14 +7,14 @@
 # =============================================================================
 
 # Core ML Framework
-torch>=2.1.0,<2.5.0
+torch>=2.1.0,<2.11.0
 torchvision>=0.16.0; platform_system != "Darwin" or platform_machine != "arm64"
 
 # HuggingFace Ecosystem
 transformers>=4.36.0,<4.45.0
 datasets>=2.16.0,<3.0.0
 tokenizers>=0.15.0,<0.20.0
-sentencepiece>=0.1.99,<0.2.0
+sentencepiece>=0.2.0,<0.3.0
 accelerate>=0.25.0,<0.35.0
 
 # Parameter-Efficient Fine-Tuning
@@ -40,8 +40,9 @@ pyyaml>=6.0.0,<7.0.0
 rich>=13.0.0,<14.0.0
 
 # Data Processing
-numpy>=1.24.0,<2.0.0
+numpy>=1.24.0,<3.0.0
 pandas>=2.0.0,<2.3.0
+protobuf>=5.28.3,<7.0.0
 tqdm>=4.66.0,<5.0.0
 
 # Visualization

result/evaluation_results.json

@@ -0,0 +1,10 @@
+{
+  "bleu": 15.1403,
+  "bleu_1": 43.0105,
+  "bleu_2": 20.1492,
+  "bleu_3": 10.4122,
+  "bleu_4": 5.8232,
+  "bp": 1.0,
+  "chrf": 15.2513,
+  "ter": 177.5966
+}

result/training_summary.json

@@ -0,0 +1,30 @@
+{
+  "best_epoch": 3,
+  "best_metric": Infinity,
+  "metric_name": "bleu",
+  "total_steps": 7630,
+  "train_loss_history": [
+    8.249846095573231,
+    6.610892885478108,
+    5.28016960105296,
+    4.47492950561006,
+    4.051334980548131
+  ],
+  "val_metrics_history": [
+    {
+      "val_loss": 7.903
+    },
+    {
+      "val_loss": 6.9446
+    },
+    {
+      "val_loss": 6.0977
+    },
+    {
+      "val_loss": 5.6509
+    },
+    {
+      "val_loss": 5.4433
+    }
+  ]
+}

result/translation_examples.json

@@ -0,0 +1,18 @@
+[
+  {
+    "source_en": "Hello, how are you today?",
+    "target_zh": "今天,你如何?"
+  },
+  {
+    "source_en": "Machine translation is an important field of natural language processing.",
+    "target_zh": "生物交流是自然语言处理的重要领域。"
+  },
+  {
+    "source_en": "The weather is beautiful and I want to go for a walk.",
+    "target_zh": "天气是美丽的,我想去。"
+  },
+  {
+    "source_en": "Deep learning has revolutionized artificial intelligence research.",
+    "target_zh": "深度学习推动了人工智能情报研究。"
+  }
+]

setup.py

@@ -14,16 +14,17 @@ setup(
     package_dir={"": "src"},
     python_requires=">=3.10",
     install_requires=[
-        "torch>=2.1.0,<2.5.0",
+        "torch>=2.1.0,<2.11.0",
         "transformers>=4.36.0,<4.45.0",
         "datasets>=2.16.0,<3.0.0",
         "tokenizers>=0.15.0,<0.20.0",
-        "sentencepiece>=0.1.99,<0.2.0",
+        "sentencepiece>=0.2.0,<0.3.0",
         "accelerate>=0.25.0,<0.35.0",
         "peft>=0.7.0,<0.12.0",
         "sacrebleu>=2.4.0,<3.0.0",
         "unbabel-comet>=2.2.0,<2.3.0",
-        "numpy>=1.24.0,<2.0.0",
+        "numpy>=1.24.0,<3.0.0",
+        "protobuf>=5.28.3,<7.0.0",
         "omegaconf>=2.3.0,<3.0.0",
         "rich>=13.0.0,<14.0.0",
         "tqdm>=4.66.0,<5.0.0",
@@ -41,11 +42,12 @@ setup(
             "transformers>=4.36.0,<4.45.0",
             "datasets>=2.16.0,<3.0.0",
             "tokenizers>=0.15.0,<0.20.0",
-            "sentencepiece>=0.1.99,<0.2.0",
+            "sentencepiece>=0.2.0,<0.3.0",
             "accelerate>=0.25.0,<0.35.0",
             "peft>=0.7.0,<0.12.0",
             "sacrebleu>=2.4.0,<3.0.0",
             "unbabel-comet>=2.2.0,<2.3.0",
+            "protobuf>=5.28.3,<7.0.0",
             "omegaconf>=2.3.0,<3.0.0",
             "rich>=13.0.0,<14.0.0",
             "tqdm>=4.66.0,<5.0.0",

src/easytranslate/evaluation/metrics.py

@@ -21,11 +21,20 @@ import logging
 import time
 from typing import Optional
 
-import sacrebleu
-
 logger = logging.getLogger(__name__)
 
 
+def _require_sacrebleu():
+    try:
+        import sacrebleu
+    except ImportError as exc:
+        raise ImportError(
+            "sacrebleu is required for BLEU/chrF/TER evaluation. "
+            "Install it with `pip install sacrebleu`."
+        ) from exc
+    return sacrebleu
+
+
 def compute_bleu(
     hypotheses: list[str],
     references: list[str],
@@ -53,7 +62,8 @@ def compute_bleu(
             f"hypotheses and references length mismatch: "
             f"{len(hypotheses)} vs {len(references)}"
         )
-    
+
+    sacrebleu = _require_sacrebleu()
     bleu = sacrebleu.corpus_bleu(hypotheses, [references], tokenize=tokenize)
     return {
         "bleu": round(float(bleu.score), 4),
@@ -144,7 +154,8 @@ def compute_chrf(
             f"hypotheses and references length mismatch: "
             f"{len(hypotheses)} vs {len(references)}"
         )
-    
+
+    sacrebleu = _require_sacrebleu()
     chrf = sacrebleu.corpus_chrf(hypotheses, [references])
     return {"chrf": round(float(chrf.score), 4)}
 
@@ -174,7 +185,8 @@ def compute_ter(
             f"hypotheses and references length mismatch: "
             f"{len(hypotheses)} vs {len(references)}"
         )
-    
+
+    sacrebleu = _require_sacrebleu()
     ter = sacrebleu.corpus_ter(hypotheses, [references])
     return {"ter": round(float(ter.score), 4)}