[Person D] Implement evaluation module: metrics, decoding, evaluator, scripts

scripts/evaluate.py

@@ -10,11 +10,17 @@
 """
 
 import argparse
+import json
 import sys
 from pathlib import Path
 
+import torch
+import yaml
+
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
 
+from easytranslate.evaluation.evaluator import Evaluator
+
 
 def parse_args():
     parser = argparse.ArgumentParser(description="EasyTranslate Evaluation")
@@ -43,7 +49,71 @@ def main():
     print("  EasyTranslate - Evaluation")
     print("=" * 60)
 
-    raise NotImplementedError("TODO: Person D 实现评估主流程")
+    # 1. 加载配置
+    with open(args.config, "r", encoding="utf-8") as f:
+        config = yaml.safe_load(f)
+
+    # 应用命令行覆盖 (格式: key.subkey=value)
+    for override in cli_overrides:
+        if "=" in override:
+            key, value = override.split("=", 1)
+            keys = key.split(".")
+            d = config
+            for k in keys[:-1]:
+                d = d.setdefault(k, {})
+            d[keys[-1]] = yaml.safe_load(value)
+
+    # 2. 加载检查点并重建模型
+    checkpoint = torch.load(args.checkpoint, map_location="cpu")
+    model_config = config.get("model", {})
+
+    from easytranslate.model.transformer import Transformer
+    model = Transformer(model_config)
+    model.load_state_dict(checkpoint["model_state_dict"])
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    model = model.to(device)
+    model.eval()
+
+    # 3. 加载 tokenizer 和测试数据
+    from easytranslate.data.tokenizer import build_tokenizer
+    from easytranslate.data.dataset import TranslationDataset
+    from torch.utils.data import DataLoader
+
+    tokenizer = build_tokenizer(config.get("data", {}).get("tokenizer", {}))
+
+    test_dataset = TranslationDataset(
+        config=config.get("data", {}),
+        tokenizer=tokenizer,
+        split="test",
+    )
+    test_loader = DataLoader(
+        test_dataset,
+        batch_size=config.get("evaluation", {}).get("batch_size", 32),
+        shuffle=False,
+    )
+
+    # 4. 构建 Evaluator
+    evaluator = Evaluator(model=model, tokenizer=tokenizer, config=config)
+
+    # 5. 运行评估
+    src_texts = [sample["src"] for sample in test_dataset.raw_data]
+    ref_texts = [sample["tgt"] for sample in test_dataset.raw_data]
+    results = evaluator.evaluate(test_loader, src_texts=src_texts, ref_texts=ref_texts)
+
+    # 6. 打印和保存结果
+    print("\n" + "=" * 60)
+    print("  Evaluation Results")
+    print("=" * 60)
+    for metric, score in results.items():
+        if not isinstance(score, list):
+            print(f"  {metric:>10s}: {score:.4f}")
+
+    output_path = Path(args.output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(output_path, "w", encoding="utf-8") as f:
+        json.dump(results, f, indent=2, ensure_ascii=False)
+    print(f"\n  Results saved to: {output_path}")
 
 
 if __name__ == "__main__":

scripts/translate.py

@@ -16,8 +16,13 @@ import argparse
 import sys
 from pathlib import Path
 
+import torch
+import yaml
+
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
 
+from easytranslate.evaluation.evaluator import Evaluator
+
 
 def parse_args():
     parser = argparse.ArgumentParser(description="EasyTranslate Inference")
@@ -39,7 +44,25 @@ def interactive_translate(evaluator):
     3. 打印翻译结果
     4. 输入 'quit' 退出
     """
-    raise NotImplementedError("TODO: Person D 实现 interactive_translate")
+    print("\nInteractive Translation Mode (type 'quit' to exit)")
+    print("-" * 40)
+
+    while True:
+        try:
+            text = input("\n[EN] > ").strip()
+        except (EOFError, KeyboardInterrupt):
+            print("\nBye!")
+            break
+
+        if text.lower() in ("quit", "exit", "q"):
+            print("Bye!")
+            break
+
+        if not text:
+            continue
+
+        translation = evaluator.translate_single(text)
+        print(f"[ZH] > {translation}")
 
 
 def translate_file(evaluator, input_path: str, output_path: str):
@@ -51,7 +74,33 @@ def translate_file(evaluator, input_path: str, output_path: str):
     2. 批量翻译
     3. 将结果写入输出文件
     """
-    raise NotImplementedError("TODO: Person D 实现 translate_file")
+    input_file = Path(input_path)
+    if not input_file.exists():
+        print(f"Error: input file not found: {input_path}")
+        return
+
+    with open(input_file, "r", encoding="utf-8") as f:
+        lines = [line.strip() for line in f if line.strip()]
+
+    print(f"Translating {len(lines)} sentences...")
+
+    # 分批翻译
+    batch_size = 32
+    translations = []
+    for i in range(0, len(lines), batch_size):
+        batch = lines[i : i + batch_size]
+        batch_translations = evaluator.translate(batch)
+        translations.extend(batch_translations)
+        print(f"  Translated {min(i + batch_size, len(lines))}/{len(lines)}")
+
+    # 写入输出文件
+    out_file = Path(output_path)
+    out_file.parent.mkdir(parents=True, exist_ok=True)
+    with open(out_file, "w", encoding="utf-8") as f:
+        for t in translations:
+            f.write(t + "\n")
+
+    print(f"Results saved to: {output_path}")
 
 
 def launch_web_ui(evaluator):
@@ -64,7 +113,25 @@ def launch_web_ui(evaluator):
     3. 输出: 中文翻译结果
     4. 调用 evaluator.translate_single()
     """
-    raise NotImplementedError("TODO: Person D 实现 launch_web_ui")
+    try:
+        import gradio as gr
+    except ImportError:
+        print("Error: gradio is not installed. Install with: pip install gradio")
+        return
+
+    def translate_fn(text):
+        if not text.strip():
+            return ""
+        return evaluator.translate_single(text)
+
+    interface = gr.Interface(
+        fn=translate_fn,
+        inputs=gr.Textbox(label="English", placeholder="Enter English text..."),
+        outputs=gr.Textbox(label="Chinese Translation"),
+        title="EasyTranslate - English to Chinese",
+        description="Transformer-based English to Chinese translation system.",
+    )
+    interface.launch()
 
 
 def main():
@@ -74,8 +141,37 @@ def main():
     print("  EasyTranslate - Translation")
     print("=" * 60)
 
-    # TODO: 加载模型、构建 evaluator，然后根据参数选择模式
-    raise NotImplementedError("TODO: 实现推理主流程")
+    # 加载配置
+    with open(args.config, "r", encoding="utf-8") as f:
+        config = yaml.safe_load(f)
+
+    # 加载模型
+    checkpoint = torch.load(args.checkpoint, map_location="cpu")
+
+    from easytranslate.model.transformer import Transformer
+    from easytranslate.data.tokenizer import build_tokenizer
+
+    model_config = config.get("model", {})
+    model = Transformer(model_config)
+    model.load_state_dict(checkpoint["model_state_dict"])
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    model = model.to(device)
+    model.eval()
+
+    tokenizer = build_tokenizer(config.get("data", {}).get("tokenizer", {}))
+
+    # 构建 evaluator
+    evaluator = Evaluator(model=model, tokenizer=tokenizer, config=config)
+
+    # 根据参数选择模式
+    if args.web:
+        launch_web_ui(evaluator)
+    elif args.input:
+        output_path = args.output or args.input.replace(".txt", "_translated.txt")
+        translate_file(evaluator, args.input, output_path)
+    else:
+        interactive_translate(evaluator)
 
 
 if __name__ == "__main__":

src/easytranslate/evaluation/decoding.py

@@ -47,7 +47,22 @@ def greedy_decode(
        d. 如果所有序列都生成了 eos_id，则提前终止
     4. 返回生成的 token ids [B, T]
     """
-    raise NotImplementedError("TODO: Person D 实现 greedy_decode")
+    encoder_output = model.encode(src_ids, src_padding_mask)
+    batch_size = src_ids.size(0)
+    device = src_ids.device
+
+    decoder_input = torch.full((batch_size, 1), bos_id, dtype=torch.long, device=device)
+    finished = torch.zeros(batch_size, dtype=torch.bool, device=device)
+
+    for _ in range(max_len):
+        logits = model.decode_step(decoder_input, encoder_output, src_padding_mask)
+        next_token = logits.argmax(dim=-1, keepdim=True)
+        decoder_input = torch.cat([decoder_input, next_token], dim=1)
+        finished = finished | next_token.squeeze(-1).eq(eos_id)
+        if finished.all():
+            break
+
+    return decoder_input
 
 
 @torch.no_grad()
@@ -86,7 +101,76 @@ def beam_search_decode(
 
     这是翻译任务最关键的解码算法，请仔细实现。
     """
-    raise NotImplementedError("TODO: Person D 实现 beam_search_decode")
+    batch_size, seq_len = src_ids.size()
+    device = src_ids.device
+
+    # 1. Encode
+    encoder_output = encoder_output = model.encode(src_ids, src_padding_mask)
+
+    # 2. Expand encoder output for beam search: [B*beam, S, D]
+    encoder_output = encoder_output.unsqueeze(1).expand(-1, beam_size, -1, -1)
+    encoder_output = encoder_output.reshape(batch_size * beam_size, seq_len, -1)
+    src_padding_mask_expanded = src_padding_mask.unsqueeze(1).expand(-1, beam_size, -1)
+    src_padding_mask_expanded = src_padding_mask_expanded.reshape(batch_size * beam_size, seq_len)
+
+    # 3. Initialize beams
+    beam_scores = torch.zeros(batch_size, beam_size, device=device)
+    beam_scores[:, 1:] = float("-inf")  # Only first beam is active initially
+    beam_tokens = torch.full((batch_size, beam_size, 1), bos_id, dtype=torch.long, device=device)
+    finished = torch.zeros(batch_size, beam_size, dtype=torch.bool, device=device)
+
+    # 4. Iterative decoding
+    for _ in range(max_len):
+        flat_tokens = beam_tokens.view(batch_size * beam_size, -1)
+        logits = model.decode_step(flat_tokens, encoder_output, src_padding_mask_expanded)
+        log_probs = F.log_softmax(logits, dim=-1)
+        vocab_size = log_probs.size(-1)
+
+        # (c) Apply no_repeat_ngram constraint
+        if no_repeat_ngram_size > 0:
+            log_probs = _apply_no_repeat_ngram(log_probs, flat_tokens, no_repeat_ngram_size)
+
+        # Mask finished beams
+        finished_flat = finished.view(batch_size * beam_size)
+        if finished_flat.any():
+            log_probs[finished_flat] = float("-inf")
+            log_probs[finished_flat, eos_id] = 0.0
+
+        # (d) Compute scores
+        scores = beam_scores.unsqueeze(-1) + log_probs.view(batch_size, beam_size, vocab_size)
+        scores = scores.view(batch_size, -1)  # [B, beam * vocab]
+
+        # (e) Select top-k
+        topk_scores, topk_indices = scores.topk(beam_size, dim=-1)
+        beam_indices = topk_indices // vocab_size
+        token_indices = topk_indices % vocab_size
+
+        # (f) Update beam tokens and scores
+        new_tokens = []
+        new_finished = []
+        for b in range(batch_size):
+            prev_seqs = beam_tokens[b][beam_indices[b]]
+            next_tokens = token_indices[b].unsqueeze(-1)
+            new_tokens.append(torch.cat([prev_seqs, next_tokens], dim=-1))
+            new_finished.append(finished[b][beam_indices[b]] | token_indices[b].eq(eos_id))
+
+        beam_tokens = torch.stack(new_tokens, dim=0)
+        finished = torch.stack(new_finished, dim=0)
+        beam_scores = topk_scores
+
+        if finished.all():
+            break
+
+    # 5. Apply length penalty
+    lengths = beam_tokens.size(-1) - 1  # Exclude BOS
+    penalties = lengths ** length_penalty
+    final_scores = beam_scores / penalties
+
+    # 6. Select best beam for each batch
+    best_indices = final_scores.argmax(dim=-1)
+    best_sequences = beam_tokens[torch.arange(batch_size, device=device), best_indices]
+
+    return best_sequences
 
 
 @torch.no_grad()
@@ -111,7 +195,46 @@ def sample_decode(
     4. 应用 top-p (nucleus): 只保留累积概率达到 p 的 token
     5. 从过滤后的分布中采样: torch.multinomial
     """
-    raise NotImplementedError("TODO: Person D 实现 sample_decode")
+    encoder_output = model.encode(src_ids, src_padding_mask)
+    batch_size = src_ids.size(0)
+    device = src_ids.device
+
+    decoder_input = torch.full((batch_size, 1), bos_id, dtype=torch.long, device=device)
+    finished = torch.zeros(batch_size, dtype=torch.bool, device=device)
+
+    for _ in range(max_len):
+        logits = model.decode_step(decoder_input, encoder_output, src_padding_mask)
+
+        # 2. Apply temperature
+        logits = logits / max(temperature, 1e-8)
+
+        # 3. Apply top-k filtering
+        if top_k > 0:
+            k = min(top_k, logits.size(-1))
+            topk_values, _ = torch.topk(logits, k, dim=-1)
+            threshold = topk_values[:, -1].unsqueeze(-1)
+            logits[logits < threshold] = float("-inf")
+
+        # 4. Apply top-p (nucleus) filtering
+        if 0.0 < top_p < 1.0:
+            sorted_logits, sorted_indices = torch.sort(logits, descending=True, dim=-1)
+            cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
+            mask = cumulative_probs - F.softmax(sorted_logits, dim=-1) >= top_p
+            sorted_logits[mask] = float("-inf")
+            logits = sorted_logits.scatter(1, sorted_indices.argsort(1), sorted_logits)
+
+        # 5. Sample from filtered distribution
+        probs = F.softmax(logits, dim=-1)
+        next_token = torch.multinomial(probs, num_samples=1)
+
+        next_token = torch.where(finished.unsqueeze(-1), torch.full_like(next_token, eos_id), next_token)
+        decoder_input = torch.cat([decoder_input, next_token], dim=1)
+        finished = finished | next_token.squeeze(-1).eq(eos_id)
+
+        if finished.all():
+            break
+
+    return decoder_input
 
 
 def _apply_no_repeat_ngram(
@@ -126,4 +249,26 @@ def _apply_no_repeat_ngram(
     1. 从 generated_tokens 中提取所有已出现的 (ngram_size-1)-gram
     2. 对于每个可能导致重复 ngram 的 next token，将其 logits 设为 -inf
     """
-    raise NotImplementedError("TODO: Person D 实现 _apply_no_repeat_ngram")
+    if ngram_size <= 0:
+        return logits
+
+    batch_size = logits.size(0)
+    for batch_idx in range(batch_size):
+        tokens = generated_tokens[batch_idx].tolist()
+        if len(tokens) < ngram_size - 1:
+            continue
+
+        # Build map of (n-1)-gram prefix -> set of next tokens that appeared
+        ngram_map: dict[tuple, set] = {}
+        for i in range(len(tokens) - ngram_size + 1):
+            prefix = tuple(tokens[i : i + ngram_size - 1])
+            next_tok = tokens[i + ngram_size - 1]
+            ngram_map.setdefault(prefix, set()).add(next_tok)
+
+        # Check current prefix and ban tokens that would create repeated n-grams
+        current_prefix = tuple(tokens[-(ngram_size - 1):])
+        if current_prefix in ngram_map:
+            banned = list(ngram_map[current_prefix])
+            logits[batch_idx, banned] = float("-inf")
+
+    return logits

src/easytranslate/evaluation/evaluator.py

@@ -39,7 +39,65 @@ class Evaluator:
         2. 从 config 读取解码策略和评估指标配置
         3. 根据策略选择解码函数
         """
-        raise NotImplementedError("TODO: Person D 实现 Evaluator.__init__")
+        self.model = model
+        self.tokenizer = tokenizer
+        self.config = config
+
+        # 从 config 读取评估和解码配置
+        eval_config = config.get("evaluation", {})
+        decoding_config = eval_config.get("decoding", {})
+
+        self.strategy = decoding_config.get("strategy", "beam_search")
+        self.max_len = decoding_config.get("max_decode_len", 256)
+        self.beam_size = decoding_config.get("beam_size", 5)
+        self.length_penalty = decoding_config.get("length_penalty", 1.0)
+        self.no_repeat_ngram_size = decoding_config.get("no_repeat_ngram_size", 0)
+
+        sampling_config = decoding_config.get("sampling", {})
+        self.temperature = sampling_config.get("temperature", 1.0)
+        self.top_k = sampling_config.get("top_k", 0)
+        self.top_p = sampling_config.get("top_p", 1.0)
+
+        self.metrics = eval_config.get("metrics", ["bleu", "comet", "chrf", "ter"])
+
+        self.bos_id = tokenizer.bos_token_id
+        self.eos_id = tokenizer.eos_token_id
+        self.pad_id = tokenizer.pad_token_id
+
+        # 根据策略选择解码函数
+        self.decode_fn = self._get_decode_fn()
+
+    def _get_decode_fn(self):
+        """根据策略选择解码函数。"""
+        if self.strategy == "greedy":
+            return self._greedy
+        elif self.strategy == "sampling":
+            return self._sample
+        else:
+            return self._beam_search
+
+    def _greedy(self, src_ids, src_padding_mask):
+        return greedy_decode(
+            self.model, src_ids, src_padding_mask,
+            self.bos_id, self.eos_id, max_len=self.max_len,
+        )
+
+    def _beam_search(self, src_ids, src_padding_mask):
+        return beam_search_decode(
+            self.model, src_ids, src_padding_mask,
+            self.bos_id, self.eos_id,
+            beam_size=self.beam_size, max_len=self.max_len,
+            length_penalty=self.length_penalty,
+            no_repeat_ngram_size=self.no_repeat_ngram_size,
+        )
+
+    def _sample(self, src_ids, src_padding_mask):
+        return sample_decode(
+            self.model, src_ids, src_padding_mask,
+            self.bos_id, self.eos_id,
+            max_len=self.max_len, temperature=self.temperature,
+            top_k=self.top_k, top_p=self.top_p,
+        )
 
     def evaluate(
         self,
@@ -57,7 +115,35 @@ class Evaluator:
         4. 调用 compute_all_metrics 计算指标
         5. 返回评估结果 dict
         """
-        raise NotImplementedError("TODO: Person D 实现 evaluate")
+        self.model.eval()
+        device = next(self.model.parameters()).device
+
+        hypotheses = []
+
+        with torch.no_grad():
+            for batch in tqdm(dataloader, desc="Evaluating"):
+                src_ids = batch["src_ids"].to(device)
+                src_padding_mask = batch.get("src_padding_mask")
+                if src_padding_mask is None:
+                    src_padding_mask = src_ids.eq(self.pad_id)
+                else:
+                    src_padding_mask = src_padding_mask.to(device)
+
+                output_ids = self.decode_fn(src_ids, src_padding_mask)
+
+                for i in range(output_ids.size(0)):
+                    text = self.tokenizer.decode(
+                        output_ids[i].tolist(), skip_special_tokens=True
+                    )
+                    hypotheses.append(text)
+
+        results = compute_all_metrics(
+            sources=src_texts,
+            hypotheses=hypotheses,
+            references=ref_texts,
+            metrics=self.metrics,
+        )
+        return results
 
     def translate(self, texts: list[str]) -> list[str]:
         """
@@ -69,7 +155,28 @@ class Evaluator:
         3. 解码为文本
         4. 返回翻译结果列表
         """
-        raise NotImplementedError("TODO: Person D 实现 translate")
+        self.model.eval()
+        device = next(self.model.parameters()).device
+
+        # 1. Tokenize
+        encoded = [self.tokenizer.encode(t, add_special_tokens=True) for t in texts]
+        max_len_src = max(len(ids) for ids in encoded)
+        src_ids = torch.full((len(texts), max_len_src), self.pad_id, dtype=torch.long, device=device)
+        for i, ids in enumerate(encoded):
+            src_ids[i, :len(ids)] = torch.tensor(ids, dtype=torch.long)
+        src_padding_mask = src_ids.eq(self.pad_id)
+
+        # 2. Decode
+        with torch.no_grad():
+            output_ids = self.decode_fn(src_ids, src_padding_mask)
+
+        # 3. Convert to text
+        translations = []
+        for i in range(output_ids.size(0)):
+            text = self.tokenizer.decode(output_ids[i].tolist(), skip_special_tokens=True)
+            translations.append(text)
+
+        return translations
 
     def translate_single(self, text: str) -> str:
         """翻译单��文本。"""

src/easytranslate/evaluation/metrics.py

@@ -18,8 +18,11 @@
 from __future__ import annotations
 
 import logging
+import time
 from typing import Optional
 
+import sacrebleu
+
 logger = logging.getLogger(__name__)
 
 
@@ -38,7 +41,15 @@ def compute_bleu(
 
     注意: references 需要包装为 list of list (支持多参考)
     """
-    raise NotImplementedError("TODO: Person D 实现 compute_bleu")
+    bleu = sacrebleu.corpus_bleu(hypotheses, [references], tokenize=tokenize)
+    return {
+        "bleu": round(float(bleu.score), 4),
+        "bleu_1": round(float(bleu.precisions[0]), 4),
+        "bleu_2": round(float(bleu.precisions[1]), 4),
+        "bleu_3": round(float(bleu.precisions[2]), 4),
+        "bleu_4": round(float(bleu.precisions[3]), 4),
+        "bp": round(float(bleu.bp), 4),
+    }
 
 
 def compute_comet(
@@ -60,7 +71,17 @@ def compute_comet(
 
     COMET 需要源语言、翻译结果和参考翻译三者。
     """
-    raise NotImplementedError("TODO: Person D 实现 compute_comet")
+    from comet import download_model, load_from_checkpoint
+
+    model_path = download_model(model_name)
+    model = load_from_checkpoint(model_path)
+
+    data = [{"src": s, "mt": h, "ref": r} for s, h, r in zip(sources, hypotheses, references)]
+    prediction = model.predict(data, batch_size=batch_size, gpus=gpus)
+
+    system_score = prediction.system_score
+    segment_scores = [float(s) for s in prediction.scores]
+    return {"comet": round(float(system_score), 4), "comet_scores": segment_scores}
 
 
 def compute_chrf(
@@ -74,7 +95,8 @@ def compute_chrf(
     1. 使用 sacrebleu.corpus_chrf(hypotheses, [references])
     2. 返回 {"chrf": score}
     """
-    raise NotImplementedError("TODO: Person D 实现 compute_chrf")
+    chrf = sacrebleu.corpus_chrf(hypotheses, [references])
+    return {"chrf": round(float(chrf.score), 4)}
 
 
 def compute_ter(
@@ -88,7 +110,8 @@ def compute_ter(
     1. 使用 sacrebleu.corpus_ter(hypotheses, [references])
     2. 返回 {"ter": score}
     """
-    raise NotImplementedError("TODO: Person D 实现 compute_ter")
+    ter = sacrebleu.corpus_ter(hypotheses, [references])
+    return {"ter": round(float(ter.score), 4)}
 
 
 def compute_all_metrics(
@@ -106,4 +129,21 @@ def compute_all_metrics(
     3. 合并结果并返回
     4. 记录每个指标的计算时间
     """
-    raise NotImplementedError("TODO: Person D 实现 compute_all_metrics")
+    results = {}
+    metric_funcs = {
+        "bleu": lambda: compute_bleu(hypotheses, references),
+        "comet": lambda: compute_comet(sources, hypotheses, references),
+        "chrf": lambda: compute_chrf(hypotheses, references),
+        "ter": lambda: compute_ter(hypotheses, references),
+    }
+
+    for metric in metrics:
+        if metric not in metric_funcs:
+            logger.warning("Unknown metric: %s, skipping", metric)
+            continue
+        start = time.time()
+        results.update(metric_funcs[metric]())
+        elapsed = time.time() - start
+        logger.info("Computed %s in %.2fs", metric, elapsed)
+
+    return results