from __future__ import annotations import argparse import json import logging from pathlib import Path from .common import ARTIFACT_DIR, existing_default_checkpoint LOGGER = logging.getLogger(__name__) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Generate an HTML evaluation report.") parser.add_argument( "--checkpoint-dir", default=existing_default_checkpoint(), help="Path to the trained model checkpoint directory containing metrics.", ) parser.add_argument( "--output-file", default=str(ARTIFACT_DIR / "eval_report.html"), help="Output HTML file path.", ) return parser.parse_args() def load_metrics(checkpoint_dir: Path) -> dict[str, dict[str, float]]: metrics = {} metrics_dir = checkpoint_dir / "metrics" if not metrics_dir.exists(): return metrics for split in ["train", "validation", "test"]: file_path = metrics_dir / f"{split}_metrics.json" if file_path.exists(): try: metrics[split] = json.loads(file_path.read_text(encoding="utf-8")) except Exception as e: LOGGER.warning(f"Failed to load {file_path}: {e}") return metrics def load_predictions(checkpoint_dir: Path) -> list[dict]: # We look for the predictions file in the artifact directory, # since eval.py writes it there by default. pred_file = ARTIFACT_DIR / "sample_predictions.jsonl" preds = [] if pred_file.exists(): try: for line in pred_file.read_text(encoding="utf-8").splitlines(): if line.strip(): preds.append(json.loads(line)) except Exception as e: LOGGER.warning(f"Failed to load predictions from {pred_file}: {e}") return preds def generate_html(checkpoint_name: str, metrics: dict, predictions: list) -> str: html = f""" Evaluation Report - {checkpoint_name}

Model Evaluation Report

Checkpoint: {checkpoint_name}

Overall Metrics

""" for split in ["train", "validation", "test"]: m = metrics.get(split, {}) if not m: continue prefix = split + "_" if split != "train" else "" loss = m.get(f"{prefix}loss", m.get("train_loss", "-")) r1 = m.get(f"{prefix}rouge1", "-") r2 = m.get(f"{prefix}rouge2", "-") rl = m.get(f"{prefix}rougeL", "-") bf1 = m.get(f"{prefix}bertscore_f1", "-") glen = m.get(f"{prefix}gen_len", "-") def fmt(v): return f"{v:.4f}" if isinstance(v, float) else str(v) html += f""" """ html += """
Split Loss ROUGE-1 ROUGE-2 ROUGE-L BERTScore F1 Avg Gen Length
{split.title()} {fmt(loss)} {fmt(r1)} {fmt(r2)} {fmt(rl)} {fmt(bf1)} {fmt(glen)}

Sample Predictions

""" if not predictions: html += "

No predictions found.

" else: for i, p in enumerate(predictions): empty_tag = " (EMPTY PREDICTION)" if p.get("empty_prediction") else "" html += f"""
Source: {p.get("source", "")}
Target: {p.get("reference", "")}
Model:{empty_tag} {p.get("prediction", "")}
""" html += """ """ return html def main() -> None: logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") args = parse_args() if not args.checkpoint_dir: LOGGER.error("No checkpoint directory provided or found.") return checkpoint_path = Path(args.checkpoint_dir) if not checkpoint_path.exists(): LOGGER.error(f"Checkpoint directory not found: {checkpoint_path}") return metrics = load_metrics(checkpoint_path) predictions = load_predictions(checkpoint_path) html_content = generate_html(checkpoint_path.name, metrics, predictions) out_file = Path(args.output_file) out_file.parent.mkdir(parents=True, exist_ok=True) out_file.write_text(html_content, encoding="utf-8") LOGGER.info(f"Evaluation report generated at: {out_file.absolute()}") if __name__ == "__main__": main()