Add application file

.dockerignore

@@ -0,0 +1,10 @@
+venv
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+.git
+.gitignore
+node_modules
+.env
+.cache

.gitignore

@@ -0,0 +1,26 @@
+.DS_Store
+node_modules/
+dist/
+build/
+*.log
+.env
+.env.local
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+env/
+venv/
+ENV/
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+.pytest_cache/
+.coverage
+htmlcov/
+.venv
+pip-log.txt
+pip-delete-this-directory.txt

Dockerfile

@@ -0,0 +1,34 @@
+# Use Python base image
+FROM python:3.12-slim
+
+# Prevent Python from writing .pyc files
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# Prevent Python from buffering stdout/stderr
+ENV PYTHONUNBUFFERED=1
+
+# Set working directory
+WORKDIR /app
+
+# Install system dependencies needed by some ML libraries
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements first (better caching)
+COPY requirements.txt .
+
+RUN pip install --upgrade pip
+RUN pip install setuptools==69.5.1
+RUN pip install --no-cache-dir -r requirements.txt
+
+RUN python -c "from comet import download_model; download_model('Unbabel/wmt20-comet-da')"
+# Copy project files
+COPY . .
+
+# Expose API port
+EXPOSE 8000
+
+# Run FastAPI
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

main.py

@@ -0,0 +1,61 @@
+"""
+Real-Time Multilingual Metric Evaluator — Backend
+FastAPI server that computes ROUGE, BERTScore, and COMET scores live.
+
+Install dependencies:
+    pip install -r requirements.txt
+
+Run:
+    uvicorn main:app --reload --port 8000
+"""
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from fastapi.exceptions import RequestValidationError
+from routes import router
+
+# Create FastAPI app
+app = FastAPI(title="Multilingual Metric Evaluator API")
+
+# Allow requests from the React frontend and local dev
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=[
+        "https://metric-evaluator.vercel.app",
+        "http://localhost:5173",
+        "http://localhost:3000",
+        "http://127.0.0.1:5173",
+        "http://127.0.0.1:3000",
+    ],
+    allow_credentials=True,
+    allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"],
+    allow_headers=["*"],
+    max_age=3600,
+)
+
+# Custom exception handler to ensure CORS headers are always sent
+@app.exception_handler(Exception)
+async def global_exception_handler(request, exc):
+    return JSONResponse(
+        status_code=500,
+        content={"detail": str(exc)},
+        headers={
+            "Access-Control-Allow-Origin": request.headers.get("origin", "*"),
+            "Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, OPTIONS",
+            "Access-Control-Allow-Headers": "*",
+        }
+    )
+
+# health check endpoint
+@app.get("/health")
+def health():
+    return {"status": "ok"}
+
+# Include routes
+app.include_router(router)
+
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000, reload=True)

metrics/__init__.py

@@ -0,0 +1,16 @@
+"""
+Metrics package for NLP evaluation
+"""
+from .rouge import compute_rouge
+from .bertscore import compute_bertscore
+from .comet import compute_comet
+from .chrf import compute_chrf
+from .bleu import compute_bleu
+
+__all__ = [
+    "compute_rouge",
+    "compute_bertscore",
+    "compute_comet",
+    "compute_chrf",
+    "compute_bleu",
+]

metrics/bertscore.py

@@ -0,0 +1,48 @@
+"""
+BERTScore metric computation
+"""
+import time
+from models import MetricResult
+
+
+def compute_bertscore(hypothesis: str, reference: str, language: str) -> MetricResult:
+    """Compute BERTScore"""
+    t0 = time.time()
+    try:
+        import bert_score
+        # Map common language codes to BERTScore model names
+        lang_model_map = {
+            "ar": "asafaya/bert-base-arabic",
+            "he": "avichr/heBERT",
+            "zh": "bert-base-chinese",
+            "ja": "cl-tohoku/bert-base-japanese-v3",
+            "tr": "dbmdz/bert-base-turkish-cased",
+            "es": "dccuchile/bert-base-spanish-wwm-cased",
+        }
+        model = lang_model_map.get(language, "bert-base-multilingual-cased")
+
+        P, R, F = bert_score.score(
+            [hypothesis], [reference],
+            model_type=model,
+            lang=language,
+            verbose=False,
+        )
+        f = float(F[0])
+        return MetricResult(
+            name="BERTScore",
+            score=round(f, 4),
+            subscores={
+                "precision": round(float(P[0]), 4),
+                "recall":    round(float(R[0]), 4),
+                "f1":        round(f, 4),
+                "model":     model,
+            },
+            duration_ms=round((time.time() - t0) * 1000, 1),
+        )
+    except Exception as e:
+        return MetricResult(
+            name="BERTScore",
+            score=0.0,
+            error=str(e),
+            duration_ms=round((time.time() - t0) * 1000, 1)
+        )

metrics/bleu.py

@@ -0,0 +1,27 @@
+"""
+BLEU metric computation
+"""
+import time
+from models import MetricResult
+
+
+def compute_bleu(hypothesis: str, reference: str) -> MetricResult:
+    """BLEU score"""
+    t0 = time.time()
+    try:
+        from sacrebleu.metrics import BLEU
+        metric = BLEU(effective_order=True)
+        score  = metric.sentence_score(hypothesis, [reference]).score / 100.0
+        return MetricResult(
+            name="BLEU",
+            score=round(score, 4),
+            subscores={"bleu": round(score, 4)},
+            duration_ms=round((time.time() - t0) * 1000, 1),
+        )
+    except Exception as e:
+        return MetricResult(
+            name="BLEU",
+            score=0.0,
+            error=str(e),
+            duration_ms=round((time.time() - t0) * 1000, 1)
+        )

metrics/chrf.py

@@ -0,0 +1,27 @@
+"""
+CHRF metric computation
+"""
+import time
+from models import MetricResult
+
+
+def compute_chrf(hypothesis: str, reference: str) -> MetricResult:
+    """Character n-gram F-score"""
+    t0 = time.time()
+    try:
+        from sacrebleu.metrics import CHRF
+        metric = CHRF()
+        score  = metric.sentence_score(hypothesis, [reference]).score / 100.0
+        return MetricResult(
+            name="CHRF",
+            score=round(score, 4),
+            subscores={"chrf": round(score, 4)},
+            duration_ms=round((time.time() - t0) * 1000, 1),
+        )
+    except Exception as e:
+        return MetricResult(
+            name="CHRF",
+            score=0.0,
+            error=str(e),
+            duration_ms=round((time.time() - t0) * 1000, 1)
+        )

metrics/comet.py

@@ -0,0 +1,53 @@
+"""
+COMET metric computation
+"""
+
+import time
+from typing import Optional
+from models import MetricResult
+from utils import get_comet
+
+
+def compute_comet(hypothesis: str, reference: str, source: Optional[str]) -> MetricResult:
+    """Compute COMET score"""
+
+    t0 = time.time()
+
+    try:
+        model = get_comet()
+
+        if model is None:
+            raise RuntimeError("COMET model not loaded — check installation.")
+
+        # COMET expects source text
+        src = source if source else reference
+
+        data = [{
+            "src": src,
+            "mt": hypothesis,
+            "ref": reference
+        }]
+
+        output = model.predict(
+            data,
+            batch_size=1,
+            accelerator="cpu",
+            progress_bar=False
+        )
+
+        seg_score = float(output["scores"][0])
+
+        return MetricResult(
+            name="COMET",
+            score=round(seg_score, 4),
+            subscores={"segment_score": round(seg_score, 4)},
+            duration_ms=round((time.time() - t0) * 1000, 1),
+        )
+
+    except Exception as e:
+        return MetricResult(
+            name="COMET",
+            score=0.0,
+            error=str(e),
+            duration_ms=round((time.time() - t0) * 1000, 1),
+        )

metrics/rouge.py

@@ -0,0 +1,35 @@
+"""
+ROUGE metric computation
+"""
+import time
+from models import MetricResult
+from utils import get_rouge
+
+
+def compute_rouge(hypothesis: str, reference: str) -> MetricResult:
+    """Compute ROUGE score"""
+    t0 = time.time()
+    try:
+        scorer  = get_rouge()
+        scores  = scorer.score(reference, hypothesis)
+        r1      = scores["rouge1"].fmeasure
+        r2      = scores["rouge2"].fmeasure
+        rL      = scores["rougeL"].fmeasure
+        overall = (r1 + r2 + rL) / 3
+        return MetricResult(
+            name="ROUGE",
+            score=round(overall, 4),
+            subscores={
+                "rouge1": round(r1, 4),
+                "rouge2": round(r2, 4),
+                "rougeL": round(rL, 4),
+            },
+            duration_ms=round((time.time() - t0) * 1000, 1),
+        )
+    except Exception as e:
+        return MetricResult(
+            name="ROUGE",
+            score=0.0,
+            error=str(e),
+            duration_ms=round((time.time() - t0) * 1000, 1)
+        )

models.py

@@ -0,0 +1,40 @@
+"""
+Pydantic models for request/response validation
+"""
+
+from pydantic import BaseModel, Field
+from typing import Optional, List, Dict
+
+
+class EvalRequest(BaseModel):
+    """Request model for evaluation endpoint"""
+
+    hypothesis: str          # generated summary
+    reference: str           # reference summary
+    source: Optional[str] = None   # original article (used by COMET)
+
+    language: str = "en"
+
+    metrics: List[str] = Field(
+        default_factory=lambda: ["rouge", "bertscore", "comet"]
+    )
+
+
+class MetricResult(BaseModel):
+    """Result for a single metric"""
+
+    name: str
+    score: float
+
+    subscores: Dict = Field(default_factory=dict)
+
+    error: Optional[str] = None
+    duration_ms: float = 0.0
+
+
+class EvalResponse(BaseModel):
+    """Response model for evaluation endpoint"""
+
+    results: List[MetricResult]
+    language: str
+    total_ms: float

requirements.txt

@@ -0,0 +1,17 @@
+setuptools==69.5.1
+
+fastapi==0.111.0
+uvicorn[standard]==0.29.0
+
+rouge-score==0.1.2
+bert-score==0.3.13
+unbabel-comet==2.2.7
+
+torch==2.2.2 --index-url https://download.pytorch.org/whl/cpu
+transformers==4.38.2
+sacrebleu==2.6.0
+
+pydantic>=2.0.0
+sentencepiece
+protobuf
+sympy==1.12

routes.py

@@ -0,0 +1,76 @@
+"""
+API routes for the metric evaluator
+"""
+import time
+from fastapi import APIRouter, HTTPException
+from models import EvalRequest, EvalResponse
+from metrics import compute_rouge, compute_bertscore, compute_comet, compute_chrf, compute_bleu
+
+router = APIRouter()
+
+
+@router.get("/")
+def root():
+    """Root endpoint"""
+    return {"status": "ok", "message": "Multilingual Metric Evaluator API"}
+
+
+@router.get("/health")
+def health():
+    """Health check endpoint"""
+    return {"status": "healthy"}
+
+
+@router.post("/evaluate", response_model=EvalResponse)
+def evaluate(req: EvalRequest):
+    """Main evaluation endpoint"""
+    t_total = time.time()
+    results = []
+
+    metric_set = set(m.lower() for m in req.metrics)
+
+    try:
+        if "rouge" in metric_set:
+            results.append(compute_rouge(req.hypothesis, req.reference))
+
+        if "bertscore" in metric_set:
+            try:
+                results.append(compute_bertscore(req.hypothesis, req.reference, req.language))
+            except Exception as e:
+                raise HTTPException(status_code=500, detail=f"BERTScore error: {str(e)}")
+
+        if "comet" in metric_set:
+            try:
+                results.append(compute_comet(req.hypothesis, req.reference, req.source))
+            except Exception as e:
+                raise HTTPException(status_code=500, detail=f"COMET error: {str(e)}")
+
+        if "chrf" in metric_set:
+            results.append(compute_chrf(req.hypothesis, req.reference))
+
+        if "bleu" in metric_set:
+            results.append(compute_bleu(req.hypothesis, req.reference))
+
+        return EvalResponse(
+            results=results,
+            language=req.language,
+            total_ms=round((time.time() - t_total) * 1000, 1),
+        )
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Evaluation error: {str(e)}")
+
+
+@router.get("/metrics")
+def list_metrics():
+    """Returns all available metrics and their descriptions"""
+    return {
+        "metrics": [
+            {"id": "rouge",      "name": "ROUGE",      "type": "N-Gram",  "description": "Recall-Oriented Understudy for Gisting Evaluation. Measures n-gram overlap between hypothesis and reference."},
+            {"id": "bertscore",  "name": "BERTScore",  "type": "Neural",  "description": "Computes similarity using BERT token embeddings. Uses language-specific models when available."},
+            {"id": "comet",      "name": "COMET",       "type": "Neural",  "description": "Trained neural metric with human quality score regression. Best overall correlation across language families."},
+            {"id": "chrf",       "name": "CHRF",        "type": "N-Gram",  "description": "Character n-gram F-score. More robust than word-level metrics for morphologically rich languages."},
+            {"id": "bleu",       "name": "BLEU",        "type": "N-Gram",  "description": "Bilingual Evaluation Understudy. Precision-based n-gram overlap metric."},
+        ]
+    }

utils.py

@@ -0,0 +1,46 @@
+"""
+Utilities for lazy loading models and helpers
+"""
+
+# Lazy-loaded model caches
+_rouge_scorer = None
+_comet_model = None
+
+
+def get_rouge():
+    """Get or create ROUGE scorer (lazy loaded)"""
+    global _rouge_scorer
+
+    if _rouge_scorer is None:
+        from rouge_score import rouge_scorer
+
+        _rouge_scorer = rouge_scorer.RougeScorer(
+            ["rouge1", "rouge2", "rougeL"],
+            use_stemmer=False
+        )
+
+    return _rouge_scorer
+
+
+def get_comet():
+    """Get or create COMET model (lazy loaded)"""
+
+    global _comet_model
+
+    if _comet_model is None:
+        try:
+            from comet import download_model, load_from_checkpoint
+
+            print("Loading COMET model... (first run may download)")
+
+            model_path = download_model("Unbabel/wmt20-comet-da")
+
+            _comet_model = load_from_checkpoint(model_path)
+
+            print("COMET model loaded successfully")
+
+        except Exception as e:
+            print(f"COMET not available: {e}")
+            _comet_model = None
+
+    return _comet_model