evaluation/metrics.py

import json
import os
from datetime import datetime
from rag.llm import generate

LOG_PATH = "logs/metrics.jsonl"


def ensure_log_dir():
    """Cria o diretório de logs se não existir"""
    os.makedirs(os.path.dirname(LOG_PATH), exist_ok=True)


def evaluate_and_log(question, context, answer):
    ensure_log_dir()
    
    prompt = (
        "Evaluate the answer based on faithfulness, clarity, usefulness and completeness. "
        "Provide a short justification.\n\n"
        f"Context:\n{context}\n\n"
        f"Answer:\n{answer}\n\n"
        "Evaluation:"
    )

    try:
        evaluation = generate(prompt, max_tokens=200)
    except Exception as e:
        evaluation = f"Evaluation failed: {str(e)}"

    record = {
        "timestamp": datetime.utcnow().isoformat(),
        "question": question,
        "answer": answer,
        "evaluation": evaluation
    }

    try:
        with open(LOG_PATH, "a", encoding="utf-8") as f:
            f.write(json.dumps(record, ensure_ascii=False) + "\n")
    except Exception as e:
        print(f"Warning: Could not write to log file: {e}")

    return evaluation