File size: 3,430 Bytes
644f326
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
import copy
import json
import os
import time
import uuid

import requests
from dotenv import load_dotenv

load_dotenv()

from src.core.logger import logger

URL = os.getenv("EVAL_CHAT_URL", "http://localhost:7860/api/v1/chat")
REQUEST_TIMEOUT = float(os.getenv("EVAL_REQUEST_TIMEOUT", "120"))
DELAY = float(os.getenv("EVAL_REQUEST_DELAY", "10"))

def _context_text(context) -> str:
    if isinstance(context, str):
        return context
    if isinstance(context, dict):
        return str(context.get("page_content") or context.get("content") or "")
    return str(getattr(context, "page_content", context))


def run_pipeline(golden_dataset: dict) -> dict:
    """Populate RAG samples by querying the local chat API."""
    golden_dataset = copy.deepcopy(golden_dataset)
    rag_samples = golden_dataset.get("rag_samples") or []
    logger.info("Starting evaluation data collection", sample_count=len(rag_samples), url=URL)

    for index, sample in enumerate(rag_samples, start=1):
        question = sample.get("question")
        if not isinstance(question, str) or not question.strip():
            raise ValueError(f"RAG sample {sample.get('id', index)} has no valid question.")

        thread_id = str(uuid.uuid4())
        payload = {
            "message": question,
            "thread_id": thread_id,
            "stream": False,
        }

        logger.info(
            "Collecting evaluation sample",
            sample_id=sample.get("id", index),
            progress=f"{index}/{len(rag_samples)}",
            thread_id=thread_id,
        )
        try:
            response = requests.post(URL, json=payload, timeout=REQUEST_TIMEOUT)
            response.raise_for_status()
            result = response.json()
        except (requests.RequestException, ValueError) as exc:
            logger.error(
                "Evaluation sample request failed",
                sample_id=sample.get("id", index),
                error=str(exc),
            )
            raise RuntimeError(
                f"Could not collect evaluation sample {sample.get('id', index)}: {exc}"
            ) from exc

        if not result.get("success"):
            raise RuntimeError(f"Chat API did not complete sample {sample.get('id', index)} successfully.")

        data = result.get("data") or {}
        contexts = data.get("retreived_content") or data.get("retrieved_content") or []
        sample["actual_response"] = result.get("response") or ""
        sample["actual_contexts"] = [text for item in contexts if (text := _context_text(item))]
        logger.info(
            "Evaluation sample collected",
            sample_id=sample.get("id", index),
            context_count=len(sample["actual_contexts"]),
        )

        if index < len(rag_samples) and DELAY > 0:
            time.sleep(DELAY)

    logger.info("Evaluation data collection complete", sample_count=len(rag_samples))
    return golden_dataset


def save_dict(golden_dataset: dict, file_path: str):
    """Save the populated golden dataset as JSON."""
    os.makedirs(os.path.dirname(file_path) or ".", exist_ok=True)
    with open(file_path, "w", encoding="utf-8") as f:
        json.dump(golden_dataset, f, indent=4)
    logger.info("Evaluation dataset saved", path=file_path)


def load_dataset(file_path: str) -> dict:
    """Load a golden dataset from JSON."""
    with open(file_path, "r", encoding="utf-8") as f:
        return json.load(f)