import sys import types # --- PATCH START: Fix ragas missing langchain_community.chat_models.vertexai import error --- try: import langchain_community.chat_models.vertexai except ImportError: vertexai_mock = types.ModuleType("langchain_community.chat_models.vertexai") vertexai_mock.ChatVertexAI = type("ChatVertexAI", (object,), {}) sys.modules["langchain_community.chat_models.vertexai"] = vertexai_mock # --- PATCH END --- from dotenv import load_dotenv load_dotenv() import os import asyncio import time import pandas as pd from openai import AsyncOpenAI from src.core.logger import logger from ragas.llms import llm_factory from ragas.embeddings import HuggingFaceEmbeddings from ragas.metrics.collections import ( Faithfulness, AnswerRelevancy, ContextPrecision, ContextRecall, AnswerCorrectness, ) GROQ_BASE_URL = "https://api.groq.com/openai/v1" JUDGE_MODEL = "qwen/qwen3.8-27b" COOLDOWN = 5 # Simple rate-limit pause # def _build_judge(): # logger.info( # "Configuring evaluation judge", # provider="nvidia", # model="minimaxai/minimax-m3.1-flash", # api_key_configured=bool(os.getenv("NVIDIA_API_KEY")), # ) # async_client = AsyncOpenAI( # api_key=os.getenv("NVIDIA_API_KEY"), # base_url="https://integrate.api.nvidia.com/v1" # ) # # Added max_tokens to prevent IncompleteOutputException length limits # judge_llm = llm_factory( # "minimaxai/minimax-m3.1-flash", # provider="openai", # client=async_client, # max_tokens=8192 # ) # embeddings = HuggingFaceEmbeddings(model="sentence-transformers/all-MiniLM-L6-v2", use_api=False) # return judge_llm, embeddings def _build_judge(): logger.info( "Configuring evaluation judge", provider="ollama", model="llama3", # Ya jo model aapne pull kiya ho (e.g., "qwen2.5", "mistral") ) # Ollama ka local OpenAI-compatible endpoint async_client = AsyncOpenAI( api_key=os.getenv("OLLAMA_API_KEY"), base_url="http://localhost:11434/v1" ) # Ollama par pull kiya hua model name yahan pass karein judge_llm = llm_factory( "gemma4:31b-cloud", # Yahan apna local Ollama model name dein (jaise "llama3" ya "qwen2.5") provider="openai", client=async_client, max_tokens=8192 ) embeddings = HuggingFaceEmbeddings(model="sentence-transformers/all-MiniLM-L6-v2", use_api=False) return judge_llm, embeddings def _score_df(metric_key: str, samples: list, scores) -> pd.DataFrame: rows = [] for s, r in zip(samples, scores): # Handle cases where score might be None or failed val = 0.0 if r is not None: try: val = round(float(getattr(r, "value", r)), 3) except Exception: val = 0.0 rows.append({"question": s["question"], metric_key: val}) return pd.DataFrame(rows) async def run_all_metrics(golden_dataset: dict, status_cb=None) -> dict: judge_llm, embeddings = _build_judge() samples = golden_dataset.get("rag_samples", []) if not samples: logger.error("Evaluation dataset is empty or missing rag_samples") raise ValueError("Dataset is empty or missing 'rag_samples'.") logger.info("Starting Ragas evaluation", sample_count=len(samples)) results = {} async def run_metric(name, metric_instance, inputs_builder): started_at = time.perf_counter() logger.info("Starting evaluation metric", metric=name, sample_count=len(samples)) if status_cb: status_cb(f"Running {name}...") scores = [] # Process sample by sample so an error in one doesn't crash the whole run for idx, s in enumerate(samples): single_input = [inputs_builder(s)] try: # abatch_score for a single item list res = await metric_instance.abatch_score(single_input) if res and len(res) > 0: scores.append(res[0]) else: scores.append(0.0) except Exception as exc: logger.error( "Sample evaluation failed, skipping sample", metric=name, sample_index=idx, error=str(exc), ) # Append a fallback/default score object or value so length matches class FallbackScore: value = 0.0 scores.append(FallbackScore()) key = name.lower().replace(" ", "_") results[key] = _score_df(key, samples, scores) logger.info( "Evaluation metric completed", metric=name, sample_count=len(samples), elapsed_seconds=round(time.perf_counter() - started_at, 2), ) await asyncio.sleep(COOLDOWN) logger.info("Evaluation Faithfulness") # 1. Faithfulness await run_metric( "Faithfulness", Faithfulness(llm=judge_llm), lambda s: {"user_input": s["question"], "response": s["actual_response"], "retrieved_contexts": s["actual_contexts"]} ) logger.info("Evaluation Answer Relevancy") # 2. Answer Relevancy await run_metric( "Answer Relevancy", AnswerRelevancy(llm=judge_llm, embeddings=embeddings), lambda s: {"user_input": s["question"], "response": s["actual_response"]} ) logger.info("Evaluation Context Precision") # 3. Context Precision await run_metric( "Context Precision", ContextPrecision(llm=judge_llm), lambda s: {"user_input": s["question"], "reference": s["reference"], "retrieved_contexts": s["actual_contexts"]} ) logger.info("Evaluation Context Recall") # 4. Context Recall await run_metric( "Context Recall", ContextRecall(llm=judge_llm), lambda s: {"user_input": s["question"], "reference": s["reference"], "retrieved_contexts": s["actual_contexts"]} ) logger.info("Evaluation Answer Correctness") # 5. Answer Correctness await run_metric( "Answer Correctness", AnswerCorrectness(llm=judge_llm, embeddings=embeddings), lambda s: {"user_input": s["question"], "response": s["actual_response"], "reference": s["reference"]} ) if status_cb: status_cb("Evaluation complete!") logger.info("Ragas evaluation completed", metric_count=len(results)) return results if __name__ == "__main__": import json import sys from evals.harness import run_pipeline, save_dict, load_dataset if len(sys.argv) != 2: print("Usage: python metrics.py ") sys.exit(1) dataset_path = sys.argv[1] logger.info("Loading evaluation dataset", path=dataset_path) if not os.path.exists(dataset_path): logger.info("Evaluation dataset not found; collecting it from the chat API", path=dataset_path) print(f"File not found: {dataset_path}") print("Creating dataset file...") dataset = run_pipeline(load_dataset("evals/goldens/golden_dataset.json")) save_dict(dataset, dataset_path) with open(dataset_path, "r") as f: golden_dataset = json.load(f) logger.info( "Evaluation dataset loaded", path=dataset_path, sample_count=len(golden_dataset.get("rag_samples", [])), ) # Run evaluations safely try: results = asyncio.run(run_all_metrics(golden_dataset)) except Exception as e: logger.error("Critical error during evaluation run", error=str(e)) results = {} os.makedirs("evals/results", exist_ok=True) i = 0 while os.path.exists(f"evals/results/metrics_results_{i}.json"): i += 1 output_path = f"evals/results/metrics_results_{i}.json" # Save whatever results were collected even if some samples failed if results: with open(output_path, "w") as f: json.dump({k: v.to_dict(orient="records") for k, v in results.items()}, f, indent=4) logger.info("Evaluation results saved", path=output_path, metric_count=len(results)) print(f"Evaluation results saved to {output_path}") else: print("No evaluation results were generated to save.")