Spaces:
Running
Running
File size: 3,120 Bytes
263c06c 599225c 263c06c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 | from pilotcore.runtime.pipeline import run_pipeline
from .benchmark_models import BenchmarkResult
def run_benchmark(
questions,
configs,
user_id,
source=None,
):
results = []
for config in configs:
config.emit_trace = False
print(f"\n===== RUNNING CONFIG: {config.experiment_name} =====\n")
evaluations = []
latencies = []
for question in questions:
trace = run_pipeline(
query=question,
user_id=user_id,
source=source,
experiment_config=config,
)
chunk_count = 0
if trace.retrieval_result and trace.retrieval_result.retrieved_chunks:
chunk_count = len(trace.retrieval_result.retrieved_chunks)
print(
f"{config.experiment_name} | " f"{question} | " f"chunks={chunk_count}"
)
print("\n=== EVALUATION ===")
print(trace.evaluation)
print("==================\n")
evaluations.append(trace.evaluation)
latencies.append(trace.latency_ms or 0.0)
total_questions = max(
len(evaluations),
1,
)
result = BenchmarkResult(
config_name=config.experiment_name,
faithfulness=sum(
e.get(
"faithfulness_score",
0.0,
)
for e in evaluations
)
/ total_questions,
semantic_grounding=sum(
e.get(
"semantic_grounding",
0.0,
)
for e in evaluations
)
/ total_questions,
semantic_query_coverage=sum(
e.get(
"semantic_query_coverage",
0.0,
)
for e in evaluations
)
/ total_questions,
retrieval_quality_score=sum(
e.get(
"retrieval_quality_score",
0.0,
)
for e in evaluations
)
/ total_questions,
latency=sum(latencies) / max(len(latencies), 1),
grounded_rate=sum(
1
for e in evaluations
if e.get(
"grounded",
False,
)
)
/ total_questions,
abstain_rate=sum(
1
for e in evaluations
if e.get(
"abstained",
False,
)
)
/ total_questions,
)
print(f"\nRESULT: {result.config_name}")
print(f"Faithfulness: {result.faithfulness:.4f}")
print(f"Grounding: {result.semantic_grounding:.4f}")
print(f"Retrieval Quality: {result.retrieval_quality_score:.4f}")
print(f"Latency: {result.latency:.2f} ms")
results.append(result)
return results
|