Spaces:
Running
Running
File size: 5,601 Bytes
0772b5a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 | """
Pipeline Evaluation Runner.
Runs evaluation suites over benchmark datasets and computes structured performance metrics.
"""
from __future__ import annotations
import asyncio
import logging
import time
from typing import Any, Dict, List, Optional
from eval.benchmark import BenchmarkDataset, BenchmarkSample
from eval.metrics import (
OCRMetrics,
ParserMetrics,
PipelineEvalSummary,
SolverMetrics,
compute_cer,
compute_wer,
)
from solver.dsl_parser import DSLParser
from solver.engine import GeometryEngine
from solver.validator import GeometryStatus, GeometryValidator
logger = logging.getLogger(__name__)
class EvalRunner:
"""Runs pipeline evaluation over benchmark datasets."""
def __init__(
self,
dsl_parser: Optional[DSLParser] = None,
geometry_engine: Optional[GeometryEngine] = None,
geometry_validator: Optional[GeometryValidator] = None,
):
self.dsl_parser = dsl_parser or DSLParser()
self.geometry_engine = geometry_engine or GeometryEngine()
self.geometry_validator = geometry_validator or GeometryValidator()
def evaluate_dsl_deterministic(self, dataset: BenchmarkDataset) -> ParserMetrics:
"""
Evaluates DSL parsing, solving, and geometric invariant validation
deterministically without LLM calls.
"""
total = len(dataset)
if total == 0:
return ParserMetrics()
valid_dsl_count = 0
solvable_count = 0
validated_count = 0
degraded_count = 0
for sample in dataset:
dsl = sample.expected_dsl or ""
if not dsl:
continue
try:
points, constraints, is_3d = self.dsl_parser.parse(dsl)
valid_dsl_count += 1
engine_res = self.geometry_engine.solve(points, constraints, is_3d)
if engine_res and engine_res.get("coordinates"):
solvable_count += 1
val_res = self.geometry_validator.validate(engine_res, constraints, is_3d)
if val_res.is_valid:
validated_count += 1
else:
degraded_count += 1
except Exception as e:
logger.debug(f"[EvalRunner] Sample {sample.id} evaluation error: {e}")
return ParserMetrics(
total_samples=total,
json_valid_rate=1.0,
dsl_valid_rate=valid_dsl_count / total,
solvability_rate=solvable_count / total,
validation_pass_rate=validated_count / total,
degradation_rate=degraded_count / total,
)
async def evaluate_full_pipeline(
self,
dataset: BenchmarkDataset,
orchestrator: Any = None,
) -> PipelineEvalSummary:
"""
Executes end-to-end evaluation using Orchestrator across benchmark samples.
"""
from agents.orchestrator import Orchestrator
orch = orchestrator or Orchestrator()
total = len(dataset)
if total == 0:
return PipelineEvalSummary()
e2e_successes = 0
total_latency_ms = 0.0
parser_metrics = ParserMetrics(total_samples=total)
solver_metrics = SolverMetrics(total_samples=total)
ocr_metrics = OCRMetrics(total_samples=total)
valid_dsl_count = 0
solvable_count = 0
validated_count = 0
degraded_count = 0
correct_answer_count = 0
for sample in dataset:
t0 = time.time()
try:
result = await orch.run(
text=sample.problem_text,
image_url=sample.image_url,
generate_video=False,
)
latency = (time.time() - t0) * 1000
total_latency_ms += latency
if result.get("status") == "success":
e2e_successes += 1
# Check geometry status
geo_status = result.get("geometry_status")
if result.get("geometry_dsl"):
valid_dsl_count += 1
if result.get("coordinates"):
solvable_count += 1
if geo_status == GeometryStatus.VALID.value:
validated_count += 1
elif geo_status == GeometryStatus.DEGRADED.value:
degraded_count += 1
# Check answer if expected_answer is present
if sample.expected_answer:
actual_ans = str((result.get("solution") or {}).get("answer", ""))
if sample.expected_answer.strip() in actual_ans or actual_ans.strip() in sample.expected_answer:
correct_answer_count += 1
except Exception as e:
logger.error(f"[EvalRunner] Full pipeline run failed on sample {sample.id}: {e}")
parser_metrics.dsl_valid_rate = valid_dsl_count / total
parser_metrics.solvability_rate = solvable_count / total
parser_metrics.validation_pass_rate = validated_count / total
parser_metrics.degradation_rate = degraded_count / total
solver_metrics.answer_exact_match_rate = correct_answer_count / max(total, 1)
return PipelineEvalSummary(
ocr=ocr_metrics,
parser=parser_metrics,
solver=solver_metrics,
e2e_success_rate=e2e_successes / total,
avg_latency_ms=total_latency_ms / max(total, 1),
total_samples=total,
)
|