Spaces:
Running
Running
File size: 4,484 Bytes
3dfc93b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 | """νμ μ΄μΒ·μμ°μ μ§κ³ 쑰립 (κ³ν 2026-09-26 L3b Β§2.4Β·D2Β·D4).
- κ²°κ³Ό νλ§λ€ νμ 3κ°, μμ = `exact` β `lcs_f1` β `spans`. μ€ν¨ νμ μΈ μ§ν λͺ¨λ `status="error"`Β·κ°μ `reason` μ΄λ€.
- `producer_aggregates.json`: `exact`Β·`lcs_f1` μ `mean` μ νμ μ°Έμ‘° ꡬν `derive("mean")`, `spans` μ `micro_prf@1:{f1,precision,
recall}` μ **λ κ±°μ ꡬν** `LCSDiffEvaluator().aggregate()` μ ok ν μΉ΄μ΄νΈλ₯Ό λ£μ κ°μ΄λ€(D4) β hub μ μ¬μ
`producer_aggregate_mismatch` λμ‘°κ° "SQL νμ β λ κ±°μ μ½λ" λμ‘°κ° λλ€. ok νμ΄ 0 μ΄λ©΄ `spans` νλͺ©μ μ£μ§ μλλ€
(λ κ±°μμ 0.0 μ "μΈ‘μ μμ" κ³Ό ꡬλ³λμ§ μλλ€).
"""
from __future__ import annotations
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import Any
from poc_bundle.scoring.derivations import derive
from poc_bundle.scoring.writer import JudgmentDraft
from solar_eval.evaluators.lcs_diff import LCSDiffEvaluator
from solar_eval.rescore.evaluator_doc import ACTOR
from solar_eval.rescore.proofread_diff_v2 import LABEL_CORRECT, RowError, RowScore
EXACT, LCS_F1, SPANS = "exact", "lcs_f1", "spans"
#: νμ νμΌ μ μ§ν μμ.
CRITERIA: tuple[str, ...] = (EXACT, LCS_F1, SPANS)
PRODUCER_AGGREGATES_SCHEMA = "producer-aggregates/v1"
#: λ κ±°μ `aggregate()["scores"]` ν€ β `micro_prf@1` μΆλ ₯ μ΄λ¦.
SPAN_OUTPUTS: tuple[str, ...] = ("f1", "precision", "recall")
@dataclass(frozen=True)
class ScoredRow:
"""μ±μ ν κ²°κ³Ό ν νλ β `sample_idx`, run λ²λ€ `items.jsonl` μ ν€ μ, μ±μ κ²°κ³Ό."""
sample_idx: int
input_key: str
golden_key: str
outcome: RowScore | RowError
def judgment_drafts(run_id: str, rows: Sequence[ScoredRow], created_at: str) -> list[JudgmentDraft]:
"""νλ§λ€ `exact`Β·`lcs_f1`Β·`spans` μ΄μ 3κ°(ν μμ κ·Έλλ‘)."""
drafts: list[JudgmentDraft] = []
for row in rows:
common = {
"level": "item",
"run_id": run_id,
"sample_idx": row.sample_idx,
"input_key": row.input_key,
"golden_key": row.golden_key,
"source_kind": "code",
"actor": ACTOR,
"created_at": created_at,
}
for criterion in CRITERIA:
fields = {**common, "criterion_id": criterion, **_result_fields(criterion, row.outcome)}
drafts.append(JudgmentDraft(fields))
return drafts
def _result_fields(criterion: str, outcome: RowScore | RowError) -> dict[str, Any]:
if isinstance(outcome, RowError):
return {"status": "error", "reason": outcome.reason}
if criterion == EXACT:
return {
"score": outcome.score_exact,
"label": outcome.label,
"passed": outcome.label == LABEL_CORRECT,
"status": "ok",
}
if criterion == LCS_F1:
return {"score": outcome.lcs_f1, "status": "ok"}
return {"value": dict(outcome.spans), "status": "ok"}
def producer_aggregates(
rows: Sequence[ScoredRow], drafts: Sequence[JudgmentDraft]
) -> dict[str, Any]:
"""`producer-aggregates/v1` λ¬Έμ β κ°μ (metric_key, aggregation) μ¬μ μ."""
values = [_mean_value(EXACT, drafts), _mean_value(LCS_F1, drafts)]
ok = [row.outcome for row in rows if isinstance(row.outcome, RowScore)]
if ok:
scores = legacy_span_scores(ok)
for output in SPAN_OUTPUTS:
values.append(_value(SPANS, f"micro_prf@1:{output}", scores[output]))
values.sort(key=lambda v: (v["metric_key"], v["aggregation"]))
return {"schema": PRODUCER_AGGREGATES_SCHEMA, "values": values}
def legacy_span_scores(ok: Sequence[RowScore]) -> Mapping[str, Any]:
"""λ κ±°μ `LCSDiffEvaluator.aggregate` κ° ok ν μΉ΄μ΄νΈλ‘ λ΄λ run μΈ΅ `scores`(νΌμΌνΈ)."""
aggregated = LCSDiffEvaluator().aggregate([{"details": dict(row.spans)} for row in ok])
return aggregated["scores"]
def _mean_value(criterion: str, drafts: Sequence[JudgmentDraft]) -> dict[str, Any]:
rows = [d.fields for d in drafts if d.fields.get("criterion_id") == criterion]
return _value(criterion, "mean", derive("mean", rows)["value"])
def _value(metric_key: str, aggregation: str, value: Any) -> dict[str, Any]:
return {
"metric_key": metric_key,
"aggregation": aggregation,
"variant": "raw",
"error_policy": "exclude",
"value": value,
}
|