dev-strender's picture
feat: switch demo to FT + Opus article review (pipeline_exp_261021_ft_hybrid)
3dfc93b verified
Raw History Blame Contribute Delete
4.48 kB
"""νŒμ • μ΄ˆμ•ˆΒ·μƒμ‚°μž 집계 쑰립 (κ³„νš 2026-09-26 L3b Β§2.4Β·D2Β·D4).
- κ²°κ³Ό ν–‰λ§ˆλ‹€ νŒμ • 3개, μˆœμ„œ = `exact` β†’ `lcs_f1` β†’ `spans`. μ‹€νŒ¨ 행은 μ„Έ μ§€ν‘œ λͺ¨λ‘ `status="error"`·같은 `reason` 이닀.
- `producer_aggregates.json`: `exact`Β·`lcs_f1` 의 `mean` 은 νŒŒμƒ μ°Έμ‘° κ΅¬ν˜„ `derive("mean")`, `spans` 의 `micro_prf@1:{f1,precision,
recall}` 은 **λ ˆκ±°μ‹œ κ΅¬ν˜„** `LCSDiffEvaluator().aggregate()` 에 ok ν–‰ 카운트λ₯Ό 넣은 값이닀(D4) β€” hub 적재의
`producer_aggregate_mismatch` λŒ€μ‘°κ°€ "SQL νŒŒμƒ ↔ λ ˆκ±°μ‹œ μ½”λ“œ" λŒ€μ‘°κ°€ λœλ‹€. ok 행이 0 이면 `spans` ν•­λͺ©μ„ μ‹£μ§€ μ•ŠλŠ”λ‹€
(λ ˆκ±°μ‹œμ˜ 0.0 은 "μΈ‘μ • μ—†μŒ" κ³Ό κ΅¬λ³„λ˜μ§€ μ•ŠλŠ”λ‹€).
"""
from __future__ import annotations
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import Any
from poc_bundle.scoring.derivations import derive
from poc_bundle.scoring.writer import JudgmentDraft
from solar_eval.evaluators.lcs_diff import LCSDiffEvaluator
from solar_eval.rescore.evaluator_doc import ACTOR
from solar_eval.rescore.proofread_diff_v2 import LABEL_CORRECT, RowError, RowScore
EXACT, LCS_F1, SPANS = "exact", "lcs_f1", "spans"
#: νŒμ • 파일 μ•ˆ μ§€ν‘œ μˆœμ„œ.
CRITERIA: tuple[str, ...] = (EXACT, LCS_F1, SPANS)
PRODUCER_AGGREGATES_SCHEMA = "producer-aggregates/v1"
#: λ ˆκ±°μ‹œ `aggregate()["scores"]` ν‚€ β†’ `micro_prf@1` 좜λ ₯ 이름.
SPAN_OUTPUTS: tuple[str, ...] = ("f1", "precision", "recall")
@dataclass(frozen=True)
class ScoredRow:
"""μ±„μ ν•œ κ²°κ³Ό ν–‰ ν•˜λ‚˜ β€” `sample_idx`, run λ²ˆλ“€ `items.jsonl` 의 ν‚€ 쌍, 채점 κ²°κ³Ό."""
sample_idx: int
input_key: str
golden_key: str
outcome: RowScore | RowError
def judgment_drafts(run_id: str, rows: Sequence[ScoredRow], created_at: str) -> list[JudgmentDraft]:
"""ν–‰λ§ˆλ‹€ `exact`Β·`lcs_f1`Β·`spans` μ΄ˆμ•ˆ 3개(ν–‰ μˆœμ„œ κ·ΈλŒ€λ‘œ)."""
drafts: list[JudgmentDraft] = []
for row in rows:
common = {
"level": "item",
"run_id": run_id,
"sample_idx": row.sample_idx,
"input_key": row.input_key,
"golden_key": row.golden_key,
"source_kind": "code",
"actor": ACTOR,
"created_at": created_at,
}
for criterion in CRITERIA:
fields = {**common, "criterion_id": criterion, **_result_fields(criterion, row.outcome)}
drafts.append(JudgmentDraft(fields))
return drafts
def _result_fields(criterion: str, outcome: RowScore | RowError) -> dict[str, Any]:
if isinstance(outcome, RowError):
return {"status": "error", "reason": outcome.reason}
if criterion == EXACT:
return {
"score": outcome.score_exact,
"label": outcome.label,
"passed": outcome.label == LABEL_CORRECT,
"status": "ok",
}
if criterion == LCS_F1:
return {"score": outcome.lcs_f1, "status": "ok"}
return {"value": dict(outcome.spans), "status": "ok"}
def producer_aggregates(
rows: Sequence[ScoredRow], drafts: Sequence[JudgmentDraft]
) -> dict[str, Any]:
"""`producer-aggregates/v1` λ¬Έμ„œ β€” 값은 (metric_key, aggregation) μ‚¬μ „μˆœ."""
values = [_mean_value(EXACT, drafts), _mean_value(LCS_F1, drafts)]
ok = [row.outcome for row in rows if isinstance(row.outcome, RowScore)]
if ok:
scores = legacy_span_scores(ok)
for output in SPAN_OUTPUTS:
values.append(_value(SPANS, f"micro_prf@1:{output}", scores[output]))
values.sort(key=lambda v: (v["metric_key"], v["aggregation"]))
return {"schema": PRODUCER_AGGREGATES_SCHEMA, "values": values}
def legacy_span_scores(ok: Sequence[RowScore]) -> Mapping[str, Any]:
"""λ ˆκ±°μ‹œ `LCSDiffEvaluator.aggregate` κ°€ ok ν–‰ 카운트둜 λ‚΄λŠ” run μΈ΅ `scores`(νΌμ„ΌνŠΈ)."""
aggregated = LCSDiffEvaluator().aggregate([{"details": dict(row.spans)} for row in ok])
return aggregated["scores"]
def _mean_value(criterion: str, drafts: Sequence[JudgmentDraft]) -> dict[str, Any]:
rows = [d.fields for d in drafts if d.fields.get("criterion_id") == criterion]
return _value(criterion, "mean", derive("mean", rows)["value"])
def _value(metric_key: str, aggregation: str, value: Any) -> dict[str, Any]:
return {
"metric_key": metric_key,
"aggregation": aggregation,
"variant": "raw",
"error_policy": "exclude",
"value": value,
}