File size: 4,484 Bytes
3dfc93b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
"""νŒμ • μ΄ˆμ•ˆΒ·μƒμ‚°μž 집계 쑰립 (κ³„νš 2026-09-26 L3b Β§2.4Β·D2Β·D4).

- κ²°κ³Ό ν–‰λ§ˆλ‹€ νŒμ • 3개, μˆœμ„œ = `exact` β†’ `lcs_f1` β†’ `spans`. μ‹€νŒ¨ 행은 μ„Έ μ§€ν‘œ λͺ¨λ‘ `status="error"`·같은 `reason` 이닀.
- `producer_aggregates.json`: `exact`Β·`lcs_f1` 의 `mean` 은 νŒŒμƒ μ°Έμ‘° κ΅¬ν˜„ `derive("mean")`, `spans` 의 `micro_prf@1:{f1,precision,
  recall}` 은 **λ ˆκ±°μ‹œ κ΅¬ν˜„** `LCSDiffEvaluator().aggregate()` 에 ok ν–‰ 카운트λ₯Ό 넣은 값이닀(D4) β€” hub 적재의
  `producer_aggregate_mismatch` λŒ€μ‘°κ°€ "SQL νŒŒμƒ ↔ λ ˆκ±°μ‹œ μ½”λ“œ" λŒ€μ‘°κ°€ λœλ‹€. ok 행이 0 이면 `spans` ν•­λͺ©μ„ μ‹£μ§€ μ•ŠλŠ”λ‹€
  (λ ˆκ±°μ‹œμ˜ 0.0 은 "μΈ‘μ • μ—†μŒ" κ³Ό κ΅¬λ³„λ˜μ§€ μ•ŠλŠ”λ‹€).
"""

from __future__ import annotations

from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import Any

from poc_bundle.scoring.derivations import derive
from poc_bundle.scoring.writer import JudgmentDraft

from solar_eval.evaluators.lcs_diff import LCSDiffEvaluator
from solar_eval.rescore.evaluator_doc import ACTOR
from solar_eval.rescore.proofread_diff_v2 import LABEL_CORRECT, RowError, RowScore

EXACT, LCS_F1, SPANS = "exact", "lcs_f1", "spans"
#: νŒμ • 파일 μ•ˆ μ§€ν‘œ μˆœμ„œ.
CRITERIA: tuple[str, ...] = (EXACT, LCS_F1, SPANS)
PRODUCER_AGGREGATES_SCHEMA = "producer-aggregates/v1"
#: λ ˆκ±°μ‹œ `aggregate()["scores"]` ν‚€ β†’ `micro_prf@1` 좜λ ₯ 이름.
SPAN_OUTPUTS: tuple[str, ...] = ("f1", "precision", "recall")


@dataclass(frozen=True)
class ScoredRow:
    """μ±„μ ν•œ κ²°κ³Ό ν–‰ ν•˜λ‚˜ β€” `sample_idx`, run λ²ˆλ“€ `items.jsonl` 의 ν‚€ 쌍, 채점 κ²°κ³Ό."""

    sample_idx: int
    input_key: str
    golden_key: str
    outcome: RowScore | RowError


def judgment_drafts(run_id: str, rows: Sequence[ScoredRow], created_at: str) -> list[JudgmentDraft]:
    """ν–‰λ§ˆλ‹€ `exact`Β·`lcs_f1`Β·`spans` μ΄ˆμ•ˆ 3개(ν–‰ μˆœμ„œ κ·ΈλŒ€λ‘œ)."""
    drafts: list[JudgmentDraft] = []
    for row in rows:
        common = {
            "level": "item",
            "run_id": run_id,
            "sample_idx": row.sample_idx,
            "input_key": row.input_key,
            "golden_key": row.golden_key,
            "source_kind": "code",
            "actor": ACTOR,
            "created_at": created_at,
        }
        for criterion in CRITERIA:
            fields = {**common, "criterion_id": criterion, **_result_fields(criterion, row.outcome)}
            drafts.append(JudgmentDraft(fields))
    return drafts


def _result_fields(criterion: str, outcome: RowScore | RowError) -> dict[str, Any]:
    if isinstance(outcome, RowError):
        return {"status": "error", "reason": outcome.reason}
    if criterion == EXACT:
        return {
            "score": outcome.score_exact,
            "label": outcome.label,
            "passed": outcome.label == LABEL_CORRECT,
            "status": "ok",
        }
    if criterion == LCS_F1:
        return {"score": outcome.lcs_f1, "status": "ok"}
    return {"value": dict(outcome.spans), "status": "ok"}


def producer_aggregates(
    rows: Sequence[ScoredRow], drafts: Sequence[JudgmentDraft]
) -> dict[str, Any]:
    """`producer-aggregates/v1` λ¬Έμ„œ β€” 값은 (metric_key, aggregation) μ‚¬μ „μˆœ."""
    values = [_mean_value(EXACT, drafts), _mean_value(LCS_F1, drafts)]
    ok = [row.outcome for row in rows if isinstance(row.outcome, RowScore)]
    if ok:
        scores = legacy_span_scores(ok)
        for output in SPAN_OUTPUTS:
            values.append(_value(SPANS, f"micro_prf@1:{output}", scores[output]))
    values.sort(key=lambda v: (v["metric_key"], v["aggregation"]))
    return {"schema": PRODUCER_AGGREGATES_SCHEMA, "values": values}


def legacy_span_scores(ok: Sequence[RowScore]) -> Mapping[str, Any]:
    """λ ˆκ±°μ‹œ `LCSDiffEvaluator.aggregate` κ°€ ok ν–‰ 카운트둜 λ‚΄λŠ” run μΈ΅ `scores`(νΌμ„ΌνŠΈ)."""
    aggregated = LCSDiffEvaluator().aggregate([{"details": dict(row.spans)} for row in ok])
    return aggregated["scores"]


def _mean_value(criterion: str, drafts: Sequence[JudgmentDraft]) -> dict[str, Any]:
    rows = [d.fields for d in drafts if d.fields.get("criterion_id") == criterion]
    return _value(criterion, "mean", derive("mean", rows)["value"])


def _value(metric_key: str, aggregation: str, value: Any) -> dict[str, Any]:
    return {
        "metric_key": metric_key,
        "aggregation": aggregation,
        "variant": "raw",
        "error_policy": "exclude",
        "value": value,
    }