Spaces:
Running
Running
Download solar_eval/rescore/drafts.py from dev-strender/proofread-demo: direct link, hf CLI and curl.
- Browser
- Download file 4.48 kB
-
https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/rescore/drafts.py
- Command line
-
hf download hf://spaces/dev-strender/proofread-demo/solar_eval/rescore/drafts.py
-
curl -L -o drafts.py https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/rescore/drafts.py
4.48 kB
| """νμ μ΄μΒ·μμ°μ μ§κ³ 쑰립 (κ³ν 2026-09-26 L3b Β§2.4Β·D2Β·D4). | |
| - κ²°κ³Ό νλ§λ€ νμ 3κ°, μμ = `exact` β `lcs_f1` β `spans`. μ€ν¨ νμ μΈ μ§ν λͺ¨λ `status="error"`Β·κ°μ `reason` μ΄λ€. | |
| - `producer_aggregates.json`: `exact`Β·`lcs_f1` μ `mean` μ νμ μ°Έμ‘° ꡬν `derive("mean")`, `spans` μ `micro_prf@1:{f1,precision, | |
| recall}` μ **λ κ±°μ ꡬν** `LCSDiffEvaluator().aggregate()` μ ok ν μΉ΄μ΄νΈλ₯Ό λ£μ κ°μ΄λ€(D4) β hub μ μ¬μ | |
| `producer_aggregate_mismatch` λμ‘°κ° "SQL νμ β λ κ±°μ μ½λ" λμ‘°κ° λλ€. ok νμ΄ 0 μ΄λ©΄ `spans` νλͺ©μ μ£μ§ μλλ€ | |
| (λ κ±°μμ 0.0 μ "μΈ‘μ μμ" κ³Ό ꡬλ³λμ§ μλλ€). | |
| """ | |
| from __future__ import annotations | |
| from collections.abc import Mapping, Sequence | |
| from dataclasses import dataclass | |
| from typing import Any | |
| from poc_bundle.scoring.derivations import derive | |
| from poc_bundle.scoring.writer import JudgmentDraft | |
| from solar_eval.evaluators.lcs_diff import LCSDiffEvaluator | |
| from solar_eval.rescore.evaluator_doc import ACTOR | |
| from solar_eval.rescore.proofread_diff_v2 import LABEL_CORRECT, RowError, RowScore | |
| EXACT, LCS_F1, SPANS = "exact", "lcs_f1", "spans" | |
| #: νμ νμΌ μ μ§ν μμ. | |
| CRITERIA: tuple[str, ...] = (EXACT, LCS_F1, SPANS) | |
| PRODUCER_AGGREGATES_SCHEMA = "producer-aggregates/v1" | |
| #: λ κ±°μ `aggregate()["scores"]` ν€ β `micro_prf@1` μΆλ ₯ μ΄λ¦. | |
| SPAN_OUTPUTS: tuple[str, ...] = ("f1", "precision", "recall") | |
| class ScoredRow: | |
| """μ±μ ν κ²°κ³Ό ν νλ β `sample_idx`, run λ²λ€ `items.jsonl` μ ν€ μ, μ±μ κ²°κ³Ό.""" | |
| sample_idx: int | |
| input_key: str | |
| golden_key: str | |
| outcome: RowScore | RowError | |
| def judgment_drafts(run_id: str, rows: Sequence[ScoredRow], created_at: str) -> list[JudgmentDraft]: | |
| """νλ§λ€ `exact`Β·`lcs_f1`Β·`spans` μ΄μ 3κ°(ν μμ κ·Έλλ‘).""" | |
| drafts: list[JudgmentDraft] = [] | |
| for row in rows: | |
| common = { | |
| "level": "item", | |
| "run_id": run_id, | |
| "sample_idx": row.sample_idx, | |
| "input_key": row.input_key, | |
| "golden_key": row.golden_key, | |
| "source_kind": "code", | |
| "actor": ACTOR, | |
| "created_at": created_at, | |
| } | |
| for criterion in CRITERIA: | |
| fields = {**common, "criterion_id": criterion, **_result_fields(criterion, row.outcome)} | |
| drafts.append(JudgmentDraft(fields)) | |
| return drafts | |
| def _result_fields(criterion: str, outcome: RowScore | RowError) -> dict[str, Any]: | |
| if isinstance(outcome, RowError): | |
| return {"status": "error", "reason": outcome.reason} | |
| if criterion == EXACT: | |
| return { | |
| "score": outcome.score_exact, | |
| "label": outcome.label, | |
| "passed": outcome.label == LABEL_CORRECT, | |
| "status": "ok", | |
| } | |
| if criterion == LCS_F1: | |
| return {"score": outcome.lcs_f1, "status": "ok"} | |
| return {"value": dict(outcome.spans), "status": "ok"} | |
| def producer_aggregates( | |
| rows: Sequence[ScoredRow], drafts: Sequence[JudgmentDraft] | |
| ) -> dict[str, Any]: | |
| """`producer-aggregates/v1` λ¬Έμ β κ°μ (metric_key, aggregation) μ¬μ μ.""" | |
| values = [_mean_value(EXACT, drafts), _mean_value(LCS_F1, drafts)] | |
| ok = [row.outcome for row in rows if isinstance(row.outcome, RowScore)] | |
| if ok: | |
| scores = legacy_span_scores(ok) | |
| for output in SPAN_OUTPUTS: | |
| values.append(_value(SPANS, f"micro_prf@1:{output}", scores[output])) | |
| values.sort(key=lambda v: (v["metric_key"], v["aggregation"])) | |
| return {"schema": PRODUCER_AGGREGATES_SCHEMA, "values": values} | |
| def legacy_span_scores(ok: Sequence[RowScore]) -> Mapping[str, Any]: | |
| """λ κ±°μ `LCSDiffEvaluator.aggregate` κ° ok ν μΉ΄μ΄νΈλ‘ λ΄λ run μΈ΅ `scores`(νΌμΌνΈ).""" | |
| aggregated = LCSDiffEvaluator().aggregate([{"details": dict(row.spans)} for row in ok]) | |
| return aggregated["scores"] | |
| def _mean_value(criterion: str, drafts: Sequence[JudgmentDraft]) -> dict[str, Any]: | |
| rows = [d.fields for d in drafts if d.fields.get("criterion_id") == criterion] | |
| return _value(criterion, "mean", derive("mean", rows)["value"]) | |
| def _value(metric_key: str, aggregation: str, value: Any) -> dict[str, Any]: | |
| return { | |
| "metric_key": metric_key, | |
| "aggregation": aggregation, | |
| "variant": "raw", | |
| "error_policy": "exclude", | |
| "value": value, | |
| } | |