Spaces:
Running
Running
File size: 6,399 Bytes
3dfc93b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 | """`proofread_diff@2` β κ²°κ³Ό ν νλμ μμ μ±μ (κ³ν 2026-09-26 L3b D1Β·D2Β·D3).
**μ΄ λͺ¨λμ νμ€ λΌμ΄λΈλ¬λ¦¬μ `solar_eval.evaluators.scoring` λ§ import νλ€.** νκ°κΈ° `impl_hash` λ μ΄ λͺ¨λμμ μμν
import νν¬(`core/snapshot/impl_hash.py`)λ‘ λ¨λ―λ‘, λ²λ€ μ½κΈ°Β·writerΒ·λ―Έλ¬ κ°μ I/O μ½λλ `from_bundle.py`Β·`case.py` μ λλ€ β
κ·ΈλμΌ μ
μΆλ ₯ μ½λλ₯Ό κ³ μ³λ νκ°κΈ° ν΄μκ° λ°λμ§ μκ³ , μ±μ κ·μΉμ΄ λ°λλ©΄ λ°λμ λ°λλ€(MB1).
μ±μ κ·μΉμ λ κ±°μ `lcs_diff` μ **κ°μ ν¨μ**(`compute_proofread_metrics`)λ€ β λ λ² λμ§ μλλ€. μ μ 쑰건λ λ κ±°μ
`runs eval`(`cli/commands/run_eval.py` `_score_all` β `validate_required_fields` β `LCSDiffEvaluator.evaluate`)κ³Ό κ°κ² λ§μΆλ€:
- `output is None` β `missing_output`, `golden is None` β `missing_golden`(λ κ±°μ `validate_required_fields` μ ValueError).
- `output` μ΄ λ¬Έμμ΄μ΄ μλ β `output_not_text`(λ κ±°μλ `.split()` μ AttributeError λ‘ μ€ν¨).
- κ·Έ λ°μ μμΈ(μ
λ ₯μ΄ κ°μ²΄κ° μλ, `original` μ΄ λ¬Έμμ΄μ΄ μλ λ±) β `scoring_failed`.
- `input` μ `original` μ΄ μμΌλ©΄ `""`, `golden` μ΄ λ¬Έμμ΄μ΄ μλλ©΄ `str(golden)` β λ κ±°μκ° λ΄λ κ°κ³Ό κ°κ².
`exact` λΌλ²¨(D3): μ κ·ν(`strip@1` β `ws_collapse@1`)ν μΆλ ₯μ΄ λΉ λ¬Έμμ΄ β `noanswer`, μ κ·νν μΆλ ₯ = μ κ·νν μ λ΅ β
`correct`, `spans.tp > 0` β `partial`, λλ¨Έμ§ β `incorrect`. μ κ·νκΈ° μ μμ μ λ³Έμ μ΄ λͺ¨λ νλλ€(λ¬Έμλ
`PS/SCORING.md` Β§4.4 κ° κ°μ λ§μ μ κ³ , ν
μ€νΈκ° λ μ΄νλ₯Ό λμ‘°νλ€).
"""
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Any
from solar_eval.evaluators.scoring import compute_proofread_metrics
#: μ κ·νκΈ° μ΄λ¦ β νκ°κΈ° λ¬Έμ `params.exact_compare.normalizers` μ μ΄ μμλ‘ μ€λ¦°λ€(ν΄μ μ¬λ£).
NORMALIZERS: tuple[str, ...] = ("strip@1", "ws_collapse@1")
#: μλ¬Έμ μ½λ μ
λ ₯ νλ β νκ°κΈ° λ¬Έμ `params.original_field`.
ORIGINAL_FIELD = "original"
LABEL_CORRECT = "correct"
LABEL_PARTIAL = "partial"
LABEL_INCORRECT = "incorrect"
LABEL_NOANSWER = "noanswer"
#: `exact` μ§ν `levels` μμ(λ²‘ν° `scoring/bundles/02` μ κ°λ€).
LABELS: tuple[str, ...] = (LABEL_CORRECT, LABEL_PARTIAL, LABEL_INCORRECT, LABEL_NOANSWER)
#: λΌλ²¨ β μ μ(`label_scores`). correct λ§ 1.
LABEL_SCORES: dict[str, int] = {
LABEL_CORRECT: 1,
LABEL_PARTIAL: 0,
LABEL_INCORRECT: 0,
LABEL_NOANSWER: 0,
}
REASON_MISSING_OUTPUT = "missing_output"
REASON_MISSING_GOLDEN = "missing_golden"
REASON_OUTPUT_NOT_TEXT = "output_not_text"
REASON_SCORING_FAILED = "scoring_failed"
#: μ€ν¨ νμ `reason` μ΄ν(ν λ¨μ).
REASONS: tuple[str, ...] = (
REASON_MISSING_OUTPUT,
REASON_MISSING_GOLDEN,
REASON_OUTPUT_NOT_TEXT,
REASON_SCORING_FAILED,
)
#: `spans` νμ `value` μ ν€ β λ κ±°μ `compute_proofread_metrics` μΉ΄μ΄νΈ μ΄λ¦ κ·Έλλ‘.
SPAN_KEYS: tuple[str, ...] = ("tp", "fp", "missing", "redundant")
@dataclass(frozen=True)
class RowScore:
"""μ±μ μ±κ³΅ β `exact` λΌλ²¨Β·μ μ, `lcs_f1`(λ κ±°μ `f1`, 0~100), `spans` μΉ΄μ΄νΈ."""
label: str
score_exact: int
lcs_f1: float
spans: dict[str, int]
@dataclass(frozen=True)
class RowError:
"""μ±μ μ€ν¨ β μΈ μ§ν λͺ¨λ `status="error"` μ΄κ³ κ°μ `reason` μ΄λ€(D2)."""
reason: str
def strip_ws(text: str) -> str:
"""`strip@1` β μλ€μ `str.isspace()` λ¬Έμλ₯Ό μ§μ΄λ€(U+3000 μ κ° κ³΅λ°± ν¬ν¨)."""
start, end = 0, len(text)
while start < end and text[start].isspace():
start += 1
while end > start and text[end - 1].isspace():
end -= 1
return text[start:end]
def collapse_ws(text: str) -> str:
"""`ws_collapse@1` β `str.isspace()` λ¬Έμμ μ°μμ U+0020 νλλ‘ λ°κΎΌλ€."""
out: list[str] = []
in_space = False
for ch in text:
if ch.isspace():
if not in_space:
out.append(" ")
in_space = True
else:
out.append(ch)
in_space = False
return "".join(out)
def normalize(text: str) -> str:
"""`NORMALIZERS` μμ(strip β collapse)λ‘ μ κ·ννλ€."""
return collapse_ws(strip_ws(text))
def label_of(norm_out: str, norm_gold: str, tp: int) -> str:
"""D3 λΌλ²¨ κ·μΉ β μ κ·νν κ°μ λ°λλ€. μμκ° κ·μΉμ΄λ€(λΉ μΆλ ₯μ΄ λ¨Όμ )."""
if norm_out == "":
return LABEL_NOANSWER
if norm_out == norm_gold:
return LABEL_CORRECT
if tp > 0:
return LABEL_PARTIAL
return LABEL_INCORRECT
def has_original(result_row: Mapping[str, Any]) -> bool:
"""μ
λ ₯ κ°μ²΄μ `original` ν€κ° μλκ° β μμΌλ©΄ λ κ±°μμ²λΌ `""` λ‘ μ±μ νλ€(λ³΄κ³ μ© κ±΄μ)."""
source = result_row.get("input")
return isinstance(source, Mapping) and ORIGINAL_FIELD in source
def score_row(result_row: Mapping[str, Any]) -> RowScore | RowError:
"""`results.jsonl` ν νλλ₯Ό μ±μ νλ€. μ
λ ₯μ λ°κΎΈμ§ μλλ€.
λ κ±°μκ° μ€ν¨λ‘ λͺ¨μΌλ νμ μ¬κΈ°μλ `RowError` λ€ β μ€ν¨ μ§ν©μ΄ κ°μμΌ parity κ° λ»μ κ°μ§λ€.
"""
output = result_row.get("output")
golden = result_row.get("golden")
if output is None:
return RowError(REASON_MISSING_OUTPUT)
if golden is None:
return RowError(REASON_MISSING_GOLDEN)
if not isinstance(output, str):
return RowError(REASON_OUTPUT_NOT_TEXT)
reference = str(golden)
try:
# λ κ±°μ `LCSDiffEvaluator.evaluate` μ κ°μ μ β `sample.input.get("original", "")`.
original = result_row.get("input").get(ORIGINAL_FIELD, "") # type: ignore[union-attr]
metrics = compute_proofread_metrics(original, reference, output)
except Exception: # noqa: BLE001 β λ κ±°μ `_score_all` λ λͺ¨λ μμΈλ₯Ό μ€ν¨ μνλ‘ λͺ¨μλ€.
return RowError(REASON_SCORING_FAILED)
spans = {key: int(metrics[key]) for key in SPAN_KEYS}
label = label_of(normalize(output), normalize(reference), spans["tp"])
return RowScore(
label=label,
score_exact=LABEL_SCORES[label],
lcs_f1=float(metrics["f1"]),
spans=spans,
)
|