proofread-demo / solar_eval /rescore /proofread_diff_v2.py
dev-strender's picture
feat: switch demo to FT + Opus article review (pipeline_exp_261021_ft_hybrid)
3dfc93b verified
Raw History Blame Contribute Delete
6.4 kB
"""`proofread_diff@2` β€” κ²°κ³Ό ν–‰ ν•˜λ‚˜μ˜ 순수 채점 (κ³„νš 2026-09-26 L3b D1Β·D2Β·D3).
**이 λͺ¨λ“ˆμ€ ν‘œμ€€ λΌμ΄λΈŒλŸ¬λ¦¬μ™€ `solar_eval.evaluators.scoring` 만 import ν•œλ‹€.** 평가기 `impl_hash` λŠ” 이 λͺ¨λ“ˆμ—μ„œ μ‹œμž‘ν•œ
import 폐포(`core/snapshot/impl_hash.py`)둜 λœ¨λ―€λ‘œ, λ²ˆλ“€ 읽기·writer·미러 같은 I/O μ½”λ“œλŠ” `from_bundle.py`Β·`case.py` 에 λ‘”λ‹€ β€”
κ·Έλž˜μ•Ό μž…μΆœλ ₯ μ½”λ“œλ₯Ό 고쳐도 평가기 ν•΄μ‹œκ°€ λ°”λ€Œμ§€ μ•Šκ³ , 채점 κ·œμΉ™μ΄ λ°”λ€Œλ©΄ λ°˜λ“œμ‹œ 바뀐닀(MB1).
채점 κ·œμΉ™μ€ λ ˆκ±°μ‹œ `lcs_diff` 와 **같은 ν•¨μˆ˜**(`compute_proofread_metrics`)λ‹€ β€” 두 벌 두지 μ•ŠλŠ”λ‹€. μ „μ œμ‘°κ±΄λ„ λ ˆκ±°μ‹œ
`runs eval`(`cli/commands/run_eval.py` `_score_all` β†’ `validate_required_fields` β†’ `LCSDiffEvaluator.evaluate`)κ³Ό κ°™κ²Œ λ§žμΆ˜λ‹€:
- `output is None` β†’ `missing_output`, `golden is None` β†’ `missing_golden`(λ ˆκ±°μ‹œ `validate_required_fields` 의 ValueError).
- `output` 이 λ¬Έμžμ—΄μ΄ μ•„λ‹˜ β†’ `output_not_text`(λ ˆκ±°μ‹œλŠ” `.split()` 의 AttributeError 둜 μ‹€νŒ¨).
- κ·Έ λ°–μ˜ μ˜ˆμ™Έ(μž…λ ₯이 객체가 μ•„λ‹˜, `original` 이 λ¬Έμžμ—΄μ΄ μ•„λ‹˜ λ“±) β†’ `scoring_failed`.
- `input` 에 `original` 이 μ—†μœΌλ©΄ `""`, `golden` 이 λ¬Έμžμ—΄μ΄ μ•„λ‹ˆλ©΄ `str(golden)` β€” λ ˆκ±°μ‹œκ°€ λ‚΄λŠ” κ°’κ³Ό κ°™κ²Œ.
`exact` 라벨(D3): μ •κ·œν™”(`strip@1` β†’ `ws_collapse@1`)ν•œ 좜λ ₯이 빈 λ¬Έμžμ—΄ β†’ `noanswer`, μ •κ·œν™”ν•œ 좜λ ₯ = μ •κ·œν™”ν•œ μ •λ‹΅ β†’
`correct`, `spans.tp > 0` β†’ `partial`, λ‚˜λ¨Έμ§€ β†’ `incorrect`. μ •κ·œν™”κΈ° μ •μ˜μ˜ 정본은 이 λͺ¨λ“ˆ ν•˜λ‚˜λ‹€(λ¬Έμ„œλŠ”
`PS/SCORING.md` Β§4.4 κ°€ 같은 말을 적고, ν…ŒμŠ€νŠΈκ°€ 두 μ–΄νœ˜λ₯Ό λŒ€μ‘°ν•œλ‹€).
"""
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Any
from solar_eval.evaluators.scoring import compute_proofread_metrics
#: μ •κ·œν™”κΈ° 이름 β€” 평가기 λ¬Έμ„œ `params.exact_compare.normalizers` 에 이 μˆœμ„œλ‘œ μ‹€λ¦°λ‹€(ν•΄μ‹œ 재료).
NORMALIZERS: tuple[str, ...] = ("strip@1", "ws_collapse@1")
#: 원문을 μ½λŠ” μž…λ ₯ ν•„λ“œ β€” 평가기 λ¬Έμ„œ `params.original_field`.
ORIGINAL_FIELD = "original"
LABEL_CORRECT = "correct"
LABEL_PARTIAL = "partial"
LABEL_INCORRECT = "incorrect"
LABEL_NOANSWER = "noanswer"
#: `exact` μ§€ν‘œ `levels` μˆœμ„œ(벑터 `scoring/bundles/02` 와 κ°™λ‹€).
LABELS: tuple[str, ...] = (LABEL_CORRECT, LABEL_PARTIAL, LABEL_INCORRECT, LABEL_NOANSWER)
#: 라벨 β†’ 점수(`label_scores`). correct 만 1.
LABEL_SCORES: dict[str, int] = {
LABEL_CORRECT: 1,
LABEL_PARTIAL: 0,
LABEL_INCORRECT: 0,
LABEL_NOANSWER: 0,
}
REASON_MISSING_OUTPUT = "missing_output"
REASON_MISSING_GOLDEN = "missing_golden"
REASON_OUTPUT_NOT_TEXT = "output_not_text"
REASON_SCORING_FAILED = "scoring_failed"
#: μ‹€νŒ¨ νŒμ • `reason` μ–΄νœ˜(ν–‰ λ‹¨μœ„).
REASONS: tuple[str, ...] = (
REASON_MISSING_OUTPUT,
REASON_MISSING_GOLDEN,
REASON_OUTPUT_NOT_TEXT,
REASON_SCORING_FAILED,
)
#: `spans` νŒμ • `value` 의 ν‚€ β€” λ ˆκ±°μ‹œ `compute_proofread_metrics` 카운트 이름 κ·ΈλŒ€λ‘œ.
SPAN_KEYS: tuple[str, ...] = ("tp", "fp", "missing", "redundant")
@dataclass(frozen=True)
class RowScore:
"""채점 성곡 β€” `exact` 라벨·점수, `lcs_f1`(λ ˆκ±°μ‹œ `f1`, 0~100), `spans` 카운트."""
label: str
score_exact: int
lcs_f1: float
spans: dict[str, int]
@dataclass(frozen=True)
class RowError:
"""채점 μ‹€νŒ¨ β€” μ„Έ μ§€ν‘œ λͺ¨λ‘ `status="error"` 이고 같은 `reason` 이닀(D2)."""
reason: str
def strip_ws(text: str) -> str:
"""`strip@1` β€” μ•žλ’€μ˜ `str.isspace()` 문자λ₯Ό μ§€μš΄λ‹€(U+3000 전각 곡백 포함)."""
start, end = 0, len(text)
while start < end and text[start].isspace():
start += 1
while end > start and text[end - 1].isspace():
end -= 1
return text[start:end]
def collapse_ws(text: str) -> str:
"""`ws_collapse@1` β€” `str.isspace()` 문자의 연속을 U+0020 ν•˜λ‚˜λ‘œ λ°”κΎΌλ‹€."""
out: list[str] = []
in_space = False
for ch in text:
if ch.isspace():
if not in_space:
out.append(" ")
in_space = True
else:
out.append(ch)
in_space = False
return "".join(out)
def normalize(text: str) -> str:
"""`NORMALIZERS` μˆœμ„œ(strip β†’ collapse)둜 μ •κ·œν™”ν•œλ‹€."""
return collapse_ws(strip_ws(text))
def label_of(norm_out: str, norm_gold: str, tp: int) -> str:
"""D3 라벨 κ·œμΉ™ β€” μ •κ·œν™”ν•œ 값을 λ°›λŠ”λ‹€. μˆœμ„œκ°€ κ·œμΉ™μ΄λ‹€(빈 좜λ ₯이 λ¨Όμ €)."""
if norm_out == "":
return LABEL_NOANSWER
if norm_out == norm_gold:
return LABEL_CORRECT
if tp > 0:
return LABEL_PARTIAL
return LABEL_INCORRECT
def has_original(result_row: Mapping[str, Any]) -> bool:
"""μž…λ ₯ 객체에 `original` ν‚€κ°€ μžˆλŠ”κ°€ β€” μ—†μœΌλ©΄ λ ˆκ±°μ‹œμ²˜λŸΌ `""` 둜 μ±„μ ν•œλ‹€(보고용 건수)."""
source = result_row.get("input")
return isinstance(source, Mapping) and ORIGINAL_FIELD in source
def score_row(result_row: Mapping[str, Any]) -> RowScore | RowError:
"""`results.jsonl` ν–‰ ν•˜λ‚˜λ₯Ό μ±„μ ν•œλ‹€. μž…λ ₯을 λ°”κΎΈμ§€ μ•ŠλŠ”λ‹€.
λ ˆκ±°μ‹œκ°€ μ‹€νŒ¨λ‘œ λͺ¨μœΌλŠ” 행은 μ—¬κΈ°μ„œλ„ `RowError` λ‹€ β€” μ‹€νŒ¨ 집합이 κ°™μ•„μ•Ό parity κ°€ λœ»μ„ κ°€μ§„λ‹€.
"""
output = result_row.get("output")
golden = result_row.get("golden")
if output is None:
return RowError(REASON_MISSING_OUTPUT)
if golden is None:
return RowError(REASON_MISSING_GOLDEN)
if not isinstance(output, str):
return RowError(REASON_OUTPUT_NOT_TEXT)
reference = str(golden)
try:
# λ ˆκ±°μ‹œ `LCSDiffEvaluator.evaluate` 와 같은 식 β€” `sample.input.get("original", "")`.
original = result_row.get("input").get(ORIGINAL_FIELD, "") # type: ignore[union-attr]
metrics = compute_proofread_metrics(original, reference, output)
except Exception: # noqa: BLE001 β€” λ ˆκ±°μ‹œ `_score_all` 도 λͺ¨λ“  μ˜ˆμ™Έλ₯Ό μ‹€νŒ¨ μƒ˜ν”Œλ‘œ λͺ¨μ€λ‹€.
return RowError(REASON_SCORING_FAILED)
spans = {key: int(metrics[key]) for key in SPAN_KEYS}
label = label_of(normalize(output), normalize(reference), spans["tp"])
return RowScore(
label=label,
score_exact=LABEL_SCORES[label],
lcs_f1=float(metrics["f1"]),
spans=spans,
)