File size: 6,399 Bytes
3dfc93b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
"""`proofread_diff@2` β€” κ²°κ³Ό ν–‰ ν•˜λ‚˜μ˜ 순수 채점 (κ³„νš 2026-09-26 L3b D1Β·D2Β·D3).

**이 λͺ¨λ“ˆμ€ ν‘œμ€€ λΌμ΄λΈŒλŸ¬λ¦¬μ™€ `solar_eval.evaluators.scoring` 만 import ν•œλ‹€.** 평가기 `impl_hash` λŠ” 이 λͺ¨λ“ˆμ—μ„œ μ‹œμž‘ν•œ
import 폐포(`core/snapshot/impl_hash.py`)둜 λœ¨λ―€λ‘œ, λ²ˆλ“€ 읽기·writer·미러 같은 I/O μ½”λ“œλŠ” `from_bundle.py`Β·`case.py` 에 λ‘”λ‹€ β€”
κ·Έλž˜μ•Ό μž…μΆœλ ₯ μ½”λ“œλ₯Ό 고쳐도 평가기 ν•΄μ‹œκ°€ λ°”λ€Œμ§€ μ•Šκ³ , 채점 κ·œμΉ™μ΄ λ°”λ€Œλ©΄ λ°˜λ“œμ‹œ 바뀐닀(MB1).

채점 κ·œμΉ™μ€ λ ˆκ±°μ‹œ `lcs_diff` 와 **같은 ν•¨μˆ˜**(`compute_proofread_metrics`)λ‹€ β€” 두 벌 두지 μ•ŠλŠ”λ‹€. μ „μ œμ‘°κ±΄λ„ λ ˆκ±°μ‹œ
`runs eval`(`cli/commands/run_eval.py` `_score_all` β†’ `validate_required_fields` β†’ `LCSDiffEvaluator.evaluate`)κ³Ό κ°™κ²Œ λ§žμΆ˜λ‹€:

- `output is None` β†’ `missing_output`, `golden is None` β†’ `missing_golden`(λ ˆκ±°μ‹œ `validate_required_fields` 의 ValueError).
- `output` 이 λ¬Έμžμ—΄μ΄ μ•„λ‹˜ β†’ `output_not_text`(λ ˆκ±°μ‹œλŠ” `.split()` 의 AttributeError 둜 μ‹€νŒ¨).
- κ·Έ λ°–μ˜ μ˜ˆμ™Έ(μž…λ ₯이 객체가 μ•„λ‹˜, `original` 이 λ¬Έμžμ—΄μ΄ μ•„λ‹˜ λ“±) β†’ `scoring_failed`.
- `input` 에 `original` 이 μ—†μœΌλ©΄ `""`, `golden` 이 λ¬Έμžμ—΄μ΄ μ•„λ‹ˆλ©΄ `str(golden)` β€” λ ˆκ±°μ‹œκ°€ λ‚΄λŠ” κ°’κ³Ό κ°™κ²Œ.

`exact` 라벨(D3): μ •κ·œν™”(`strip@1` β†’ `ws_collapse@1`)ν•œ 좜λ ₯이 빈 λ¬Έμžμ—΄ β†’ `noanswer`, μ •κ·œν™”ν•œ 좜λ ₯ = μ •κ·œν™”ν•œ μ •λ‹΅ β†’
`correct`, `spans.tp > 0` β†’ `partial`, λ‚˜λ¨Έμ§€ β†’ `incorrect`. μ •κ·œν™”κΈ° μ •μ˜μ˜ 정본은 이 λͺ¨λ“ˆ ν•˜λ‚˜λ‹€(λ¬Έμ„œλŠ”
`PS/SCORING.md` Β§4.4 κ°€ 같은 말을 적고, ν…ŒμŠ€νŠΈκ°€ 두 μ–΄νœ˜λ₯Ό λŒ€μ‘°ν•œλ‹€).
"""

from __future__ import annotations

from collections.abc import Mapping
from dataclasses import dataclass
from typing import Any

from solar_eval.evaluators.scoring import compute_proofread_metrics

#: μ •κ·œν™”κΈ° 이름 β€” 평가기 λ¬Έμ„œ `params.exact_compare.normalizers` 에 이 μˆœμ„œλ‘œ μ‹€λ¦°λ‹€(ν•΄μ‹œ 재료).
NORMALIZERS: tuple[str, ...] = ("strip@1", "ws_collapse@1")
#: 원문을 μ½λŠ” μž…λ ₯ ν•„λ“œ β€” 평가기 λ¬Έμ„œ `params.original_field`.
ORIGINAL_FIELD = "original"

LABEL_CORRECT = "correct"
LABEL_PARTIAL = "partial"
LABEL_INCORRECT = "incorrect"
LABEL_NOANSWER = "noanswer"
#: `exact` μ§€ν‘œ `levels` μˆœμ„œ(벑터 `scoring/bundles/02` 와 κ°™λ‹€).
LABELS: tuple[str, ...] = (LABEL_CORRECT, LABEL_PARTIAL, LABEL_INCORRECT, LABEL_NOANSWER)
#: 라벨 β†’ 점수(`label_scores`). correct 만 1.
LABEL_SCORES: dict[str, int] = {
    LABEL_CORRECT: 1,
    LABEL_PARTIAL: 0,
    LABEL_INCORRECT: 0,
    LABEL_NOANSWER: 0,
}

REASON_MISSING_OUTPUT = "missing_output"
REASON_MISSING_GOLDEN = "missing_golden"
REASON_OUTPUT_NOT_TEXT = "output_not_text"
REASON_SCORING_FAILED = "scoring_failed"
#: μ‹€νŒ¨ νŒμ • `reason` μ–΄νœ˜(ν–‰ λ‹¨μœ„).
REASONS: tuple[str, ...] = (
    REASON_MISSING_OUTPUT,
    REASON_MISSING_GOLDEN,
    REASON_OUTPUT_NOT_TEXT,
    REASON_SCORING_FAILED,
)
#: `spans` νŒμ • `value` 의 ν‚€ β€” λ ˆκ±°μ‹œ `compute_proofread_metrics` 카운트 이름 κ·ΈλŒ€λ‘œ.
SPAN_KEYS: tuple[str, ...] = ("tp", "fp", "missing", "redundant")


@dataclass(frozen=True)
class RowScore:
    """채점 성곡 β€” `exact` 라벨·점수, `lcs_f1`(λ ˆκ±°μ‹œ `f1`, 0~100), `spans` 카운트."""

    label: str
    score_exact: int
    lcs_f1: float
    spans: dict[str, int]


@dataclass(frozen=True)
class RowError:
    """채점 μ‹€νŒ¨ β€” μ„Έ μ§€ν‘œ λͺ¨λ‘ `status="error"` 이고 같은 `reason` 이닀(D2)."""

    reason: str


def strip_ws(text: str) -> str:
    """`strip@1` β€” μ•žλ’€μ˜ `str.isspace()` 문자λ₯Ό μ§€μš΄λ‹€(U+3000 전각 곡백 포함)."""
    start, end = 0, len(text)
    while start < end and text[start].isspace():
        start += 1
    while end > start and text[end - 1].isspace():
        end -= 1
    return text[start:end]


def collapse_ws(text: str) -> str:
    """`ws_collapse@1` β€” `str.isspace()` 문자의 연속을 U+0020 ν•˜λ‚˜λ‘œ λ°”κΎΌλ‹€."""
    out: list[str] = []
    in_space = False
    for ch in text:
        if ch.isspace():
            if not in_space:
                out.append(" ")
            in_space = True
        else:
            out.append(ch)
            in_space = False
    return "".join(out)


def normalize(text: str) -> str:
    """`NORMALIZERS` μˆœμ„œ(strip β†’ collapse)둜 μ •κ·œν™”ν•œλ‹€."""
    return collapse_ws(strip_ws(text))


def label_of(norm_out: str, norm_gold: str, tp: int) -> str:
    """D3 라벨 κ·œμΉ™ β€” μ •κ·œν™”ν•œ 값을 λ°›λŠ”λ‹€. μˆœμ„œκ°€ κ·œμΉ™μ΄λ‹€(빈 좜λ ₯이 λ¨Όμ €)."""
    if norm_out == "":
        return LABEL_NOANSWER
    if norm_out == norm_gold:
        return LABEL_CORRECT
    if tp > 0:
        return LABEL_PARTIAL
    return LABEL_INCORRECT


def has_original(result_row: Mapping[str, Any]) -> bool:
    """μž…λ ₯ 객체에 `original` ν‚€κ°€ μžˆλŠ”κ°€ β€” μ—†μœΌλ©΄ λ ˆκ±°μ‹œμ²˜λŸΌ `""` 둜 μ±„μ ν•œλ‹€(보고용 건수)."""
    source = result_row.get("input")
    return isinstance(source, Mapping) and ORIGINAL_FIELD in source


def score_row(result_row: Mapping[str, Any]) -> RowScore | RowError:
    """`results.jsonl` ν–‰ ν•˜λ‚˜λ₯Ό μ±„μ ν•œλ‹€. μž…λ ₯을 λ°”κΎΈμ§€ μ•ŠλŠ”λ‹€.

    λ ˆκ±°μ‹œκ°€ μ‹€νŒ¨λ‘œ λͺ¨μœΌλŠ” 행은 μ—¬κΈ°μ„œλ„ `RowError` λ‹€ β€” μ‹€νŒ¨ 집합이 κ°™μ•„μ•Ό parity κ°€ λœ»μ„ κ°€μ§„λ‹€.
    """
    output = result_row.get("output")
    golden = result_row.get("golden")
    if output is None:
        return RowError(REASON_MISSING_OUTPUT)
    if golden is None:
        return RowError(REASON_MISSING_GOLDEN)
    if not isinstance(output, str):
        return RowError(REASON_OUTPUT_NOT_TEXT)
    reference = str(golden)
    try:
        # λ ˆκ±°μ‹œ `LCSDiffEvaluator.evaluate` 와 같은 식 β€” `sample.input.get("original", "")`.
        original = result_row.get("input").get(ORIGINAL_FIELD, "")  # type: ignore[union-attr]
        metrics = compute_proofread_metrics(original, reference, output)
    except Exception:  # noqa: BLE001 β€” λ ˆκ±°μ‹œ `_score_all` 도 λͺ¨λ“  μ˜ˆμ™Έλ₯Ό μ‹€νŒ¨ μƒ˜ν”Œλ‘œ λͺ¨μ€λ‹€.
        return RowError(REASON_SCORING_FAILED)
    spans = {key: int(metrics[key]) for key in SPAN_KEYS}
    label = label_of(normalize(output), normalize(reference), spans["tp"])
    return RowScore(
        label=label,
        score_exact=LABEL_SCORES[label],
        lcs_f1=float(metrics["f1"]),
        spans=spans,
    )