Spaces:
Running
Running
Download solar_eval/rescore/proofread_diff_v2.py from dev-strender/proofread-demo: direct link, hf CLI and curl.
- Browser
- Download file 6.4 kB
-
https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/rescore/proofread_diff_v2.py
- Command line
-
hf download hf://spaces/dev-strender/proofread-demo/solar_eval/rescore/proofread_diff_v2.py
-
curl -L -o proofread_diff_v2.py https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/rescore/proofread_diff_v2.py
6.4 kB
| """`proofread_diff@2` β κ²°κ³Ό ν νλμ μμ μ±μ (κ³ν 2026-09-26 L3b D1Β·D2Β·D3). | |
| **μ΄ λͺ¨λμ νμ€ λΌμ΄λΈλ¬λ¦¬μ `solar_eval.evaluators.scoring` λ§ import νλ€.** νκ°κΈ° `impl_hash` λ μ΄ λͺ¨λμμ μμν | |
| import νν¬(`core/snapshot/impl_hash.py`)λ‘ λ¨λ―λ‘, λ²λ€ μ½κΈ°Β·writerΒ·λ―Έλ¬ κ°μ I/O μ½λλ `from_bundle.py`Β·`case.py` μ λλ€ β | |
| κ·ΈλμΌ μ μΆλ ₯ μ½λλ₯Ό κ³ μ³λ νκ°κΈ° ν΄μκ° λ°λμ§ μκ³ , μ±μ κ·μΉμ΄ λ°λλ©΄ λ°λμ λ°λλ€(MB1). | |
| μ±μ κ·μΉμ λ κ±°μ `lcs_diff` μ **κ°μ ν¨μ**(`compute_proofread_metrics`)λ€ β λ λ² λμ§ μλλ€. μ μ 쑰건λ λ κ±°μ | |
| `runs eval`(`cli/commands/run_eval.py` `_score_all` β `validate_required_fields` β `LCSDiffEvaluator.evaluate`)κ³Ό κ°κ² λ§μΆλ€: | |
| - `output is None` β `missing_output`, `golden is None` β `missing_golden`(λ κ±°μ `validate_required_fields` μ ValueError). | |
| - `output` μ΄ λ¬Έμμ΄μ΄ μλ β `output_not_text`(λ κ±°μλ `.split()` μ AttributeError λ‘ μ€ν¨). | |
| - κ·Έ λ°μ μμΈ(μ λ ₯μ΄ κ°μ²΄κ° μλ, `original` μ΄ λ¬Έμμ΄μ΄ μλ λ±) β `scoring_failed`. | |
| - `input` μ `original` μ΄ μμΌλ©΄ `""`, `golden` μ΄ λ¬Έμμ΄μ΄ μλλ©΄ `str(golden)` β λ κ±°μκ° λ΄λ κ°κ³Ό κ°κ². | |
| `exact` λΌλ²¨(D3): μ κ·ν(`strip@1` β `ws_collapse@1`)ν μΆλ ₯μ΄ λΉ λ¬Έμμ΄ β `noanswer`, μ κ·νν μΆλ ₯ = μ κ·νν μ λ΅ β | |
| `correct`, `spans.tp > 0` β `partial`, λλ¨Έμ§ β `incorrect`. μ κ·νκΈ° μ μμ μ λ³Έμ μ΄ λͺ¨λ νλλ€(λ¬Έμλ | |
| `PS/SCORING.md` Β§4.4 κ° κ°μ λ§μ μ κ³ , ν μ€νΈκ° λ μ΄νλ₯Ό λμ‘°νλ€). | |
| """ | |
| from __future__ import annotations | |
| from collections.abc import Mapping | |
| from dataclasses import dataclass | |
| from typing import Any | |
| from solar_eval.evaluators.scoring import compute_proofread_metrics | |
| #: μ κ·νκΈ° μ΄λ¦ β νκ°κΈ° λ¬Έμ `params.exact_compare.normalizers` μ μ΄ μμλ‘ μ€λ¦°λ€(ν΄μ μ¬λ£). | |
| NORMALIZERS: tuple[str, ...] = ("strip@1", "ws_collapse@1") | |
| #: μλ¬Έμ μ½λ μ λ ₯ νλ β νκ°κΈ° λ¬Έμ `params.original_field`. | |
| ORIGINAL_FIELD = "original" | |
| LABEL_CORRECT = "correct" | |
| LABEL_PARTIAL = "partial" | |
| LABEL_INCORRECT = "incorrect" | |
| LABEL_NOANSWER = "noanswer" | |
| #: `exact` μ§ν `levels` μμ(λ²‘ν° `scoring/bundles/02` μ κ°λ€). | |
| LABELS: tuple[str, ...] = (LABEL_CORRECT, LABEL_PARTIAL, LABEL_INCORRECT, LABEL_NOANSWER) | |
| #: λΌλ²¨ β μ μ(`label_scores`). correct λ§ 1. | |
| LABEL_SCORES: dict[str, int] = { | |
| LABEL_CORRECT: 1, | |
| LABEL_PARTIAL: 0, | |
| LABEL_INCORRECT: 0, | |
| LABEL_NOANSWER: 0, | |
| } | |
| REASON_MISSING_OUTPUT = "missing_output" | |
| REASON_MISSING_GOLDEN = "missing_golden" | |
| REASON_OUTPUT_NOT_TEXT = "output_not_text" | |
| REASON_SCORING_FAILED = "scoring_failed" | |
| #: μ€ν¨ νμ `reason` μ΄ν(ν λ¨μ). | |
| REASONS: tuple[str, ...] = ( | |
| REASON_MISSING_OUTPUT, | |
| REASON_MISSING_GOLDEN, | |
| REASON_OUTPUT_NOT_TEXT, | |
| REASON_SCORING_FAILED, | |
| ) | |
| #: `spans` νμ `value` μ ν€ β λ κ±°μ `compute_proofread_metrics` μΉ΄μ΄νΈ μ΄λ¦ κ·Έλλ‘. | |
| SPAN_KEYS: tuple[str, ...] = ("tp", "fp", "missing", "redundant") | |
| class RowScore: | |
| """μ±μ μ±κ³΅ β `exact` λΌλ²¨Β·μ μ, `lcs_f1`(λ κ±°μ `f1`, 0~100), `spans` μΉ΄μ΄νΈ.""" | |
| label: str | |
| score_exact: int | |
| lcs_f1: float | |
| spans: dict[str, int] | |
| class RowError: | |
| """μ±μ μ€ν¨ β μΈ μ§ν λͺ¨λ `status="error"` μ΄κ³ κ°μ `reason` μ΄λ€(D2).""" | |
| reason: str | |
| def strip_ws(text: str) -> str: | |
| """`strip@1` β μλ€μ `str.isspace()` λ¬Έμλ₯Ό μ§μ΄λ€(U+3000 μ κ° κ³΅λ°± ν¬ν¨).""" | |
| start, end = 0, len(text) | |
| while start < end and text[start].isspace(): | |
| start += 1 | |
| while end > start and text[end - 1].isspace(): | |
| end -= 1 | |
| return text[start:end] | |
| def collapse_ws(text: str) -> str: | |
| """`ws_collapse@1` β `str.isspace()` λ¬Έμμ μ°μμ U+0020 νλλ‘ λ°κΎΌλ€.""" | |
| out: list[str] = [] | |
| in_space = False | |
| for ch in text: | |
| if ch.isspace(): | |
| if not in_space: | |
| out.append(" ") | |
| in_space = True | |
| else: | |
| out.append(ch) | |
| in_space = False | |
| return "".join(out) | |
| def normalize(text: str) -> str: | |
| """`NORMALIZERS` μμ(strip β collapse)λ‘ μ κ·ννλ€.""" | |
| return collapse_ws(strip_ws(text)) | |
| def label_of(norm_out: str, norm_gold: str, tp: int) -> str: | |
| """D3 λΌλ²¨ κ·μΉ β μ κ·νν κ°μ λ°λλ€. μμκ° κ·μΉμ΄λ€(λΉ μΆλ ₯μ΄ λ¨Όμ ).""" | |
| if norm_out == "": | |
| return LABEL_NOANSWER | |
| if norm_out == norm_gold: | |
| return LABEL_CORRECT | |
| if tp > 0: | |
| return LABEL_PARTIAL | |
| return LABEL_INCORRECT | |
| def has_original(result_row: Mapping[str, Any]) -> bool: | |
| """μ λ ₯ κ°μ²΄μ `original` ν€κ° μλκ° β μμΌλ©΄ λ κ±°μμ²λΌ `""` λ‘ μ±μ νλ€(λ³΄κ³ μ© κ±΄μ).""" | |
| source = result_row.get("input") | |
| return isinstance(source, Mapping) and ORIGINAL_FIELD in source | |
| def score_row(result_row: Mapping[str, Any]) -> RowScore | RowError: | |
| """`results.jsonl` ν νλλ₯Ό μ±μ νλ€. μ λ ₯μ λ°κΎΈμ§ μλλ€. | |
| λ κ±°μκ° μ€ν¨λ‘ λͺ¨μΌλ νμ μ¬κΈ°μλ `RowError` λ€ β μ€ν¨ μ§ν©μ΄ κ°μμΌ parity κ° λ»μ κ°μ§λ€. | |
| """ | |
| output = result_row.get("output") | |
| golden = result_row.get("golden") | |
| if output is None: | |
| return RowError(REASON_MISSING_OUTPUT) | |
| if golden is None: | |
| return RowError(REASON_MISSING_GOLDEN) | |
| if not isinstance(output, str): | |
| return RowError(REASON_OUTPUT_NOT_TEXT) | |
| reference = str(golden) | |
| try: | |
| # λ κ±°μ `LCSDiffEvaluator.evaluate` μ κ°μ μ β `sample.input.get("original", "")`. | |
| original = result_row.get("input").get(ORIGINAL_FIELD, "") # type: ignore[union-attr] | |
| metrics = compute_proofread_metrics(original, reference, output) | |
| except Exception: # noqa: BLE001 β λ κ±°μ `_score_all` λ λͺ¨λ μμΈλ₯Ό μ€ν¨ μνλ‘ λͺ¨μλ€. | |
| return RowError(REASON_SCORING_FAILED) | |
| spans = {key: int(metrics[key]) for key in SPAN_KEYS} | |
| label = label_of(normalize(output), normalize(reference), spans["tp"]) | |
| return RowScore( | |
| label=label, | |
| score_exact=LABEL_SCORES[label], | |
| lcs_f1=float(metrics["f1"]), | |
| spans=spans, | |
| ) | |