proofread-demo / solar_eval /rescore /evaluator_doc.py
dev-strender's picture
feat: switch demo to FT + Opus article review (pipeline_exp_261021_ft_hybrid)
3dfc93b verified
Raw History Blame Contribute Delete
5.11 kB
"""`proofread_diff@2` 평가기 λ¬Έμ„œμ™€ κ΅¬ν˜„ ν•΄μ‹œ (κ³„νš 2026-09-26 L3b Β§2.4Β·MB1).
- `impl_hash` = `json_fingerprint(["proofread-diff-impl/v1", closure_hash])` β€” `closure_hash` λŠ” `proofread_diff_v2` μ—μ„œ μ‹œμž‘ν•œ
AST import 폐포(`core/snapshot/impl_hash.py`, μŠ€ν…Β·ν‰κ°€κΈ° ν•΄μ‹œμ™€ 같은 κ·œμΉ™)의 파일 λ°”μ΄νŠΈ 지문이닀. `dependency_lock` 은
μ„žμ§€ μ•ŠλŠ”λ‹€: 폐포가 ν‘œμ€€ 라이브러리·`solar_eval` 밖을 import ν•˜μ§€ μ•ŠμŒμ„ ν…ŒμŠ€νŠΈκ°€ κ³ μ •ν•˜κ³ (`test_impl_hash.py`), μ„œλ“œνŒŒν‹°κ°€
λ“€μ–΄μ˜€λŠ” λ‚  μ„žλŠ”λ‹€(μ—΄λ¦° 질문 Q-L3b-1).
- **νν¬λŠ” λ„“νžˆμ§€ μ•ŠλŠ”λ‹€**(κ³„νš 2026-09-27 L6 C-L6-7Β·Q-L6-8, L3b 후속 6). β‘  `drafts.py` λŠ” `poc_bundle` 을 import ν•΄ λ„£μœΌλ©΄
μ„œλ“œνŒŒν‹°κ°€ 폐포에 듀어와 `dependency_lock` 이 ν•„μš”ν•˜λ‹€(U-L3b-4 둜 λ„£μ§€ μ•ŠκΈ°λ‘œ ν–ˆλ‹€). `evaluators/lcs_diff.py` λŠ” 폐포 λ°–μ΄μ§€λ§Œ
μž¬μ±„μ  경둜 **μ•ˆ**이닀 β€” `drafts.legacy_span_scores` κ°€ `LCSDiffEvaluator.aggregate` 둜 `producer_aggregates.json` 의 `spans`
집계λ₯Ό λ‚Έλ‹€. β‘‘ 폐포λ₯Ό λ°”κΎΈλ©΄ `evaluator_config_hash` κ°€ λ°”λ€Œμ–΄ κΈ°μ‘΄ μž¬μ±„μ μ΄ μƒˆ ν‰κ°€κΈ°λ‘œ κ°ˆλΌμ§„λ‹€. β‘’ 두 파일의 λ³€ν™”λŠ”
`from_bundle._same_as` κ°€ νŒμ • ν–‰κ³Ό `producer_aggregates.json` λ°”μ΄νŠΈλ₯Ό ν•¨κ»˜ 비ꡐ해 같은 `evaluation_run_id` μ•ˆμ˜
`supersedes` 둜 μž‘λŠ”λ‹€ β€” `test_impl_hash.py` κ°€ 셋을 κ³ μ •ν•œλ‹€(νŒμ • ν–‰λ§Œ 바뀐 κ²½μš°Β·μ§‘κ³„λ§Œ 바뀐 경우 각각).
- `metrics` λŠ” 벑터 `scoring/bundles/02-proofread-diff-v2-shape` 와 **같은 λ°°μ—΄**이닀(ν…ŒμŠ€νŠΈκ°€ 벑터와 λŒ€μ‘°ν•œλ‹€). λ¬Έμ„œκ°€ 벑터와
λ‹€λ₯Έ 점은 `impl_hash`Β·`params.original_field`Β·`provenance` 셋이닀.
"""
from __future__ import annotations
import copy
from typing import Any
from poc_bundle.canonical import json_fingerprint
from solar_eval._version import __version__
from solar_eval.core.snapshot.impl_hash import ImplRoots, closure_hash, default_roots
from solar_eval.rescore.proofread_diff_v2 import (
LABEL_CORRECT,
LABEL_SCORES,
LABELS,
NORMALIZERS,
ORIGINAL_FIELD,
)
#: 폐포 μ‹œμž‘ λͺ¨λ“ˆ β€” 평가기 λ¬Έμ„œ `impl.name` κ³Ό κ°™λ‹€.
IMPL_START = "solar_eval.rescore.proofread_diff_v2"
#: `impl_hash` νŠœν”Œ 머리의 도메인 νƒœκ·Έ. νŠœν”Œμ„ λ°”κΎΈλ©΄ μƒˆ νƒœκ·Έλ‹€.
IMPL_TAG = "proofread-diff-impl/v1"
IMPL_VERSION = "2"
EVALUATOR_KIND = "proofread_diff"
#: μ‚¬λžŒμ΄ λΆ€λ₯΄λŠ” 평가기 이름 β€” νŒμ • `actor` λŠ” `solar_eval.<ALIAS>`.
ALIAS = "proofread_diff@2"
ACTOR = f"solar_eval.{ALIAS}"
#: `params.noanswer` β€” μ •κ·œν™”ν•œ 좜λ ₯이 빈 λ¬Έμžμ—΄μ΄λ©΄ `noanswer`(D3).
NOANSWER_RULE = "empty_output"
#: μ§€ν‘œ μ„ μ–Έ β€” 벑터 02 와 같은 λ°°μ—΄(ν‚€ μˆœμ„œκΉŒμ§€). `exact` λŠ” categorical, `lcs_f1` 은 continuous, `spans` λŠ” counts.
METRICS: list[dict[str, Any]] = [
{
"key": "exact",
"level": "item",
"score_type": "categorical",
"unit": "ratio",
"levels": list(LABELS),
"label_scores": dict(LABEL_SCORES),
"passing_labels": [LABEL_CORRECT],
"aggregations": ["mean"],
"primary": True,
"primary_aggregation": "mean",
},
{
"key": "lcs_f1",
"level": "item",
"score_type": "continuous",
"unit": "percent",
"min": 0,
"max": 100,
"aggregations": ["mean"],
},
{
"key": "spans",
"level": "item",
"score_type": "counts",
"unit": "percent",
"count_roles": {"tp": "tp", "fp": "fd", "missing": "fn", "redundant": "fa"},
"empty_policy": "one",
"scale": 100,
"aggregations": ["micro_prf@1:f1", "micro_prf@1:precision", "micro_prf@1:recall"],
},
]
def implementation_hash(roots: ImplRoots | None = None) -> tuple[str, list[str]]:
"""(`impl_hash`, μ •λ ¬λœ 폐포 라벨). `roots` λ₯Ό μ£Όλ©΄ κ·Έ 해석 루트둜 λœ¬λ‹€(ν…ŒμŠ€νŠΈκ°€ tmp 사본을 λΌμš΄λ‹€)."""
closure, labels = closure_hash([IMPL_START], roots or default_roots())
return json_fingerprint([IMPL_TAG, closure]), labels
def evaluator_document(impl_hash: str) -> dict[str, Any]:
"""`evaluator/v1` λ¬Έμ„œ(Β§2.4). ν•΄μ‹œλŠ” `config` μ—λ§Œ λœ¨λ―€λ‘œ `provenance` 의 버전 ν‘œκΈ°λŠ” id λ₯Ό λ°”κΎΈμ§€ μ•ŠλŠ”λ‹€."""
return {
"schema": "evaluator/v1",
"config": {
"kind": EVALUATOR_KIND,
"source_kind": "code",
"impl": {"name": IMPL_START, "version": IMPL_VERSION, "impl_hash": impl_hash},
"params": {
"exact_compare": {"normalizers": list(NORMALIZERS)},
"noanswer": NOANSWER_RULE,
"original_field": ORIGINAL_FIELD,
},
"metrics": copy.deepcopy(METRICS),
},
"provenance": {"alias": ALIAS, "version_label": f"solar-eval {__version__}"},
}
def current_evaluator_document() -> dict[str, Any]:
"""μ§€κΈˆ λ””μŠ€ν¬μ˜ κ΅¬ν˜„μœΌλ‘œ 뜬 평가기 λ¬Έμ„œ β€” μž¬μ±„μ  ν•œ 번(case 전체)에 ν•œ 번 λΆ€λ₯Έλ‹€."""
impl_hash, _ = implementation_hash()
return evaluator_document(impl_hash)