Spaces:
Running
Running
Download solar_eval/rescore/evaluator_doc.py from dev-strender/proofread-demo: direct link, hf CLI and curl.
- Browser
- Download file 5.11 kB
-
https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/rescore/evaluator_doc.py
- Command line
-
hf download hf://spaces/dev-strender/proofread-demo/solar_eval/rescore/evaluator_doc.py
-
curl -L -o evaluator_doc.py https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/rescore/evaluator_doc.py
5.11 kB
| """`proofread_diff@2` νκ°κΈ° λ¬Έμμ ꡬν ν΄μ (κ³ν 2026-09-26 L3b Β§2.4Β·MB1). | |
| - `impl_hash` = `json_fingerprint(["proofread-diff-impl/v1", closure_hash])` β `closure_hash` λ `proofread_diff_v2` μμ μμν | |
| AST import νν¬(`core/snapshot/impl_hash.py`, μ€ν Β·νκ°κΈ° ν΄μμ κ°μ κ·μΉ)μ νμΌ λ°μ΄νΈ μ§λ¬Έμ΄λ€. `dependency_lock` μ | |
| μμ§ μλλ€: νν¬κ° νμ€ λΌμ΄λΈλ¬λ¦¬Β·`solar_eval` λ°μ import νμ§ μμμ ν μ€νΈκ° κ³ μ νκ³ (`test_impl_hash.py`), μλνν°κ° | |
| λ€μ΄μ€λ λ μλλ€(μ΄λ¦° μ§λ¬Έ Q-L3b-1). | |
| - **νν¬λ λνμ§ μλλ€**(κ³ν 2026-09-27 L6 C-L6-7Β·Q-L6-8, L3b νμ 6). β `drafts.py` λ `poc_bundle` μ import ν΄ λ£μΌλ©΄ | |
| μλνν°κ° νν¬μ λ€μ΄μ `dependency_lock` μ΄ νμνλ€(U-L3b-4 λ‘ λ£μ§ μκΈ°λ‘ νλ€). `evaluators/lcs_diff.py` λ νν¬ λ°μ΄μ§λ§ | |
| μ¬μ±μ κ²½λ‘ **μ**μ΄λ€ β `drafts.legacy_span_scores` κ° `LCSDiffEvaluator.aggregate` λ‘ `producer_aggregates.json` μ `spans` | |
| μ§κ³λ₯Ό λΈλ€. β‘ νν¬λ₯Ό λ°κΎΈλ©΄ `evaluator_config_hash` κ° λ°λμ΄ κΈ°μ‘΄ μ¬μ±μ μ΄ μ νκ°κΈ°λ‘ κ°λΌμ§λ€. β’ λ νμΌμ λ³νλ | |
| `from_bundle._same_as` κ° νμ νκ³Ό `producer_aggregates.json` λ°μ΄νΈλ₯Ό ν¨κ» λΉκ΅ν΄ κ°μ `evaluation_run_id` μμ | |
| `supersedes` λ‘ μ‘λλ€ β `test_impl_hash.py` κ° μ μ κ³ μ νλ€(νμ νλ§ λ°λ κ²½μ°Β·μ§κ³λ§ λ°λ κ²½μ° κ°κ°). | |
| - `metrics` λ λ²‘ν° `scoring/bundles/02-proofread-diff-v2-shape` μ **κ°μ λ°°μ΄**μ΄λ€(ν μ€νΈκ° 벑ν°μ λμ‘°νλ€). λ¬Έμκ° λ²‘ν°μ | |
| λ€λ₯Έ μ μ `impl_hash`Β·`params.original_field`Β·`provenance` μ μ΄λ€. | |
| """ | |
| from __future__ import annotations | |
| import copy | |
| from typing import Any | |
| from poc_bundle.canonical import json_fingerprint | |
| from solar_eval._version import __version__ | |
| from solar_eval.core.snapshot.impl_hash import ImplRoots, closure_hash, default_roots | |
| from solar_eval.rescore.proofread_diff_v2 import ( | |
| LABEL_CORRECT, | |
| LABEL_SCORES, | |
| LABELS, | |
| NORMALIZERS, | |
| ORIGINAL_FIELD, | |
| ) | |
| #: νν¬ μμ λͺ¨λ β νκ°κΈ° λ¬Έμ `impl.name` κ³Ό κ°λ€. | |
| IMPL_START = "solar_eval.rescore.proofread_diff_v2" | |
| #: `impl_hash` νν 머리μ λλ©μΈ νκ·Έ. ννμ λ°κΎΈλ©΄ μ νκ·Έλ€. | |
| IMPL_TAG = "proofread-diff-impl/v1" | |
| IMPL_VERSION = "2" | |
| EVALUATOR_KIND = "proofread_diff" | |
| #: μ¬λμ΄ λΆλ₯΄λ νκ°κΈ° μ΄λ¦ β νμ `actor` λ `solar_eval.<ALIAS>`. | |
| ALIAS = "proofread_diff@2" | |
| ACTOR = f"solar_eval.{ALIAS}" | |
| #: `params.noanswer` β μ κ·νν μΆλ ₯μ΄ λΉ λ¬Έμμ΄μ΄λ©΄ `noanswer`(D3). | |
| NOANSWER_RULE = "empty_output" | |
| #: μ§ν μ μΈ β λ²‘ν° 02 μ κ°μ λ°°μ΄(ν€ μμκΉμ§). `exact` λ categorical, `lcs_f1` μ continuous, `spans` λ counts. | |
| METRICS: list[dict[str, Any]] = [ | |
| { | |
| "key": "exact", | |
| "level": "item", | |
| "score_type": "categorical", | |
| "unit": "ratio", | |
| "levels": list(LABELS), | |
| "label_scores": dict(LABEL_SCORES), | |
| "passing_labels": [LABEL_CORRECT], | |
| "aggregations": ["mean"], | |
| "primary": True, | |
| "primary_aggregation": "mean", | |
| }, | |
| { | |
| "key": "lcs_f1", | |
| "level": "item", | |
| "score_type": "continuous", | |
| "unit": "percent", | |
| "min": 0, | |
| "max": 100, | |
| "aggregations": ["mean"], | |
| }, | |
| { | |
| "key": "spans", | |
| "level": "item", | |
| "score_type": "counts", | |
| "unit": "percent", | |
| "count_roles": {"tp": "tp", "fp": "fd", "missing": "fn", "redundant": "fa"}, | |
| "empty_policy": "one", | |
| "scale": 100, | |
| "aggregations": ["micro_prf@1:f1", "micro_prf@1:precision", "micro_prf@1:recall"], | |
| }, | |
| ] | |
| def implementation_hash(roots: ImplRoots | None = None) -> tuple[str, list[str]]: | |
| """(`impl_hash`, μ λ ¬λ νν¬ λΌλ²¨). `roots` λ₯Ό μ£Όλ©΄ κ·Έ ν΄μ 루νΈλ‘ λ¬λ€(ν μ€νΈκ° tmp μ¬λ³Έμ λΌμ΄λ€).""" | |
| closure, labels = closure_hash([IMPL_START], roots or default_roots()) | |
| return json_fingerprint([IMPL_TAG, closure]), labels | |
| def evaluator_document(impl_hash: str) -> dict[str, Any]: | |
| """`evaluator/v1` λ¬Έμ(Β§2.4). ν΄μλ `config` μλ§ λ¨λ―λ‘ `provenance` μ λ²μ νκΈ°λ id λ₯Ό λ°κΎΈμ§ μλλ€.""" | |
| return { | |
| "schema": "evaluator/v1", | |
| "config": { | |
| "kind": EVALUATOR_KIND, | |
| "source_kind": "code", | |
| "impl": {"name": IMPL_START, "version": IMPL_VERSION, "impl_hash": impl_hash}, | |
| "params": { | |
| "exact_compare": {"normalizers": list(NORMALIZERS)}, | |
| "noanswer": NOANSWER_RULE, | |
| "original_field": ORIGINAL_FIELD, | |
| }, | |
| "metrics": copy.deepcopy(METRICS), | |
| }, | |
| "provenance": {"alias": ALIAS, "version_label": f"solar-eval {__version__}"}, | |
| } | |
| def current_evaluator_document() -> dict[str, Any]: | |
| """μ§κΈ λμ€ν¬μ ꡬνμΌλ‘ λ¬ νκ°κΈ° λ¬Έμ β μ¬μ±μ ν λ²(case μ 체)μ ν λ² λΆλ₯Έλ€.""" | |
| impl_hash, _ = implementation_hash() | |
| return evaluator_document(impl_hash) | |