Spaces:
Running
Running
Download solar_eval/rescore/case.py from dev-strender/proofread-demo: direct link, hf CLI and curl.
- Browser
- Download file 9.03 kB
-
https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/rescore/case.py
- Command line
-
hf download hf://spaces/dev-strender/proofread-demo/solar_eval/rescore/case.py
-
curl -L -o case.py https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/rescore/case.py
9.03 kB
| """case μ 체 μ¬μ±μ β `solar-eval score proofread-diff --case a/c --all` (κ³ν 2026-09-26 L3b Β§3.2Β·D6). | |
| λ―Έλ¬μ `accounts/<a>/<c>/eval/bundles` μμ run_id λ³ μ΅μ μ»€λ° λ²λ€μ `poc_bundle.adopt` λ‘ κ³ λ₯΄κ³ (μ΅μ μ΄ invalid λ©΄ μ΄μ μΌλ‘ | |
| λμ΄κ°μ§ μκ³ `latest_bundle_invalid` skip), μ±νλ λ²λ€λ§λ€ `rescore_run_bundle` μ λΆλ₯Έλ€. | |
| - `bundles` μμ λλ ν λ¦¬κ° μμΌλ©΄ case μ€λ₯ `case_area_missing`(ν΄μν κ²½λ‘λ₯Ό μ£λλ€) β case μ€νλ `--mirror-root` μΈ΅ | |
| μ΄κΈλ¨(Β§2.1 "λ―Έλ¬ λ£¨νΈ μΈ΅")μ΄ "ν μΌ μμΒ·μ’ λ£ 0" μΌλ‘ μ¨μ§ μκ². | |
| - μμμ μλλ° μ»€λ°λ run λ²λ€μ΄ 0 μ΄λ©΄ case μ€λ₯ `case_area_empty`(ν΄μν κ²½λ‘) β κ°μ μ΄μ λ€(κ³ν 2026-09-27 L6 Β§2.11, | |
| L3b νμ 3). run νν°κ° μμ΄λ μ΄ νμ μ΄ λ¨Όμ λ€. | |
| - μμμ λ§€λνμ€νΈλ₯Ό λͺ» μ½λ λ²λ€μ΄ νλλΌλ μμΌλ©΄(`AdoptError`) case μ μ²΄κ° μ€λ₯λ€ β μ΄λ run μ λ²λ€μΈμ§ λͺ¨λ₯΄λ μ±λ‘ | |
| μ±ννμ§ μλλ€(fail closed). run νν°κ° μμ΄λ μμ μ 체λ₯Ό μ½λλ€ β νν°κ° fail closed λ₯Ό μ½νκ² λ§λ€μ§ μκ². | |
| - **run νν°**(`run_ids`, L6 Β§2.11 β μμ§ μ’ λ£ ν μ΄ run νλλ§ μ¬μ±μ νλ€): μ±ν λͺ©λ‘(run_id λ³ μ»€λ° λ²λ€)μ κ±°λ₯Έλ€. μ±ν | |
| κ·μΉμ νν° μλ case λͺ¨λμ κ°λ€. μμ²ν run μ΄ μμμ μμΌλ©΄ run λ§λ€ μ€λ₯ `run_not_in_mirror`. κ° κ·μΉ | |
| (`is_valid_run_id` β 곡백λ§μ΄ μλΒ·μ μ΄ λ¬Έμ μμΒ·512μ μ΄ν)μ CLI `--run` κ³Ό μ΄ ν¨μ κ²½κ³κ° ν¨κ» μ΄λ€. | |
| - run νλμ μ€λ₯(`RescoreError`)λ κ·Έ run λ§ μ€λ₯λ‘ μΈκ³ λ€μ run μ κ³μνλ€ β μ’ λ£ μ½λλ CLI κ° `errors` λ‘ μ νλ€. | |
| - νκ°κΈ° λ¬Έμλ case ν λ²μ ν λ² λ¬λ€(λͺ¨λ run μ΄ κ°μ `evaluator_config_hash` λ₯Ό μ΄λ€). | |
| """ | |
| from __future__ import annotations | |
| import re | |
| from collections import Counter | |
| from collections.abc import Collection, Mapping, Sequence | |
| from dataclasses import dataclass, field | |
| from datetime import datetime | |
| from pathlib import Path | |
| from typing import Any | |
| from poc_bundle.adopt import AdoptError, Committed, adopt_run_bundle, committed_run_bundles | |
| from solar_eval.rescore.evaluator_doc import current_evaluator_document | |
| from solar_eval.rescore.from_bundle import ( | |
| ERR_ADOPT, | |
| SKIP_LATEST_INVALID, | |
| RescoreError, | |
| RescoreOutcome, | |
| rescore_run_bundle, | |
| ) | |
| from solar_eval.rescore.paths import BUNDLES_AREA, mirror_area_dir, resolve_mirror_root | |
| #: case μ€λ₯ β λ―Έλ¬ `bundles` μμμ΄ μλ€(case μ€νΒ·`--mirror-root` μΈ΅ μ΄κΈλ¨). μ무κ²λ μ ν μ± μ±κ³΅νμ§ μκ². | |
| ERR_CASE_AREA_MISSING = "case_area_missing" | |
| #: case μ€λ₯ β `bundles` μμμ μλλ° μ»€λ°λ run λ²λ€μ΄ 0 μ΄λ€(L6 Β§2.11). | |
| ERR_CASE_AREA_EMPTY = "case_area_empty" | |
| #: run μ€λ₯ β run νν°λ‘ μμ²ν run μ μ»€λ° λ²λ€μ΄ λ―Έλ¬ μμμ μλ€(L6 Β§2.11). | |
| ERR_RUN_NOT_IN_MIRROR = "run_not_in_mirror" | |
| #: run νν° κ° νλμ κΈμ μ μν(μ½λ ν¬μΈνΈ) β ES `hub ingest --expect-run` κ³Ό κ°λ€(L6 Β§2.9). | |
| MAX_RUN_ID_LENGTH = 512 | |
| # C0Β·DELΒ·C1 μ μ΄ λ¬Έμ β ES `--expect-run` κ³Ό κ°μ λ²μ. stderr `error <run_id> β¦` μ€μ μμ‘°νμ§ λͺ»νκ² νλ€ | |
| _CONTROL_CHARACTER = re.compile(r"[\x00-\x1f\x7f-\x9f]") | |
| def is_valid_run_id(value: object) -> bool: | |
| """run νν° κ° νλ β 곡백λ§μ΄ μλ λ¬Έμμ΄, μ μ΄ λ¬Έμ μμ, `MAX_RUN_ID_LENGTH` μ μ΄ν(CLI `--run` κ³Ό κ°μ κ·μΉ).""" | |
| return ( | |
| isinstance(value, str) | |
| and bool(value.strip()) | |
| and len(value) <= MAX_RUN_ID_LENGTH | |
| and _CONTROL_CHARACTER.search(value) is None | |
| ) | |
| class RunFailure: | |
| """run νλ(λλ case μ 체, `run_id=None`)μ μ€λ₯ β idΒ·μ½λμ, μμΌλ©΄ κ²½λ‘λ§(λ©μμ§ μλ¬Έμ μ£μ§ μλλ€).""" | |
| run_id: str | None | |
| code: str | |
| detail: str | None = None | |
| class CaseSummary: | |
| """case μ¬μ±μ κ²°κ³Ό β run κ²°κ³Ό λͺ©λ‘κ³Ό μ€λ₯.""" | |
| outcomes: list[RescoreOutcome] = field(default_factory=list) | |
| failures: list[RunFailure] = field(default_factory=list) | |
| def to_json(self) -> dict[str, Any]: | |
| """Β§2.1 μμ½ μ€ `{written, exists, skipped:{code:n}, errors, n_original_missing}`.""" | |
| statuses = Counter(o.status for o in self.outcomes) | |
| skipped = Counter(o.code for o in self.outcomes if o.status == "skip") | |
| return { | |
| "written": statuses.get("written", 0), | |
| "exists": statuses.get("exists", 0), | |
| "skipped": dict(sorted(skipped.items())), | |
| "errors": len(self.failures), | |
| "n_original_missing": sum(o.n_original_missing for o in self.outcomes), | |
| } | |
| def rescore_case( | |
| account: str, | |
| case: str, | |
| *, | |
| mirror_root: Path | None = None, | |
| out_dir: Path | None = None, | |
| now: datetime | None = None, | |
| repo_root: Path | None = None, | |
| run_ids: Collection[str] | None = None, | |
| ) -> CaseSummary: | |
| """λ―Έλ¬μ ν case λ₯Ό μ¬μ±μ νλ€. `out_dir` κ° None μ΄λ©΄ λ―Έλ¬ `evaluations` μ λλλ€(`mirror_bundle`). | |
| `mirror_root` λ λ―Έλ¬ λ²ν·μ λΆλͺ¨(κΈ°λ³Έ λ°μ΄ν° ν `blob-mirror`) β μ λ ₯ run λ²λ€λ μ¬κΈ°μ μ½λλ€. | |
| `run_ids` λ₯Ό μ£Όλ©΄ κ·Έ run λ€λ§ μ¬μ±μ νλ€(μλ run μ `run_not_in_mirror`). λ¬Έμμ΄ νλ(`TypeError`)λ λΉ λͺ¨μΒ· | |
| `is_valid_run_id` λ₯Ό λͺ» λλ μμ(`ValueError`)λ νΈμΆμ λ²κ·ΈλΌ μ¦μ κ±°λΆνλ€ β κΈμ λ¨μ νν°λ "ν μΌ μμΒ·μ±κ³΅", | |
| stderr μ€ μμ‘°κ° λμ§ μκ². | |
| """ | |
| wanted = _checked_run_ids(run_ids) | |
| summary = CaseSummary() | |
| by_run = _committed_by_run(mirror_root, account, case, summary) | |
| if by_run is None: | |
| return summary | |
| selected = sorted(by_run) if wanted is None else sorted(wanted & by_run.keys()) | |
| if wanted is not None: | |
| missing = sorted(wanted - by_run.keys()) | |
| summary.failures.extend(RunFailure(r, ERR_RUN_NOT_IN_MIRROR) for r in missing) | |
| evaluator = current_evaluator_document() | |
| root = resolve_mirror_root(mirror_root) | |
| for run_id in selected: | |
| _rescore_one(run_id, by_run[run_id], summary, out_dir, root, now, evaluator, repo_root) | |
| return summary | |
| def _checked_run_ids(run_ids: Collection[str] | None) -> frozenset[str] | None: | |
| """run νν° κ²½κ³ κ²μ¬ β None(νν° μμ) λλ `is_valid_run_id` λ₯Ό λλ κ°μ λΉμ΄ μμ§ μμ λͺ¨μ.""" | |
| if run_ids is None: | |
| return None | |
| if isinstance(run_ids, str | bytes): | |
| raise TypeError("run_ids must be a collection of run ids, not a single string") | |
| wanted = frozenset(run_ids) | |
| if not wanted: | |
| raise ValueError("run_ids must not be empty (pass None for every run)") | |
| if not all(is_valid_run_id(r) for r in wanted): | |
| raise ValueError( | |
| "run_ids must be non-blank strings without control characters," | |
| f" at most {MAX_RUN_ID_LENGTH} characters" | |
| ) | |
| return wanted | |
| def _committed_by_run( | |
| mirror_root: Path | None, account: str, case: str, summary: CaseSummary | |
| ) -> Mapping[str, Sequence[Committed]] | None: | |
| """μμμ μ»€λ° run λ²λ€(run_id λ³). case μ€λ₯λ©΄ `summary` μ μ£κ³ None.""" | |
| bundles_dir = mirror_area_dir(mirror_root, account, case, BUNDLES_AREA) | |
| if not bundles_dir.is_dir(): | |
| # `committed_run_bundles` λ μλ μμμ λΉ dict λ‘ λ³Έλ€ β κ·Έλλ‘ λλ©΄ μμ½μ΄ μ λΆ 0 μΈ μ± μ’ λ£ 0 μ΄λ€ | |
| summary.failures.append(RunFailure(None, ERR_CASE_AREA_MISSING, str(bundles_dir))) | |
| return None | |
| try: | |
| by_run = committed_run_bundles(bundles_dir) | |
| except AdoptError: | |
| summary.failures.append(RunFailure(None, ERR_ADOPT)) | |
| return None | |
| if not by_run: | |
| # μλλ° λΉμλ€ β λ°±ν μ μ΄κ±°λ μμ κ²½λ‘κ° μ΄κΈλ¬λ€. ν μΌ μμΒ·μ’ λ£ 0 μΌλ‘ μ¨κΈ°μ§ μλλ€ | |
| summary.failures.append(RunFailure(None, ERR_CASE_AREA_EMPTY, str(bundles_dir))) | |
| return None | |
| return by_run | |
| def _rescore_one( | |
| run_id: str, | |
| cands: Sequence[Committed], | |
| summary: CaseSummary, | |
| out_dir: Path | None, | |
| root: Path, | |
| now: datetime | None, | |
| evaluator: Mapping[str, Any], | |
| repo_root: Path | None, | |
| ) -> None: | |
| """run νλ β μ΅μ λ²λ€ μ±ν(invalid λ©΄ skip) λ€ μ¬μ±μ . κ²°κ³ΌΒ·μ€λ₯λ₯Ό `summary` μ μ£λλ€.""" | |
| adoption = adopt_run_bundle(cands) | |
| if adoption.status != "ok": | |
| summary.outcomes.append(RescoreOutcome("skip", run_id, code=SKIP_LATEST_INVALID)) | |
| return | |
| try: | |
| outcome = rescore_run_bundle( | |
| adoption.committed.path, | |
| out_dir, | |
| mirror_root=None if out_dir is not None else root, | |
| now=now, | |
| evaluator=evaluator, | |
| repo_root=repo_root, | |
| ) | |
| except RescoreError as exc: | |
| summary.failures.append(RunFailure(exc.run_id or run_id, exc.code)) | |
| return | |
| summary.outcomes.append(outcome) | |