Spaces:
Running
Running
File size: 9,030 Bytes
3dfc93b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 | """case μ 체 μ¬μ±μ β `solar-eval score proofread-diff --case a/c --all` (κ³ν 2026-09-26 L3b Β§3.2Β·D6).
λ―Έλ¬μ `accounts/<a>/<c>/eval/bundles` μμ run_id λ³ μ΅μ μ»€λ° λ²λ€μ `poc_bundle.adopt` λ‘ κ³ λ₯΄κ³ (μ΅μ μ΄ invalid λ©΄ μ΄μ μΌλ‘
λμ΄κ°μ§ μκ³ `latest_bundle_invalid` skip), μ±νλ λ²λ€λ§λ€ `rescore_run_bundle` μ λΆλ₯Έλ€.
- `bundles` μμ λλ ν λ¦¬κ° μμΌλ©΄ case μ€λ₯ `case_area_missing`(ν΄μν κ²½λ‘λ₯Ό μ£λλ€) β case μ€νλ `--mirror-root` μΈ΅
μ΄κΈλ¨(Β§2.1 "λ―Έλ¬ λ£¨νΈ μΈ΅")μ΄ "ν μΌ μμΒ·μ’
λ£ 0" μΌλ‘ μ¨μ§ μκ².
- μμμ μλλ° μ»€λ°λ run λ²λ€μ΄ 0 μ΄λ©΄ case μ€λ₯ `case_area_empty`(ν΄μν κ²½λ‘) β κ°μ μ΄μ λ€(κ³ν 2026-09-27 L6 Β§2.11,
L3b νμ 3). run νν°κ° μμ΄λ μ΄ νμ μ΄ λ¨Όμ λ€.
- μμμ λ§€λνμ€νΈλ₯Ό λͺ» μ½λ λ²λ€μ΄ νλλΌλ μμΌλ©΄(`AdoptError`) case μ μ²΄κ° μ€λ₯λ€ β μ΄λ run μ λ²λ€μΈμ§ λͺ¨λ₯΄λ μ±λ‘
μ±ννμ§ μλλ€(fail closed). run νν°κ° μμ΄λ μμ μ 체λ₯Ό μ½λλ€ β νν°κ° fail closed λ₯Ό μ½νκ² λ§λ€μ§ μκ².
- **run νν°**(`run_ids`, L6 Β§2.11 β μμ§ μ’
λ£ ν
μ΄ run νλλ§ μ¬μ±μ νλ€): μ±ν λͺ©λ‘(run_id λ³ μ»€λ° λ²λ€)μ κ±°λ₯Έλ€. μ±ν
κ·μΉμ νν° μλ case λͺ¨λμ κ°λ€. μμ²ν run μ΄ μμμ μμΌλ©΄ run λ§λ€ μ€λ₯ `run_not_in_mirror`. κ° κ·μΉ
(`is_valid_run_id` β 곡백λ§μ΄ μλΒ·μ μ΄ λ¬Έμ μμΒ·512μ μ΄ν)μ CLI `--run` κ³Ό μ΄ ν¨μ κ²½κ³κ° ν¨κ» μ΄λ€.
- run νλμ μ€λ₯(`RescoreError`)λ κ·Έ run λ§ μ€λ₯λ‘ μΈκ³ λ€μ run μ κ³μνλ€ β μ’
λ£ μ½λλ CLI κ° `errors` λ‘ μ νλ€.
- νκ°κΈ° λ¬Έμλ case ν λ²μ ν λ² λ¬λ€(λͺ¨λ run μ΄ κ°μ `evaluator_config_hash` λ₯Ό μ΄λ€).
"""
from __future__ import annotations
import re
from collections import Counter
from collections.abc import Collection, Mapping, Sequence
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
from typing import Any
from poc_bundle.adopt import AdoptError, Committed, adopt_run_bundle, committed_run_bundles
from solar_eval.rescore.evaluator_doc import current_evaluator_document
from solar_eval.rescore.from_bundle import (
ERR_ADOPT,
SKIP_LATEST_INVALID,
RescoreError,
RescoreOutcome,
rescore_run_bundle,
)
from solar_eval.rescore.paths import BUNDLES_AREA, mirror_area_dir, resolve_mirror_root
#: case μ€λ₯ β λ―Έλ¬ `bundles` μμμ΄ μλ€(case μ€νΒ·`--mirror-root` μΈ΅ μ΄κΈλ¨). μ무κ²λ μ ν μ± μ±κ³΅νμ§ μκ².
ERR_CASE_AREA_MISSING = "case_area_missing"
#: case μ€λ₯ β `bundles` μμμ μλλ° μ»€λ°λ run λ²λ€μ΄ 0 μ΄λ€(L6 Β§2.11).
ERR_CASE_AREA_EMPTY = "case_area_empty"
#: run μ€λ₯ β run νν°λ‘ μμ²ν run μ μ»€λ° λ²λ€μ΄ λ―Έλ¬ μμμ μλ€(L6 Β§2.11).
ERR_RUN_NOT_IN_MIRROR = "run_not_in_mirror"
#: run νν° κ° νλμ κΈμ μ μν(μ½λ ν¬μΈνΈ) β ES `hub ingest --expect-run` κ³Ό κ°λ€(L6 Β§2.9).
MAX_RUN_ID_LENGTH = 512
# C0Β·DELΒ·C1 μ μ΄ λ¬Έμ β ES `--expect-run` κ³Ό κ°μ λ²μ. stderr `error <run_id> β¦` μ€μ μμ‘°νμ§ λͺ»νκ² νλ€
_CONTROL_CHARACTER = re.compile(r"[\x00-\x1f\x7f-\x9f]")
def is_valid_run_id(value: object) -> bool:
"""run νν° κ° νλ β 곡백λ§μ΄ μλ λ¬Έμμ΄, μ μ΄ λ¬Έμ μμ, `MAX_RUN_ID_LENGTH` μ μ΄ν(CLI `--run` κ³Ό κ°μ κ·μΉ)."""
return (
isinstance(value, str)
and bool(value.strip())
and len(value) <= MAX_RUN_ID_LENGTH
and _CONTROL_CHARACTER.search(value) is None
)
@dataclass(frozen=True)
class RunFailure:
"""run νλ(λλ case μ 체, `run_id=None`)μ μ€λ₯ β idΒ·μ½λμ, μμΌλ©΄ κ²½λ‘λ§(λ©μμ§ μλ¬Έμ μ£μ§ μλλ€)."""
run_id: str | None
code: str
detail: str | None = None
@dataclass
class CaseSummary:
"""case μ¬μ±μ κ²°κ³Ό β run κ²°κ³Ό λͺ©λ‘κ³Ό μ€λ₯."""
outcomes: list[RescoreOutcome] = field(default_factory=list)
failures: list[RunFailure] = field(default_factory=list)
def to_json(self) -> dict[str, Any]:
"""Β§2.1 μμ½ μ€ `{written, exists, skipped:{code:n}, errors, n_original_missing}`."""
statuses = Counter(o.status for o in self.outcomes)
skipped = Counter(o.code for o in self.outcomes if o.status == "skip")
return {
"written": statuses.get("written", 0),
"exists": statuses.get("exists", 0),
"skipped": dict(sorted(skipped.items())),
"errors": len(self.failures),
"n_original_missing": sum(o.n_original_missing for o in self.outcomes),
}
def rescore_case(
account: str,
case: str,
*,
mirror_root: Path | None = None,
out_dir: Path | None = None,
now: datetime | None = None,
repo_root: Path | None = None,
run_ids: Collection[str] | None = None,
) -> CaseSummary:
"""λ―Έλ¬μ ν case λ₯Ό μ¬μ±μ νλ€. `out_dir` κ° None μ΄λ©΄ λ―Έλ¬ `evaluations` μ λλλ€(`mirror_bundle`).
`mirror_root` λ λ―Έλ¬ λ²ν·μ λΆλͺ¨(κΈ°λ³Έ λ°μ΄ν° ν `blob-mirror`) β μ
λ ₯ run λ²λ€λ μ¬κΈ°μ μ½λλ€.
`run_ids` λ₯Ό μ£Όλ©΄ κ·Έ run λ€λ§ μ¬μ±μ νλ€(μλ run μ `run_not_in_mirror`). λ¬Έμμ΄ νλ(`TypeError`)λ λΉ λͺ¨μΒ·
`is_valid_run_id` λ₯Ό λͺ» λλ μμ(`ValueError`)λ νΈμΆμ λ²κ·ΈλΌ μ¦μ κ±°λΆνλ€ β κΈμ λ¨μ νν°λ "ν μΌ μμΒ·μ±κ³΅",
stderr μ€ μμ‘°κ° λμ§ μκ².
"""
wanted = _checked_run_ids(run_ids)
summary = CaseSummary()
by_run = _committed_by_run(mirror_root, account, case, summary)
if by_run is None:
return summary
selected = sorted(by_run) if wanted is None else sorted(wanted & by_run.keys())
if wanted is not None:
missing = sorted(wanted - by_run.keys())
summary.failures.extend(RunFailure(r, ERR_RUN_NOT_IN_MIRROR) for r in missing)
evaluator = current_evaluator_document()
root = resolve_mirror_root(mirror_root)
for run_id in selected:
_rescore_one(run_id, by_run[run_id], summary, out_dir, root, now, evaluator, repo_root)
return summary
def _checked_run_ids(run_ids: Collection[str] | None) -> frozenset[str] | None:
"""run νν° κ²½κ³ κ²μ¬ β None(νν° μμ) λλ `is_valid_run_id` λ₯Ό λλ κ°μ λΉμ΄ μμ§ μμ λͺ¨μ."""
if run_ids is None:
return None
if isinstance(run_ids, str | bytes):
raise TypeError("run_ids must be a collection of run ids, not a single string")
wanted = frozenset(run_ids)
if not wanted:
raise ValueError("run_ids must not be empty (pass None for every run)")
if not all(is_valid_run_id(r) for r in wanted):
raise ValueError(
"run_ids must be non-blank strings without control characters,"
f" at most {MAX_RUN_ID_LENGTH} characters"
)
return wanted
def _committed_by_run(
mirror_root: Path | None, account: str, case: str, summary: CaseSummary
) -> Mapping[str, Sequence[Committed]] | None:
"""μμμ μ»€λ° run λ²λ€(run_id λ³). case μ€λ₯λ©΄ `summary` μ μ£κ³ None."""
bundles_dir = mirror_area_dir(mirror_root, account, case, BUNDLES_AREA)
if not bundles_dir.is_dir():
# `committed_run_bundles` λ μλ μμμ λΉ dict λ‘ λ³Έλ€ β κ·Έλλ‘ λλ©΄ μμ½μ΄ μ λΆ 0 μΈ μ± μ’
λ£ 0 μ΄λ€
summary.failures.append(RunFailure(None, ERR_CASE_AREA_MISSING, str(bundles_dir)))
return None
try:
by_run = committed_run_bundles(bundles_dir)
except AdoptError:
summary.failures.append(RunFailure(None, ERR_ADOPT))
return None
if not by_run:
# μλλ° λΉμλ€ β λ°±ν μ μ΄κ±°λ μμ κ²½λ‘κ° μ΄κΈλ¬λ€. ν μΌ μμΒ·μ’
λ£ 0 μΌλ‘ μ¨κΈ°μ§ μλλ€
summary.failures.append(RunFailure(None, ERR_CASE_AREA_EMPTY, str(bundles_dir)))
return None
return by_run
def _rescore_one(
run_id: str,
cands: Sequence[Committed],
summary: CaseSummary,
out_dir: Path | None,
root: Path,
now: datetime | None,
evaluator: Mapping[str, Any],
repo_root: Path | None,
) -> None:
"""run νλ β μ΅μ λ²λ€ μ±ν(invalid λ©΄ skip) λ€ μ¬μ±μ . κ²°κ³ΌΒ·μ€λ₯λ₯Ό `summary` μ μ£λλ€."""
adoption = adopt_run_bundle(cands)
if adoption.status != "ok":
summary.outcomes.append(RescoreOutcome("skip", run_id, code=SKIP_LATEST_INVALID))
return
try:
outcome = rescore_run_bundle(
adoption.committed.path,
out_dir,
mirror_root=None if out_dir is not None else root,
now=now,
evaluator=evaluator,
repo_root=repo_root,
)
except RescoreError as exc:
summary.failures.append(RunFailure(exc.run_id or run_id, exc.code))
return
summary.outcomes.append(outcome)
|