dev-strender's picture
feat: switch demo to FT + Opus article review (pipeline_exp_261021_ft_hybrid)
3dfc93b verified
Raw History Blame Contribute Delete
9.03 kB
"""case 전체 μž¬μ±„μ  β€” `solar-eval score proofread-diff --case a/c --all` (κ³„νš 2026-09-26 L3b Β§3.2Β·D6).
미러의 `accounts/<a>/<c>/eval/bundles` μ—μ„œ run_id 별 μ΅œμ‹  컀밋 λ²ˆλ“€μ„ `poc_bundle.adopt` 둜 κ³ λ₯΄κ³ (μ΅œμ‹ μ΄ invalid λ©΄ μ΄μ „μœΌλ‘œ
λ„˜μ–΄κ°€μ§€ μ•Šκ³  `latest_bundle_invalid` skip), μ±„νƒλœ λ²ˆλ“€λ§ˆλ‹€ `rescore_run_bundle` 을 λΆ€λ₯Έλ‹€.
- `bundles` μ˜μ—­ 디렉토리가 μ—†μœΌλ©΄ case 였λ₯˜ `case_area_missing`(ν•΄μ„ν•œ 경둜λ₯Ό μ‹£λŠ”λ‹€) β€” case μ˜€νƒ€λ‚˜ `--mirror-root` μΈ΅
어긋남(Β§2.1 "미러 루트 μΈ΅")이 "ν•  일 μ—†μŒΒ·μ’…λ£Œ 0" 으둜 μˆ¨μ§€ μ•Šκ²Œ.
- μ˜μ—­μ€ μžˆλŠ”λ° μ»€λ°‹λœ run λ²ˆλ“€μ΄ 0 이면 case 였λ₯˜ `case_area_empty`(ν•΄μ„ν•œ 경둜) β€” 같은 μ΄μœ λ‹€(κ³„νš 2026-09-27 L6 Β§2.11,
L3b 후속 3). run ν•„ν„°κ°€ μžˆμ–΄λ„ 이 νŒμ •μ΄ λ¨Όμ €λ‹€.
- μ˜μ—­μ— λ§€λ‹ˆνŽ˜μŠ€νŠΈλ₯Ό λͺ» μ½λŠ” λ²ˆλ“€μ΄ ν•˜λ‚˜λΌλ„ 있으면(`AdoptError`) case 전체가 였λ₯˜λ‹€ β€” μ–΄λŠ run 의 λ²ˆλ“€μΈμ§€ λͺ¨λ₯΄λŠ” μ±„λ‘œ
μ±„νƒν•˜μ§€ μ•ŠλŠ”λ‹€(fail closed). run ν•„ν„°κ°€ μžˆμ–΄λ„ μ˜μ—­ 전체λ₯Ό μ½λŠ”λ‹€ β€” ν•„ν„°κ°€ fail closed λ₯Ό μ•½ν•˜κ²Œ λ§Œλ“€μ§€ μ•Šκ²Œ.
- **run ν•„ν„°**(`run_ids`, L6 Β§2.11 β€” μ—”μ§„ μ’…λ£Œ 훅이 run ν•˜λ‚˜λ§Œ μž¬μ±„μ ν•œλ‹€): 채택 λͺ©λ‘(run_id 별 컀밋 λ²ˆλ“€)을 κ±°λ₯Έλ‹€. 채택
κ·œμΉ™μ€ ν•„ν„° μ—†λŠ” case λͺ¨λ“œμ™€ κ°™λ‹€. μš”μ²­ν•œ run 이 μ˜μ—­μ— μ—†μœΌλ©΄ run λ§ˆλ‹€ 였λ₯˜ `run_not_in_mirror`. κ°’ κ·œμΉ™
(`is_valid_run_id` β€” 곡백만이 μ•„λ‹˜Β·μ œμ–΄ 문자 μ—†μŒΒ·512자 μ΄ν•˜)은 CLI `--run` κ³Ό 이 ν•¨μˆ˜ 경계가 ν•¨κ»˜ μ“΄λ‹€.
- run ν•˜λ‚˜μ˜ 였λ₯˜(`RescoreError`)λŠ” κ·Έ run 만 였λ₯˜λ‘œ μ„Έκ³  λ‹€μŒ run 을 κ³„μ†ν•œλ‹€ β€” μ’…λ£Œ μ½”λ“œλŠ” CLI κ°€ `errors` 둜 μ •ν•œλ‹€.
- 평가기 λ¬Έμ„œλŠ” case ν•œ λ²ˆμ— ν•œ 번 λœ¬λ‹€(λͺ¨λ“  run 이 같은 `evaluator_config_hash` λ₯Ό μ“΄λ‹€).
"""
from __future__ import annotations
import re
from collections import Counter
from collections.abc import Collection, Mapping, Sequence
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
from typing import Any
from poc_bundle.adopt import AdoptError, Committed, adopt_run_bundle, committed_run_bundles
from solar_eval.rescore.evaluator_doc import current_evaluator_document
from solar_eval.rescore.from_bundle import (
ERR_ADOPT,
SKIP_LATEST_INVALID,
RescoreError,
RescoreOutcome,
rescore_run_bundle,
)
from solar_eval.rescore.paths import BUNDLES_AREA, mirror_area_dir, resolve_mirror_root
#: case 였λ₯˜ β€” 미러 `bundles` μ˜μ—­μ΄ μ—†λ‹€(case μ˜€νƒ€Β·`--mirror-root` μΈ΅ 어긋남). 아무것도 μ•ˆ ν•œ 채 μ„±κ³΅ν•˜μ§€ μ•Šκ²Œ.
ERR_CASE_AREA_MISSING = "case_area_missing"
#: case 였λ₯˜ β€” `bundles` μ˜μ—­μ€ μžˆλŠ”λ° μ»€λ°‹λœ run λ²ˆλ“€μ΄ 0 이닀(L6 Β§2.11).
ERR_CASE_AREA_EMPTY = "case_area_empty"
#: run 였λ₯˜ β€” run ν•„ν„°λ‘œ μš”μ²­ν•œ run 의 컀밋 λ²ˆλ“€μ΄ 미러 μ˜μ—­μ— μ—†λ‹€(L6 Β§2.11).
ERR_RUN_NOT_IN_MIRROR = "run_not_in_mirror"
#: run ν•„ν„° κ°’ ν•˜λ‚˜μ˜ κΈ€μž 수 μƒν•œ(μ½”λ“œ 포인트) β€” ES `hub ingest --expect-run` κ³Ό κ°™λ‹€(L6 Β§2.9).
MAX_RUN_ID_LENGTH = 512
# C0Β·DELΒ·C1 μ œμ–΄ 문자 β€” ES `--expect-run` κ³Ό 같은 λ²”μœ„. stderr `error <run_id> …` 쀄을 μœ„μ‘°ν•˜μ§€ λͺ»ν•˜κ²Œ ν•œλ‹€
_CONTROL_CHARACTER = re.compile(r"[\x00-\x1f\x7f-\x9f]")
def is_valid_run_id(value: object) -> bool:
"""run ν•„ν„° κ°’ ν•˜λ‚˜ β€” 곡백만이 μ•„λ‹Œ λ¬Έμžμ—΄, μ œμ–΄ 문자 μ—†μŒ, `MAX_RUN_ID_LENGTH` 자 μ΄ν•˜(CLI `--run` κ³Ό 같은 κ·œμΉ™)."""
return (
isinstance(value, str)
and bool(value.strip())
and len(value) <= MAX_RUN_ID_LENGTH
and _CONTROL_CHARACTER.search(value) is None
)
@dataclass(frozen=True)
class RunFailure:
"""run ν•˜λ‚˜(λ˜λŠ” case 전체, `run_id=None`)의 였λ₯˜ β€” idΒ·μ½”λ“œμ™€, 있으면 경둜만(λ©”μ‹œμ§€ 원문은 μ‹£μ§€ μ•ŠλŠ”λ‹€)."""
run_id: str | None
code: str
detail: str | None = None
@dataclass
class CaseSummary:
"""case μž¬μ±„μ  κ²°κ³Ό β€” run κ²°κ³Ό λͺ©λ‘κ³Ό 였λ₯˜."""
outcomes: list[RescoreOutcome] = field(default_factory=list)
failures: list[RunFailure] = field(default_factory=list)
def to_json(self) -> dict[str, Any]:
"""Β§2.1 μš”μ•½ 쀄 `{written, exists, skipped:{code:n}, errors, n_original_missing}`."""
statuses = Counter(o.status for o in self.outcomes)
skipped = Counter(o.code for o in self.outcomes if o.status == "skip")
return {
"written": statuses.get("written", 0),
"exists": statuses.get("exists", 0),
"skipped": dict(sorted(skipped.items())),
"errors": len(self.failures),
"n_original_missing": sum(o.n_original_missing for o in self.outcomes),
}
def rescore_case(
account: str,
case: str,
*,
mirror_root: Path | None = None,
out_dir: Path | None = None,
now: datetime | None = None,
repo_root: Path | None = None,
run_ids: Collection[str] | None = None,
) -> CaseSummary:
"""미러의 ν•œ case λ₯Ό μž¬μ±„μ ν•œλ‹€. `out_dir` κ°€ None 이면 미러 `evaluations` 에 λ†“λŠ”λ‹€(`mirror_bundle`).
`mirror_root` λŠ” 미러 λ²„ν‚·μ˜ λΆ€λͺ¨(κΈ°λ³Έ 데이터 ν™ˆ `blob-mirror`) β€” μž…λ ₯ run λ²ˆλ“€λ„ μ—¬κΈ°μ„œ μ½λŠ”λ‹€.
`run_ids` λ₯Ό μ£Όλ©΄ κ·Έ run λ“€λ§Œ μž¬μ±„μ ν•œλ‹€(μ—†λŠ” run 은 `run_not_in_mirror`). λ¬Έμžμ—΄ ν•˜λ‚˜(`TypeError`)λ‚˜ 빈 λͺ¨μŒΒ·
`is_valid_run_id` λ₯Ό λͺ» λ„˜λŠ” μ›μ†Œ(`ValueError`)λŠ” 호좜자 버그라 μ¦‰μ‹œ κ±°λΆ€ν•œλ‹€ β€” κΈ€μž λ‹¨μœ„ ν•„ν„°λ‚˜ "ν•  일 μ—†μŒΒ·μ„±κ³΅",
stderr 쀄 μœ„μ‘°κ°€ λ˜μ§€ μ•Šκ²Œ.
"""
wanted = _checked_run_ids(run_ids)
summary = CaseSummary()
by_run = _committed_by_run(mirror_root, account, case, summary)
if by_run is None:
return summary
selected = sorted(by_run) if wanted is None else sorted(wanted & by_run.keys())
if wanted is not None:
missing = sorted(wanted - by_run.keys())
summary.failures.extend(RunFailure(r, ERR_RUN_NOT_IN_MIRROR) for r in missing)
evaluator = current_evaluator_document()
root = resolve_mirror_root(mirror_root)
for run_id in selected:
_rescore_one(run_id, by_run[run_id], summary, out_dir, root, now, evaluator, repo_root)
return summary
def _checked_run_ids(run_ids: Collection[str] | None) -> frozenset[str] | None:
"""run ν•„ν„° 경계 검사 β€” None(ν•„ν„° μ—†μŒ) λ˜λŠ” `is_valid_run_id` λ₯Ό λ„˜λŠ” κ°’μ˜ λΉ„μ–΄ μžˆμ§€ μ•Šμ€ λͺ¨μŒ."""
if run_ids is None:
return None
if isinstance(run_ids, str | bytes):
raise TypeError("run_ids must be a collection of run ids, not a single string")
wanted = frozenset(run_ids)
if not wanted:
raise ValueError("run_ids must not be empty (pass None for every run)")
if not all(is_valid_run_id(r) for r in wanted):
raise ValueError(
"run_ids must be non-blank strings without control characters,"
f" at most {MAX_RUN_ID_LENGTH} characters"
)
return wanted
def _committed_by_run(
mirror_root: Path | None, account: str, case: str, summary: CaseSummary
) -> Mapping[str, Sequence[Committed]] | None:
"""μ˜μ—­μ˜ 컀밋 run λ²ˆλ“€(run_id 별). case 였λ₯˜λ©΄ `summary` 에 μ‹£κ³  None."""
bundles_dir = mirror_area_dir(mirror_root, account, case, BUNDLES_AREA)
if not bundles_dir.is_dir():
# `committed_run_bundles` λŠ” μ—†λŠ” μ˜μ—­μ„ 빈 dict 둜 λ³Έλ‹€ β€” κ·ΈλŒ€λ‘œ 두면 μš”μ•½μ΄ μ „λΆ€ 0 인 채 μ’…λ£Œ 0 이닀
summary.failures.append(RunFailure(None, ERR_CASE_AREA_MISSING, str(bundles_dir)))
return None
try:
by_run = committed_run_bundles(bundles_dir)
except AdoptError:
summary.failures.append(RunFailure(None, ERR_ADOPT))
return None
if not by_run:
# μžˆλŠ”λ° λΉ„μ—ˆλ‹€ β€” λ°±ν•„ μ „μ΄κ±°λ‚˜ μ˜μ—­ κ²½λ‘œκ°€ 어긋났닀. ν•  일 μ—†μŒΒ·μ’…λ£Œ 0 으둜 μˆ¨κΈ°μ§€ μ•ŠλŠ”λ‹€
summary.failures.append(RunFailure(None, ERR_CASE_AREA_EMPTY, str(bundles_dir)))
return None
return by_run
def _rescore_one(
run_id: str,
cands: Sequence[Committed],
summary: CaseSummary,
out_dir: Path | None,
root: Path,
now: datetime | None,
evaluator: Mapping[str, Any],
repo_root: Path | None,
) -> None:
"""run ν•˜λ‚˜ β€” μ΅œμ‹  λ²ˆλ“€ 채택(invalid λ©΄ skip) λ’€ μž¬μ±„μ . 결과·였λ₯˜λ₯Ό `summary` 에 μ‹£λŠ”λ‹€."""
adoption = adopt_run_bundle(cands)
if adoption.status != "ok":
summary.outcomes.append(RescoreOutcome("skip", run_id, code=SKIP_LATEST_INVALID))
return
try:
outcome = rescore_run_bundle(
adoption.committed.path,
out_dir,
mirror_root=None if out_dir is not None else root,
now=now,
evaluator=evaluator,
repo_root=repo_root,
)
except RescoreError as exc:
summary.failures.append(RunFailure(exc.run_id or run_id, exc.code))
return
summary.outcomes.append(outcome)