"""run 번들 하나 → `proofread_diff@2` 채점 번들 (계획 2026-09-26 L3b §2.4·D5). 순서: subject verify → 생산자 판별(`is_legacy_results_producer`) → `results.jsonl` 레코드 줄 순서대로 읽기 → 행마다 채점(순수 모듈) → 판정 초안·생산자 집계 → 같은 `evaluation_run_id` 의 기존 최신과 비교 → 새 번들 쓰기(`write_scoring_bundle`, subject 를 넘겨 E1·E2 까지 커밋 전에 본다) → (`mirror_root` 가 있으면) `mirror_bundle(area="evaluations")`. - `evaluation_run_id` = `code_rescore_run_id(subject bundle_id, evaluator_config_hash)` — subject 번들이 바뀌면 새 id 다. - **멱등(D5).** 기존 최신(같은 id)이 subject 와 함께 `verify_scoring_bundle` 을 통과하고(E4 로 `evaluator_config.json` 파일의 config 해시 = 매니페스트 값까지 본다), 평가기 해시가 같고, `producer_aggregates.json` 바이트가 같고, `judgments.jsonl` 행에서 `created_at`(재채점 시각)을 뺀 목록이 새 목록과 같으면 새 번들을 쓰지 않고 기존 경로를 `exists` 로 돌려준다. 하나라도 다르면(기존 최신이 깨졌어도) `supersedes` = 기존 최신으로 새 ULID 번들을 쓴다 — 새 ULID ≤ 기존이면 `clock_skew`(eval-wrap· wrap 과 같은 규칙). 비교가 id 순서 검사보다 먼저다. 생산자 집계는 폐포(`impl_hash`) 밖의 레거시 집계 코드가 내므로 판정만 보면 그 코드가 바뀐 것을 놓친다. - **subject 경계.** `manifest.json` 이 없는 디렉토리는 `not_a_run_bundle` 오류(번들이 아니다 — 경로 실수), verify 실패는 `latest_bundle_invalid` skip(`items.jsonl` 이 깨진 경우 포함 — case 모드의 채택과 같은 분류). - 출력·오류 메시지에는 id·코드·경로만 싣는다. 입력·출력·정답 원문은 어디에도 쓰지 않는다(census D-04 와 같은 규칙). """ from __future__ import annotations import re import shutil import tempfile from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import Any, Literal from poc_bundle.adopt import AdoptError, Committed, committed_scoring_bundles, latest from poc_bundle.bundle import verify_bundle from poc_bundle.canonical import load_json_strict from poc_bundle.jsonl import iter_record_lines from poc_bundle.legacy_run import is_legacy_results_producer from poc_bundle.mirror import MirrorError, mirror_bundle from poc_bundle.scoring.ids import code_rescore_run_id, evaluator_config_hash from poc_bundle.scoring.verify import RunBundleRef, load_run_bundle_ref, verify_scoring_bundle from poc_bundle.scoring.writer import JudgmentDraft, ScoringWriteError, write_scoring_bundle from ulid import ULID from solar_eval._version import __version__ from solar_eval.rescore.drafts import ScoredRow, judgment_drafts, producer_aggregates from solar_eval.rescore.evaluator_doc import current_evaluator_document from solar_eval.rescore.paths import EVALUATIONS_AREA, RescorePathError, mirror_area_dir from solar_eval.rescore.proofread_diff_v2 import has_original, score_row PRODUCER_NAME = "solar-eval score" PURPOSE = "rescore_after_fix" RESULTS_FILE = "results.jsonl" JUDGMENTS_FILE = "judgments.jsonl" AGGREGATES_FILE = "producer_aggregates.json" #: 멱등 비교용 임시 번들 id — 가장 큰 ULID 라 어떤 `supersedes` 보다도 뒤에 온다. 출력 디렉토리에는 쓰지 않는다. STAGING_BUNDLE_ID = "7ZZZZZZZZZZZZZZZZZZZZZZZZZ" # run 단위 skip 코드(종료 0 에 포함)와 오류 코드(종료 1) — 계획 §2.2. SKIP_LATEST_INVALID = "latest_bundle_invalid" SKIP_UNKNOWN_FORMAT = "unknown_results_format" SKIP_NO_RESULTS = "no_results" ERR_ADOPT = "adopt_unreadable_manifest" ERR_CLOCK_SKEW = "clock_skew" ERR_MIRROR = "mirror_failed" ERR_RESULTS = "results_unreadable" ERR_OUT_INSIDE = "out_inside_bundle" ERR_NOT_A_BUNDLE = "not_a_run_bundle" WRITE_FAILED_PREFIX = "write_failed:" _RULE_ID = re.compile(r"^\[([A-Z]+[0-9]*)\]") Status = Literal["written", "exists", "skip"] class RescoreError(RuntimeError): """이 run 을 재채점할 수 없다(종료 1). `code` 는 §2.2 오류 어휘, `run_id` 는 알면 싣는다.""" def __init__(self, code: str, message: str, run_id: str | None = None) -> None: super().__init__(f"{code}: {message}") self.code = code self.run_id = run_id @dataclass(frozen=True) class RescoreOutcome: """run 하나의 결과 — `written`·`exists` 는 번들 경로, `skip` 은 코드.""" status: Status run_id: str | None path: Path | None = None code: str | None = None n_original_missing: int = 0 class _Skip(Exception): def __init__(self, code: str) -> None: super().__init__(code) self.code = code @dataclass(frozen=True) class _Subject: bundle_dir: Path run_id: str bundle_id: str case: dict[str, Any] ref: RunBundleRef def rescore_run_bundle( run_bundle_dir: Path, out_dir: Path | None = None, *, mirror_root: Path | None = None, now: datetime | None = None, bundle_id: str | None = None, evaluator: Mapping[str, Any] | None = None, repo_root: Path | None = None, ) -> RescoreOutcome: """run 번들 하나를 재채점한다. `out_dir`(채점 번들들의 부모) 와 `mirror_root` 중 **정확히 하나**를 준다. Args: mirror_root: 미러 버킷의 부모 — 주면 출력 영역은 subject `case` 의 미러 `evaluations` 이고 `mirror_bundle` 로 놓는다. now · bundle_id: 재현 가능한 테스트 전용. 비우면 지금 시각·새 ULID. evaluator: 평가기 문서(case 전체에 한 번 뜬 것). 비우면 지금 디스크의 구현으로 뜬다. repo_root: `mirror_bundle` 의 레포 밖 검사 기준(기본: 레포를 찾아 본다). Raises: RescoreError: §2.2 오류(채택 영역 매니페스트·시계 역행·쓰기·미러·results 읽기). """ if (out_dir is None) == (mirror_root is None): raise ValueError("give exactly one of out_dir / mirror_root") try: subject = _load_subject(Path(run_bundle_dir)) rows, n_missing = _scored_rows(subject) except _Skip as skip: run_id = _run_id_or_none(Path(run_bundle_dir)) return RescoreOutcome("skip", run_id, code=skip.code) doc = dict(evaluator) if evaluator is not None else current_evaluator_document() at = _utc_seconds(now) drafts = judgment_drafts(subject.run_id, rows, at) target = out_dir if out_dir is not None else _mirror_evaluations(mirror_root, subject) _check_out_dir(Path(target), subject) plan = _Plan(subject, doc, drafts, producer_aggregates(rows, drafts), at) status, path = _commit(plan, Path(target), bundle_id, mirror_root, repo_root) return RescoreOutcome(status, subject.run_id, path=path, n_original_missing=n_missing) # ── 읽기 ───────────────────────────────────────────────────────────────────── def _load_subject(bundle_dir: Path) -> _Subject: ref = _subject_ref(bundle_dir) report = ref.verify_report if not report.is_bundle: raise RescoreError(ERR_NOT_A_BUNDLE, f"no manifest.json in {bundle_dir}") if not report.ok or report.manifest is None: raise _Skip(SKIP_LATEST_INVALID) manifest = report.manifest if not is_legacy_results_producer(manifest.producer.name): raise _Skip(SKIP_UNKNOWN_FORMAT) return _Subject( bundle_dir=bundle_dir, run_id=manifest.run_id, bundle_id=manifest.bundle_id, case=manifest.case.model_dump(mode="json"), ref=ref, ) def _subject_ref(bundle_dir: Path) -> RunBundleRef: """`RunBundleRef` — `items.jsonl` 을 못 읽으면 verify 결과로 가른다: 실패면 skip, 통과인데 못 읽으면 오류. `load_run_bundle_ref` 는 verify 결과와 무관하게 items 파싱 오류를 던지므로, 그대로 두면 깨진 번들이 case 모드(채택이 verify 를 먼저 본다)와 달리 `results_unreadable` 로 분류된다. """ try: return load_run_bundle_ref(bundle_dir) except ValueError as exc: if not verify_bundle(bundle_dir).ok: raise _Skip(SKIP_LATEST_INVALID) from exc raise RescoreError(ERR_RESULTS, f"items.jsonl unreadable in {bundle_dir.name}") from exc def _scored_rows(subject: _Subject) -> tuple[list[ScoredRow], int]: """results 레코드 줄 순서대로 채점한 행과 `original` 이 없는 행 수. 레코드가 0이면 `no_results`.""" rows: list[ScoredRow] = [] missing = 0 path = subject.bundle_dir / RESULTS_FILE for line_no, raw in iter_record_lines(path): result = _parse_result(raw, line_no, subject) sample_idx = result["sample_idx"] input_key, golden_key = subject.ref.items[sample_idx] rows.append(ScoredRow(sample_idx, input_key, golden_key, score_row(result))) missing += 0 if has_original(result) else 1 if not rows: raise _Skip(SKIP_NO_RESULTS) return rows, missing def _parse_result(raw: bytes, line_no: int, subject: _Subject) -> Mapping[str, Any]: """레코드 한 줄 → 객체. 메시지에는 줄 번호만 싣는다(행 내용 없음).""" where = f"{RESULTS_FILE}:{line_no} of run bundle {subject.bundle_id}" try: row = load_json_strict(raw, safe_integers=False) except (ValueError, RecursionError) as exc: raise RescoreError(ERR_RESULTS, f"{where} is not strict JSON", subject.run_id) from exc sample_idx = row.get("sample_idx") if isinstance(row, dict) else None if isinstance(sample_idx, bool) or not isinstance(sample_idx, int): raise RescoreError(ERR_RESULTS, f"{where} has no integer sample_idx", subject.run_id) if sample_idx not in subject.ref.items: raise RescoreError(ERR_RESULTS, f"{where} sample_idx is not in items.jsonl", subject.run_id) return row def _run_id_or_none(bundle_dir: Path) -> str | None: """skip 줄에 실을 run_id — 매니페스트를 읽을 수 있을 때만(검증 실패여도).""" try: data = load_json_strict((bundle_dir / "manifest.json").read_bytes()) except (OSError, ValueError, RecursionError): return None run_id = data.get("run_id") if isinstance(data, dict) else None return run_id if isinstance(run_id, str) else None def _mirror_evaluations(mirror_root: Path | None, subject: _Subject) -> Path: try: return mirror_area_dir( mirror_root, subject.case["account"], subject.case["case"], EVALUATIONS_AREA ) except RescorePathError as exc: raise RescoreError(ERR_MIRROR, str(exc), subject.run_id) from exc def _check_out_dir(out: Path, subject: _Subject) -> None: target, run_abs = out.resolve(), subject.bundle_dir.resolve() if target == run_abs or run_abs in target.parents: raise RescoreError( ERR_OUT_INSIDE, f"output {target} is inside the run bundle", subject.run_id ) def _utc_seconds(now: datetime | None) -> str: moment = (now or datetime.now(UTC)).astimezone(UTC) return moment.strftime("%Y-%m-%dT%H:%M:%SZ") # ── 멱등·커밋 ───────────────────────────────────────────────────────────────── @dataclass(frozen=True) class _Plan: subject: _Subject evaluator: Mapping[str, Any] drafts: Sequence[JudgmentDraft] aggregates: Mapping[str, Any] created_at: str @property def config_hash(self) -> str: return evaluator_config_hash(self.evaluator) @property def evaluation_run_id(self) -> str: return code_rescore_run_id(self.subject.bundle_id, self.config_hash) def _commit( plan: _Plan, out: Path, bundle_id: str | None, mirror_root: Path | None, repo_root: Path | None, ) -> tuple[Status, Path]: previous = _latest_committed(out, plan) if previous is not None and _same_as(previous, plan): return "exists", previous.path new_id = bundle_id or str(ULID()) if previous is not None and previous.bundle_id >= new_id: raise RescoreError( ERR_CLOCK_SKEW, f"new bundle_id {new_id} is not newer than committed {previous.bundle_id}", plan.subject.run_id, ) supersedes = None if previous is None else previous.bundle_id fields = _manifest_fields(plan, new_id, supersedes) if mirror_root is None: dest = out / new_id _write(dest, fields, plan) return "written", dest return "written", _write_and_mirror(fields, plan, new_id, Path(mirror_root), repo_root) def _latest_committed(out: Path, plan: _Plan) -> Committed | None: try: committed = committed_scoring_bundles(out) except AdoptError as exc: raise RescoreError(ERR_ADOPT, str(exc), plan.subject.run_id) from exc cands = committed.get(plan.evaluation_run_id, []) return latest(cands) if cands else None def _same_as(previous: Committed, plan: _Plan) -> bool: """D5 — 기존 최신이 온전하고 평가기 해시·생산자 집계·`created_at` 을 뺀 판정 행이 같은가. 자리표시 id 로 써 본 번들과 비교한다. 기존 최신이 verify 에 실패하면 같지 않다 — `exists` 로 넘기면 깨진 번들이 영영 최신으로 남는다(hub 는 최신이 invalid 면 이전으로 넘어가지 않는다). """ evaluator_ref = previous.manifest.get("evaluator") if ( not isinstance(evaluator_ref, Mapping) or evaluator_ref.get("config_hash") != plan.config_hash or not _previous_verifies(previous, plan) ): return False fields = _manifest_fields(plan, STAGING_BUNDLE_ID, previous.bundle_id) with tempfile.TemporaryDirectory(prefix="solar-eval-score-") as tmp: staged = Path(tmp) / STAGING_BUNDLE_ID _write(staged, fields, plan) new_rows = _rows_without_time(staged / JUDGMENTS_FILE) new_aggregates = _bytes_or_none(staged / AGGREGATES_FILE) old_rows = _rows_without_time(previous.path / JUDGMENTS_FILE) old_aggregates = _bytes_or_none(previous.path / AGGREGATES_FILE) return ( old_rows is not None and old_rows == new_rows and old_aggregates is not None and old_aggregates == new_aggregates ) def _previous_verifies(previous: Committed, plan: _Plan) -> bool: """기존 최신을 subject 와 함께 검증한다 — SB1(파일 sha)·E4(`evaluator_config.json` config 해시)·E1·E2 까지.""" subject = plan.subject report = verify_scoring_bundle(previous.path, run_bundles={subject.bundle_id: subject.ref}) return report.ok def _bytes_or_none(path: Path) -> bytes | None: """파일 바이트. 못 읽으면 None(비교가 거짓 → 새 번들).""" try: return path.read_bytes() except OSError: return None def _rows_without_time(path: Path) -> list[dict[str, Any]] | None: """판정 행에서 `created_at` 을 뺀 목록. 못 읽으면 None(비교가 거짓 → 새 번들).""" try: rows = [load_json_strict(raw) for _, raw in iter_record_lines(path)] except (OSError, ValueError, RecursionError): return None if not all(isinstance(row, dict) for row in rows): return None return [{k: v for k, v in row.items() if k != "created_at"} for row in rows] def _manifest_fields(plan: _Plan, bundle_id: str, supersedes: str | None) -> dict[str, Any]: subject = plan.subject return { "bundle_id": bundle_id, "evaluation_run_id": plan.evaluation_run_id, "id_origin": "deterministic", "supersedes": supersedes, "created_at": plan.created_at, "producer": {"name": PRODUCER_NAME, "version": __version__}, "case": subject.case, "subject": { "kind": "run", "runs": [{"run_id": subject.run_id, "bundle_id": subject.bundle_id, "role": "subject"}], }, "purpose": PURPOSE, "retry_of": None, "golden_basis": "run", "status": "completed", "finished_at": plan.created_at, "reproducible": {"snapshot_complete": True, "missing": []}, "known_defects": [], "observed": None, "usage": None, } def _write(dest: Path, fields: Mapping[str, Any], plan: _Plan) -> None: try: write_scoring_bundle( dest, fields=fields, evaluator=plan.evaluator, drafts=plan.drafts, producer_aggregates=plan.aggregates, run_bundle=plan.subject.ref, ) except ScoringWriteError as exc: _remove_uncommitted(dest) raise RescoreError(_write_code(exc.issues), str(exc), plan.subject.run_id) from exc except OSError as exc: _remove_uncommitted(dest) raise RescoreError(f"{WRITE_FAILED_PREFIX}io_error", str(exc), plan.subject.run_id) from exc def _write_code(issues: Sequence[str]) -> str: """`write_failed:<규칙 id 목록>` — 이슈 문장의 `[]` 머리만 모은다(없으면 `input`).""" ids = sorted({m.group(1) for issue in issues if (m := _RULE_ID.match(issue))}) return WRITE_FAILED_PREFIX + (",".join(ids) if ids else "input") def _write_and_mirror( fields: Mapping[str, Any], plan: _Plan, new_id: str, mirror_root: Path, repo_root: Path | None ) -> Path: """임시 디렉토리에 쓴 뒤 미러에 놓는다 — 미러의 커밋 규칙(manifest 마지막)·레포 밖 검사를 `mirror_bundle` 이 맡는다.""" case = plan.subject.case with tempfile.TemporaryDirectory(prefix="solar-eval-score-") as tmp: staged = Path(tmp) / new_id _write(staged, fields, plan) try: return mirror_bundle( staged, account=case["account"], case=case["case"], area=EVALUATIONS_AREA, root=mirror_root, repo_root=repo_root, ) except (MirrorError, OSError) as exc: raise RescoreError(ERR_MIRROR, str(exc), plan.subject.run_id) from exc def _remove_uncommitted(staging: Path) -> None: """writer 가 manifest 없이 남긴 디렉토리를 지운다 — 커밋된 번들(manifest 있음)은 건드리지 않는다.""" if staging.is_dir() and not (staging / "manifest.json").exists(): shutil.rmtree(staging)