File size: 9,030 Bytes
3dfc93b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
"""case 전체 μž¬μ±„μ  β€” `solar-eval score proofread-diff --case a/c --all` (κ³„νš 2026-09-26 L3b Β§3.2Β·D6).

미러의 `accounts/<a>/<c>/eval/bundles` μ—μ„œ run_id 별 μ΅œμ‹  컀밋 λ²ˆλ“€μ„ `poc_bundle.adopt` 둜 κ³ λ₯΄κ³ (μ΅œμ‹ μ΄ invalid λ©΄ μ΄μ „μœΌλ‘œ
λ„˜μ–΄κ°€μ§€ μ•Šκ³  `latest_bundle_invalid` skip), μ±„νƒλœ λ²ˆλ“€λ§ˆλ‹€ `rescore_run_bundle` 을 λΆ€λ₯Έλ‹€.

- `bundles` μ˜μ—­ 디렉토리가 μ—†μœΌλ©΄ case 였λ₯˜ `case_area_missing`(ν•΄μ„ν•œ 경둜λ₯Ό μ‹£λŠ”λ‹€) β€” case μ˜€νƒ€λ‚˜ `--mirror-root` μΈ΅
  어긋남(Β§2.1 "미러 루트 μΈ΅")이 "ν•  일 μ—†μŒΒ·μ’…λ£Œ 0" 으둜 μˆ¨μ§€ μ•Šκ²Œ.
- μ˜μ—­μ€ μžˆλŠ”λ° μ»€λ°‹λœ run λ²ˆλ“€μ΄ 0 이면 case 였λ₯˜ `case_area_empty`(ν•΄μ„ν•œ 경둜) β€” 같은 μ΄μœ λ‹€(κ³„νš 2026-09-27 L6 Β§2.11,
  L3b 후속 3). run ν•„ν„°κ°€ μžˆμ–΄λ„ 이 νŒμ •μ΄ λ¨Όμ €λ‹€.
- μ˜μ—­μ— λ§€λ‹ˆνŽ˜μŠ€νŠΈλ₯Ό λͺ» μ½λŠ” λ²ˆλ“€μ΄ ν•˜λ‚˜λΌλ„ 있으면(`AdoptError`) case 전체가 였λ₯˜λ‹€ β€” μ–΄λŠ run 의 λ²ˆλ“€μΈμ§€ λͺ¨λ₯΄λŠ” μ±„λ‘œ
  μ±„νƒν•˜μ§€ μ•ŠλŠ”λ‹€(fail closed). run ν•„ν„°κ°€ μžˆμ–΄λ„ μ˜μ—­ 전체λ₯Ό μ½λŠ”λ‹€ β€” ν•„ν„°κ°€ fail closed λ₯Ό μ•½ν•˜κ²Œ λ§Œλ“€μ§€ μ•Šκ²Œ.
- **run ν•„ν„°**(`run_ids`, L6 Β§2.11 β€” μ—”μ§„ μ’…λ£Œ 훅이 run ν•˜λ‚˜λ§Œ μž¬μ±„μ ν•œλ‹€): 채택 λͺ©λ‘(run_id 별 컀밋 λ²ˆλ“€)을 κ±°λ₯Έλ‹€. 채택
  κ·œμΉ™μ€ ν•„ν„° μ—†λŠ” case λͺ¨λ“œμ™€ κ°™λ‹€. μš”μ²­ν•œ run 이 μ˜μ—­μ— μ—†μœΌλ©΄ run λ§ˆλ‹€ 였λ₯˜ `run_not_in_mirror`. κ°’ κ·œμΉ™
  (`is_valid_run_id` β€” 곡백만이 μ•„λ‹˜Β·μ œμ–΄ 문자 μ—†μŒΒ·512자 μ΄ν•˜)은 CLI `--run` κ³Ό 이 ν•¨μˆ˜ 경계가 ν•¨κ»˜ μ“΄λ‹€.
- run ν•˜λ‚˜μ˜ 였λ₯˜(`RescoreError`)λŠ” κ·Έ run 만 였λ₯˜λ‘œ μ„Έκ³  λ‹€μŒ run 을 κ³„μ†ν•œλ‹€ β€” μ’…λ£Œ μ½”λ“œλŠ” CLI κ°€ `errors` 둜 μ •ν•œλ‹€.
- 평가기 λ¬Έμ„œλŠ” case ν•œ λ²ˆμ— ν•œ 번 λœ¬λ‹€(λͺ¨λ“  run 이 같은 `evaluator_config_hash` λ₯Ό μ“΄λ‹€).
"""

from __future__ import annotations

import re
from collections import Counter
from collections.abc import Collection, Mapping, Sequence
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
from typing import Any

from poc_bundle.adopt import AdoptError, Committed, adopt_run_bundle, committed_run_bundles

from solar_eval.rescore.evaluator_doc import current_evaluator_document
from solar_eval.rescore.from_bundle import (
    ERR_ADOPT,
    SKIP_LATEST_INVALID,
    RescoreError,
    RescoreOutcome,
    rescore_run_bundle,
)
from solar_eval.rescore.paths import BUNDLES_AREA, mirror_area_dir, resolve_mirror_root

#: case 였λ₯˜ β€” 미러 `bundles` μ˜μ—­μ΄ μ—†λ‹€(case μ˜€νƒ€Β·`--mirror-root` μΈ΅ 어긋남). 아무것도 μ•ˆ ν•œ 채 μ„±κ³΅ν•˜μ§€ μ•Šκ²Œ.
ERR_CASE_AREA_MISSING = "case_area_missing"
#: case 였λ₯˜ β€” `bundles` μ˜μ—­μ€ μžˆλŠ”λ° μ»€λ°‹λœ run λ²ˆλ“€μ΄ 0 이닀(L6 Β§2.11).
ERR_CASE_AREA_EMPTY = "case_area_empty"
#: run 였λ₯˜ β€” run ν•„ν„°λ‘œ μš”μ²­ν•œ run 의 컀밋 λ²ˆλ“€μ΄ 미러 μ˜μ—­μ— μ—†λ‹€(L6 Β§2.11).
ERR_RUN_NOT_IN_MIRROR = "run_not_in_mirror"
#: run ν•„ν„° κ°’ ν•˜λ‚˜μ˜ κΈ€μž 수 μƒν•œ(μ½”λ“œ 포인트) β€” ES `hub ingest --expect-run` κ³Ό κ°™λ‹€(L6 Β§2.9).
MAX_RUN_ID_LENGTH = 512
# C0Β·DELΒ·C1 μ œμ–΄ 문자 β€” ES `--expect-run` κ³Ό 같은 λ²”μœ„. stderr `error <run_id> …` 쀄을 μœ„μ‘°ν•˜μ§€ λͺ»ν•˜κ²Œ ν•œλ‹€
_CONTROL_CHARACTER = re.compile(r"[\x00-\x1f\x7f-\x9f]")


def is_valid_run_id(value: object) -> bool:
    """run ν•„ν„° κ°’ ν•˜λ‚˜ β€” 곡백만이 μ•„λ‹Œ λ¬Έμžμ—΄, μ œμ–΄ 문자 μ—†μŒ, `MAX_RUN_ID_LENGTH` 자 μ΄ν•˜(CLI `--run` κ³Ό 같은 κ·œμΉ™)."""
    return (
        isinstance(value, str)
        and bool(value.strip())
        and len(value) <= MAX_RUN_ID_LENGTH
        and _CONTROL_CHARACTER.search(value) is None
    )


@dataclass(frozen=True)
class RunFailure:
    """run ν•˜λ‚˜(λ˜λŠ” case 전체, `run_id=None`)의 였λ₯˜ β€” idΒ·μ½”λ“œμ™€, 있으면 경둜만(λ©”μ‹œμ§€ 원문은 μ‹£μ§€ μ•ŠλŠ”λ‹€)."""

    run_id: str | None
    code: str
    detail: str | None = None


@dataclass
class CaseSummary:
    """case μž¬μ±„μ  κ²°κ³Ό β€” run κ²°κ³Ό λͺ©λ‘κ³Ό 였λ₯˜."""

    outcomes: list[RescoreOutcome] = field(default_factory=list)
    failures: list[RunFailure] = field(default_factory=list)

    def to_json(self) -> dict[str, Any]:
        """Β§2.1 μš”μ•½ 쀄 `{written, exists, skipped:{code:n}, errors, n_original_missing}`."""
        statuses = Counter(o.status for o in self.outcomes)
        skipped = Counter(o.code for o in self.outcomes if o.status == "skip")
        return {
            "written": statuses.get("written", 0),
            "exists": statuses.get("exists", 0),
            "skipped": dict(sorted(skipped.items())),
            "errors": len(self.failures),
            "n_original_missing": sum(o.n_original_missing for o in self.outcomes),
        }


def rescore_case(
    account: str,
    case: str,
    *,
    mirror_root: Path | None = None,
    out_dir: Path | None = None,
    now: datetime | None = None,
    repo_root: Path | None = None,
    run_ids: Collection[str] | None = None,
) -> CaseSummary:
    """미러의 ν•œ case λ₯Ό μž¬μ±„μ ν•œλ‹€. `out_dir` κ°€ None 이면 미러 `evaluations` 에 λ†“λŠ”λ‹€(`mirror_bundle`).

    `mirror_root` λŠ” 미러 λ²„ν‚·μ˜ λΆ€λͺ¨(κΈ°λ³Έ 데이터 ν™ˆ `blob-mirror`) β€” μž…λ ₯ run λ²ˆλ“€λ„ μ—¬κΈ°μ„œ μ½λŠ”λ‹€.
    `run_ids` λ₯Ό μ£Όλ©΄ κ·Έ run λ“€λ§Œ μž¬μ±„μ ν•œλ‹€(μ—†λŠ” run 은 `run_not_in_mirror`). λ¬Έμžμ—΄ ν•˜λ‚˜(`TypeError`)λ‚˜ 빈 λͺ¨μŒΒ·
    `is_valid_run_id` λ₯Ό λͺ» λ„˜λŠ” μ›μ†Œ(`ValueError`)λŠ” 호좜자 버그라 μ¦‰μ‹œ κ±°λΆ€ν•œλ‹€ β€” κΈ€μž λ‹¨μœ„ ν•„ν„°λ‚˜ "ν•  일 μ—†μŒΒ·μ„±κ³΅",
    stderr 쀄 μœ„μ‘°κ°€ λ˜μ§€ μ•Šκ²Œ.
    """
    wanted = _checked_run_ids(run_ids)
    summary = CaseSummary()
    by_run = _committed_by_run(mirror_root, account, case, summary)
    if by_run is None:
        return summary
    selected = sorted(by_run) if wanted is None else sorted(wanted & by_run.keys())
    if wanted is not None:
        missing = sorted(wanted - by_run.keys())
        summary.failures.extend(RunFailure(r, ERR_RUN_NOT_IN_MIRROR) for r in missing)
    evaluator = current_evaluator_document()
    root = resolve_mirror_root(mirror_root)
    for run_id in selected:
        _rescore_one(run_id, by_run[run_id], summary, out_dir, root, now, evaluator, repo_root)
    return summary


def _checked_run_ids(run_ids: Collection[str] | None) -> frozenset[str] | None:
    """run ν•„ν„° 경계 검사 β€” None(ν•„ν„° μ—†μŒ) λ˜λŠ” `is_valid_run_id` λ₯Ό λ„˜λŠ” κ°’μ˜ λΉ„μ–΄ μžˆμ§€ μ•Šμ€ λͺ¨μŒ."""
    if run_ids is None:
        return None
    if isinstance(run_ids, str | bytes):
        raise TypeError("run_ids must be a collection of run ids, not a single string")
    wanted = frozenset(run_ids)
    if not wanted:
        raise ValueError("run_ids must not be empty (pass None for every run)")
    if not all(is_valid_run_id(r) for r in wanted):
        raise ValueError(
            "run_ids must be non-blank strings without control characters,"
            f" at most {MAX_RUN_ID_LENGTH} characters"
        )
    return wanted


def _committed_by_run(
    mirror_root: Path | None, account: str, case: str, summary: CaseSummary
) -> Mapping[str, Sequence[Committed]] | None:
    """μ˜μ—­μ˜ 컀밋 run λ²ˆλ“€(run_id 별). case 였λ₯˜λ©΄ `summary` 에 μ‹£κ³  None."""
    bundles_dir = mirror_area_dir(mirror_root, account, case, BUNDLES_AREA)
    if not bundles_dir.is_dir():
        # `committed_run_bundles` λŠ” μ—†λŠ” μ˜μ—­μ„ 빈 dict 둜 λ³Έλ‹€ β€” κ·ΈλŒ€λ‘œ 두면 μš”μ•½μ΄ μ „λΆ€ 0 인 채 μ’…λ£Œ 0 이닀
        summary.failures.append(RunFailure(None, ERR_CASE_AREA_MISSING, str(bundles_dir)))
        return None
    try:
        by_run = committed_run_bundles(bundles_dir)
    except AdoptError:
        summary.failures.append(RunFailure(None, ERR_ADOPT))
        return None
    if not by_run:
        # μžˆλŠ”λ° λΉ„μ—ˆλ‹€ β€” λ°±ν•„ μ „μ΄κ±°λ‚˜ μ˜μ—­ κ²½λ‘œκ°€ 어긋났닀. ν•  일 μ—†μŒΒ·μ’…λ£Œ 0 으둜 μˆ¨κΈ°μ§€ μ•ŠλŠ”λ‹€
        summary.failures.append(RunFailure(None, ERR_CASE_AREA_EMPTY, str(bundles_dir)))
        return None
    return by_run


def _rescore_one(
    run_id: str,
    cands: Sequence[Committed],
    summary: CaseSummary,
    out_dir: Path | None,
    root: Path,
    now: datetime | None,
    evaluator: Mapping[str, Any],
    repo_root: Path | None,
) -> None:
    """run ν•˜λ‚˜ β€” μ΅œμ‹  λ²ˆλ“€ 채택(invalid λ©΄ skip) λ’€ μž¬μ±„μ . 결과·였λ₯˜λ₯Ό `summary` 에 μ‹£λŠ”λ‹€."""
    adoption = adopt_run_bundle(cands)
    if adoption.status != "ok":
        summary.outcomes.append(RescoreOutcome("skip", run_id, code=SKIP_LATEST_INVALID))
        return
    try:
        outcome = rescore_run_bundle(
            adoption.committed.path,
            out_dir,
            mirror_root=None if out_dir is not None else root,
            now=now,
            evaluator=evaluator,
            repo_root=repo_root,
        )
    except RescoreError as exc:
        summary.failures.append(RunFailure(exc.run_id or run_id, exc.code))
        return
    summary.outcomes.append(outcome)