File size: 8,673 Bytes
3dfc93b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
"""`solar-eval score` โ€” ์˜คํ”„๋ผ์ธ ์žฌ์ฑ„์  ๋ช…๋ น ๊ทธ๋ฃน (๊ณ„ํš 2026-09-26 L3b ยง2.1).

`score proofread-diff` ๋Š” run ๋ฒˆ๋“ค์˜ `results.jsonl` ์„ ๋ ˆ๊ฑฐ์‹œ `lcs_diff` ์™€ ๊ฐ™์€ ํ•จ์ˆ˜๋กœ ๋‹ค์‹œ ์ฑ„์ ํ•ด ์ฑ„์  ๋ฒˆ๋“ค
(`poc-scoring/v1`, `purpose=rescore_after_fix`)๋กœ ๋‚ธ๋‹ค. ๋‘ ๋ชจ๋“œ:

- `<run_bundle_dir>โ€ฆ` ๋ชจ๋“œ: ๋ฒˆ๋“ค์ด `โ€ฆ/eval/bundles/<id>` ์— ์žˆ์œผ๋ฉด ๊ธฐ๋ณธ ์ถœ๋ ฅ์€ `โ€ฆ/eval/evaluations`, ์•„๋‹ˆ๋ฉด `--out` ํ•„์ˆ˜.
  `--mirror` ๋ฉด ๊ฐ ๋ฒˆ๋“ค ๋งค๋‹ˆํŽ˜์ŠคํŠธ `case` ์˜ ๋ฏธ๋Ÿฌ `evaluations` ์— ๋†“๋Š”๋‹ค.
- `--case a/c --all` ๋ชจ๋“œ: ๋ฏธ๋Ÿฌ์˜ ๊ทธ case run ๋ฒˆ๋“ค ์ „๋ถ€(`poc_bundle.adopt` ๋กœ ์ฑ„ํƒ). `--mirror` ๋˜๋Š” `--out` ์ด ํ•„์š”ํ•˜๋‹ค.
  `--run <run_id>`(๋ฐ˜๋ณต)๋ฅผ ์ฃผ๋ฉด ๊ทธ run ๋“ค๋งŒ(๊ณ„ํš 2026-09-27 L6 ยง2.11 โ€” ์ฑ„ํƒ ๊ทœ์น™์€ ๊ฐ™๊ณ , ์˜์—ญ์— ์—†๋Š” run ์€ ์˜ค๋ฅ˜
  `run_not_in_mirror`). `--run` ์€ ์ด ๋ชจ๋“œ์—์„œ๋งŒ ๋ฐ›๊ณ , ๋นˆ ๊ฐ’ยท๊ณต๋ฐฑ๋งŒยท์ œ์–ด ๋ฌธ์žยท512์ž ์ดˆ๊ณผ๋Š” ์‚ฌ์šฉ ์˜ค๋ฅ˜๋‹ค(ES
  `hub ingest --expect-run` ๊ณผ ๊ฐ™์€ ๋ฒ”์œ„ โ€” ์—†๋Š” run ์ด stderr ์— ๊ทธ๋Œ€๋กœ ์ฐํžˆ๋ฏ€๋กœ ์ค„ ์œ„์กฐ๋ฅผ ๋ง‰๋Š”๋‹ค).
- `--out DIR` ๊ณผ `--mirror` ๋Š” ํ•จ๊ป˜ ์“ฐ์ง€ ์•Š๋Š”๋‹ค. `--mirror-root` ๋Š” ๋ฏธ๋Ÿฌ **๋ฒ„ํ‚ท์˜ ๋ถ€๋ชจ**๋‹ค(ยง2.1 "๋ฏธ๋Ÿฌ ๋ฃจํŠธ ์ธต").
  ๋””๋ ‰ํ† ๋ฆฌ ๋ชจ๋“œ์—์„œ๋Š” `--mirror` ์™€ ํ•จ๊ป˜๋งŒ ๋ฐ›๋Š”๋‹ค. ์—†๋Š” run ๋ฒˆ๋“ค ๊ฒฝ๋กœ๋Š” ์‚ฌ์šฉ ์˜ค๋ฅ˜(์ข…๋ฃŒ 2)๋‹ค.

stdout ์€ run ๋งˆ๋‹ค ํ•œ ์ค„ `written <path>` ยท `exists <path>` ยท `skip <run_id> <code>` ์ด๊ณ , case ๋ชจ๋“œ๋Š” ๋งˆ์ง€๋ง‰ ์ค„์— JSON ์š”์•ฝ
`{written, exists, skipped:{code:n}, errors, n_original_missing}` ์„ ๋‚ธ๋‹ค. ์˜ค๋ฅ˜๋Š” stderr ์— `error <run_id> <code>` ํ•œ ์ค„์”ฉ
(case ์ „์ฒด ์˜ค๋ฅ˜๋Š” run_id ์ž๋ฆฌ์— `-`, `case_area_missing`ยท`case_area_empty` ๋Š” ๋’ค์— ํ•ด์„ํ•œ ์˜์—ญ ๊ฒฝ๋กœ๋ฅผ ๋ถ™์ธ๋‹ค).
์–ด๋А ์ชฝ์—๋„ ์ž…๋ ฅยท์ถœ๋ ฅยท์ •๋‹ต ์›๋ฌธ์€ ์‹ฃ์ง€ ์•Š๋Š”๋‹ค.

์ข…๋ฃŒ ์ฝ”๋“œ: 0 = ์ „๋ถ€ written/exists/๊ทœ์น™ skip ยท 1 = ์˜ค๋ฅ˜ 1๊ฑด ์ด์ƒ ยท 2 = ์‚ฌ์šฉ ์˜ค๋ฅ˜.

import ๊ฒฝ๊ณ„(D1): ์ด ๋ชจ๋“ˆ์€ click ๋งŒ ๋ชจ๋“ˆ ์ตœ์ƒ์œ„์—์„œ import ํ•œ๋‹ค. `solar_eval.rescore`(๊ทธ๋ฆฌ๊ณ  ๊ทธ๊ฒƒ์ด ๋Œ์–ด์˜ค๋Š”
`poc_bundle`)๋Š” **๋ช…๋ น ํ•จ์ˆ˜ ์•ˆ์—์„œ** import ํ•œ๋‹ค โ€” `solar-eval` CLI ๋ฅผ ๋„์šฐ๋Š” ๊ฒƒ๋งŒ์œผ๋กœ ์žฌ์ฑ„์ ๊ธฐยท๋ฒˆ๋“ค writer ๊ฐ€
๋กœ๋”ฉ๋˜์ง€ ์•Š๊ฒŒ.
"""

from __future__ import annotations

import json
import sys
from pathlib import Path
from typing import Any

import click

_DIR = click.Path(file_okay=False, path_type=Path)
# ์ž…๋ ฅ run ๋ฒˆ๋“ค์€ ์žˆ์–ด์•ผ ํ•œ๋‹ค โ€” ๊ฒฝ๋กœ ์˜คํƒ€๊ฐ€ ๊ทœ์น™ skip(`latest_bundle_invalid`)ยท์ข…๋ฃŒ 0 ์œผ๋กœ ๋๋‚˜์ง€ ์•Š๊ฒŒ click ์ด ์ข…๋ฃŒ 2 ๋กœ ๋ง‰๋Š”๋‹ค.
_EXISTING_DIR = click.Path(exists=True, file_okay=False, path_type=Path)
EXIT_ERRORS = 1
EXIT_USAGE = 2


@click.group("score")
def score_group() -> None:
    """Offline rescoring of run bundles (scoring bundles, poc-scoring/v1)."""


@score_group.command("proofread-diff")
@click.argument("run_bundle_dirs", nargs=-1, type=_EXISTING_DIR)
@click.option("--case", "case_key", default=None, help="accounts/<a>/<c> as a/c (with --all).")
@click.option("--all", "all_runs", is_flag=True, default=False, help="Every adopted run bundle.")
@click.option("--out", "out_dir", type=_DIR, default=None, help="Parent dir for scoring bundles.")
@click.option("--mirror", is_flag=True, default=False, help="Write into the local mirror.")
@click.option(
    "--mirror-root",
    type=_DIR,
    default=None,
    help="Parent of the mirror bucket (default: data home blob-mirror).",
)
@click.option(
    "--run",
    "run_ids",
    multiple=True,
    help="Only this run_id (repeatable; with --case --all).",
)
def proofread_diff(
    run_bundle_dirs: tuple[Path, ...],
    case_key: str | None,
    all_runs: bool,
    out_dir: Path | None,
    mirror: bool,
    mirror_root: Path | None,
    run_ids: tuple[str, ...],
) -> None:
    """Rescore run bundles with proofread_diff@2 (same scoring as legacy lcs_diff)."""
    _check_usage(run_bundle_dirs, case_key, all_runs, out_dir, mirror, mirror_root, run_ids)
    if case_key is not None:
        code = _run_case(case_key, out_dir, mirror_root, frozenset(run_ids) or None)
    else:
        code = _run_dirs(run_bundle_dirs, out_dir, mirror_root if mirror else None, mirror)
    sys.exit(code)


def _usage(message: str) -> None:
    click.echo(f"usage error: {message}", err=True)
    sys.exit(EXIT_USAGE)


def _check_usage(
    dirs: tuple[Path, ...],
    case_key: str | None,
    all_runs: bool,
    out_dir: Path | None,
    mirror: bool,
    mirror_root: Path | None,
    run_ids: tuple[str, ...] = (),
) -> None:
    """๋‘ ๋ชจ๋“œ์˜ ์ธ์ž ์กฐํ•ฉ โ€” ์–ด๊ธ‹๋‚˜๋ฉด ์ข…๋ฃŒ 2."""
    if out_dir is not None and mirror:
        _usage("--out and --mirror are mutually exclusive")
    if case_key is not None or all_runs:
        if dirs:
            _usage("give run bundle directories or --case/--all, not both")
        if case_key is None or not all_runs:
            _usage("--case a/c needs --all (and --all needs --case)")
        if out_dir is None and not mirror:
            _usage("--case --all needs --mirror or --out DIR")
    elif not dirs:
        _usage("give run bundle directories, or --case a/c --all")
    elif mirror_root is not None and not mirror:
        # ๋””๋ ‰ํ† ๋ฆฌ ๋ชจ๋“œ์—์„œ --mirror-root ๋Š” --mirror ์˜ ์ถœ๋ ฅ ์ž๋ฆฌ์ผ ๋ฟ์ด๋‹ค โ€” ์กฐ์šฉํžˆ ๋ฒ„๋ฆฌ์ง€ ์•Š๋Š”๋‹ค
        _usage("--mirror-root needs --mirror when rescoring run bundle directories")
    if run_ids:
        _check_run_filter(case_key, run_ids)


def _check_run_filter(case_key: str | None, run_ids: tuple[str, ...]) -> None:
    """`--run` ์€ case ๋ชจ๋“œ์˜ run ํ•„ํ„ฐ๋‹ค(L6 ยง2.11) โ€” ๋””๋ ‰ํ† ๋ฆฌ ๋ชจ๋“œ์—์„œ ์กฐ์šฉํžˆ ๋ฒ„๋ฆฌ์ง€ ์•Š๊ณ , ๊ฐ’์€ ๊ฒฝ๊ณ„์—์„œ ๊ฑฐ๋ฅธ๋‹ค.

    case ๋ชจ๋“œ์˜ `--case`/`--all` ์ง์€ `_check_usage` ๊ฐ€ ๋จผ์ € ๋ง‰์œผ๋ฏ€๋กœ ์—ฌ๊ธฐ์„œ `case_key is None` ์ด๋ฉด ๋””๋ ‰ํ† ๋ฆฌ ๋ชจ๋“œ๋‹ค.
    ๊ฐ’ ๊ทœ์น™์€ ๋ผ์ด๋ธŒ๋Ÿฌ๋ฆฌ ๊ฒฝ๊ณ„(`rescore_case`)์™€ ๊ฐ™์€ `is_valid_run_id` ๋‹ค. ๊ฑฐ๋ถ€ ๋ฌธ๊ตฌ์— ๊ฐ’ ์›๋ฌธ์€ ์‹ฃ์ง€ ์•Š๋Š”๋‹ค.
    """
    from solar_eval.rescore.case import MAX_RUN_ID_LENGTH, is_valid_run_id

    if case_key is None:
        _usage("--run needs --case a/c --all (not run bundle directories)")
    if not all(is_valid_run_id(r) for r in run_ids):
        _usage(
            "--run needs a non-blank run_id without control characters,"
            f" at most {MAX_RUN_ID_LENGTH} characters"
        )


def _run_case(
    case_key: str,
    out_dir: Path | None,
    mirror_root: Path | None,
    run_ids: frozenset[str] | None = None,
) -> int:
    from solar_eval.rescore.case import rescore_case
    from solar_eval.rescore.paths import RescorePathError, parse_case_key

    try:
        account, case = parse_case_key(case_key)
    except RescorePathError as exc:
        _usage(str(exc))
    summary = rescore_case(account, case, mirror_root=mirror_root, out_dir=out_dir, run_ids=run_ids)
    for outcome in summary.outcomes:
        _echo_outcome(outcome)
    for failure in summary.failures:
        suffix = f" {failure.detail}" if failure.detail else ""
        click.echo(f"error {failure.run_id or '-'} {failure.code}{suffix}", err=True)
    report: dict[str, Any] = summary.to_json()
    click.echo(json.dumps(report, ensure_ascii=False, sort_keys=False))
    return EXIT_ERRORS if report["errors"] else 0


def _run_dirs(
    dirs: tuple[Path, ...], out_dir: Path | None, mirror_root: Path | None, mirror: bool
) -> int:
    from solar_eval.rescore.evaluator_doc import current_evaluator_document
    from solar_eval.rescore.from_bundle import RescoreError, rescore_run_bundle
    from solar_eval.rescore.paths import default_evaluations_dir, resolve_mirror_root

    targets = [(d, out_dir or (None if mirror else default_evaluations_dir(d))) for d in dirs]
    missing = [d for d, target in targets if target is None and not mirror]
    if missing:
        _usage(f"{missing[0]} is not under โ€ฆ/eval/bundles/ โ€” give --out DIR")
    root = resolve_mirror_root(mirror_root) if mirror else None
    evaluator = current_evaluator_document()
    errors = 0
    for bundle_dir, target in targets:
        try:
            outcome = rescore_run_bundle(bundle_dir, target, mirror_root=root, evaluator=evaluator)
        except RescoreError as exc:
            click.echo(f"error {exc.run_id or bundle_dir.name} {exc.code}", err=True)
            errors += 1
            continue
        _echo_outcome(outcome, fallback=bundle_dir.name)
    return EXIT_ERRORS if errors else 0


def _echo_outcome(outcome: Any, fallback: str = "-") -> None:
    if outcome.status == "skip":
        click.echo(f"skip {outcome.run_id or fallback} {outcome.code}")
    else:
        click.echo(f"{outcome.status} {outcome.path}")