File size: 12,356 Bytes
3dfc93b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
"""ν›… μ½˜μ†” β€” 닀리별 ν•œ 쀄과 λ˜μ‚΄λ¦¬κΈ° λͺ…λ Ή 문ꡬ (κ³„νš 2026-09-27 L6 Β§2.6 ν‘œ Β· Β§2.7 Β· κ²€ν†  F16).

λ˜μ‚΄λ¦¬κΈ° λͺ…λ Ήμ˜ URL μžλ¦¬λŠ” 늘 `HUB_DATABASE_URL=…` 이닀 β€” 값은 어디에도 μ‹£μ§€ μ•ŠλŠ”λ‹€(D-L6-4). μ‹€νŒ¨ 쀄은 λΉ¨κ°• + λ˜μ‚΄λ¦¬κΈ°.
λ ˆκ±°μ‹œ ν™”λ©΄(`public`)은 L10 μ»·μ˜€λ²„λ‘œ μ½”λ“œμ—μ„œ μ‚¬λΌμ‘Œλ‹€ β€” λ ˆκ±°μ‹œ μˆ˜λ™ 적재 쀄은 μ—†λ‹€(κ³„νš 2026-09-28 L10 Β§2.6 Β· L6b 후속 1).
hub opt-in 이 μ—†μœΌλ©΄(`not_opted_in`) λΉ¨κ°„ μ€„λ‘œ "이 run 은 hub(/v1)에 μ—†λ‹€" 와 μΌœλŠ” λ²•Β·μ§€κΈˆ μ‹£λŠ” λͺ…령을 μ•ˆλ‚΄ν•œλ‹€(λ§ˆμŠ€ν„° D-M2-18).
μš”μ•½ λ¨Έλ¦¬μ—λŠ” ν›… 전체 κ²½κ³Ό 초(μƒνƒœ 파일의 `started_at`β†’`finished_at`, 잠금 λŒ€κΈ° 포함)λ₯Ό 뢙인닀 β€” 첫 싀데이터 run(U-L6a-1)μ—μ„œ
μ½˜μ†”λ§ŒμœΌλ‘œ hub 적재 μ‹œκ°„μ„ μ•Œ 수 μ—†μ—ˆκΈ° λ•Œλ¬Έμ΄λ‹€. μ‹œκ°μ΄ μ—†κ±°λ‚˜ ν˜•μ‹μ΄ λ‹€λ₯΄λ©΄ 뢙이지 μ•ŠλŠ”λ‹€.
"""

from __future__ import annotations

from collections.abc import Iterable, Mapping
from datetime import datetime
from typing import Any

import click

from solar_eval.hub_hook import optin
from solar_eval.hub_hook.types import (
    CASE_BLOCKED_PREFIX,
    CASE_FAILED_PREFIX,
    FILTER_NO_MATCH,
    HUB_ERR_CONFIRMATION_REQUIRED,
    HUB_ERR_CONNECTION_FAILED,
    HUB_FAILED_PREFIX,
    LEG_BUNDLE,
    LEG_HOOK,
    LEG_HUB,
    LEG_LOCK,
    LEG_PUBLISH,
    LOCK_TIMEOUT,
    NOT_OPTED_IN,
    RUN_NOT_OK_PREFIX,
    SCHEMA_MISMATCH,
    SKIP_CASE_UNMAPPED,
    SKIP_DATA_HOME_INSIDE_REPO,
    SKIP_DATA_HOME_MISMATCH,
    SKIP_NO_REPO_ROOT,
    SKIP_NOT_REPO_MANAGED,
    SMOKE_RUN,
    STAGE_CASE_LOCK,
    STATUS_FAILED,
    STATUS_OK,
    STATUS_SKIPPED,
    HookResult,
    LegResult,
    Skip,
)

URL_PLACEHOLDER = "HUB_DATABASE_URL=…"
#: μƒνƒœ 파일 μ‹œκ° ν˜•μ‹(`hook._now` 와 κ°™λ‹€).
HOOK_TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
LEG_LABELS = {
    LEG_BUNDLE: "λ²ˆλ“€",
    LEG_HUB: "hub 적재",
    LEG_LOCK: "case 잠금",
    LEG_HOOK: "ν›…",
    LEG_PUBLISH: "채점 λ°œν–‰",
}
SKIP_REASONS = {
    SKIP_NOT_REPO_MANAGED: "레포 관리 ν”„λ‘œμ νŠΈκ°€ μ•„λ‹ˆλ‹€",
    SKIP_NO_REPO_ROOT: "레포 config λ£¨νŠΈκ°€ μ—†λ‹€(--projects-dir λ“±)",
    SKIP_CASE_UNMAPPED: "project κ°€ case 에 λ§€ν•‘λ˜μ§€ μ•ŠλŠ”λ‹€ β€” make bundle-census 둜 확인",
    SKIP_DATA_HOME_MISMATCH: "μ—”μ§„ μ•„μΉ΄μ΄λΈŒκ°€ 데이터 ν™ˆ 밖이닀",
    SKIP_DATA_HOME_INSIDE_REPO: "데이터 ν™ˆμ΄ 레포 μ•ˆμ΄λ‹€ β€” ν›… νŒŒμƒλ¬Όμ€ git 밖에 λ‘”λ‹€(CBS_DATA_HOME)",
}
#: μš”μ•½ 쀄에 μ‹£λŠ” λ²ˆλ“€ 닀리 detail ν‚€(μˆœμ„œλŒ€λ‘œ).
_BUNDLE_SUMMARY_KEYS = (
    (("backfill", "wrap"), "wrap"),
    (("backfill", "eval_wrap"), "eval-wrap"),
    (("rescore", "status"), "μž¬μ±„μ "),
    (("indexes",), "색인"),
)


# ── λ˜μ‚΄λ¦¬κΈ° λͺ…λ Ή ────────────────────────────────────────────────────────


def hub_ingest_command(case_ref: str) -> str:
    return f"{URL_PLACEHOLDER} make hub-ingest CASE={case_ref} CONFIRM=local"


def hub_backfill_command(case_ref: str, run_id: str) -> str:
    return f'make hub-backfill CASE={case_ref} ARGS="--run {run_id}"'


def hub_recovery(code: str, case_ref: str, run_id: str) -> str:
    """hub 닀리 μ‹€νŒ¨ μ½”λ“œ β†’ λ˜μ‚΄λ¦¬κΈ° 문ꡬ(Β§2.6 ν‘œ)."""
    ingest = hub_ingest_command(case_ref)
    if code == f"{HUB_FAILED_PREFIX}{HUB_ERR_CONFIRMATION_REQUIRED}":
        return _confirmation_hint(ingest)
    if code == f"{HUB_FAILED_PREFIX}{HUB_ERR_CONNECTION_FAILED}":
        return f"make experiment-hub-up λ’€ {ingest}"
    if code == f"{CASE_FAILED_PREFIX}{STAGE_CASE_LOCK}":
        return f"μ•ž μ μž¬κ°€ λλ‚œ λ’€ {ingest}"
    if code.startswith((CASE_BLOCKED_PREFIX, CASE_FAILED_PREFIX)):
        return f"{ingest} ν›„ 이슈 ν‘œ(/v1/{case_ref}) 확인"
    if code == SCHEMA_MISMATCH:
        return (
            f"hub μŠ€ν‚€λ§ˆ 판이 λ‹€λ₯΄λ‹€ β€” {URL_PLACEHOLDER} make hub-m1-local CASE={case_ref} "
            "CONFIRM=local SKIP_PULL=1 FROM=backfill (주의: μž¬κ΅¬μ„±μ€ λͺ¨λ“  case λ₯Ό μ§€μš΄λ‹€, case λ§ˆλ‹€ λ‹€μ‹œ)"
        )
    if code.startswith(RUN_NOT_OK_PREFIX):
        return f"{hub_backfill_command(case_ref, run_id)} λ’€ {ingest}"
    if code == FILTER_NO_MATCH:
        return f"pnpm μ›Œν¬μŠ€νŽ˜μ΄μŠ€μ— @poc/eval-store κ°€ μ—†λ‹€ β€” pnpm install λ’€ {ingest}"
    return ingest


#: μ‚¬μš©μž DB μ“°κΈ° 확인 μ•ˆλ‚΄(μ‚¬λžŒ μ…Έ) β€” μ˜μ† `hub.confirm` 은 λ§ˆμŠ€ν„° Β§4 κ·œμΉ™ 9 의 μ˜λ„λœ μ˜ˆμ™Έλ‹€(L10 Β§2.6 Β· κ²€ν†  F1-3).
CONFIRMATION_HINT_HUMAN = (
    "μ‚¬μš©μž DB 에 μ“°λ €λ©΄ λͺ…λ Ή μ•žμ— SOLAR_EVAL_HUB_CONFIRM=local 을 λΆ™μ΄κ±°λ‚˜ μ„€μ • 파일 hub.confirm: local"
    "(μ‚¬λžŒ μ…Έλ§Œ Β· URL 도 섀정에 μžˆμ„ λ•Œλ§Œ β€” κ·œμΉ™ 9 의 μ˜λ„λœ μ˜ˆμ™Έ)"
)


#: `not_opted_in` λΉ¨κ°„ μ€„μ˜ 머리(μ‚¬λžŒΒ·μ—μ΄μ „νŠΈ 곡톡).
NOT_OPTED_IN_HEAD = "hub opt-in μ—†μŒ β€” 이 run 은 hub(/v1)에 μ—†λ‹€."


def not_opted_in_hint(case_ref: str) -> str:
    """hub opt-in 이 μ—†λ‹€ β€” λΉ¨κ°„ 쀄 μ•ˆλ‚΄(λ§ˆμŠ€ν„° D-M2-18 Β· L10 Β§2.6 ν‘œ "λ‘˜ λ‹€ μ—†μŒ"). URL 값은 μ‹£μ§€ μ•ŠλŠ”λ‹€.

    μ—μ΄μ „νŠΈ μ„Έμ…˜(`optin.AGENT_SESSION_MARKERS`)은 μ„€μ • 파일 `hub.database_url` 을 λ¬΄μ‹œν•˜λ―€λ‘œ(D-L10-6) "μ„€μ • 파일둜 켜라" λ₯Ό
    λ§ν•˜μ§€ μ•Šκ³  μ‚¬λžŒμ΄ λΆ€λ₯Ό λͺ…λ Ήλ§Œ μ€€λ‹€ β€” λ°”λ‘œ μ•ž "μ„€μ • νŒŒμΌμ„ λ¬΄μ‹œν•œλ‹€" 경고와 μ–΄κΈ‹λ‚˜μ§€ μ•Šκ²Œ(리뷰 L10 지적 3,
    `_confirmation_hint` 와 같은 갈래).
    """
    ingest = hub_ingest_command(case_ref)
    if optin.is_agent_session():
        return f"{NOT_OPTED_IN_HEAD} μ—μ΄μ „νŠΈ μ„Έμ…˜μ€ μ„€μ • 파일 opt-in 을 λ”°λ₯΄μ§€ μ•ŠλŠ”λ‹€ β€” μ‚¬λžŒμ΄ 직접: {ingest}"
    return (
        f"{NOT_OPTED_IN_HEAD} 켜렀면 μ„€μ • 파일 hub.database_url λ˜λŠ” λͺ…λ Ή μ•ž "
        f"{URL_PLACEHOLDER}. μ§€κΈˆ μ‹£κΈ°: {ingest}"
    )


def _confirmation_hint(ingest: str) -> str:
    """hub κ°€ μ‚¬μš©μž DB μ“°κΈ° 확인을 μš”κ΅¬ν–ˆλ‹€. μ—μ΄μ „νŠΈ μ„Έμ…˜(`optin.AGENT_SESSION_MARKERS`)은 확인을 λ¬΄μ‹œν•˜λ―€λ‘œ μ‚¬λžŒμ΄ λΆ€λ₯Ό λͺ…령을 μ•ˆλ‚΄ν•œλ‹€."""
    if optin.is_agent_session():
        return f"μ—μ΄μ „νŠΈ μ„Έμ…˜μ€ μ‚¬μš©μž DB μ“°κΈ° 확인을 λ„˜κΈ°μ§€ μ•ŠλŠ”λ‹€ β€” μ‚¬λžŒμ΄ 직접: {ingest}"
    return CONFIRMATION_HINT_HUMAN


def default_hint(leg_name: str, leg: LegResult, case_ref: str | None, run_id: str) -> str | None:
    """닀리가 hint λ₯Ό μ‹£μ§€ μ•Šμ•˜μ„ λ•Œ μ½˜μ†”μ΄ λΆ™μ΄λŠ” μ•ˆλ‚΄."""
    if leg.hint is not None or case_ref is None:
        return leg.hint
    if leg.status == STATUS_SKIPPED and leg_name == LEG_HUB and leg.code == NOT_OPTED_IN:
        return not_opted_in_hint(case_ref)
    if leg.status != STATUS_FAILED:
        return None
    if leg_name == LEG_HUB:
        return hub_recovery(leg.code, case_ref, run_id)
    if leg_name == LEG_LOCK and leg.code == LOCK_TIMEOUT:
        return f"같은 case 의 λ‹€λ₯Έ 훅이 λλ‚œ λ’€ {hub_backfill_command(case_ref, run_id)} β†’ {hub_ingest_command(case_ref)}"
    return f"{hub_backfill_command(case_ref, run_id)} β†’ {hub_ingest_command(case_ref)}"


# ── 좜λ ₯ ─────────────────────────────────────────────────────────────────


def echo_no_ingest(project: str, run_id: str, case_ref: str | None) -> None:
    """`--no-ingest` β€” hub ν›… 전체λ₯Ό κ±΄λ„ˆλ›΄λ‹€λŠ” ν•œ 쀄(Q-M2-4). `project` λŠ” 호좜자 μ‹œκ·Έλ‹ˆμ²˜ μœ μ§€μš©μ΄λ‹€(L10 β€” λ ˆκ±°μ‹œ 쀄 μ‚­μ œ)."""
    del project  # λ ˆκ±°μ‹œ ν™”λ©΄ μˆ˜λ™ 적재 쀄이 L10 μ—μ„œ μ‚¬λΌμ‘Œλ‹€(κ³„νš 2026-09-28 L10 Β§2.6)
    if case_ref is None:
        click.secho("\nhub ν›… κ±΄λ„ˆλœ€(--no-ingest)", fg="yellow")
        return
    revive = f"{hub_backfill_command(case_ref, run_id)} β†’ {hub_ingest_command(case_ref)}"
    click.secho(f"\nhub ν›… κ±΄λ„ˆλœ€(--no-ingest) β€” λ˜μ‚΄λ¦¬κΈ°: {revive}", fg="yellow")


def echo_skip(skip: Skip, project: str) -> None:
    """hub λŒ€μƒμ΄ μ•„λ‹Œ run β€” ν•œ 쀄(standalone 은 λΆ€λ₯΄μ§€ μ•ŠλŠ”λ‹€)."""
    reason = SKIP_REASONS.get(skip.code, skip.code)
    paths = " Β· ".join(f"{k}={v}" for k, v in skip.detail.items())
    suffix = f" ({paths})" if paths else ""
    click.secho(f"\nhub ν›… κ±΄λ„ˆλœ€: {skip.code} β€” {reason}{suffix}", fg="yellow")


def echo_smoke(case_ref: str) -> None:
    """`--limit` 슀λͺ¨ν¬ run β€” ν•œ 쀄(μž κΈˆΒ·μƒνƒœ 파일 μ—†μŒ, D-L6-10 Β· μ™„λ£Œ μ •μ˜ Β§0.4-2)."""
    click.secho(
        f"\nhub ν›… κ±΄λ„ˆλœ€: {SMOKE_RUN} β€” --limit 슀λͺ¨ν¬ run 은 λ²ˆλ“€Β·hub 에 μ‹£μ§€ μ•ŠλŠ”λ‹€ ({case_ref})",
        fg="yellow",
    )


def elapsed_seconds(result: HookResult) -> int | None:
    """ν›… 전체 κ²½κ³Ό 초(`started_at`β†’`finished_at`). μ‹œκ°μ΄ μ—†κ±°λ‚˜ ν˜•μ‹μ΄ λ‹€λ₯΄κ±°λ‚˜ 음수면 None."""
    if not result.started_at or not result.finished_at:
        return None
    try:
        started = datetime.strptime(result.started_at, HOOK_TIME_FORMAT)
        finished = datetime.strptime(result.finished_at, HOOK_TIME_FORMAT)
    except ValueError:
        return None
    seconds = int((finished - started).total_seconds())
    return seconds if seconds >= 0 else None


def echo_summary(result: HookResult, project: str) -> None:
    """λ‹€λ¦¬λ§ˆλ‹€ ν•œ 쀄 + μ‹€νŒ¨λ©΄ λ˜μ‚΄λ¦¬κΈ°(Β§2.7). 머리에 κ²½κ³Ό 초. λ ˆκ±°μ‹œ ν™”λ©΄ 쀄은 L10 μ—μ„œ μ—†μ–΄μ‘Œλ‹€."""
    elapsed = elapsed_seconds(result)
    suffix = f" Β· κ²½κ³Ό {elapsed} s" if elapsed is not None else ""
    click.secho(
        f"\nExperiment Hub ν›… β€” {result.case or project} Β· {result.run_id}{suffix}", fg="blue", bold=True
    )
    for name, leg in result.legs.items():
        _echo_leg(name, leg, result.case, result.run_id)


def _echo_leg(name: str, leg: LegResult, case_ref: str | None, run_id: str) -> None:
    label = f"  {LEG_LABELS.get(name, name):<10}"
    hint = default_hint(name, leg, case_ref, run_id)
    if leg.status == STATUS_OK:
        details = _ok_details(leg.detail)
        click.secho(f"{label} ok{f' ({details})' if details else ''}", fg="green")
    elif leg.status == STATUS_FAILED:
        click.secho(f"{label} μ‹€νŒ¨ β€” {leg.code}", fg="red", bold=True)
        if hint:
            click.secho(f"  {'':<10} λ˜μ‚΄λ¦¬κΈ°: {hint}", fg="red")
    elif leg.status == STATUS_SKIPPED:
        # hub opt-in μ—†μŒμ€ λΉ¨κ°„ 쀄(D-M2-18 β€” 이 run 은 /v1 에 μ—†λ‹€), κ·Έ λ°– κ±΄λ„ˆλœ€μ€ λ…Έλž‘
        color = "red" if (name, leg.code) == (LEG_HUB, NOT_OPTED_IN) else "yellow"
        click.secho(f"{label} κ±΄λ„ˆλœ€ β€” {leg.code}{f'. {hint}' if hint else ''}", fg=color)
    else:
        click.secho(f"{label} λŒμ§€ μ•ŠμŒ β€” {leg.code}", fg="yellow")


def _ok_details(detail: Mapping[str, Any]) -> str:
    parts = []
    for path, label in _BUNDLE_SUMMARY_KEYS:
        value: Any = detail
        for key in path:
            value = value.get(key) if isinstance(value, Mapping) else None
        if isinstance(value, str):
            parts.append(f"{label} {value}")
    return " Β· ".join(parts)


def echo_batch_hub(leg: LegResult, case_ref: str, run_ids: Iterable[str]) -> None:
    """batch 끝 hub 닀리 ν•œ 쀄(D-L6-12)."""
    ids = list(run_ids)
    _echo_leg(LEG_HUB, leg, case_ref, ids[0] if len(ids) == 1 else "<run>")


def echo_batch_summary(results: Iterable[HookResult | None]) -> None:
    """batch 끝 μš”μ•½ "ν›…: ok N Β· μ‹€νŒ¨ M Β· κ±΄λ„ˆλœ€ K"(κ±΄λ„ˆλœ€ = 훅이 돌 닀리가 μ—†μ—ˆλ˜ run)."""
    ok = failed = skipped = 0
    for result in results:
        if result is not None and not result.ok:
            failed += 1
        elif result is not None and any(leg.status == STATUS_OK for leg in result.legs.values()):
            ok += 1
        else:
            skipped += 1
    color = "red" if failed else "green"
    click.secho(f"ν›…: ok {ok} Β· μ‹€νŒ¨ {failed} Β· κ±΄λ„ˆλœ€ {skipped}", fg=color)


def echo_warning(message: str) -> None:
    click.secho(f"  ⚠ {message}", fg="yellow", err=True)


def echo_hook_crashed(code: str) -> None:
    """ν›… λ°”κΉ₯ 겹이 μ˜ˆμ™Έλ₯Ό μž‘μ•˜λ‹€ β€” run κ²°κ³ΌλŠ” κ·ΈλŒ€λ‘œλ‹€(D-L6-2)."""
    click.secho(f"\nExperiment Hub 훅이 예기치 μ•Šκ²Œ λ©ˆμ·„λ‹€ β€” {code} (run κ²°κ³ΌλŠ” λ””μŠ€ν¬μ— μžˆλ‹€)", fg="red")


def echo_status_write_failed(code: str) -> None:
    echo_warning(f"ν›… μƒνƒœ νŒŒμΌμ„ μ“°μ§€ λͺ»ν–ˆλ‹€ β€” {code}")