dev-strender's picture
feat: switch demo to FT + Opus article review (pipeline_exp_261021_ft_hybrid)
3dfc93b verified
Raw History Blame Contribute Delete
12.4 kB
"""ν›… μ½˜μ†” β€” 닀리별 ν•œ 쀄과 λ˜μ‚΄λ¦¬κΈ° λͺ…λ Ή 문ꡬ (κ³„νš 2026-09-27 L6 Β§2.6 ν‘œ Β· Β§2.7 Β· κ²€ν†  F16).
λ˜μ‚΄λ¦¬κΈ° λͺ…λ Ήμ˜ URL μžλ¦¬λŠ” 늘 `HUB_DATABASE_URL=…` 이닀 β€” 값은 어디에도 μ‹£μ§€ μ•ŠλŠ”λ‹€(D-L6-4). μ‹€νŒ¨ 쀄은 λΉ¨κ°• + λ˜μ‚΄λ¦¬κΈ°.
λ ˆκ±°μ‹œ ν™”λ©΄(`public`)은 L10 μ»·μ˜€λ²„λ‘œ μ½”λ“œμ—μ„œ μ‚¬λΌμ‘Œλ‹€ β€” λ ˆκ±°μ‹œ μˆ˜λ™ 적재 쀄은 μ—†λ‹€(κ³„νš 2026-09-28 L10 Β§2.6 Β· L6b 후속 1).
hub opt-in 이 μ—†μœΌλ©΄(`not_opted_in`) λΉ¨κ°„ μ€„λ‘œ "이 run 은 hub(/v1)에 μ—†λ‹€" 와 μΌœλŠ” λ²•Β·μ§€κΈˆ μ‹£λŠ” λͺ…령을 μ•ˆλ‚΄ν•œλ‹€(λ§ˆμŠ€ν„° D-M2-18).
μš”μ•½ λ¨Έλ¦¬μ—λŠ” ν›… 전체 κ²½κ³Ό 초(μƒνƒœ 파일의 `started_at`β†’`finished_at`, 잠금 λŒ€κΈ° 포함)λ₯Ό 뢙인닀 β€” 첫 싀데이터 run(U-L6a-1)μ—μ„œ
μ½˜μ†”λ§ŒμœΌλ‘œ hub 적재 μ‹œκ°„μ„ μ•Œ 수 μ—†μ—ˆκΈ° λ•Œλ¬Έμ΄λ‹€. μ‹œκ°μ΄ μ—†κ±°λ‚˜ ν˜•μ‹μ΄ λ‹€λ₯΄λ©΄ 뢙이지 μ•ŠλŠ”λ‹€.
"""
from __future__ import annotations
from collections.abc import Iterable, Mapping
from datetime import datetime
from typing import Any
import click
from solar_eval.hub_hook import optin
from solar_eval.hub_hook.types import (
CASE_BLOCKED_PREFIX,
CASE_FAILED_PREFIX,
FILTER_NO_MATCH,
HUB_ERR_CONFIRMATION_REQUIRED,
HUB_ERR_CONNECTION_FAILED,
HUB_FAILED_PREFIX,
LEG_BUNDLE,
LEG_HOOK,
LEG_HUB,
LEG_LOCK,
LEG_PUBLISH,
LOCK_TIMEOUT,
NOT_OPTED_IN,
RUN_NOT_OK_PREFIX,
SCHEMA_MISMATCH,
SKIP_CASE_UNMAPPED,
SKIP_DATA_HOME_INSIDE_REPO,
SKIP_DATA_HOME_MISMATCH,
SKIP_NO_REPO_ROOT,
SKIP_NOT_REPO_MANAGED,
SMOKE_RUN,
STAGE_CASE_LOCK,
STATUS_FAILED,
STATUS_OK,
STATUS_SKIPPED,
HookResult,
LegResult,
Skip,
)
URL_PLACEHOLDER = "HUB_DATABASE_URL=…"
#: μƒνƒœ 파일 μ‹œκ° ν˜•μ‹(`hook._now` 와 κ°™λ‹€).
HOOK_TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
LEG_LABELS = {
LEG_BUNDLE: "λ²ˆλ“€",
LEG_HUB: "hub 적재",
LEG_LOCK: "case 잠금",
LEG_HOOK: "ν›…",
LEG_PUBLISH: "채점 λ°œν–‰",
}
SKIP_REASONS = {
SKIP_NOT_REPO_MANAGED: "레포 관리 ν”„λ‘œμ νŠΈκ°€ μ•„λ‹ˆλ‹€",
SKIP_NO_REPO_ROOT: "레포 config λ£¨νŠΈκ°€ μ—†λ‹€(--projects-dir λ“±)",
SKIP_CASE_UNMAPPED: "project κ°€ case 에 λ§€ν•‘λ˜μ§€ μ•ŠλŠ”λ‹€ β€” make bundle-census 둜 확인",
SKIP_DATA_HOME_MISMATCH: "μ—”μ§„ μ•„μΉ΄μ΄λΈŒκ°€ 데이터 ν™ˆ 밖이닀",
SKIP_DATA_HOME_INSIDE_REPO: "데이터 ν™ˆμ΄ 레포 μ•ˆμ΄λ‹€ β€” ν›… νŒŒμƒλ¬Όμ€ git 밖에 λ‘”λ‹€(CBS_DATA_HOME)",
}
#: μš”μ•½ 쀄에 μ‹£λŠ” λ²ˆλ“€ 닀리 detail ν‚€(μˆœμ„œλŒ€λ‘œ).
_BUNDLE_SUMMARY_KEYS = (
(("backfill", "wrap"), "wrap"),
(("backfill", "eval_wrap"), "eval-wrap"),
(("rescore", "status"), "μž¬μ±„μ "),
(("indexes",), "색인"),
)
# ── λ˜μ‚΄λ¦¬κΈ° λͺ…λ Ή ────────────────────────────────────────────────────────
def hub_ingest_command(case_ref: str) -> str:
return f"{URL_PLACEHOLDER} make hub-ingest CASE={case_ref} CONFIRM=local"
def hub_backfill_command(case_ref: str, run_id: str) -> str:
return f'make hub-backfill CASE={case_ref} ARGS="--run {run_id}"'
def hub_recovery(code: str, case_ref: str, run_id: str) -> str:
"""hub 닀리 μ‹€νŒ¨ μ½”λ“œ β†’ λ˜μ‚΄λ¦¬κΈ° 문ꡬ(Β§2.6 ν‘œ)."""
ingest = hub_ingest_command(case_ref)
if code == f"{HUB_FAILED_PREFIX}{HUB_ERR_CONFIRMATION_REQUIRED}":
return _confirmation_hint(ingest)
if code == f"{HUB_FAILED_PREFIX}{HUB_ERR_CONNECTION_FAILED}":
return f"make experiment-hub-up λ’€ {ingest}"
if code == f"{CASE_FAILED_PREFIX}{STAGE_CASE_LOCK}":
return f"μ•ž μ μž¬κ°€ λλ‚œ λ’€ {ingest}"
if code.startswith((CASE_BLOCKED_PREFIX, CASE_FAILED_PREFIX)):
return f"{ingest} ν›„ 이슈 ν‘œ(/v1/{case_ref}) 확인"
if code == SCHEMA_MISMATCH:
return (
f"hub μŠ€ν‚€λ§ˆ 판이 λ‹€λ₯΄λ‹€ β€” {URL_PLACEHOLDER} make hub-m1-local CASE={case_ref} "
"CONFIRM=local SKIP_PULL=1 FROM=backfill (주의: μž¬κ΅¬μ„±μ€ λͺ¨λ“  case λ₯Ό μ§€μš΄λ‹€, case λ§ˆλ‹€ λ‹€μ‹œ)"
)
if code.startswith(RUN_NOT_OK_PREFIX):
return f"{hub_backfill_command(case_ref, run_id)} λ’€ {ingest}"
if code == FILTER_NO_MATCH:
return f"pnpm μ›Œν¬μŠ€νŽ˜μ΄μŠ€μ— @poc/eval-store κ°€ μ—†λ‹€ β€” pnpm install λ’€ {ingest}"
return ingest
#: μ‚¬μš©μž DB μ“°κΈ° 확인 μ•ˆλ‚΄(μ‚¬λžŒ μ…Έ) β€” μ˜μ† `hub.confirm` 은 λ§ˆμŠ€ν„° Β§4 κ·œμΉ™ 9 의 μ˜λ„λœ μ˜ˆμ™Έλ‹€(L10 Β§2.6 Β· κ²€ν†  F1-3).
CONFIRMATION_HINT_HUMAN = (
"μ‚¬μš©μž DB 에 μ“°λ €λ©΄ λͺ…λ Ή μ•žμ— SOLAR_EVAL_HUB_CONFIRM=local 을 λΆ™μ΄κ±°λ‚˜ μ„€μ • 파일 hub.confirm: local"
"(μ‚¬λžŒ μ…Έλ§Œ Β· URL 도 섀정에 μžˆμ„ λ•Œλ§Œ β€” κ·œμΉ™ 9 의 μ˜λ„λœ μ˜ˆμ™Έ)"
)
#: `not_opted_in` λΉ¨κ°„ μ€„μ˜ 머리(μ‚¬λžŒΒ·μ—μ΄μ „νŠΈ 곡톡).
NOT_OPTED_IN_HEAD = "hub opt-in μ—†μŒ β€” 이 run 은 hub(/v1)에 μ—†λ‹€."
def not_opted_in_hint(case_ref: str) -> str:
"""hub opt-in 이 μ—†λ‹€ β€” λΉ¨κ°„ 쀄 μ•ˆλ‚΄(λ§ˆμŠ€ν„° D-M2-18 Β· L10 Β§2.6 ν‘œ "λ‘˜ λ‹€ μ—†μŒ"). URL 값은 μ‹£μ§€ μ•ŠλŠ”λ‹€.
μ—μ΄μ „νŠΈ μ„Έμ…˜(`optin.AGENT_SESSION_MARKERS`)은 μ„€μ • 파일 `hub.database_url` 을 λ¬΄μ‹œν•˜λ―€λ‘œ(D-L10-6) "μ„€μ • 파일둜 켜라" λ₯Ό
λ§ν•˜μ§€ μ•Šκ³  μ‚¬λžŒμ΄ λΆ€λ₯Ό λͺ…λ Ήλ§Œ μ€€λ‹€ β€” λ°”λ‘œ μ•ž "μ„€μ • νŒŒμΌμ„ λ¬΄μ‹œν•œλ‹€" 경고와 μ–΄κΈ‹λ‚˜μ§€ μ•Šκ²Œ(리뷰 L10 지적 3,
`_confirmation_hint` 와 같은 갈래).
"""
ingest = hub_ingest_command(case_ref)
if optin.is_agent_session():
return f"{NOT_OPTED_IN_HEAD} μ—μ΄μ „νŠΈ μ„Έμ…˜μ€ μ„€μ • 파일 opt-in 을 λ”°λ₯΄μ§€ μ•ŠλŠ”λ‹€ β€” μ‚¬λžŒμ΄ 직접: {ingest}"
return (
f"{NOT_OPTED_IN_HEAD} 켜렀면 μ„€μ • 파일 hub.database_url λ˜λŠ” λͺ…λ Ή μ•ž "
f"{URL_PLACEHOLDER}. μ§€κΈˆ μ‹£κΈ°: {ingest}"
)
def _confirmation_hint(ingest: str) -> str:
"""hub κ°€ μ‚¬μš©μž DB μ“°κΈ° 확인을 μš”κ΅¬ν–ˆλ‹€. μ—μ΄μ „νŠΈ μ„Έμ…˜(`optin.AGENT_SESSION_MARKERS`)은 확인을 λ¬΄μ‹œν•˜λ―€λ‘œ μ‚¬λžŒμ΄ λΆ€λ₯Ό λͺ…령을 μ•ˆλ‚΄ν•œλ‹€."""
if optin.is_agent_session():
return f"μ—μ΄μ „νŠΈ μ„Έμ…˜μ€ μ‚¬μš©μž DB μ“°κΈ° 확인을 λ„˜κΈ°μ§€ μ•ŠλŠ”λ‹€ β€” μ‚¬λžŒμ΄ 직접: {ingest}"
return CONFIRMATION_HINT_HUMAN
def default_hint(leg_name: str, leg: LegResult, case_ref: str | None, run_id: str) -> str | None:
"""닀리가 hint λ₯Ό μ‹£μ§€ μ•Šμ•˜μ„ λ•Œ μ½˜μ†”μ΄ λΆ™μ΄λŠ” μ•ˆλ‚΄."""
if leg.hint is not None or case_ref is None:
return leg.hint
if leg.status == STATUS_SKIPPED and leg_name == LEG_HUB and leg.code == NOT_OPTED_IN:
return not_opted_in_hint(case_ref)
if leg.status != STATUS_FAILED:
return None
if leg_name == LEG_HUB:
return hub_recovery(leg.code, case_ref, run_id)
if leg_name == LEG_LOCK and leg.code == LOCK_TIMEOUT:
return f"같은 case 의 λ‹€λ₯Έ 훅이 λλ‚œ λ’€ {hub_backfill_command(case_ref, run_id)} β†’ {hub_ingest_command(case_ref)}"
return f"{hub_backfill_command(case_ref, run_id)} β†’ {hub_ingest_command(case_ref)}"
# ── 좜λ ₯ ─────────────────────────────────────────────────────────────────
def echo_no_ingest(project: str, run_id: str, case_ref: str | None) -> None:
"""`--no-ingest` β€” hub ν›… 전체λ₯Ό κ±΄λ„ˆλ›΄λ‹€λŠ” ν•œ 쀄(Q-M2-4). `project` λŠ” 호좜자 μ‹œκ·Έλ‹ˆμ²˜ μœ μ§€μš©μ΄λ‹€(L10 β€” λ ˆκ±°μ‹œ 쀄 μ‚­μ œ)."""
del project # λ ˆκ±°μ‹œ ν™”λ©΄ μˆ˜λ™ 적재 쀄이 L10 μ—μ„œ μ‚¬λΌμ‘Œλ‹€(κ³„νš 2026-09-28 L10 Β§2.6)
if case_ref is None:
click.secho("\nhub ν›… κ±΄λ„ˆλœ€(--no-ingest)", fg="yellow")
return
revive = f"{hub_backfill_command(case_ref, run_id)} β†’ {hub_ingest_command(case_ref)}"
click.secho(f"\nhub ν›… κ±΄λ„ˆλœ€(--no-ingest) β€” λ˜μ‚΄λ¦¬κΈ°: {revive}", fg="yellow")
def echo_skip(skip: Skip, project: str) -> None:
"""hub λŒ€μƒμ΄ μ•„λ‹Œ run β€” ν•œ 쀄(standalone 은 λΆ€λ₯΄μ§€ μ•ŠλŠ”λ‹€)."""
reason = SKIP_REASONS.get(skip.code, skip.code)
paths = " Β· ".join(f"{k}={v}" for k, v in skip.detail.items())
suffix = f" ({paths})" if paths else ""
click.secho(f"\nhub ν›… κ±΄λ„ˆλœ€: {skip.code} β€” {reason}{suffix}", fg="yellow")
def echo_smoke(case_ref: str) -> None:
"""`--limit` 슀λͺ¨ν¬ run β€” ν•œ 쀄(μž κΈˆΒ·μƒνƒœ 파일 μ—†μŒ, D-L6-10 Β· μ™„λ£Œ μ •μ˜ Β§0.4-2)."""
click.secho(
f"\nhub ν›… κ±΄λ„ˆλœ€: {SMOKE_RUN} β€” --limit 슀λͺ¨ν¬ run 은 λ²ˆλ“€Β·hub 에 μ‹£μ§€ μ•ŠλŠ”λ‹€ ({case_ref})",
fg="yellow",
)
def elapsed_seconds(result: HookResult) -> int | None:
"""ν›… 전체 κ²½κ³Ό 초(`started_at`β†’`finished_at`). μ‹œκ°μ΄ μ—†κ±°λ‚˜ ν˜•μ‹μ΄ λ‹€λ₯΄κ±°λ‚˜ 음수면 None."""
if not result.started_at or not result.finished_at:
return None
try:
started = datetime.strptime(result.started_at, HOOK_TIME_FORMAT)
finished = datetime.strptime(result.finished_at, HOOK_TIME_FORMAT)
except ValueError:
return None
seconds = int((finished - started).total_seconds())
return seconds if seconds >= 0 else None
def echo_summary(result: HookResult, project: str) -> None:
"""λ‹€λ¦¬λ§ˆλ‹€ ν•œ 쀄 + μ‹€νŒ¨λ©΄ λ˜μ‚΄λ¦¬κΈ°(Β§2.7). 머리에 κ²½κ³Ό 초. λ ˆκ±°μ‹œ ν™”λ©΄ 쀄은 L10 μ—μ„œ μ—†μ–΄μ‘Œλ‹€."""
elapsed = elapsed_seconds(result)
suffix = f" Β· κ²½κ³Ό {elapsed} s" if elapsed is not None else ""
click.secho(
f"\nExperiment Hub ν›… β€” {result.case or project} Β· {result.run_id}{suffix}", fg="blue", bold=True
)
for name, leg in result.legs.items():
_echo_leg(name, leg, result.case, result.run_id)
def _echo_leg(name: str, leg: LegResult, case_ref: str | None, run_id: str) -> None:
label = f" {LEG_LABELS.get(name, name):<10}"
hint = default_hint(name, leg, case_ref, run_id)
if leg.status == STATUS_OK:
details = _ok_details(leg.detail)
click.secho(f"{label} ok{f' ({details})' if details else ''}", fg="green")
elif leg.status == STATUS_FAILED:
click.secho(f"{label} μ‹€νŒ¨ β€” {leg.code}", fg="red", bold=True)
if hint:
click.secho(f" {'':<10} λ˜μ‚΄λ¦¬κΈ°: {hint}", fg="red")
elif leg.status == STATUS_SKIPPED:
# hub opt-in μ—†μŒμ€ λΉ¨κ°„ 쀄(D-M2-18 β€” 이 run 은 /v1 에 μ—†λ‹€), κ·Έ λ°– κ±΄λ„ˆλœ€μ€ λ…Έλž‘
color = "red" if (name, leg.code) == (LEG_HUB, NOT_OPTED_IN) else "yellow"
click.secho(f"{label} κ±΄λ„ˆλœ€ β€” {leg.code}{f'. {hint}' if hint else ''}", fg=color)
else:
click.secho(f"{label} λŒμ§€ μ•ŠμŒ β€” {leg.code}", fg="yellow")
def _ok_details(detail: Mapping[str, Any]) -> str:
parts = []
for path, label in _BUNDLE_SUMMARY_KEYS:
value: Any = detail
for key in path:
value = value.get(key) if isinstance(value, Mapping) else None
if isinstance(value, str):
parts.append(f"{label} {value}")
return " Β· ".join(parts)
def echo_batch_hub(leg: LegResult, case_ref: str, run_ids: Iterable[str]) -> None:
"""batch 끝 hub 닀리 ν•œ 쀄(D-L6-12)."""
ids = list(run_ids)
_echo_leg(LEG_HUB, leg, case_ref, ids[0] if len(ids) == 1 else "<run>")
def echo_batch_summary(results: Iterable[HookResult | None]) -> None:
"""batch 끝 μš”μ•½ "ν›…: ok N Β· μ‹€νŒ¨ M Β· κ±΄λ„ˆλœ€ K"(κ±΄λ„ˆλœ€ = 훅이 돌 닀리가 μ—†μ—ˆλ˜ run)."""
ok = failed = skipped = 0
for result in results:
if result is not None and not result.ok:
failed += 1
elif result is not None and any(leg.status == STATUS_OK for leg in result.legs.values()):
ok += 1
else:
skipped += 1
color = "red" if failed else "green"
click.secho(f"ν›…: ok {ok} Β· μ‹€νŒ¨ {failed} Β· κ±΄λ„ˆλœ€ {skipped}", fg=color)
def echo_warning(message: str) -> None:
click.secho(f" ⚠ {message}", fg="yellow", err=True)
def echo_hook_crashed(code: str) -> None:
"""ν›… λ°”κΉ₯ 겹이 μ˜ˆμ™Έλ₯Ό μž‘μ•˜λ‹€ β€” run κ²°κ³ΌλŠ” κ·ΈλŒ€λ‘œλ‹€(D-L6-2)."""
click.secho(f"\nExperiment Hub 훅이 예기치 μ•Šκ²Œ λ©ˆμ·„λ‹€ β€” {code} (run κ²°κ³ΌλŠ” λ””μŠ€ν¬μ— μžˆλ‹€)", fg="red")
def echo_status_write_failed(code: str) -> None:
echo_warning(f"ν›… μƒνƒœ νŒŒμΌμ„ μ“°μ§€ λͺ»ν–ˆλ‹€ β€” {code}")