dev-strender's picture
feat: switch demo to FT + Opus article review (pipeline_exp_261021_ft_hybrid)
3dfc93b verified
Raw History Blame Contribute Delete
12.2 kB
"""run ์ข…๋ฃŒ ํ›… ์˜ค์ผ€์ŠคํŠธ๋ ˆ์ดํ„ฐ (๊ณ„ํš 2026-09-27 L6 ยง2.1 ยท D-L6-2 ยท D-L6-3 ยท D-L6-11 ยท D-L6-12).
์ˆœ์„œ = **๋ฒˆ๋“ค โ†’ hub**(๋ ˆ๊ฑฐ์‹œ ์ ์žฌ๋Š” ๋ถ€๋ฅด์ง€ ์•Š๋Š”๋‹ค, D-M2-4 (b)). hub ๋Œ€์ƒ run ์€ case ์ž ๊ธˆ **์•ˆ**์—์„œ ๋‘ ๋‹ค๋ฆฌ๋ฅผ ๋Œ๊ณ  ์ƒํƒœ ํŒŒ์ผ์„ ์“ด
๋’ค ์ž ๊ธˆ ๋ฐ–์—์„œ ์ฝ˜์†” ์š”์•ฝ์„ ๋‚ธ๋‹ค.
**์˜ˆ์™ธ ๋ด‰์‡„(D-L6-2 ยท ๊ฒ€ํ†  F1).** ๋‹ค๋ฆฌ ํ˜ธ์ถœ๋งˆ๋‹ค `except Exception` โ†’ `failed:<leg>_exception:<Type>`(ํƒ€์ž… ์ด๋ฆ„๋งŒ). ์ƒํƒœ ํŒŒ์ผ ์“ฐ๊ธฐ
์‹คํŒจ๋Š” ์ฝ˜์†” ๊ฒฝ๊ณ ๋กœ ๊ฐ•๋“ฑํ•˜๊ณ , ํ›… ์ „์ฒด๋„ ํ•œ ๊ฒน ๋” ๊ฐ์‹ธ run ๋ช…๋ น์ด ํ›… ๋•Œ๋ฌธ์— ์ข…๋ฃŒ 1ยทํŠธ๋ ˆ์ด์Šค๋ฐฑ์ด ๋˜์ง€ ์•Š๊ฒŒ ํ•œ๋‹ค. batch ์—์„œ
ํ›… ์˜ˆ์™ธ๊ฐ€ ํ”„๋กฌํ”„ํŠธ ์‹คํŒจ๋กœ ์„ธ์ง€์ง€๋„ ์•Š๋Š”๋‹ค. `BaseException`(KeyboardInterruptยทpytest tripwire)์€ ์žก์ง€ ์•Š๋Š”๋‹ค.
**์˜ˆ์™ธ ์ƒ์„ธ๋Š” ๋กœ๊ทธ๋กœ๋งŒ(๋ฆฌ๋ทฐ ์ง€์  7).** ์ฝ˜์†”ยท์ƒํƒœ ํŒŒ์ผ์—๋Š” ํƒ€์ž… ์ด๋ฆ„๋งŒ ์‹ฃ๊ณ (D-L6-2), ํŠธ๋ ˆ์ด์Šค๋ฐฑ์€ ์ด ๋ชจ๋“ˆ ๋กœ๊ฑฐ์— ๋‚จ๊ธด๋‹ค โ€”
๊ธฐ๋ณธ์€ DEBUG(๋ณดํ†ต ์„ค์ •์—์„œ๋Š” ๋ณด์ด์ง€ ์•Š๋Š”๋‹ค), `SOLAR_EVAL_HOOK_DEBUG=1` ์ด๋ฉด WARNING(stderr). ํŠธ๋ ˆ์ด์Šค๋ฐฑ์€ ์ž…๋ ฅ ์กฐ๊ฐ์„ ๋‹ด์„ ์ˆ˜
์žˆ์œผ๋‹ˆ ํ˜„์žฅ ์ง„๋‹จ ๋•Œ๋งŒ ์ผ ๋‹ค.
**์Šค๋ชจํฌ run ์€ ์ž ๊ทธ์ง€ ์•Š๋Š”๋‹ค(D-L6-3 "๋Œ ๋‹ค๋ฆฌ๊ฐ€ ์—†์œผ๋ฉด ์ž ๊ทธ์ง€ ์•Š๋Š”๋‹ค" ยท D-L6-10).** hub ๋Œ€์ƒ์ด์–ด๋„ `--limit` ์Šค๋ชจํฌ๋ฉด ์ž ๊ธˆยท์ƒํƒœ
ํŒŒ์ผ ์—†์ด ๋‘ ๋‹ค๋ฆฌ `skipped:smoke_run` ํ•œ ์ค„๋กœ ๋๋‚œ๋‹ค โ€” ๋‹ค๋ฅธ ํ›…์ด ์ž ๊ธˆ์„ ์ฅ๊ณ  ์žˆ์–ด๋„ ๊ธฐ๋‹ค๋ฆฌ๊ฑฐ๋‚˜ `lock_timeout` ์ด ๋˜์ง€ ์•Š๋Š”๋‹ค.
์ข…๋ฃŒ ์ฝ”๋“œ๋Š” ํ˜ธ์ถœ์ž๊ฐ€ `exit_code_for(result, strict)` ๋กœ ์ •ํ•œ๋‹ค โ€” ๊ธฐ๋ณธ 0, strict ์ด๊ณ  ์‹คํŒจ ๋‹ค๋ฆฌ๊ฐ€ ์žˆ์œผ๋ฉด 3.
**opt-in ํ•œ ๊ณณ(๊ณ„ํš 2026-09-28 L10 ยง2.6 ยท D-L10-6).** `gated_hub_leg` ๊ฐ€ `optin.resolve_hub_optin(config=t.hub_config)` ๋กœ
ํ™˜๊ฒฝ ๋ณ€์ˆ˜ > ์„ค์ • ํŒŒ์ผ `hub:` ์ ˆ์„ ํ‘ผ๋‹ค. ์„ค์ • ์ถœ์ฒ˜๋ฉด URL ์€ ์ž์‹ ํ™˜๊ฒฝ(`child_env`)์œผ๋กœ๋งŒ ๊ฐ€๊ณ , ์„ค์ • ๊ฐ’์€ ์ง„๋‹จ ๊ฐ€๋ฆผยท์ƒํƒœ ํŒŒ์ผ
๊ฑฐ๋ถ€ ๋ชฉ๋ก(`optin.config_secrets`)์— ๋”ํ•œ๋‹ค. ํ˜ธ์ถœ์ž(`runs start`ยท`resume`ยทbatch ๋ยท`runs eval` ๋ฐœํ–‰) ์‹œ๊ทธ๋‹ˆ์ฒ˜๋Š” ๊ทธ๋Œ€๋กœ๋‹ค.
"""
from __future__ import annotations
import logging
from collections.abc import Callable, Sequence
from contextlib import ExitStack
from datetime import datetime, timezone
from pathlib import Path
from typing import TYPE_CHECKING
from solar_eval.hub_hook import bundle_leg, console, hub_leg, lock, optin, status, target
from solar_eval.hub_hook.types import (
BATCH_PENDING,
EXIT_HOOK_FAILED,
HOOK_LOCK_TIMEOUT_SECONDS,
LEG_BUNDLE,
LEG_HOOK,
LEG_HUB,
LEG_LOCK,
LEG_STATUS_FILE,
LOCK_TIMEOUT,
NO_BUNDLE,
NO_WORKSPACE,
NOT_OPTED_IN,
SKIP_STANDALONE,
SMOKE_RUN,
HookMode,
HookOptions,
HookResult,
LegResult,
Skip,
Target,
bundle_ready,
exception_code,
hub_run_id,
)
if TYPE_CHECKING:
from solar_eval.cli.config import CLIConfig
_LOG = logging.getLogger(__name__)
def _now() -> str:
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def _contained_code(leg: str, exc: Exception) -> str:
"""๋ด‰์‡„ํ•œ ์˜ˆ์™ธ โ†’ ์ฝ”๋“œ(ํƒ€์ž… ์ด๋ฆ„๋งŒ, D-L6-2). ํŠธ๋ ˆ์ด์Šค๋ฐฑ์€ ๋กœ๊ฑฐ๋กœ๋งŒ โ€” ์ฝ˜์†”ยท์ƒํƒœ ํŒŒ์ผ์—๋Š” ์‹ฃ์ง€ ์•Š๋Š”๋‹ค."""
code = exception_code(leg, exc)
level = logging.WARNING if optin.hook_debug_requested() else logging.DEBUG
_LOG.log(level, "Experiment Hub ํ›… ์˜ˆ์™ธ ๋ด‰์‡„ โ€” %s", code, exc_info=exc)
return code
def _contained(leg: str, fn: Callable[..., LegResult], *args, **kwargs) -> LegResult:
"""๋‹ค๋ฆฌ ํ•˜๋‚˜๋ฅผ ๋Œ๋ฆฌ๊ณ  ์˜ˆ๊ธฐ์น˜ ์•Š์€ ์˜ˆ์™ธ๋ฅผ `<leg>_exception:<Type>` ์‹คํŒจ๋กœ ๊ฐ•๋“ฑํ•œ๋‹ค(D-L6-2)."""
try:
return fn(*args, **kwargs)
except Exception as exc: # ๋‹ค๋ฆฌ ๊ฒฝ๊ณ„ โ€” ์–ด๋–ค ์˜ˆ์™ธ๋„ run ๋ช…๋ น์„ ์‹คํŒจ๋กœ ๋งŒ๋“ค์ง€ ์•Š๋Š”๋‹ค
return LegResult.failed(_contained_code(leg, exc))
def exit_code_for(result: HookResult | None, strict: bool) -> int:
"""ํ›… ๊ฒฐ๊ณผ โ†’ run ๋ช…๋ น ์ข…๋ฃŒ ์ฝ”๋“œ(D-L6-11). ๊ธฐ๋ณธ 0, strict ์ด๊ณ  ์‹คํŒจ ๋‹ค๋ฆฌ๊ฐ€ ์žˆ์œผ๋ฉด 3."""
return EXIT_HOOK_FAILED if strict and result is not None and not result.ok else 0
def run_hook(
cfg: CLIConfig, project: str, run_dir: Path, options: HookOptions
) -> HookResult | None:
"""run ํ•˜๋‚˜๊ฐ€ ๋๋‚œ ๋’ค์˜ ํ›…. hub ๋Œ€์ƒ์ด ์•„๋‹ˆ๋ฉด(standaloneยท`--no-ingest`) None ๋˜๋Š” ๊ฑด๋„ˆ๋œ€ ๊ฒฐ๊ณผ."""
run_id = Path(run_dir).name
try:
return _run_hook(cfg, project, Path(run_dir), run_id, options)
except Exception as exc: # ํ›… ๋ฐ”๊นฅ ๊ฒน(D-L6-2) โ€” ์ฝ˜์†”์— ์•Œ๋ฆฌ๊ณ  ๊ฒฐ๊ณผ๋กœ ๋Œ๋ ค์ค€๋‹ค
code = _contained_code(LEG_HOOK, exc)
console.echo_hook_crashed(code)
return HookResult(run_id, None, {LEG_HOOK: LegResult.failed(code)})
def _run_hook(
cfg: CLIConfig, project: str, run_dir: Path, run_id: str, options: HookOptions
) -> HookResult | None:
if options.no_ingest:
if cfg.repo_root is not None:
console.echo_no_ingest(project, run_id, target.case_ref_hint(cfg, project))
return None
resolved = target.resolve_target(cfg, project)
if isinstance(resolved, Skip):
if resolved.code == SKIP_STANDALONE:
return None
console.echo_skip(resolved, project)
legs = {
LEG_BUNDLE: LegResult.skipped(resolved.code),
LEG_HUB: LegResult.skipped(resolved.code),
}
return HookResult(run_id, None, legs, _now(), _now())
if options.smoke:
# ๋Œ ๋‹ค๋ฆฌ๊ฐ€ ์—†๋‹ค โ€” ์ž ๊ทธ์ง€ ์•Š๊ณ  ์ƒํƒœ ํŒŒ์ผ๋„ ์“ฐ์ง€ ์•Š๋Š”๋‹ค(D-L6-3 ยท D-L6-10, ๋ฆฌ๋ทฐ ์ง€์  1).
console.echo_smoke(resolved.case_ref)
smoke = {LEG_BUNDLE: LegResult.skipped(SMOKE_RUN), LEG_HUB: LegResult.skipped(SMOKE_RUN)}
return HookResult(run_id, resolved.case_ref, smoke, _now(), _now())
result = _run_target(resolved, run_dir, run_id, options)
console.echo_summary(result, project)
return result
def _run_target(t: Target, run_dir: Path, run_id: str, options: HookOptions) -> HookResult:
"""case ์ž ๊ธˆ ์•ˆ์—์„œ ๋‘ ๋‹ค๋ฆฌ โ†’ ์ƒํƒœ ํŒŒ์ผ(D-L6-3). ์ž ๊ธˆ ํ•œ๋„๋ฅผ ๋„˜๊ธฐ๋ฉด ๋‘ ๋‹ค๋ฆฌ ๋ชจ๋‘ not_run."""
started = _now()
with ExitStack() as stack:
try:
held = stack.enter_context(
lock.case_lock(t.account, t.case, timeout=HOOK_LOCK_TIMEOUT_SECONDS)
)
except Exception as exc: # ์ž ๊ธˆ ํŒŒ์ผ์„ ๋งŒ๋“ค ์ˆ˜ ์—†๋‹ค โ€” ๋‹ค๋ฆฌ๋ฅผ ๋Œ๋ฆฌ์ง€ ์•Š๋Š”๋‹ค
legs = _lock_failed(_contained_code(LEG_LOCK, exc))
else:
legs = (
_legs_in_lock(t, run_dir, run_id, options) if held else _lock_failed(LOCK_TIMEOUT)
)
result = HookResult(run_id, t.case_ref, legs, started, _now())
_write_status(t, result, options)
return result
def _lock_failed(code: str) -> dict[str, LegResult]:
return {
LEG_LOCK: LegResult.failed(code),
LEG_BUNDLE: LegResult.not_run(),
LEG_HUB: LegResult.not_run(),
}
def _legs_in_lock(
t: Target, run_dir: Path, run_id: str, options: HookOptions
) -> dict[str, LegResult]:
bundle = _contained(LEG_BUNDLE, bundle_leg.run_bundle_leg, t, run_dir, run_id)
if options.mode is HookMode.BATCH_ITEM:
hub = LegResult.not_run(BATCH_PENDING)
elif not bundle_ready(bundle):
hub = LegResult.skipped(NO_BUNDLE)
else:
# `--expect-run` ์€ ๋ฒˆ๋“ค์˜ run_id(run.json) โ€” ๋””๋ ‰ํ† ๋ฆฌ ์ด๋ฆ„๊ณผ ๋‹ค๋ฅผ ์ˆ˜ ์žˆ๋‹ค(๋ฆฌ๋ทฐ ์ง€์  3). ์ƒํƒœ ํŒŒ์ผ ์ด๋ฆ„์€ ๋””๋ ‰ํ† ๋ฆฌ ์ด๋ฆ„.
hub = gated_hub_leg(t, (hub_run_id(bundle, run_id),))
return {LEG_BUNDLE: bundle, LEG_HUB: hub}
def gated_hub_leg(t: Target, run_ids: Sequence[str]) -> LegResult:
"""hub ๋‹ค๋ฆฌ ๊ฒŒ์ดํŠธ ํ•œ ๊ณณ โ€” opt-in(D-L6-4 ยท L10 D-L10-6)ยท์›Œํฌ์ŠคํŽ˜์ด์Šค(C-L6-9)๋ฅผ ๋ณธ ๋’ค `hub_leg.run_hub_leg`(์˜ˆ์™ธ ๋ด‰์‡„).
ํ˜ธ์ถœ์ž๊ฐ€ case ์ž ๊ธˆ์„ ์ฅ๊ณ  ๋ฒˆ๋“ค ์œ ๋ฌด๋ฅผ ์ด๋ฏธ ๋ดค๋‹ค๊ณ  ๊ฐ€์ •ํ•œ๋‹ค. L6b `runs eval` ์ƒˆ ๊ฒฝ๋กœ โ‘ฃ ๋„ ์ด ํ•จ์ˆ˜๋ฅผ ๋ถ€๋ฅธ๋‹ค.
opt-in ํ•ด์„ ๊ฒฝ๊ณ (์„ค์ • ๋ชจ์–‘ ์˜ค๋ฅ˜ยท์—์ด์ „ํŠธ ์„ธ์…˜์—์„œ ์„ค์ • ๋ฌด์‹œ)๋Š” ๊ฑด๋„ˆ๋›ฐ๊ธฐ ์ „์— ๋จผ์ € ์ฐ๋Š”๋‹ค โ€” ๊ฐ’์€ ์‹ฃ์ง€ ์•Š๋Š”๋‹ค.
"""
resolved = optin.resolve_hub_optin(config=t.hub_config)
for warning in resolved.warnings:
console.echo_warning(warning)
if not resolved.opted_in:
return LegResult.skipped(NOT_OPTED_IN)
if target.workspace_root() is None:
return LegResult.skipped(NO_WORKSPACE)
if resolved.confirm.warning is not None:
console.echo_warning(resolved.confirm.warning)
return _contained(
LEG_HUB,
hub_leg.run_hub_leg,
t,
tuple(run_ids),
confirm_local=resolved.confirm.confirm_local,
child_env=resolved.child_env,
secrets=resolved.secrets,
)
def status_secrets(t: Target) -> tuple[str, ...]:
"""์ƒํƒœ ํŒŒ์ผ ๊ฑฐ๋ถ€ ๋ชฉ๋ก์— ๋”ํ•  ์„ค์ • ์ถœ์ฒ˜ ๋น„๋ฐ€(ยง2.6 "๋น„๋ฐ€") โ€” `runs eval` ๋ฐœํ–‰๋„ ๊ฐ™์€ ํ•จ์ˆ˜๋ฅผ ์“ด๋‹ค."""
return optin.config_secrets(t.hub_config)
def _write_status(t: Target, result: HookResult, options: HookOptions) -> None:
try:
status.write_status(
t, result, options.command, strict=options.strict, secrets=status_secrets(t)
)
except Exception as exc: # ์ƒํƒœ ํŒŒ์ผ์€ ๋ณด์กฐ ๊ธฐ๋ก โ€” ์“ฐ๊ธฐ ์‹คํŒจ๋Š” ๊ฒฝ๊ณ ๋กœ ๊ฐ•๋“ฑ(D-L6-2)
console.echo_status_write_failed(_contained_code(LEG_STATUS_FILE, exc))
def finish_batch(
cfg: CLIConfig,
project: str,
results: Sequence[HookResult | None],
options: HookOptions,
) -> LegResult | None:
"""batch ๋ hub ๋‹ค๋ฆฌ ํ•œ ๋ฒˆ(D-L6-12) โ€” ๋ฒˆ๋“ค์ด ์ƒ๊ธด run ๋“ค์„ `--expect-run` ์œผ๋กœ, case ์ž ๊ธˆ ์•ˆ์—์„œ.
๋๋‚˜๋ฉด run ๋งˆ๋‹ค ์ƒํƒœ ํŒŒ์ผ์˜ `hub` ๋ฅผ `status.update_legs` ๋กœ ๋‹ค์‹œ ์“ด๋‹ค โ€” ๋ฒˆ๋“ค์ด ์—†์–ด ๋ ๋‹ค๋ฆฌ์— ๋“ค์ง€ ๋ชปํ•œ run ์€
`skipped:no_bundle` ๋กœ(์ƒํƒœ ํŒŒ์ผ์— `batch_pending` ์ด ๋‚จ์ง€ ์•Š๊ฒŒ). ๋ ๋‹ค๋ฆฌ๋ฅผ ๋Œ์ง€ ์•Š์•˜์œผ๋ฉด None.
"""
try:
return _finish_batch(cfg, project, results, options)
except Exception as exc: # ํ›… ๋ฐ”๊นฅ ๊ฒน(D-L6-2)
code = _contained_code(LEG_HOOK, exc)
console.echo_hook_crashed(code)
return LegResult.failed(code)
def _finish_batch(
cfg: CLIConfig,
project: str,
results: Sequence[HookResult | None],
options: HookOptions,
) -> LegResult | None:
targeted = [r for r in results if r is not None and r.case is not None]
ready = [r for r in targeted if bundle_ready(r.legs.get(LEG_BUNDLE))]
# ๋ฒˆ๋“ค์ด ์—†์–ด ๋ ๋‹ค๋ฆฌ์— ๋“ค์ง€ ๋ชปํ•œ run โ€” ์ƒํƒœ ํŒŒ์ผ์˜ `batch_pending` ์„ `no_bundle` ๋กœ ๋‹ซ๋Š”๋‹ค(l6a-verify).
no_bundle = [r.run_id for r in targeted if r not in ready and _hub_pending(r)]
if options.no_ingest or not (ready or no_bundle):
return None
resolved = target.resolve_target(cfg, project)
if isinstance(resolved, Skip):
return None
# `--expect-run` ์€ ๋ฒˆ๋“ค์˜ run_id(run.json), ์ƒํƒœ ํŒŒ์ผ์€ ๋””๋ ‰ํ† ๋ฆฌ ์ด๋ฆ„(๋ฆฌ๋ทฐ ์ง€์  3).
expect = [hub_run_id(r.legs.get(LEG_BUNDLE), r.run_id) for r in ready]
leg: LegResult | None = None
with lock.case_lock(resolved.account, resolved.case, timeout=HOOK_LOCK_TIMEOUT_SECONDS) as held:
if ready:
leg = gated_hub_leg(resolved, expect) if held else LegResult.failed(LOCK_TIMEOUT)
for result in ready:
_update_hub_leg(resolved, result.run_id, leg)
for run_id in no_bundle:
_update_hub_leg(resolved, run_id, LegResult.skipped(NO_BUNDLE))
if leg is not None:
console.echo_batch_hub(leg, resolved.case_ref, expect)
return leg
def _hub_pending(result: HookResult) -> bool:
"""batch ํ•œ run ์˜ hub ๋‹ค๋ฆฌ๊ฐ€ ๋ ๋‹ค๋ฆฌ๋ฅผ ๊ธฐ๋‹ค๋ฆฌ๋Š” ์ค‘์ธ๊ฐ€(`not_run:batch_pending`)."""
hub = result.legs.get(LEG_HUB)
return hub is not None and hub.code == BATCH_PENDING
def _update_hub_leg(t: Target, run_id: str, leg: LegResult) -> None:
try:
status.update_legs(t, run_id, {LEG_HUB: leg}, secrets=status_secrets(t))
except Exception as exc: # ์ƒํƒœ ํŒŒ์ผ ๋ณด์กฐ ๊ธฐ๋ก โ€” ๊ฒฝ๊ณ ๋กœ ๊ฐ•๋“ฑ(D-L6-2)
console.echo_status_write_failed(_contained_code(LEG_STATUS_FILE, exc))