Spaces:
Running
Running
Download solar_eval/hub_hook/hook.py from dev-strender/proofread-demo: direct link, hf CLI and curl.
- Browser
- Download file 12.2 kB
-
https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/hub_hook/hook.py
- Command line
-
hf download hf://spaces/dev-strender/proofread-demo/solar_eval/hub_hook/hook.py
-
curl -L -o hook.py https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/hub_hook/hook.py
12.2 kB
| """run ์ข ๋ฃ ํ ์ค์ผ์คํธ๋ ์ดํฐ (๊ณํ 2026-09-27 L6 ยง2.1 ยท D-L6-2 ยท D-L6-3 ยท D-L6-11 ยท D-L6-12). | |
| ์์ = **๋ฒ๋ค โ hub**(๋ ๊ฑฐ์ ์ ์ฌ๋ ๋ถ๋ฅด์ง ์๋๋ค, D-M2-4 (b)). hub ๋์ run ์ case ์ ๊ธ **์**์์ ๋ ๋ค๋ฆฌ๋ฅผ ๋๊ณ ์ํ ํ์ผ์ ์ด | |
| ๋ค ์ ๊ธ ๋ฐ์์ ์ฝ์ ์์ฝ์ ๋ธ๋ค. | |
| **์์ธ ๋ด์(D-L6-2 ยท ๊ฒํ F1).** ๋ค๋ฆฌ ํธ์ถ๋ง๋ค `except Exception` โ `failed:<leg>_exception:<Type>`(ํ์ ์ด๋ฆ๋ง). ์ํ ํ์ผ ์ฐ๊ธฐ | |
| ์คํจ๋ ์ฝ์ ๊ฒฝ๊ณ ๋ก ๊ฐ๋ฑํ๊ณ , ํ ์ ์ฒด๋ ํ ๊ฒน ๋ ๊ฐ์ธ run ๋ช ๋ น์ด ํ ๋๋ฌธ์ ์ข ๋ฃ 1ยทํธ๋ ์ด์ค๋ฐฑ์ด ๋์ง ์๊ฒ ํ๋ค. batch ์์ | |
| ํ ์์ธ๊ฐ ํ๋กฌํํธ ์คํจ๋ก ์ธ์ง์ง๋ ์๋๋ค. `BaseException`(KeyboardInterruptยทpytest tripwire)์ ์ก์ง ์๋๋ค. | |
| **์์ธ ์์ธ๋ ๋ก๊ทธ๋ก๋ง(๋ฆฌ๋ทฐ ์ง์ 7).** ์ฝ์ยท์ํ ํ์ผ์๋ ํ์ ์ด๋ฆ๋ง ์ฃ๊ณ (D-L6-2), ํธ๋ ์ด์ค๋ฐฑ์ ์ด ๋ชจ๋ ๋ก๊ฑฐ์ ๋จ๊ธด๋ค โ | |
| ๊ธฐ๋ณธ์ DEBUG(๋ณดํต ์ค์ ์์๋ ๋ณด์ด์ง ์๋๋ค), `SOLAR_EVAL_HOOK_DEBUG=1` ์ด๋ฉด WARNING(stderr). ํธ๋ ์ด์ค๋ฐฑ์ ์ ๋ ฅ ์กฐ๊ฐ์ ๋ด์ ์ | |
| ์์ผ๋ ํ์ฅ ์ง๋จ ๋๋ง ์ผ ๋ค. | |
| **์ค๋ชจํฌ run ์ ์ ๊ทธ์ง ์๋๋ค(D-L6-3 "๋ ๋ค๋ฆฌ๊ฐ ์์ผ๋ฉด ์ ๊ทธ์ง ์๋๋ค" ยท D-L6-10).** hub ๋์์ด์ด๋ `--limit` ์ค๋ชจํฌ๋ฉด ์ ๊ธยท์ํ | |
| ํ์ผ ์์ด ๋ ๋ค๋ฆฌ `skipped:smoke_run` ํ ์ค๋ก ๋๋๋ค โ ๋ค๋ฅธ ํ ์ด ์ ๊ธ์ ์ฅ๊ณ ์์ด๋ ๊ธฐ๋ค๋ฆฌ๊ฑฐ๋ `lock_timeout` ์ด ๋์ง ์๋๋ค. | |
| ์ข ๋ฃ ์ฝ๋๋ ํธ์ถ์๊ฐ `exit_code_for(result, strict)` ๋ก ์ ํ๋ค โ ๊ธฐ๋ณธ 0, strict ์ด๊ณ ์คํจ ๋ค๋ฆฌ๊ฐ ์์ผ๋ฉด 3. | |
| **opt-in ํ ๊ณณ(๊ณํ 2026-09-28 L10 ยง2.6 ยท D-L10-6).** `gated_hub_leg` ๊ฐ `optin.resolve_hub_optin(config=t.hub_config)` ๋ก | |
| ํ๊ฒฝ ๋ณ์ > ์ค์ ํ์ผ `hub:` ์ ์ ํผ๋ค. ์ค์ ์ถ์ฒ๋ฉด URL ์ ์์ ํ๊ฒฝ(`child_env`)์ผ๋ก๋ง ๊ฐ๊ณ , ์ค์ ๊ฐ์ ์ง๋จ ๊ฐ๋ฆผยท์ํ ํ์ผ | |
| ๊ฑฐ๋ถ ๋ชฉ๋ก(`optin.config_secrets`)์ ๋ํ๋ค. ํธ์ถ์(`runs start`ยท`resume`ยทbatch ๋ยท`runs eval` ๋ฐํ) ์๊ทธ๋์ฒ๋ ๊ทธ๋๋ก๋ค. | |
| """ | |
| from __future__ import annotations | |
| import logging | |
| from collections.abc import Callable, Sequence | |
| from contextlib import ExitStack | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| from typing import TYPE_CHECKING | |
| from solar_eval.hub_hook import bundle_leg, console, hub_leg, lock, optin, status, target | |
| from solar_eval.hub_hook.types import ( | |
| BATCH_PENDING, | |
| EXIT_HOOK_FAILED, | |
| HOOK_LOCK_TIMEOUT_SECONDS, | |
| LEG_BUNDLE, | |
| LEG_HOOK, | |
| LEG_HUB, | |
| LEG_LOCK, | |
| LEG_STATUS_FILE, | |
| LOCK_TIMEOUT, | |
| NO_BUNDLE, | |
| NO_WORKSPACE, | |
| NOT_OPTED_IN, | |
| SKIP_STANDALONE, | |
| SMOKE_RUN, | |
| HookMode, | |
| HookOptions, | |
| HookResult, | |
| LegResult, | |
| Skip, | |
| Target, | |
| bundle_ready, | |
| exception_code, | |
| hub_run_id, | |
| ) | |
| if TYPE_CHECKING: | |
| from solar_eval.cli.config import CLIConfig | |
| _LOG = logging.getLogger(__name__) | |
| def _now() -> str: | |
| return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") | |
| def _contained_code(leg: str, exc: Exception) -> str: | |
| """๋ด์ํ ์์ธ โ ์ฝ๋(ํ์ ์ด๋ฆ๋ง, D-L6-2). ํธ๋ ์ด์ค๋ฐฑ์ ๋ก๊ฑฐ๋ก๋ง โ ์ฝ์ยท์ํ ํ์ผ์๋ ์ฃ์ง ์๋๋ค.""" | |
| code = exception_code(leg, exc) | |
| level = logging.WARNING if optin.hook_debug_requested() else logging.DEBUG | |
| _LOG.log(level, "Experiment Hub ํ ์์ธ ๋ด์ โ %s", code, exc_info=exc) | |
| return code | |
| def _contained(leg: str, fn: Callable[..., LegResult], *args, **kwargs) -> LegResult: | |
| """๋ค๋ฆฌ ํ๋๋ฅผ ๋๋ฆฌ๊ณ ์๊ธฐ์น ์์ ์์ธ๋ฅผ `<leg>_exception:<Type>` ์คํจ๋ก ๊ฐ๋ฑํ๋ค(D-L6-2).""" | |
| try: | |
| return fn(*args, **kwargs) | |
| except Exception as exc: # ๋ค๋ฆฌ ๊ฒฝ๊ณ โ ์ด๋ค ์์ธ๋ run ๋ช ๋ น์ ์คํจ๋ก ๋ง๋ค์ง ์๋๋ค | |
| return LegResult.failed(_contained_code(leg, exc)) | |
| def exit_code_for(result: HookResult | None, strict: bool) -> int: | |
| """ํ ๊ฒฐ๊ณผ โ run ๋ช ๋ น ์ข ๋ฃ ์ฝ๋(D-L6-11). ๊ธฐ๋ณธ 0, strict ์ด๊ณ ์คํจ ๋ค๋ฆฌ๊ฐ ์์ผ๋ฉด 3.""" | |
| return EXIT_HOOK_FAILED if strict and result is not None and not result.ok else 0 | |
| def run_hook( | |
| cfg: CLIConfig, project: str, run_dir: Path, options: HookOptions | |
| ) -> HookResult | None: | |
| """run ํ๋๊ฐ ๋๋ ๋ค์ ํ . hub ๋์์ด ์๋๋ฉด(standaloneยท`--no-ingest`) None ๋๋ ๊ฑด๋๋ ๊ฒฐ๊ณผ.""" | |
| run_id = Path(run_dir).name | |
| try: | |
| return _run_hook(cfg, project, Path(run_dir), run_id, options) | |
| except Exception as exc: # ํ ๋ฐ๊นฅ ๊ฒน(D-L6-2) โ ์ฝ์์ ์๋ฆฌ๊ณ ๊ฒฐ๊ณผ๋ก ๋๋ ค์ค๋ค | |
| code = _contained_code(LEG_HOOK, exc) | |
| console.echo_hook_crashed(code) | |
| return HookResult(run_id, None, {LEG_HOOK: LegResult.failed(code)}) | |
| def _run_hook( | |
| cfg: CLIConfig, project: str, run_dir: Path, run_id: str, options: HookOptions | |
| ) -> HookResult | None: | |
| if options.no_ingest: | |
| if cfg.repo_root is not None: | |
| console.echo_no_ingest(project, run_id, target.case_ref_hint(cfg, project)) | |
| return None | |
| resolved = target.resolve_target(cfg, project) | |
| if isinstance(resolved, Skip): | |
| if resolved.code == SKIP_STANDALONE: | |
| return None | |
| console.echo_skip(resolved, project) | |
| legs = { | |
| LEG_BUNDLE: LegResult.skipped(resolved.code), | |
| LEG_HUB: LegResult.skipped(resolved.code), | |
| } | |
| return HookResult(run_id, None, legs, _now(), _now()) | |
| if options.smoke: | |
| # ๋ ๋ค๋ฆฌ๊ฐ ์๋ค โ ์ ๊ทธ์ง ์๊ณ ์ํ ํ์ผ๋ ์ฐ์ง ์๋๋ค(D-L6-3 ยท D-L6-10, ๋ฆฌ๋ทฐ ์ง์ 1). | |
| console.echo_smoke(resolved.case_ref) | |
| smoke = {LEG_BUNDLE: LegResult.skipped(SMOKE_RUN), LEG_HUB: LegResult.skipped(SMOKE_RUN)} | |
| return HookResult(run_id, resolved.case_ref, smoke, _now(), _now()) | |
| result = _run_target(resolved, run_dir, run_id, options) | |
| console.echo_summary(result, project) | |
| return result | |
| def _run_target(t: Target, run_dir: Path, run_id: str, options: HookOptions) -> HookResult: | |
| """case ์ ๊ธ ์์์ ๋ ๋ค๋ฆฌ โ ์ํ ํ์ผ(D-L6-3). ์ ๊ธ ํ๋๋ฅผ ๋๊ธฐ๋ฉด ๋ ๋ค๋ฆฌ ๋ชจ๋ not_run.""" | |
| started = _now() | |
| with ExitStack() as stack: | |
| try: | |
| held = stack.enter_context( | |
| lock.case_lock(t.account, t.case, timeout=HOOK_LOCK_TIMEOUT_SECONDS) | |
| ) | |
| except Exception as exc: # ์ ๊ธ ํ์ผ์ ๋ง๋ค ์ ์๋ค โ ๋ค๋ฆฌ๋ฅผ ๋๋ฆฌ์ง ์๋๋ค | |
| legs = _lock_failed(_contained_code(LEG_LOCK, exc)) | |
| else: | |
| legs = ( | |
| _legs_in_lock(t, run_dir, run_id, options) if held else _lock_failed(LOCK_TIMEOUT) | |
| ) | |
| result = HookResult(run_id, t.case_ref, legs, started, _now()) | |
| _write_status(t, result, options) | |
| return result | |
| def _lock_failed(code: str) -> dict[str, LegResult]: | |
| return { | |
| LEG_LOCK: LegResult.failed(code), | |
| LEG_BUNDLE: LegResult.not_run(), | |
| LEG_HUB: LegResult.not_run(), | |
| } | |
| def _legs_in_lock( | |
| t: Target, run_dir: Path, run_id: str, options: HookOptions | |
| ) -> dict[str, LegResult]: | |
| bundle = _contained(LEG_BUNDLE, bundle_leg.run_bundle_leg, t, run_dir, run_id) | |
| if options.mode is HookMode.BATCH_ITEM: | |
| hub = LegResult.not_run(BATCH_PENDING) | |
| elif not bundle_ready(bundle): | |
| hub = LegResult.skipped(NO_BUNDLE) | |
| else: | |
| # `--expect-run` ์ ๋ฒ๋ค์ run_id(run.json) โ ๋๋ ํ ๋ฆฌ ์ด๋ฆ๊ณผ ๋ค๋ฅผ ์ ์๋ค(๋ฆฌ๋ทฐ ์ง์ 3). ์ํ ํ์ผ ์ด๋ฆ์ ๋๋ ํ ๋ฆฌ ์ด๋ฆ. | |
| hub = gated_hub_leg(t, (hub_run_id(bundle, run_id),)) | |
| return {LEG_BUNDLE: bundle, LEG_HUB: hub} | |
| def gated_hub_leg(t: Target, run_ids: Sequence[str]) -> LegResult: | |
| """hub ๋ค๋ฆฌ ๊ฒ์ดํธ ํ ๊ณณ โ opt-in(D-L6-4 ยท L10 D-L10-6)ยท์ํฌ์คํ์ด์ค(C-L6-9)๋ฅผ ๋ณธ ๋ค `hub_leg.run_hub_leg`(์์ธ ๋ด์). | |
| ํธ์ถ์๊ฐ case ์ ๊ธ์ ์ฅ๊ณ ๋ฒ๋ค ์ ๋ฌด๋ฅผ ์ด๋ฏธ ๋ดค๋ค๊ณ ๊ฐ์ ํ๋ค. L6b `runs eval` ์ ๊ฒฝ๋ก โฃ ๋ ์ด ํจ์๋ฅผ ๋ถ๋ฅธ๋ค. | |
| opt-in ํด์ ๊ฒฝ๊ณ (์ค์ ๋ชจ์ ์ค๋ฅยท์์ด์ ํธ ์ธ์ ์์ ์ค์ ๋ฌด์)๋ ๊ฑด๋๋ฐ๊ธฐ ์ ์ ๋จผ์ ์ฐ๋๋ค โ ๊ฐ์ ์ฃ์ง ์๋๋ค. | |
| """ | |
| resolved = optin.resolve_hub_optin(config=t.hub_config) | |
| for warning in resolved.warnings: | |
| console.echo_warning(warning) | |
| if not resolved.opted_in: | |
| return LegResult.skipped(NOT_OPTED_IN) | |
| if target.workspace_root() is None: | |
| return LegResult.skipped(NO_WORKSPACE) | |
| if resolved.confirm.warning is not None: | |
| console.echo_warning(resolved.confirm.warning) | |
| return _contained( | |
| LEG_HUB, | |
| hub_leg.run_hub_leg, | |
| t, | |
| tuple(run_ids), | |
| confirm_local=resolved.confirm.confirm_local, | |
| child_env=resolved.child_env, | |
| secrets=resolved.secrets, | |
| ) | |
| def status_secrets(t: Target) -> tuple[str, ...]: | |
| """์ํ ํ์ผ ๊ฑฐ๋ถ ๋ชฉ๋ก์ ๋ํ ์ค์ ์ถ์ฒ ๋น๋ฐ(ยง2.6 "๋น๋ฐ") โ `runs eval` ๋ฐํ๋ ๊ฐ์ ํจ์๋ฅผ ์ด๋ค.""" | |
| return optin.config_secrets(t.hub_config) | |
| def _write_status(t: Target, result: HookResult, options: HookOptions) -> None: | |
| try: | |
| status.write_status( | |
| t, result, options.command, strict=options.strict, secrets=status_secrets(t) | |
| ) | |
| except Exception as exc: # ์ํ ํ์ผ์ ๋ณด์กฐ ๊ธฐ๋ก โ ์ฐ๊ธฐ ์คํจ๋ ๊ฒฝ๊ณ ๋ก ๊ฐ๋ฑ(D-L6-2) | |
| console.echo_status_write_failed(_contained_code(LEG_STATUS_FILE, exc)) | |
| def finish_batch( | |
| cfg: CLIConfig, | |
| project: str, | |
| results: Sequence[HookResult | None], | |
| options: HookOptions, | |
| ) -> LegResult | None: | |
| """batch ๋ hub ๋ค๋ฆฌ ํ ๋ฒ(D-L6-12) โ ๋ฒ๋ค์ด ์๊ธด run ๋ค์ `--expect-run` ์ผ๋ก, case ์ ๊ธ ์์์. | |
| ๋๋๋ฉด run ๋ง๋ค ์ํ ํ์ผ์ `hub` ๋ฅผ `status.update_legs` ๋ก ๋ค์ ์ด๋ค โ ๋ฒ๋ค์ด ์์ด ๋ ๋ค๋ฆฌ์ ๋ค์ง ๋ชปํ run ์ | |
| `skipped:no_bundle` ๋ก(์ํ ํ์ผ์ `batch_pending` ์ด ๋จ์ง ์๊ฒ). ๋ ๋ค๋ฆฌ๋ฅผ ๋์ง ์์์ผ๋ฉด None. | |
| """ | |
| try: | |
| return _finish_batch(cfg, project, results, options) | |
| except Exception as exc: # ํ ๋ฐ๊นฅ ๊ฒน(D-L6-2) | |
| code = _contained_code(LEG_HOOK, exc) | |
| console.echo_hook_crashed(code) | |
| return LegResult.failed(code) | |
| def _finish_batch( | |
| cfg: CLIConfig, | |
| project: str, | |
| results: Sequence[HookResult | None], | |
| options: HookOptions, | |
| ) -> LegResult | None: | |
| targeted = [r for r in results if r is not None and r.case is not None] | |
| ready = [r for r in targeted if bundle_ready(r.legs.get(LEG_BUNDLE))] | |
| # ๋ฒ๋ค์ด ์์ด ๋ ๋ค๋ฆฌ์ ๋ค์ง ๋ชปํ run โ ์ํ ํ์ผ์ `batch_pending` ์ `no_bundle` ๋ก ๋ซ๋๋ค(l6a-verify). | |
| no_bundle = [r.run_id for r in targeted if r not in ready and _hub_pending(r)] | |
| if options.no_ingest or not (ready or no_bundle): | |
| return None | |
| resolved = target.resolve_target(cfg, project) | |
| if isinstance(resolved, Skip): | |
| return None | |
| # `--expect-run` ์ ๋ฒ๋ค์ run_id(run.json), ์ํ ํ์ผ์ ๋๋ ํ ๋ฆฌ ์ด๋ฆ(๋ฆฌ๋ทฐ ์ง์ 3). | |
| expect = [hub_run_id(r.legs.get(LEG_BUNDLE), r.run_id) for r in ready] | |
| leg: LegResult | None = None | |
| with lock.case_lock(resolved.account, resolved.case, timeout=HOOK_LOCK_TIMEOUT_SECONDS) as held: | |
| if ready: | |
| leg = gated_hub_leg(resolved, expect) if held else LegResult.failed(LOCK_TIMEOUT) | |
| for result in ready: | |
| _update_hub_leg(resolved, result.run_id, leg) | |
| for run_id in no_bundle: | |
| _update_hub_leg(resolved, run_id, LegResult.skipped(NO_BUNDLE)) | |
| if leg is not None: | |
| console.echo_batch_hub(leg, resolved.case_ref, expect) | |
| return leg | |
| def _hub_pending(result: HookResult) -> bool: | |
| """batch ํ run ์ hub ๋ค๋ฆฌ๊ฐ ๋ ๋ค๋ฆฌ๋ฅผ ๊ธฐ๋ค๋ฆฌ๋ ์ค์ธ๊ฐ(`not_run:batch_pending`).""" | |
| hub = result.legs.get(LEG_HUB) | |
| return hub is not None and hub.code == BATCH_PENDING | |
| def _update_hub_leg(t: Target, run_id: str, leg: LegResult) -> None: | |
| try: | |
| status.update_legs(t, run_id, {LEG_HUB: leg}, secrets=status_secrets(t)) | |
| except Exception as exc: # ์ํ ํ์ผ ๋ณด์กฐ ๊ธฐ๋ก โ ๊ฒฝ๊ณ ๋ก ๊ฐ๋ฑ(D-L6-2) | |
| console.echo_status_write_failed(_contained_code(LEG_STATUS_FILE, exc)) | |