Spaces:
Running
Running
Download solar_eval/hub_hook/console.py from dev-strender/proofread-demo: direct link, hf CLI and curl.
- Browser
- Download file 12.4 kB
-
https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/hub_hook/console.py
- Command line
-
hf download hf://spaces/dev-strender/proofread-demo/solar_eval/hub_hook/console.py
-
curl -L -o console.py https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/hub_hook/console.py
12.4 kB
| """ν μ½μ β λ€λ¦¬λ³ ν μ€κ³Ό λμ΄λ¦¬κΈ° λͺ λ Ή 문ꡬ (κ³ν 2026-09-27 L6 Β§2.6 ν Β· Β§2.7 Β· κ²ν F16). | |
| λμ΄λ¦¬κΈ° λͺ λ Ήμ URL μ리λ λ `HUB_DATABASE_URL=β¦` μ΄λ€ β κ°μ μ΄λμλ μ£μ§ μλλ€(D-L6-4). μ€ν¨ μ€μ λΉ¨κ° + λμ΄λ¦¬κΈ°. | |
| λ κ±°μ νλ©΄(`public`)μ L10 μ»·μ€λ²λ‘ μ½λμμ μ¬λΌμ‘λ€ β λ κ±°μ μλ μ μ¬ μ€μ μλ€(κ³ν 2026-09-28 L10 Β§2.6 Β· L6b νμ 1). | |
| hub opt-in μ΄ μμΌλ©΄(`not_opted_in`) λΉ¨κ° μ€λ‘ "μ΄ run μ hub(/v1)μ μλ€" μ μΌλ λ²Β·μ§κΈ μ£λ λͺ λ Ήμ μλ΄νλ€(λ§μ€ν° D-M2-18). | |
| μμ½ λ¨Έλ¦¬μλ ν μ 체 κ²½κ³Ό μ΄(μν νμΌμ `started_at`β`finished_at`, μ κΈ λκΈ° ν¬ν¨)λ₯Ό λΆμΈλ€ β 첫 μ€λ°μ΄ν° run(U-L6a-1)μμ | |
| μ½μλ§μΌλ‘ hub μ μ¬ μκ°μ μ μ μμκΈ° λλ¬Έμ΄λ€. μκ°μ΄ μκ±°λ νμμ΄ λ€λ₯΄λ©΄ λΆμ΄μ§ μλλ€. | |
| """ | |
| from __future__ import annotations | |
| from collections.abc import Iterable, Mapping | |
| from datetime import datetime | |
| from typing import Any | |
| import click | |
| from solar_eval.hub_hook import optin | |
| from solar_eval.hub_hook.types import ( | |
| CASE_BLOCKED_PREFIX, | |
| CASE_FAILED_PREFIX, | |
| FILTER_NO_MATCH, | |
| HUB_ERR_CONFIRMATION_REQUIRED, | |
| HUB_ERR_CONNECTION_FAILED, | |
| HUB_FAILED_PREFIX, | |
| LEG_BUNDLE, | |
| LEG_HOOK, | |
| LEG_HUB, | |
| LEG_LOCK, | |
| LEG_PUBLISH, | |
| LOCK_TIMEOUT, | |
| NOT_OPTED_IN, | |
| RUN_NOT_OK_PREFIX, | |
| SCHEMA_MISMATCH, | |
| SKIP_CASE_UNMAPPED, | |
| SKIP_DATA_HOME_INSIDE_REPO, | |
| SKIP_DATA_HOME_MISMATCH, | |
| SKIP_NO_REPO_ROOT, | |
| SKIP_NOT_REPO_MANAGED, | |
| SMOKE_RUN, | |
| STAGE_CASE_LOCK, | |
| STATUS_FAILED, | |
| STATUS_OK, | |
| STATUS_SKIPPED, | |
| HookResult, | |
| LegResult, | |
| Skip, | |
| ) | |
| URL_PLACEHOLDER = "HUB_DATABASE_URL=β¦" | |
| #: μν νμΌ μκ° νμ(`hook._now` μ κ°λ€). | |
| HOOK_TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" | |
| LEG_LABELS = { | |
| LEG_BUNDLE: "λ²λ€", | |
| LEG_HUB: "hub μ μ¬", | |
| LEG_LOCK: "case μ κΈ", | |
| LEG_HOOK: "ν ", | |
| LEG_PUBLISH: "μ±μ λ°ν", | |
| } | |
| SKIP_REASONS = { | |
| SKIP_NOT_REPO_MANAGED: "λ ν¬ κ΄λ¦¬ νλ‘μ νΈκ° μλλ€", | |
| SKIP_NO_REPO_ROOT: "λ ν¬ config 루νΈκ° μλ€(--projects-dir λ±)", | |
| SKIP_CASE_UNMAPPED: "project κ° case μ λ§€νλμ§ μλλ€ β make bundle-census λ‘ νμΈ", | |
| SKIP_DATA_HOME_MISMATCH: "μμ§ μμΉ΄μ΄λΈκ° λ°μ΄ν° ν λ°μ΄λ€", | |
| SKIP_DATA_HOME_INSIDE_REPO: "λ°μ΄ν° νμ΄ λ ν¬ μμ΄λ€ β ν νμλ¬Όμ git λ°μ λλ€(CBS_DATA_HOME)", | |
| } | |
| #: μμ½ μ€μ μ£λ λ²λ€ λ€λ¦¬ detail ν€(μμλλ‘). | |
| _BUNDLE_SUMMARY_KEYS = ( | |
| (("backfill", "wrap"), "wrap"), | |
| (("backfill", "eval_wrap"), "eval-wrap"), | |
| (("rescore", "status"), "μ¬μ±μ "), | |
| (("indexes",), "μμΈ"), | |
| ) | |
| # ββ λμ΄λ¦¬κΈ° λͺ λ Ή ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def hub_ingest_command(case_ref: str) -> str: | |
| return f"{URL_PLACEHOLDER} make hub-ingest CASE={case_ref} CONFIRM=local" | |
| def hub_backfill_command(case_ref: str, run_id: str) -> str: | |
| return f'make hub-backfill CASE={case_ref} ARGS="--run {run_id}"' | |
| def hub_recovery(code: str, case_ref: str, run_id: str) -> str: | |
| """hub λ€λ¦¬ μ€ν¨ μ½λ β λμ΄λ¦¬κΈ° 문ꡬ(Β§2.6 ν).""" | |
| ingest = hub_ingest_command(case_ref) | |
| if code == f"{HUB_FAILED_PREFIX}{HUB_ERR_CONFIRMATION_REQUIRED}": | |
| return _confirmation_hint(ingest) | |
| if code == f"{HUB_FAILED_PREFIX}{HUB_ERR_CONNECTION_FAILED}": | |
| return f"make experiment-hub-up λ€ {ingest}" | |
| if code == f"{CASE_FAILED_PREFIX}{STAGE_CASE_LOCK}": | |
| return f"μ μ μ¬κ° λλ λ€ {ingest}" | |
| if code.startswith((CASE_BLOCKED_PREFIX, CASE_FAILED_PREFIX)): | |
| return f"{ingest} ν μ΄μ ν(/v1/{case_ref}) νμΈ" | |
| if code == SCHEMA_MISMATCH: | |
| return ( | |
| f"hub μ€ν€λ§ νμ΄ λ€λ₯΄λ€ β {URL_PLACEHOLDER} make hub-m1-local CASE={case_ref} " | |
| "CONFIRM=local SKIP_PULL=1 FROM=backfill (μ£Όμ: μ¬κ΅¬μ±μ λͺ¨λ case λ₯Ό μ§μ΄λ€, case λ§λ€ λ€μ)" | |
| ) | |
| if code.startswith(RUN_NOT_OK_PREFIX): | |
| return f"{hub_backfill_command(case_ref, run_id)} λ€ {ingest}" | |
| if code == FILTER_NO_MATCH: | |
| return f"pnpm μν¬μ€νμ΄μ€μ @poc/eval-store κ° μλ€ β pnpm install λ€ {ingest}" | |
| return ingest | |
| #: μ¬μ©μ DB μ°κΈ° νμΈ μλ΄(μ¬λ μ Έ) β μμ `hub.confirm` μ λ§μ€ν° Β§4 κ·μΉ 9 μ μλλ μμΈλ€(L10 Β§2.6 Β· κ²ν F1-3). | |
| CONFIRMATION_HINT_HUMAN = ( | |
| "μ¬μ©μ DB μ μ°λ €λ©΄ λͺ λ Ή μμ SOLAR_EVAL_HUB_CONFIRM=local μ λΆμ΄κ±°λ μ€μ νμΌ hub.confirm: local" | |
| "(μ¬λ μ Έλ§ Β· URL λ μ€μ μ μμ λλ§ β κ·μΉ 9 μ μλλ μμΈ)" | |
| ) | |
| #: `not_opted_in` λΉ¨κ° μ€μ 머리(μ¬λΒ·μμ΄μ νΈ κ³΅ν΅). | |
| NOT_OPTED_IN_HEAD = "hub opt-in μμ β μ΄ run μ hub(/v1)μ μλ€." | |
| def not_opted_in_hint(case_ref: str) -> str: | |
| """hub opt-in μ΄ μλ€ β λΉ¨κ° μ€ μλ΄(λ§μ€ν° D-M2-18 Β· L10 Β§2.6 ν "λ λ€ μμ"). URL κ°μ μ£μ§ μλλ€. | |
| μμ΄μ νΈ μΈμ (`optin.AGENT_SESSION_MARKERS`)μ μ€μ νμΌ `hub.database_url` μ 무μνλ―λ‘(D-L10-6) "μ€μ νμΌλ‘ μΌλΌ" λ₯Ό | |
| λ§νμ§ μκ³ μ¬λμ΄ λΆλ₯Ό λͺ λ Ήλ§ μ€λ€ β λ°λ‘ μ "μ€μ νμΌμ 무μνλ€" κ²½κ³ μ μ΄κΈλμ§ μκ²(리뷰 L10 μ§μ 3, | |
| `_confirmation_hint` μ κ°μ κ°λ). | |
| """ | |
| ingest = hub_ingest_command(case_ref) | |
| if optin.is_agent_session(): | |
| return f"{NOT_OPTED_IN_HEAD} μμ΄μ νΈ μΈμ μ μ€μ νμΌ opt-in μ λ°λ₯΄μ§ μλλ€ β μ¬λμ΄ μ§μ : {ingest}" | |
| return ( | |
| f"{NOT_OPTED_IN_HEAD} μΌλ €λ©΄ μ€μ νμΌ hub.database_url λλ λͺ λ Ή μ " | |
| f"{URL_PLACEHOLDER}. μ§κΈ μ£κΈ°: {ingest}" | |
| ) | |
| def _confirmation_hint(ingest: str) -> str: | |
| """hub κ° μ¬μ©μ DB μ°κΈ° νμΈμ μꡬνλ€. μμ΄μ νΈ μΈμ (`optin.AGENT_SESSION_MARKERS`)μ νμΈμ 무μνλ―λ‘ μ¬λμ΄ λΆλ₯Ό λͺ λ Ήμ μλ΄νλ€.""" | |
| if optin.is_agent_session(): | |
| return f"μμ΄μ νΈ μΈμ μ μ¬μ©μ DB μ°κΈ° νμΈμ λκΈ°μ§ μλλ€ β μ¬λμ΄ μ§μ : {ingest}" | |
| return CONFIRMATION_HINT_HUMAN | |
| def default_hint(leg_name: str, leg: LegResult, case_ref: str | None, run_id: str) -> str | None: | |
| """λ€λ¦¬κ° hint λ₯Ό μ£μ§ μμμ λ μ½μμ΄ λΆμ΄λ μλ΄.""" | |
| if leg.hint is not None or case_ref is None: | |
| return leg.hint | |
| if leg.status == STATUS_SKIPPED and leg_name == LEG_HUB and leg.code == NOT_OPTED_IN: | |
| return not_opted_in_hint(case_ref) | |
| if leg.status != STATUS_FAILED: | |
| return None | |
| if leg_name == LEG_HUB: | |
| return hub_recovery(leg.code, case_ref, run_id) | |
| if leg_name == LEG_LOCK and leg.code == LOCK_TIMEOUT: | |
| return f"κ°μ case μ λ€λ₯Έ ν μ΄ λλ λ€ {hub_backfill_command(case_ref, run_id)} β {hub_ingest_command(case_ref)}" | |
| return f"{hub_backfill_command(case_ref, run_id)} β {hub_ingest_command(case_ref)}" | |
| # ββ μΆλ ₯ βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def echo_no_ingest(project: str, run_id: str, case_ref: str | None) -> None: | |
| """`--no-ingest` β hub ν μ 체λ₯Ό 건λλ΄λ€λ ν μ€(Q-M2-4). `project` λ νΈμΆμ μκ·Έλμ² μ μ§μ©μ΄λ€(L10 β λ κ±°μ μ€ μμ ).""" | |
| del project # λ κ±°μ νλ©΄ μλ μ μ¬ μ€μ΄ L10 μμ μ¬λΌμ‘λ€(κ³ν 2026-09-28 L10 Β§2.6) | |
| if case_ref is None: | |
| click.secho("\nhub ν 건λλ(--no-ingest)", fg="yellow") | |
| return | |
| revive = f"{hub_backfill_command(case_ref, run_id)} β {hub_ingest_command(case_ref)}" | |
| click.secho(f"\nhub ν 건λλ(--no-ingest) β λμ΄λ¦¬κΈ°: {revive}", fg="yellow") | |
| def echo_skip(skip: Skip, project: str) -> None: | |
| """hub λμμ΄ μλ run β ν μ€(standalone μ λΆλ₯΄μ§ μλλ€).""" | |
| reason = SKIP_REASONS.get(skip.code, skip.code) | |
| paths = " Β· ".join(f"{k}={v}" for k, v in skip.detail.items()) | |
| suffix = f" ({paths})" if paths else "" | |
| click.secho(f"\nhub ν 건λλ: {skip.code} β {reason}{suffix}", fg="yellow") | |
| def echo_smoke(case_ref: str) -> None: | |
| """`--limit` μ€λͺ¨ν¬ run β ν μ€(μ κΈΒ·μν νμΌ μμ, D-L6-10 Β· μλ£ μ μ Β§0.4-2).""" | |
| click.secho( | |
| f"\nhub ν 건λλ: {SMOKE_RUN} β --limit μ€λͺ¨ν¬ run μ λ²λ€Β·hub μ μ£μ§ μλλ€ ({case_ref})", | |
| fg="yellow", | |
| ) | |
| def elapsed_seconds(result: HookResult) -> int | None: | |
| """ν μ 체 κ²½κ³Ό μ΄(`started_at`β`finished_at`). μκ°μ΄ μκ±°λ νμμ΄ λ€λ₯΄κ±°λ μμλ©΄ None.""" | |
| if not result.started_at or not result.finished_at: | |
| return None | |
| try: | |
| started = datetime.strptime(result.started_at, HOOK_TIME_FORMAT) | |
| finished = datetime.strptime(result.finished_at, HOOK_TIME_FORMAT) | |
| except ValueError: | |
| return None | |
| seconds = int((finished - started).total_seconds()) | |
| return seconds if seconds >= 0 else None | |
| def echo_summary(result: HookResult, project: str) -> None: | |
| """λ€λ¦¬λ§λ€ ν μ€ + μ€ν¨λ©΄ λμ΄λ¦¬κΈ°(Β§2.7). 머리μ κ²½κ³Ό μ΄. λ κ±°μ νλ©΄ μ€μ L10 μμ μμ΄μ‘λ€.""" | |
| elapsed = elapsed_seconds(result) | |
| suffix = f" Β· κ²½κ³Ό {elapsed} s" if elapsed is not None else "" | |
| click.secho( | |
| f"\nExperiment Hub ν β {result.case or project} Β· {result.run_id}{suffix}", fg="blue", bold=True | |
| ) | |
| for name, leg in result.legs.items(): | |
| _echo_leg(name, leg, result.case, result.run_id) | |
| def _echo_leg(name: str, leg: LegResult, case_ref: str | None, run_id: str) -> None: | |
| label = f" {LEG_LABELS.get(name, name):<10}" | |
| hint = default_hint(name, leg, case_ref, run_id) | |
| if leg.status == STATUS_OK: | |
| details = _ok_details(leg.detail) | |
| click.secho(f"{label} ok{f' ({details})' if details else ''}", fg="green") | |
| elif leg.status == STATUS_FAILED: | |
| click.secho(f"{label} μ€ν¨ β {leg.code}", fg="red", bold=True) | |
| if hint: | |
| click.secho(f" {'':<10} λμ΄λ¦¬κΈ°: {hint}", fg="red") | |
| elif leg.status == STATUS_SKIPPED: | |
| # hub opt-in μμμ λΉ¨κ° μ€(D-M2-18 β μ΄ run μ /v1 μ μλ€), κ·Έ λ° κ±΄λλμ λ Έλ | |
| color = "red" if (name, leg.code) == (LEG_HUB, NOT_OPTED_IN) else "yellow" | |
| click.secho(f"{label} 건λλ β {leg.code}{f'. {hint}' if hint else ''}", fg=color) | |
| else: | |
| click.secho(f"{label} λμ§ μμ β {leg.code}", fg="yellow") | |
| def _ok_details(detail: Mapping[str, Any]) -> str: | |
| parts = [] | |
| for path, label in _BUNDLE_SUMMARY_KEYS: | |
| value: Any = detail | |
| for key in path: | |
| value = value.get(key) if isinstance(value, Mapping) else None | |
| if isinstance(value, str): | |
| parts.append(f"{label} {value}") | |
| return " Β· ".join(parts) | |
| def echo_batch_hub(leg: LegResult, case_ref: str, run_ids: Iterable[str]) -> None: | |
| """batch λ hub λ€λ¦¬ ν μ€(D-L6-12).""" | |
| ids = list(run_ids) | |
| _echo_leg(LEG_HUB, leg, case_ref, ids[0] if len(ids) == 1 else "<run>") | |
| def echo_batch_summary(results: Iterable[HookResult | None]) -> None: | |
| """batch λ μμ½ "ν : ok N Β· μ€ν¨ M Β· 건λλ K"(건λλ = ν μ΄ λ λ€λ¦¬κ° μμλ run).""" | |
| ok = failed = skipped = 0 | |
| for result in results: | |
| if result is not None and not result.ok: | |
| failed += 1 | |
| elif result is not None and any(leg.status == STATUS_OK for leg in result.legs.values()): | |
| ok += 1 | |
| else: | |
| skipped += 1 | |
| color = "red" if failed else "green" | |
| click.secho(f"ν : ok {ok} Β· μ€ν¨ {failed} Β· 건λλ {skipped}", fg=color) | |
| def echo_warning(message: str) -> None: | |
| click.secho(f" β {message}", fg="yellow", err=True) | |
| def echo_hook_crashed(code: str) -> None: | |
| """ν λ°κΉ₯ κ²Ήμ΄ μμΈλ₯Ό μ‘μλ€ β run κ²°κ³Όλ κ·Έλλ‘λ€(D-L6-2).""" | |
| click.secho(f"\nExperiment Hub ν μ΄ μκΈ°μΉ μκ² λ©μ·λ€ β {code} (run κ²°κ³Όλ λμ€ν¬μ μλ€)", fg="red") | |
| def echo_status_write_failed(code: str) -> None: | |
| echo_warning(f"ν μν νμΌμ μ°μ§ λͺ»νλ€ β {code}") | |