dev-strender's picture
feat: switch demo to FT + Opus article review (pipeline_exp_261021_ft_hybrid)
3dfc93b verified
Raw History Blame Contribute Delete
9.26 kB
"""์ƒํƒœ ํŒŒ์ผ (๊ณ„ํš 2026-09-27 L6 ยง2.7 ยท D-L6-12).
`<hub_reports_dir>/<a>/<c>/hook/<run_id>.json`, ์Šคํ‚ค๋งˆ `STATUS_SCHEMA`(`solar-eval-hook/v1`), ์›์ž์  ์“ฐ๊ธฐ(๊ฐ™์€ ๋””๋ ‰ํ† ๋ฆฌ์˜ ์ž„์‹œ
ํŒŒ์ผ โ†’ fsync โ†’ `os.replace`). ํ‚ค: `schema ยท run_id ยท project ยท case ยท command ยท started_at ยท finished_at ยท strict ยท
legs{<leg>: {status, code, โ€ฆdetail}} ยท ok`(`ok` = ์‹คํŒจ ๋‹ค๋ฆฌ 0). batch run ์€ run ๋งˆ๋‹ค ๋ฒˆ๋“ค ๋‹ค๋ฆฌ ๊ฒฐ๊ณผ๋กœ ๋จผ์ € ์“ฐ๊ณ ,
`hook.finish_batch` ๊ฐ€ ๋ ๋‹ค๋ฆฌ ๊ฒฐ๊ณผ๋กœ `update_legs` ๋ฅผ ๋ถ€๋ฅธ๋‹ค. hub ๋Œ€์ƒ์ด ์•„๋‹Œ run ์€ ์ƒํƒœ ํŒŒ์ผ์ด ์—†๋‹ค.
๊ฐ’์€ ์ฝ”๋“œยท๊ฑด์ˆ˜ยทidยท์‹œ๊ฐ๋งŒ โ€” ๋‹ค๋ฆฌ์˜ `hint`(๋˜์‚ด๋ฆฌ๊ธฐ ๋ฌธ๊ตฌ)๋Š” ์ฝ˜์†” ๋ชซ์ด๋ผ ์‹ฃ์ง€ ์•Š๊ณ , ๊ณ ๊ฐ ํ…์ŠคํŠธยทURLยท์˜ˆ์™ธ ์›๋ฌธ๋„ ์—†๋‹ค.
๋งˆ์ง€๋ง‰ ๋ฐฉ์–ด๋กœ ์ง๋ ฌํ™”ํ•œ ๋ฌธ์„œ์— hubยท๋ ˆ๊ฑฐ์‹œ DB URL ๊ฐ’์ด ๋“ค์–ด ์žˆ์œผ๋ฉด ์“ฐ์ง€ ์•Š๊ณ  `ValueError`(๊ฐ’์€ ๋ฌธ๊ตฌ์— ์‹ฃ์ง€ ์•Š๋Š”๋‹ค, D-L6-4).
ํ™˜๊ฒฝ ๋ณ€์ˆ˜ ๊ฐ’๋งŒ์œผ๋กœ๋Š” ์„ค์ • ํŒŒ์ผ `hub.database_url` ๊ฐ’์„ ๋ชจ๋ฅด๋ฏ€๋กœ ํ˜ธ์ถœ์ž๊ฐ€ `secrets=`(`optin.config_secrets`)๋กœ ๋„˜๊ธด๋‹ค โ€”
๊ทธ ๊ฐ’๋„ ๊ฐ™์€ ๊ทœ์น™์œผ๋กœ ๊ฑฐ๋ถ€ยท๊ฐ€๋ฆฐ๋‹ค(๊ณ„ํš 2026-09-28 L10 ยง2.6 "๋น„๋ฐ€" ยท ๊ฒ€ํ†  F1-4).
์“ฐ๊ธฐ ์‹คํŒจ๋Š” hook ์ด ์ฝ˜์†” ๊ฒฝ๊ณ ๋กœ ๊ฐ•๋“ฑํ•œ๋‹ค(D-L6-2) โ€” ์ด ๋ชจ๋“ˆ์€ ์˜ˆ์™ธ๋ฅผ ๊ทธ๋Œ€๋กœ ๋‚ธ๋‹ค.
"""
from __future__ import annotations
import json
import os
import re
import tempfile
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Any
from solar_eval.hub_hook.optin import ENV_HUB_DATABASE_URL
from solar_eval.hub_hook.types import (
HOOK_DIRNAME,
STATUS_FAILED,
STATUS_SCHEMA,
Command,
HookResult,
LegResult,
Target,
)
#: ๋ฌธ์„œ์— ๊ฐ’์ด ์ƒˆ๋ฉด ์•ˆ ๋˜๋Š” ํ™˜๊ฒฝ ๋ณ€์ˆ˜(์ž๊ฒฉ์ฆ๋ช…์„ ๋‹ด์€ URL).
# ์˜› ์ด๋ฆ„(EVALHUB_DATABASE_URL)์€ ๋” ์ด์ƒ ์ฝ์ง€ ์•Š์ง€๋งŒ, ๊ฐœ๋ช… ์ „ ์…ธ์— ๋‚จ์•„ ์žˆ์„ ์ˆ˜ ์žˆ์–ด ๊ฐ’์ด ์ƒˆ์ง€ ์•Š๊ฒŒ ๊ณ„์† ๊ฐ€๋ฆฐ๋‹ค.
SECRET_ENV_NAMES = (ENV_HUB_DATABASE_URL, "EXPERIMENT_HUB_DATABASE_URL", "EVALHUB_DATABASE_URL")
#: ์ด๋ณด๋‹ค ์งง์€ ๊ฐ’์€ ์šฐ์—ฐํžˆ ๊ฒน์น  ์ˆ˜ ์žˆ์–ด ๊ฒ€์‚ฌํ•˜์ง€ ์•Š๋Š”๋‹ค(URL ์€ ์ด๋ณด๋‹ค ๊ธธ๋‹ค).
MIN_SECRET_LENGTH = 8
#: ํŒŒ์ผ ์ด๋ฆ„์ด ๋˜๋Š” run_id โ€” ๊ฒฝ๋กœ ๊ตฌ๋ถ„์žยท์ œ์–ด ๋ฌธ์žยท`.`/`..` ๊ธˆ์ง€.
_UNSAFE_RUN_ID = re.compile(r"[/\\\x00-\x1f\x7f]")
#: ์ž„์‹œ ํŒŒ์ผ ์ด๋ฆ„์€ run_id ์— ๊ธฐ๋Œ€์ง€ ์•Š๋Š”๋‹ค โ€” `<run_id>.json` ์ด ์ด๋ฆ„ ํ•œ๋„(255๋ฐ”์ดํŠธ) ์•ˆ์ด์–ด๋„ `.<run_id>.json.XXXX.tmp`
#: ๋Š” ๋„˜์น  ์ˆ˜ ์žˆ๋‹ค(๋ฆฌ๋ทฐ ์ง€์  1 โ€” 237~250๋ฐ”์ดํŠธ run_id ์—์„œ ENAMETOOLONG).
_TMP_PREFIX = ".hook-"
_TMP_SUFFIX = ".tmp"
#: URL ์˜ `์‚ฌ์šฉ์ž:๋น„๋ฐ€๋ฒˆํ˜ธ@` โ€” ๋น„๋ฐ€๋ฒˆํ˜ธ ์ž๋ฆฌ๋งŒ ๊ฐ€๋ฆฐ๋‹ค.
_URL_PASSWORD = re.compile(r"(?P<head>[A-Za-z][A-Za-z0-9+.-]*://[^\s:/@]*:)[^\s@]*@")
_REDACTED = "***"
#: ์„ค์ • ์ถœ์ฒ˜ ๋น„๋ฐ€์˜ ์ด๋ฆ„ํ‘œ โ€” ๊ฑฐ๋ถ€ ๋ฌธ๊ตฌยท๊ฐ€๋ฆผ ์ž๋ฆฌ์— ๊ฐ’ ๋Œ€์‹  ์‹ฃ๋Š”๋‹ค.
CONFIG_SECRET_LABEL = "hub.database_url"
def status_path(target: Target, run_id: str) -> Path:
"""`<hub_reports_dir>/<a>/<c>/hook/<run_id>.json`. run_id ๊ฐ€ ํŒŒ์ผ ์ด๋ฆ„์œผ๋กœ ์•ˆ์ „ํ•˜์ง€ ์•Š์œผ๋ฉด `ValueError`."""
from poc_bundle.home import hub_reports_dir
_check_run_id(run_id)
return hub_reports_dir() / target.account / target.case / HOOK_DIRNAME / f"{run_id}.json"
def _check_run_id(run_id: str) -> None:
if not isinstance(run_id, str) or run_id in ("", ".", "..") or _UNSAFE_RUN_ID.search(run_id):
raise ValueError("hook status: run_id is not a safe file name")
def leg_document(leg: LegResult) -> dict[str, Any]:
"""๋‹ค๋ฆฌ ํ•˜๋‚˜ โ†’ `{status, code, โ€ฆdetail}`. detail ์ด `status`ยท`code` ๋ฅผ ๋ฎ์ง€ ๋ชปํ•œ๋‹ค. hint ๋Š” ์‹ฃ์ง€ ์•Š๋Š”๋‹ค."""
return {**dict(leg.detail), "status": leg.status, "code": leg.code}
def _ok(legs: Mapping[str, Mapping[str, Any]]) -> bool:
return all(leg.get("status") != STATUS_FAILED for leg in legs.values())
def status_document(
target: Target, result: HookResult, command: Command, *, strict: bool
) -> dict[str, Any]:
"""์ƒํƒœ ํŒŒ์ผ ๋ฌธ์„œ(์ˆœ์ˆ˜)."""
legs = {name: leg_document(leg) for name, leg in result.legs.items()}
return {
"schema": STATUS_SCHEMA,
"run_id": result.run_id,
"project": target.project,
"case": target.case_ref,
"command": command,
"started_at": result.started_at,
"finished_at": result.finished_at,
"strict": bool(strict),
"legs": legs,
"ok": _ok(legs),
}
def write_status(
target: Target,
result: HookResult,
command: Command,
*,
strict: bool,
secrets: Sequence[str] = (),
) -> Path:
"""run ํ•˜๋‚˜์˜ ํ›… ๊ฒฐ๊ณผ๋ฅผ ์›์ž์ ์œผ๋กœ ์“ฐ๊ณ  ๊ฒฝ๋กœ๋ฅผ ๋Œ๋ ค์ค€๋‹ค(๊ฐ™์€ run ์€ ๋ฎ์–ด์“ด๋‹ค). `secrets` = ์„ค์ • ์ถœ์ฒ˜ ๋น„๋ฐ€(๊ฑฐ๋ถ€ ๋ชฉ๋ก)."""
path = status_path(target, result.run_id)
_atomic_write(path, status_document(target, result, command, strict=strict), secrets)
return path
def update_legs(
target: Target,
run_id: str,
legs: Mapping[str, LegResult],
*,
secrets: Sequence[str] = (),
) -> None:
"""์ด๋ฏธ ์žˆ๋Š” ์ƒํƒœ ํŒŒ์ผ์˜ ๋‹ค๋ฆฌ ๋ช‡ ๊ฐœ๋ฅผ ์›์ž์ ์œผ๋กœ ๋ฐ”๊ฟ” ์“ฐ๊ณ  `ok` ๋ฅผ ๋‹ค์‹œ ์„ผ๋‹ค(batch ๋ hub ๋‹ค๋ฆฌ, D-L6-12).
ํŒŒ์ผ์ด ์—†์œผ๋ฉด `FileNotFoundError`, ์Šคํ‚ค๋งˆ๋‚˜ run_id ๊ฐ€ ๋‹ค๋ฅด๋ฉด `ValueError` โ€” ํ˜ธ์ถœ์ž(hook)๊ฐ€ ๊ฒฝ๊ณ ๋กœ ๊ฐ•๋“ฑํ•œ๋‹ค.
"""
path = status_path(target, run_id)
doc = json.loads(path.read_text("utf-8"))
if (
not isinstance(doc, dict)
or doc.get("schema") != STATUS_SCHEMA
or doc.get("run_id") != run_id
):
raise ValueError(f"hook status: {path.name} is not a {STATUS_SCHEMA} file for this run")
current = doc.get("legs")
merged = dict(current) if isinstance(current, dict) else {}
merged.update({name: leg_document(leg) for name, leg in legs.items()})
_atomic_write(path, {**doc, "legs": merged, "ok": _ok(merged)}, secrets)
def _atomic_write(path: Path, doc: Mapping[str, Any], secrets: Sequence[str] = ()) -> None:
"""์ง๋ ฌํ™”ยท๋น„๋ฐ€ ๊ฒ€์‚ฌ โ†’ ๊ฐ™์€ ๋””๋ ‰ํ† ๋ฆฌ ์ž„์‹œ ํŒŒ์ผ โ†’ fsync โ†’ `os.replace`. ์‹คํŒจํ•˜๋ฉด ์ž„์‹œ ํŒŒ์ผ์„ ์ง€์šฐ๊ณ  ๋‹ค์‹œ ๋‚ธ๋‹ค."""
text = json.dumps(doc, ensure_ascii=False, indent=2) + "\n"
_refuse_secrets(text, secrets)
path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp_name = tempfile.mkstemp(dir=path.parent, prefix=_TMP_PREFIX, suffix=_TMP_SUFFIX)
tmp = Path(tmp_name)
try:
with os.fdopen(fd, "w", encoding="utf-8") as fh:
fh.write(text)
fh.flush()
os.fsync(fh.fileno())
os.replace(tmp, path)
except BaseException:
tmp.unlink(missing_ok=True)
raise
def _secret_values(extra: Sequence[str] = ()) -> list[tuple[str, str]]:
"""(์ด๋ฆ„ํ‘œ, ๊ฐ’) โ€” ํ™˜๊ฒฝ ๋ณ€์ˆ˜ ๊ฐ’ + ์„ค์ • ์ถœ์ฒ˜ ๊ฐ’(`extra`, ์ด๋ฆ„ํ‘œ `hub.database_url`). ๊ฒ€์‚ฌํ•  ๋งŒํผ ๊ธด ๊ฐ’๋งŒ."""
pairs = [(name, os.environ.get(name, "").strip()) for name in SECRET_ENV_NAMES]
pairs += [(CONFIG_SECRET_LABEL, value.strip()) for value in extra if isinstance(value, str)]
return [(name, value) for name, value in pairs if len(value) >= MIN_SECRET_LENGTH]
def _refuse_secrets(text: str, extra: Sequence[str] = ()) -> None:
for name, value in _secret_values(extra):
if value in text:
raise ValueError(f"hook status: refusing to store the value of {name}")
def _mask_url_password(text: str) -> str:
return _URL_PASSWORD.sub(lambda m: f"{m.group('head')}{_REDACTED}@", text)
def _whole_value_replacement(name: str, value: str) -> str:
"""์•Œ๋ ค์ง„ ๋น„๋ฐ€ ๊ฐ’ ํ•˜๋‚˜๋ฅผ ๋ฌด์—‡์œผ๋กœ ๋ฐ”๊ฟ€๊นŒ. ๋น„๋ฐ€์ด ๋น„๋ฐ€๋ฒˆํ˜ธ ์ž๋ฆฌ๋ฟ์ด๋ฉด(์ฟผ๋ฆฌยท์กฐ๊ฐ ์—†์Œ) ๋น„๋ฐ€๋ฒˆํ˜ธ๋งŒ ๊ฐ€๋ฆฐ URL โ€” ํ˜ธ์ŠคํŠธยทDB
์ด๋ฆ„์ด ์ง„๋‹จ์— ๋‚จ๋Š”๋‹ค. ๊ทธ ๋ฐ–(์ฟผ๋ฆฌ ๋ฌธ์ž์—ด ๋น„๋ฐ€๋ฒˆํ˜ธ ๋“ฑ โ€” ์–ด๋””๊ฐ€ ๋น„๋ฐ€์ธ์ง€ ๋ชจ๋ฅธ๋‹ค)์€ ํ†ต์งธ๋กœ `<์ด๋ฆ„ํ‘œ>`."""
masked = _mask_url_password(value)
only_password = masked != value and "?" not in value and "#" not in value
return masked if only_password else f"<{name}>"
def redact_secrets(text: str, extra: Sequence[str] = ()) -> str:
"""์ฝ˜์†”์— ์˜ฎ๊ธฐ๋Š” ์™ธ๋ถ€ ์ถœ๋ ฅ(hub CLI stderr)์—์„œ ๋น„๋ฐ€์„ ๊ฐ€๋ฆฐ๋‹ค(๋ฆฌ๋ทฐ ์ง€์  6).
**์•Œ๋ ค์ง„ ๊ฐ’์„ ๋จผ์ €**(๋ฆฌ๋ทฐ L10 ์ง€์  4): `SECRET_ENV_NAMES` ๊ฐ’์ด๋‚˜ ์„ค์ • ์ถœ์ฒ˜ ๊ฐ’(`extra`)์ด ํ†ต์งธ๋กœ ์žˆ์œผ๋ฉด
`_whole_value_replacement` ๋กœ ๋ฐ”๊พธ๊ณ , hub CLI `redactUrl` ์ด ๋น„๋ฐ€๋ฒˆํ˜ธ๋งŒ ๊ฐ€๋ฆฐ ๋ชจ์–‘(`u:***@โ€ฆ?sslpassword=โ€ฆ`)์œผ๋กœ ์žˆ์–ด๋„
๊ฐ™์€ ์ด๋ฆ„ํ‘œ๋กœ ๋ฐ”๊พผ๋‹ค. ๊ทธ **๋’ค์—** ๋‚จ์€ ๋ชจ๋ฅด๋Š” URL ์˜ ๋น„๋ฐ€๋ฒˆํ˜ธ ์ž๋ฆฌ๋ฅผ `***` ๋กœ(`redactUrl` ๊ณผ ๊ฐ™์€ ๋ชจ์–‘ โ€” ํ˜ธ์ŠคํŠธยทDB ์ด๋ฆ„์€
์ง„๋‹จ์— ๋‚จ๋Š”๋‹ค). ์ˆœ์„œ๊ฐ€ ๊ฑฐ๊พธ๋กœ๋ฉด ๋น„๋ฐ€๋ฒˆํ˜ธ๋ฅผ ๊ฐ€๋ฆฐ ์ˆœ๊ฐ„ ๊ฐ’์ด ํ†ต์งธ๋กœ ๋งž์ง€ ์•Š์•„ ์ฟผ๋ฆฌ ๋น„๋ฐ€์ด ์ƒŒ๋‹ค.
์ƒํƒœ ํŒŒ์ผ์€ ์ด ํ•จ์ˆ˜๋ฅผ ์“ฐ์ง€ ์•Š๋Š”๋‹ค โ€” ๊ฑฐ๊ธฐ์—๋Š” ์™ธ๋ถ€ ์ถœ๋ ฅ์„ ์•„์˜ˆ ์‹ฃ์ง€ ์•Š๋Š”๋‹ค(`_refuse_secrets` ๊ฐ€ ๋งˆ์ง€๋ง‰ ๋ฐฉ์–ด).
"""
for name, value in _secret_values(extra):
replacement = _whole_value_replacement(name, value)
text = text.replace(value, replacement)
masked = _mask_url_password(value)
if masked not in (value, replacement):
text = text.replace(masked, replacement)
return _mask_url_password(text)