dev-strender's picture
feat: switch demo to FT + Opus article review (pipeline_exp_261021_ft_hybrid)
3dfc93b verified
Raw History Blame Contribute Delete
11.9 kB
"""ํ‰๊ฐ€ ๋””๋ ‰ํ† ๋ฆฌ โ€” `runs eval` ์ƒˆ ๊ฒฝ๋กœ์˜ ์ฑ„์  ๊ฒฐ๊ณผ ์ž๋ฆฌ (๊ณ„ํš 2026-09-27 L6 ยง2.12 B4 โ‘ก ยท D-L6-15).
์ž๋ฆฌ๋Š” `<data_home>/solar-eval-evaluations/<project>/<run_id>/<eval_ulid>/` ๋‹ค. ๋ ˆ๊ฑฐ์‹œ ์•„์นด์ด๋ธŒ ํŠธ๋ฆฌ(`solar-eval-projects/`)
**๋ฐ–**์ด๋ผ ๋ ˆ๊ฑฐ์‹œ ์ ์žฌ๊ธฐ๊ฐ€ ํ”„๋กœ์ ํŠธ ์ž์›์œผ๋กœ ์˜ค์ธํ•˜์ง€ ์•Š๊ณ , run ๋””๋ ‰ํ† ๋ฆฌ ๋ฐ–์ด๋ผ run ๋ฒˆ๋“ค์ด ๊ทธ๋Œ€๋กœ๋‹ค.
- `evaluation.json`ยท`eval_details.jsonl` ์€ ๋ ˆ๊ฑฐ์‹œ ๋ชจ์–‘ ๊ทธ๋Œ€๋กœ๋‹ค โ€” run ๋””๋ ‰ํ† ๋ฆฌ์™€ ๊ฐ™์€ `JsonlStore` ๋กœ ์“ด๋‹ค
(`build_evaluation_doc`ยท`build_eval_detail_docs` ์žฌ์‚ฌ์šฉ). eval-wrap ์ด ์ด ๋””๋ ‰ํ† ๋ฆฌ๋ฅผ ํ‰๊ฐ€ ์›์ฒœ์œผ๋กœ ์ฝ๋Š”๋‹ค(B3).
- `eval-run.json` ์€ ์ด ํ‰๊ฐ€์˜ ์ •์ฒด โ€” ํ‰๊ฐ€ ์‹œ๊ฐยทํ‰๊ฐ€๊ธฐ ์„ค์ •ยทjudge ๋ชจ๋ธยท์›์ฒœ run ๋ฒˆ๋“ค id. ์›์ž์  ์“ฐ๊ธฐ(์ž„์‹œ โ†’ `os.replace`).
๋‹ค๋ฅธ ๋‘ ํŒŒ์ผ ๋’ค์— ๋งˆ์ง€๋ง‰์œผ๋กœ ์“ด๋‹ค โ€” ์ด ํŒŒ์ผ์ด ์—†๋Š” ๋””๋ ‰ํ† ๋ฆฌ๋Š” ๋๋‚˜์ง€ ์•Š์€ ํ‰๊ฐ€๋‹ค.
- judge ๊ฒฝ๋กœ(๊ณ„ํš 2026-09-28 L12 ยง2.7)๋Š” `verdicts.jsonl`(๊ธฐ์ค€ ํŒ์ • ์ „๋ถ€ โ€” ๊ทผ๊ฑฐยท์› ์‘๋‹ต ํฌํ•จ, ๋ฐ์ดํ„ฐ ํ™ˆ ์ „์šฉ)์„ ๋” ์“ฐ๊ณ 
`eval-run.json` ์— `purpose`ยท`judge{evaluation_run_id, scoring_bundle_id, retry_of}` ๋ฅผ ์‹ฃ๋Š”๋‹ค(๋ฐœํ–‰ **๋’ค**์— ์“ด๋‹ค). lcs ๊ฒฝ๋กœ๋Š”
์„ ํƒ ์ธ์ž๋ฅผ ์ฃผ์ง€ ์•Š์•„ ๋ฐ”์ดํŠธ๊ฐ€ ๊ทธ๋Œ€๋กœ๋‹ค.
- ์ด๋ฆ„ ์กฐ๊ฐ(projectยทrun_id)์€ ๊ฒฝ๋กœ ๊ตฌ๋ถ„์žยท์ œ์–ด ๋ฌธ์žยท`.`/`..` ๋ฅผ ๊ฑฐ๋ถ€ํ•˜๊ณ , ๋ฃจํŠธ๊ฐ€ ๋ ˆํฌ(๋งํฌ๋œ ์›ŒํฌํŠธ๋ฆฌ๋ฉด ๋ฉ”์ธ ์ž‘์—… ํŠธ๋ฆฌ๋„)
์•ˆ์ด๋ฉด ์“ฐ๊ธฐ ์ „์— ๋ฉˆ์ถ˜๋‹ค โ€” hub ๋Œ€์ƒ ํŒ์ •(`hub_hook.target`, D-L6-1)์ด ์ด๋ฏธ ๋ณด์ง€๋งŒ ์ด ๋ชจ๋“ˆ์ด ์ง์ ‘ ์“ฐ๋ฏ€๋กœ ํ•œ ๋ฒˆ ๋” ๋ณธ๋‹ค.
`poc_bundle`ยท`ulid` ๋Š” ํ•จ์ˆ˜ ์•ˆ์—์„œ๋งŒ import ํ•œ๋‹ค(๋งˆ์Šคํ„ฐ ยง4 ๊ทœ์น™ 11 ยท D-L6-14).
"""
from __future__ import annotations
import dataclasses
import json
import os
import re
import shutil
import tempfile
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import Any
from solar_eval.core.run_records import build_eval_detail_docs, build_evaluation_doc
from solar_eval.stores import JsonlStore
#: ๋ฐ์ดํ„ฐ ํ™ˆ ์•„๋ž˜ ํ‰๊ฐ€ ๋””๋ ‰ํ† ๋ฆฌ ๋ฃจํŠธ ์ด๋ฆ„(D-L6-15).
EVALUATIONS_DIRNAME = "solar-eval-evaluations"
EVAL_RUN_FILE = "eval-run.json"
EVAL_RUN_SCHEMA = "solar-eval-eval-run/v1"
#: ์ด ๋””๋ ‰ํ† ๋ฆฌ๊ฐ€ ์‹ฃ๋Š” ํ‰๊ฐ€์˜ ๋ชฉ์  โ€” eval-wrap ๋งค๋‹ˆํŽ˜์ŠคํŠธ `purpose` ์™€ ๊ฐ™์€ ๊ฐ’(PB `PURPOSE_RESCORE_AFTER_FIX`).
EVAL_RUN_PURPOSE = "rescore_after_fix"
#: judge ๊ฒฝ๋กœ(๊ณ„ํš 2026-09-28 L12 ยง2.7) โ€” ๊ธฐ์ค€ ํŒ์ • ์ „๋ถ€(๊ทผ๊ฑฐยท์› ์‘๋‹ต ํฌํ•จ)์™€ `eval-run.json` ์˜ `judge` ์นธ ํ‚ค.
VERDICTS_FILE = "verdicts.jsonl"
JUDGE_BLOCK_KEYS = ("evaluation_run_id", "scoring_bundle_id", "retry_of")
#: ๋ ˆํฌ ์•ˆ ์“ฐ๊ธฐ ๊ฒ€์‚ฌ์— ์ฃผ๋Š” ์ด๋ฆ„ํ‘œ.
WRITE_LABEL = "evaluation directory"
_UNSAFE_SEGMENT = re.compile(r"[/\\\x00-\x1f\x7f]")
_TMP_PREFIX = ".eval-run-"
_TMP_SUFFIX = ".tmp"
#: `EvalDirError.reason` โ€” ๋ฐœํ–‰ ๋‹ค๋ฆฌ ์ฝ”๋“œ `publish_eval_dir:<reason>` ์˜ ๊ผฌ๋ฆฌ. ๋ชจ๋‘ ์„ค์ •ยท์ž…๋ ฅ ์˜ค๋ฅ˜๋ผ ๋‹ค์‹œ ๋Œ๋ ค๋„ ๊ฐ™๋‹ค.
REASON_UNSAFE_SEGMENT = "unsafe_segment"
REASON_INSIDE_REPO = "inside_repo"
REASON_BAD_DOCUMENT = "bad_document"
REASON_OUTSIDE_ROOT = "outside_root"
class EvalDirError(ValueError):
"""ํ‰๊ฐ€ ๋””๋ ‰ํ† ๋ฆฌ๋ฅผ ๋งŒ๋“ค๊ฑฐ๋‚˜ ์“ธ ์ˆ˜ ์—†๋‹ค(์ด๋ฆ„ ์กฐ๊ฐยท๋ ˆํฌ ์•ˆยท์ž˜๋ชป๋œ ๋ฌธ์„œ). `reason` ์€ ์œ„ `REASON_*` ํ•˜๋‚˜."""
def __init__(self, message: str, reason: str) -> None:
super().__init__(message)
self.reason = reason
#: ์ฝ˜์†” ๋ณด๊ณ ์šฉ ์ง‘๊ณ„์˜ ์ฑ„์  ์ž…๋ ฅ ํ–‰ ์ˆ˜(์„ฑ๊ณต + ์‹คํŒจ) โ€” ์ฑ„ํƒ๋œ run ๋ฒˆ๋“ค์˜ ์ •๊ทœํ™” ์‚ฌ๋ณธ ๊ธฐ์ค€(l6b-verify ๋ฆฌ๋ทฐ ์ง€์  4).
REPORT_SAMPLES = "input_samples"
@dataclass(frozen=True)
class WrittenEvaluation:
"""`write_evaluation` ๊ฒฐ๊ณผ โ€” ์ƒˆ ํ‰๊ฐ€ id ์™€ ์ฝ˜์†” ๋ณด๊ณ ์šฉ ์ง‘๊ณ„(`overall_score`ยท`failed`ยท`REPORT_SAMPLES` ํฌํ•จ)."""
evaluation_id: str
aggregated: Mapping[str, Any]
def evaluations_root(env: Mapping[str, str] | None = None) -> Path:
"""`<data_home>/solar-eval-evaluations`(์ ˆ๋Œ€ ๊ฒฝ๋กœ)."""
from poc_bundle.home import data_home
return data_home(env) / EVALUATIONS_DIRNAME
def _check_segment(label: str, value: str) -> str:
if not isinstance(value, str) or value in ("", ".", "..") or _UNSAFE_SEGMENT.search(value):
raise EvalDirError(
f"evaluation directory: {label} is not a safe path segment", REASON_UNSAFE_SEGMENT
)
return value
def _require_outside_repo(root: Path, repo_root: Path | None) -> None:
"""๋ฃจํŠธ๊ฐ€ ๋ ˆํฌ(๋งํฌ๋œ ์›ŒํฌํŠธ๋ฆฌ๋ฉด ๋ฉ”์ธ ์ž‘์—… ํŠธ๋ฆฌ๋„) ์•ˆ์ด๋ฉด `EvalDirError`."""
from poc_bundle.home import HomeError, require_outside_repo
from poc_bundle.mirror import linked_main_work_tree
if repo_root is None:
return
main = linked_main_work_tree(Path(repo_root))
for tree in (Path(repo_root),) if main is None else (Path(repo_root), main):
try:
require_outside_repo(root, tree, WRITE_LABEL)
except HomeError as exc:
raise EvalDirError(
f"evaluation directory is inside the repo: {exc}", REASON_INSIDE_REPO
) from None
def create_evaluation_dir(
project: str,
run_id: str,
*,
repo_root: Path | None,
env: Mapping[str, str] | None = None,
) -> Path:
"""์ƒˆ ํ‰๊ฐ€ ๋””๋ ‰ํ† ๋ฆฌ(๋นˆ ๋””๋ ‰ํ† ๋ฆฌ)๋ฅผ ๋งŒ๋“ค์–ด ๋Œ๋ ค์ค€๋‹ค. ์ด๋ฆ„์€ ์ƒˆ ULID โ€” ๊ฐ™์€ run ์˜ ์žฌํ‰๊ฐ€๋Š” ์‹œ๊ฐ ์ˆœ์œผ๋กœ ์Œ“์ธ๋‹ค."""
from ulid import ULID
parent = (
evaluations_root(env)
/ _check_segment("project", project)
/ _check_segment("run_id", run_id)
)
_require_outside_repo(evaluations_root(env), repo_root)
path = parent / str(ULID())
path.mkdir(parents=True, exist_ok=False)
return path
async def write_evaluation(
path: Path,
run_id: str,
evaluator: Any,
eval_results: Sequence[Mapping[str, Any]],
eval_failures: Sequence[Mapping[str, Any]],
) -> WrittenEvaluation:
"""๋ ˆ๊ฑฐ์‹œ ๋ชจ์–‘ `evaluation.json`ยท`eval_details.jsonl` ์„ ์“ด๋‹ค(`run_eval._evaluate_run` ๊ณผ ๊ฐ™์€ ์กฐ๋ฆฝ)."""
aggregated = evaluator.aggregate(list(eval_results))
doc = build_evaluation_doc(
run_id, aggregated, eval_results, eval_failures, evaluator.judge_model
)
store = JsonlStore(path)
evaluation_id = await store.create_evaluation(doc)
await store.insert_eval_details(
build_eval_detail_docs(evaluation_id, eval_results, eval_failures)
)
report = {
**aggregated,
"overall_score": doc["overall_score"],
"failed": len(eval_failures),
REPORT_SAMPLES: len(eval_results) + len(eval_failures),
}
return WrittenEvaluation(str(evaluation_id), report)
def eval_run_document(
*,
project: str,
run_id: str,
run_bundle_id: str,
evaluation_id: str,
evaluated_at: str,
evaluator_config: Mapping[str, Any] | None,
eval_model: str | None,
purpose: str = EVAL_RUN_PURPOSE,
judge: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
"""`eval-run.json` ๋ฌธ์„œ(์ˆœ์ˆ˜). ์‹œ๊ฐ์€ ์˜คํ”„์…‹์ด ์žˆ์–ด์•ผ ํ•˜๊ณ  ํ‰๊ฐ€๊ธฐ ์„ค์ •์€ JSON ๋งคํ•‘์ด์–ด์•ผ ํ•œ๋‹ค.
`purpose`ยท`judge` ๋Š” judge ๊ฒฝ๋กœ(๊ณ„ํš 2026-09-28 L12 ยง2.7)๋งŒ ์ค€๋‹ค โ€” lcs ๊ฒฝ๋กœ ํ˜ธ์ถœ์€ ์ธ์ž๋ฅผ ์ฃผ์ง€ ์•Š์•„ ๋ฐ”์ดํŠธ๊ฐ€ ๊ทธ๋Œ€๋กœ๋‹ค.
`judge` ๊ฐ€ ์žˆ์œผ๋ฉด ๋์— `{"evaluation_run_id", "scoring_bundle_id", "retry_of"}` ๋ฅผ ๋”ํ•œ๋‹ค(๋ฐœํ–‰ ์‹คํŒจ๋ฉด bundle id ๋Š” null).
"""
_check_offset(evaluated_at)
doc: dict[str, Any] = {
"schema": EVAL_RUN_SCHEMA,
"project": project,
"run_id": run_id,
"run_bundle_id": run_bundle_id,
"evaluation_id": evaluation_id,
"evaluated_at": evaluated_at,
"purpose": purpose,
"evaluator_config": _json_mapping(evaluator_config),
"eval_model": eval_model,
}
if judge is not None:
doc["judge"] = _judge_block(judge)
return doc
def _judge_block(judge: Mapping[str, Any]) -> dict[str, Any]:
"""`eval-run.json` ์˜ `judge` ์นธ โ€” ๋‹ซํžŒ ํ‚ค ์…‹, ๊ฐ’์€ ๋ฌธ์ž์—ด id ๋˜๋Š” null."""
if not isinstance(judge, Mapping) or set(judge) != set(JUDGE_BLOCK_KEYS):
raise EvalDirError(
f"eval-run: judge must have exactly the keys {list(JUDGE_BLOCK_KEYS)}",
REASON_BAD_DOCUMENT,
)
if not all(judge[k] is None or isinstance(judge[k], str) for k in JUDGE_BLOCK_KEYS):
raise EvalDirError("eval-run: judge ids must be strings or null", REASON_BAD_DOCUMENT)
return {key: judge[key] for key in JUDGE_BLOCK_KEYS}
def write_verdicts(path: Path, verdicts: Iterable[Any]) -> Path:
"""`verdicts.jsonl` โ€” ๊ธฐ์ค€ ํŒ์ • ์ „๋ถ€(๊ทผ๊ฑฐยท์› ์‘๋‹ต ํฌํ•จ, ๋ฐ์ดํ„ฐ ํ™ˆ ์ „์šฉ โ€” ๊ณ„ํš L12 ยง2.7 ยท D-L12-8).
ํ•œ ์ค„ = ํŒ์ • dataclass ์˜ JSON(ํ‚ค ์ˆœ์„œ = ํ•„๋“œ ์ˆœ์„œ, compact, `ensure_ascii=False`). UTF-8 ๋กœ ์˜ฎ๊ธธ ์ˆ˜ ์—†๋Š” ๋ฌธ์ž์—ด(์ง ์—†๋Š”
์„œ๋กœ๊ฒŒ์ดํŠธ)์ด ๋“  ์ค„๋งŒ `ensure_ascii=True` ๋กœ ์ ๋Š”๋‹ค โ€” ์œ ๋ฃŒ ํŒ์ • ํ•œ ๊ฑด์ด ํŒŒ์ผ ์ „์ฒด ์“ฐ๊ธฐ๋ฅผ ๊นจ๋œจ๋ฆฌ์ง€ ์•Š๊ฒŒ. ๋ ˆํฌ ๋ฐ– ๊ฒ€์‚ฌ๋Š”
๋””๋ ‰ํ† ๋ฆฌ๋ฅผ ๋งŒ๋“ค ๋•Œ ์ด๋ฏธ ํ–ˆ๋‹ค(`create_evaluation_dir`).
"""
lines: list[str] = []
for verdict in verdicts:
record = dataclasses.asdict(verdict)
line = json.dumps(record, ensure_ascii=False, separators=(",", ":"), allow_nan=False)
try:
line.encode("utf-8")
except UnicodeEncodeError:
line = json.dumps(record, ensure_ascii=True, separators=(",", ":"), allow_nan=False)
lines.append(line + "\n")
target = Path(path) / VERDICTS_FILE
target.write_text("".join(lines), encoding="utf-8")
return target
def _check_offset(value: str) -> None:
try:
parsed = datetime.fromisoformat(value)
except (TypeError, ValueError):
raise EvalDirError(
"eval-run: evaluated_at is not an ISO 8601 time", REASON_BAD_DOCUMENT
) from None
if parsed.tzinfo is None:
raise EvalDirError("eval-run: evaluated_at needs a UTC offset", REASON_BAD_DOCUMENT)
def _json_mapping(value: Mapping[str, Any] | None) -> dict[str, Any] | None:
"""ํ‰๊ฐ€๊ธฐ ์„ค์ •์„ JSON ์œผ๋กœ ์˜ฎ๊ธธ ์ˆ˜ ์žˆ๋Š” ์‚ฌ๋ณธ์œผ๋กœ. ์„ค์ •์ด ์—†์œผ๋ฉด(๊ธฐ๋ณธ ํ‰๊ฐ€๊ธฐ) None."""
if value is None:
return None
if not isinstance(value, Mapping):
raise EvalDirError("eval-run: evaluator_config must be a mapping", REASON_BAD_DOCUMENT)
try:
return json.loads(json.dumps(dict(value), ensure_ascii=False))
except (TypeError, ValueError):
raise EvalDirError(
"eval-run: evaluator_config is not JSON serializable", REASON_BAD_DOCUMENT
) from None
def write_eval_run(path: Path, doc: Mapping[str, Any]) -> Path:
"""`eval-run.json` ์›์ž์  ์“ฐ๊ธฐ(์ง๋ ฌํ™” โ†’ ๊ฐ™์€ ๋””๋ ‰ํ† ๋ฆฌ ์ž„์‹œ ํŒŒ์ผ โ†’ fsync โ†’ `os.replace`)."""
text = json.dumps(doc, ensure_ascii=False, indent=2) + "\n"
target = Path(path) / EVAL_RUN_FILE
fd, tmp_name = tempfile.mkstemp(dir=path, prefix=_TMP_PREFIX, suffix=_TMP_SUFFIX)
tmp = Path(tmp_name)
try:
with os.fdopen(fd, "w", encoding="utf-8") as fh:
fh.write(text)
fh.flush()
os.fsync(fh.fileno())
os.replace(tmp, target)
except BaseException:
tmp.unlink(missing_ok=True)
raise
return target
def remove_partial(path: Path, env: Mapping[str, str] | None = None) -> None:
"""๋๋‚˜์ง€ ๋ชปํ•œ ํ‰๊ฐ€ ๋””๋ ‰ํ† ๋ฆฌ๋ฅผ ์น˜์šด๋‹ค โ€” ํ‰๊ฐ€ ๋ฃจํŠธ ์•„๋ž˜์˜ ๋””๋ ‰ํ† ๋ฆฌ๋งŒ(๋‹ค๋ฅธ ๊ณณ์€ `EvalDirError`)."""
resolved = Path(path).resolve()
root = evaluations_root(env).resolve()
if root not in resolved.parents:
raise EvalDirError(
"evaluation directory: refusing to remove a path outside the root", REASON_OUTSIDE_ROOT
)
shutil.rmtree(resolved)