dev-strender's picture
feat: switch demo to FT + Opus article review (pipeline_exp_261021_ft_hybrid)
3dfc93b verified
Raw History Blame Contribute Delete
23.1 kB
"""run ์‹œ์ž‘ยทresume ์ด ๊ณต์œ ํ•˜๋Š” ์ค€๋น„ ๋‹จ๊ณ„์™€ run.json ๋ฉ”ํƒ€ ์กฐ๋ฆฝ (๊ณ„ํš 2026-09-24-phase1a D3ยทD5ยทD12ยทD17ยทD19).
`runs start` ๋Š” run ๋””๋ ‰ํ† ๋ฆฌ๋ฅผ ๋งŒ๋“ค๊ธฐ **์ „์—** ์ด ๋ชจ๋“ˆ๋กœ ๋ชจ๋“  ๊ฒƒ์„ ์ •ํ•œ๋‹ค:
ํƒœ์Šคํฌยทํ”„๋กฌํ”„ํŠธยทํŒŒ์ดํ”„๋ผ์ธ ํ•ด์„ โ†’ ํ–‰ ์„ ํƒ(`where`ยท`limit`) โ†’ ๋ฐ์ดํ„ฐ์…‹ ํ•œ ๋ฒˆ ์ฝ๊ธฐ โ†’ ํŒŒ์ดํ”„๋ผ์ธ ์ƒ์„ฑ โ†’
์Šค๋ƒ…์ƒท(run.json `dataset`ยท`snapshot` ๋ธ”๋ก, ์ž๊ธฐ ๊ฒ€์ฆ๊นŒ์ง€). ๊ทธ๋ž˜์„œ 0ํ–‰ ์„ ํƒยท๋ผ๋ฒจ ๋ˆ„๋ฝยท์—†๋Š” ์‚ฌ์ „ยท๊นจ์ง„ lock
๊ฐ™์€ ์˜ค๋ฅ˜๊ฐ€ ๋นˆ run ๋””๋ ‰ํ† ๋ฆฌ๋ฅผ ๋‚จ๊ธฐ์ง€ ์•Š๊ณ , run.json ์ด ์ ์€ sha256ยท์Šค๋ƒ…์ƒท๊ณผ ๋Ÿฌ๋„ˆ๊ฐ€ ์‹ค์ œ๋กœ ๋„๋Š”
๋ฐ์ดํ„ฐยทํŒŒ์ดํ”„๋ผ์ธ์ด **๊ฐ™์€ ๊ฐ์ฒด**์—์„œ ๋‚˜์˜จ๋‹ค.
resume ์€ ๊ฐ™์€ ์กฐ๋ฆฝ ํ•จ์ˆ˜(`build_snapshot`)๋กœ ์ง€๊ธˆ ๋””์Šคํฌ ๊ธฐ์ค€ ์Šค๋ƒ…์ƒท์„ ๋‹ค์‹œ ๋–  ์„ค์ • ๋“œ๋ฆฌํ”„ํŠธ๋ฅผ ๋ง‰๋Š”๋‹ค(D17).
`poc_bundle` import ๋Š” ์ด ๋ชจ๋“ˆ(๊ณผ `run_resume`ยท`run_eval`)๊ณผ `core/snapshot/**` ์—์„œ๋งŒ ํ•œ๋‹ค(D18).
"""
from __future__ import annotations
import subprocess
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import click
from poc_bundle.data.errors import DataError
from solar_eval.core.dataset_loader import DatasetLoader, LoadedJsonl
from solar_eval.core.dataset_prep import (
PreparedDataset,
build_selection,
prepare_dataset,
reject_defaults_where,
)
from solar_eval.core.pipeline_factory import build_task_pipeline
from solar_eval.core.row_filter import RowSelection, parse_cli_where
from solar_eval.core.snapshot import EngineSnapshotInputs, build_engine_snapshot, self_check
from solar_eval.core.snapshot import environment
from solar_eval.core.snapshot.dataset_block import describe_dataset
from solar_eval.evaluators.judge_config import JudgeConfigError, validate_judge_config
#: run ์„ ์‹œ์ž‘ํ•˜๊ธฐ ์ „์— ์‚ฌ๋žŒ์ด ์ฝ์„ ๋ฉ”์‹œ์ง€๋กœ ๋ฐ”๊ฟ” ๋ฉˆ์ถ”๋Š” ์˜ค๋ฅ˜ โ€” ํ–‰ ์„ ํƒยทmeta_fieldsยทdefaults.where
#: (ValueError ๊ณ„์—ด), ์—†๋Š” ๋ฐ์ดํ„ฐ์…‹ยท์ง€์‹ ์ž์‚ฐ(FileNotFoundError ๊ณ„์—ด), ๊นจ์ง„ datasets.lock(DataError).
PREP_ERRORS: tuple[type[Exception], ...] = (ValueError, FileNotFoundError, DataError)
#: ์ง€๋ฌธ์„ ์‚ฌ๋žŒ์ด ๋ˆˆ์œผ๋กœ ๋Œ€์กฐํ•  ๋•Œ ๋ณด์—ฌ ์ฃผ๋Š” ์•ž์ž๋ฆฌ ์ˆ˜ โ€” legacy `prompt_sha` ํ˜•์‹๊ณผ ๋งž์ถ˜๋‹ค.
FINGERPRINT_DISPLAY_CHARS = 12
def resolve_prompt(prompts_dir: Path, prompt_name: str) -> Path:
"""Resolve prompt from prompts/ directory (directory or YAML file).
Resolution order:
1. prompts/{name}/ (directory TXT format)
2. prompts/{name}.yaml (flat YAML layout)
3. prompts/_library/{name}.yaml (legacy layout)
"""
dir_path = prompts_dir / prompt_name
if dir_path.is_dir():
return dir_path
yaml_path = prompts_dir / f"{prompt_name}.yaml"
if yaml_path.exists():
return yaml_path
legacy_path = prompts_dir / "_library" / f"{prompt_name}.yaml"
if legacy_path.exists():
return legacy_path
raise click.ClickException(
f"Prompt '{prompt_name}' not found. Searched:\n"
f" {dir_path}/ (directory)\n"
f" {yaml_path} (YAML)\n"
f" {legacy_path} (legacy)"
)
def resolve_pipeline_file(project_dir: Path, pipeline_name: str) -> Path:
"""Resolve pipeline YAML from pipelines/ directory."""
path = project_dir / "pipelines" / f"{pipeline_name}.yaml"
if not path.exists():
raise click.ClickException(f"Pipeline '{pipeline_name}' not found: {path}")
return path
def compose_pipeline_config(project_dir: Path, pipeline_name: str, overrides: dict) -> dict:
"""ํŒŒ์ดํ”„๋ผ์ธ์„ `extends`/override ๊นŒ์ง€ ํ•ด์„ํ•ด ๋Œ๋ ค์ค€๋‹ค.
์กฐ๋ฆฝ ์‹คํŒจ(์—†๋Š” base, ์—†๋Š” ์Šคํ…์„ ๊ฐ€๋ฆฌํ‚ค๋Š” override ๋“ฑ)๋Š” run ์„ ์‹œ์ž‘ํ•˜๊ธฐ ์ „์—
์ฃฝ์ธ๋‹ค โ€” ํšจ๊ณผ ์—†๋Š” override ๋กœ ๋Œ์•„๊ฐ„ run ์€ ๊ฒฐ๋ก ์„ ์กฐ์šฉํžˆ ์˜ค์—ผ์‹œํ‚จ๋‹ค.
"""
from solar_eval.core.pipeline_compose import PipelineCompositionError, compose_pipeline
from solar_eval.core.project_loader import load_pipeline_file
pipelines_dir = resolve_pipeline_file(project_dir, pipeline_name).parent
try:
composed = load_pipeline_file(pipelines_dir, pipeline_name)
if overrides:
composed = compose_pipeline(composed, overrides=overrides)
return composed
except (PipelineCompositionError, FileNotFoundError) as e:
raise click.ClickException(str(e)) from None
def config_source(cfg, project: str) -> str:
"""run ์˜ config ์ถœ์ฒ˜ โ€” ๋ ˆํฌ ๊ด€๋ฆฌ ํ”„๋กœ์ ํŠธ๋ฉด ๋ ˆํฌ ์ƒ๋Œ€ ๊ฒฝ๋กœ, ์•„๋‹ˆ๋ฉด ๋ฐ์ดํ„ฐ ํ™ˆ ์ ˆ๋Œ€ ๊ฒฝ๋กœ."""
config_dir = cfg.config_dir(project)
if cfg.is_repo_managed(project) and cfg.repo_root is not None:
return str(config_dir.relative_to(cfg.repo_root))
return str(config_dir)
def git_commit(repo_root: Path | None) -> str | None:
"""์žฌํ˜„์šฉ ๋„์žฅ: ๋ ˆํฌ HEAD ์ปค๋ฐ‹. ์ž‘์—… ํŠธ๋ฆฌ๊ฐ€ ๋”๋Ÿฌ์šฐ๋ฉด '-dirty' ๋ฅผ ๋ถ™์ธ๋‹ค (๋ ˆํฌ ์ „์ฒด โ€” D8 ๋ถˆ๋ณ€)."""
if repo_root is None:
return None
try:
head = subprocess.run(
["git", "-C", str(repo_root), "rev-parse", "--short", "HEAD"],
capture_output=True,
text=True,
check=True,
).stdout.strip()
status = subprocess.run(
["git", "-C", str(repo_root), "status", "--porcelain"],
capture_output=True,
text=True,
check=True,
).stdout.strip()
return f"{head}-dirty" if status else head
except (OSError, subprocess.CalledProcessError):
return None
def find_task(cfg, project: str, task: str) -> tuple[dict[str, Any], dict[str, Any]]:
"""๋กœ๋“œ๋œ ํ”„๋กœ์ ํŠธ ์„ค์ •๊ณผ ํƒœ์Šคํฌ ์„ค์ •. ์—†์œผ๋ฉด ClickException."""
from solar_eval.core.project_loader import load_all_project_configs
configs = load_all_project_configs(cfg.projects_dir, cfg.config_dirs)
project_config = next((c for c in configs if c["name"] == project), None)
if not project_config:
raise click.ClickException(f"Project not found in {cfg.projects_dir}: {project}")
task_config = next((t for t in project_config.get("tasks", []) if t["name"] == task), None)
if not task_config:
raise click.ClickException(f"Task '{task}' not found in project '{project}'")
return project_config, task_config
def load_run(cfg, project: str, run_id: str) -> tuple[Path, Any, dict[str, Any]]:
"""(run ๋””๋ ‰ํ† ๋ฆฌ, ์Šคํ† ์–ด, run.json). ์—†์œผ๋ฉด ClickException."""
from solar_eval.stores import JsonlStore
run_dir = cfg.artifacts_dir(project) / run_id
if not run_dir.exists():
raise click.ClickException(f"Run directory not found: {run_dir}")
store = JsonlStore(run_dir)
run_meta = store.load_run_meta()
if not run_meta:
raise click.ClickException(f"No run.json found in {run_dir}")
if not run_meta.get("task"):
raise click.ClickException("Cannot determine task from run.json")
return run_dir, store, run_meta
def load_prompt(
prompt_path: Path,
model: str,
temperature: float,
max_tokens: int,
reasoning_effort: str | None,
) -> tuple[dict[str, Any], str]:
"""ํ”„๋กฌํ”„ํŠธ ํŒŒ์ผ์„ ๋Ÿฌ๋„ˆ๊ฐ€ ๋ฐ›๋Š” dict ๋กœ โ€” (prompt, ํฌ๋งท `multi_step`|`single_step`)."""
from solar_eval.models.prompt_version import (
detect_prompt_format,
load_prompt_messages,
load_step_prompts,
)
fmt = detect_prompt_format(prompt_path)
if fmt == "multi_step":
prompt: dict[str, Any] = {"step_prompts": load_step_prompts(prompt_path)}
else:
prompt = {"messages": load_prompt_messages(prompt_path)}
prompt.update({"model": model, "temperature": temperature, "max_tokens": max_tokens})
if reasoning_effort:
prompt["reasoning_effort"] = reasoning_effort
return prompt, fmt
def make_providers(evaluator_config: Mapping[str, Any] | None = None) -> tuple[Any, Any]:
"""(์ถ”๋ก  ํ”„๋กœ๋ฐ”์ด๋”, judge ํ”„๋กœ๋ฐ”์ด๋”). ํ…Œ์ŠคํŠธ๋Š” ์ด ํ•จ์ˆ˜๋ฅผ ๋ฐ”๊ฟ” ๋ผ์›Œ ๋„คํŠธ์›Œํฌ ์—†์ด ๋ˆ๋‹ค.
`evaluator_config` ๋Š” ํƒœ์Šคํฌ์˜ ํ‰๊ฐ€๊ธฐ ์„ค์ •(ํ‚ค๊ฐ€ ์—†์œผ๋ฉด None) โ€” judge ์ธ์Šคํ„ด์Šค๋ฅผ ์„ ์–ธ(`judge_provider`)์œผ๋กœ ๊ณ ๋ฅด๋Š”
์ž๋ฆฌ๋‹ค(๊ณ„ํš 2026-09-28 L12 ยง2.8 ยท D-L12-10). ์„ ์–ธ์ด ์—†๊ฑฐ๋‚˜ ํŒ์ • ์—†๋Š” ํ‰๊ฐ€๊ธฐ๋Š” ์˜ค๋Š˜์ฒ˜๋Ÿผ `OpenAIProvider()` โ€”
run.json ์Šค๋ƒ…์ƒท `provenance.providers.judge` ๋ฐ”์ดํŠธ๊ฐ€ ๋ฐ”๋€Œ์ง€ ์•Š๋Š”๋‹ค. ์ถ”๋ก  ํ”„๋กœ๋ฐ”์ด๋”๋Š” ๋Š˜ `UpstageProvider()`.
Raises:
JudgeConfigError: ๋ชจ๋ฅด๋Š” providerยทํ•ฉ์„ฑ ํ‰๊ฐ€๊ธฐ ์•ˆ์˜ provider ์ถฉ๋Œ.
"""
from solar_eval.evaluators.judge_config import (
PROVIDER_OPENAI,
PROVIDER_UPSTAGE,
judge_provider_name,
)
from solar_eval.providers import UpstageProvider
from solar_eval.providers.openai_provider import OpenAIProvider
judge_classes = {PROVIDER_UPSTAGE: UpstageProvider, PROVIDER_OPENAI: OpenAIProvider}
name = judge_provider_name(evaluator_config) or PROVIDER_OPENAI
return UpstageProvider(), judge_classes[name]()
def route_inference_provider(provider: Any, step_models: Sequence[str]) -> Any:
"""์Šคํ… ๋ชจ๋ธ์— ๋ผ์šฐํŠธ ์ ‘๋‘์–ด(`timely:<๋ชจ๋ธ>`)๊ฐ€ ํ•˜๋‚˜๋ผ๋„ ์žˆ์œผ๋ฉด ์ถ”๋ก  ์ œ๊ณต์ž๋ฅผ `RoutedProvider` ๋กœ ๊ฐ์‹ผ๋‹ค.
์—†์œผ๋ฉด ๊ทธ๋Œ€๋กœ ๋Œ๋ ค์ค€๋‹ค โ€” ๋ผ์šฐํŠธ ์—†๋Š” run ์˜ provenance(`providers.inference` ๋ชจ๋“ˆ)๊ฐ€ ์˜ˆ์ „ ๋ฐ”์ดํŠธ ๊ทธ๋Œ€๋กœ์—ฌ์•ผ ํ•œ๋‹ค.
์™ธ๋ถ€ ๋ผ์šฐํŠธ๋ฅผ ์จ๋„ ๋˜๋Š”๊ฐ€๋Š” ์—”์ง„์ด ์•„๋‹ˆ๋ผ case ์˜ ๋ฐ์ดํ„ฐ ์ •์ฑ…(๊ณ„์•ฝ)์ด ์ •ํ•œ๋‹ค โ€” 2026-10 judge egress ์ •์ฑ… ํ์ง€(#353)์™€ ๊ฐ™์€ ์ž๋ฆฌ.
"""
from solar_eval.providers import INFERENCE_ROUTES, RoutedProvider
from solar_eval.providers.routed import route_prefixes
if not route_prefixes(list(step_models), INFERENCE_ROUTES):
return provider
return RoutedProvider(provider, {k: cls() for k, cls in INFERENCE_ROUTES.items()})
@dataclass(frozen=True)
class RunPlan:
"""run ํ•˜๋‚˜๊ฐ€ ๋ฌด์—‡์œผ๋กœ ๋„๋Š”๊ฐ€ โ€” run ๋””๋ ‰ํ† ๋ฆฌ๋ฅผ ๋งŒ๋“ค๊ธฐ ์ „์— ์ „๋ถ€ ์ •ํ•ด์ง„๋‹ค."""
project: str
task: str
project_config: dict[str, Any]
task_config: dict[str, Any]
prompt_name: str
prompt_path: Path
prompt_format: str
prompt: dict[str, Any]
model: str
temperature: float
max_tokens: int
reasoning_effort: str | None
pipeline_name: str
pipeline_config: dict[str, Any]
pipeline_overrides: dict[str, Any] | None
max_workers: int
dataset_loader: DatasetLoader
prepared: PreparedDataset
pipeline: Any
provider: Any
judge_provider: Any
@property
def selection(self) -> RowSelection | None:
return self.prepared.selection
def resolve_pipeline(
cfg, project: str, task_config: dict[str, Any], pipeline_name: str | None, overrides: dict
) -> tuple[dict[str, Any], str, dict[str, Any]]:
"""(ํŒŒ์ดํ”„๋ผ์ธ์„ ์–น์€ task_config, ์‹คํšจ ํŒŒ์ดํ”„๋ผ์ธ ์ด๋ฆ„, ํ•ด์„๋œ ํŒŒ์ดํ”„๋ผ์ธ dict).
`-P` ๊ฐ€ ์žˆ์œผ๋ฉด config ์ •๋ณธ์˜ pipelines/ ์—์„œ `extends`ยท`--set` ๊นŒ์ง€ ํ•ด์„ํ•˜๊ณ , ์—†์œผ๋ฉด
project.yaml ๊ธฐ๋ณธ๊ฐ’(project_loader ๊ฐ€ ์ด๋ฏธ ํ•ด์„)์„ ์“ด๋‹ค. config ๋Š” ๋ ˆํฌ ๊ด€๋ฆฌ ํ”„๋กœ์ ํŠธ๋ฉด git
์ชฝ์ด ์ •๋ณธ์ด๋‹ค. `extends` ์ƒ์†๊ณผ `--set` override ๋Š” ์—ฌ๊ธฐ์„œ ์ „๋ถ€ ํ•ด์„๋ผ, ์‹คํ–‰ ์—”์ง„์—๋Š” ์™„์„ฑ๋œ
steps ๋ชฉ๋ก๋งŒ ๋„˜์–ด๊ฐ„๋‹ค.
"""
if pipeline_name:
pipeline_config = compose_pipeline_config(cfg.config_dir(project), pipeline_name, overrides)
return {**task_config, "pipeline_config": pipeline_config}, pipeline_name, pipeline_config
if overrides:
raise click.ClickException("--set requires an explicit --pipeline/-P")
pc = task_config.get("pipeline_config", {})
effective = pc.get("name", "default") if isinstance(pc, dict) else str(pc)
return task_config, effective, pc if isinstance(pc, dict) else {}
class _CheckedReader:
"""`prepare_dataset` ์— ๋„˜๊ธฐ๋Š” ์–‡์€ ๋Œ€๋ฆฌ์ž โ€” ํŒŒ์ผ์„ ์ฝ์€ **์งํ›„**, ํ–‰์„ ๊ณ ๋ฅด๊ธฐ **์ „์—** `on_read` ๋ฅผ ๋ถ€๋ฅธ๋‹ค.
resume ์˜ ๋ฐ์ดํ„ฐ์…‹ sha ๊ฐ€๋“œ(D17 ์ˆœ์„œ์˜ ์ฒซ์งธ)๊ฐ€ ์„ ํƒยท๋ผ๋ฒจ ๊ฒ€์‚ฌ๋ณด๋‹ค ๋จผ์ € ๊ฑธ๋ฆฌ๊ฒŒ ํ•˜๋ ค๋Š” ๊ฒƒ์ด๋‹ค โ€”
๋ฐ์ดํ„ฐ์…‹์ด ๋ฐ”๋€Œ์–ด ์„ ํƒ์ด 0ํ–‰์ด ๋˜๋ฉด ์›์ธ์€ "where ๊ฐ€ 0ํ–‰" ์ด ์•„๋‹ˆ๋ผ "๋ฐ์ดํ„ฐ์…‹์ด ๋ฐ”๋€Œ์—ˆ๋‹ค" ๋‹ค.
๋ฐ”์ดํŠธ๋Š” ์—ฌ์ „ํžˆ ํ•œ ๋ฒˆ๋งŒ ์ฝ๋Š”๋‹ค(D16).
"""
def __init__(self, loader: DatasetLoader, on_read: Callable[[LoadedJsonl], None]) -> None:
self._loader = loader
self._on_read = on_read
def read_jsonl(self, repo: str, path: str, source: str) -> LoadedJsonl:
loaded = self._loader.read_jsonl(repo, path, source=source)
self._on_read(loaded)
return loaded
def _step_models(pipeline: Any) -> list[str]:
"""ํŒŒ์ดํ”„๋ผ์ธ ์Šคํ…๋“ค์ด ๋ช…์‹œํ•œ ๋ชจ๋ธ ๋ฌธ์ž์—ด โ€” ๋ผ์šฐํŠธ ์ ‘๋‘์–ด ๊ฒ€์‚ฌ์šฉ(์—†๋Š” ์Šคํ…์€ run ์˜ `-m` ๋ชจ๋ธ = Upstage)."""
return [
str(step.config["model"])
for step in getattr(pipeline, "steps", [])
if isinstance(getattr(step, "config", None), dict) and step.config.get("model")
]
def assemble_plan(
cfg,
selection: RowSelection | None,
*,
on_read: Callable[[LoadedJsonl], None] | None = None,
**fields: Any,
) -> RunPlan:
"""๋ฐ์ดํ„ฐ์…‹์„ ํ•œ ๋ฒˆ ์ฝ์–ด ๊ณ ๋ฅด๊ณ , ๋Ÿฌ๋„ˆ๊ฐ€ ๋Œ ํŒŒ์ดํ”„๋ผ์ธ ์ธ์Šคํ„ด์Šค๋ฅผ ๋งŒ๋“ ๋‹ค(D6ยทD16).
`fields` ๋Š” `RunPlan` ์˜ ๋‚˜๋จธ์ง€ ํ•„๋“œ(projectยทtask_configยทprompt ๋“ฑ)๋‹ค. `on_read` ๋Š” ๋ฐ์ดํ„ฐ์…‹์„
์ฝ์€ ์งํ›„ยท์„ ํƒ ์ „์— ๋ถˆ๋ฆฐ๋‹ค(resume ์˜ sha ๊ฐ€๋“œ โ€” ๊ฑฐ๊ธฐ์„œ ๋‚œ ClickException ์€ ๊ทธ๋Œ€๋กœ ์˜ฌ๋ผ๊ฐ„๋‹ค).
judge ์„ค์ • ์˜ค๋ฅ˜๋Š” ๋ฐ์ดํ„ฐ์…‹ ์ฝ๊ธฐยทํŒŒ์ดํ”„๋ผ์ธยทprovider ์ƒ์„ฑ๋ณด๋‹ค ๋จผ์ € ํ™•์ธํ•œ๋‹ค.
`runs start`ยท`quick-run`ยท`runs batch`(ํ”„๋กฌํ”„ํŠธ๋งˆ๋‹ค)ยท`runs resume` ์ด ๋ชจ๋‘ ์ด ์ž๋ฆฌ๋ฅผ ์ง€๋‚œ๋‹ค.
Raises:
click.ClickException: judge ์„ค์ •ยท์„ ํƒยทmeta_fieldsยท๋ฐ์ดํ„ฐ์…‹ยทํŒŒ์ดํ”„๋ผ์ธ ์ƒ์„ฑ ์˜ค๋ฅ˜(run ์‹œ์ž‘ ์ „).
"""
preflight_judge_config(fields["task_config"])
loader = DatasetLoader(base_dir=cfg.projects_dir)
reader: Any = loader if on_read is None else _CheckedReader(loader, on_read)
try:
prepared = prepare_dataset(
reader, fields["project_config"], fields["task_config"], selection
)
pipeline = build_task_pipeline(
fields["project_config"], fields["task_config"], fields["prompt"], loader
)
except PREP_ERRORS as e:
raise click.ClickException(str(e)) from None
provider, judge_provider = make_providers(fields["task_config"].get("evaluator"))
provider = route_inference_provider(provider, _step_models(pipeline))
return RunPlan(
dataset_loader=loader,
prepared=prepared,
pipeline=pipeline,
provider=provider,
judge_provider=judge_provider,
**fields,
)
def preflight_judge_config(task_config: Mapping[str, Any]) -> None:
"""Judge ์„ค์ • ์˜ค๋ฅ˜๋ฅผ CLI ์˜ค๋ฅ˜๋กœ ๋ฐ”๊พธ๋˜ provider ๋Š” ๋งŒ๋“ค์ง€ ์•Š๋Š”๋‹ค."""
try:
validate_judge_config(task_config)
except JudgeConfigError as exc:
raise click.ClickException(str(exc)) from None
def start_options(
project_config: dict[str, Any],
task_config: dict[str, Any],
set_options: tuple[str, ...],
where: tuple[str, ...],
limit: int | None,
) -> tuple[dict[str, Any], RowSelection]:
"""(`--set` override, ํ–‰ ์„ ํƒ) โ€” ํ”„๋กฌํ”„ํŠธ์™€ ๋ฌด๊ด€ํ•œ ์‹œ์ž‘ ๊ฒ€์‚ฌ. `runs batch` ๋Š” ๋ฃจํ”„ ์ „์— ํ•œ ๋ฒˆ ๋ถ€๋ฅธ๋‹ค.
Raises:
click.ClickException: `defaults.where`ยทwhere ๋ฌธ๋ฒ•ยท`--set` ๋ฌธ๋ฒ• ์˜ค๋ฅ˜.
"""
from solar_eval.core.pipeline_compose import PipelineCompositionError, parse_set_option
try:
reject_defaults_where(project_config)
overrides = parse_set_option(set_options)
selection = build_selection(task_config, parse_cli_where(where), limit)
except (PipelineCompositionError, *PREP_ERRORS) as e:
raise click.ClickException(str(e)) from None
return overrides, selection
def plan_run(
cfg,
*,
project: str,
task: str,
prompt_name: str,
model: str,
pipeline_name: str | None,
reasoning_effort: str | None,
temperature: float,
max_tokens: int,
max_workers: int,
limit: int | None,
set_options: tuple[str, ...] = (),
where: tuple[str, ...] = (),
) -> RunPlan:
"""`runs start` ์˜ ์ค€๋น„ ๋‹จ๊ณ„ ์ „๋ถ€ โ€” `reject_defaults_where` โ†’ ์„ ํƒ โ†’ `prepare_dataset` โ†’
`build_task_pipeline` (๊ณ„ํš ยง9 G2 ์ˆœ์„œ). ๋ชจ๋“  ์˜ค๋ฅ˜๋Š” run ๋””๋ ‰ํ† ๋ฆฌ๋ฅผ ๋งŒ๋“ค๊ธฐ ์ „์— ๋‚œ๋‹ค.
"""
project_config, task_config = find_task(cfg, project, task)
overrides, selection = start_options(project_config, task_config, set_options, where, limit)
prompt_path = resolve_prompt(cfg.prompts_dir(project), prompt_name)
prompt, fmt = load_prompt(prompt_path, model, temperature, max_tokens, reasoning_effort)
task_config, effective, pipeline_config = resolve_pipeline(
cfg, project, task_config, pipeline_name, overrides
)
return assemble_plan(
cfg,
selection,
project=project,
task=task,
project_config=project_config,
task_config=task_config,
prompt_name=prompt_name,
prompt_path=prompt_path,
prompt_format=fmt,
prompt=prompt,
model=model,
temperature=temperature,
max_tokens=max_tokens,
reasoning_effort=reasoning_effort,
pipeline_name=effective,
pipeline_config=pipeline_config,
pipeline_overrides=overrides or None,
max_workers=max_workers,
)
def dataset_block(cfg, plan: RunPlan) -> dict[str, Any]:
"""run.json `dataset` ๋ธ”๋ก โ€” ์ฝ์€ ๋ฐ”์ดํŠธ์˜ sha256ยทํ–‰ ์ˆ˜ยท์ •๊ทœํ™”๋œ ์„ ํƒยทlock ๋Œ€์กฐ(S6)."""
prepared = plan.prepared
selection = plan.selection or RowSelection()
try:
return describe_dataset(
sha256=prepared.sha256,
rows_total=prepared.rows_total,
rows_selected=prepared.rows_selected,
selection=selection.to_json(),
meta_fields=prepared.meta_fields,
task_config=plan.task_config,
loader_cfg=plan.project_config.get("dataset") or {},
config_dir=cfg.config_dir(plan.project),
project_name=plan.project,
repo_root=environment.engine_repo(),
)
except PREP_ERRORS as e:
raise click.ClickException(f"dataset lock check failed: {e}") from None
def build_snapshot(
cfg,
plan: RunPlan,
dataset: dict[str, Any] | None,
*,
source: str | None,
commit: str | None,
) -> dict[str, Any]:
"""run.json `snapshot` ๋ธ”๋ก(engine-snapshot/v1). start ์™€ resume ์ด ๊ฐ™์€ ํ•จ์ˆ˜๋ฅผ ์“ด๋‹ค(D17).
Raises:
click.ClickException: ์„ ์–ธํ•œ ์ž์›์ด ์—†๊ฑฐ๋‚˜ ์Šค๋ƒ…์ƒท์ด ์ž๊ธฐ ๊ฒ€์ฆ(D5)์„ ํ†ต๊ณผํ•˜์ง€ ๋ชปํ•  ๋•Œ.
"""
inputs = EngineSnapshotInputs(
model=plan.model,
temperature=plan.temperature,
max_tokens=plan.max_tokens,
reasoning_effort=plan.reasoning_effort,
pipeline_type=plan.task_config.get("pipeline", "single_step"),
input_fields=plan.task_config.get("input_fields", []),
resolved_pipeline=plan.pipeline_config,
prompt_path=plan.prompt_path,
pipeline=plan.pipeline,
evaluator_config=plan.task_config.get("evaluator"),
inference_provider=plan.provider,
judge_provider=plan.judge_provider,
prompt_name=plan.prompt_name,
pipeline_name=plan.pipeline_name,
config_source=source,
git_commit=commit,
pipeline_overrides=plan.pipeline_overrides,
max_workers=plan.max_workers,
config_dir=cfg.config_dir(plan.project),
repo_managed=cfg.is_repo_managed(plan.project),
dataset=dataset,
)
try:
snapshot = build_engine_snapshot(inputs)
self_check(snapshot)
except PREP_ERRORS as e:
raise click.ClickException(f"cannot snapshot this run's config: {e}") from None
return snapshot
def auto_note(note: str | None, plan: RunPlan) -> str | None:
"""`--where` run ์˜ note ์ž๋™ ๊ธฐ์ž…(D19) โ€” Experiment Hub ๋ชฉ๋ก์—์„œ ์ „์ฒด์…‹ ๋ฐ˜๋ณต๊ณผ ์‚ฌ๋žŒ์ด ๊ตฌ๋ถ„ํ•˜๊ฒŒ.
note ๊ฐ€ ๋น„๋ฉด `where tags=yul-ryul ยท 12/159 rows`, ์‚ฌ์šฉ์ž๊ฐ€ ์ฃผ๋ฉด ๊ทธ ๋’ค์— ` [where โ€ฆ]`.
where ๊ฐ€ ์—†๋Š” run ์€ note ๋ฅผ ๊ทธ๋Œ€๋กœ ๋‘”๋‹ค(๋‹ค๋ฅธ run ์˜ run.json ๋ฐ”์ดํŠธ ๋ถˆ๋ณ€).
"""
selection = plan.selection
if selection is None or selection.where is None:
return note
prepared = plan.prepared
label = (
f"where {selection.where.describe()} ยท {prepared.rows_selected}/{prepared.rows_total} rows"
)
return f"{note} [{label}]" if note else label
def build_run_meta(cfg, plan: RunPlan, note: str | None) -> dict[str, Any]:
"""run.json ์ดˆ๊ธฐ ๋‚ด์šฉ โ€” legacy ํ‚ค(๊ฐ’ ๊ทœ์น™ ๋ถˆ๋ณ€, D12) ๋’ค์— `dataset`ยท`snapshot`(๊ณ„ํš ยง3.2 ์ˆœ์„œ)."""
from solar_eval.core.fingerprint import pipeline_fingerprint, prompt_fingerprint
source = config_source(cfg, plan.project)
commit = git_commit(cfg.repo_root) if cfg.is_repo_managed(plan.project) else None
dataset = dataset_block(cfg, plan)
return {
"task": plan.task,
"model": plan.model,
"prompt": plan.prompt_name,
"pipeline": plan.pipeline_name,
"temperature": plan.temperature,
"max_tokens": plan.max_tokens,
"reasoning_effort": plan.reasoning_effort,
# ์™œ ๋Œ๋ ธ๋Š”์ง€ โ€” ์˜ˆ์ „์—๋Š” ๋ณ„๋„ ๋žฉ๋…ธํŠธ(experiments/*.yaml)๊ฐ€ ๋‹ด์•˜์ง€๋งŒ run ๊ณผ ๋ถ„๋ฆฌ๋ผ ์žˆ์–ด
# ์ฐธ์กฐ๊ฐ€ ๋Š์–ด์ง€๊ณ  ์ฉ์—ˆ๋‹ค. run ๊ณผ ๊ฐ™์€ ํŒŒ์ผ์— ๋‘๋ฉด ๋Š์–ด์งˆ ์ˆ˜๊ฐ€ ์—†๋‹ค. where run ์€ ์ž๋™ ๊ธฐ์ž…(D19).
"note": auto_note(note, plan),
# provenance: ์ด run ์˜ config(ํ”„๋กฌํ”„ํŠธยทํŒŒ์ดํ”„๋ผ์ธ)๊ฐ€ ์–ด๋””์„œ ์™”๋Š”์ง€ (W&B/DVC ์‹ ๋„์žฅ)
"config_source": source,
"git_commit": commit,
# ๋‚ด์šฉ ์ง€๋ฌธ โ€” ์ด๋ฆ„์ด ์•„๋‹ˆ๋ผ ๋‚ด์šฉ์œผ๋กœ ์„ค์ •์„ ์‹๋ณ„ํ•œ๋‹ค. ์ด๋ฆ„๋งŒ ๋‹ค๋ฅธ ์‚ฌ๋ณธ,
# `extends` ๋กœ ์กฐ๋ฆฝํ•œ ๊ฒƒ, `--set` ์œผ๋กœ ์ฆ‰์„์—์„œ ๋งŒ๋“  ๊ฒƒ์ด ๋ชจ๋‘ ๊ฐ™์€ ๊ฐ’์ด๋ฉด
# ๊ฐ™์€ ์ง€๋ฌธ์„ ๊ฐ–๋Š”๋‹ค (core/fingerprint.py). legacy ๊ฐ’ ๊ทœ์น™ ๊ทธ๋Œ€๋กœ(D12) โ€” ์ƒˆ ์ง€๋ฌธ์€ snapshot ์—.
"prompt_sha": prompt_fingerprint(plan.prompt_path),
"pipeline_sha": pipeline_fingerprint(plan.pipeline_config),
# ํŒŒ์ผ ์—†์ด ๋Œ๋ฆฐ ์‹คํ—˜์ด๋ฉด ๋ฌด์—‡์„ ๋ฐ”๊ฟจ๋Š”์ง€๊ฐ€ ์—ฌ๊ธฐ ๋‚จ๋Š”๋‹ค.
"pipeline_overrides": plan.pipeline_overrides,
"dataset": dataset,
"snapshot": build_snapshot(cfg, plan, dataset, source=source, commit=commit),
}
def short_fingerprint(value: str | None) -> str:
"""64-hex ์ง€๋ฌธ์„ `sha256:<12>` ๋กœ โ€” legacy `prompt_sha` ์™€ ๊ฐ™์€ ๋ชจ์–‘์œผ๋กœ ๋ˆˆ ๋Œ€์กฐ."""
if not value:
return "-"
return f"sha256:{value[:FINGERPRINT_DISPLAY_CHARS]}"