Spaces:
Running
Running
Download solar_eval/cli/commands/run_meta.py from dev-strender/proofread-demo: direct link, hf CLI and curl.
- Browser
- Download file 23.1 kB
-
https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/cli/commands/run_meta.py
- Command line
-
hf download hf://spaces/dev-strender/proofread-demo/solar_eval/cli/commands/run_meta.py
-
curl -L -o run_meta.py https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/cli/commands/run_meta.py
23.1 kB
| """run ์์ยทresume ์ด ๊ณต์ ํ๋ ์ค๋น ๋จ๊ณ์ run.json ๋ฉํ ์กฐ๋ฆฝ (๊ณํ 2026-09-24-phase1a D3ยทD5ยทD12ยทD17ยทD19). | |
| `runs start` ๋ run ๋๋ ํ ๋ฆฌ๋ฅผ ๋ง๋ค๊ธฐ **์ ์** ์ด ๋ชจ๋๋ก ๋ชจ๋ ๊ฒ์ ์ ํ๋ค: | |
| ํ์คํฌยทํ๋กฌํํธยทํ์ดํ๋ผ์ธ ํด์ โ ํ ์ ํ(`where`ยท`limit`) โ ๋ฐ์ดํฐ์ ํ ๋ฒ ์ฝ๊ธฐ โ ํ์ดํ๋ผ์ธ ์์ฑ โ | |
| ์ค๋ ์ท(run.json `dataset`ยท`snapshot` ๋ธ๋ก, ์๊ธฐ ๊ฒ์ฆ๊น์ง). ๊ทธ๋์ 0ํ ์ ํยท๋ผ๋ฒจ ๋๋ฝยท์๋ ์ฌ์ ยท๊นจ์ง lock | |
| ๊ฐ์ ์ค๋ฅ๊ฐ ๋น run ๋๋ ํ ๋ฆฌ๋ฅผ ๋จ๊ธฐ์ง ์๊ณ , run.json ์ด ์ ์ sha256ยท์ค๋ ์ท๊ณผ ๋ฌ๋๊ฐ ์ค์ ๋ก ๋๋ | |
| ๋ฐ์ดํฐยทํ์ดํ๋ผ์ธ์ด **๊ฐ์ ๊ฐ์ฒด**์์ ๋์จ๋ค. | |
| resume ์ ๊ฐ์ ์กฐ๋ฆฝ ํจ์(`build_snapshot`)๋ก ์ง๊ธ ๋์คํฌ ๊ธฐ์ค ์ค๋ ์ท์ ๋ค์ ๋ ์ค์ ๋๋ฆฌํํธ๋ฅผ ๋ง๋๋ค(D17). | |
| `poc_bundle` import ๋ ์ด ๋ชจ๋(๊ณผ `run_resume`ยท`run_eval`)๊ณผ `core/snapshot/**` ์์๋ง ํ๋ค(D18). | |
| """ | |
| from __future__ import annotations | |
| import subprocess | |
| from collections.abc import Callable, Mapping, Sequence | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| from typing import Any | |
| import click | |
| from poc_bundle.data.errors import DataError | |
| from solar_eval.core.dataset_loader import DatasetLoader, LoadedJsonl | |
| from solar_eval.core.dataset_prep import ( | |
| PreparedDataset, | |
| build_selection, | |
| prepare_dataset, | |
| reject_defaults_where, | |
| ) | |
| from solar_eval.core.pipeline_factory import build_task_pipeline | |
| from solar_eval.core.row_filter import RowSelection, parse_cli_where | |
| from solar_eval.core.snapshot import EngineSnapshotInputs, build_engine_snapshot, self_check | |
| from solar_eval.core.snapshot import environment | |
| from solar_eval.core.snapshot.dataset_block import describe_dataset | |
| from solar_eval.evaluators.judge_config import JudgeConfigError, validate_judge_config | |
| #: run ์ ์์ํ๊ธฐ ์ ์ ์ฌ๋์ด ์ฝ์ ๋ฉ์์ง๋ก ๋ฐ๊ฟ ๋ฉ์ถ๋ ์ค๋ฅ โ ํ ์ ํยทmeta_fieldsยทdefaults.where | |
| #: (ValueError ๊ณ์ด), ์๋ ๋ฐ์ดํฐ์ ยท์ง์ ์์ฐ(FileNotFoundError ๊ณ์ด), ๊นจ์ง datasets.lock(DataError). | |
| PREP_ERRORS: tuple[type[Exception], ...] = (ValueError, FileNotFoundError, DataError) | |
| #: ์ง๋ฌธ์ ์ฌ๋์ด ๋์ผ๋ก ๋์กฐํ ๋ ๋ณด์ฌ ์ฃผ๋ ์์๋ฆฌ ์ โ legacy `prompt_sha` ํ์๊ณผ ๋ง์ถ๋ค. | |
| FINGERPRINT_DISPLAY_CHARS = 12 | |
| def resolve_prompt(prompts_dir: Path, prompt_name: str) -> Path: | |
| """Resolve prompt from prompts/ directory (directory or YAML file). | |
| Resolution order: | |
| 1. prompts/{name}/ (directory TXT format) | |
| 2. prompts/{name}.yaml (flat YAML layout) | |
| 3. prompts/_library/{name}.yaml (legacy layout) | |
| """ | |
| dir_path = prompts_dir / prompt_name | |
| if dir_path.is_dir(): | |
| return dir_path | |
| yaml_path = prompts_dir / f"{prompt_name}.yaml" | |
| if yaml_path.exists(): | |
| return yaml_path | |
| legacy_path = prompts_dir / "_library" / f"{prompt_name}.yaml" | |
| if legacy_path.exists(): | |
| return legacy_path | |
| raise click.ClickException( | |
| f"Prompt '{prompt_name}' not found. Searched:\n" | |
| f" {dir_path}/ (directory)\n" | |
| f" {yaml_path} (YAML)\n" | |
| f" {legacy_path} (legacy)" | |
| ) | |
| def resolve_pipeline_file(project_dir: Path, pipeline_name: str) -> Path: | |
| """Resolve pipeline YAML from pipelines/ directory.""" | |
| path = project_dir / "pipelines" / f"{pipeline_name}.yaml" | |
| if not path.exists(): | |
| raise click.ClickException(f"Pipeline '{pipeline_name}' not found: {path}") | |
| return path | |
| def compose_pipeline_config(project_dir: Path, pipeline_name: str, overrides: dict) -> dict: | |
| """ํ์ดํ๋ผ์ธ์ `extends`/override ๊น์ง ํด์ํด ๋๋ ค์ค๋ค. | |
| ์กฐ๋ฆฝ ์คํจ(์๋ base, ์๋ ์คํ ์ ๊ฐ๋ฆฌํค๋ override ๋ฑ)๋ run ์ ์์ํ๊ธฐ ์ ์ | |
| ์ฃฝ์ธ๋ค โ ํจ๊ณผ ์๋ override ๋ก ๋์๊ฐ run ์ ๊ฒฐ๋ก ์ ์กฐ์ฉํ ์ค์ผ์ํจ๋ค. | |
| """ | |
| from solar_eval.core.pipeline_compose import PipelineCompositionError, compose_pipeline | |
| from solar_eval.core.project_loader import load_pipeline_file | |
| pipelines_dir = resolve_pipeline_file(project_dir, pipeline_name).parent | |
| try: | |
| composed = load_pipeline_file(pipelines_dir, pipeline_name) | |
| if overrides: | |
| composed = compose_pipeline(composed, overrides=overrides) | |
| return composed | |
| except (PipelineCompositionError, FileNotFoundError) as e: | |
| raise click.ClickException(str(e)) from None | |
| def config_source(cfg, project: str) -> str: | |
| """run ์ config ์ถ์ฒ โ ๋ ํฌ ๊ด๋ฆฌ ํ๋ก์ ํธ๋ฉด ๋ ํฌ ์๋ ๊ฒฝ๋ก, ์๋๋ฉด ๋ฐ์ดํฐ ํ ์ ๋ ๊ฒฝ๋ก.""" | |
| config_dir = cfg.config_dir(project) | |
| if cfg.is_repo_managed(project) and cfg.repo_root is not None: | |
| return str(config_dir.relative_to(cfg.repo_root)) | |
| return str(config_dir) | |
| def git_commit(repo_root: Path | None) -> str | None: | |
| """์ฌํ์ฉ ๋์ฅ: ๋ ํฌ HEAD ์ปค๋ฐ. ์์ ํธ๋ฆฌ๊ฐ ๋๋ฌ์ฐ๋ฉด '-dirty' ๋ฅผ ๋ถ์ธ๋ค (๋ ํฌ ์ ์ฒด โ D8 ๋ถ๋ณ).""" | |
| if repo_root is None: | |
| return None | |
| try: | |
| head = subprocess.run( | |
| ["git", "-C", str(repo_root), "rev-parse", "--short", "HEAD"], | |
| capture_output=True, | |
| text=True, | |
| check=True, | |
| ).stdout.strip() | |
| status = subprocess.run( | |
| ["git", "-C", str(repo_root), "status", "--porcelain"], | |
| capture_output=True, | |
| text=True, | |
| check=True, | |
| ).stdout.strip() | |
| return f"{head}-dirty" if status else head | |
| except (OSError, subprocess.CalledProcessError): | |
| return None | |
| def find_task(cfg, project: str, task: str) -> tuple[dict[str, Any], dict[str, Any]]: | |
| """๋ก๋๋ ํ๋ก์ ํธ ์ค์ ๊ณผ ํ์คํฌ ์ค์ . ์์ผ๋ฉด ClickException.""" | |
| from solar_eval.core.project_loader import load_all_project_configs | |
| configs = load_all_project_configs(cfg.projects_dir, cfg.config_dirs) | |
| project_config = next((c for c in configs if c["name"] == project), None) | |
| if not project_config: | |
| raise click.ClickException(f"Project not found in {cfg.projects_dir}: {project}") | |
| task_config = next((t for t in project_config.get("tasks", []) if t["name"] == task), None) | |
| if not task_config: | |
| raise click.ClickException(f"Task '{task}' not found in project '{project}'") | |
| return project_config, task_config | |
| def load_run(cfg, project: str, run_id: str) -> tuple[Path, Any, dict[str, Any]]: | |
| """(run ๋๋ ํ ๋ฆฌ, ์คํ ์ด, run.json). ์์ผ๋ฉด ClickException.""" | |
| from solar_eval.stores import JsonlStore | |
| run_dir = cfg.artifacts_dir(project) / run_id | |
| if not run_dir.exists(): | |
| raise click.ClickException(f"Run directory not found: {run_dir}") | |
| store = JsonlStore(run_dir) | |
| run_meta = store.load_run_meta() | |
| if not run_meta: | |
| raise click.ClickException(f"No run.json found in {run_dir}") | |
| if not run_meta.get("task"): | |
| raise click.ClickException("Cannot determine task from run.json") | |
| return run_dir, store, run_meta | |
| def load_prompt( | |
| prompt_path: Path, | |
| model: str, | |
| temperature: float, | |
| max_tokens: int, | |
| reasoning_effort: str | None, | |
| ) -> tuple[dict[str, Any], str]: | |
| """ํ๋กฌํํธ ํ์ผ์ ๋ฌ๋๊ฐ ๋ฐ๋ dict ๋ก โ (prompt, ํฌ๋งท `multi_step`|`single_step`).""" | |
| from solar_eval.models.prompt_version import ( | |
| detect_prompt_format, | |
| load_prompt_messages, | |
| load_step_prompts, | |
| ) | |
| fmt = detect_prompt_format(prompt_path) | |
| if fmt == "multi_step": | |
| prompt: dict[str, Any] = {"step_prompts": load_step_prompts(prompt_path)} | |
| else: | |
| prompt = {"messages": load_prompt_messages(prompt_path)} | |
| prompt.update({"model": model, "temperature": temperature, "max_tokens": max_tokens}) | |
| if reasoning_effort: | |
| prompt["reasoning_effort"] = reasoning_effort | |
| return prompt, fmt | |
| def make_providers(evaluator_config: Mapping[str, Any] | None = None) -> tuple[Any, Any]: | |
| """(์ถ๋ก ํ๋ก๋ฐ์ด๋, judge ํ๋ก๋ฐ์ด๋). ํ ์คํธ๋ ์ด ํจ์๋ฅผ ๋ฐ๊ฟ ๋ผ์ ๋คํธ์ํฌ ์์ด ๋๋ค. | |
| `evaluator_config` ๋ ํ์คํฌ์ ํ๊ฐ๊ธฐ ์ค์ (ํค๊ฐ ์์ผ๋ฉด None) โ judge ์ธ์คํด์ค๋ฅผ ์ ์ธ(`judge_provider`)์ผ๋ก ๊ณ ๋ฅด๋ | |
| ์๋ฆฌ๋ค(๊ณํ 2026-09-28 L12 ยง2.8 ยท D-L12-10). ์ ์ธ์ด ์๊ฑฐ๋ ํ์ ์๋ ํ๊ฐ๊ธฐ๋ ์ค๋์ฒ๋ผ `OpenAIProvider()` โ | |
| run.json ์ค๋ ์ท `provenance.providers.judge` ๋ฐ์ดํธ๊ฐ ๋ฐ๋์ง ์๋๋ค. ์ถ๋ก ํ๋ก๋ฐ์ด๋๋ ๋ `UpstageProvider()`. | |
| Raises: | |
| JudgeConfigError: ๋ชจ๋ฅด๋ providerยทํฉ์ฑ ํ๊ฐ๊ธฐ ์์ provider ์ถฉ๋. | |
| """ | |
| from solar_eval.evaluators.judge_config import ( | |
| PROVIDER_OPENAI, | |
| PROVIDER_UPSTAGE, | |
| judge_provider_name, | |
| ) | |
| from solar_eval.providers import UpstageProvider | |
| from solar_eval.providers.openai_provider import OpenAIProvider | |
| judge_classes = {PROVIDER_UPSTAGE: UpstageProvider, PROVIDER_OPENAI: OpenAIProvider} | |
| name = judge_provider_name(evaluator_config) or PROVIDER_OPENAI | |
| return UpstageProvider(), judge_classes[name]() | |
| def route_inference_provider(provider: Any, step_models: Sequence[str]) -> Any: | |
| """์คํ ๋ชจ๋ธ์ ๋ผ์ฐํธ ์ ๋์ด(`timely:<๋ชจ๋ธ>`)๊ฐ ํ๋๋ผ๋ ์์ผ๋ฉด ์ถ๋ก ์ ๊ณต์๋ฅผ `RoutedProvider` ๋ก ๊ฐ์ผ๋ค. | |
| ์์ผ๋ฉด ๊ทธ๋๋ก ๋๋ ค์ค๋ค โ ๋ผ์ฐํธ ์๋ run ์ provenance(`providers.inference` ๋ชจ๋)๊ฐ ์์ ๋ฐ์ดํธ ๊ทธ๋๋ก์ฌ์ผ ํ๋ค. | |
| ์ธ๋ถ ๋ผ์ฐํธ๋ฅผ ์จ๋ ๋๋๊ฐ๋ ์์ง์ด ์๋๋ผ case ์ ๋ฐ์ดํฐ ์ ์ฑ (๊ณ์ฝ)์ด ์ ํ๋ค โ 2026-10 judge egress ์ ์ฑ ํ์ง(#353)์ ๊ฐ์ ์๋ฆฌ. | |
| """ | |
| from solar_eval.providers import INFERENCE_ROUTES, RoutedProvider | |
| from solar_eval.providers.routed import route_prefixes | |
| if not route_prefixes(list(step_models), INFERENCE_ROUTES): | |
| return provider | |
| return RoutedProvider(provider, {k: cls() for k, cls in INFERENCE_ROUTES.items()}) | |
| class RunPlan: | |
| """run ํ๋๊ฐ ๋ฌด์์ผ๋ก ๋๋๊ฐ โ run ๋๋ ํ ๋ฆฌ๋ฅผ ๋ง๋ค๊ธฐ ์ ์ ์ ๋ถ ์ ํด์ง๋ค.""" | |
| project: str | |
| task: str | |
| project_config: dict[str, Any] | |
| task_config: dict[str, Any] | |
| prompt_name: str | |
| prompt_path: Path | |
| prompt_format: str | |
| prompt: dict[str, Any] | |
| model: str | |
| temperature: float | |
| max_tokens: int | |
| reasoning_effort: str | None | |
| pipeline_name: str | |
| pipeline_config: dict[str, Any] | |
| pipeline_overrides: dict[str, Any] | None | |
| max_workers: int | |
| dataset_loader: DatasetLoader | |
| prepared: PreparedDataset | |
| pipeline: Any | |
| provider: Any | |
| judge_provider: Any | |
| def selection(self) -> RowSelection | None: | |
| return self.prepared.selection | |
| def resolve_pipeline( | |
| cfg, project: str, task_config: dict[str, Any], pipeline_name: str | None, overrides: dict | |
| ) -> tuple[dict[str, Any], str, dict[str, Any]]: | |
| """(ํ์ดํ๋ผ์ธ์ ์น์ task_config, ์คํจ ํ์ดํ๋ผ์ธ ์ด๋ฆ, ํด์๋ ํ์ดํ๋ผ์ธ dict). | |
| `-P` ๊ฐ ์์ผ๋ฉด config ์ ๋ณธ์ pipelines/ ์์ `extends`ยท`--set` ๊น์ง ํด์ํ๊ณ , ์์ผ๋ฉด | |
| project.yaml ๊ธฐ๋ณธ๊ฐ(project_loader ๊ฐ ์ด๋ฏธ ํด์)์ ์ด๋ค. config ๋ ๋ ํฌ ๊ด๋ฆฌ ํ๋ก์ ํธ๋ฉด git | |
| ์ชฝ์ด ์ ๋ณธ์ด๋ค. `extends` ์์๊ณผ `--set` override ๋ ์ฌ๊ธฐ์ ์ ๋ถ ํด์๋ผ, ์คํ ์์ง์๋ ์์ฑ๋ | |
| steps ๋ชฉ๋ก๋ง ๋์ด๊ฐ๋ค. | |
| """ | |
| if pipeline_name: | |
| pipeline_config = compose_pipeline_config(cfg.config_dir(project), pipeline_name, overrides) | |
| return {**task_config, "pipeline_config": pipeline_config}, pipeline_name, pipeline_config | |
| if overrides: | |
| raise click.ClickException("--set requires an explicit --pipeline/-P") | |
| pc = task_config.get("pipeline_config", {}) | |
| effective = pc.get("name", "default") if isinstance(pc, dict) else str(pc) | |
| return task_config, effective, pc if isinstance(pc, dict) else {} | |
| class _CheckedReader: | |
| """`prepare_dataset` ์ ๋๊ธฐ๋ ์์ ๋๋ฆฌ์ โ ํ์ผ์ ์ฝ์ **์งํ**, ํ์ ๊ณ ๋ฅด๊ธฐ **์ ์** `on_read` ๋ฅผ ๋ถ๋ฅธ๋ค. | |
| resume ์ ๋ฐ์ดํฐ์ sha ๊ฐ๋(D17 ์์์ ์ฒซ์งธ)๊ฐ ์ ํยท๋ผ๋ฒจ ๊ฒ์ฌ๋ณด๋ค ๋จผ์ ๊ฑธ๋ฆฌ๊ฒ ํ๋ ค๋ ๊ฒ์ด๋ค โ | |
| ๋ฐ์ดํฐ์ ์ด ๋ฐ๋์ด ์ ํ์ด 0ํ์ด ๋๋ฉด ์์ธ์ "where ๊ฐ 0ํ" ์ด ์๋๋ผ "๋ฐ์ดํฐ์ ์ด ๋ฐ๋์๋ค" ๋ค. | |
| ๋ฐ์ดํธ๋ ์ฌ์ ํ ํ ๋ฒ๋ง ์ฝ๋๋ค(D16). | |
| """ | |
| def __init__(self, loader: DatasetLoader, on_read: Callable[[LoadedJsonl], None]) -> None: | |
| self._loader = loader | |
| self._on_read = on_read | |
| def read_jsonl(self, repo: str, path: str, source: str) -> LoadedJsonl: | |
| loaded = self._loader.read_jsonl(repo, path, source=source) | |
| self._on_read(loaded) | |
| return loaded | |
| def _step_models(pipeline: Any) -> list[str]: | |
| """ํ์ดํ๋ผ์ธ ์คํ ๋ค์ด ๋ช ์ํ ๋ชจ๋ธ ๋ฌธ์์ด โ ๋ผ์ฐํธ ์ ๋์ด ๊ฒ์ฌ์ฉ(์๋ ์คํ ์ run ์ `-m` ๋ชจ๋ธ = Upstage).""" | |
| return [ | |
| str(step.config["model"]) | |
| for step in getattr(pipeline, "steps", []) | |
| if isinstance(getattr(step, "config", None), dict) and step.config.get("model") | |
| ] | |
| def assemble_plan( | |
| cfg, | |
| selection: RowSelection | None, | |
| *, | |
| on_read: Callable[[LoadedJsonl], None] | None = None, | |
| **fields: Any, | |
| ) -> RunPlan: | |
| """๋ฐ์ดํฐ์ ์ ํ ๋ฒ ์ฝ์ด ๊ณ ๋ฅด๊ณ , ๋ฌ๋๊ฐ ๋ ํ์ดํ๋ผ์ธ ์ธ์คํด์ค๋ฅผ ๋ง๋ ๋ค(D6ยทD16). | |
| `fields` ๋ `RunPlan` ์ ๋๋จธ์ง ํ๋(projectยทtask_configยทprompt ๋ฑ)๋ค. `on_read` ๋ ๋ฐ์ดํฐ์ ์ | |
| ์ฝ์ ์งํยท์ ํ ์ ์ ๋ถ๋ฆฐ๋ค(resume ์ sha ๊ฐ๋ โ ๊ฑฐ๊ธฐ์ ๋ ClickException ์ ๊ทธ๋๋ก ์ฌ๋ผ๊ฐ๋ค). | |
| judge ์ค์ ์ค๋ฅ๋ ๋ฐ์ดํฐ์ ์ฝ๊ธฐยทํ์ดํ๋ผ์ธยทprovider ์์ฑ๋ณด๋ค ๋จผ์ ํ์ธํ๋ค. | |
| `runs start`ยท`quick-run`ยท`runs batch`(ํ๋กฌํํธ๋ง๋ค)ยท`runs resume` ์ด ๋ชจ๋ ์ด ์๋ฆฌ๋ฅผ ์ง๋๋ค. | |
| Raises: | |
| click.ClickException: judge ์ค์ ยท์ ํยทmeta_fieldsยท๋ฐ์ดํฐ์ ยทํ์ดํ๋ผ์ธ ์์ฑ ์ค๋ฅ(run ์์ ์ ). | |
| """ | |
| preflight_judge_config(fields["task_config"]) | |
| loader = DatasetLoader(base_dir=cfg.projects_dir) | |
| reader: Any = loader if on_read is None else _CheckedReader(loader, on_read) | |
| try: | |
| prepared = prepare_dataset( | |
| reader, fields["project_config"], fields["task_config"], selection | |
| ) | |
| pipeline = build_task_pipeline( | |
| fields["project_config"], fields["task_config"], fields["prompt"], loader | |
| ) | |
| except PREP_ERRORS as e: | |
| raise click.ClickException(str(e)) from None | |
| provider, judge_provider = make_providers(fields["task_config"].get("evaluator")) | |
| provider = route_inference_provider(provider, _step_models(pipeline)) | |
| return RunPlan( | |
| dataset_loader=loader, | |
| prepared=prepared, | |
| pipeline=pipeline, | |
| provider=provider, | |
| judge_provider=judge_provider, | |
| **fields, | |
| ) | |
| def preflight_judge_config(task_config: Mapping[str, Any]) -> None: | |
| """Judge ์ค์ ์ค๋ฅ๋ฅผ CLI ์ค๋ฅ๋ก ๋ฐ๊พธ๋ provider ๋ ๋ง๋ค์ง ์๋๋ค.""" | |
| try: | |
| validate_judge_config(task_config) | |
| except JudgeConfigError as exc: | |
| raise click.ClickException(str(exc)) from None | |
| def start_options( | |
| project_config: dict[str, Any], | |
| task_config: dict[str, Any], | |
| set_options: tuple[str, ...], | |
| where: tuple[str, ...], | |
| limit: int | None, | |
| ) -> tuple[dict[str, Any], RowSelection]: | |
| """(`--set` override, ํ ์ ํ) โ ํ๋กฌํํธ์ ๋ฌด๊ดํ ์์ ๊ฒ์ฌ. `runs batch` ๋ ๋ฃจํ ์ ์ ํ ๋ฒ ๋ถ๋ฅธ๋ค. | |
| Raises: | |
| click.ClickException: `defaults.where`ยทwhere ๋ฌธ๋ฒยท`--set` ๋ฌธ๋ฒ ์ค๋ฅ. | |
| """ | |
| from solar_eval.core.pipeline_compose import PipelineCompositionError, parse_set_option | |
| try: | |
| reject_defaults_where(project_config) | |
| overrides = parse_set_option(set_options) | |
| selection = build_selection(task_config, parse_cli_where(where), limit) | |
| except (PipelineCompositionError, *PREP_ERRORS) as e: | |
| raise click.ClickException(str(e)) from None | |
| return overrides, selection | |
| def plan_run( | |
| cfg, | |
| *, | |
| project: str, | |
| task: str, | |
| prompt_name: str, | |
| model: str, | |
| pipeline_name: str | None, | |
| reasoning_effort: str | None, | |
| temperature: float, | |
| max_tokens: int, | |
| max_workers: int, | |
| limit: int | None, | |
| set_options: tuple[str, ...] = (), | |
| where: tuple[str, ...] = (), | |
| ) -> RunPlan: | |
| """`runs start` ์ ์ค๋น ๋จ๊ณ ์ ๋ถ โ `reject_defaults_where` โ ์ ํ โ `prepare_dataset` โ | |
| `build_task_pipeline` (๊ณํ ยง9 G2 ์์). ๋ชจ๋ ์ค๋ฅ๋ run ๋๋ ํ ๋ฆฌ๋ฅผ ๋ง๋ค๊ธฐ ์ ์ ๋๋ค. | |
| """ | |
| project_config, task_config = find_task(cfg, project, task) | |
| overrides, selection = start_options(project_config, task_config, set_options, where, limit) | |
| prompt_path = resolve_prompt(cfg.prompts_dir(project), prompt_name) | |
| prompt, fmt = load_prompt(prompt_path, model, temperature, max_tokens, reasoning_effort) | |
| task_config, effective, pipeline_config = resolve_pipeline( | |
| cfg, project, task_config, pipeline_name, overrides | |
| ) | |
| return assemble_plan( | |
| cfg, | |
| selection, | |
| project=project, | |
| task=task, | |
| project_config=project_config, | |
| task_config=task_config, | |
| prompt_name=prompt_name, | |
| prompt_path=prompt_path, | |
| prompt_format=fmt, | |
| prompt=prompt, | |
| model=model, | |
| temperature=temperature, | |
| max_tokens=max_tokens, | |
| reasoning_effort=reasoning_effort, | |
| pipeline_name=effective, | |
| pipeline_config=pipeline_config, | |
| pipeline_overrides=overrides or None, | |
| max_workers=max_workers, | |
| ) | |
| def dataset_block(cfg, plan: RunPlan) -> dict[str, Any]: | |
| """run.json `dataset` ๋ธ๋ก โ ์ฝ์ ๋ฐ์ดํธ์ sha256ยทํ ์ยท์ ๊ทํ๋ ์ ํยทlock ๋์กฐ(S6).""" | |
| prepared = plan.prepared | |
| selection = plan.selection or RowSelection() | |
| try: | |
| return describe_dataset( | |
| sha256=prepared.sha256, | |
| rows_total=prepared.rows_total, | |
| rows_selected=prepared.rows_selected, | |
| selection=selection.to_json(), | |
| meta_fields=prepared.meta_fields, | |
| task_config=plan.task_config, | |
| loader_cfg=plan.project_config.get("dataset") or {}, | |
| config_dir=cfg.config_dir(plan.project), | |
| project_name=plan.project, | |
| repo_root=environment.engine_repo(), | |
| ) | |
| except PREP_ERRORS as e: | |
| raise click.ClickException(f"dataset lock check failed: {e}") from None | |
| def build_snapshot( | |
| cfg, | |
| plan: RunPlan, | |
| dataset: dict[str, Any] | None, | |
| *, | |
| source: str | None, | |
| commit: str | None, | |
| ) -> dict[str, Any]: | |
| """run.json `snapshot` ๋ธ๋ก(engine-snapshot/v1). start ์ resume ์ด ๊ฐ์ ํจ์๋ฅผ ์ด๋ค(D17). | |
| Raises: | |
| click.ClickException: ์ ์ธํ ์์์ด ์๊ฑฐ๋ ์ค๋ ์ท์ด ์๊ธฐ ๊ฒ์ฆ(D5)์ ํต๊ณผํ์ง ๋ชปํ ๋. | |
| """ | |
| inputs = EngineSnapshotInputs( | |
| model=plan.model, | |
| temperature=plan.temperature, | |
| max_tokens=plan.max_tokens, | |
| reasoning_effort=plan.reasoning_effort, | |
| pipeline_type=plan.task_config.get("pipeline", "single_step"), | |
| input_fields=plan.task_config.get("input_fields", []), | |
| resolved_pipeline=plan.pipeline_config, | |
| prompt_path=plan.prompt_path, | |
| pipeline=plan.pipeline, | |
| evaluator_config=plan.task_config.get("evaluator"), | |
| inference_provider=plan.provider, | |
| judge_provider=plan.judge_provider, | |
| prompt_name=plan.prompt_name, | |
| pipeline_name=plan.pipeline_name, | |
| config_source=source, | |
| git_commit=commit, | |
| pipeline_overrides=plan.pipeline_overrides, | |
| max_workers=plan.max_workers, | |
| config_dir=cfg.config_dir(plan.project), | |
| repo_managed=cfg.is_repo_managed(plan.project), | |
| dataset=dataset, | |
| ) | |
| try: | |
| snapshot = build_engine_snapshot(inputs) | |
| self_check(snapshot) | |
| except PREP_ERRORS as e: | |
| raise click.ClickException(f"cannot snapshot this run's config: {e}") from None | |
| return snapshot | |
| def auto_note(note: str | None, plan: RunPlan) -> str | None: | |
| """`--where` run ์ note ์๋ ๊ธฐ์ (D19) โ Experiment Hub ๋ชฉ๋ก์์ ์ ์ฒด์ ๋ฐ๋ณต๊ณผ ์ฌ๋์ด ๊ตฌ๋ถํ๊ฒ. | |
| note ๊ฐ ๋น๋ฉด `where tags=yul-ryul ยท 12/159 rows`, ์ฌ์ฉ์๊ฐ ์ฃผ๋ฉด ๊ทธ ๋ค์ ` [where โฆ]`. | |
| where ๊ฐ ์๋ run ์ note ๋ฅผ ๊ทธ๋๋ก ๋๋ค(๋ค๋ฅธ run ์ run.json ๋ฐ์ดํธ ๋ถ๋ณ). | |
| """ | |
| selection = plan.selection | |
| if selection is None or selection.where is None: | |
| return note | |
| prepared = plan.prepared | |
| label = ( | |
| f"where {selection.where.describe()} ยท {prepared.rows_selected}/{prepared.rows_total} rows" | |
| ) | |
| return f"{note} [{label}]" if note else label | |
| def build_run_meta(cfg, plan: RunPlan, note: str | None) -> dict[str, Any]: | |
| """run.json ์ด๊ธฐ ๋ด์ฉ โ legacy ํค(๊ฐ ๊ท์น ๋ถ๋ณ, D12) ๋ค์ `dataset`ยท`snapshot`(๊ณํ ยง3.2 ์์).""" | |
| from solar_eval.core.fingerprint import pipeline_fingerprint, prompt_fingerprint | |
| source = config_source(cfg, plan.project) | |
| commit = git_commit(cfg.repo_root) if cfg.is_repo_managed(plan.project) else None | |
| dataset = dataset_block(cfg, plan) | |
| return { | |
| "task": plan.task, | |
| "model": plan.model, | |
| "prompt": plan.prompt_name, | |
| "pipeline": plan.pipeline_name, | |
| "temperature": plan.temperature, | |
| "max_tokens": plan.max_tokens, | |
| "reasoning_effort": plan.reasoning_effort, | |
| # ์ ๋๋ ธ๋์ง โ ์์ ์๋ ๋ณ๋ ๋ฉ๋ ธํธ(experiments/*.yaml)๊ฐ ๋ด์์ง๋ง run ๊ณผ ๋ถ๋ฆฌ๋ผ ์์ด | |
| # ์ฐธ์กฐ๊ฐ ๋์ด์ง๊ณ ์ฉ์๋ค. run ๊ณผ ๊ฐ์ ํ์ผ์ ๋๋ฉด ๋์ด์ง ์๊ฐ ์๋ค. where run ์ ์๋ ๊ธฐ์ (D19). | |
| "note": auto_note(note, plan), | |
| # provenance: ์ด run ์ config(ํ๋กฌํํธยทํ์ดํ๋ผ์ธ)๊ฐ ์ด๋์ ์๋์ง (W&B/DVC ์ ๋์ฅ) | |
| "config_source": source, | |
| "git_commit": commit, | |
| # ๋ด์ฉ ์ง๋ฌธ โ ์ด๋ฆ์ด ์๋๋ผ ๋ด์ฉ์ผ๋ก ์ค์ ์ ์๋ณํ๋ค. ์ด๋ฆ๋ง ๋ค๋ฅธ ์ฌ๋ณธ, | |
| # `extends` ๋ก ์กฐ๋ฆฝํ ๊ฒ, `--set` ์ผ๋ก ์ฆ์์์ ๋ง๋ ๊ฒ์ด ๋ชจ๋ ๊ฐ์ ๊ฐ์ด๋ฉด | |
| # ๊ฐ์ ์ง๋ฌธ์ ๊ฐ๋๋ค (core/fingerprint.py). legacy ๊ฐ ๊ท์น ๊ทธ๋๋ก(D12) โ ์ ์ง๋ฌธ์ snapshot ์. | |
| "prompt_sha": prompt_fingerprint(plan.prompt_path), | |
| "pipeline_sha": pipeline_fingerprint(plan.pipeline_config), | |
| # ํ์ผ ์์ด ๋๋ฆฐ ์คํ์ด๋ฉด ๋ฌด์์ ๋ฐ๊ฟจ๋์ง๊ฐ ์ฌ๊ธฐ ๋จ๋๋ค. | |
| "pipeline_overrides": plan.pipeline_overrides, | |
| "dataset": dataset, | |
| "snapshot": build_snapshot(cfg, plan, dataset, source=source, commit=commit), | |
| } | |
| def short_fingerprint(value: str | None) -> str: | |
| """64-hex ์ง๋ฌธ์ `sha256:<12>` ๋ก โ legacy `prompt_sha` ์ ๊ฐ์ ๋ชจ์์ผ๋ก ๋ ๋์กฐ.""" | |
| if not value: | |
| return "-" | |
| return f"sha256:{value[:FINGERPRINT_DISPLAY_CHARS]}" | |