Spaces:
Running
Running
File size: 6,994 Bytes
3dfc93b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 | """run μμ μ λ°μ΄ν°μ
μ€λΉ -- ν λ² μ½κ³ , κ³ λ₯΄κ³ , ν λ©ν κ·μΉμ κ²μ¦νλ€ (κ³ν D3Β·D13Β·D16).
CLI(`runs start/batch`)λ run λλ ν 리λ₯Ό λ§λ€κΈ° **μ μ** `prepare_dataset` μ λΆλ¬
0ν μ νΒ·λΌλ²¨ λλ½Β·`meta_fields` μ€λ₯λ₯Ό λ¨Όμ ν°λ¨λ¦¬κ³ , κ°μ `PreparedDataset` μ
λ¬λ(`execute_run(prepared=...)`)μ run.json κΈ°λ‘(sha256Β·ν μΒ·μ ν)μ ν¨κ» λκΈ΄λ€.
κ·Έλμ run.json μ sha256 κ³Ό μ€μ λ‘ λ νμ΄ **κ°μ λ°μ΄νΈ**μμ λμ¨λ€.
λ¬λλ `prepared` κ° μμΌλ©΄(μλΉμ€Β·μ νΈμΆμ) κ°μ ν¨μλ‘ μ€μ€λ‘ μ€λΉνλ€.
μ΄ λͺ¨λμ poc-bundle ν¨ν€μ§μ packaging μ κ°μ Έμ€μ§ μλλ€(D18).
"""
from __future__ import annotations
import copy
from dataclasses import dataclass
from typing import Any, Mapping
from solar_eval.core.dataset_loader import DatasetLoader, LoadedJsonl
from solar_eval.core.row_filter import (
RowSelection,
SelectionError,
Where,
combine,
format_indices,
parse_where,
select,
)
class DatasetPrepError(ValueError):
"""νμ€ν¬μ ν λ©νΒ·where μ μΈμ΄ κ·μΉμ λ§μ§ μμ λ -- run μ μμνμ§ μλλ€."""
@dataclass(frozen=True)
class PreparedDataset:
"""run μ΄ λ ν μ§ν©. `selected` λ `(μλ³Έ νμΌμ 0-based μμΉ, ν)` ννλ€μ΄λ€."""
loaded: LoadedJsonl
selected: tuple[tuple[int, dict[str, Any]], ...]
rows_total: int
meta_fields: tuple[str, ...] | None
selection: RowSelection | None = None
@property
def rows_selected(self) -> int:
return len(self.selected)
@property
def sha256(self) -> str:
return self.loaded.sha256
@property
def selected_indices(self) -> frozenset[int]:
return frozenset(idx for idx, _ in self.selected)
def reject_defaults_where(project_config: Mapping[str, Any]) -> None:
"""`defaults.where` λ₯Ό κ±°λΆνλ€ (D1).
`_resolve_defaults` λ ν€ λ¨μ μμ λ³ν©μ΄λΌ `defaults.where` κ° λΌλ²¨ μλ νμ€ν¬μλ
νΌμ Έ κ·Έ run μ΄ μ λΆ μ£½κ³ , νμ€ν¬μ where κ° μμΌλ©΄ ν΅μ§Έλ‘ 무μλλ€("λ³ν©"μ΄ μλλΌ
"λ체"). λ‘λλ dict μ `defaults` ν€κ° λ³ν© λ€μλ λ¨μ μμΌλ―λ‘ κ·Έκ²μ λ³Έλ€.
λ‘λ μμ μ΄ μλλΌ run μμ μ κ²μ¬νλ€ -- ν νλ‘μ νΈμ μλͺ»λ μ μΈμ΄ λ€λ₯Έ
νλ‘μ νΈ λ‘λλ₯Ό λ§μ§ μκ².
"""
defaults = project_config.get("defaults") or {}
if isinstance(defaults, Mapping) and "where" in defaults:
name = project_config.get("name", "?")
raise DatasetPrepError(
f"project {name!r}: defaults.where is not allowed -- "
"declare where on a task or pass --where on the CLI"
)
def _data_keys(task_config: Mapping[str, Any]) -> set[str]:
"""μ
λ ₯Β·μ λ΅μΌλ‘ μ΄λ―Έ μ°λ λ°μ΄ν° ν ν€ -- meta λ‘ μ€λ³΅ 볡μ νμ§ μλλ€."""
keys = set(task_config.get("input_fields") or [])
golden_field = task_config.get("golden_field")
if golden_field:
keys.add(golden_field)
golden_fields = task_config.get("golden_fields") or {}
if isinstance(golden_fields, Mapping):
keys.update(v for v in golden_fields.values() if isinstance(v, str))
return keys
def validate_meta_fields(task_config: Mapping[str, Any]) -> tuple[str, ...] | None:
"""νμ€ν¬ `meta_fields` κ²μ¦ (D13) -- λΉμ΄ μμ§ μμ λ¬Έμμ΄ λ¦¬μ€νΈ, μ€λ³΅Β·μ
λ ₯/μ λ΅ κ²ΉμΉ¨ λΆκ°.
μ μΈμ΄ μμΌλ©΄ `None`(results νμ `meta` ν€λ₯Ό μ°μ§ μλλ€ -- λ€λ₯Έ νλ‘μ νΈμ λ°μ΄νΈ λΆλ³).
"""
raw = task_config.get("meta_fields")
if raw is None:
return None
task = task_config.get("name", "?")
if not isinstance(raw, list) or not raw:
raise DatasetPrepError(f"task {task!r}: meta_fields must be a non-empty list of strings")
if not all(isinstance(f, str) and f.strip() for f in raw):
raise DatasetPrepError(f"task {task!r}: meta_fields values must be non-empty strings")
duplicates = sorted({f for f in raw if raw.count(f) > 1})
if duplicates:
raise DatasetPrepError(f"task {task!r}: meta_fields has duplicates: {duplicates}")
overlap = sorted(set(raw) & _data_keys(task_config))
if overlap:
raise DatasetPrepError(f"task {task!r}: meta_fields overlap input/golden fields: {overlap}")
return tuple(raw)
def row_meta(row: Mapping[str, Any], meta_fields: tuple[str, ...] | None) -> dict[str, Any] | None:
"""λ°μ΄ν° νμμ `meta_fields` ν€λ§ κ³¨λΌ λ³΅μ¬νλ€. νμ μλ ν€λ **μλ΅**νλ€.
κ°μ κΉμ 볡μ¬λ³Έμ΄λ€ -- κ²°κ³Ό νμ΄ λ°μ΄ν°μ
ν κ°μ²΄λ₯Ό 곡μ νμ§ μκ².
"""
if meta_fields is None:
return None
return {key: copy.deepcopy(row[key]) for key in meta_fields if key in row}
def build_selection(
task_config: Mapping[str, Any], cli_where: Where | None = None, limit: int | None = None
) -> RowSelection:
"""νμ€ν¬ `where` μ CLI where(`Where` λλ `None`)λ₯Ό ν©μ³ run μ μ νμ λ§λ λ€.
μ νμ CLI κ° **λͺ
μμ μΌλ‘** λ§λ€μ΄ λ¬λμ λκΈ΄λ€ -- λ¬λλ `task_config["where"]` λ₯Ό
μ€μ€λ‘ μ½μ§ μλλ€(D3).
"""
task_name = task_config.get("name", "?")
task_where = parse_where(task_config.get("where"), origin=f"task {task_name!r}")
return RowSelection(where=combine(task_where, cli_where), limit=limit or None)
def prepare_dataset(
loader: DatasetLoader,
project_config: Mapping[str, Any],
task_config: Mapping[str, Any],
selection: RowSelection | None,
) -> PreparedDataset:
"""λ°μ΄ν°μ
μ ν λ² μ½κ³ μ νμ μ μ©νλ€. 0νΒ·λΌλ²¨ λλ½Β·meta μ€λ₯λ μ¬κΈ°μ ν°μ§λ€."""
meta_fields = validate_meta_fields(task_config)
dataset_config = project_config.get("dataset") or {}
loaded = loader.read_jsonl(
dataset_config.get("repo", ""),
task_config.get("dataset_path", ""),
source=dataset_config.get("source", "huggingface"),
)
selected = select(loaded.records, selection)
return PreparedDataset(
loaded=loaded,
selected=tuple(selected),
rows_total=len(loaded.records),
meta_fields=meta_fields,
selection=selection,
)
def check_resume_indices(prepared: PreparedDataset, completed_indices: set[int]) -> None:
"""μ΄λ―Έ λλ `sample_idx` κ° μ ν λ°μ΄λ©΄ μ€ν¨νλ€ (D3 resume κ°λ).
μ ν λ° κ²°κ³Όκ° μμΈ results λ μ΄ run μ μ νμΌλ‘ μ€λͺ
λμ§ μλλ€ -- μ νμ΄λ
λ°μ΄ν°μ
μ΄ run μμ λ€ λ°λμλ€λ λ»μ΄λ€.
"""
outside = sorted(completed_indices - prepared.selected_indices)
if outside:
raise SelectionError(
"completed results are outside this run's selection "
f"({format_indices(outside)}) "
"-- the selection or dataset changed since the run started; start a new run"
)
|