proofread-demo / solar_eval /core /dataset_prep.py
dev-strender's picture
feat: switch demo to FT + Opus article review (pipeline_exp_261021_ft_hybrid)
3dfc93b verified
Raw History Blame Contribute Delete
6.99 kB
"""run μ‹œμž‘ μ „ 데이터셋 μ€€λΉ„ -- ν•œ 번 읽고, κ³ λ₯΄κ³ , ν–‰ 메타 κ·œμΉ™μ„ κ²€μ¦ν•œλ‹€ (κ³„νš D3Β·D13Β·D16).
CLI(`runs start/batch`)λŠ” run 디렉토리λ₯Ό λ§Œλ“€κΈ° **전에** `prepare_dataset` 을 뢈러
0ν–‰ 선택·라벨 λˆ„λ½Β·`meta_fields` 였λ₯˜λ₯Ό λ¨Όμ € ν„°λœ¨λ¦¬κ³ , 같은 `PreparedDataset` 을
λŸ¬λ„ˆ(`execute_run(prepared=...)`)와 run.json 기둝(sha256Β·ν–‰ μˆ˜Β·μ„ νƒ)에 ν•¨κ»˜ λ„˜κΈ΄λ‹€.
κ·Έλž˜μ„œ run.json 의 sha256 κ³Ό μ‹€μ œλ‘œ 돈 행이 **같은 λ°”μ΄νŠΈ**μ—μ„œ λ‚˜μ˜¨λ‹€.
λŸ¬λ„ˆλŠ” `prepared` κ°€ μ—†μœΌλ©΄(μ„œλΉ„μŠ€Β·μ˜› 호좜자) 같은 ν•¨μˆ˜λ‘œ 슀슀둜 μ€€λΉ„ν•œλ‹€.
이 λͺ¨λ“ˆμ€ poc-bundle νŒ¨ν‚€μ§€μ™€ packaging 을 κ°€μ Έμ˜€μ§€ μ•ŠλŠ”λ‹€(D18).
"""
from __future__ import annotations
import copy
from dataclasses import dataclass
from typing import Any, Mapping
from solar_eval.core.dataset_loader import DatasetLoader, LoadedJsonl
from solar_eval.core.row_filter import (
RowSelection,
SelectionError,
Where,
combine,
format_indices,
parse_where,
select,
)
class DatasetPrepError(ValueError):
"""νƒœμŠ€ν¬μ˜ ν–‰ 메타·where 선언이 κ·œμΉ™μ— λ§žμ§€ μ•Šμ„ λ•Œ -- run 을 μ‹œμž‘ν•˜μ§€ μ•ŠλŠ”λ‹€."""
@dataclass(frozen=True)
class PreparedDataset:
"""run 이 돌 ν–‰ μ§‘ν•©. `selected` λŠ” `(원본 파일의 0-based μœ„μΉ˜, ν–‰)` νŠœν”Œλ“€μ΄λ‹€."""
loaded: LoadedJsonl
selected: tuple[tuple[int, dict[str, Any]], ...]
rows_total: int
meta_fields: tuple[str, ...] | None
selection: RowSelection | None = None
@property
def rows_selected(self) -> int:
return len(self.selected)
@property
def sha256(self) -> str:
return self.loaded.sha256
@property
def selected_indices(self) -> frozenset[int]:
return frozenset(idx for idx, _ in self.selected)
def reject_defaults_where(project_config: Mapping[str, Any]) -> None:
"""`defaults.where` λ₯Ό κ±°λΆ€ν•œλ‹€ (D1).
`_resolve_defaults` λŠ” ν‚€ λ‹¨μœ„ 얕은 병합이라 `defaults.where` κ°€ 라벨 μ—†λŠ” νƒœμŠ€ν¬μ—λ„
퍼져 κ·Έ run 이 μ „λΆ€ μ£½κ³ , νƒœμŠ€ν¬μ— where κ°€ 있으면 ν†΅μ§Έλ‘œ λ¬΄μ‹œλœλ‹€("병합"이 μ•„λ‹ˆλΌ
"λŒ€μ²΄"). λ‘œλ“œλœ dict 에 `defaults` ν‚€κ°€ 병합 뒀에도 남아 μžˆμœΌλ―€λ‘œ 그것을 λ³Έλ‹€.
λ‘œλ“œ μ‹œμ μ΄ μ•„λ‹ˆλΌ run μ‹œμž‘ μ‹œ κ²€μ‚¬ν•œλ‹€ -- ν•œ ν”„λ‘œμ νŠΈμ˜ 잘λͺ»λœ 선언이 λ‹€λ₯Έ
ν”„λ‘œμ νŠΈ λ‘œλ“œλ₯Ό 막지 μ•Šκ²Œ.
"""
defaults = project_config.get("defaults") or {}
if isinstance(defaults, Mapping) and "where" in defaults:
name = project_config.get("name", "?")
raise DatasetPrepError(
f"project {name!r}: defaults.where is not allowed -- "
"declare where on a task or pass --where on the CLI"
)
def _data_keys(task_config: Mapping[str, Any]) -> set[str]:
"""μž…λ ₯Β·μ •λ‹΅μœΌλ‘œ 이미 μ“°λŠ” 데이터 ν–‰ ν‚€ -- meta 둜 쀑볡 λ³΅μ œν•˜μ§€ μ•ŠλŠ”λ‹€."""
keys = set(task_config.get("input_fields") or [])
golden_field = task_config.get("golden_field")
if golden_field:
keys.add(golden_field)
golden_fields = task_config.get("golden_fields") or {}
if isinstance(golden_fields, Mapping):
keys.update(v for v in golden_fields.values() if isinstance(v, str))
return keys
def validate_meta_fields(task_config: Mapping[str, Any]) -> tuple[str, ...] | None:
"""νƒœμŠ€ν¬ `meta_fields` 검증 (D13) -- λΉ„μ–΄ μžˆμ§€ μ•Šμ€ λ¬Έμžμ—΄ 리슀트, μ€‘λ³΅Β·μž…λ ₯/μ •λ‹΅ κ²ΉμΉ¨ λΆˆκ°€.
선언이 μ—†μœΌλ©΄ `None`(results 행에 `meta` ν‚€λ₯Ό μ“°μ§€ μ•ŠλŠ”λ‹€ -- λ‹€λ₯Έ ν”„λ‘œμ νŠΈμ˜ λ°”μ΄νŠΈ λΆˆλ³€).
"""
raw = task_config.get("meta_fields")
if raw is None:
return None
task = task_config.get("name", "?")
if not isinstance(raw, list) or not raw:
raise DatasetPrepError(f"task {task!r}: meta_fields must be a non-empty list of strings")
if not all(isinstance(f, str) and f.strip() for f in raw):
raise DatasetPrepError(f"task {task!r}: meta_fields values must be non-empty strings")
duplicates = sorted({f for f in raw if raw.count(f) > 1})
if duplicates:
raise DatasetPrepError(f"task {task!r}: meta_fields has duplicates: {duplicates}")
overlap = sorted(set(raw) & _data_keys(task_config))
if overlap:
raise DatasetPrepError(f"task {task!r}: meta_fields overlap input/golden fields: {overlap}")
return tuple(raw)
def row_meta(row: Mapping[str, Any], meta_fields: tuple[str, ...] | None) -> dict[str, Any] | None:
"""데이터 ν–‰μ—μ„œ `meta_fields` ν‚€λ§Œ 골라 λ³΅μ‚¬ν•œλ‹€. 행에 μ—†λŠ” ν‚€λŠ” **μƒλž΅**ν•œλ‹€.
값은 κΉŠμ€ 볡사본이닀 -- κ²°κ³Ό 행이 데이터셋 ν–‰ 객체λ₯Ό κ³΅μœ ν•˜μ§€ μ•Šκ²Œ.
"""
if meta_fields is None:
return None
return {key: copy.deepcopy(row[key]) for key in meta_fields if key in row}
def build_selection(
task_config: Mapping[str, Any], cli_where: Where | None = None, limit: int | None = None
) -> RowSelection:
"""νƒœμŠ€ν¬ `where` 와 CLI where(`Where` λ˜λŠ” `None`)λ₯Ό 합쳐 run 의 선택을 λ§Œλ“ λ‹€.
선택은 CLI κ°€ **λͺ…μ‹œμ μœΌλ‘œ** λ§Œλ“€μ–΄ λŸ¬λ„ˆμ— λ„˜κΈ΄λ‹€ -- λŸ¬λ„ˆλŠ” `task_config["where"]` λ₯Ό
슀슀둜 읽지 μ•ŠλŠ”λ‹€(D3).
"""
task_name = task_config.get("name", "?")
task_where = parse_where(task_config.get("where"), origin=f"task {task_name!r}")
return RowSelection(where=combine(task_where, cli_where), limit=limit or None)
def prepare_dataset(
loader: DatasetLoader,
project_config: Mapping[str, Any],
task_config: Mapping[str, Any],
selection: RowSelection | None,
) -> PreparedDataset:
"""데이터셋을 ν•œ 번 읽고 선택을 μ μš©ν•œλ‹€. 0행·라벨 λˆ„λ½Β·meta 였λ₯˜λŠ” μ—¬κΈ°μ„œ ν„°μ§„λ‹€."""
meta_fields = validate_meta_fields(task_config)
dataset_config = project_config.get("dataset") or {}
loaded = loader.read_jsonl(
dataset_config.get("repo", ""),
task_config.get("dataset_path", ""),
source=dataset_config.get("source", "huggingface"),
)
selected = select(loaded.records, selection)
return PreparedDataset(
loaded=loaded,
selected=tuple(selected),
rows_total=len(loaded.records),
meta_fields=meta_fields,
selection=selection,
)
def check_resume_indices(prepared: PreparedDataset, completed_indices: set[int]) -> None:
"""이미 λλ‚œ `sample_idx` κ°€ 선택 밖이면 μ‹€νŒ¨ν•œλ‹€ (D3 resume κ°€λ“œ).
선택 λ°– κ²°κ³Όκ°€ μ„žμΈ results λŠ” 이 run 의 μ„ νƒμœΌλ‘œ μ„€λͺ…λ˜μ§€ μ•ŠλŠ”λ‹€ -- μ„ νƒμ΄λ‚˜
데이터셋이 run μ‹œμž‘ λ’€ λ°”λ€Œμ—ˆλ‹€λŠ” λœ»μ΄λ‹€.
"""
outside = sorted(completed_indices - prepared.selected_indices)
if outside:
raise SelectionError(
"completed results are outside this run's selection "
f"({format_indices(outside)}) "
"-- the selection or dataset changed since the run started; start a new run"
)