File size: 6,994 Bytes
3dfc93b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
"""run μ‹œμž‘ μ „ 데이터셋 μ€€λΉ„ -- ν•œ 번 읽고, κ³ λ₯΄κ³ , ν–‰ 메타 κ·œμΉ™μ„ κ²€μ¦ν•œλ‹€ (κ³„νš D3Β·D13Β·D16).

CLI(`runs start/batch`)λŠ” run 디렉토리λ₯Ό λ§Œλ“€κΈ° **전에** `prepare_dataset` 을 뢈러
0ν–‰ 선택·라벨 λˆ„λ½Β·`meta_fields` 였λ₯˜λ₯Ό λ¨Όμ € ν„°λœ¨λ¦¬κ³ , 같은 `PreparedDataset` 을
λŸ¬λ„ˆ(`execute_run(prepared=...)`)와 run.json 기둝(sha256Β·ν–‰ μˆ˜Β·μ„ νƒ)에 ν•¨κ»˜ λ„˜κΈ΄λ‹€.
κ·Έλž˜μ„œ run.json 의 sha256 κ³Ό μ‹€μ œλ‘œ 돈 행이 **같은 λ°”μ΄νŠΈ**μ—μ„œ λ‚˜μ˜¨λ‹€.

λŸ¬λ„ˆλŠ” `prepared` κ°€ μ—†μœΌλ©΄(μ„œλΉ„μŠ€Β·μ˜› 호좜자) 같은 ν•¨μˆ˜λ‘œ 슀슀둜 μ€€λΉ„ν•œλ‹€.

이 λͺ¨λ“ˆμ€ poc-bundle νŒ¨ν‚€μ§€μ™€ packaging 을 κ°€μ Έμ˜€μ§€ μ•ŠλŠ”λ‹€(D18).
"""

from __future__ import annotations

import copy
from dataclasses import dataclass
from typing import Any, Mapping

from solar_eval.core.dataset_loader import DatasetLoader, LoadedJsonl
from solar_eval.core.row_filter import (
    RowSelection,
    SelectionError,
    Where,
    combine,
    format_indices,
    parse_where,
    select,
)


class DatasetPrepError(ValueError):
    """νƒœμŠ€ν¬μ˜ ν–‰ 메타·where 선언이 κ·œμΉ™μ— λ§žμ§€ μ•Šμ„ λ•Œ -- run 을 μ‹œμž‘ν•˜μ§€ μ•ŠλŠ”λ‹€."""


@dataclass(frozen=True)
class PreparedDataset:
    """run 이 돌 ν–‰ μ§‘ν•©. `selected` λŠ” `(원본 파일의 0-based μœ„μΉ˜, ν–‰)` νŠœν”Œλ“€μ΄λ‹€."""

    loaded: LoadedJsonl
    selected: tuple[tuple[int, dict[str, Any]], ...]
    rows_total: int
    meta_fields: tuple[str, ...] | None
    selection: RowSelection | None = None

    @property
    def rows_selected(self) -> int:
        return len(self.selected)

    @property
    def sha256(self) -> str:
        return self.loaded.sha256

    @property
    def selected_indices(self) -> frozenset[int]:
        return frozenset(idx for idx, _ in self.selected)


def reject_defaults_where(project_config: Mapping[str, Any]) -> None:
    """`defaults.where` λ₯Ό κ±°λΆ€ν•œλ‹€ (D1).

    `_resolve_defaults` λŠ” ν‚€ λ‹¨μœ„ 얕은 병합이라 `defaults.where` κ°€ 라벨 μ—†λŠ” νƒœμŠ€ν¬μ—λ„
    퍼져 κ·Έ run 이 μ „λΆ€ μ£½κ³ , νƒœμŠ€ν¬μ— where κ°€ 있으면 ν†΅μ§Έλ‘œ λ¬΄μ‹œλœλ‹€("병합"이 μ•„λ‹ˆλΌ
    "λŒ€μ²΄"). λ‘œλ“œλœ dict 에 `defaults` ν‚€κ°€ 병합 뒀에도 남아 μžˆμœΌλ―€λ‘œ 그것을 λ³Έλ‹€.
    λ‘œλ“œ μ‹œμ μ΄ μ•„λ‹ˆλΌ run μ‹œμž‘ μ‹œ κ²€μ‚¬ν•œλ‹€ -- ν•œ ν”„λ‘œμ νŠΈμ˜ 잘λͺ»λœ 선언이 λ‹€λ₯Έ
    ν”„λ‘œμ νŠΈ λ‘œλ“œλ₯Ό 막지 μ•Šκ²Œ.
    """
    defaults = project_config.get("defaults") or {}
    if isinstance(defaults, Mapping) and "where" in defaults:
        name = project_config.get("name", "?")
        raise DatasetPrepError(
            f"project {name!r}: defaults.where is not allowed -- "
            "declare where on a task or pass --where on the CLI"
        )


def _data_keys(task_config: Mapping[str, Any]) -> set[str]:
    """μž…λ ₯Β·μ •λ‹΅μœΌλ‘œ 이미 μ“°λŠ” 데이터 ν–‰ ν‚€ -- meta 둜 쀑볡 λ³΅μ œν•˜μ§€ μ•ŠλŠ”λ‹€."""
    keys = set(task_config.get("input_fields") or [])
    golden_field = task_config.get("golden_field")
    if golden_field:
        keys.add(golden_field)
    golden_fields = task_config.get("golden_fields") or {}
    if isinstance(golden_fields, Mapping):
        keys.update(v for v in golden_fields.values() if isinstance(v, str))
    return keys


def validate_meta_fields(task_config: Mapping[str, Any]) -> tuple[str, ...] | None:
    """νƒœμŠ€ν¬ `meta_fields` 검증 (D13) -- λΉ„μ–΄ μžˆμ§€ μ•Šμ€ λ¬Έμžμ—΄ 리슀트, μ€‘λ³΅Β·μž…λ ₯/μ •λ‹΅ κ²ΉμΉ¨ λΆˆκ°€.

    선언이 μ—†μœΌλ©΄ `None`(results 행에 `meta` ν‚€λ₯Ό μ“°μ§€ μ•ŠλŠ”λ‹€ -- λ‹€λ₯Έ ν”„λ‘œμ νŠΈμ˜ λ°”μ΄νŠΈ λΆˆλ³€).
    """
    raw = task_config.get("meta_fields")
    if raw is None:
        return None
    task = task_config.get("name", "?")
    if not isinstance(raw, list) or not raw:
        raise DatasetPrepError(f"task {task!r}: meta_fields must be a non-empty list of strings")
    if not all(isinstance(f, str) and f.strip() for f in raw):
        raise DatasetPrepError(f"task {task!r}: meta_fields values must be non-empty strings")
    duplicates = sorted({f for f in raw if raw.count(f) > 1})
    if duplicates:
        raise DatasetPrepError(f"task {task!r}: meta_fields has duplicates: {duplicates}")
    overlap = sorted(set(raw) & _data_keys(task_config))
    if overlap:
        raise DatasetPrepError(f"task {task!r}: meta_fields overlap input/golden fields: {overlap}")
    return tuple(raw)


def row_meta(row: Mapping[str, Any], meta_fields: tuple[str, ...] | None) -> dict[str, Any] | None:
    """데이터 ν–‰μ—μ„œ `meta_fields` ν‚€λ§Œ 골라 λ³΅μ‚¬ν•œλ‹€. 행에 μ—†λŠ” ν‚€λŠ” **μƒλž΅**ν•œλ‹€.

    값은 κΉŠμ€ 볡사본이닀 -- κ²°κ³Ό 행이 데이터셋 ν–‰ 객체λ₯Ό κ³΅μœ ν•˜μ§€ μ•Šκ²Œ.
    """
    if meta_fields is None:
        return None
    return {key: copy.deepcopy(row[key]) for key in meta_fields if key in row}


def build_selection(
    task_config: Mapping[str, Any], cli_where: Where | None = None, limit: int | None = None
) -> RowSelection:
    """νƒœμŠ€ν¬ `where` 와 CLI where(`Where` λ˜λŠ” `None`)λ₯Ό 합쳐 run 의 선택을 λ§Œλ“ λ‹€.

    선택은 CLI κ°€ **λͺ…μ‹œμ μœΌλ‘œ** λ§Œλ“€μ–΄ λŸ¬λ„ˆμ— λ„˜κΈ΄λ‹€ -- λŸ¬λ„ˆλŠ” `task_config["where"]` λ₯Ό
    슀슀둜 읽지 μ•ŠλŠ”λ‹€(D3).
    """
    task_name = task_config.get("name", "?")
    task_where = parse_where(task_config.get("where"), origin=f"task {task_name!r}")
    return RowSelection(where=combine(task_where, cli_where), limit=limit or None)


def prepare_dataset(
    loader: DatasetLoader,
    project_config: Mapping[str, Any],
    task_config: Mapping[str, Any],
    selection: RowSelection | None,
) -> PreparedDataset:
    """데이터셋을 ν•œ 번 읽고 선택을 μ μš©ν•œλ‹€. 0행·라벨 λˆ„λ½Β·meta 였λ₯˜λŠ” μ—¬κΈ°μ„œ ν„°μ§„λ‹€."""
    meta_fields = validate_meta_fields(task_config)
    dataset_config = project_config.get("dataset") or {}
    loaded = loader.read_jsonl(
        dataset_config.get("repo", ""),
        task_config.get("dataset_path", ""),
        source=dataset_config.get("source", "huggingface"),
    )
    selected = select(loaded.records, selection)
    return PreparedDataset(
        loaded=loaded,
        selected=tuple(selected),
        rows_total=len(loaded.records),
        meta_fields=meta_fields,
        selection=selection,
    )


def check_resume_indices(prepared: PreparedDataset, completed_indices: set[int]) -> None:
    """이미 λλ‚œ `sample_idx` κ°€ 선택 밖이면 μ‹€νŒ¨ν•œλ‹€ (D3 resume κ°€λ“œ).

    선택 λ°– κ²°κ³Όκ°€ μ„žμΈ results λŠ” 이 run 의 μ„ νƒμœΌλ‘œ μ„€λͺ…λ˜μ§€ μ•ŠλŠ”λ‹€ -- μ„ νƒμ΄λ‚˜
    데이터셋이 run μ‹œμž‘ λ’€ λ°”λ€Œμ—ˆλ‹€λŠ” λœ»μ΄λ‹€.
    """
    outside = sorted(completed_indices - prepared.selected_indices)
    if outside:
        raise SelectionError(
            "completed results are outside this run's selection "
            f"({format_indices(outside)}) "
            "-- the selection or dataset changed since the run started; start a new run"
        )