File size: 7,099 Bytes
3dfc93b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
"""์ถ”๋ก  ์‹คํŒจ ์ƒ˜ํ”Œ์˜ ์ง€๋ฌธ โ€” run.json `failed_sample_keys` (LP5 D-LP5-4 ยท ๊ณ„ํš ยง2.4).

๋Ÿฌ๋„ˆ๋Š” ์‹คํŒจ ์œ„์น˜(`failed_sample_indices`)์™€ ์˜ˆ์™ธ ํƒ€์ž…(`failed_sample_errors`)๋งŒ run.json ์— ์“ด๋‹ค โ€” ๋Ÿฌ๋„ˆ ๊ฒฝ๋กœ๋Š”
`poc_bundle` ์„ import ํ•˜์ง€ ์•Š๋Š”๋‹ค(D18). ์ด ๋ชจ๋“ˆ์€ CLI ์ธต์ด๋ผ `poc_bundle.keys` ๋กœ ์ง€๋ฌธ์„ ๋œฐ ์ˆ˜ ์žˆ๋‹ค:
`execute_plan` ์ด ๋Ÿฌ๋„ˆ๊ฐ€ **์ •์ƒ ๋ฐ˜ํ™˜ํ•œ ๋’ค** ์‹คํŒจ ์œ„์น˜๋งˆ๋‹ค ์ค€๋น„๋œ ๋ฐ์ดํ„ฐ์…‹์˜ ์›๋ณธ ํ–‰์— ๊ฒฐ๊ณผ ํ–‰๊ณผ ๊ฐ™์€
`extract_input` ยท `extract_golden` ์„ ์ ์šฉํ•ด `{sample_idx, input_key, golden_key, error_type}` ๋ฅผ ๋”ํ•œ๋‹ค.

ํ…์ŠคํŠธ๋Š” ์‹ฃ์ง€ ์•Š๋Š”๋‹ค โ€” hex ์ง€๋ฌธ๊ณผ ํƒ€์ž… ์ด๋ฆ„๋ฟ์ด๋‹ค. ๊ณ„์‚ฐํ•  ์ˆ˜ ์—†์œผ๋ฉด(์œ„์น˜๊ฐ€ ์„ ํƒ ๋ฐ– ยท ์ž…๋ ฅ์ด ๋นˆ ๊ฐ์ฒด ยท
JCS ๊ฐ€ ๊ฑฐ๋ถ€ํ•˜๋Š” ๊ฐ’) **์•„๋ฌด๊ฒƒ๋„ ์“ฐ์ง€ ์•Š๋Š”๋‹ค**(= ๋ชจ๋ฆ„). wrap ์€ ๊ทธ๋•Œ `failures.jsonl` ์„ ์ƒ๋žตํ•œ๋‹ค.
๋‹จ ์žฌ๊ฐœ ์ „ ์‹œ๋„๊ฐ€ ๋‚จ๊ธด ๋ชฉ๋ก์ด run ๋ฌธ์„œ์— ์žˆ์œผ๋ฉด `[]` ๋กœ ๋ฎ๋Š”๋‹ค(`record_failure_keys` โ€” ๋‚ก์€ ์ง€๋ฌธ์„ ๋‚จ๊ธฐ์ง€ ์•Š๋Š”๋‹ค).
๊ฒฝ๊ณ  ํ•œ ์ค„์—๋Š” ์ด์œ ๋งŒ ์ ๊ณ  ๊ฐ’์€ ์ ์ง€ ์•Š๋Š”๋‹ค.
"""

from __future__ import annotations

import asyncio
import inspect
import json
from collections.abc import Mapping
from typing import Any

import click

from poc_bundle.keys import golden_key, input_key
from poc_bundle.v11 import FAILED_SAMPLE_KEYS_KEY
from solar_eval.core.run_records import FAILED_SAMPLE_ERRORS_KEY, extract_golden, extract_input

#: `failed_sample_errors` ์— ๊ทธ ์œ„์น˜๊ฐ€ ์—†์„ ๋•Œ์˜ ํƒ€์ž… ์ด๋ฆ„(wrap ์€ `exception:unknown` ์œผ๋กœ ์ ๋Š”๋‹ค).
UNKNOWN_ERROR_TYPE = "unknown"


class _KeysUnknown(Exception):
    """์ง€๋ฌธ์„ ๋œฐ ์ˆ˜ ์—†๋‹ค โ€” ์ด์œ  ํ•œ ๋งˆ๋””๋งŒ ์‹ฃ๋Š”๋‹ค(๊ฐ’์€ ์‹ฃ์ง€ ์•Š๋Š”๋‹ค)."""


def failure_keys(
    run_doc: Mapping[str, Any] | None, prepared: Any, task_config: Mapping[str, Any]
) -> list[dict[str, Any]] | None:
    """run ๋ฌธ์„œ์˜ ์‹คํŒจ ์œ„์น˜ โ†’ `failed_sample_keys` ์›์†Œ๋“ค(sample_idx ์ˆœ). ๋ชจ๋ฅด๋ฉด None.

    `prepared` ๋Š” `PreparedDataset`(๋˜๋Š” `selected` ๊ฐ€ `(์›๋ณธ ์œ„์น˜, ํ–‰)` ๋ชฉ๋ก์ธ ๊ฐ์ฒด)์ด๋‹ค.
    ์‹คํŒจ๊ฐ€ ์—†์œผ๋ฉด `[]` ์ด๋‹ค.
    """
    try:
        indices = _failed_indices(run_doc)
        rows = dict(prepared.selected)
        types = _error_types(run_doc.get(FAILED_SAMPLE_ERRORS_KEY))
        return [_entry(idx, rows, task_config, types) for idx in sorted(indices)]
    except _KeysUnknown as unknown:
        click.secho(f"  failure keys not recorded ({unknown})", fg="yellow", err=True)
        return None


def record_failure_keys(
    store: Any, run_id: str, prepared: Any, task_config: Mapping[str, Any]
) -> None:
    """run ๋ฌธ์„œ๋ฅผ ์ฝ์–ด ์‹คํŒจ ์ง€๋ฌธ์„ ๊ณ„์‚ฐํ•˜๊ณ  `update_run` ์œผ๋กœ ๋”ํ•œ๋‹ค.

    run ๋ฌธ์„œ์— ์•ž์„  ์‹œ๋„(์žฌ๊ฐœ ์ „)๊ฐ€ ๋‚จ๊ธด ๋น„์ง€ ์•Š์€ ๋ชฉ๋ก์ด ์žˆ์œผ๋ฉด **๋Š˜ ๋ฎ๋Š”๋‹ค** โ€” ์ €์žฅ์†Œ `update_run` ์€ ๋ณ‘ํ•ฉ์ด๋ผ
    ๋ฎ์ง€ ์•Š์œผ๋ฉด ๋‚ก์€ ๋ชฉ๋ก์ด ๋‚จ๋Š”๋‹ค. ์ƒˆ ๋ชฉ๋ก์ด ์žˆ์œผ๋ฉด ๊ทธ๊ฒƒ์œผ๋กœ, ์‹คํŒจ๊ฐ€ ์—†์œผ๋ฉด(`[]`, ์žฌ๊ฐœ๊ฐ€ ์‹คํŒจ๋ฅผ ๋ชจ๋‘ ํšŒ๋ณต) `[]` ๋กœ
    (wrap ์ด 0ํ–‰์„ ์“ด๋‹ค), ๋ชจ๋ฅด๋ฉด(None) ์—ญ์‹œ `[]` ๋กœ ๋ฎ๋Š”๋‹ค โ€” `[]` ๋Š” ๋น„์ง€ ์•Š์€ `failed_sample_indices` ์™€ ์ง‘ํ•ฉ์ด ๋‹ฌ๋ผ
    wrap ์ด ์ƒ๋žต(๋ชจ๋ฆ„)ํ•˜๋ฏ€๋กœ, ๋‚ก์€ ์ง€๋ฌธ์ด ์šฐ์—ฐํžˆ ๊ฐ™์€ ์œ„์น˜ ์ง‘ํ•ฉ๊ณผ ๋งž์•„ ํ‹€๋ฆฐ ์Œ์œผ๋กœ ์“ฐ์ด๋Š” ๊ธธ์ด ์—†๋‹ค.
    ๋‚ก์€ ๋ชฉ๋ก์ด ์—†์œผ๋ฉด: ๋ชจ๋ฅด๋ฉด ์“ฐ์ง€ ์•Š๊ณ , ์‹คํŒจ๊ฐ€ ์—†์–ด๋„ ์“ฐ์ง€ ์•Š๋Š”๋‹ค โ€” ์‹คํŒจ ์—†์Œ์€ `failed_sample_indices = []` ๊ฐ€
    ์ด๋ฏธ ๋งํ•œ๋‹ค(์„ฑ๊ณต run ๋งˆ๋‹ค run.json ์„ ํ•œ ๋ฒˆ ๋” ์“ฐ์ง€ ์•Š๋Š”๋‹ค). run ๋ฌธ์„œ๋ฅผ ์ฝ์„ ์ˆ˜ ์—†๋Š” ์ €์žฅ์†Œ(`load_run_meta`
    ์—†์Œ)๋Š” ๋ชจ๋ฆ„์ด๋‹ค.
    """
    loader = getattr(store, "load_run_meta", None)
    run_doc = loader() if callable(loader) else None
    keys = failure_keys(run_doc, prepared, task_config)
    if _has_stale_keys(run_doc):
        keys = keys or []
    elif not keys:
        return
    pending = store.update_run(run_id, {FAILED_SAMPLE_KEYS_KEY: keys})
    if inspect.isawaitable(pending):
        asyncio.run(pending)


def _has_stale_keys(run_doc: Mapping[str, Any] | None) -> bool:
    """run ๋ฌธ์„œ์— `[]` ๊ฐ€ ์•„๋‹Œ `failed_sample_keys` ๊ฐ€ ์ด๋ฏธ ์žˆ๋‹ค(์•ž์„  ์‹œ๋„์˜ ๋ชฉ๋ก โ€” ๋ชจ์–‘์€ ๋”ฐ์ง€์ง€ ์•Š๋Š”๋‹ค)."""
    return isinstance(run_doc, Mapping) and run_doc.get(FAILED_SAMPLE_KEYS_KEY, []) != []


def _failed_indices(run_doc: Mapping[str, Any] | None) -> list[int]:
    if not isinstance(run_doc, Mapping):
        raise _KeysUnknown("run document unreadable")
    indices = run_doc.get("failed_sample_indices")
    if not isinstance(indices, list):
        raise _KeysUnknown("no failed_sample_indices list")
    if not all(isinstance(i, int) and not isinstance(i, bool) and i >= 0 for i in indices):
        raise _KeysUnknown("failed_sample_indices is not a list of positions")
    if len(set(indices)) != len(indices):
        raise _KeysUnknown("failed_sample_indices has duplicates")
    return indices


def _error_types(errors: Any) -> dict[int, str]:
    """`failed_sample_errors` โ†’ `{sample_idx: ํƒ€์ž… ์ด๋ฆ„}`. ๋ชจ์–‘์ด ํ‹€๋ฆฐ ์›์†Œ๋Š” ๊ฑด๋„ˆ๋›ด๋‹ค(โ†’ unknown)."""
    if not isinstance(errors, list):
        return {}
    types: dict[int, str] = {}
    for element in errors:
        if not isinstance(element, Mapping):
            continue
        idx, name = element.get("sample_idx"), element.get("error_type")
        if isinstance(idx, int) and not isinstance(idx, bool) and isinstance(name, str) and name:
            types[idx] = name
    return types


def _entry(
    idx: int, rows: Mapping[int, Any], task_config: Mapping[str, Any], types: Mapping[int, str]
) -> dict[str, Any]:
    sample = rows.get(idx)
    if not isinstance(sample, Mapping):
        raise _KeysUnknown("failed position outside the selection")
    input_data = _as_stored(extract_input(sample, task_config.get("input_fields", [])))
    if not input_data:
        raise _KeysUnknown("empty input")
    golden = _as_stored(extract_golden(sample, task_config))
    try:
        # items ๊ทœ์น™๊ณผ ๊ฐ™๋‹ค โ€” ์—”์ง„ run ์˜ ์ž…๋ ฅ ์ •์ฒด์„ฑ์€ ์—”์ง„์ด ๊ณ ๋ฅธ input ๊ฐ์ฒด ์ „์ฒด(fields=None).
        ik, gk = input_key(input_data, None), golden_key(golden)
    except (TypeError, ValueError) as exc:  # CanonicalizationError ๋Š” ValueError ๋‹ค
        raise _KeysUnknown(f"input not fingerprintable: {type(exc).__name__}") from None
    return {
        "sample_idx": idx,
        "input_key": ik,
        "golden_key": gk,
        "error_type": types.get(idx, UNKNOWN_ERROR_TYPE),
    }


def _as_stored(value: Any) -> Any:
    """๊ฒฐ๊ณผ ์ €์žฅ์†Œ(`JsonlStore.insert_run_result`)๊ฐ€ ์“ฐ๋Š” ์ง๋ ฌํ™”๋ฅผ ํ•œ ๋ฒˆ ๊ฑฐ์นœ ๊ฐ’.

    items ๋Š” results.jsonl ์„ ๋‹ค์‹œ ์ฝ์–ด ์ง€๋ฌธ์„ ๋œจ๋ฏ€๋กœ, ๊ฐ™์€ ์›๋ณธ ํ–‰์ด๋ฉด ๊ฐ™์€ ํ‚ค๊ฐ€ ๋‚˜์˜ค๊ฒŒ ๊ฐ™์€
    `json.dumps` ์™•๋ณต์„ ๊ฑฐ์นœ๋‹ค(ํŠœํ”Œ โ†’ ๋ชฉ๋ก ยท ๋ฌธ์ž์—ด์ด ์•„๋‹Œ ํ‚ค โ†’ ๋ฌธ์ž์—ด). ์ง๋ ฌํ™”ํ•  ์ˆ˜ ์—†์œผ๋ฉด ๋ชจ๋ฆ„.
    """
    try:
        return json.loads(json.dumps(value, ensure_ascii=False))
    except (TypeError, ValueError) as exc:
        raise _KeysUnknown(f"input not serializable: {type(exc).__name__}") from None