proofread-demo / solar_eval /cli /commands /failure_keys.py
dev-strender's picture
feat: switch demo to FT + Opus article review (pipeline_exp_261021_ft_hybrid)
3dfc93b verified
Raw History Blame Contribute Delete
7.1 kB
"""์ถ”๋ก  ์‹คํŒจ ์ƒ˜ํ”Œ์˜ ์ง€๋ฌธ โ€” run.json `failed_sample_keys` (LP5 D-LP5-4 ยท ๊ณ„ํš ยง2.4).
๋Ÿฌ๋„ˆ๋Š” ์‹คํŒจ ์œ„์น˜(`failed_sample_indices`)์™€ ์˜ˆ์™ธ ํƒ€์ž…(`failed_sample_errors`)๋งŒ run.json ์— ์“ด๋‹ค โ€” ๋Ÿฌ๋„ˆ ๊ฒฝ๋กœ๋Š”
`poc_bundle` ์„ import ํ•˜์ง€ ์•Š๋Š”๋‹ค(D18). ์ด ๋ชจ๋“ˆ์€ CLI ์ธต์ด๋ผ `poc_bundle.keys` ๋กœ ์ง€๋ฌธ์„ ๋œฐ ์ˆ˜ ์žˆ๋‹ค:
`execute_plan` ์ด ๋Ÿฌ๋„ˆ๊ฐ€ **์ •์ƒ ๋ฐ˜ํ™˜ํ•œ ๋’ค** ์‹คํŒจ ์œ„์น˜๋งˆ๋‹ค ์ค€๋น„๋œ ๋ฐ์ดํ„ฐ์…‹์˜ ์›๋ณธ ํ–‰์— ๊ฒฐ๊ณผ ํ–‰๊ณผ ๊ฐ™์€
`extract_input` ยท `extract_golden` ์„ ์ ์šฉํ•ด `{sample_idx, input_key, golden_key, error_type}` ๋ฅผ ๋”ํ•œ๋‹ค.
ํ…์ŠคํŠธ๋Š” ์‹ฃ์ง€ ์•Š๋Š”๋‹ค โ€” hex ์ง€๋ฌธ๊ณผ ํƒ€์ž… ์ด๋ฆ„๋ฟ์ด๋‹ค. ๊ณ„์‚ฐํ•  ์ˆ˜ ์—†์œผ๋ฉด(์œ„์น˜๊ฐ€ ์„ ํƒ ๋ฐ– ยท ์ž…๋ ฅ์ด ๋นˆ ๊ฐ์ฒด ยท
JCS ๊ฐ€ ๊ฑฐ๋ถ€ํ•˜๋Š” ๊ฐ’) **์•„๋ฌด๊ฒƒ๋„ ์“ฐ์ง€ ์•Š๋Š”๋‹ค**(= ๋ชจ๋ฆ„). wrap ์€ ๊ทธ๋•Œ `failures.jsonl` ์„ ์ƒ๋žตํ•œ๋‹ค.
๋‹จ ์žฌ๊ฐœ ์ „ ์‹œ๋„๊ฐ€ ๋‚จ๊ธด ๋ชฉ๋ก์ด run ๋ฌธ์„œ์— ์žˆ์œผ๋ฉด `[]` ๋กœ ๋ฎ๋Š”๋‹ค(`record_failure_keys` โ€” ๋‚ก์€ ์ง€๋ฌธ์„ ๋‚จ๊ธฐ์ง€ ์•Š๋Š”๋‹ค).
๊ฒฝ๊ณ  ํ•œ ์ค„์—๋Š” ์ด์œ ๋งŒ ์ ๊ณ  ๊ฐ’์€ ์ ์ง€ ์•Š๋Š”๋‹ค.
"""
from __future__ import annotations
import asyncio
import inspect
import json
from collections.abc import Mapping
from typing import Any
import click
from poc_bundle.keys import golden_key, input_key
from poc_bundle.v11 import FAILED_SAMPLE_KEYS_KEY
from solar_eval.core.run_records import FAILED_SAMPLE_ERRORS_KEY, extract_golden, extract_input
#: `failed_sample_errors` ์— ๊ทธ ์œ„์น˜๊ฐ€ ์—†์„ ๋•Œ์˜ ํƒ€์ž… ์ด๋ฆ„(wrap ์€ `exception:unknown` ์œผ๋กœ ์ ๋Š”๋‹ค).
UNKNOWN_ERROR_TYPE = "unknown"
class _KeysUnknown(Exception):
"""์ง€๋ฌธ์„ ๋œฐ ์ˆ˜ ์—†๋‹ค โ€” ์ด์œ  ํ•œ ๋งˆ๋””๋งŒ ์‹ฃ๋Š”๋‹ค(๊ฐ’์€ ์‹ฃ์ง€ ์•Š๋Š”๋‹ค)."""
def failure_keys(
run_doc: Mapping[str, Any] | None, prepared: Any, task_config: Mapping[str, Any]
) -> list[dict[str, Any]] | None:
"""run ๋ฌธ์„œ์˜ ์‹คํŒจ ์œ„์น˜ โ†’ `failed_sample_keys` ์›์†Œ๋“ค(sample_idx ์ˆœ). ๋ชจ๋ฅด๋ฉด None.
`prepared` ๋Š” `PreparedDataset`(๋˜๋Š” `selected` ๊ฐ€ `(์›๋ณธ ์œ„์น˜, ํ–‰)` ๋ชฉ๋ก์ธ ๊ฐ์ฒด)์ด๋‹ค.
์‹คํŒจ๊ฐ€ ์—†์œผ๋ฉด `[]` ์ด๋‹ค.
"""
try:
indices = _failed_indices(run_doc)
rows = dict(prepared.selected)
types = _error_types(run_doc.get(FAILED_SAMPLE_ERRORS_KEY))
return [_entry(idx, rows, task_config, types) for idx in sorted(indices)]
except _KeysUnknown as unknown:
click.secho(f" failure keys not recorded ({unknown})", fg="yellow", err=True)
return None
def record_failure_keys(
store: Any, run_id: str, prepared: Any, task_config: Mapping[str, Any]
) -> None:
"""run ๋ฌธ์„œ๋ฅผ ์ฝ์–ด ์‹คํŒจ ์ง€๋ฌธ์„ ๊ณ„์‚ฐํ•˜๊ณ  `update_run` ์œผ๋กœ ๋”ํ•œ๋‹ค.
run ๋ฌธ์„œ์— ์•ž์„  ์‹œ๋„(์žฌ๊ฐœ ์ „)๊ฐ€ ๋‚จ๊ธด ๋น„์ง€ ์•Š์€ ๋ชฉ๋ก์ด ์žˆ์œผ๋ฉด **๋Š˜ ๋ฎ๋Š”๋‹ค** โ€” ์ €์žฅ์†Œ `update_run` ์€ ๋ณ‘ํ•ฉ์ด๋ผ
๋ฎ์ง€ ์•Š์œผ๋ฉด ๋‚ก์€ ๋ชฉ๋ก์ด ๋‚จ๋Š”๋‹ค. ์ƒˆ ๋ชฉ๋ก์ด ์žˆ์œผ๋ฉด ๊ทธ๊ฒƒ์œผ๋กœ, ์‹คํŒจ๊ฐ€ ์—†์œผ๋ฉด(`[]`, ์žฌ๊ฐœ๊ฐ€ ์‹คํŒจ๋ฅผ ๋ชจ๋‘ ํšŒ๋ณต) `[]` ๋กœ
(wrap ์ด 0ํ–‰์„ ์“ด๋‹ค), ๋ชจ๋ฅด๋ฉด(None) ์—ญ์‹œ `[]` ๋กœ ๋ฎ๋Š”๋‹ค โ€” `[]` ๋Š” ๋น„์ง€ ์•Š์€ `failed_sample_indices` ์™€ ์ง‘ํ•ฉ์ด ๋‹ฌ๋ผ
wrap ์ด ์ƒ๋žต(๋ชจ๋ฆ„)ํ•˜๋ฏ€๋กœ, ๋‚ก์€ ์ง€๋ฌธ์ด ์šฐ์—ฐํžˆ ๊ฐ™์€ ์œ„์น˜ ์ง‘ํ•ฉ๊ณผ ๋งž์•„ ํ‹€๋ฆฐ ์Œ์œผ๋กœ ์“ฐ์ด๋Š” ๊ธธ์ด ์—†๋‹ค.
๋‚ก์€ ๋ชฉ๋ก์ด ์—†์œผ๋ฉด: ๋ชจ๋ฅด๋ฉด ์“ฐ์ง€ ์•Š๊ณ , ์‹คํŒจ๊ฐ€ ์—†์–ด๋„ ์“ฐ์ง€ ์•Š๋Š”๋‹ค โ€” ์‹คํŒจ ์—†์Œ์€ `failed_sample_indices = []` ๊ฐ€
์ด๋ฏธ ๋งํ•œ๋‹ค(์„ฑ๊ณต run ๋งˆ๋‹ค run.json ์„ ํ•œ ๋ฒˆ ๋” ์“ฐ์ง€ ์•Š๋Š”๋‹ค). run ๋ฌธ์„œ๋ฅผ ์ฝ์„ ์ˆ˜ ์—†๋Š” ์ €์žฅ์†Œ(`load_run_meta`
์—†์Œ)๋Š” ๋ชจ๋ฆ„์ด๋‹ค.
"""
loader = getattr(store, "load_run_meta", None)
run_doc = loader() if callable(loader) else None
keys = failure_keys(run_doc, prepared, task_config)
if _has_stale_keys(run_doc):
keys = keys or []
elif not keys:
return
pending = store.update_run(run_id, {FAILED_SAMPLE_KEYS_KEY: keys})
if inspect.isawaitable(pending):
asyncio.run(pending)
def _has_stale_keys(run_doc: Mapping[str, Any] | None) -> bool:
"""run ๋ฌธ์„œ์— `[]` ๊ฐ€ ์•„๋‹Œ `failed_sample_keys` ๊ฐ€ ์ด๋ฏธ ์žˆ๋‹ค(์•ž์„  ์‹œ๋„์˜ ๋ชฉ๋ก โ€” ๋ชจ์–‘์€ ๋”ฐ์ง€์ง€ ์•Š๋Š”๋‹ค)."""
return isinstance(run_doc, Mapping) and run_doc.get(FAILED_SAMPLE_KEYS_KEY, []) != []
def _failed_indices(run_doc: Mapping[str, Any] | None) -> list[int]:
if not isinstance(run_doc, Mapping):
raise _KeysUnknown("run document unreadable")
indices = run_doc.get("failed_sample_indices")
if not isinstance(indices, list):
raise _KeysUnknown("no failed_sample_indices list")
if not all(isinstance(i, int) and not isinstance(i, bool) and i >= 0 for i in indices):
raise _KeysUnknown("failed_sample_indices is not a list of positions")
if len(set(indices)) != len(indices):
raise _KeysUnknown("failed_sample_indices has duplicates")
return indices
def _error_types(errors: Any) -> dict[int, str]:
"""`failed_sample_errors` โ†’ `{sample_idx: ํƒ€์ž… ์ด๋ฆ„}`. ๋ชจ์–‘์ด ํ‹€๋ฆฐ ์›์†Œ๋Š” ๊ฑด๋„ˆ๋›ด๋‹ค(โ†’ unknown)."""
if not isinstance(errors, list):
return {}
types: dict[int, str] = {}
for element in errors:
if not isinstance(element, Mapping):
continue
idx, name = element.get("sample_idx"), element.get("error_type")
if isinstance(idx, int) and not isinstance(idx, bool) and isinstance(name, str) and name:
types[idx] = name
return types
def _entry(
idx: int, rows: Mapping[int, Any], task_config: Mapping[str, Any], types: Mapping[int, str]
) -> dict[str, Any]:
sample = rows.get(idx)
if not isinstance(sample, Mapping):
raise _KeysUnknown("failed position outside the selection")
input_data = _as_stored(extract_input(sample, task_config.get("input_fields", [])))
if not input_data:
raise _KeysUnknown("empty input")
golden = _as_stored(extract_golden(sample, task_config))
try:
# items ๊ทœ์น™๊ณผ ๊ฐ™๋‹ค โ€” ์—”์ง„ run ์˜ ์ž…๋ ฅ ์ •์ฒด์„ฑ์€ ์—”์ง„์ด ๊ณ ๋ฅธ input ๊ฐ์ฒด ์ „์ฒด(fields=None).
ik, gk = input_key(input_data, None), golden_key(golden)
except (TypeError, ValueError) as exc: # CanonicalizationError ๋Š” ValueError ๋‹ค
raise _KeysUnknown(f"input not fingerprintable: {type(exc).__name__}") from None
return {
"sample_idx": idx,
"input_key": ik,
"golden_key": gk,
"error_type": types.get(idx, UNKNOWN_ERROR_TYPE),
}
def _as_stored(value: Any) -> Any:
"""๊ฒฐ๊ณผ ์ €์žฅ์†Œ(`JsonlStore.insert_run_result`)๊ฐ€ ์“ฐ๋Š” ์ง๋ ฌํ™”๋ฅผ ํ•œ ๋ฒˆ ๊ฑฐ์นœ ๊ฐ’.
items ๋Š” results.jsonl ์„ ๋‹ค์‹œ ์ฝ์–ด ์ง€๋ฌธ์„ ๋œจ๋ฏ€๋กœ, ๊ฐ™์€ ์›๋ณธ ํ–‰์ด๋ฉด ๊ฐ™์€ ํ‚ค๊ฐ€ ๋‚˜์˜ค๊ฒŒ ๊ฐ™์€
`json.dumps` ์™•๋ณต์„ ๊ฑฐ์นœ๋‹ค(ํŠœํ”Œ โ†’ ๋ชฉ๋ก ยท ๋ฌธ์ž์—ด์ด ์•„๋‹Œ ํ‚ค โ†’ ๋ฌธ์ž์—ด). ์ง๋ ฌํ™”ํ•  ์ˆ˜ ์—†์œผ๋ฉด ๋ชจ๋ฆ„.
"""
try:
return json.loads(json.dumps(value, ensure_ascii=False))
except (TypeError, ValueError) as exc:
raise _KeysUnknown(f"input not serializable: {type(exc).__name__}") from None