Spaces:
Running
Running
Download solar_eval/cli/commands/failure_keys.py from dev-strender/proofread-demo: direct link, hf CLI and curl.
- Browser
- Download file 7.1 kB
-
https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/cli/commands/failure_keys.py
- Command line
-
hf download hf://spaces/dev-strender/proofread-demo/solar_eval/cli/commands/failure_keys.py
-
curl -L -o failure_keys.py https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/cli/commands/failure_keys.py
7.1 kB
| """์ถ๋ก ์คํจ ์ํ์ ์ง๋ฌธ โ run.json `failed_sample_keys` (LP5 D-LP5-4 ยท ๊ณํ ยง2.4). | |
| ๋ฌ๋๋ ์คํจ ์์น(`failed_sample_indices`)์ ์์ธ ํ์ (`failed_sample_errors`)๋ง run.json ์ ์ด๋ค โ ๋ฌ๋ ๊ฒฝ๋ก๋ | |
| `poc_bundle` ์ import ํ์ง ์๋๋ค(D18). ์ด ๋ชจ๋์ CLI ์ธต์ด๋ผ `poc_bundle.keys` ๋ก ์ง๋ฌธ์ ๋ฐ ์ ์๋ค: | |
| `execute_plan` ์ด ๋ฌ๋๊ฐ **์ ์ ๋ฐํํ ๋ค** ์คํจ ์์น๋ง๋ค ์ค๋น๋ ๋ฐ์ดํฐ์ ์ ์๋ณธ ํ์ ๊ฒฐ๊ณผ ํ๊ณผ ๊ฐ์ | |
| `extract_input` ยท `extract_golden` ์ ์ ์ฉํด `{sample_idx, input_key, golden_key, error_type}` ๋ฅผ ๋ํ๋ค. | |
| ํ ์คํธ๋ ์ฃ์ง ์๋๋ค โ hex ์ง๋ฌธ๊ณผ ํ์ ์ด๋ฆ๋ฟ์ด๋ค. ๊ณ์ฐํ ์ ์์ผ๋ฉด(์์น๊ฐ ์ ํ ๋ฐ ยท ์ ๋ ฅ์ด ๋น ๊ฐ์ฒด ยท | |
| JCS ๊ฐ ๊ฑฐ๋ถํ๋ ๊ฐ) **์๋ฌด๊ฒ๋ ์ฐ์ง ์๋๋ค**(= ๋ชจ๋ฆ). wrap ์ ๊ทธ๋ `failures.jsonl` ์ ์๋ตํ๋ค. | |
| ๋จ ์ฌ๊ฐ ์ ์๋๊ฐ ๋จ๊ธด ๋ชฉ๋ก์ด run ๋ฌธ์์ ์์ผ๋ฉด `[]` ๋ก ๋ฎ๋๋ค(`record_failure_keys` โ ๋ก์ ์ง๋ฌธ์ ๋จ๊ธฐ์ง ์๋๋ค). | |
| ๊ฒฝ๊ณ ํ ์ค์๋ ์ด์ ๋ง ์ ๊ณ ๊ฐ์ ์ ์ง ์๋๋ค. | |
| """ | |
| from __future__ import annotations | |
| import asyncio | |
| import inspect | |
| import json | |
| from collections.abc import Mapping | |
| from typing import Any | |
| import click | |
| from poc_bundle.keys import golden_key, input_key | |
| from poc_bundle.v11 import FAILED_SAMPLE_KEYS_KEY | |
| from solar_eval.core.run_records import FAILED_SAMPLE_ERRORS_KEY, extract_golden, extract_input | |
| #: `failed_sample_errors` ์ ๊ทธ ์์น๊ฐ ์์ ๋์ ํ์ ์ด๋ฆ(wrap ์ `exception:unknown` ์ผ๋ก ์ ๋๋ค). | |
| UNKNOWN_ERROR_TYPE = "unknown" | |
| class _KeysUnknown(Exception): | |
| """์ง๋ฌธ์ ๋ฐ ์ ์๋ค โ ์ด์ ํ ๋ง๋๋ง ์ฃ๋๋ค(๊ฐ์ ์ฃ์ง ์๋๋ค).""" | |
| def failure_keys( | |
| run_doc: Mapping[str, Any] | None, prepared: Any, task_config: Mapping[str, Any] | |
| ) -> list[dict[str, Any]] | None: | |
| """run ๋ฌธ์์ ์คํจ ์์น โ `failed_sample_keys` ์์๋ค(sample_idx ์). ๋ชจ๋ฅด๋ฉด None. | |
| `prepared` ๋ `PreparedDataset`(๋๋ `selected` ๊ฐ `(์๋ณธ ์์น, ํ)` ๋ชฉ๋ก์ธ ๊ฐ์ฒด)์ด๋ค. | |
| ์คํจ๊ฐ ์์ผ๋ฉด `[]` ์ด๋ค. | |
| """ | |
| try: | |
| indices = _failed_indices(run_doc) | |
| rows = dict(prepared.selected) | |
| types = _error_types(run_doc.get(FAILED_SAMPLE_ERRORS_KEY)) | |
| return [_entry(idx, rows, task_config, types) for idx in sorted(indices)] | |
| except _KeysUnknown as unknown: | |
| click.secho(f" failure keys not recorded ({unknown})", fg="yellow", err=True) | |
| return None | |
| def record_failure_keys( | |
| store: Any, run_id: str, prepared: Any, task_config: Mapping[str, Any] | |
| ) -> None: | |
| """run ๋ฌธ์๋ฅผ ์ฝ์ด ์คํจ ์ง๋ฌธ์ ๊ณ์ฐํ๊ณ `update_run` ์ผ๋ก ๋ํ๋ค. | |
| run ๋ฌธ์์ ์์ ์๋(์ฌ๊ฐ ์ )๊ฐ ๋จ๊ธด ๋น์ง ์์ ๋ชฉ๋ก์ด ์์ผ๋ฉด **๋ ๋ฎ๋๋ค** โ ์ ์ฅ์ `update_run` ์ ๋ณํฉ์ด๋ผ | |
| ๋ฎ์ง ์์ผ๋ฉด ๋ก์ ๋ชฉ๋ก์ด ๋จ๋๋ค. ์ ๋ชฉ๋ก์ด ์์ผ๋ฉด ๊ทธ๊ฒ์ผ๋ก, ์คํจ๊ฐ ์์ผ๋ฉด(`[]`, ์ฌ๊ฐ๊ฐ ์คํจ๋ฅผ ๋ชจ๋ ํ๋ณต) `[]` ๋ก | |
| (wrap ์ด 0ํ์ ์ด๋ค), ๋ชจ๋ฅด๋ฉด(None) ์ญ์ `[]` ๋ก ๋ฎ๋๋ค โ `[]` ๋ ๋น์ง ์์ `failed_sample_indices` ์ ์งํฉ์ด ๋ฌ๋ผ | |
| wrap ์ด ์๋ต(๋ชจ๋ฆ)ํ๋ฏ๋ก, ๋ก์ ์ง๋ฌธ์ด ์ฐ์ฐํ ๊ฐ์ ์์น ์งํฉ๊ณผ ๋ง์ ํ๋ฆฐ ์์ผ๋ก ์ฐ์ด๋ ๊ธธ์ด ์๋ค. | |
| ๋ก์ ๋ชฉ๋ก์ด ์์ผ๋ฉด: ๋ชจ๋ฅด๋ฉด ์ฐ์ง ์๊ณ , ์คํจ๊ฐ ์์ด๋ ์ฐ์ง ์๋๋ค โ ์คํจ ์์์ `failed_sample_indices = []` ๊ฐ | |
| ์ด๋ฏธ ๋งํ๋ค(์ฑ๊ณต run ๋ง๋ค run.json ์ ํ ๋ฒ ๋ ์ฐ์ง ์๋๋ค). run ๋ฌธ์๋ฅผ ์ฝ์ ์ ์๋ ์ ์ฅ์(`load_run_meta` | |
| ์์)๋ ๋ชจ๋ฆ์ด๋ค. | |
| """ | |
| loader = getattr(store, "load_run_meta", None) | |
| run_doc = loader() if callable(loader) else None | |
| keys = failure_keys(run_doc, prepared, task_config) | |
| if _has_stale_keys(run_doc): | |
| keys = keys or [] | |
| elif not keys: | |
| return | |
| pending = store.update_run(run_id, {FAILED_SAMPLE_KEYS_KEY: keys}) | |
| if inspect.isawaitable(pending): | |
| asyncio.run(pending) | |
| def _has_stale_keys(run_doc: Mapping[str, Any] | None) -> bool: | |
| """run ๋ฌธ์์ `[]` ๊ฐ ์๋ `failed_sample_keys` ๊ฐ ์ด๋ฏธ ์๋ค(์์ ์๋์ ๋ชฉ๋ก โ ๋ชจ์์ ๋ฐ์ง์ง ์๋๋ค).""" | |
| return isinstance(run_doc, Mapping) and run_doc.get(FAILED_SAMPLE_KEYS_KEY, []) != [] | |
| def _failed_indices(run_doc: Mapping[str, Any] | None) -> list[int]: | |
| if not isinstance(run_doc, Mapping): | |
| raise _KeysUnknown("run document unreadable") | |
| indices = run_doc.get("failed_sample_indices") | |
| if not isinstance(indices, list): | |
| raise _KeysUnknown("no failed_sample_indices list") | |
| if not all(isinstance(i, int) and not isinstance(i, bool) and i >= 0 for i in indices): | |
| raise _KeysUnknown("failed_sample_indices is not a list of positions") | |
| if len(set(indices)) != len(indices): | |
| raise _KeysUnknown("failed_sample_indices has duplicates") | |
| return indices | |
| def _error_types(errors: Any) -> dict[int, str]: | |
| """`failed_sample_errors` โ `{sample_idx: ํ์ ์ด๋ฆ}`. ๋ชจ์์ด ํ๋ฆฐ ์์๋ ๊ฑด๋๋ด๋ค(โ unknown).""" | |
| if not isinstance(errors, list): | |
| return {} | |
| types: dict[int, str] = {} | |
| for element in errors: | |
| if not isinstance(element, Mapping): | |
| continue | |
| idx, name = element.get("sample_idx"), element.get("error_type") | |
| if isinstance(idx, int) and not isinstance(idx, bool) and isinstance(name, str) and name: | |
| types[idx] = name | |
| return types | |
| def _entry( | |
| idx: int, rows: Mapping[int, Any], task_config: Mapping[str, Any], types: Mapping[int, str] | |
| ) -> dict[str, Any]: | |
| sample = rows.get(idx) | |
| if not isinstance(sample, Mapping): | |
| raise _KeysUnknown("failed position outside the selection") | |
| input_data = _as_stored(extract_input(sample, task_config.get("input_fields", []))) | |
| if not input_data: | |
| raise _KeysUnknown("empty input") | |
| golden = _as_stored(extract_golden(sample, task_config)) | |
| try: | |
| # items ๊ท์น๊ณผ ๊ฐ๋ค โ ์์ง run ์ ์ ๋ ฅ ์ ์ฒด์ฑ์ ์์ง์ด ๊ณ ๋ฅธ input ๊ฐ์ฒด ์ ์ฒด(fields=None). | |
| ik, gk = input_key(input_data, None), golden_key(golden) | |
| except (TypeError, ValueError) as exc: # CanonicalizationError ๋ ValueError ๋ค | |
| raise _KeysUnknown(f"input not fingerprintable: {type(exc).__name__}") from None | |
| return { | |
| "sample_idx": idx, | |
| "input_key": ik, | |
| "golden_key": gk, | |
| "error_type": types.get(idx, UNKNOWN_ERROR_TYPE), | |
| } | |
| def _as_stored(value: Any) -> Any: | |
| """๊ฒฐ๊ณผ ์ ์ฅ์(`JsonlStore.insert_run_result`)๊ฐ ์ฐ๋ ์ง๋ ฌํ๋ฅผ ํ ๋ฒ ๊ฑฐ์น ๊ฐ. | |
| items ๋ results.jsonl ์ ๋ค์ ์ฝ์ด ์ง๋ฌธ์ ๋จ๋ฏ๋ก, ๊ฐ์ ์๋ณธ ํ์ด๋ฉด ๊ฐ์ ํค๊ฐ ๋์ค๊ฒ ๊ฐ์ | |
| `json.dumps` ์๋ณต์ ๊ฑฐ์น๋ค(ํํ โ ๋ชฉ๋ก ยท ๋ฌธ์์ด์ด ์๋ ํค โ ๋ฌธ์์ด). ์ง๋ ฌํํ ์ ์์ผ๋ฉด ๋ชจ๋ฆ. | |
| """ | |
| try: | |
| return json.loads(json.dumps(value, ensure_ascii=False)) | |
| except (TypeError, ValueError) as exc: | |
| raise _KeysUnknown(f"input not serializable: {type(exc).__name__}") from None | |