Spaces:
Running
Running
Download solar_eval/scoring_publish/eval_dir.py from dev-strender/proofread-demo: direct link, hf CLI and curl.
- Browser
- Download file 11.9 kB
-
https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/scoring_publish/eval_dir.py
- Command line
-
hf download hf://spaces/dev-strender/proofread-demo/solar_eval/scoring_publish/eval_dir.py
-
curl -L -o eval_dir.py https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/scoring_publish/eval_dir.py
11.9 kB
| """ํ๊ฐ ๋๋ ํ ๋ฆฌ โ `runs eval` ์ ๊ฒฝ๋ก์ ์ฑ์ ๊ฒฐ๊ณผ ์๋ฆฌ (๊ณํ 2026-09-27 L6 ยง2.12 B4 โก ยท D-L6-15). | |
| ์๋ฆฌ๋ `<data_home>/solar-eval-evaluations/<project>/<run_id>/<eval_ulid>/` ๋ค. ๋ ๊ฑฐ์ ์์นด์ด๋ธ ํธ๋ฆฌ(`solar-eval-projects/`) | |
| **๋ฐ**์ด๋ผ ๋ ๊ฑฐ์ ์ ์ฌ๊ธฐ๊ฐ ํ๋ก์ ํธ ์์์ผ๋ก ์ค์ธํ์ง ์๊ณ , run ๋๋ ํ ๋ฆฌ ๋ฐ์ด๋ผ run ๋ฒ๋ค์ด ๊ทธ๋๋ก๋ค. | |
| - `evaluation.json`ยท`eval_details.jsonl` ์ ๋ ๊ฑฐ์ ๋ชจ์ ๊ทธ๋๋ก๋ค โ run ๋๋ ํ ๋ฆฌ์ ๊ฐ์ `JsonlStore` ๋ก ์ด๋ค | |
| (`build_evaluation_doc`ยท`build_eval_detail_docs` ์ฌ์ฌ์ฉ). eval-wrap ์ด ์ด ๋๋ ํ ๋ฆฌ๋ฅผ ํ๊ฐ ์์ฒ์ผ๋ก ์ฝ๋๋ค(B3). | |
| - `eval-run.json` ์ ์ด ํ๊ฐ์ ์ ์ฒด โ ํ๊ฐ ์๊ฐยทํ๊ฐ๊ธฐ ์ค์ ยทjudge ๋ชจ๋ธยท์์ฒ run ๋ฒ๋ค id. ์์์ ์ฐ๊ธฐ(์์ โ `os.replace`). | |
| ๋ค๋ฅธ ๋ ํ์ผ ๋ค์ ๋ง์ง๋ง์ผ๋ก ์ด๋ค โ ์ด ํ์ผ์ด ์๋ ๋๋ ํ ๋ฆฌ๋ ๋๋์ง ์์ ํ๊ฐ๋ค. | |
| - judge ๊ฒฝ๋ก(๊ณํ 2026-09-28 L12 ยง2.7)๋ `verdicts.jsonl`(๊ธฐ์ค ํ์ ์ ๋ถ โ ๊ทผ๊ฑฐยท์ ์๋ต ํฌํจ, ๋ฐ์ดํฐ ํ ์ ์ฉ)์ ๋ ์ฐ๊ณ | |
| `eval-run.json` ์ `purpose`ยท`judge{evaluation_run_id, scoring_bundle_id, retry_of}` ๋ฅผ ์ฃ๋๋ค(๋ฐํ **๋ค**์ ์ด๋ค). lcs ๊ฒฝ๋ก๋ | |
| ์ ํ ์ธ์๋ฅผ ์ฃผ์ง ์์ ๋ฐ์ดํธ๊ฐ ๊ทธ๋๋ก๋ค. | |
| - ์ด๋ฆ ์กฐ๊ฐ(projectยทrun_id)์ ๊ฒฝ๋ก ๊ตฌ๋ถ์ยท์ ์ด ๋ฌธ์ยท`.`/`..` ๋ฅผ ๊ฑฐ๋ถํ๊ณ , ๋ฃจํธ๊ฐ ๋ ํฌ(๋งํฌ๋ ์ํฌํธ๋ฆฌ๋ฉด ๋ฉ์ธ ์์ ํธ๋ฆฌ๋) | |
| ์์ด๋ฉด ์ฐ๊ธฐ ์ ์ ๋ฉ์ถ๋ค โ hub ๋์ ํ์ (`hub_hook.target`, D-L6-1)์ด ์ด๋ฏธ ๋ณด์ง๋ง ์ด ๋ชจ๋์ด ์ง์ ์ฐ๋ฏ๋ก ํ ๋ฒ ๋ ๋ณธ๋ค. | |
| `poc_bundle`ยท`ulid` ๋ ํจ์ ์์์๋ง import ํ๋ค(๋ง์คํฐ ยง4 ๊ท์น 11 ยท D-L6-14). | |
| """ | |
| from __future__ import annotations | |
| import dataclasses | |
| import json | |
| import os | |
| import re | |
| import shutil | |
| import tempfile | |
| from collections.abc import Iterable, Mapping, Sequence | |
| from dataclasses import dataclass | |
| from datetime import datetime | |
| from pathlib import Path | |
| from typing import Any | |
| from solar_eval.core.run_records import build_eval_detail_docs, build_evaluation_doc | |
| from solar_eval.stores import JsonlStore | |
| #: ๋ฐ์ดํฐ ํ ์๋ ํ๊ฐ ๋๋ ํ ๋ฆฌ ๋ฃจํธ ์ด๋ฆ(D-L6-15). | |
| EVALUATIONS_DIRNAME = "solar-eval-evaluations" | |
| EVAL_RUN_FILE = "eval-run.json" | |
| EVAL_RUN_SCHEMA = "solar-eval-eval-run/v1" | |
| #: ์ด ๋๋ ํ ๋ฆฌ๊ฐ ์ฃ๋ ํ๊ฐ์ ๋ชฉ์ โ eval-wrap ๋งค๋ํ์คํธ `purpose` ์ ๊ฐ์ ๊ฐ(PB `PURPOSE_RESCORE_AFTER_FIX`). | |
| EVAL_RUN_PURPOSE = "rescore_after_fix" | |
| #: judge ๊ฒฝ๋ก(๊ณํ 2026-09-28 L12 ยง2.7) โ ๊ธฐ์ค ํ์ ์ ๋ถ(๊ทผ๊ฑฐยท์ ์๋ต ํฌํจ)์ `eval-run.json` ์ `judge` ์นธ ํค. | |
| VERDICTS_FILE = "verdicts.jsonl" | |
| JUDGE_BLOCK_KEYS = ("evaluation_run_id", "scoring_bundle_id", "retry_of") | |
| #: ๋ ํฌ ์ ์ฐ๊ธฐ ๊ฒ์ฌ์ ์ฃผ๋ ์ด๋ฆํ. | |
| WRITE_LABEL = "evaluation directory" | |
| _UNSAFE_SEGMENT = re.compile(r"[/\\\x00-\x1f\x7f]") | |
| _TMP_PREFIX = ".eval-run-" | |
| _TMP_SUFFIX = ".tmp" | |
| #: `EvalDirError.reason` โ ๋ฐํ ๋ค๋ฆฌ ์ฝ๋ `publish_eval_dir:<reason>` ์ ๊ผฌ๋ฆฌ. ๋ชจ๋ ์ค์ ยท์ ๋ ฅ ์ค๋ฅ๋ผ ๋ค์ ๋๋ ค๋ ๊ฐ๋ค. | |
| REASON_UNSAFE_SEGMENT = "unsafe_segment" | |
| REASON_INSIDE_REPO = "inside_repo" | |
| REASON_BAD_DOCUMENT = "bad_document" | |
| REASON_OUTSIDE_ROOT = "outside_root" | |
| class EvalDirError(ValueError): | |
| """ํ๊ฐ ๋๋ ํ ๋ฆฌ๋ฅผ ๋ง๋ค๊ฑฐ๋ ์ธ ์ ์๋ค(์ด๋ฆ ์กฐ๊ฐยท๋ ํฌ ์ยท์๋ชป๋ ๋ฌธ์). `reason` ์ ์ `REASON_*` ํ๋.""" | |
| def __init__(self, message: str, reason: str) -> None: | |
| super().__init__(message) | |
| self.reason = reason | |
| #: ์ฝ์ ๋ณด๊ณ ์ฉ ์ง๊ณ์ ์ฑ์ ์ ๋ ฅ ํ ์(์ฑ๊ณต + ์คํจ) โ ์ฑํ๋ run ๋ฒ๋ค์ ์ ๊ทํ ์ฌ๋ณธ ๊ธฐ์ค(l6b-verify ๋ฆฌ๋ทฐ ์ง์ 4). | |
| REPORT_SAMPLES = "input_samples" | |
| class WrittenEvaluation: | |
| """`write_evaluation` ๊ฒฐ๊ณผ โ ์ ํ๊ฐ id ์ ์ฝ์ ๋ณด๊ณ ์ฉ ์ง๊ณ(`overall_score`ยท`failed`ยท`REPORT_SAMPLES` ํฌํจ).""" | |
| evaluation_id: str | |
| aggregated: Mapping[str, Any] | |
| def evaluations_root(env: Mapping[str, str] | None = None) -> Path: | |
| """`<data_home>/solar-eval-evaluations`(์ ๋ ๊ฒฝ๋ก).""" | |
| from poc_bundle.home import data_home | |
| return data_home(env) / EVALUATIONS_DIRNAME | |
| def _check_segment(label: str, value: str) -> str: | |
| if not isinstance(value, str) or value in ("", ".", "..") or _UNSAFE_SEGMENT.search(value): | |
| raise EvalDirError( | |
| f"evaluation directory: {label} is not a safe path segment", REASON_UNSAFE_SEGMENT | |
| ) | |
| return value | |
| def _require_outside_repo(root: Path, repo_root: Path | None) -> None: | |
| """๋ฃจํธ๊ฐ ๋ ํฌ(๋งํฌ๋ ์ํฌํธ๋ฆฌ๋ฉด ๋ฉ์ธ ์์ ํธ๋ฆฌ๋) ์์ด๋ฉด `EvalDirError`.""" | |
| from poc_bundle.home import HomeError, require_outside_repo | |
| from poc_bundle.mirror import linked_main_work_tree | |
| if repo_root is None: | |
| return | |
| main = linked_main_work_tree(Path(repo_root)) | |
| for tree in (Path(repo_root),) if main is None else (Path(repo_root), main): | |
| try: | |
| require_outside_repo(root, tree, WRITE_LABEL) | |
| except HomeError as exc: | |
| raise EvalDirError( | |
| f"evaluation directory is inside the repo: {exc}", REASON_INSIDE_REPO | |
| ) from None | |
| def create_evaluation_dir( | |
| project: str, | |
| run_id: str, | |
| *, | |
| repo_root: Path | None, | |
| env: Mapping[str, str] | None = None, | |
| ) -> Path: | |
| """์ ํ๊ฐ ๋๋ ํ ๋ฆฌ(๋น ๋๋ ํ ๋ฆฌ)๋ฅผ ๋ง๋ค์ด ๋๋ ค์ค๋ค. ์ด๋ฆ์ ์ ULID โ ๊ฐ์ run ์ ์ฌํ๊ฐ๋ ์๊ฐ ์์ผ๋ก ์์ธ๋ค.""" | |
| from ulid import ULID | |
| parent = ( | |
| evaluations_root(env) | |
| / _check_segment("project", project) | |
| / _check_segment("run_id", run_id) | |
| ) | |
| _require_outside_repo(evaluations_root(env), repo_root) | |
| path = parent / str(ULID()) | |
| path.mkdir(parents=True, exist_ok=False) | |
| return path | |
| async def write_evaluation( | |
| path: Path, | |
| run_id: str, | |
| evaluator: Any, | |
| eval_results: Sequence[Mapping[str, Any]], | |
| eval_failures: Sequence[Mapping[str, Any]], | |
| ) -> WrittenEvaluation: | |
| """๋ ๊ฑฐ์ ๋ชจ์ `evaluation.json`ยท`eval_details.jsonl` ์ ์ด๋ค(`run_eval._evaluate_run` ๊ณผ ๊ฐ์ ์กฐ๋ฆฝ).""" | |
| aggregated = evaluator.aggregate(list(eval_results)) | |
| doc = build_evaluation_doc( | |
| run_id, aggregated, eval_results, eval_failures, evaluator.judge_model | |
| ) | |
| store = JsonlStore(path) | |
| evaluation_id = await store.create_evaluation(doc) | |
| await store.insert_eval_details( | |
| build_eval_detail_docs(evaluation_id, eval_results, eval_failures) | |
| ) | |
| report = { | |
| **aggregated, | |
| "overall_score": doc["overall_score"], | |
| "failed": len(eval_failures), | |
| REPORT_SAMPLES: len(eval_results) + len(eval_failures), | |
| } | |
| return WrittenEvaluation(str(evaluation_id), report) | |
| def eval_run_document( | |
| *, | |
| project: str, | |
| run_id: str, | |
| run_bundle_id: str, | |
| evaluation_id: str, | |
| evaluated_at: str, | |
| evaluator_config: Mapping[str, Any] | None, | |
| eval_model: str | None, | |
| purpose: str = EVAL_RUN_PURPOSE, | |
| judge: Mapping[str, Any] | None = None, | |
| ) -> dict[str, Any]: | |
| """`eval-run.json` ๋ฌธ์(์์). ์๊ฐ์ ์คํ์ ์ด ์์ด์ผ ํ๊ณ ํ๊ฐ๊ธฐ ์ค์ ์ JSON ๋งคํ์ด์ด์ผ ํ๋ค. | |
| `purpose`ยท`judge` ๋ judge ๊ฒฝ๋ก(๊ณํ 2026-09-28 L12 ยง2.7)๋ง ์ค๋ค โ lcs ๊ฒฝ๋ก ํธ์ถ์ ์ธ์๋ฅผ ์ฃผ์ง ์์ ๋ฐ์ดํธ๊ฐ ๊ทธ๋๋ก๋ค. | |
| `judge` ๊ฐ ์์ผ๋ฉด ๋์ `{"evaluation_run_id", "scoring_bundle_id", "retry_of"}` ๋ฅผ ๋ํ๋ค(๋ฐํ ์คํจ๋ฉด bundle id ๋ null). | |
| """ | |
| _check_offset(evaluated_at) | |
| doc: dict[str, Any] = { | |
| "schema": EVAL_RUN_SCHEMA, | |
| "project": project, | |
| "run_id": run_id, | |
| "run_bundle_id": run_bundle_id, | |
| "evaluation_id": evaluation_id, | |
| "evaluated_at": evaluated_at, | |
| "purpose": purpose, | |
| "evaluator_config": _json_mapping(evaluator_config), | |
| "eval_model": eval_model, | |
| } | |
| if judge is not None: | |
| doc["judge"] = _judge_block(judge) | |
| return doc | |
| def _judge_block(judge: Mapping[str, Any]) -> dict[str, Any]: | |
| """`eval-run.json` ์ `judge` ์นธ โ ๋ซํ ํค ์ , ๊ฐ์ ๋ฌธ์์ด id ๋๋ null.""" | |
| if not isinstance(judge, Mapping) or set(judge) != set(JUDGE_BLOCK_KEYS): | |
| raise EvalDirError( | |
| f"eval-run: judge must have exactly the keys {list(JUDGE_BLOCK_KEYS)}", | |
| REASON_BAD_DOCUMENT, | |
| ) | |
| if not all(judge[k] is None or isinstance(judge[k], str) for k in JUDGE_BLOCK_KEYS): | |
| raise EvalDirError("eval-run: judge ids must be strings or null", REASON_BAD_DOCUMENT) | |
| return {key: judge[key] for key in JUDGE_BLOCK_KEYS} | |
| def write_verdicts(path: Path, verdicts: Iterable[Any]) -> Path: | |
| """`verdicts.jsonl` โ ๊ธฐ์ค ํ์ ์ ๋ถ(๊ทผ๊ฑฐยท์ ์๋ต ํฌํจ, ๋ฐ์ดํฐ ํ ์ ์ฉ โ ๊ณํ L12 ยง2.7 ยท D-L12-8). | |
| ํ ์ค = ํ์ dataclass ์ JSON(ํค ์์ = ํ๋ ์์, compact, `ensure_ascii=False`). UTF-8 ๋ก ์ฎ๊ธธ ์ ์๋ ๋ฌธ์์ด(์ง ์๋ | |
| ์๋ก๊ฒ์ดํธ)์ด ๋ ์ค๋ง `ensure_ascii=True` ๋ก ์ ๋๋ค โ ์ ๋ฃ ํ์ ํ ๊ฑด์ด ํ์ผ ์ ์ฒด ์ฐ๊ธฐ๋ฅผ ๊นจ๋จ๋ฆฌ์ง ์๊ฒ. ๋ ํฌ ๋ฐ ๊ฒ์ฌ๋ | |
| ๋๋ ํ ๋ฆฌ๋ฅผ ๋ง๋ค ๋ ์ด๋ฏธ ํ๋ค(`create_evaluation_dir`). | |
| """ | |
| lines: list[str] = [] | |
| for verdict in verdicts: | |
| record = dataclasses.asdict(verdict) | |
| line = json.dumps(record, ensure_ascii=False, separators=(",", ":"), allow_nan=False) | |
| try: | |
| line.encode("utf-8") | |
| except UnicodeEncodeError: | |
| line = json.dumps(record, ensure_ascii=True, separators=(",", ":"), allow_nan=False) | |
| lines.append(line + "\n") | |
| target = Path(path) / VERDICTS_FILE | |
| target.write_text("".join(lines), encoding="utf-8") | |
| return target | |
| def _check_offset(value: str) -> None: | |
| try: | |
| parsed = datetime.fromisoformat(value) | |
| except (TypeError, ValueError): | |
| raise EvalDirError( | |
| "eval-run: evaluated_at is not an ISO 8601 time", REASON_BAD_DOCUMENT | |
| ) from None | |
| if parsed.tzinfo is None: | |
| raise EvalDirError("eval-run: evaluated_at needs a UTC offset", REASON_BAD_DOCUMENT) | |
| def _json_mapping(value: Mapping[str, Any] | None) -> dict[str, Any] | None: | |
| """ํ๊ฐ๊ธฐ ์ค์ ์ JSON ์ผ๋ก ์ฎ๊ธธ ์ ์๋ ์ฌ๋ณธ์ผ๋ก. ์ค์ ์ด ์์ผ๋ฉด(๊ธฐ๋ณธ ํ๊ฐ๊ธฐ) None.""" | |
| if value is None: | |
| return None | |
| if not isinstance(value, Mapping): | |
| raise EvalDirError("eval-run: evaluator_config must be a mapping", REASON_BAD_DOCUMENT) | |
| try: | |
| return json.loads(json.dumps(dict(value), ensure_ascii=False)) | |
| except (TypeError, ValueError): | |
| raise EvalDirError( | |
| "eval-run: evaluator_config is not JSON serializable", REASON_BAD_DOCUMENT | |
| ) from None | |
| def write_eval_run(path: Path, doc: Mapping[str, Any]) -> Path: | |
| """`eval-run.json` ์์์ ์ฐ๊ธฐ(์ง๋ ฌํ โ ๊ฐ์ ๋๋ ํ ๋ฆฌ ์์ ํ์ผ โ fsync โ `os.replace`).""" | |
| text = json.dumps(doc, ensure_ascii=False, indent=2) + "\n" | |
| target = Path(path) / EVAL_RUN_FILE | |
| fd, tmp_name = tempfile.mkstemp(dir=path, prefix=_TMP_PREFIX, suffix=_TMP_SUFFIX) | |
| tmp = Path(tmp_name) | |
| try: | |
| with os.fdopen(fd, "w", encoding="utf-8") as fh: | |
| fh.write(text) | |
| fh.flush() | |
| os.fsync(fh.fileno()) | |
| os.replace(tmp, target) | |
| except BaseException: | |
| tmp.unlink(missing_ok=True) | |
| raise | |
| return target | |
| def remove_partial(path: Path, env: Mapping[str, str] | None = None) -> None: | |
| """๋๋์ง ๋ชปํ ํ๊ฐ ๋๋ ํ ๋ฆฌ๋ฅผ ์น์ด๋ค โ ํ๊ฐ ๋ฃจํธ ์๋์ ๋๋ ํ ๋ฆฌ๋ง(๋ค๋ฅธ ๊ณณ์ `EvalDirError`).""" | |
| resolved = Path(path).resolve() | |
| root = evaluations_root(env).resolve() | |
| if root not in resolved.parents: | |
| raise EvalDirError( | |
| "evaluation directory: refusing to remove a path outside the root", REASON_OUTSIDE_ROOT | |
| ) | |
| shutil.rmtree(resolved) | |