Spaces:
Running
Running
Download solar_eval/cli/commands/run_progress.py from dev-strender/proofread-demo: direct link, hf CLI and curl.
- Browser
- Download file 9.23 kB
-
https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/cli/commands/run_progress.py
- Command line
-
hf download hf://spaces/dev-strender/proofread-demo/solar_eval/cli/commands/run_progress.py
-
curl -L -o run_progress.py https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/cli/commands/run_progress.py
9.23 kB
| """run ์คํ + ์งํ ํ์ (Rich progress bar) โ `runs start`ยท`runs resume` ์ด ๊ณต์ ํ๋ค. | |
| `runs.py` ๊ฐ 800์ค ์ํ์ ๋์ด ๋ผ์ด ๋๋ค(๊ณํ 2026-09-24-phase1a E1). ๋์์ ์ฎ๊ธฐ๊ธฐ ์ ๊ณผ ๊ฐ๊ณ , | |
| ๋ฌ๋์ ์ ํยท์ค๋น๋ ๋ฐ์ดํฐ์ ยทํ์ดํ๋ผ์ธ ์ธ์คํด์ค๋ฅผ ๋๊ธฐ๋ ์ธ์๋ง ๋ํ๋ค(D3ยทD6). | |
| """ | |
| from __future__ import annotations | |
| import asyncio | |
| import logging | |
| from typing import Any | |
| import click | |
| from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn | |
| from solar_eval.cli.formatters import format_scores | |
| #: ์งํ ํ์ ์๋ก ๋์ด์ฌ๋ ค ์ฐ๋ ๋ก๊ฑฐ๋ค. | |
| _LOG_NAMES = ( | |
| "solar_eval.pipelines.pipeline", | |
| "solar_eval.pipelines.steps.llm", | |
| "solar_eval.core.runner", | |
| "solar_eval.providers.upstage", | |
| ) | |
| class _ProgressView: | |
| """Rich ์งํ ํ์(๊ฐ๋ฅํ ๋)์ ํ๋ฌธ ์ถ๋ ฅ ์ฌ์ด๋ฅผ ๊ฐ๋ฅด๋ ์์ ์ํ ๋ฌถ์.""" | |
| def __init__(self) -> None: | |
| from rich.console import Console | |
| from rich.live import Live | |
| self.bar = Progress( | |
| SpinnerColumn(), | |
| TextColumn("[progress.description]{task.description}"), | |
| BarColumn(), | |
| TextColumn("{task.completed}/{task.total}"), | |
| TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), | |
| TimeElapsedColumn(), | |
| ) | |
| self.live: Live | None = None | |
| self.use_rich = True | |
| self.inference_task_id: Any = None | |
| self.eval_task_id: Any = None | |
| self._handler: logging.Handler | None = None | |
| self._closed = False | |
| try: | |
| self.live = Live(self.bar, console=Console(stderr=True), refresh_per_second=8) | |
| self.live.start() | |
| except Exception as e: # ํฐ๋ฏธ๋์ด Rich ๋ฅผ ๋ชป ๋ฐ์ผ๋ฉด ํ๋ฌธ์ผ๋ก โ ์์ธ์ ํ ์ค ๋จ๊ธด๋ค. | |
| click.secho(f" (์งํ ํ์ ์์ด ํ๋ฌธ์ผ๋ก ์ถ๋ ฅํฉ๋๋ค: {e})", fg="yellow") | |
| self.use_rich = False | |
| if self.use_rich: | |
| self._redirect_logs() | |
| def _redirect_logs(self) -> None: | |
| """ํ์ดํ๋ผ์ธยท๋ฌ๋ ๋ก๊ทธ๋ฅผ ์งํ ํ์ ์๋ก ์ฌ๋ฆฐ๋ค.""" | |
| from rich.text import Text | |
| live = self.live | |
| class _LiveLogHandler(logging.Handler): | |
| def emit(self, record): | |
| if live: | |
| live.console.print(Text(record.getMessage(), style="dim")) | |
| self._handler = _LiveLogHandler() | |
| for name in _LOG_NAMES: | |
| lg = logging.getLogger(name) | |
| lg.addHandler(self._handler) | |
| lg.propagate = False | |
| logging.getLogger("httpx").setLevel(logging.WARNING) | |
| def cleanup(self) -> None: | |
| """์งํ ํ์๋ฅผ ๋ซ๊ณ ๋ก๊ฑฐ๋ฅผ ๋๋๋ฆฐ๋ค. ์ฌ๋ฌ ๋ฒ ๋ถ๋ฌ๋ ๋๋ค(๋ ์ด๋ฒคํธ + finally).""" | |
| if self.use_rich and self.live is not None and not self._closed: | |
| self._closed = True | |
| self.live.stop() | |
| for name in _LOG_NAMES: | |
| logging.getLogger(name).removeHandler(self._handler) | |
| logging.getLogger(name).propagate = True | |
| def _on_inference(view: _ProgressView, etype: str, event: dict[str, Any]) -> None: | |
| if etype == "dataset_loaded": | |
| if view.use_rich: | |
| view.inference_task_id = view.bar.add_task("Inference", total=event["total"]) | |
| else: | |
| click.secho(f" Dataset loaded: {event['total']} samples", fg="blue") | |
| elif etype == "inference_progress": | |
| if view.use_rich and view.inference_task_id is not None: | |
| view.bar.update(view.inference_task_id, completed=event["completed"]) | |
| elif not view.use_rich: | |
| c, t = event["completed"], event["total"] | |
| if c % 10 == 0 or c == t: | |
| click.echo(f" Inference: {c}/{t}") | |
| elif etype == "inference_complete": | |
| if view.use_rich and view.inference_task_id is not None: | |
| view.bar.update(view.inference_task_id, completed=event["total"]) | |
| else: | |
| click.secho(f" Inference complete: {event['total']} samples", fg="green") | |
| def _on_evaluation(view: _ProgressView, etype: str, event: dict[str, Any]) -> None: | |
| if etype == "evaluation_start": | |
| if view.use_rich: | |
| view.eval_task_id = view.bar.add_task("Evaluation", total=event.get("total", 0) or 1) | |
| else: | |
| click.secho(" Starting evaluation...", fg="blue") | |
| elif etype == "evaluation_progress" and view.use_rich and view.eval_task_id is not None: | |
| view.bar.update(view.eval_task_id, completed=event["completed"]) | |
| def _on_finish(view: _ProgressView, etype: str, event: dict[str, Any]) -> None: | |
| if etype == "done": | |
| if view.use_rich and view.eval_task_id is not None: | |
| view.bar.update(view.eval_task_id, completed=view.bar.tasks[view.eval_task_id].total) | |
| view.cleanup() | |
| click.echo() | |
| click.secho("Run completed!", fg="green", bold=True) | |
| overall = event.get("overall_score") | |
| if overall is not None: | |
| click.echo(f" Overall Score: {overall:.4f}") | |
| scores = event.get("scores", {}) | |
| if scores: | |
| click.echo(f" Scores: {format_scores(scores)}") | |
| elif etype == "error": | |
| view.cleanup() | |
| click.echo() | |
| click.secho(f" Run failed: {event.get('error')}", fg="red", bold=True) | |
| def _run_with_progress( | |
| runner, | |
| run_id, | |
| project_config, | |
| task_config, | |
| prompt, | |
| max_workers, | |
| limit, | |
| completed_results: list[dict[str, Any]] | None = None, | |
| selection=None, | |
| prepared=None, | |
| pipeline=None, | |
| ): | |
| """Execute a run with Rich progress bar display. | |
| `completed_results` ๋ resume ์ด ์ด๋ฏธ ๋๋ ์ํ์ ๋ค์ ๋๋ฆฌ์ง ์๊ฒ ํ๋ ๋ค๋ฆฌ๋ค -- | |
| ์ฌ๊ธฐ์ ์ ๋๊ธฐ๋ฉด runner ๊ฐ ํ๋ผ๋ฏธํฐ๋ฅผ ๊ฐ๊ณ ์์ด๋ ์์ฉ์ด ์๋ค(F2). `selection`ยท`prepared`ยท | |
| `pipeline` ์ CLI ๊ฐ run ์์ ์ ์ ์ ํ ํ ์ ํยท๋ฐ์ดํฐ์ ยทํ์ดํ๋ผ์ธ ์ธ์คํด์ค๋ค(D3ยทD6). | |
| `execute_run` ์ ์น๋ช ์ ์คํจ์์ raise ํ๋ฏ๋ก ์ด ํจ์๋ raise ํ๋ค -- | |
| ์ผ๋ถ๋ฌ ๊ฐ์ธ์ง ์๋๋ค. "error" ์ด๋ฒคํธ๊ฐ ์ด๋ฏธ ์ฌ๋์ด ์ฝ์ ๋ฉ์์ง๋ฅผ ์ฐ์๊ณ , | |
| ์ฌ๊ธฐ์ ๋ ๊ฐ์ธ๋ฉด ๊ฐ์ ๋ด์ฉ์ด ๋ ๋ฒ ๋์จ๋ค. | |
| """ | |
| view = _ProgressView() | |
| async def on_progress(event): | |
| etype = event.get("type") | |
| _on_inference(view, etype, event) | |
| _on_evaluation(view, etype, event) | |
| _on_finish(view, etype, event) | |
| try: | |
| asyncio.run( | |
| runner.execute_run( | |
| run_id=run_id, | |
| project_config=project_config, | |
| task_config=task_config, | |
| prompt=prompt, | |
| on_progress=on_progress, | |
| max_workers=max_workers, | |
| limit=limit, | |
| completed_results=completed_results, | |
| selection=selection, | |
| prepared=prepared, | |
| pipeline=pipeline, | |
| ) | |
| ) | |
| finally: | |
| # ๋ ์ด๋ฒคํธ ์์ด ๋๋๋(๋ฌ๋๊ฐ ์ด๋ฒคํธ ์ ์ ์ฃฝ์) ํฐ๋ฏธ๋ยท๋ก๊ฑฐ๋ฅผ ์๋๋๋ก ๋๋ฆฐ๋ค. | |
| view.cleanup() | |
| def execute_plan( | |
| plan, run_id: str, store, completed_results: list[dict[str, Any]] | None = None | |
| ) -> None: | |
| """์ค๋น๋ `RunPlan` ์ ๋ฌ๋๋ก ๋๋ฆฐ๋ค โ `runs start`ยท`runs resume` ๊ณต์ฉ. | |
| ๋ฌ๋๋ plan ์ด ์ฅ **๊ฐ์ ์ธ์คํด์ค**(์ค๋น๋ ๋ฐ์ดํฐ์ ยทํ์ดํ๋ผ์ธยทํ๋ก๋ฐ์ด๋)๋ก ๋๋ค(D6ยทD16). | |
| `limit` ์ `prepared.selection` ์์ ์์ผ๋ฏ๋ก ๋ฐ๋ก ๋๊ธฐ์ง ์๋๋ค. | |
| """ | |
| from solar_eval.core.runner import BatchRunner | |
| runner = BatchRunner( | |
| store=store, | |
| inference_provider=plan.provider, | |
| judge_provider=plan.judge_provider, | |
| dataset_loader=plan.dataset_loader, | |
| ) | |
| _run_with_progress( | |
| runner, | |
| run_id, | |
| plan.project_config, | |
| plan.task_config, | |
| plan.prompt, | |
| plan.max_workers, | |
| limit=None, | |
| completed_results=completed_results, | |
| prepared=plan.prepared, | |
| pipeline=plan.pipeline, | |
| ) | |
| # ๋ฌ๋๊ฐ raise ํ๋ฉด(FAILED) ์ฌ๊ธฐ ์ค์ง ์๋๋ค. ํ ์ ํธ์ถ์๊ฐ ์ด ํจ์ **๋ค์** ๋ถ๋ฅธ๋ค โ ์์ ๊ทธ๋๋ก. | |
| _record_failure_keys(plan, run_id, store) | |
| def _record_failure_keys(plan, run_id: str, store) -> None: | |
| """์คํจ ์ํ ์ง๋ฌธ์ run.json ์ ๋ํ๋ค(LP5 D-LP5-4). ๋ชป ํด๋ run ์ ๊ทธ๋๋ก ๋๋๋ค. | |
| ์คํจ ์ง๋ฌธ์ ์์ด๋ ๋๋ ์ ๋ณด๋ค โ ์์ผ๋ฉด wrap ์ด `failures.jsonl` ์ ์๋ต(๋ชจ๋ฆ)ํ๋ค. ๊ทธ๋์ ๋์ฐ๋ฏธ์ | |
| ์์ธ๋ ๊ฒฝ๊ณ ํ ์ค(์์ธ **ํ์ ๋ง** โ ๋ฉ์์ง๋ ์ ๋ ฅ ์กฐ๊ฐ์ ๋ด์ ์ ์๋ค)๋ก ๋๋ฌ๋ด๊ณ run ์ ๊ณ์ ๋๋ธ๋ค. | |
| `poc_bundle` ์ ์ฌ๊ธฐ์ ์ฒ์ import ํ๋ค โ CLI ๋ฅผ ๋์ฐ๋ ๊ฒ๋ง์ผ๋ก ๋ฒ๋ค ๊ณ์ธต์ ์ฃ์ง ์๋๋ค. | |
| """ | |
| from solar_eval.cli.commands import failure_keys | |
| try: | |
| failure_keys.record_failure_keys(store, run_id, plan.prepared, plan.task_config) | |
| except Exception as exc: # ์คํจ ์ง๋ฌธ์ ์ ํ ์ ๋ณด โ run ์ ์คํจ๋ก ๋ง๋ค์ง ์๋๋ค(๊ณํ ยง2.4) | |
| click.secho(f" failure keys not recorded ({type(exc).__name__})", fg="yellow") | |