proofread-demo / solar_eval /cli /commands /run_progress.py
dev-strender's picture
feat: switch demo to FT + Opus article review (pipeline_exp_261021_ft_hybrid)
3dfc93b verified
Raw History Blame Contribute Delete
9.23 kB
"""run ์‹คํ–‰ + ์ง„ํ–‰ ํ‘œ์‹œ (Rich progress bar) โ€” `runs start`ยท`runs resume` ์ด ๊ณต์œ ํ•œ๋‹ค.
`runs.py` ๊ฐ€ 800์ค„ ์ƒํ•œ์„ ๋„˜์–ด ๋–ผ์–ด ๋ƒˆ๋‹ค(๊ณ„ํš 2026-09-24-phase1a E1). ๋™์ž‘์€ ์˜ฎ๊ธฐ๊ธฐ ์ „๊ณผ ๊ฐ™๊ณ ,
๋Ÿฌ๋„ˆ์— ์„ ํƒยท์ค€๋น„๋œ ๋ฐ์ดํ„ฐ์…‹ยทํŒŒ์ดํ”„๋ผ์ธ ์ธ์Šคํ„ด์Šค๋ฅผ ๋„˜๊ธฐ๋Š” ์ธ์ž๋งŒ ๋”ํ–ˆ๋‹ค(D3ยทD6).
"""
from __future__ import annotations
import asyncio
import logging
from typing import Any
import click
from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
from solar_eval.cli.formatters import format_scores
#: ์ง„ํ–‰ ํ‘œ์‹œ ์œ„๋กœ ๋Œ์–ด์˜ฌ๋ ค ์ฐ๋Š” ๋กœ๊ฑฐ๋“ค.
_LOG_NAMES = (
"solar_eval.pipelines.pipeline",
"solar_eval.pipelines.steps.llm",
"solar_eval.core.runner",
"solar_eval.providers.upstage",
)
class _ProgressView:
"""Rich ์ง„ํ–‰ ํ‘œ์‹œ(๊ฐ€๋Šฅํ•  ๋•Œ)์™€ ํ‰๋ฌธ ์ถœ๋ ฅ ์‚ฌ์ด๋ฅผ ๊ฐ€๋ฅด๋Š” ์ž‘์€ ์ƒํƒœ ๋ฌถ์Œ."""
def __init__(self) -> None:
from rich.console import Console
from rich.live import Live
self.bar = Progress(
SpinnerColumn(),
TextColumn("[progress.description]{task.description}"),
BarColumn(),
TextColumn("{task.completed}/{task.total}"),
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
TimeElapsedColumn(),
)
self.live: Live | None = None
self.use_rich = True
self.inference_task_id: Any = None
self.eval_task_id: Any = None
self._handler: logging.Handler | None = None
self._closed = False
try:
self.live = Live(self.bar, console=Console(stderr=True), refresh_per_second=8)
self.live.start()
except Exception as e: # ํ„ฐ๋ฏธ๋„์ด Rich ๋ฅผ ๋ชป ๋ฐ›์œผ๋ฉด ํ‰๋ฌธ์œผ๋กœ โ€” ์›์ธ์€ ํ•œ ์ค„ ๋‚จ๊ธด๋‹ค.
click.secho(f" (์ง„ํ–‰ ํ‘œ์‹œ ์—†์ด ํ‰๋ฌธ์œผ๋กœ ์ถœ๋ ฅํ•ฉ๋‹ˆ๋‹ค: {e})", fg="yellow")
self.use_rich = False
if self.use_rich:
self._redirect_logs()
def _redirect_logs(self) -> None:
"""ํŒŒ์ดํ”„๋ผ์ธยท๋Ÿฌ๋„ˆ ๋กœ๊ทธ๋ฅผ ์ง„ํ–‰ ํ‘œ์‹œ ์œ„๋กœ ์˜ฌ๋ฆฐ๋‹ค."""
from rich.text import Text
live = self.live
class _LiveLogHandler(logging.Handler):
def emit(self, record):
if live:
live.console.print(Text(record.getMessage(), style="dim"))
self._handler = _LiveLogHandler()
for name in _LOG_NAMES:
lg = logging.getLogger(name)
lg.addHandler(self._handler)
lg.propagate = False
logging.getLogger("httpx").setLevel(logging.WARNING)
def cleanup(self) -> None:
"""์ง„ํ–‰ ํ‘œ์‹œ๋ฅผ ๋‹ซ๊ณ  ๋กœ๊ฑฐ๋ฅผ ๋˜๋Œ๋ฆฐ๋‹ค. ์—ฌ๋Ÿฌ ๋ฒˆ ๋ถˆ๋Ÿฌ๋„ ๋œ๋‹ค(๋ ์ด๋ฒคํŠธ + finally)."""
if self.use_rich and self.live is not None and not self._closed:
self._closed = True
self.live.stop()
for name in _LOG_NAMES:
logging.getLogger(name).removeHandler(self._handler)
logging.getLogger(name).propagate = True
def _on_inference(view: _ProgressView, etype: str, event: dict[str, Any]) -> None:
if etype == "dataset_loaded":
if view.use_rich:
view.inference_task_id = view.bar.add_task("Inference", total=event["total"])
else:
click.secho(f" Dataset loaded: {event['total']} samples", fg="blue")
elif etype == "inference_progress":
if view.use_rich and view.inference_task_id is not None:
view.bar.update(view.inference_task_id, completed=event["completed"])
elif not view.use_rich:
c, t = event["completed"], event["total"]
if c % 10 == 0 or c == t:
click.echo(f" Inference: {c}/{t}")
elif etype == "inference_complete":
if view.use_rich and view.inference_task_id is not None:
view.bar.update(view.inference_task_id, completed=event["total"])
else:
click.secho(f" Inference complete: {event['total']} samples", fg="green")
def _on_evaluation(view: _ProgressView, etype: str, event: dict[str, Any]) -> None:
if etype == "evaluation_start":
if view.use_rich:
view.eval_task_id = view.bar.add_task("Evaluation", total=event.get("total", 0) or 1)
else:
click.secho(" Starting evaluation...", fg="blue")
elif etype == "evaluation_progress" and view.use_rich and view.eval_task_id is not None:
view.bar.update(view.eval_task_id, completed=event["completed"])
def _on_finish(view: _ProgressView, etype: str, event: dict[str, Any]) -> None:
if etype == "done":
if view.use_rich and view.eval_task_id is not None:
view.bar.update(view.eval_task_id, completed=view.bar.tasks[view.eval_task_id].total)
view.cleanup()
click.echo()
click.secho("Run completed!", fg="green", bold=True)
overall = event.get("overall_score")
if overall is not None:
click.echo(f" Overall Score: {overall:.4f}")
scores = event.get("scores", {})
if scores:
click.echo(f" Scores: {format_scores(scores)}")
elif etype == "error":
view.cleanup()
click.echo()
click.secho(f" Run failed: {event.get('error')}", fg="red", bold=True)
def _run_with_progress(
runner,
run_id,
project_config,
task_config,
prompt,
max_workers,
limit,
completed_results: list[dict[str, Any]] | None = None,
selection=None,
prepared=None,
pipeline=None,
):
"""Execute a run with Rich progress bar display.
`completed_results` ๋Š” resume ์ด ์ด๋ฏธ ๋๋‚œ ์ƒ˜ํ”Œ์„ ๋‹ค์‹œ ๋Œ๋ฆฌ์ง€ ์•Š๊ฒŒ ํ•˜๋Š” ๋‹ค๋ฆฌ๋‹ค --
์—ฌ๊ธฐ์„œ ์•ˆ ๋„˜๊ธฐ๋ฉด runner ๊ฐ€ ํŒŒ๋ผ๋ฏธํ„ฐ๋ฅผ ๊ฐ–๊ณ  ์žˆ์–ด๋„ ์†Œ์šฉ์ด ์—†๋‹ค(F2). `selection`ยท`prepared`ยท
`pipeline` ์€ CLI ๊ฐ€ run ์‹œ์ž‘ ์ „์— ์ •ํ•œ ํ–‰ ์„ ํƒยท๋ฐ์ดํ„ฐ์…‹ยทํŒŒ์ดํ”„๋ผ์ธ ์ธ์Šคํ„ด์Šค๋‹ค(D3ยทD6).
`execute_run` ์€ ์น˜๋ช…์  ์‹คํŒจ์—์„œ raise ํ•˜๋ฏ€๋กœ ์ด ํ•จ์ˆ˜๋„ raise ํ•œ๋‹ค --
์ผ๋ถ€๋Ÿฌ ๊ฐ์‹ธ์ง€ ์•Š๋Š”๋‹ค. "error" ์ด๋ฒคํŠธ๊ฐ€ ์ด๋ฏธ ์‚ฌ๋žŒ์ด ์ฝ์„ ๋ฉ”์‹œ์ง€๋ฅผ ์ฐ์—ˆ๊ณ ,
์—ฌ๊ธฐ์„œ ๋˜ ๊ฐ์‹ธ๋ฉด ๊ฐ™์€ ๋‚ด์šฉ์ด ๋‘ ๋ฒˆ ๋‚˜์˜จ๋‹ค.
"""
view = _ProgressView()
async def on_progress(event):
etype = event.get("type")
_on_inference(view, etype, event)
_on_evaluation(view, etype, event)
_on_finish(view, etype, event)
try:
asyncio.run(
runner.execute_run(
run_id=run_id,
project_config=project_config,
task_config=task_config,
prompt=prompt,
on_progress=on_progress,
max_workers=max_workers,
limit=limit,
completed_results=completed_results,
selection=selection,
prepared=prepared,
pipeline=pipeline,
)
)
finally:
# ๋ ์ด๋ฒคํŠธ ์—†์ด ๋๋‚˜๋„(๋Ÿฌ๋„ˆ๊ฐ€ ์ด๋ฒคํŠธ ์ „์— ์ฃฝ์Œ) ํ„ฐ๋ฏธ๋„ยท๋กœ๊ฑฐ๋ฅผ ์›๋ž˜๋Œ€๋กœ ๋Œ๋ฆฐ๋‹ค.
view.cleanup()
def execute_plan(
plan, run_id: str, store, completed_results: list[dict[str, Any]] | None = None
) -> None:
"""์ค€๋น„๋œ `RunPlan` ์„ ๋Ÿฌ๋„ˆ๋กœ ๋Œ๋ฆฐ๋‹ค โ€” `runs start`ยท`runs resume` ๊ณต์šฉ.
๋Ÿฌ๋„ˆ๋Š” plan ์ด ์ฅ” **๊ฐ™์€ ์ธ์Šคํ„ด์Šค**(์ค€๋น„๋œ ๋ฐ์ดํ„ฐ์…‹ยทํŒŒ์ดํ”„๋ผ์ธยทํ”„๋กœ๋ฐ”์ด๋”)๋กœ ๋ˆ๋‹ค(D6ยทD16).
`limit` ์€ `prepared.selection` ์•ˆ์— ์žˆ์œผ๋ฏ€๋กœ ๋”ฐ๋กœ ๋„˜๊ธฐ์ง€ ์•Š๋Š”๋‹ค.
"""
from solar_eval.core.runner import BatchRunner
runner = BatchRunner(
store=store,
inference_provider=plan.provider,
judge_provider=plan.judge_provider,
dataset_loader=plan.dataset_loader,
)
_run_with_progress(
runner,
run_id,
plan.project_config,
plan.task_config,
plan.prompt,
plan.max_workers,
limit=None,
completed_results=completed_results,
prepared=plan.prepared,
pipeline=plan.pipeline,
)
# ๋Ÿฌ๋„ˆ๊ฐ€ raise ํ•˜๋ฉด(FAILED) ์—ฌ๊ธฐ ์˜ค์ง€ ์•Š๋Š”๋‹ค. ํ›…์€ ํ˜ธ์ถœ์ž๊ฐ€ ์ด ํ•จ์ˆ˜ **๋’ค์—** ๋ถ€๋ฅธ๋‹ค โ€” ์ˆœ์„œ ๊ทธ๋Œ€๋กœ.
_record_failure_keys(plan, run_id, store)
def _record_failure_keys(plan, run_id: str, store) -> None:
"""์‹คํŒจ ์ƒ˜ํ”Œ ์ง€๋ฌธ์„ run.json ์— ๋”ํ•œ๋‹ค(LP5 D-LP5-4). ๋ชป ํ•ด๋„ run ์€ ๊ทธ๋Œ€๋กœ ๋๋‚œ๋‹ค.
์‹คํŒจ ์ง€๋ฌธ์€ ์—†์–ด๋„ ๋˜๋Š” ์ •๋ณด๋‹ค โ€” ์—†์œผ๋ฉด wrap ์ด `failures.jsonl` ์„ ์ƒ๋žต(๋ชจ๋ฆ„)ํ•œ๋‹ค. ๊ทธ๋ž˜์„œ ๋„์šฐ๋ฏธ์˜
์˜ˆ์™ธ๋Š” ๊ฒฝ๊ณ  ํ•œ ์ค„(์˜ˆ์™ธ **ํƒ€์ž…๋งŒ** โ€” ๋ฉ”์‹œ์ง€๋Š” ์ž…๋ ฅ ์กฐ๊ฐ์„ ๋‹ด์„ ์ˆ˜ ์žˆ๋‹ค)๋กœ ๋“œ๋Ÿฌ๋‚ด๊ณ  run ์€ ๊ณ„์† ๋๋‚ธ๋‹ค.
`poc_bundle` ์€ ์—ฌ๊ธฐ์„œ ์ฒ˜์Œ import ํ•œ๋‹ค โ€” CLI ๋ฅผ ๋„์šฐ๋Š” ๊ฒƒ๋งŒ์œผ๋กœ ๋ฒˆ๋“ค ๊ณ„์ธต์„ ์‹ฃ์ง€ ์•Š๋Š”๋‹ค.
"""
from solar_eval.cli.commands import failure_keys
try:
failure_keys.record_failure_keys(store, run_id, plan.prepared, plan.task_config)
except Exception as exc: # ์‹คํŒจ ์ง€๋ฌธ์€ ์„ ํƒ ์ •๋ณด โ€” run ์„ ์‹คํŒจ๋กœ ๋งŒ๋“ค์ง€ ์•Š๋Š”๋‹ค(๊ณ„ํš ยง2.4)
click.secho(f" failure keys not recorded ({type(exc).__name__})", fg="yellow")