Spaces:
Running
Running
| """ | |
| ์๋ฎฌ๋ ์ด์ ๊ฒฐ๊ณผ ์ ์ฅ ๋ฐ ์์ฝ ์ถ๋ ฅ. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| from datetime import datetime | |
| from pathlib import Path | |
| from app.config import KST, LLM_MODEL, MAX_TURNS, SIMULATION_RESULTS_DIR | |
| def save_simul_result(result: dict, | |
| save_dir: str = SIMULATION_RESULTS_DIR) -> str: | |
| """ | |
| ๋ฉํฐ์ธ์ ์๋ฎฌ๋ ์ด์ ๊ฒฐ๊ณผ(multi_session_simulator ๋ฐํ๊ฐ)๋ฅผ JSON์ผ๋ก ์ ์ฅํจ. | |
| ํ์ผ๋ช : simul_{persona_id}_{timestamp}.json | |
| """ | |
| Path(save_dir).mkdir(parents=True, exist_ok=True) | |
| persona_id = result["persona_id"] | |
| timestamp = datetime.now(tz=KST).strftime("%Y%m%d_%H%M%S") | |
| filename = f"simul_{persona_id}_{timestamp}.json" | |
| filepath = os.path.join(save_dir, filename) | |
| with open(filepath, "w", encoding="utf-8") as f: | |
| json.dump(result, f, ensure_ascii=False, indent=2, default=str) | |
| print(f"\n[์ ์ฅ ์๋ฃ] {filepath}") | |
| return filepath | |
| def save_results(results: list, | |
| save_dir: str = SIMULATION_RESULTS_DIR, | |
| filename: str = None) -> str: | |
| """ | |
| ๋จ์ผ์ธ์ ๊ฒฐ๊ณผ list(run_session๋ค์ list)๋ฅผ JSON์ผ๋ก ์ ์ฅํจ. | |
| metadata๋ก model, max_turns, version ๋ฑ์ ํจ๊ป ๊ธฐ๋ก. | |
| """ | |
| Path(save_dir).mkdir(parents=True, exist_ok=True) | |
| if filename is None: | |
| ts = datetime.now(tz=KST).strftime("%Y%m%d_%H%M%S") | |
| filename = f"sim_results_{ts}.json" | |
| output = { | |
| "metadata": { | |
| "version": "v2", | |
| "simulated_at": datetime.now(tz=KST).isoformat(), | |
| "llm_model": LLM_MODEL, | |
| "max_turns": MAX_TURNS, | |
| "note": "PeekaReader self-eval + PeekaJudge ๋ ๋ฆฝํ๊ฐ (Claude Haiku 4.5).", | |
| }, | |
| "results": results, | |
| } | |
| filepath = os.path.join(save_dir, filename) | |
| with open(filepath, "w", encoding="utf-8") as f: | |
| json.dump(output, f, ensure_ascii=False, indent=2, default=str) | |
| print(f"\n[์ ์ฅ ์๋ฃ] {filepath}") | |
| return filepath | |
| def print_summary(results: list) -> None: | |
| """ | |
| ๋จ์ผ์ธ์ ๊ฒฐ๊ณผ list์์ ํ๋ฅด์๋๋ณ ๋งค์น์จ ์์ฝ์ ์ฝ์์ ์ถ๋ ฅํจ. | |
| ๋ฉํฐ์ธ์ ๊ฒฐ๊ณผ์ ๊ฒฝ์ฐ run_multi_session์์ ์ด๋ฏธ ์ฝ์์ ์์ฝ ์ถ๋ ฅ๋จ โ ๋จ์ผ์ธ์ list ์ฉ. | |
| """ | |
| print("\n" + "=" * 60) | |
| print("์๋ฎฌ๋ ์ด์ ๊ฒฐ๊ณผ ์์ฝ") | |
| print("=" * 60) | |
| for r in results: | |
| ev = r.get("self_evaluation") or r.get("evaluation") or {} | |
| books = ev.get("books_evaluated", []) | |
| matched = sum(1 for b in books if b.get("match")) | |
| total = len(books) | |
| print(f"\n[{r.get('persona_id', '?')}] " | |
| f"{r.get('status', '?')} | " | |
| f"{r.get('total_turns', 0)}ํด | " | |
| f"{r.get('response_time_sec', 0)}์ด") | |
| if books: | |
| print(f" ๋์ ์ผ์น: {matched}/{total}๊ถ ({matched / total:.0%})") | |
| print(f" ํ๊ฐ ๋ชจ๋: {r.get('eval_mode', '-')} " | |
| f"(book_intro {r.get('book_intro_loaded', 0)}๊ถ)") | |
| print(f" ์ดํ: {ev.get('overall_reason', '-')}") | |
| elif r.get("status") == "timeout": | |
| print(f" ์ฌ๋กฏ ์ฑ์ฐ๊ธฐ {MAX_TURNS}ํด ์ด๊ณผ โ ์ถ์ฒ ๋๋ฌ ๋ชป ํจ") | |
| elif str(r.get("status", "")).startswith("error"): | |
| print(f" ์ค๋ฅ: {r['status']}") | |
| def print_multi_session_summary(results: list) -> None: | |
| """ | |
| ์ฌ๋ฌ ํ๋ฅด์๋์ run_multi_session ๊ฒฐ๊ณผ list ์์ฝ ์ถ๋ ฅ. | |
| ๊ฐ ํ๋ฅด์๋์ ํ๊ท judge_match_rate, final verdict ๋ถํฌ ๋ฑ. | |
| """ | |
| print("\n" + "=" * 60) | |
| print("๋ฉํฐ์ธ์ ์๋ฎฌ๋ ์ด์ ์ข ํฉ ์์ฝ") | |
| print("=" * 60) | |
| for r in results: | |
| pid = r.get("persona_id", "?") | |
| sessions = r.get("sessions", []) | |
| if not sessions: | |
| print(f"\n[{pid}] ์ธ์ ์์") | |
| continue | |
| judge_rates = [ | |
| s.get("judge_match_rate", 0.0) | |
| for s in sessions | |
| if s.get("judge_match_rate") is not None | |
| ] | |
| avg_judge = sum(judge_rates) / len(judge_rates) if judge_rates else 0.0 | |
| verdicts = [s.get("verdict") for s in sessions if s.get("verdict")] | |
| verdict_counts: dict = {} | |
| for v in verdicts: | |
| verdict_counts[v] = verdict_counts.get(v, 0) + 1 | |
| successful = sum(1 for s in sessions if s.get("status") == "success") | |
| print(f"\n[{pid}] {successful}/{len(sessions)}์ธ์ ์ฑ๊ณต") | |
| print(f" ํ๊ท judge match_rate: {avg_judge:.0%}") | |
| print(f" verdict ๋ถํฌ: {dict(sorted(verdict_counts.items()))}") | |