peekabook-api / app /simulation /multi_sim_logger.py
lael
feat: initial deploy
3960e2b
Raw
History Blame Contribute Delete
4.64 kB
"""
์‹œ๋ฎฌ๋ ˆ์ด์…˜ ๊ฒฐ๊ณผ ์ €์žฅ ๋ฐ ์š”์•ฝ ์ถœ๋ ฅ.
"""
from __future__ import annotations
import json
import os
from datetime import datetime
from pathlib import Path
from app.config import KST, LLM_MODEL, MAX_TURNS, SIMULATION_RESULTS_DIR
def save_simul_result(result: dict,
save_dir: str = SIMULATION_RESULTS_DIR) -> str:
"""
๋ฉ€ํ‹ฐ์„ธ์…˜ ์‹œ๋ฎฌ๋ ˆ์ด์…˜ ๊ฒฐ๊ณผ(multi_session_simulator ๋ฐ˜ํ™˜๊ฐ’)๋ฅผ JSON์œผ๋กœ ์ €์žฅํ•จ.
ํŒŒ์ผ๋ช…: simul_{persona_id}_{timestamp}.json
"""
Path(save_dir).mkdir(parents=True, exist_ok=True)
persona_id = result["persona_id"]
timestamp = datetime.now(tz=KST).strftime("%Y%m%d_%H%M%S")
filename = f"simul_{persona_id}_{timestamp}.json"
filepath = os.path.join(save_dir, filename)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(result, f, ensure_ascii=False, indent=2, default=str)
print(f"\n[์ €์žฅ ์™„๋ฃŒ] {filepath}")
return filepath
def save_results(results: list,
save_dir: str = SIMULATION_RESULTS_DIR,
filename: str = None) -> str:
"""
๋‹จ์ผ์„ธ์…˜ ๊ฒฐ๊ณผ list(run_session๋“ค์˜ list)๋ฅผ JSON์œผ๋กœ ์ €์žฅํ•จ.
metadata๋กœ model, max_turns, version ๋“ฑ์„ ํ•จ๊ป˜ ๊ธฐ๋ก.
"""
Path(save_dir).mkdir(parents=True, exist_ok=True)
if filename is None:
ts = datetime.now(tz=KST).strftime("%Y%m%d_%H%M%S")
filename = f"sim_results_{ts}.json"
output = {
"metadata": {
"version": "v2",
"simulated_at": datetime.now(tz=KST).isoformat(),
"llm_model": LLM_MODEL,
"max_turns": MAX_TURNS,
"note": "PeekaReader self-eval + PeekaJudge ๋…๋ฆฝํ‰๊ฐ€ (Claude Haiku 4.5).",
},
"results": results,
}
filepath = os.path.join(save_dir, filename)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(output, f, ensure_ascii=False, indent=2, default=str)
print(f"\n[์ €์žฅ ์™„๋ฃŒ] {filepath}")
return filepath
def print_summary(results: list) -> None:
"""
๋‹จ์ผ์„ธ์…˜ ๊ฒฐ๊ณผ list์—์„œ ํŽ˜๋ฅด์†Œ๋‚˜๋ณ„ ๋งค์น˜์œจ ์š”์•ฝ์„ ์ฝ˜์†”์— ์ถœ๋ ฅํ•จ.
๋ฉ€ํ‹ฐ์„ธ์…˜ ๊ฒฐ๊ณผ์˜ ๊ฒฝ์šฐ run_multi_session์—์„œ ์ด๋ฏธ ์ฝ˜์†”์— ์š”์•ฝ ์ถœ๋ ฅ๋จ โ€” ๋‹จ์ผ์„ธ์…˜ list ์šฉ.
"""
print("\n" + "=" * 60)
print("์‹œ๋ฎฌ๋ ˆ์ด์…˜ ๊ฒฐ๊ณผ ์š”์•ฝ")
print("=" * 60)
for r in results:
ev = r.get("self_evaluation") or r.get("evaluation") or {}
books = ev.get("books_evaluated", [])
matched = sum(1 for b in books if b.get("match"))
total = len(books)
print(f"\n[{r.get('persona_id', '?')}] "
f"{r.get('status', '?')} | "
f"{r.get('total_turns', 0)}ํ„ด | "
f"{r.get('response_time_sec', 0)}์ดˆ")
if books:
print(f" ๋„์„œ ์ผ์น˜: {matched}/{total}๊ถŒ ({matched / total:.0%})")
print(f" ํ‰๊ฐ€ ๋ชจ๋“œ: {r.get('eval_mode', '-')} "
f"(book_intro {r.get('book_intro_loaded', 0)}๊ถŒ)")
print(f" ์ดํ‰: {ev.get('overall_reason', '-')}")
elif r.get("status") == "timeout":
print(f" ์Šฌ๋กฏ ์ฑ„์šฐ๊ธฐ {MAX_TURNS}ํ„ด ์ดˆ๊ณผ โ€” ์ถ”์ฒœ ๋„๋‹ฌ ๋ชป ํ•จ")
elif str(r.get("status", "")).startswith("error"):
print(f" ์˜ค๋ฅ˜: {r['status']}")
def print_multi_session_summary(results: list) -> None:
"""
์—ฌ๋Ÿฌ ํŽ˜๋ฅด์†Œ๋‚˜์˜ run_multi_session ๊ฒฐ๊ณผ list ์š”์•ฝ ์ถœ๋ ฅ.
๊ฐ ํŽ˜๋ฅด์†Œ๋‚˜์˜ ํ‰๊ท  judge_match_rate, final verdict ๋ถ„ํฌ ๋“ฑ.
"""
print("\n" + "=" * 60)
print("๋ฉ€ํ‹ฐ์„ธ์…˜ ์‹œ๋ฎฌ๋ ˆ์ด์…˜ ์ข…ํ•ฉ ์š”์•ฝ")
print("=" * 60)
for r in results:
pid = r.get("persona_id", "?")
sessions = r.get("sessions", [])
if not sessions:
print(f"\n[{pid}] ์„ธ์…˜ ์—†์Œ")
continue
judge_rates = [
s.get("judge_match_rate", 0.0)
for s in sessions
if s.get("judge_match_rate") is not None
]
avg_judge = sum(judge_rates) / len(judge_rates) if judge_rates else 0.0
verdicts = [s.get("verdict") for s in sessions if s.get("verdict")]
verdict_counts: dict = {}
for v in verdicts:
verdict_counts[v] = verdict_counts.get(v, 0) + 1
successful = sum(1 for s in sessions if s.get("status") == "success")
print(f"\n[{pid}] {successful}/{len(sessions)}์„ธ์…˜ ์„ฑ๊ณต")
print(f" ํ‰๊ท  judge match_rate: {avg_judge:.0%}")
print(f" verdict ๋ถ„ํฌ: {dict(sorted(verdict_counts.items()))}")