peekabook-api / app /simulation /profile_visualizer.py
lael
feat: initial deploy
3960e2b
Raw
History Blame Contribute Delete
6.31 kB
"""
inspect_viz.py: ChromaDB ์‚ฌ์šฉ์ž ํ”„๋กœํŒŒ์ผ ๊ฒ€์‚ฌ ๋ฐ ํด๋Ÿฌ์Šคํ„ฐ ์‹œ๊ฐํ™”.
์›๋ณธ ๋…ธํŠธ๋ถ: peekareader_sim_multi_v3.ipynb (cell 13, 20)
ํ•จ์ˆ˜:
- inspect_profile_by_session: SQLite ์ง์ ‘ ์ ‘๊ทผ์œผ๋กœ ์„ธ์…˜๋ณ„ ํ”„๋กœํŒŒ์ผ dump
- visualize_profile_clusters: ์ž„๋ฒ ๋”ฉ์„ KMeans + PCA๋กœ 2D ์‹œ๊ฐํ™” ํ›„ PNG ์ €์žฅ
- log_clusters_to_wandb: (์„ ํƒ) wandb.Image๋กœ W&B์— push
"""
from __future__ import annotations
import json
import sqlite3
from pathlib import Path
from typing import Optional
def inspect_profile_by_session(chroma_path: str) -> list[dict]:
"""
Chroma SQLite๋ฅผ ์ง์ ‘ ์ฝ์–ด ์„ธ์…˜๋ณ„ ์‚ฌ์šฉ์ž ํ”„๋กœํŒŒ์ผ์„ ์ฝ˜์†”์— ์ถœ๋ ฅํ•˜๊ณ  list๋กœ ๋ฐ˜ํ™˜ํ•จ.
ํ”„๋กœํŒŒ์ผ์€ ChromaDB์˜ embedding_fulltext_search_content ํ…Œ์ด๋ธ”์— JSON ๋ฌธ์ž์—ด๋กœ ์ €์žฅ๋จ.
Args:
chroma_path: Chroma persist_directory ๊ฒฝ๋กœ (์˜ˆ: backend/chroma_db_runs/...)
Returns:
[{"session_seq": int, "profile": dict | str}, ...]
"""
db_file = Path(chroma_path) / "chroma.sqlite3"
if not db_file.exists():
print(f"[inspect ์Šคํ‚ต] Chroma DB ์—†์Œ: {db_file}")
return []
conn = sqlite3.connect(str(db_file))
cur = conn.cursor()
try:
cur.execute("SELECT id, c0 FROM embedding_fulltext_search_content ORDER BY id")
rows = cur.fetchall()
finally:
conn.close()
print(f"\n{'='*60}")
print(f"์„ธ์…˜๋ณ„ ์œ ์ € ํ”„๋กœํŒŒ์ผ ({len(rows)}๊ฐœ ์„ธ์…˜)")
print(f"chroma_path: {chroma_path}")
print(f"{'='*60}")
profiles: list[dict] = []
for seq_id, content in rows:
try:
data = json.loads(content)
profile = data.get("profile", {})
print(f"\n[์„ธ์…˜ {seq_id}]")
for k, v in profile.items():
print(f" {k}: {v}")
profiles.append({"session_seq": seq_id, "profile": profile})
except Exception:
print(f"\n[์„ธ์…˜ {seq_id}] {content[:100]}...")
profiles.append({"session_seq": seq_id, "profile": content})
return profiles
def visualize_profile_clusters(chroma_path: str,
persona_id: str,
collection_name: str = "user_profile_memory",
save_path: Optional[str] = None) -> Optional[str]:
"""
Chroma์— ์ €์žฅ๋œ ์„ธ์…˜๋ณ„ ํ”„๋กœํŒŒ์ผ ์ž„๋ฒ ๋”ฉ์„ KMeans + PCA๋กœ 2D ์‹œ๊ฐํ™”ํ•จ.
์—˜๋ณด์šฐ ๋ฐฉ๋ฒ•์œผ๋กœ ์ตœ์  k๋ฅผ ์ž๋™ ์„ ํƒ.
Args:
chroma_path: Chroma persist_directory
persona_id: ํ”Œ๋กฏ ์ œ๋ชฉ์šฉ
collection_name: ChromaDB collection ์ด๋ฆ„ (default 'user_profile_memory')
save_path: PNG ์ €์žฅ ๊ฒฝ๋กœ. None์ด๋ฉด chroma_path ์•ˆ์— ์ €์žฅํ•จ
Returns:
์ €์žฅ๋œ PNG ๊ฒฝ๋กœ (๋˜๋Š” ์ž„๋ฒ ๋”ฉ ์—†์œผ๋ฉด None)
"""
# heavy ์˜์กด์„ฑ์€ ํ•จ์ˆ˜ ์•ˆ์—์„œ import (์‚ฌ์šฉ์ž๊ฐ€ ์ด ํ•จ์ˆ˜ ์•ˆ ๋ถ€๋ฅด๋ฉด sklearn/matplotlib ์•ˆ ๊น”๋ ค ์žˆ์–ด๋„ OK)
import chromadb
import matplotlib.pyplot as plt
import numpy as np
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
client = chromadb.PersistentClient(path=chroma_path)
try:
collection = client.get_collection(collection_name)
except Exception as e:
print(f"[viz ์Šคํ‚ต] collection '{collection_name}' ์—†์Œ: {e}")
return None
result = collection.get(include=["embeddings", "documents"])
embeddings = result.get("embeddings")
documents = result.get("documents")
if embeddings is None or len(embeddings) == 0:
print("[viz ์Šคํ‚ต] ์ž„๋ฒ ๋”ฉ ์—†์Œ โ€” ์‹œ๋ฎฌ๋ ˆ์ด์…˜ ๋จผ์ € ์‹คํ–‰ ํ•„์š”")
return None
vectors = np.array(embeddings)
# 1) ๋ ˆ์ด๋ธ” ์ถ”์ถœ (์„ธ์…˜ ๋ฒˆํ˜ธ + preferred_genre ์•ž 10์ž)
labels = []
for i, doc in enumerate(documents):
try:
data = json.loads(doc)
genre = data.get("profile", {}).get("preferred_genre", f"์„ธ์…˜ {i+1}")
except Exception:
genre = f"์„ธ์…˜ {i+1}"
labels.append(f"S{i+1}: {genre[:10]}")
# 2) ์—˜๋ณด์šฐ ๋ฐฉ๋ฒ•์œผ๋กœ ์ตœ์  k ์ž๋™ ์„ ํƒ
n = len(vectors)
k_range = range(2, min(11, n))
if len(k_range) == 0:
print(f"[viz ์Šคํ‚ต] ์„ธ์…˜ ์ˆ˜ ๋ถ€์กฑ ({n}๊ฐœ)")
return None
inertias = []
for k in k_range:
km = KMeans(n_clusters=k, random_state=42, n_init=10)
km.fit(vectors)
inertias.append(km.inertia_)
if len(inertias) < 2:
best_k = k_range[0]
else:
diffs = [inertias[i] - inertias[i+1] for i in range(len(inertias)-1)]
best_k = list(k_range)[diffs.index(max(diffs)) + 1]
print(f"[์—˜๋ณด์šฐ ๋ฐฉ๋ฒ•] ์ตœ์  ํด๋Ÿฌ์Šคํ„ฐ ์ˆ˜: {best_k}")
# 3) ์ตœ์  k๋กœ ํด๋Ÿฌ์Šคํ„ฐ๋ง
km = KMeans(n_clusters=best_k, random_state=42, n_init=10)
cluster_ids = km.fit_predict(vectors)
# 4) PCA 2D ์ถ•์†Œ
pca = PCA(n_components=2)
vecs_2d = pca.fit_transform(vectors)
# 5) ์‹œ๊ฐํ™”
fig, ax = plt.subplots(figsize=(14, 9))
colors = plt.cm.tab10.colors
for i, (x, y) in enumerate(vecs_2d):
c = colors[cluster_ids[i] % len(colors)]
ax.scatter(x, y, color=c, s=120, zorder=3)
ax.annotate(
labels[i], (x, y),
fontsize=8, xytext=(6, 6), textcoords="offset points",
)
ax.set_title(
f"{persona_id} ์„ธ์…˜๋ณ„ ํ”„๋กœํŒŒ์ผ ํด๋Ÿฌ์Šคํ„ฐ๋ง\n"
f"(PCA + KMeans, k={best_k}, ์ด {n}์„ธ์…˜)",
fontsize=12,
)
ax.set_xlabel(f"PC1 (์„ค๋ช…๋ ฅ: {pca.explained_variance_ratio_[0]:.1%})")
ax.set_ylabel(f"PC2 (์„ค๋ช…๋ ฅ: {pca.explained_variance_ratio_[1]:.1%})")
plt.tight_layout()
if save_path is None:
save_path = str(Path(chroma_path) / f"cluster_{persona_id}.png")
Path(save_path).parent.mkdir(parents=True, exist_ok=True)
plt.savefig(save_path, dpi=150)
plt.close(fig)
print(f"[viz ์ €์žฅ] {save_path}")
return save_path
def log_clusters_to_wandb(png_path: str, key: str = "cluster_plot") -> None:
"""
ํด๋Ÿฌ์Šคํ„ฐ PNG๋ฅผ W&B์— wandb.Image๋กœ pushํ•จ.
wandb.run์ด None์ด๋ฉด no-op.
"""
import wandb
if wandb.run is None:
return
wandb.log({key: wandb.Image(png_path)})