Spaces:
Running
Running
| """ | |
| inspect_viz.py: ChromaDB ์ฌ์ฉ์ ํ๋กํ์ผ ๊ฒ์ฌ ๋ฐ ํด๋ฌ์คํฐ ์๊ฐํ. | |
| ์๋ณธ ๋ ธํธ๋ถ: peekareader_sim_multi_v3.ipynb (cell 13, 20) | |
| ํจ์: | |
| - inspect_profile_by_session: SQLite ์ง์ ์ ๊ทผ์ผ๋ก ์ธ์ ๋ณ ํ๋กํ์ผ dump | |
| - visualize_profile_clusters: ์๋ฒ ๋ฉ์ KMeans + PCA๋ก 2D ์๊ฐํ ํ PNG ์ ์ฅ | |
| - log_clusters_to_wandb: (์ ํ) wandb.Image๋ก W&B์ push | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import sqlite3 | |
| from pathlib import Path | |
| from typing import Optional | |
| def inspect_profile_by_session(chroma_path: str) -> list[dict]: | |
| """ | |
| Chroma SQLite๋ฅผ ์ง์ ์ฝ์ด ์ธ์ ๋ณ ์ฌ์ฉ์ ํ๋กํ์ผ์ ์ฝ์์ ์ถ๋ ฅํ๊ณ list๋ก ๋ฐํํจ. | |
| ํ๋กํ์ผ์ ChromaDB์ embedding_fulltext_search_content ํ ์ด๋ธ์ JSON ๋ฌธ์์ด๋ก ์ ์ฅ๋จ. | |
| Args: | |
| chroma_path: Chroma persist_directory ๊ฒฝ๋ก (์: backend/chroma_db_runs/...) | |
| Returns: | |
| [{"session_seq": int, "profile": dict | str}, ...] | |
| """ | |
| db_file = Path(chroma_path) / "chroma.sqlite3" | |
| if not db_file.exists(): | |
| print(f"[inspect ์คํต] Chroma DB ์์: {db_file}") | |
| return [] | |
| conn = sqlite3.connect(str(db_file)) | |
| cur = conn.cursor() | |
| try: | |
| cur.execute("SELECT id, c0 FROM embedding_fulltext_search_content ORDER BY id") | |
| rows = cur.fetchall() | |
| finally: | |
| conn.close() | |
| print(f"\n{'='*60}") | |
| print(f"์ธ์ ๋ณ ์ ์ ํ๋กํ์ผ ({len(rows)}๊ฐ ์ธ์ )") | |
| print(f"chroma_path: {chroma_path}") | |
| print(f"{'='*60}") | |
| profiles: list[dict] = [] | |
| for seq_id, content in rows: | |
| try: | |
| data = json.loads(content) | |
| profile = data.get("profile", {}) | |
| print(f"\n[์ธ์ {seq_id}]") | |
| for k, v in profile.items(): | |
| print(f" {k}: {v}") | |
| profiles.append({"session_seq": seq_id, "profile": profile}) | |
| except Exception: | |
| print(f"\n[์ธ์ {seq_id}] {content[:100]}...") | |
| profiles.append({"session_seq": seq_id, "profile": content}) | |
| return profiles | |
| def visualize_profile_clusters(chroma_path: str, | |
| persona_id: str, | |
| collection_name: str = "user_profile_memory", | |
| save_path: Optional[str] = None) -> Optional[str]: | |
| """ | |
| Chroma์ ์ ์ฅ๋ ์ธ์ ๋ณ ํ๋กํ์ผ ์๋ฒ ๋ฉ์ KMeans + PCA๋ก 2D ์๊ฐํํจ. | |
| ์๋ณด์ฐ ๋ฐฉ๋ฒ์ผ๋ก ์ต์ k๋ฅผ ์๋ ์ ํ. | |
| Args: | |
| chroma_path: Chroma persist_directory | |
| persona_id: ํ๋กฏ ์ ๋ชฉ์ฉ | |
| collection_name: ChromaDB collection ์ด๋ฆ (default 'user_profile_memory') | |
| save_path: PNG ์ ์ฅ ๊ฒฝ๋ก. None์ด๋ฉด chroma_path ์์ ์ ์ฅํจ | |
| Returns: | |
| ์ ์ฅ๋ PNG ๊ฒฝ๋ก (๋๋ ์๋ฒ ๋ฉ ์์ผ๋ฉด None) | |
| """ | |
| # heavy ์์กด์ฑ์ ํจ์ ์์์ import (์ฌ์ฉ์๊ฐ ์ด ํจ์ ์ ๋ถ๋ฅด๋ฉด sklearn/matplotlib ์ ๊น๋ ค ์์ด๋ OK) | |
| import chromadb | |
| import matplotlib.pyplot as plt | |
| import numpy as np | |
| from sklearn.cluster import KMeans | |
| from sklearn.decomposition import PCA | |
| client = chromadb.PersistentClient(path=chroma_path) | |
| try: | |
| collection = client.get_collection(collection_name) | |
| except Exception as e: | |
| print(f"[viz ์คํต] collection '{collection_name}' ์์: {e}") | |
| return None | |
| result = collection.get(include=["embeddings", "documents"]) | |
| embeddings = result.get("embeddings") | |
| documents = result.get("documents") | |
| if embeddings is None or len(embeddings) == 0: | |
| print("[viz ์คํต] ์๋ฒ ๋ฉ ์์ โ ์๋ฎฌ๋ ์ด์ ๋จผ์ ์คํ ํ์") | |
| return None | |
| vectors = np.array(embeddings) | |
| # 1) ๋ ์ด๋ธ ์ถ์ถ (์ธ์ ๋ฒํธ + preferred_genre ์ 10์) | |
| labels = [] | |
| for i, doc in enumerate(documents): | |
| try: | |
| data = json.loads(doc) | |
| genre = data.get("profile", {}).get("preferred_genre", f"์ธ์ {i+1}") | |
| except Exception: | |
| genre = f"์ธ์ {i+1}" | |
| labels.append(f"S{i+1}: {genre[:10]}") | |
| # 2) ์๋ณด์ฐ ๋ฐฉ๋ฒ์ผ๋ก ์ต์ k ์๋ ์ ํ | |
| n = len(vectors) | |
| k_range = range(2, min(11, n)) | |
| if len(k_range) == 0: | |
| print(f"[viz ์คํต] ์ธ์ ์ ๋ถ์กฑ ({n}๊ฐ)") | |
| return None | |
| inertias = [] | |
| for k in k_range: | |
| km = KMeans(n_clusters=k, random_state=42, n_init=10) | |
| km.fit(vectors) | |
| inertias.append(km.inertia_) | |
| if len(inertias) < 2: | |
| best_k = k_range[0] | |
| else: | |
| diffs = [inertias[i] - inertias[i+1] for i in range(len(inertias)-1)] | |
| best_k = list(k_range)[diffs.index(max(diffs)) + 1] | |
| print(f"[์๋ณด์ฐ ๋ฐฉ๋ฒ] ์ต์ ํด๋ฌ์คํฐ ์: {best_k}") | |
| # 3) ์ต์ k๋ก ํด๋ฌ์คํฐ๋ง | |
| km = KMeans(n_clusters=best_k, random_state=42, n_init=10) | |
| cluster_ids = km.fit_predict(vectors) | |
| # 4) PCA 2D ์ถ์ | |
| pca = PCA(n_components=2) | |
| vecs_2d = pca.fit_transform(vectors) | |
| # 5) ์๊ฐํ | |
| fig, ax = plt.subplots(figsize=(14, 9)) | |
| colors = plt.cm.tab10.colors | |
| for i, (x, y) in enumerate(vecs_2d): | |
| c = colors[cluster_ids[i] % len(colors)] | |
| ax.scatter(x, y, color=c, s=120, zorder=3) | |
| ax.annotate( | |
| labels[i], (x, y), | |
| fontsize=8, xytext=(6, 6), textcoords="offset points", | |
| ) | |
| ax.set_title( | |
| f"{persona_id} ์ธ์ ๋ณ ํ๋กํ์ผ ํด๋ฌ์คํฐ๋ง\n" | |
| f"(PCA + KMeans, k={best_k}, ์ด {n}์ธ์ )", | |
| fontsize=12, | |
| ) | |
| ax.set_xlabel(f"PC1 (์ค๋ช ๋ ฅ: {pca.explained_variance_ratio_[0]:.1%})") | |
| ax.set_ylabel(f"PC2 (์ค๋ช ๋ ฅ: {pca.explained_variance_ratio_[1]:.1%})") | |
| plt.tight_layout() | |
| if save_path is None: | |
| save_path = str(Path(chroma_path) / f"cluster_{persona_id}.png") | |
| Path(save_path).parent.mkdir(parents=True, exist_ok=True) | |
| plt.savefig(save_path, dpi=150) | |
| plt.close(fig) | |
| print(f"[viz ์ ์ฅ] {save_path}") | |
| return save_path | |
| def log_clusters_to_wandb(png_path: str, key: str = "cluster_plot") -> None: | |
| """ | |
| ํด๋ฌ์คํฐ PNG๋ฅผ W&B์ wandb.Image๋ก pushํจ. | |
| wandb.run์ด None์ด๋ฉด no-op. | |
| """ | |
| import wandb | |
| if wandb.run is None: | |
| return | |
| wandb.log({key: wandb.Image(png_path)}) | |