Initial commit: NBS Persona Survey System

.gitattributes

@@ -0,0 +1 @@
+*.parquet filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,15 @@
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.env
+.venv
+venv/
+ENV/
+.DS_Store
+*.log
+survey_results_*.json
+전국지표조사_원본/
+전국지표조사_json/
+*.md~
+.ipynb_checkpoints/

Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+# 시스템 패키지 설치
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+# Python 패키지 설치
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# 애플리케이션 파일 복사
+COPY app.py .
+COPY rag_engine.py .
+COPY avatar_synthetic.py .
+COPY index.html .
+COPY consolidated_nbs_data.parquet .
+COPY nbs_questions_index.parquet .
+
+# 포트 노출
+EXPOSE 7860
+
+# FastAPI 실행
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -0,0 +1,67 @@
+---
+title: NBS Persona Survey
+emoji: 🎭
+colorFrom: blue
+colorTo: purple
+sdk: docker
+pinned: false
+---
+
+# NBS 페르소나 설문조사 시스템
+
+전국지표조사(NBS) 데이터 기반 페르소나 아바타 설문 시뮬레이션 API
+
+## 기능
+
+- **실제 응답자 모드**: 16만 건의 실제 응답자 데이터 기반 시뮬레이션
+- **통계 기반 모드**: 특정 그룹의 통계적 경향성을 반영한 시뮬레이션
+- **RAG 기반 응답 생성**: 과거 응답 이력을 참조하여 일관성 있는 답변 생성
+
+## 데이터
+
+- 총 응답자: 166,721명
+- 설문 회차: 163회 (2020년~현재)
+- 고유 질문: 1,219개
+
+## API 사용법
+
+### Health Check
+```bash
+GET /health
+```
+
+### 실제 응답자 시뮬레이션
+```bash
+POST /simulate/actual
+{
+  "question": "정년연장에 대해 어떻게 생각하십니까?",
+  "gender": "남자",
+  "age": "31~40",
+  "region": "서울",
+  "job": "사무/기술직",
+  "sample": 5
+}
+```
+
+### 통계 기반 시뮬레이션
+```bash
+POST /simulate/synthetic
+{
+  "question": "기본소득제 도입에 대한 의견은?",
+  "gender": "여자",
+  "age": "20~29",
+  "region": "경기",
+  "sample": 3
+}
+```
+
+## 기술 스택
+
+- FastAPI
+- Sentence Transformers (KR-SBERT)
+- Google Gemini 2.5 Flash
+- Pandas + PyArrow (Parquet)
+
+## 라이선스
+
+데이터는 전국지표조사(NBS) 원본 데이터를 기반으로 합니다.

app.py

@@ -0,0 +1,127 @@
+
+import os
+from fastapi import FastAPI, HTTPException, Header
+from fastapi.responses import FileResponse
+from fastapi.staticfiles import StaticFiles
+from pydantic import BaseModel
+from typing import Optional, List
+from rag_engine import NBSRagEngine
+import pandas as pd
+import json
+
+# Import helpers from existing scripts if possible, or redefine for standalone stability
+from avatar_synthetic import get_statistical_context
+
+app = FastAPI(title="NBS Persona Survey API", description="전국지표조사(NBS) 기반 페르소나 아바타 설문 시뮬레이션 API")
+
+# Initialize Engine (Singleton-like)
+PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
+PARQUET_PATH = os.path.join(PROJECT_DIR, "consolidated_nbs_data.parquet")
+INDEX_PATH = os.path.join(PROJECT_DIR, "nbs_questions_index.parquet")
+MODEL_NAME = "snunlp/KR-SBERT-V40K-klueNLI-augSTS"
+
+engine = NBSRagEngine(PARQUET_PATH, INDEX_PATH, MODEL_NAME)
+
+@app.get("/")
+async def get_index():
+    return FileResponse(os.path.join(PROJECT_DIR, "index.html"))
+
+class SurveyRequest(BaseModel):
+    question: str
+    gender: Optional[str] = None
+    age: Optional[str] = None
+    region: Optional[str] = None
+    job: Optional[str] = None
+    sample: int = 5
+
+@app.post("/simulate/actual")
+async def simulate_actual(req: SurveyRequest, x_api_key: Optional[str] = Header(None)):
+    """실제 응답자 기반 시뮬레이션 (Actual Mode)"""
+    try:
+        # Restriction for simulation speed and stability
+        effective_sample = min(req.sample, 10)
+        
+        # 1. Filter candidates
+        candidates = engine.filter_respondents(
+            gender=req.gender,
+            age=req.age,
+            region=req.region,
+            job=req.job
+        )
+        
+        if len(candidates) == 0:
+            raise HTTPException(status_code=404, detail="지정한 조건에 맞는 응답자가 없습니다. 조건을 완화해 주세요.")
+
+        # 2. Sample
+        sample_size = min(len(candidates), effective_sample)
+        sample = candidates.sample(n=sample_size)
+
+        # 3. Predict
+        sim_qs = engine.find_similar_questions(req.question, top_k=5)
+        persona_desc = f"{req.region or '전국'} 거주, {req.age or '전연령'}, {req.gender or '성별무관'}, 직업: {req.job or '직업무관'}"
+        
+        results = []
+        for idx, row in sample.iterrows():
+            context = engine.get_context_for_responder(row, sim_qs)
+            response = engine.generate_response(persona_desc, context, req.question, api_key=x_api_key)
+            results.append({
+                "respondent_id": str(row.get('id', idx)),
+                "survey_round": int(row.get('survey_round', 0)),
+                "demographics": {
+                    "gender": row.get('gender'),
+                    "region": row.get('region'),
+                    "age": int(row.get('age')) if pd.notna(row.get('age')) else None,
+                    "job": row.get('job')
+                },
+                "referenced_context": context.split('\n\n'),
+                "response": response
+            })
+        return results
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/simulate/synthetic")
+async def simulate_synthetic(req: SurveyRequest, x_api_key: Optional[str] = Header(None)):
+    """가상 통계 기반 시뮬레이션 (Synthetic Mode)"""
+    try:
+        # 1. Get statistical base
+        candidates = engine.filter_respondents(
+            gender=req.gender,
+            age=req.age,
+            region=req.region,
+            job=req.job
+        )
+        
+        if len(candidates) == 0:
+            raise HTTPException(status_code=404, detail="지정한 조건에 맞는 응답자가 없습니다.")
+
+        # 2. Extract stats
+        sim_qs = engine.find_similar_questions(req.question, top_k=5)
+        stat_context = get_statistical_context(candidates, sim_qs)
+        persona_desc = f"{req.region or '전국'} 거주, {req.age or '전연령대'}, {req.gender or '성별무관'}, 직업: {req.job or '직업무관'} 그룹의 통계적 평균 모델"
+        
+        results = []
+        for i in range(req.sample):
+            response = engine.generate_response(persona_desc, stat_context, req.question, api_key=x_api_key)
+            results.append({
+                "avatar_id": f"syn_{i}",
+                "demographics": {
+                    "gender": req.gender,
+                    "age": req.age,
+                    "region": req.region,
+                    "job": req.job
+                },
+                "referenced_stat_context": stat_context.split('\n\n'),
+                "response": response
+            })
+        return results
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/health")
+async def health_check():
+    return {"status": "healthy", "total_records": len(engine.df)}
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)

avatar_synthetic.py

@@ -0,0 +1,96 @@
+
+import os
+import pandas as pd
+from rag_engine import NBSRagEngine
+from tqdm import tqdm
+import argparse
+import json
+
+def get_statistical_context(candidates, sim_qs):
+    context_lines = []
+    
+    # 1. Broad Persona Statistics
+    persona_keywords = ['정치', '성향', '이념', '지지', '후보', '경제', '부동산']
+    persona_cols = [c for c in candidates.columns if any(k in c for k in persona_keywords)]
+    
+    # Pick top 5 most 'filled' persona columns to avoid noise
+    persona_cols = sorted(persona_cols, key=lambda c: candidates[c].count(), reverse=True)[:5]
+    
+    all_qs = list(persona_cols)
+    for q in sim_qs:
+        if q not in all_qs:
+            all_qs.append(q)
+
+    for q in all_qs:
+        if q in candidates.columns:
+            # Calculate distribution
+            dist = candidates[q].value_counts(normalize=True).head(5) # Top 5 answers
+            dist_str = ", ".join([f"'{ans}' ({pct:.1%})" for ans, pct in dist.items()])
+            context_lines.append(f"과거 데이터 통계 ({q}): {dist_str}")
+    return "\n\n".join(context_lines)
+
+def run_synthetic_avatar_survey(target_question, demographics, top_k_context=5, num_avatars=1000):
+    project_dir = os.path.dirname(os.path.abspath(__file__))
+    parquet = os.path.join(project_dir, "consolidated_nbs_data.parquet")
+    db = os.path.join(project_dir, "nbs_questions_index.parquet")
+    model = "snunlp/KR-SBERT-V40K-klueNLI-augSTS"
+    
+    engine = NBSRagEngine(parquet, db, model)
+    
+    # 1. Get statistical base from candidates
+    candidates = engine.filter_respondents(
+        gender=demographics.get('gender'),
+        age=demographics.get('age'),
+        region=demographics.get('region'),
+        job=demographics.get('job')
+    )
+    
+    if len(candidates) == 0:
+        print("No candidates found for these demographics.")
+        return []
+
+    # 2. Find context questions
+    sim_qs = engine.find_similar_questions(target_question, top_k=top_k_context)
+    stat_context = get_statistical_context(candidates, sim_qs)
+
+    persona_desc = f"{demographics.get('region', '전국')} 거주, {demographics.get('age', '전연령대')}, {demographics.get('gender', '성별무관')}, 직업: {demographics.get('job', '직업무관')} 그룹의 통계적 평균 모델"
+    
+    print(f"Generating {num_avatars} synthetic responses based on group statistics...")
+    results = []
+    
+    for i in tqdm(range(num_avatars)):
+        response = engine.generate_response(persona_desc, stat_context, target_question)
+        results.append({
+            "avatar_id": f"syn_{i}",
+            "demographics": demographics,
+            "referenced_stat_context": stat_context.split('\n\n'),
+            "response": response
+        })
+
+    return results
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--question", type=str, required=True)
+    parser.add_argument("--gender", type=str, default=None)
+    parser.add_argument("--age", type=str, default=None, help="나이 또는 나이 범위 (예: 20 또는 21~29)")
+    parser.add_argument("--region", type=str, default=None)
+    parser.add_argument("--job", type=str, default=None)
+    parser.add_argument("--sample", type=int, default=10, help="Number of synthetic avatars to generate")
+    
+    args = parser.parse_args()
+    
+    demos = {
+        'gender': args.gender,
+        'age': args.age,
+        'region': args.region,
+        'job': args.job
+    }
+    
+    survey_results = run_synthetic_avatar_survey(args.question, demos, num_avatars=args.sample)
+    
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    output_f = os.path.join(current_dir, "survey_results_synthetic.json")
+    with open(output_f, "w", encoding="utf-8") as f:
+        json.dump(survey_results, f, ensure_ascii=False, indent=4)
+    print(f"Synthetic simulation finished. Results saved to {output_f}")

consolidated_nbs_data.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ae6e667dc62aa831baa866b4de965a75bc3bca6468a4a32d0b669a2e1d52322
+size 6986110

index.html

@@ -0,0 +1,736 @@
+<!DOCTYPE html>
+<html lang="ko">
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Persona Issue Survey(Beta version) - NBS Avatar System</title>
+    <link href="https://fonts.googleapis.com/css2?family=Pretendard:wght@400;500;700&display=swap" rel="stylesheet">
+    <style>
+        :root {
+            --primary: #6366f1;
+            --primary-dark: #4f46e5;
+            --secondary: #ec4899;
+            --bg: #f8fafc;
+            --sidebar-bg: #0f172a;
+            --card-bg: #ffffff;
+            --text-main: #1e293b;
+            --text-muted: #64748b;
+            --glass: rgba(255, 255, 255, 0.7);
+        }
+
+        * {
+            margin: 0;
+            padding: 0;
+            box-sizing: border-box;
+            font-family: 'Pretendard', sans-serif;
+        }
+
+        body {
+            background-color: var(--bg);
+            color: var(--text-main);
+            display: flex;
+            min-height: 100vh;
+        }
+
+        /* Sidebar Styling */
+        .sidebar {
+            width: 320px;
+            background-color: var(--sidebar-bg);
+            color: white;
+            padding: 2.5rem 1.5rem;
+            display: flex;
+            flex-direction: column;
+            position: fixed;
+            height: 100vh;
+            left: 0;
+            top: 0;
+            box-shadow: 4px 0 20px rgba(0, 0, 0, 0.1);
+            z-index: 100;
+        }
+
+        .logo {
+            font-size: 1.5rem;
+            font-weight: 700;
+            margin-bottom: 3rem;
+            background: linear-gradient(to right, #818cf8, #f472b6);
+            -webkit-background-clip: text;
+            -webkit-text-fill-color: transparent;
+            display: flex;
+            align-items: center;
+            gap: 10px;
+        }
+
+        .menu-item {
+            padding: 1rem 1.25rem;
+            border-radius: 12px;
+            margin-bottom: 0.5rem;
+            cursor: pointer;
+            transition: all 0.3s ease;
+            display: flex;
+            align-items: center;
+            gap: 12px;
+            color: #94a3b8;
+            font-weight: 500;
+        }
+
+        .menu-item:hover {
+            background: rgba(255, 255, 255, 0.05);
+            color: white;
+        }
+
+        .menu-item.active {
+            background: var(--primary);
+            color: white;
+        }
+
+        .api-key-section {
+            margin-top: auto;
+            padding: 1.5rem;
+            background: rgba(255, 255, 255, 0.05);
+            border-radius: 16px;
+            font-size: 0.875rem;
+        }
+
+        .api-key-label {
+            display: block;
+            margin-bottom: 0.75rem;
+            font-weight: 500;
+            color: #e2e8f0;
+        }
+
+        .api-key-container {
+            position: relative;
+            width: 100%;
+            margin-bottom: 1rem;
+        }
+
+        .api-key-input {
+            width: 100%;
+            padding: 0.75rem;
+            padding-right: 2.5rem;
+            /* Space for toggle icon */
+            border-radius: 8px;
+            border: 1px solid rgba(255, 255, 255, 0.2);
+            background: rgba(255, 255, 255, 0.05);
+            color: #f8fafc;
+            font-size: 0.85rem;
+            transition: all 0.2s;
+        }
+
+        .api-key-input:focus {
+            outline: none;
+            background: rgba(255, 255, 255, 0.1);
+            border-color: var(--primary);
+            box-shadow: 0 0 0 2px rgba(99, 102, 241, 0.3);
+        }
+
+        .toggle-password {
+            position: absolute;
+            right: 0.75rem;
+            top: 50%;
+            transform: translateY(-50%);
+            cursor: pointer;
+            color: #94a3b8;
+            font-size: 0.9rem;
+            user-select: none;
+        }
+
+        .toggle-password:hover {
+            color: white;
+        }
+
+        .key-guide {
+            color: #94a3b8;
+            font-size: 0.75rem;
+            line-height: 1.5;
+        }
+
+        .key-guide a {
+            color: #818cf8;
+            text-decoration: none;
+        }
+
+        /* Main Content Styling */
+        .main-content {
+            flex: 1;
+            margin-left: 320px;
+            padding: 3rem;
+            max-width: 1200px;
+        }
+
+        .header-box {
+            margin-bottom: 3rem;
+        }
+
+        .title {
+            font-size: 2.25rem;
+            font-weight: 700;
+            margin-bottom: 1rem;
+            color: #0f172a;
+        }
+
+        .description {
+            color: var(--text-muted);
+            line-height: 1.6;
+            font-size: 1.1rem;
+        }
+
+        .system-info {
+            display: grid;
+            grid-template-columns: repeat(3, 1fr);
+            gap: 1.5rem;
+            margin-top: 2rem;
+        }
+
+        .info-card {
+            background: white;
+            padding: 1.5rem;
+            border-radius: 20px;
+            box-shadow: 0 4px 15px rgba(0, 0, 0, 0.03);
+            border: 1px solid #f1f5f9;
+        }
+
+        .info-card h4 {
+            margin-bottom: 0.5rem;
+            color: var(--primary);
+        }
+
+        .info-card p {
+            font-size: 0.9rem;
+            color: var(--text-muted);
+            line-height: 1.5;
+        }
+
+        /* Form Styling */
+        .survey-form {
+            background: white;
+            padding: 2.5rem;
+            border-radius: 24px;
+            box-shadow: 0 10px 40px rgba(0, 0, 0, 0.04);
+            margin-bottom: 3rem;
+            border: 1px solid #f1f5f9;
+        }
+
+        .form-grid {
+            display: grid;
+            grid-template-columns: repeat(2, 1fr);
+            gap: 1.5rem;
+            margin-bottom: 2rem;
+        }
+
+        .input-group {
+            display: flex;
+            flex-direction: column;
+            gap: 0.5rem;
+        }
+
+        .input-group.full {
+            grid-column: span 2;
+        }
+
+        label {
+            font-weight: 600;
+            font-size: 0.9rem;
+            color: #475569;
+        }
+
+        select,
+        input[type="text"],
+        input[type="number"],
+        textarea {
+            padding: 0.875rem;
+            border-radius: 12px;
+            border: 1px solid #e2e8f0;
+            background: #f8fafc;
+            font-size: 1rem;
+            transition: all 0.2s;
+        }
+
+        select:focus,
+        input:focus,
+        textarea:focus {
+            outline: none;
+            border-color: var(--primary);
+            box-shadow: 0 0 0 4px rgba(99, 102, 241, 0.1);
+            background: white;
+        }
+
+        textarea {
+            resize: vertical;
+            min-height: 100px;
+        }
+
+        .btn-submit {
+            background: linear-gradient(135deg, var(--primary), var(--primary-dark));
+            color: white;
+            padding: 1rem 2rem;
+            border-radius: 14px;
+            border: none;
+            font-weight: 600;
+            font-size: 1.1rem;
+            cursor: pointer;
+            width: 100%;
+            transition: all 0.3s;
+            box-shadow: 0 4px 15px rgba(79, 70, 229, 0.3);
+        }
+
+        .btn-submit:hover {
+            transform: translateY(-2px);
+            box-shadow: 0 8px 25px rgba(79, 70, 229, 0.4);
+        }
+
+        .btn-submit:active {
+            transform: translateY(0);
+        }
+
+        .btn-submit:disabled {
+            background: #cbd5e1;
+            box-shadow: none;
+            cursor: not-allowed;
+        }
+
+        .btn-reset {
+            background: white;
+            color: #64748b;
+            padding: 1rem 1.5rem;
+            border-radius: 14px;
+            border: 1px solid #e2e8f0;
+            font-weight: 600;
+            font-size: 1.1rem;
+            cursor: pointer;
+            transition: all 0.2s;
+        }
+
+        .btn-reset:hover {
+            background: #f8fafc;
+            border-color: #cbd5e1;
+            color: #1e293b;
+        }
+
+        .button-row {
+            display: flex;
+            gap: 1rem;
+        }
+
+        /* Results Styling */
+        .results-container {
+            display: flex;
+            flex-direction: column;
+            gap: 1.5rem;
+        }
+
+        .result-card {
+            background: white;
+            padding: 2rem;
+            border-radius: 20px;
+            box-shadow: 0 4px 20px rgba(0, 0, 0, 0.03);
+            border: 1px solid #f1f5f9;
+            animation: slideIn 0.5s ease forwards;
+            opacity: 0;
+        }
+
+        @keyframes slideIn {
+            from {
+                opacity: 0;
+                transform: translateY(20px);
+            }
+
+            to {
+                opacity: 1;
+                transform: translateY(0);
+            }
+        }
+
+        .avatar-info {
+            display: flex;
+            align-items: center;
+            gap: 1rem;
+            margin-bottom: 1.5rem;
+            padding-bottom: 1rem;
+            border-bottom: 1px solid #f1f5f9;
+        }
+
+        .avatar-icon {
+            width: 48px;
+            height: 48px;
+            background: #e0e7ff;
+            border-radius: 12px;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            color: var(--primary);
+            font-weight: 700;
+        }
+
+        .avatar-meta {
+            font-size: 0.85rem;
+            color: var(--text-muted);
+        }
+
+        .response-text {
+            line-height: 1.8;
+            font-size: 1.05rem;
+            color: #334155;
+            white-space: pre-wrap;
+        }
+
+        .context-chips {
+            display: flex;
+            flex-wrap: wrap;
+            gap: 0.5rem;
+            margin-top: 1.5rem;
+        }
+
+        .chip {
+            padding: 0.25rem 0.75rem;
+            background: #f1f5f9;
+            border-radius: 20px;
+            font-size: 0.75rem;
+            color: var(--text-muted);
+        }
+
+        .loader {
+            display: none;
+            text-align: center;
+            padding: 2rem;
+        }
+
+        .spinner {
+            width: 40px;
+            height: 40px;
+            border: 4px solid #f3f3f3;
+            border-top: 4px solid var(--primary);
+            border-radius: 50%;
+            animation: spin 1s linear infinite;
+            margin: 0 auto 1rem;
+        }
+
+        @keyframes spin {
+            0% {
+                transform: rotate(0deg);
+            }
+
+            100% {
+                transform: rotate(360deg);
+            }
+        }
+
+        .notice-label {
+            font-size: 0.8rem;
+            color: var(--secondary);
+            font-weight: 500;
+            margin-top: 0.5rem;
+        }
+    </style>
+</head>
+
+<body>
+
+    <aside class="sidebar">
+        <div class="logo">
+            <span>✨</span> Issue Survey (Beta)
+        </div>
+
+        <div class="menu-item active" id="menu-actual" onclick="switchMode('actual')">
+            <span>👤</span> Actual Respondent
+        </div>
+        <div class="menu-item" id="menu-synthetic" onclick="switchMode('synthetic')">
+            <span>📊</span> Statistical Synthetic
+        </div>
+
+        <div class="api-key-section">
+            <label class="api-key-label">Gemini API Key</label>
+            <div class="api-key-container">
+                <input type="password" id="api-key" class="api-key-input" placeholder="여기에 API 키 입력">
+                <span class="toggle-password" id="toggle-key" onclick="toggleKeyVisibility()">👁️</span>
+            </div>
+            <div class="key-guide">
+                <p>Gemini API 키가 없으신가요?</p>
+                <p><a href="https://aistudio.google.com/app/apikey" target="_blank">Google AI Studio</a>에서 무료로 발급받으실 수
+                    있습니다.</p>
+            </div>
+        </div>
+    </aside>
+
+    <main class="main-content">
+        <div class="header-box">
+            <h1 class="title">Persona Issue Survey (Beta version)</h1>
+            <p class="description">2020년 7월(1차)부터 2025년 9월(164차)까지 수행된 <b>전국지표조사(NBS)</b> 데이터를 바탕으로 특정 계층의 목소리를
+                시뮬레이션합니다.</p>
+            <p style="margin-top: 0.5rem; font-size: 0.85rem; color: #f43f5e; font-weight: 500;">⚠️ 본 시스템은 사회적 이슈 분석용이며
+                인물 평가에는 부적합합니다. 시뮬레이션 결과는 참고용으로만 사용하십시오.</p>
+
+            <div class="system-info">
+                <div class="info-card">
+                    <h4>👤 Actual vs 📊 Synthetic</h4>
+                    <p><b>Actual</b>: 실제 개별 응답자의 이력과 가치관을 추적하여 생생한 개인의 목소리를 시나리오화 합니다. (심층 인터뷰 대용)<br>
+                        <b>Synthetic</b>: 그룹 전체의 통계적 답변 분포를 요약하여 집단의 평균적이고 전형적인 경향성을 도출합니다. (여론 지형 분석용)
+                    </p>
+                </div>
+                <div class="info-card">
+                    <h4>🛠 구현 과정</h4>
+                    <p>163회의 NBS 설문(16만명)을 통합 전처리하고, SBERT 의미론적 검색을 통해 아바타의 '기억'을 AI에게 주입합니다.</p>
+                </div>
+                <div class="info-card">
+                    <h4>💡 사용법</h4>
+                    <p>1. <b>API 키를 먼저 입력</b>하세요. 2. 인구통계 조건을 설정하고 이슈 질문을 입력(최저임금 사례 등 구체적 배경 권장) 후 시작하세요.</p>
+                </div>
+            </div>
+        </div>
+
+        <div class="survey-form">
+            <div class="form-grid">
+                <div class="input-group">
+                    <label>지역</label>
+                    <select id="region">
+                        <option value="">전체 (전국)</option>
+                        <option value="서울">서울</option>
+                        <option value="경기">경기</option>
+                        <option value="인천">인천</option>
+                        <option value="부산">부산</option>
+                        <option value="대구">대구</option>
+                        <option value="광주">광주</option>
+                        <option value="대전">대전</option>
+                        <option value="울산">울산</option>
+                        <option value="세종">세종</option>
+                        <option value="충북">충북</option>
+                        <option value="충남">충남</option>
+                        <option value="전북">전북</option>
+                        <option value="전남">전남</option>
+                        <option value="경북">경북</option>
+                        <option value="경남">경남</option>
+                        <option value="제주">제주</option>
+                        <option value="강원">강원</option>
+                    </select>
+                </div>
+                <div class="input-group">
+                    <label>성별</label>
+                    <select id="gender">
+                        <option value="">전체 (성별무관)</option>
+                        <option value="남자">남자</option>
+                        <option value="여자">여자</option>
+                    </select>
+                </div>
+                <div class="input-group">
+                    <label>연령대</label>
+                    <select id="age">
+                        <option value="">전체 (연령무관)</option>
+                        <option value="18~29">20대 (18~29)</option>
+                        <option value="30~39">30대 (30~39)</option>
+                        <option value="40~49">40대 (40~49)</option>
+                        <option value="50~59">50대 (50~59)</option>
+                        <option value="60~69">60대 (60~69)</option>
+                        <option value="70~99">70대 이상</option>
+                    </select>
+                </div>
+                <div class="input-group">
+                    <label>구체적 연령 (선택)</label>
+                    <input type="number" id="age-specific" placeholder="예: 32">
+                </div>
+                <div class="input-group">
+                    <label>직업</label>
+                    <select id="job">
+                        <option value="">전체 (직업무관)</option>
+                        <option value="학생">학생</option>
+                        <option value="사무/기술직">사무/기술직</option>
+                        <option value="자영업">자영업</option>
+                        <option value="주부">주부</option>
+                        <option value="경영/관리/전문직">경영/관리/전문직</option>
+                        <option value="생산/기능/노무직">생산/기능/노무직</option>
+                        <option value="농/림/수산업">농/림/수산업</option>
+                        <option value="무직/퇴직/기타">무직/퇴직/기타</option>
+                    </select>
+                </div>
+                <div class="input-group full">
+                    <label>분석 이슈 (질문)</label>
+                    <textarea id="question"
+                        placeholder="예: 최근 고물가 상황에서 서민 경제 안정을 위해 최저임금을 인상해야 한다는 주장에 얼마나 동의하십니까? (1: 전혀 동의 안함 ~ 5: 매우 동의)"></textarea>
+                </div>
+                <div class="input-group" id="sample-group">
+                    <label>생성할 아바타(응답) 수</label>
+                    <input type="number" id="sample" value="3" min="1" max="10">
+                    <div id="sample-notice" class="notice-label">Actual 모드는 최대 10명으로 제한됩니다.</div>
+                </div>
+            </div>
+            <div class="button-row">
+                <button class="btn-submit" id="btn-submit" onclick="runSimulation()">시뮬레이션 시작</button>
+                <button class="btn-reset" id="btn-reset" onclick="resetForm()">초기화</button>
+            </div>
+        </div>
+
+        <div class="loader" id="loader">
+            <div class="spinner"></div>
+            <p>데이터에서 페르소나를 추출하고 AI 답변을 생성하고 있습니다...</p>
+        </div>
+
+        <div class="results-container" id="results">
+            <!-- Results will be injected here -->
+        </div>
+    </main>
+
+    <script>
+        let currentMode = 'actual';
+
+        function toggleKeyVisibility() {
+            const keyInput = document.getElementById('api-key');
+            const toggleIcon = document.getElementById('toggle-key');
+            if (keyInput.type === 'password') {
+                keyInput.type = 'text';
+                toggleIcon.innerText = '🙈';
+            } else {
+                keyInput.type = 'password';
+                toggleIcon.innerText = '👁️';
+            }
+        }
+
+        function switchMode(mode) {
+            currentMode = mode;
+            document.querySelectorAll('.menu-item').forEach(el => el.classList.remove('active'));
+            document.getElementById(`menu-${mode}`).classList.add('active');
+
+            const sampleGroup = document.getElementById('sample-group');
+            const sampleInput = document.getElementById('sample');
+            const notice = document.getElementById('sample-notice');
+
+            if (mode === 'actual') {
+                sampleGroup.style.display = "flex";
+                sampleInput.max = 10;
+                if (sampleInput.value > 10) sampleInput.value = 10;
+                notice.innerText = "Actual 모드는 최대 10명으로 제한됩니다. (대규모 분석은 Synthetic 추천)";
+                notice.style.color = "#ec4899";
+            } else {
+                // Synthetic is always 1 summarized avatar per request in current implementation
+                sampleGroup.style.display = "none";
+                sampleInput.value = 1;
+            }
+        }
+
+        function resetForm() {
+            document.getElementById('question').value = "";
+            document.getElementById('region').value = "";
+            document.getElementById('gender').value = "";
+            document.getElementById('age').value = "";
+            document.getElementById('age-specific').value = "";
+            document.getElementById('job').value = "";
+            document.getElementById('sample').value = (currentMode === 'actual' ? "3" : "1");
+            document.getElementById('results').innerHTML = "";
+        }
+
+        async function runSimulation() {
+            const apiKey = document.getElementById('api-key').value.trim();
+            const question = document.getElementById('question').value.trim();
+            const region = document.getElementById('region').value;
+            const gender = document.getElementById('gender').value;
+
+            // Priority: specific age > age range
+            const ageSpecific = document.getElementById('age-specific').value.trim();
+            const ageRange = document.getElementById('age').value;
+            const age = ageSpecific || ageRange;
+
+            const job = document.getElementById('job').value;
+            const sample = parseInt(document.getElementById('sample').value);
+
+            if (!apiKey) {
+                alert("시작하려면 Gemini API 키를 입력해 주세요 (사이드바 하단).");
+                return;
+            }
+
+            if (!question) {
+                alert("질문을 입력해 주세요.");
+                return;
+            }
+
+            const resultsDiv = document.getElementById('results');
+            const loader = document.getElementById('loader');
+            const btn = document.getElementById('btn-submit');
+
+            resultsDiv.innerHTML = "";
+            loader.style.display = "block";
+            btn.disabled = true;
+
+            const payload = {
+                question,
+                region: region || null,
+                gender: gender || null,
+                age: age || null,
+                job: job || null,
+                sample
+            };
+
+            const endpoint = `/simulate/${currentMode}`;
+
+            try {
+                const response = await fetch(endpoint, {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json',
+                        'X-API-Key': apiKey || ""
+                    },
+                    body: JSON.stringify(payload)
+                });
+
+                if (!response.ok) {
+                    const err = await response.json();
+                    throw new Error(err.detail || "시뮬레이션 중 오류가 발생했습니다.");
+                }
+
+                const data = await response.json();
+                renderResults(data);
+            } catch (error) {
+                alert(error.message);
+            } finally {
+                loader.style.display = "none";
+                btn.disabled = false;
+            }
+        }
+
+        function renderResults(data) {
+            const resultsDiv = document.getElementById('results');
+
+            if (currentMode === 'synthetic') {
+                resultsDiv.innerHTML = `<h2 style="margin-bottom: 1.5rem;">집단 대표(Representative) 분석 결과</h2>`;
+            } else {
+                resultsDiv.innerHTML = `<h2 style="margin-bottom: 1.5rem;">총 ${data.length}명의 답변 결과</h2>`;
+            }
+
+            data.forEach((item, index) => {
+                const card = document.createElement('div');
+                card.className = "result-card";
+                card.style.animationDelay = `${index * 0.1}s`;
+
+                const isSynthetic = !item.respondent_id;
+                const idLabel = item.respondent_id ? `ID: ${item.respondent_id}` : `Group Representative (Representative)`;
+                const roundLabel = item.survey_round ? ` | Survey Round: ${item.survey_round}` : "";
+
+                let demographicsStr = "";
+                if (item.demographics) {
+                    const d = item.demographics;
+                    demographicsStr = `${d.region || '전국'} / ${d.age || '연령대'} / ${d.gender || '성별'} / ${d.job || '직업'}`;
+                }
+
+                const contextHtml = (item.referenced_context || item.referenced_stat_context || [])
+                    .map(c => `<span class="chip">${c.split('\n')[0].replace('Q: ', '')}</span>`)
+                    .join('');
+
+                card.innerHTML = `
+                    <div class="avatar-info">
+                        <div class="avatar-icon">${item.respondent_id ? 'P' : 'S'}</div>
+                        <div>
+                            <div style="font-weight: 700; color: #0f172a;">${idLabel}${roundLabel}</div>
+                            <div class="avatar-meta">${demographicsStr}</div>
+                        </div>
+                    </div>
+                    <div class="response-text">${item.response}</div>
+                    <div class="api-key-label" style="margin-top: 1.5rem; font-size: 0.75rem; color: #94a3b8;">참조된 과거 데이터 필드:</div>
+                    <div class="context-chips">
+                        ${contextHtml}
+                    </div>
+                `;
+                resultsDiv.appendChild(card);
+            });
+        }
+    </script>
+</body>
+
+</html>

nbs_questions_index.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fdd94e1940b369064abfc09632d968529e29a61247a07f1586dd9941fd6591b
+size 6425603

rag_engine.py

@@ -0,0 +1,184 @@
+
+import os
+import pandas as pd
+import numpy as np
+from sklearn.metrics.pairwise import cosine_similarity
+from sentence_transformers import SentenceTransformer
+import google.generativeai as genai
+from dotenv import load_dotenv
+
+# Look for .env in current and parent directories
+load_dotenv()
+if not os.getenv("GEMINI_KEY"):
+    load_dotenv(os.path.join(os.path.dirname(__file__), '..', '.env'))
+
+class NBSRagEngine:
+    def __init__(self, parquet_path, db_path, model_name):
+        print("Initializing NBSRagEngine...")
+        print("Initializing NBSRagEngine (Stable Mode)...")
+        self.df = pd.read_parquet(parquet_path)
+        # Ensure age is numeric for range filtering
+        if 'age' in self.df.columns:
+            self.df['age'] = pd.to_numeric(self.df['age'], errors='coerce')
+        
+        # Load stable question index
+        idx_path = db_path # Now interpreted as the .parquet index file
+        if not os.path.exists(idx_path):
+            # Fallback for old pathing
+            idx_path = os.path.join(os.path.dirname(idx_path), "nbs_questions_index.parquet")
+            
+        self.q_df = pd.read_parquet(idx_path)
+        self.q_vectors = np.array(self.q_df['vector'].tolist())
+        self.questions = self.q_df['question'].tolist()
+        
+        self.model = SentenceTransformer(model_name)
+        
+        # Configure Gemini
+        gemini_key = os.getenv("GEMINI_KEY")
+        if not gemini_key:
+            raise ValueError("GEMINI_KEY not found in .env")
+        genai.configure(api_key=gemini_key)
+        # Using a stable flash model with fallback
+        try:
+            self.llm = genai.GenerativeModel('gemini-2.5-flash-lite')
+            print("Using Gemini 2.5 Flash model.")
+        except Exception as e:
+            print(f"Failed to load Gemini 2.5 Flash: {e}. Falling back to Gemini 1.0 Pro.")
+            self.llm = genai.GenerativeModel('gemini-1.0-pro')
+
+    def find_similar_questions(self, target_question, top_k=5):
+        query_vec = self.model.encode([target_question])
+        sims = cosine_similarity(query_vec, self.q_vectors)[0]
+        top_indices = np.argsort(sims)[-top_k:][::-1]
+        return [self.questions[i] for i in top_indices]
+
+    def normalize_input(self, gender=None, region=None):
+        norm_gender = None
+        if gender:
+            g = str(gender).strip()
+            if g in ['남', '남자', '남성']: norm_gender = '남자'
+            elif g in ['여', '여자', '여성']: norm_gender = '여자'
+            else: norm_gender = g
+            
+        norm_region = None
+        if region:
+            r = str(region).strip()
+            # Simple alias map
+            aliases = {
+                '서울': '서울', '서울특별시': '서울',
+                '경기': '경기', '경기도': '경기',
+                '인천': '인천', '인천광역시': '인천',
+                '부산': '부산', '부산광역시': '부산',
+                '대구': '대구', '대구광역시': '대구',
+                '광주': '광주', '광주광역시': '광주',
+                '대전': '대전', '대전광역시': '대전',
+                '울산': '울산', '울산광역시': '울산',
+                '세종': '세종', '세종특별자치시': '세종',
+                '강원': '강원', '강원도': '강원',
+                '충북': '충북', '충청북도': '충북',
+                '충남': '충남', '충청남도': '충남',
+                '전북': '전북', '전라북도': '전북',
+                '전남': '전남', '전라남도': '전남',
+                '경북': '경북', '경상북도': '경북',
+                '경남': '경남', '경상남도': '경남',
+                '제주': '제주', '제주특별자치도': '제주'
+            }
+            norm_region = aliases.get(r, r)
+            
+        return norm_gender, norm_region
+
+    def filter_respondents(self, gender=None, age=None, region=None, job=None):
+        mask = pd.Series([True] * len(self.df))
+        
+        # Normalize inputs to match Parquet categories
+        norm_gender, norm_region = self.normalize_input(gender, region)
+        
+        if norm_gender:
+            mask &= (self.df['gender'] == norm_gender)
+        if age:
+            age_str = str(age).strip()
+            if '~' in age_str:
+                try:
+                    start_age, end_age = map(int, age_str.split('~'))
+                    mask &= (self.df['age'] >= start_age) & (self.df['age'] <= end_age)
+                except ValueError:
+                    # Fallback to exact match if format is wrong
+                    mask &= (self.df['age'] == float(age_str))
+            else:
+                try:
+                    mask &= (self.df['age'] == float(age_str))
+                except ValueError:
+                    pass
+        if norm_region:
+            mask &= (self.df['region'] == norm_region)
+        if job:
+            mask &= (self.df['job'] == str(job).strip())
+        
+        return self.df[mask]
+
+    def get_context_for_responder(self, responder_row, similar_questions):
+        """
+        Extracts actual answers for a given responder.
+        Includes both semantically similar questions and 'persona-defining' questions.
+        """
+        context = []
+        referenced_qs = set(similar_questions)
+        
+        # 1. Add key persona columns if they exist in this row and aren't already included
+        persona_keywords = ['정치', '성향', '이념', '지지', '후보', '경제', '부동산']
+        for col in responder_row.index:
+            if any(k in col for k in persona_keywords):
+                ans = responder_row[col]
+                if pd.notna(ans) and ans != "" and str(ans).strip() != "":
+                    if col not in referenced_qs:
+                        context.append(f"Q: {col}\nA: {ans}")
+                        referenced_qs.add(col)
+
+        # 2. Add semantically similar questions
+        for q in similar_questions:
+            if q in responder_row.index:
+                ans = responder_row[q]
+                if pd.notna(ans) and ans != "" and str(ans).strip() != "":
+                    # Only add if not already added by persona scan
+                    if f"Q: {q}\nA: {ans}" not in context:
+                        context.append(f"Q: {q}\nA: {ans}")
+        
+        return "\n\n".join(context)
+
+    def generate_response(self, persona_desc, context, target_question, api_key=None):
+        if api_key:
+            # Temporary configuration for this call
+            genai.configure(api_key=api_key)
+            model = genai.GenerativeModel('gemini-2.5-flash-lite')
+        else:
+            model = self.llm
+
+        prompt = f"""당신은 다음과 같은 페르소나를 가진 한국인 응답자입니다.
+---
+페르소나: {persona_desc}
+
+과거 당신은 유사한 질문에 대해 다음과 같이 답변한 이력이 있습니다:
+{context}
+---
+
+위의 가치관과 경험을 바탕으로, 다음의 새로운 질문에 대해 당신이 할 법한 답변을 선택하거나 작성하십시오.
+답변은 실제 설문조사에 응하듯 자연스럽고 구체적이어야 합니다.
+
+질문: {target_question}
+
+답변:"""
+        
+        response = model.generate_content(prompt)
+        return response.text
+
+if __name__ == "__main__":
+    # Test block
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    parquet = os.path.join(script_dir, "consolidated_nbs_data.parquet")
+    # Point to the root directory for the stable DB
+    db = os.path.join(os.path.dirname(script_dir), "nbs_db_stable")
+    model = "snunlp/KR-SBERT-V40K-klueNLI-augSTS"
+    
+    engine = NBSRagEngine(parquet, db, model)
+    sim_qs = engine.find_similar_questions("대통령 국정 운영에 대해 어떻게 생각하십니까?")
+    print(f"Similar Questions: {sim_qs}")

requirements.txt

@@ -0,0 +1,10 @@
+fastapi
+uvicorn
+pandas
+pyarrow
+numpy
+scikit-learn
+sentence-transformers
+google-generativeai
+python-dotenv
+tqdm