backend/scripts/build_vector_db.py

import os
import json
from pathlib import Path
from dotenv import load_dotenv

from langchain_core.documents import Document
from langchain_community.vectorstores import FAISS
from langchain_openai import OpenAIEmbeddings
from langchain_community.embeddings import FakeEmbeddings

# Load environment variables
ROOT_DIR = Path(__file__).resolve().parent.parent
load_dotenv(ROOT_DIR / ".env")

RAW_LAWS_PATH = ROOT_DIR / "data" / "raw_laws.json"
RAW_CASES_PATH = ROOT_DIR / "data" / "raw_cases.json"
FAISS_INDEX_DIR = ROOT_DIR / "data" / "faiss_index"

def load_data():
    with open(RAW_LAWS_PATH, "r", encoding="utf-8") as f:
        laws = json.load(f)
    with open(RAW_CASES_PATH, "r", encoding="utf-8") as f:
        cases = json.load(f)
    return laws, cases

def create_documents(laws, cases):
    documents = []
    
    # 1. 법령 데이터 문단 생성
    for law in laws:
        # 검색용 텍스트 구성: 조항 내용 + 관련 키워드
        page_content = f"법령명: {law['law_name']}\n조항: {law['article_no']} {law.get('paragraph_no', '')}\n제목: {law['title']}\n내용: {law['content']}\n키워드: {', '.join(law['keywords'])}"
        metadata = {
            "type": "law",
            "law_name": law["law_name"],
            "article_no": law["article_no"],
            "paragraph_no": law.get("paragraph_no", ""),
            "title": law["title"]
        }
        documents.append(Document(page_content=page_content, metadata=metadata))
        
    # 2. 공정위 제재 사례 데이터 문단 생성
    for case in cases:
        # 검색용 텍스트 구성: 사례 제목 + 사실 관계 + 판단 이유 + 키워드
        page_content = f"공정위 사례: {case['case_title']}\n위반 법령: {case['violated_law']}\n조치 내용: {case['sanction_details']}\n사실 관계: {case['facts']}\n판단 이유: {case['reasoning']}\n키워드: {', '.join(case['keywords'])}"
        metadata = {
            "type": "case",
            "case_title": case["case_title"],
            "violated_law": case["violated_law"],
            "sanction_details": case["sanction_details"]
        }
        documents.append(Document(page_content=page_content, metadata=metadata))
        
    return documents

def build_db():
    print("[Info] Loading data...")
    laws, cases = load_data()
    docs = create_documents(laws, cases)
    print(f"[Info] Loaded {len(docs)} documents (Laws: {len(laws)}, Cases: {len(cases)})")
    
    openai_key = os.getenv("OPENAI_API_KEY")
    
    # OpenAI API Key가 플레이스홀더이거나 없을 경우 가짜 임베딩 사용
    if not openai_key or "your_openai_api_key" in openai_key or openai_key.strip() == "":
        print("[Warning] OPENAI_API_KEY is not set or is using placeholder.")
        print("[Warning] Using FakeEmbeddings for testing to build FAISS index.")
        embeddings = FakeEmbeddings(size=1536)
    else:
        print("[Info] Using OpenAI text-embedding-3-small model...")
        embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
        
    print("[Info] Building FAISS Vector DB...")
    db = FAISS.from_documents(docs, embeddings)
    
    # 디렉토리 생성 후 저장
    FAISS_INDEX_DIR.mkdir(parents=True, exist_ok=True)
    db.save_local(str(FAISS_INDEX_DIR))
    print(f"[Success] FAISS index saved successfully to: {FAISS_INDEX_DIR}")

if __name__ == "__main__":
    build_db()