Update chat_a.py

chat_a.py

@@ -1,33 +1,30 @@
-#!/usr/bin/env python
-# coding: utf-8
-
-# In[10]:
-import os, io, pathlib
-from huggingface_hub import hf_hub_download
+# -*- coding: utf-8 -*-
+import os, io, json, pathlib, re, random
 import pandas as pd
+import streamlit as st
 import torch
-from sentence_transformers import SentenceTransformer, util
-from transformers import AutoTokenizer, AutoModelForSequenceClassification
 import torch.nn.functional as F
 from collections import defaultdict
 from datetime import datetime
-import random
-import re
+from huggingface_hub import hf_hub_download
+from sentence_transformers import SentenceTransformer, util
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+from css import log_and_render
 
-CACHE_DIR = os.getenv("TRANSFORMERS_CACHE", "/tmp/hf-cache")
+# ──────────────────────────────── 캐시/데이터셋 설정 ────────────────────────────────
+CACHE_DIR = os.getenv("TRANSFORMERS_CACHE", "/data/hf-cache")
 os.makedirs(CACHE_DIR, exist_ok=True)
 
-HF_DATASET_REPO = os.getenv("HF_DATASET_REPO", "emisdfde/moai-travel-data")  # ← 본인 리포
+HF_DATASET_REPO = os.getenv("HF_DATASET_REPO", "emisdfde/moai-travel-data")
 HF_DATASET_REV  = os.getenv("HF_DATASET_REV", "main")
 
 def _is_pointer_bytes(b: bytes) -> bool:
     head = b[:2048].decode(errors="ignore").lower()
-    # git-lfs / xet 포인터 텍스트 패턴 모두 감지
     return (
-        "version https://git-lfs.github.com/spec/v1" in head or
-        "git-lfs" in head or
-        "xet" in head or            # e.g. "Xet backed hash"
-        "pointer size" in head
+        "version https://git-lfs.github.com/spec/v1" in head
+        or "git-lfs" in head
+        or "xet" in head
+        or "pointer size" in head
     )
 
 def _read_csv_bytes(b: bytes) -> pd.DataFrame:
@@ -40,7 +37,7 @@ def load_csv_smart(local_path: str,
                    hub_filename: str | None = None,
                    repo_id: str = HF_DATASET_REPO,
                    repo_type: str = "dataset",
-                   revision: str = HF_DATASET_REV):
+                   revision: str = HF_DATASET_REV) -> pd.DataFrame:
     if hub_filename is None:
         hub_filename = os.path.basename(local_path)
     if os.path.exists(local_path):
@@ -58,56 +55,69 @@ def load_csv_smart(local_path: str,
     except UnicodeDecodeError:
         return pd.read_csv(cached, encoding="cp949")
 
+# ──────────────────────────────── 전역 데이터 컨테이너 (지연 초기화) ────────────────────────────────
+travel_df = festival_df = external_score_df = weather_df = package_df = master_df = None
+countries, cities = [], []
+theme_title_phrases = {}
 
-from css import log_and_render
-import streamlit as st, pandas as pd, json, requests
-# -------------------- 모델 및 데이터 로딩 --------------------
-# 모델 로딩 부분을 함수로 만들고 데코레이터 추가
+def _strip_columns(df: pd.DataFrame | None) -> pd.DataFrame | None:
+    if df is not None and hasattr(df, "columns"):
+        df.columns = df.columns.str.strip()
+    return df
+
+def init_datasets(*,
+                  travel_df: pd.DataFrame,
+                  festival_df: pd.DataFrame,
+                  external_score_df: pd.DataFrame,
+                  weather_df: pd.DataFrame,
+                  package_df: pd.DataFrame,
+                  master_df: pd.DataFrame,
+                  theme_title_phrases: dict | None = None):
+    """app.py에서 데이터 로드가 끝난 뒤 딱 한 번 호출"""
+    globals()["travel_df"] = _strip_columns(travel_df.copy())
+    globals()["festival_df"] = _strip_columns(festival_df.copy())
+    globals()["external_score_df"] = _strip_columns(external_score_df.copy())
+    globals()["weather_df"] = _strip_columns(weather_df.copy())
+    globals()["package_df"] = _strip_columns(package_df.copy())
+    globals()["master_df"] = _strip_columns(master_df.copy())
+    if theme_title_phrases is not None:
+        globals()["theme_title_phrases"] = theme_title_phrases
+
+    # 필수 컬럼 확인
+    req = ["여행나라", "여행도시", "여행지"]
+    miss = [c for c in req if c not in globals()["travel_df"].columns]
+    if miss:
+        raise KeyError(f"travel_df 필수 컬럼 누락: {miss} / 실제: {list(globals()['travel_df'].columns)}")
+
+    # 파생 목록
+    global countries, cities
+    countries = sorted(globals()["travel_df"]["여행나라"].dropna().unique().tolist())
+    cities    = sorted(globals()["travel_df"]["여행도시"].dropna().unique().tolist())
+
+def _assert_ready():
+    if globals()["travel_df"] is None:
+        raise RuntimeError("chat_a.init_datasets(...)를 먼저 호출해주세요.")
+
+# ──────────────────────────────── 모델 로더 (캐시/권한 안전) ────────────────────────────────
 @st.cache_resource(show_spinner=False)
-def load_sentiment_model():
-    repo = "hun3359/klue-bert-base-sentiment"
-    tok = AutoTokenizer.from_pretrained(repo, cache_dir=CACHE_DIR)
-    model = AutoModelForSequenceClassification.from_pretrained(repo, cache_dir=CACHE_DIR)
-    return tok, model
-    
-@st.cache_resource
-def load_sbert_model():
-    print("SBERT 모델 로딩 중... (이 메시지는 한 번만 보여야 합니다)")    
-    return SentenceTransformer("jhgan/ko-sroberta-multitask")
+def load_tokenizer():
+    return AutoTokenizer.from_pretrained("hun3359/klue-bert-base-sentiment",
+                                         cache_dir=CACHE_DIR)
 
-@st.cache_resource
+@st.cache_resource(show_spinner=False)
 def load_sentiment_model():
-    print("감성 분석 모델 로딩 중... (이 메시지는 한 번만 보여야 합니다)")    
-    model = AutoModelForSequenceClassification.from_pretrained("hun3359/klue-bert-base-sentiment")
+    model = AutoModelForSequenceClassification.from_pretrained(
+        "hun3359/klue-bert-base-sentiment",
+        cache_dir=CACHE_DIR
+    )
     model.eval()
     return model
 
-@st.cache_resource
-def load_tokenizer():
-    print("토크나이저 로딩 중... (이 메시지는 한 번만 보여야 합니다)")   
-    return AutoTokenizer.from_pretrained("hun3359/klue-bert-base-sentiment")
-
-
-@st.cache_data(show_spinner=False)
-def load_csv_any(p):
-    return pd.read_csv(p) if str(p).startswith(("http://","https://")) else pd.read_csv(p)
-
-# trip_url = st.secrets.get("TRIPDATA_URL")
-# if not trip_url:
-#     st.error("TRIPDATA_URL 미설정: Streamlit Secrets에 URL을 넣어주세요.")
-#     st.stop()
-
-travel_df = load_csv_smart("trip_emotions.csv")
-festival_df = load_csv_smart("festivals.csv")
-external_score_df = load_csv_smart("external_scores.csv")
-external_score_df.columns = external_score_df.columns.str.strip()
-weather_df = load_csv_smart("weather.csv")
-package_df = load_csv_smart("packages.csv")
-package_df.columns = package_df.columns.str.strip()
-master_df = load_csv_smart("countries_cities.csv")
-
-countries = travel_df["여행나라"].dropna().unique().tolist()
-cities = travel_df["여행도시"].dropna().unique().tolist()
+@st.cache_resource(show_spinner=False)
+def load_sbert_model():
+    # SentenceTransformer 쪽도 캐시 폴더 명시
+    return SentenceTransformer("jhgan/ko-sroberta-multitask",
+                               cache_folder=CACHE_DIR)
     
 def detect_location_filter(text, intent_score=None):
     def in_text_exact(word):