fix: tz-naive datetime crash + initial-backup safety + English-only sweep

npc_intelligence.py

@@ -1,16 +1,16 @@
 """
-🧠 NPC Intelligence Engine — 자율 지능 시스템
+🧠 NPC Intelligence Engine — Autonomous Intelligence System
 =============================================
-NPC가 스스로 뉴스를 읽고, 분석하고, 목표가를 설정하고, 투자의견을 생성하는 자율 지능 엔진.
-모든 출력은 NPC의 "개인적 분석"으로 포장됨.
+Autonomous intelligence engine where NPCs read news, analyze, set price targets, and generate investment opinions on their own.
+All outputs are framed as the NPC's "personal analysis".
 
-핵심 모듈:
-1. MarketIndexCollector: S&P 500, NASDAQ, DOW, VIX 실시간 수집
-2. ScreeningEngine: RSI, PER, 52주고점, 시가총액 확장
-3. NPCNewsEngine: Brave API 뉴스 수집 → NPC 관점 분석
-4. NPCTargetPriceEngine: 동적 목표가 + 투자의견(Strong Buy~Sell)
-5. NPCElasticityEngine: 상승/하락 확률 + 리스크-리워드
-6. NPCResearchEngine: 조사자→감사자→감독자 3단계 심층 분석
+Core modules:
+1. MarketIndexCollector: Real-time collection of S&P 500, NASDAQ, DOW, VIX
+2. ScreeningEngine: RSI, PER, 52-week high, market cap expansion
+3. NPCNewsEngine: Brave API news collection → NPC-perspective analysis
+4. NPCTargetPriceEngine: Dynamic price target + investment opinion (Strong Buy~Sell)
+5. NPCElasticityEngine: Upside/downside probability + risk-reward
+6. NPCResearchEngine: 3-stage deep analysis (researcher → auditor → supervisor)
 
 Author: Ginigen AI / NPC Autonomous System
 """
@@ -29,7 +29,7 @@ from typing import Dict, List, Optional, Tuple
 
 logger = logging.getLogger(__name__)
 
-# ===== 시장 지수 정의 =====
+# ===== Market index definitions =====
 MAJOR_INDICES = [
     {'symbol': '^GSPC', 'name': 'S&P 500', 'emoji': '📊'},
     {'symbol': '^IXIC', 'name': 'NASDAQ', 'emoji': '💻'},
@@ -37,7 +37,7 @@ MAJOR_INDICES = [
     {'symbol': '^VIX', 'name': 'VIX', 'emoji': '⚡'},
 ]
 
-# ===== 섹터별 평균 PER =====
+# ===== Average PER by sector =====
 SECTOR_AVG_PE = {
     'Technology': 28, 'Communication': 22, 'Consumer Cyclical': 20,
     'Consumer Defensive': 22, 'Healthcare': 18, 'Financial': 14,
@@ -47,10 +47,10 @@ SECTOR_AVG_PE = {
 
 
 # ===================================================================
-# 1. 시장 지수 수집기
+# 1. Market index collector
 # ===================================================================
 class MarketIndexCollector:
-    """S&P 500, NASDAQ, DOW, VIX 실시간 수집"""
+    """Real-time collection of S&P 500, NASDAQ, DOW, VIX"""
 
     @staticmethod
     def fetch_indices() -> List[Dict]:
@@ -78,7 +78,7 @@ class MarketIndexCollector:
         except Exception as e:
             logger.warning(f"Index fetch error: {e}")
 
-        # 누락 시 시뮬레이션
+        # Simulate when missing
         fetched = {r['symbol'] for r in results}
         for idx in MAJOR_INDICES:
             if idx['symbol'] not in fetched:
@@ -134,14 +134,14 @@ async def load_indices_from_db(db_path: str) -> List[Dict]:
 
 
 # ===================================================================
-# 2. 스크리닝 지표 확장 엔진
+# 2. Screening indicators extension engine
 # ===================================================================
 class ScreeningEngine:
-    """RSI, PER, 52주 고점/저점, 시가총액 확장 데이터 수집"""
+    """RSI, PER, 52-week high/low, market cap extended data collection"""
 
     @staticmethod
     def fetch_extended_data(tickers: List[Dict]) -> Dict[str, Dict]:
-        """확장 스크리닝 데이터 수집 (Yahoo Finance)"""
+        """Collect extended screening data (Yahoo Finance)"""
         results = {}
         ticker_str = ' '.join([t['ticker'] for t in tickers])
         fields = 'regularMarketPrice,regularMarketChangePercent,regularMarketVolume,marketCap,fiftyTwoWeekHigh,fiftyTwoWeekLow,trailingPE,forwardPE'
@@ -178,7 +178,7 @@ class ScreeningEngine:
         except Exception as e:
             logger.warning(f"Screening data fetch error: {e}")
 
-        # 누락 종목 시뮬레이션
+        # Simulate missing tickers
         for t in tickers:
             if t['ticker'] not in results:
                 results[t['ticker']] = ScreeningEngine._simulate_screening(t)
@@ -187,8 +187,8 @@ class ScreeningEngine:
 
     @staticmethod
     def _estimate_rsi(change_pct: float) -> float:
-        """변동률 기반 RSI 추정 (14일 평균 대용)"""
-        # 실제 14일 데이터 없이 현재 변동률로 추정
+        """Estimate RSI based on change rate (proxy for 14-day average)"""
+        # Estimate from current change rate, no real 14-day data
         base = 50
         if change_pct > 3:
             base = random.uniform(65, 80)
@@ -206,7 +206,7 @@ class ScreeningEngine:
 
     @staticmethod
     def _simulate_screening(ticker_info: Dict) -> Dict:
-        """API 실패 시 시뮬레이션 데이터"""
+        """Simulation data when API fails"""
         is_crypto = ticker_info.get('type') == 'crypto'
         return {
             'price': 0,
@@ -222,10 +222,10 @@ class ScreeningEngine:
 
 
 async def save_screening_to_db(db_path: str, screening: Dict[str, Dict]):
-    """확장 스크리닝 데이터 DB 저장"""
+    """Save extended screening data to DB"""
     async with aiosqlite.connect(db_path, timeout=30.0) as db:
         await db.execute("PRAGMA busy_timeout=30000")
-        # 컬럼 추가 (이미 있으면 무시)
+        # Add columns (ignore if already exists)
         for col in ['rsi REAL DEFAULT 50', 'pe_ratio REAL DEFAULT 0', 'high_52w REAL DEFAULT 0',
                      'low_52w REAL DEFAULT 0', 'from_high REAL DEFAULT 0', 'from_low REAL DEFAULT 0']:
             try:
@@ -246,18 +246,18 @@ async def save_screening_to_db(db_path: str, screening: Dict[str, Dict]):
 
 
 # ===================================================================
-# 3. NPC 뉴스 분석 엔진
+# 3. NPC News Analysis Engine
 # ===================================================================
 class NPCNewsEngine:
-    """NPC가 자율적으로 뉴스를 수집하고 분석하는 시스템.
-    모든 분석은 NPC의 '개인적 견해'로 포장됨."""
+    """System where NPCs autonomously collect and analyze news.
+    All analyses are framed as the NPC's 'personal view'."""
 
     def __init__(self):
         self.brave_api_key = os.getenv('BRAVE_API_KEY', '')
         self.api_available = bool(self.brave_api_key)
         self.base_url = "https://api.search.brave.com/res/v1/news/search"
         self.cache = {}
-        self.cache_ttl = 1800  # 30분
+        self.cache_ttl = 1800  # 30 min
 
     def search_news(self, query: str, count: int = 5, freshness: str = "pd") -> List[Dict]:
         if not self.api_available:
@@ -290,7 +290,7 @@ class NPCNewsEngine:
             return []
 
     async def collect_ticker_news(self, ticker: str, name: str, count: int = 3) -> List[Dict]:
-        """특정 종목 뉴스 수집"""
+        """Collect news for a specific ticker"""
         queries = [f"{ticker} stock news", f"{name} earnings analyst"]
         all_news = []
         seen = set()
@@ -304,7 +304,7 @@ class NPCNewsEngine:
         return all_news[:count]
 
     async def collect_market_news(self, count: int = 10) -> List[Dict]:
-        """시장 전체 뉴스 수집"""
+        """Collect overall market news"""
         queries = ["stock market today", "Fed interest rate", "S&P 500 NASDAQ", "AI chip semiconductor"]
         all_news = []
         seen = set()
@@ -319,11 +319,11 @@ class NPCNewsEngine:
 
     @staticmethod
     def npc_analyze_news(news: Dict, npc_identity: str, npc_name: str) -> Dict:
-        """NPC가 뉴스를 자신의 관점으로 분석 (프레이밍)"""
+        """NPC analyzes news from its own perspective (framing)"""
         title = news.get('title', '')
         desc = news.get('description', '')
 
-        # 감성 분석 (키워드 기반)
+        # Sentiment analysis (keyword-based)
         positive = ['surge', 'rally', 'beat', 'growth', 'upgrade', 'record', 'boom', 'soar']
         negative = ['crash', 'plunge', 'miss', 'warning', 'downgrade', 'fear', 'recession', 'sell']
         text = f"{title} {desc}".lower()
@@ -341,7 +341,7 @@ class NPCNewsEngine:
             sentiment = 'neutral'
             impact = 'mixed'
 
-        # NPC 성격별 해석 프레이밍
+        # Identity-specific interpretation framing
         identity_frames = {
             'skeptic': f"🤨 I'm not buying this hype. {title[:60]}... needs verification.",
             'doomer': f"💀 This confirms my thesis. Markets are fragile. {title[:50]}...",
@@ -364,7 +364,7 @@ class NPCNewsEngine:
 
 
 async def init_news_db(db_path: str):
-    """뉴스 관련 DB 테이블 생성"""
+    """Create news-related DB tables"""
     async with aiosqlite.connect(db_path, timeout=30.0) as db:
         await db.execute("PRAGMA busy_timeout=30000")
         await db.execute("""
@@ -406,7 +406,7 @@ async def save_news_to_db(db_path: str, news_list: List[Dict]) -> int:
             except:
                 pass
         await db.commit()
-        # 24시간 이상 된 뉴스 삭제
+        # Delete news older than 72 hours
         await db.execute("DELETE FROM npc_news WHERE created_at < datetime('now', '-72 hours')")
         await db.commit()
     return saved
@@ -430,14 +430,14 @@ async def load_news_from_db(db_path: str, ticker: str = None, limit: int = 50) -
 
 
 # ===================================================================
-# 4. 목표가 + 투자의견 엔진
+# 4. Target price + investment opinion engine
 # ===================================================================
 class NPCTargetPriceEngine:
-    """NPC가 자율적으로 목표가와 투자의견을 생성하는 엔진"""
+    """Engine where NPCs autonomously generate price targets and investment opinions"""
 
     @staticmethod
     def calculate_target(ticker: str, price: float, screening: Dict, ticker_type: str = 'stock') -> Dict:
-        """동적 목표가 계산 (섹터/밸류에이션/모멘텀 기반)"""
+        """Dynamic price target calculation (sector/valuation/momentum based)"""
         if price <= 0:
             return {'target_price': 0, 'upside': 0, 'rating': 'N/A', 'rating_class': 'na'}
 
@@ -447,7 +447,7 @@ class NPCTargetPriceEngine:
         sector = screening.get('sector', 'Technology')
 
         if ticker_type == 'crypto':
-            # 크립토: 변동성 높은 모델
+            # Crypto: high-volatility model
             multiplier = 1.12
             if rsi < 30:
                 multiplier += 0.10
@@ -459,13 +459,13 @@ class NPCTargetPriceEngine:
                 multiplier -= 0.05
             multiplier = max(0.85, min(1.50, multiplier))
         else:
-            # 주식: PER + 기술적 분석 기반
+            # Stocks: PER + technical analysis based
             avg_pe = SECTOR_AVG_PE.get(sector, 20)
             multiplier = 1.10
 
             if pe > 0:
                 if pe < avg_pe * 0.7:
-                    multiplier += 0.08  # 심한 저평가
+                    multiplier += 0.08  # Severely undervalued
                 elif pe < avg_pe * 0.85:
                     multiplier += 0.05
                 elif pe > avg_pe * 1.5:
@@ -496,7 +496,7 @@ class NPCTargetPriceEngine:
         target_price = round(price * multiplier, 2)
         upside = round((multiplier - 1) * 100, 1)
 
-        # 투자의견 결정
+        # Determine investment opinion
         rating, rating_class = NPCTargetPriceEngine._determine_rating(upside, rsi, from_high)
 
         return {
@@ -522,14 +522,14 @@ class NPCTargetPriceEngine:
 
 
 # ===================================================================
-# 5. 탄력성 예측 엔진
+# 5. Elasticity prediction engine
 # ===================================================================
 class NPCElasticityEngine:
-    """상승/하락 양방향 확률 예측 시스템"""
+    """Bidirectional upside/downside probability prediction system"""
 
     @staticmethod
     def calculate(price: float, screening: Dict, target_price: float = 0, ticker_type: str = 'stock') -> Dict:
-        """탄력성 예측 계산"""
+        """Calculate elasticity prediction"""
         pe = screening.get('pe_ratio', 0) or 0
         rsi = screening.get('rsi', 50) or 50
         from_high = screening.get('from_high', -10) or -10
@@ -540,7 +540,7 @@ class NPCElasticityEngine:
         upside_factors = []
         downside_factors = []
 
-        # 애널리스트 목표가 기반
+        # Based on analyst target price
         if target_price and price > 0:
             diff = ((target_price - price) / price) * 100
             if diff > 0:
@@ -548,7 +548,7 @@ class NPCElasticityEngine:
             else:
                 downside_factors.append(diff)
 
-        # PER 기반 밸류에이션
+        # PER-based valuation
         if pe > 0 and avg_pe > 0:
             fair_diff = ((avg_pe / pe) - 1) * 100
             fair_diff = max(-40, min(60, fair_diff))
@@ -557,11 +557,11 @@ class NPCElasticityEngine:
             else:
                 downside_factors.append(fair_diff * 0.6)
 
-        # 52주 고점 대비 기술적 반등 여력
+        # Technical rebound potential vs. 52-week high
         if from_high < 0:
             upside_factors.append(abs(from_high) * 0.5)
 
-        # 52주 저점 대비 하락 리스크
+        # Downside risk vs. 52-week low
         if from_low > 30:
             downside_factors.append(-from_low * 0.35)
         elif from_low > 15:
@@ -569,7 +569,7 @@ class NPCElasticityEngine:
         elif from_low > 5:
             downside_factors.append(-from_low * 0.25)
 
-        # RSI 기반
+        # RSI-based
         if rsi < 30:
             upside_factors.append(18)
         elif rsi < 40:
@@ -581,7 +581,7 @@ class NPCElasticityEngine:
         elif rsi > 60:
             downside_factors.append(-10)
 
-        # 고점 근처 리스크
+        # Near-high risk
         if from_high > -3:
             downside_factors.append(-12)
         elif from_high > -8:
@@ -593,7 +593,7 @@ class NPCElasticityEngine:
         expected_up = max(upside_factors) if upside_factors else 15
         expected_down = min(downside_factors) if downside_factors else -10
 
-        # 크립토 변동성 확대
+        # Crypto volatility expansion
         if ticker_type == 'crypto':
             expected_up = min(80, expected_up * 1.5)
             expected_down = max(-50, expected_down * 1.5)
@@ -601,7 +601,7 @@ class NPCElasticityEngine:
             expected_up = max(5, min(50, expected_up))
             expected_down = max(-35, min(-3, expected_down))
 
-        # 확률 계산
+        # Probability calculation
         up_prob = 50
         if rsi < 30:
             up_prob = 70
@@ -633,29 +633,29 @@ class NPCElasticityEngine:
 
 
 # ===================================================================
-# 6. NPC 심층 리서치 엔진 (조사자→감사자→감독자 3단계)
+# 6. NPC Deep Research Engine (3-stage: investigator → auditor → supervisor)
 # ===================================================================
 class NPCResearchEngine:
-    """NPC 자율 심층 분석 — 3단계 SOMA 협업으로 프레이밍"""
+    """NPC autonomous deep analysis — framed as 3-stage SOMA collaboration"""
 
     def __init__(self, ai_client=None):
         self.ai_client = ai_client
 
     async def generate_deep_analysis(self, ticker: str, name: str, screening: Dict,
                                      news_ctx: str = '', npc_analysts: List[Dict] = None) -> Dict:
-        """3단계 심층 분석 실행"""
+        """Run 3-stage deep analysis"""
         price = screening.get('price', 0)
         rsi = screening.get('rsi', 50)
         pe = screening.get('pe_ratio', 0)
         from_high = screening.get('from_high', 0)
         sector = screening.get('sector', 'Technology')
 
-        # 목표가 계산
+        # Calculate target price
         target = NPCTargetPriceEngine.calculate_target(ticker, price, screening)
-        # 탄력성 계산
+        # Calculate elasticity
         elasticity = NPCElasticityEngine.calculate(price, screening, target['target_price'])
 
-        # NPC 분석가 3명 선정 (또는 기본값)
+        # Select 3 NPC analysts (or use defaults)
         if npc_analysts and len(npc_analysts) >= 3:
             investigator = npc_analysts[0]
             auditor = npc_analysts[1]
@@ -665,12 +665,12 @@ class NPCResearchEngine:
             auditor = {'username': 'AuditBot_Beta', 'ai_identity': 'skeptic'}
             supervisor = {'username': 'ChiefAnalyst_Gamma', 'ai_identity': 'awakened'}
 
-        # LLM 사용 가능 시 심층 분석
+        # Deep analysis if LLM available
         inv_report = await self._run_investigator(ticker, name, screening, news_ctx)
         aud_feedback = await self._run_auditor(ticker, name, inv_report)
         final_report = await self._run_supervisor(ticker, name, screening, inv_report, aud_feedback)
 
-        # 파싱된 최종 보고서
+        # Parsed final report
         sections = self._parse_report(final_report, ticker, name, screening)
         sections.update({
             'target_price': target['target_price'],
@@ -688,7 +688,7 @@ class NPCResearchEngine:
         return sections
 
     async def _run_investigator(self, ticker: str, name: str, data: Dict, news_ctx: str) -> str:
-        """조사자 에이전트"""
+        """Investigator agent"""
         if self.ai_client:
             try:
                 messages = [
@@ -726,7 +726,7 @@ Cover: 1) Business model 2) Financials 3) Technical analysis 4) Industry 5) Risk
         if self.ai_client:
             try:
                 messages = [
-                    {"role": "system", "content": "You are a chief analyst at a global investment bank. Write final report in English with sections marked ##."},
+                    {"role": "system", "content": "You are a chief analyst at a global investment bank. Write the final report in English. Sections marked ##."},
                     {"role": "user", "content": f"""{ticker} ({name}) | ${data.get('price', 0):,.2f}
 [Investigator Summary] {inv[:1200]}
 [Auditor Feedback] {aud[:500]}
@@ -783,16 +783,16 @@ ${target['target_price']:,.2f} ({'+' if target['upside'] >= 0 else ''}{target['u
             'final_recommendation': '',
         }
         patterns = [
-            (r'##\s*(핵심\s*요약|Executive\s*Summary|Executive)', 'executive_summary'),
-            (r'##\s*(회사\s*개요|Company\s*Overview)', 'company_overview'),
-            (r'##\s*(재무\s*분석|Financial\s*Analysis)', 'financial_analysis'),
-            (r'##\s*(기술적\s*분석|Technical\s*Analysis)', 'technical_analysis'),
-            (r'##\s*(산업\s*분석|Industry\s*Analysis)', 'industry_analysis'),
-            (r'##\s*(리스크|Risk\s*Assessment|Risk)', 'risk_assessment'),
-            (r'##\s*(투자\s*논리|Investment\s*Thesis)', 'investment_thesis'),
-            (r'##\s*(목표\s*주가|Price\s*Target)', 'price_targets'),
-            (r'##\s*(카탈리스트|Catalyst)', 'catalysts'),
-            (r'##\s*(최종\s*권고|Final\s*Recommendation)', 'final_recommendation'),
+            (r'##\s*(Executive\s*Summary|Executive)', 'executive_summary'),
+            (r'##\s*(Company\s*Overview)', 'company_overview'),
+            (r'##\s*(Financial\s*Analysis)', 'financial_analysis'),
+            (r'##\s*(Technical\s*Analysis)', 'technical_analysis'),
+            (r'##\s*(Industry\s*Analysis)', 'industry_analysis'),
+            (r'##\s*(Risk\s*Assessment|Risk)', 'risk_assessment'),
+            (r'##\s*(Investment\s*Thesis)', 'investment_thesis'),
+            (r'##\s*(Price\s*Target)', 'price_targets'),
+            (r'##\s*(Catalyst)', 'catalysts'),
+            (r'##\s*(Final\s*Recommendation)', 'final_recommendation'),
         ]
         for pattern, key in patterns:
             match = re.search(f'{pattern}[\\s\\S]*?(?=##|$)', text, re.IGNORECASE)
@@ -808,7 +808,7 @@ ${target['target_price']:,.2f} ({'+' if target['upside'] >= 0 else ''}{target['u
 
 
 async def init_research_db(db_path: str):
-    """심층 분석 DB 테이블"""
+    """Deep analysis DB tables"""
     async with aiosqlite.connect(db_path, timeout=30.0) as db:
         await db.execute("PRAGMA busy_timeout=30000")
         await db.execute("""
@@ -900,28 +900,28 @@ async def load_all_analyses_from_db(db_path: str) -> List[Dict]:
 
 
 # ===================================================================
-# 통합 초기화
+# Integrated initialization
 # ===================================================================
 async def init_intelligence_db(db_path: str):
-    """Intelligence 모듈 전체 DB 초기화"""
+    """Initialize all Intelligence module DB tables"""
     await init_news_db(db_path)
     await init_research_db(db_path)
     logger.info("🧠 NPC Intelligence DB initialized")
 
 
 async def run_full_intelligence_cycle(db_path: str, all_tickers: List[Dict], ai_client=None):
-    """전체 Intelligence 사이클 실행 (스케줄러에서 호출) — ★ 비동기 안전"""
+    """Run full Intelligence cycle (called from scheduler) — ★ async-safe"""
     logger.info("🧠 Full Intelligence Cycle starting...")
 
-    # 1) 시장 지수 수집 (★ 동기 requests → to_thread로 비동기 래핑)
+    # 1) Collect market indices (★ sync requests → wrapped via to_thread)
     indices = await asyncio.to_thread(MarketIndexCollector.fetch_indices)
     await save_indices_to_db(db_path, indices)
 
-    # 2) 확장 스크리닝 데이터 (★ 동기 requests → to_thread로 비동기 래핑)
+    # 2) Extended screening data (★ sync requests → wrapped via to_thread)
     screening = await asyncio.to_thread(ScreeningEngine.fetch_extended_data, all_tickers)
     await save_screening_to_db(db_path, screening)
 
-    # 3) 뉴스 수집 + NPC 분석 (★ search_news 내부 requests → to_thread)
+    # 3) News collection + NPC analysis (★ requests inside search_news → to_thread)
     news_engine = NPCNewsEngine()
     all_news = []
 
@@ -965,7 +965,7 @@ async def run_full_intelligence_cycle(db_path: str, all_tickers: List[Dict], ai_
 
     saved = await save_news_to_db(db_path, all_news)
 
-    # 4) 상위 5개 종목 심층 분석
+    # 4) Deep analysis for top 5 tickers
     research = NPCResearchEngine(ai_client)
     for t in all_tickers[:5]:
         ticker = t['ticker']