# ============================================
# АВТО-УСТАНОВКА ПАКЕТОВ
# ============================================
import subprocess, sys, importlib

REQUIRED_PACKAGES = {
    'numpy': 'numpy',
    'pandas': 'pandas',
    'httpx': 'httpx',
    'scipy': 'scipy',
    'fastapi': 'fastapi',
    'uvicorn': 'uvicorn',
    'requests': 'requests'
}

for module_name, pip_name in REQUIRED_PACKAGES.items():
    try:
        importlib.import_module(module_name)
    except ImportError:
        print(f"📦 Устанавливаю {pip_name}...")
        subprocess.check_call([sys.executable, "-m", "pip", "install", pip_name])
        print(f"✅ {pip_name} установлен!")

# ============================================
# 👑 TOMIRIS SPACE 21 v4.0 «СТАЛЬ» — GOLD MACRO & FLOW (УСИЛЕННЫЙ)
# ============================================
import os, time, json, logging, asyncio, sqlite3
from typing import Dict, Any, List, Optional, Tuple
from datetime import datetime, timezone
from collections import deque
import numpy as np
import pandas as pd
from scipy import stats
import httpx
from fastapi import FastAPI, Query
import io

logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
logger = logging.getLogger("Space21_GoldMacro")

# ================= КОНФИГУРАЦИЯ =================
SPACE_ID = 21
SPACE_NAME = "Gold Macro & Flow"
SYMBOL = "XAU/USD"

HUB_URL = "https://TOMI-HUB-HUB-FINAL.hf.space"
HUB_SECRET = os.getenv("HUB_SECRET", "TomyrisUltraSecret2026!")

TWELVE_DATA_KEY = os.getenv("TWELVE_DATA_KEY", "")
FRED_KEY = os.getenv("FRED_KEY", "")
FRED_KEY_2 = os.getenv("FRED_KEY_2", "")
NEWSAPI_KEY = os.getenv("NEWSAPI_KEY", "")

FRED_KEYS = [k for k in [FRED_KEY, FRED_KEY_2] if k]
if not FRED_KEYS:
    logger.warning("⚠️ Нет FRED ключей!")
    FRED_KEYS = ["no_key"]

STARTUP_SLEEP = int(os.getenv("STARTUP_SLEEP", "120"))
AUTO_SEND_INTERVAL = int(os.getenv("AUTO_SEND_INTERVAL", "300"))

logger.info(f"🔗 Хаб: {HUB_URL} | Старт: {STARTUP_SLEEP}с | Интервал: {AUTO_SEND_INTERVAL}с")
logger.info(f"🔑 FRED: {len(FRED_KEYS)} | NewsAPI: {'✓' if NEWSAPI_KEY else '✗'} | TwelveData: {'✓' if TWELVE_DATA_KEY else '✗'}")

GOLD_ETFS = ["GLD", "IAU"]
CACHE_TTL = {"fred": 3600, "etf": 900, "cot": 86400, "gpr": 900}

# ================= SQLite =================
DB_FILE = "gold_macro.db"

def init_db():
    conn = sqlite3.connect(DB_FILE)
    c = conn.cursor()
    c.execute('''CREATE TABLE IF NOT EXISTS etf_history (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        timestamp TEXT NOT NULL,
        ticker TEXT NOT NULL,
        aum REAL NOT NULL
    )''')
    c.execute('''CREATE TABLE IF NOT EXISTS cot_history (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        timestamp TEXT NOT NULL,
        report_date TEXT NOT NULL,
        noncommercial_net INTEGER NOT NULL,
        percentile REAL NOT NULL,
        signal TEXT NOT NULL
    )''')
    c.execute('''CREATE TABLE IF NOT EXISTS signals_log (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        timestamp TEXT NOT NULL,
        signal TEXT NOT NULL,
        confidence REAL,
        macro_score REAL
    )''')
    conn.commit()
    conn.close()
    logger.info("🗄️ SQLite база Gold Macro инициализирована")

init_db()

# ================= HTTP КЛИЕНТ =================
http_client = httpx.AsyncClient(timeout=20.0)

def hub_headers():
    return {"X-Hub-Secret": HUB_SECRET, "Content-Type": "application/json"}

async def log_to_hub(event_type: str, message: str, details: dict = None):
    try:
        await http_client.post(
            f"{HUB_URL}/log",
            json={"space_id": str(SPACE_ID), "event_type": event_type, "message": message, "details": details or {}},
            headers=hub_headers(), timeout=5
        )
    except: pass

# ================= ИСТОРИЯ ДЛЯ Z-SCORE =================
SCORE_HISTORY = deque(maxlen=200)
TIPS_HISTORY = deque(maxlen=200)
TWD_HISTORY = deque(maxlen=200)

def calculate_zscore(current: float, history: deque) -> float:
    if len(history) < 10: return 0.0
    arr = np.array(list(history))
    mean, std = arr.mean(), arr.std()
    if std == 0: return 0.0
    return (current - mean) / std

# ================= ОТПРАВКА В HUB =================
async def send_signal_to_hub(symbol: str, signal: str, confidence: float, features: Dict = None):
    if features is None: features = {}
    payload = {
        "space_id": SPACE_ID, "space_name": SPACE_NAME,
        "symbol": symbol, "signal": signal, "confidence": round(confidence, 4),
        "features": features, "metadata": {"version": "4.0"},
        "timestamp": datetime.now(timezone.utc).isoformat()
    }
    for attempt in range(3):
        try:
            r = await http_client.post(f"{HUB_URL}/signals", json=payload, timeout=15, headers=hub_headers())
            if r.status_code == 200:
                logger.info(f"📤 {symbol}: {signal} conf={confidence:.3f}")
                return True
            await asyncio.sleep(2)
        except Exception as e:
            logger.warning(f"Попытка {attempt+1}: {e}")
            await asyncio.sleep(2)
    return False

# ================= FRED ДАННЫЕ (С РОТАЦИЕЙ КЛЮЧЕЙ) =================
async def fetch_fred_series(series_id: str, days: int = 90) -> List[Dict]:
    if not FRED_KEYS or FRED_KEYS == ["no_key"]: return []
    
    for key in FRED_KEYS:
        try:
            r = await http_client.get(
                f"https://api.stlouisfed.org/fred/series/observations",
                params={"series_id": series_id, "api_key": key, "file_type": "json", "sort_order": "desc", "limit": days}
            )
            if r.status_code == 200:
                values = []
                for obs in r.json().get('observations', []):
                    if obs['value'] != '.':
                        try: values.append({'date': obs['date'], 'value': float(obs['value'])})
                        except: continue
                if values: return values
        except: continue
    return []

async def fetch_trade_weighted_dollar() -> Dict[str, Any]:
    values = await fetch_fred_series("DTWEXBGS", 90)
    if len(values) >= 2:
        current = values[0]['value']
        month_ago = values[-1]['value']
        if month_ago > 0:
            change = ((current - month_ago) / month_ago) * 100
            TWD_HISTORY.append(current)
            zscore = calculate_zscore(current, TWD_HISTORY)
            
            if change > 3: trend, gold_signal = "STRONG_UP", "STRONG_BEARISH"
            elif change > 1: trend, gold_signal = "UP", "BEARISH"
            elif change < -3: trend, gold_signal = "STRONG_DOWN", "STRONG_BULLISH"
            elif change < -1: trend, gold_signal = "DOWN", "BULLISH"
            else: trend, gold_signal = "STABLE", "NEUTRAL"
            
            return {
                'twd': round(current, 2), 'change_1m': round(change, 2),
                'zscore': round(zscore, 2), 'trend': trend, 'gold_signal': gold_signal
            }
    return {'twd': 104.5, 'change_1m': 0, 'zscore': 0, 'gold_signal': 'NEUTRAL'}

async def fetch_tips_yield() -> Dict[str, Any]:
    tips_values = await fetch_fred_series("DFII10", 90)
    t10y_values = await fetch_fred_series("DGS10", 90)
    
    if tips_values and t10y_values and len(tips_values) >= 2 and len(t10y_values) >= 2:
        current_tips = tips_values[0]['value']
        current_t10y = t10y_values[0]['value']
        current_real_yield = current_t10y - current_tips
        
        TIPS_HISTORY.append(current_real_yield)
        zscore = calculate_zscore(current_real_yield, TIPS_HISTORY)
        
        if current_real_yield < -1.0: gold_signal = "STRONG_BULLISH"
        elif current_real_yield < 0: gold_signal = "BULLISH"
        elif current_real_yield > 2.0: gold_signal = "STRONG_BEARISH"
        elif current_real_yield > 1.0: gold_signal = "BEARISH"
        else: gold_signal = "NEUTRAL"
        
        return {
            'tips_yield': round(current_tips, 4), 't10y': round(current_t10y, 4),
            'real_yield': round(current_real_yield, 4), 'zscore': round(zscore, 2),
            'gold_signal': gold_signal
        }
    
    return {'tips_yield': 0.5, 'real_yield': 1.0, 'zscore': 0, 'gold_signal': 'NEUTRAL'}

# ================= ETF ПОТОКИ =================
async def fetch_etf_aum(ticker: str) -> Optional[float]:
    if not TWELVE_DATA_KEY: return None
    try:
        r = await http_client.get(f"https://api.twelvedata.com/statistics?symbol={ticker}&apikey={TWELVE_DATA_KEY}")
        if r.status_code == 200:
            data = r.json()
            aum = data.get("statistics", {}).get("fundamental", {}).get("total_assets", 0)
            return float(aum)
    except: pass
    return None

def get_previous_etf_aum(ticker: str) -> Optional[float]:
    try:
        conn = sqlite3.connect(DB_FILE)
        c = conn.cursor()
        c.execute("SELECT aum FROM etf_history WHERE ticker = ? ORDER BY id DESC LIMIT 2", (ticker,))
        rows = c.fetchall()
        conn.close()
        if len(rows) >= 2: return rows[1][0]
        return None
    except: return None

def save_etf_aum(ticker: str, aum: float):
    try:
        conn = sqlite3.connect(DB_FILE)
        c = conn.cursor()
        c.execute("INSERT INTO etf_history (timestamp, ticker, aum) VALUES (?, ?, ?)",
                  (datetime.now(timezone.utc).isoformat(), ticker, aum))
        conn.commit()
        conn.close()
    except: pass

async def analyze_etf_flows() -> Dict[str, Any]:
    etf_data = {}
    total_change = 0.0
    count = 0
    
    for ticker in GOLD_ETFS:
        aum_now = await fetch_etf_aum(ticker)
        if aum_now is None or aum_now <= 0: continue
        
        prev_aum = get_previous_etf_aum(ticker)
        save_etf_aum(ticker, aum_now)
        
        change_pct = ((aum_now - prev_aum) / prev_aum * 100) if prev_aum and prev_aum > 0 else 0.0
        
        etf_data[ticker] = {
            "aum": aum_now, "change_24h": round(change_pct, 2),
            "signal": "INFLOW" if change_pct > 0.5 else "OUTFLOW" if change_pct < -0.5 else "NEUTRAL"
        }
        total_change += change_pct
        count += 1

    avg_change = total_change / count if count > 0 else 0
    
    if avg_change > 3: flow_signal, gold_signal = "STRONG_INFLOW", "STRONG_BULLISH"
    elif avg_change > 1: flow_signal, gold_signal = "INFLOW", "BULLISH"
    elif avg_change < -3: flow_signal, gold_signal = "STRONG_OUTFLOW", "STRONG_BEARISH"
    elif avg_change < -1: flow_signal, gold_signal = "OUTFLOW", "BEARISH"
    else: flow_signal, gold_signal = "NEUTRAL", "NEUTRAL"

    logger.info(f"✅ ETF Flow: {flow_signal} ({round(avg_change, 2)}%)")
    return {"etfs": etf_data, "avg_change_pct": round(avg_change, 2), "flow_signal": flow_signal, "gold_signal": gold_signal}

# ================= COT ОТЧЁТ =================
async def fetch_cot_report() -> Dict[str, Any]:
    try:
        r = await http_client.get(
            "https://raw.githubusercontent.com/datasets/cftc-commitment-of-traders/main/data/gold.csv",
            timeout=15
        )
        if r.status_code == 200:
            df = pd.read_csv(io.StringIO(r.text))
            if not df.empty and 'Noncommercial_Long' in df.columns and 'Noncommercial_Short' in df.columns:
                noncomm_net = df['Noncommercial_Long'] - df['Noncommercial_Short']
                latest_net = int(noncomm_net.iloc[-1])
                prev_net = int(noncomm_net.iloc[-5]) if len(noncomm_net) >= 5 else int(noncomm_net.iloc[0])
                trend = "INCREASING" if latest_net > prev_net else "DECREASING"
                
                lookback = min(156, len(noncomm_net))
                recent = noncomm_net.iloc[-lookback:]
                percentile = (recent < latest_net).mean() * 100
                
                if percentile > 85: cot_signal = "EXTREME_BULLISH"
                elif percentile > 65: cot_signal = "BULLISH"
                elif percentile < 15: cot_signal = "EXTREME_BEARISH"
                elif percentile < 35: cot_signal = "BEARISH"
                else: cot_signal = "NEUTRAL"
                
                report_date = str(df['Date'].iloc[-1]) if 'Date' in df.columns else 'Unknown'
                save_cot_to_db(report_date, latest_net, percentile, cot_signal)
                
                logger.info(f"✅ COT: {cot_signal} (p={round(percentile, 1)}%, trend={trend})")
                return {
                    'report_date': report_date, 'noncommercial_net': latest_net,
                    'percentile': round(percentile, 1), 'trend': trend,
                    'cot_signal': cot_signal,
                    'gold_signal': 'BULLISH' if 'BULLISH' in cot_signal else 'BEARISH' if 'BEARISH' in cot_signal else 'NEUTRAL'
                }
    except Exception as e:
        logger.warning(f"COT error: {e}")
    
    return _get_last_cot_from_db()

def save_cot_to_db(report_date: str, net: int, percentile: float, signal: str):
    try:
        conn = sqlite3.connect(DB_FILE)
        c = conn.cursor()
        c.execute("INSERT INTO cot_history (timestamp, report_date, noncommercial_net, percentile, signal) VALUES (?, ?, ?, ?, ?)",
                  (datetime.now(timezone.utc).isoformat(), report_date, net, round(percentile, 1), signal))
        conn.commit()
        conn.close()
    except: pass

def _get_last_cot_from_db() -> Dict:
    try:
        conn = sqlite3.connect(DB_FILE)
        c = conn.cursor()
        c.execute("SELECT * FROM cot_history ORDER BY id DESC LIMIT 1")
        row = c.fetchone()
        conn.close()
        if row:
            return {
                'report_date': row[2], 'noncommercial_net': row[3],
                'percentile': row[4], 'cot_signal': row[5],
                'gold_signal': 'BULLISH' if 'BULLISH' in row[5] else 'BEARISH' if 'BEARISH' in row[5] else 'NEUTRAL'
            }
    except: pass
    return {'cot_signal': 'NEUTRAL', 'gold_signal': 'NEUTRAL'}

# ================= GPR =================
async def fetch_gpr() -> Dict[str, Any]:
    if not NEWSAPI_KEY: return {'gpr_level': 'UNAVAILABLE', 'gold_signal': 'NEUTRAL'}
    try:
        r = await http_client.get(f"https://newsapi.org/v2/everything?q=geopolitical+war+sanctions&pageSize=5&apiKey={NEWSAPI_KEY}")
        if r.status_code == 200:
            total = r.json().get('totalResults', 0)
            r_all = await http_client.get(f"https://newsapi.org/v2/everything?q=all&pageSize=5&apiKey={NEWSAPI_KEY}")
            total_all = r_all.json().get('totalResults', 1) if r_all.status_code == 200 else 1000
            ratio = total / max(total_all, 1)
            
            if ratio > 0.15: level, gold_signal = "HIGH", "BULLISH"
            elif ratio > 0.08: level, gold_signal = "ELEVATED", "SLIGHTLY_BULLISH"
            else: level, gold_signal = "LOW", "NEUTRAL"
            
            return {'gpr_level': level, 'mentions_share': round(ratio, 4), 'gold_signal': gold_signal}
    except: pass
    return {'gpr_level': 'UNAVAILABLE', 'gold_signal': 'NEUTRAL'}

# ================= 🔥 ГЛАВНЫЙ МАКРО‑АНАЛИЗ =================
async def analyze_gold_macro() -> Dict[str, Any]:
    logger.info("🥇 Запуск Gold Macro анализа...")
    
    twd, tips, etf, cot, gpr = await asyncio.gather(
        fetch_trade_weighted_dollar(), fetch_tips_yield(), analyze_etf_flows(), fetch_cot_report(), fetch_gpr()
    )

    signals = {}
    score = 50.0
    
    # 1. TIPS РЕАЛЬНАЯ ДОХОДНОСТЬ (Вес 30%)
    tips_signal = tips.get('gold_signal', 'NEUTRAL')
    tips_z = tips.get('zscore', 0)
    if tips_signal == 'STRONG_BULLISH':
        score += 25; signals["tips"] = ("STRONG_BUY", 25)
    elif tips_signal == 'BULLISH':
        score += 16; signals["tips"] = ("BUY", 16)
    elif tips_signal == 'STRONG_BEARISH':
        score -= 22; signals["tips"] = ("STRONG_SELL", 22)
    elif tips_signal == 'BEARISH':
        score -= 14; signals["tips"] = ("SELL", 14)
    else:
        signals["tips"] = ("NEUTRAL", 0)
    
    if abs(tips_z) > 2.0:
        if tips_z > 0: score -= 8
        else: score += 8
    
    # 2. TRADE-WEIGHTED DOLLAR (Вес 25%)
    twd_signal = twd.get('gold_signal', 'NEUTRAL')
    if twd_signal == 'STRONG_BULLISH':
        score += 18; signals["twd"] = ("STRONG_BUY", 18)
    elif twd_signal == 'BULLISH':
        score += 12; signals["twd"] = ("BUY", 12)
    elif twd_signal == 'STRONG_BEARISH':
        score -= 16; signals["twd"] = ("STRONG_SELL", 16)
    elif twd_signal == 'BEARISH':
        score -= 10; signals["twd"] = ("SELL", 10)
    else:
        signals["twd"] = ("NEUTRAL", 0)
    
    # 3. ETF FLOW (Вес 20%)
    etf_signal = etf.get('gold_signal', 'NEUTRAL')
    if etf_signal == 'STRONG_BULLISH':
        score += 14; signals["etf"] = ("STRONG_BUY", 14)
    elif etf_signal == 'BULLISH':
        score += 9; signals["etf"] = ("BUY", 9)
    elif etf_signal == 'STRONG_BEARISH':
        score -= 12; signals["etf"] = ("STRONG_SELL", 12)
    elif etf_signal == 'BEARISH':
        score -= 8; signals["etf"] = ("SELL", 8)
    else:
        signals["etf"] = ("NEUTRAL", 0)
    
    # 4. COT (Вес 15%)
    cot_signal = cot.get('gold_signal', 'NEUTRAL')
    if cot_signal == 'BULLISH':
        score += 10; signals["cot"] = ("BUY", 10)
    elif cot_signal == 'BEARISH':
        score -= 10; signals["cot"] = ("SELL", 10)
    else:
        signals["cot"] = ("NEUTRAL", 0)
    
    # 5. GPR (Вес 10%)
    gpr_signal = gpr.get('gold_signal', 'NEUTRAL')
    if gpr_signal == 'BULLISH':
        score += 6; signals["gpr"] = ("SLIGHT_BUY", 6)
    elif gpr_signal == 'SLIGHTLY_BULLISH':
        score += 3; signals["gpr"] = ("SLIGHT_BUY", 3)
    else:
        signals["gpr"] = ("NEUTRAL", 0)
    
    score = round(max(3, min(97, score)), 1)
    
    # Z-score макро-скора
    SCORE_HISTORY.append(score)
    score_z = calculate_zscore(score, SCORE_HISTORY)
    
    # Определяем сигнал
    if score > 62: signal, confidence = "BUY", min(0.92, score / 100)
    elif score > 54: signal, confidence = "BUY", min(0.68, (score - 50) / 50)
    elif score < 38: signal, confidence = "SELL", min(0.92, (100 - score) / 100)
    elif score < 46: signal, confidence = "SELL", min(0.68, (50 - score) / 50)
    else: signal, confidence = "WAIT", 0.0
    
    confidence = round(confidence, 4)

    logger.info(f"🥇 Gold Macro Score: {score} (z={score_z:.1f}) → {signal} (conf={confidence:.3f})")
    return {
        "macro_score": score, "score_zscore": round(score_z, 2),
        "signal": signal, "confidence": confidence,
        "signals_breakdown": {k: v[0] for k, v in signals.items()},
        "metrics": {"twd": twd, "tips": tips, "etf_flows": etf, "cot_report": cot, "geopolitical_risk": gpr}
    }

# ================= ГЛАВНЫЙ СИГНАЛ =================
async def get_gold_macro_signal() -> Dict[str, Any]:
    start = time.time()
    analysis = await analyze_gold_macro()
    
    features = {
        "macro_score": analysis['macro_score'],
        "score_zscore": analysis['score_zscore'],
        "tips_yield": analysis['metrics'].get('tips', {}).get('real_yield', 0),
        "twd_change": analysis['metrics'].get('twd', {}).get('change_1m', 0),
        "etf_flow": analysis['metrics'].get('etf_flows', {}).get('avg_change_pct', 0)
    }
    await send_signal_to_hub(SYMBOL, analysis['signal'], analysis['confidence'], features)
    
    # Логируем в SQLite
    try:
        conn = sqlite3.connect(DB_FILE)
        c = conn.cursor()
        c.execute("INSERT INTO signals_log (timestamp, signal, confidence, macro_score) VALUES (?, ?, ?, ?)",
                  (datetime.now(timezone.utc).isoformat(), analysis['signal'], analysis['confidence'], analysis['macro_score']))
        conn.commit()
        conn.close()
    except: pass
    
    elapsed = int((time.time() - start) * 1000)
    logger.info(f"🥇 Gold: {analysis['signal']} conf={analysis['confidence']:.3f} | {elapsed}ms")
    
    return {"space_id": SPACE_ID, "signal": analysis['signal'], "confidence": analysis['confidence'], "macro_score": analysis['macro_score']}

# ================= АВТО-ОТПРАВКА =================
async def auto_send_loop():
    logger.info(f"⏳ Стартовый сон {STARTUP_SLEEP}с...")
    await log_to_hub("STARTUP", f"Gold Macro v4.0 запущен, жду {STARTUP_SLEEP}с")
    await asyncio.sleep(STARTUP_SLEEP)
    logger.info(f"🔄 Gold Macro [интервал={AUTO_SEND_INTERVAL}с]")
    while True:
        try: await get_gold_macro_signal()
        except Exception as e:
            logger.error(f"Auto: {e}")
            await log_to_hub("ERROR", f"Ошибка: {str(e)[:200]}")
        await asyncio.sleep(AUTO_SEND_INTERVAL)

# ================= FASTAPI =================
app = FastAPI(title="Tomiris Space 21 v4.0 STEEL")

@app.on_event("startup")
async def startup():
    asyncio.create_task(auto_send_loop())
    logger.info(f"🚀 Space 21 v4.0 | Хаб: {HUB_URL}")

@app.on_event("shutdown")
async def shutdown(): await http_client.aclose()

@app.get("/health")
async def health(): return {"space_id": SPACE_ID, "status": "operational", "version": "4.0"}
@app.head("/health")
async def health_head(): return {}
@app.get("/consilium")
async def consilium(): return await get_gold_macro_signal()
@app.get("/dxy")
async def dxy(): return await fetch_trade_weighted_dollar()
@app.get("/tips")
async def tips(): return await fetch_tips_yield()
@app.get("/etf")
async def etf(): return await analyze_etf_flows()
@app.get("/cot")
async def cot(): return await fetch_cot_report()
@app.get("/gpr")
async def gpr(): return await fetch_gpr()
@app.get("/full")
async def full(): return await analyze_gold_macro()
@app.get("/send_now")
async def send_now(): return await get_gold_macro_signal()
@app.get("/")
async def root(): return {"name": "Gold Macro & Flow v4.0 STEEL", "space_id": SPACE_ID}

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=7860)

print("🚀 SPACE 21 v4.0 — GOLD MACRO STEEL — ГОТОВ!")