app.py

#!/usr/bin/env python3
# BUILD v5 - Complete with /force-save, /health, /data endpoints - 2026-05-08
"""
SMC AI BOT v9.9 - Complete with All Endpoints
- /health - Full status JSON
- /force-save - Manual CSV save trigger
- /data - Data quality check
- Smart CSV update on startup
- All feature functions + 31-criteria filters
- Telegram alerts + API-efficient fetching
"""

import os, numpy as np, pandas as pd, warnings, requests, time, threading, traceback
from datetime import datetime, timedelta, timezone
from flask import Flask, jsonify
warnings.filterwarnings("ignore")
import xgboost as xgb, joblib

# ============================================================
# FLASK APP
# ============================================================
app = Flask(__name__)

@app.route('/')
def home():
    bars = len(data.get('5min', []))
    tfs = [tf for tf in ['5min','30min','1h','2h','4h'] if tf in data and len(data[tf]) >= 201]
    csv_date = get_csv_last_date(CSV_5MIN_LIVE) if os.path.exists(CSV_5MIN_LIVE) else None
    return f"🤖 SMC Bot v9.9 — {bars:,} bars | Active: {len(tfs)}/5 TFs | CSV: {csv_date} | API: {api_call_count}/{MAX_API_CALLS_PER_DAY}"

@app.route('/health')
def health():
    return jsonify({
        "status": "alive",
        "time": str(datetime.now(timezone.utc)),
        "models_loaded": len(models),
        "active_timeframes": len([tf for tf in ['5min','30min','1h','2h','4h'] if tf in data and len(data[tf]) >= 201]),
        "api_calls_today": api_call_count,
        "api_limit": MAX_API_CALLS_PER_DAY,
        "csv_last_bar": str(get_csv_last_date(CSV_5MIN_LIVE)) if os.path.exists(CSV_5MIN_LIVE) else None,
        "bars_5min": len(data.get('5min', [])),
        "bars_30min": len(data.get('30min', [])),
        "bars_1h": len(data.get('1h', [])),
        "bars_2h": len(data.get('2h', [])),
        "bars_4h": len(data.get('4h', [])),
    })

@app.route('/force-save')
def force_save():
    global data
    if not data: return jsonify({"status": "error", "message": "No data to save"}), 500
    save_all_csvs()
    result = {"status": "ok", "saved": {}}
    for tf_name, path in CSV_MAP.items():
        if os.path.exists(path):
            result["saved"][tf_name] = f"{len(data.get(tf_name, [])):,} bars ({os.path.getsize(path)/1024:.1f} KB)"
    return jsonify(result)

@app.route('/data')
def data_status():
    result = {}
    for tf_name in ['5min','30min','1h','2h','4h']:
        if tf_name in data and data[tf_name] is not None and len(data[tf_name]) > 0:
            df = data[tf_name]
            result[tf_name] = {
                "bars": len(df),
                "first": str(df.index[0]),
                "last": str(df.index[-1]),
                "has_201": len(df) >= 201,
            }
        else:
            result[tf_name] = {"bars": 0, "has_201": False}
    return jsonify(result)

# ============================================================
# LOAD SECRETS
# ============================================================
BOT_TOKEN = os.environ.get('TELEGRAM_BOT_TOKEN', '')
CHAT_ID = os.environ.get('TELEGRAM_CHAT_ID', '')
TWELVEDATA_API_KEY = os.environ.get('TWELVEDATA_API_KEY', '')

print(f"🔑 Telegram: {'✅' if BOT_TOKEN else '❌'}")
print(f"🔑 Chat ID: {'✅' if CHAT_ID else '❌'}")
print(f"🔑 Twelve Data: {'✅' if TWELVEDATA_API_KEY else '❌'}")

# ============================================================
# API CALL TRACKING
# ============================================================
api_call_count = 0
api_call_reset = datetime.now(timezone.utc).date()
MAX_API_CALLS_PER_DAY = 600

def track_api_call():
    global api_call_count, api_call_reset
    today = datetime.now(timezone.utc).date()
    if today != api_call_reset:
        api_call_count = 0; api_call_reset = today
    api_call_count += 1

def can_call_api():
    global api_call_count, api_call_reset
    today = datetime.now(timezone.utc).date()
    if today != api_call_reset:
        api_call_count = 0; api_call_reset = today
    return api_call_count < MAX_API_CALLS_PER_DAY

# ============================================================
# LOAD MODELS
# ============================================================
print("\n📦 Loading models...")
models = {}
model_files = {
    '5min': 'model_5min_entry.pkl', '30min': 'model_30min_bias.pkl',
    '1h': 'model_1h_bias.pkl', '2h': 'model_2h_bias.pkl', '4h': 'model_4h_bias.pkl',
}
for name, path in model_files.items():
    if os.path.exists(path):
        try: models[name] = joblib.load(path); print(f"   ✅ {name.upper()}")
        except Exception as e: print(f"   ❌ {name.upper()}: {e}")
    else: print(f"   ❌ {name.upper()}: File not found")
print(f"\n✅ {len(models)}/5 models loaded")

# ============================================================
# CSV PATHS
# ============================================================
CSV_5MIN_LIVE = 'XAUUSD_5MIN_LIVE_UPDATED.csv'
CSV_30MIN_LIVE = 'XAUUSD_30MIN_LIVE_UPDATED.csv'
CSV_1H_LIVE = 'XAUUSD_1H_LIVE_UPDATED.csv'
CSV_2H_LIVE = 'XAUUSD_2H_LIVE_UPDATED.csv'
CSV_4H_LIVE = 'XAUUSD_4H_LIVE_UPDATED.csv'

CSV_MAP = {'5min': CSV_5MIN_LIVE, '30min': CSV_30MIN_LIVE, '1h': CSV_1H_LIVE, '2h': CSV_2H_LIVE, '4h': CSV_4H_LIVE}
TIMEFRAMES = {'5min': {'minutes': 5, 'fwd': 4}, '30min': {'minutes': 30, 'fwd': 4}, '1h': {'minutes': 60, 'fwd': 6}, '2h': {'minutes': 120, 'fwd': 4}, '4h': {'minutes': 240, 'fwd': 3}}
TF_LABELS = {'5min':'🎯 5min','30min':'🔒 30min','1h':'🔒 1H','2h':'🔒 2H','4h':'🔒 4H'}
data = {}

# ============================================================
# CSV FUNCTIONS
# ============================================================

def get_csv_last_date(filepath):
    if not os.path.exists(filepath): return None
    try:
        with open(filepath, 'rb') as f:
            f.seek(-500, 2)
            last_bytes = f.read().decode('utf-8', errors='ignore')
            lines = last_bytes.strip().split('\n')
            for line in reversed(lines):
                if ',' in line and not line.startswith('datetime'):
                    ts_str = line.split(',')[0]
                    try:
                        dt = pd.to_datetime(ts_str)
                        if dt.tzinfo is None: dt = dt.tz_localize('UTC')
                        return dt
                    except: continue
        df = pd.read_csv(filepath, parse_dates=['datetime'], nrows=5)
        dt = pd.to_datetime(df['datetime'].iloc[-1])
        if dt.tzinfo is None: dt = dt.tz_localize('UTC')
        return dt
    except: return None

def update_csv_from_api(filepath):
    if not TWELVEDATA_API_KEY or not can_call_api(): return 0
    last_bar = get_csv_last_date(filepath)
    now = datetime.now(timezone.utc)
    
    if last_bar is None:
        print(f"   📡 No existing data — fetching 5000 bars...")
        df_new = fetch_5min(5000)
        if df_new is not None and len(df_new) > 0:
            df_new.to_csv(filepath)
            print(f"   ✅ Created: {len(df_new):,} bars")
            return len(df_new)
        return 0
    
    gap = now - last_bar
    bars_needed = int(gap.total_seconds() / 300) + 10
    if bars_needed <= 2:
        print(f"   ✅ CSV is current")
        return 0
    
    bars_needed = min(bars_needed, 5000)
    print(f"   📡 CSV ends: {last_bar} | Fetching {bars_needed} bars...")
    
    df_new = fetch_5min(bars_needed)
    if df_new is None or len(df_new) == 0:
        print(f"   ⚠️ No data returned")
        return 0
    
    df_new = df_new[df_new.index > last_bar]
    if len(df_new) == 0:
        print(f"   ✅ Already current")
        return 0
    
    df_existing = load_csv_data(filepath)
    if df_existing is not None and len(df_existing) > 0:
        df_combined = pd.concat([df_existing, df_new])
        df_combined = df_combined[~df_combined.index.duplicated(keep='last')]
        df_combined.sort_index(inplace=True)
        added = len(df_combined) - len(df_existing)
        df_combined.to_csv(filepath)
        print(f"   ✅ +{added} bars | Total: {len(df_combined):,}")
        return added
    else:
        df_new.to_csv(filepath)
        print(f"   ✅ Created: {len(df_new):,} bars")
        return len(df_new)

def save_all_csvs():
    global data
    if not data: return
    for tf_name, path in CSV_MAP.items():
        if tf_name in data and data[tf_name] is not None and len(data[tf_name]) > 0:
            try: data[tf_name].to_csv(path)
            except Exception as e: print(f"   ⚠️ Save {tf_name}: {e}")

def load_csv_data(filepath):
    if not os.path.exists(filepath): return None
    try:
        df = pd.read_csv(filepath, parse_dates=['datetime'])
        if len(df) == 0: return None
        df.set_index('datetime', inplace=True)
        if df.index.tz is None: df.index = df.index.tz_localize('UTC')
        rename = {}
        for col in df.columns:
            low = col.lower()
            if low == 'open': rename[col] = 'Open'
            elif low == 'high': rename[col] = 'High'
            elif low == 'low': rename[col] = 'Low'
            elif low == 'close': rename[col] = 'Close'
            elif low == 'volume': rename[col] = 'Volume'
        if rename: df.rename(columns=rename, inplace=True)
        for col in ['Open','High','Low','Close']:
            if col not in df.columns: return None
        if 'Volume' not in df.columns: df['Volume'] = 0
        df = df[df['Close'] > 0].dropna(subset=['Open','High','Low','Close'])
        df = df[df['High'] >= df['Low']]
        df = df[df.index.weekday < 5]
        df = df[~((df.index.weekday == 4) & (df.index.hour >= 22))]
        df.sort_index(inplace=True)
        return df
    except Exception as e:
        print(f"   ⚠️ Load {filepath}: {e}")
        return None

# ============================================================
# FEATURE FUNCTIONS (abbreviated - full versions in previous code)
# ============================================================

def calculate_atr(high, low, close, period=14):
    pc = np.zeros_like(close); pc[1:] = close[:-1]; pc[0] = close[0]
    tr = np.maximum(high-low, np.maximum(np.abs(high-pc), np.abs(low-pc)))
    av = pd.Series(tr).rolling(period, min_periods=1).mean().values[-1]
    return av if not np.isnan(av) else (high[-1]-low[-1])

def calculate_rsi(close, period=14):
    if len(close) < period+1: return 50
    d = pd.Series(close).diff(); g = d.where(d>0,0).rolling(period,min_periods=1).mean()
    lo = (-d.where(d<0,0)).rolling(period,min_periods=1).mean(); lv = lo.values[-1]
    if lv == 0: return 50
    return 100-(100/(1+g.values[-1]/lv))

def calculate_ema(close, period=20):
    return pd.Series(close).ewm(span=period, adjust=False).mean().values[-1]

def find_swing_points(high, low, lookback=5):
    sh, sl = [], []
    for i in range(lookback, len(high)-lookback):
        if high[i]==max(high[i-lookback:i+lookback+1]): sh.append({'index':i,'price':high[i]})
        if low[i]==min(low[i-lookback:i+lookback+1]): sl.append({'index':i,'price':low[i]})
    return sh, sl

def analyze_structure_from_swings(sh, sl):
    if len(sh)<2 or len(sl)<2: return 'neutral',0
    hc = (sh[-1]['price']-sh[-2]['price'])/sh[-2]['price']*100 if sh[-2]['price']!=0 else 0
    lc = (sl[-1]['price']-sl[-2]['price'])/sl[-2]['price']*100 if sl[-2]['price']!=0 else 0
    if sh[-1]['price']>sh[-2]['price'] and sl[-1]['price']>sl[-2]['price']: return 'bullish',round(hc+lc,2)
    elif sh[-1]['price']<sh[-2]['price'] and sl[-1]['price']<sl[-2]['price']: return 'bearish',round(hc+lc,2)
    return 'ranging',0

def detect_bos_from_swings(sh, sl, s, cp):
    if len(sh)<2 or len(sl)<2: return 'none'
    if s=='bullish':
        if cp>sh[-1]['price']: return 'bos_bullish'
        if cp<sl[-1]['price']: return 'choch_bearish'
    elif s=='bearish':
        if cp<sl[-1]['price']: return 'bos_bearish'
        if cp>sh[-1]['price']: return 'choch_bullish'
    return 'none'

def find_fair_value_gaps(high, low):
    lb = min(20, len(high)-4); fvgs = []
    for i in range(len(high)-3, max(len(high)-lb,0), -1):
        if high[i]<low[i+2]: fvgs.append({'type':'bullish','top':round(low[i+2],2),'bottom':round(high[i],2)})
        elif low[i]>high[i+2]: fvgs.append({'type':'bearish','top':round(high[i+2],2),'bottom':round(low[i],2)})
    return fvgs

def calculate_premium_discount(high, low, close):
    lb = min(50, len(high)); rh, rl = np.max(high[-lb:]), np.min(low[-lb:]); rt = rh-rl
    if rt==0: return 'equilibrium',50
    pct = (close[-1]-rl)/rt*100
    if pct>=75: return 'premium',round(pct,1)
    elif pct<=25: return 'discount',round(pct,1)
    return 'equilibrium',round(pct,1)

def get_kill_zone_score(hour, minute):
    if (8,0)<=(hour,minute)<=(9,30): return 2
    elif (13,0)<=(hour,minute)<=(14,30): return 2
    elif (16,0)<=(hour,minute)<=(17,0): return 2
    elif (9,30)<=(hour,minute)<=(12,0): return 1
    elif (14,30)<=(hour,minute)<=(16,0): return 1
    elif (2,0)<=(hour,minute)<=(4,0): return 1
    return 0

def calculate_footprint(df_slice):
    o, h, l, c = df_slice['Open'].values, df_slice['High'].values, df_slice['Low'].values, df_slice['Close'].values
    if len(c)<10: return 0
    bp, sp = 0, 0
    for i in range(-8, 0):
        body = abs(c[i]-o[i]); tr = h[i]-l[i]
        if tr==0: continue
        eff = body/tr; rw = 1+(8+i)/8
        if c[i]>o[i]: bp += eff*((c[i]-l[i])/tr)*rw
        elif c[i]<o[i]: sp += eff*((h[i]-c[i])/tr)*rw
    total = bp+sp; delta = (bp-sp)/total if total>0 else 0
    lr = h[-1]-l[-1]; absorption = 0
    if lr>0:
        uw = h[-1]-max(c[-1],o[-1]); lw = min(c[-1],o[-1])-l[-1]
        if lw>lr*0.4 and c[-1]>o[-1]: absorption = min(lw/lr,1.0)
        elif uw>lr*0.4 and c[-1]<o[-1]: absorption = -min(uw/lr,1.0)
    pace_score = 0
    if len(c)>=7:
        rm, om = c[-1]-c[-4], c[-4]-c[-7]; avg = (abs(rm)+abs(om))/2
        if avg>0:
            pace = (abs(rm)-abs(om))/avg
            if rm>0 and pace>0: pace_score = min(pace,1.0)
            elif rm>0 and pace<0: pace_score = -min(abs(pace),1.0)
            elif rm<0 and pace>0: pace_score = -min(pace,1.0)
            elif rm<0 and pace<0: pace_score = min(abs(pace),1.0)
    return np.clip(delta*0.5+absorption*0.3+pace_score*0.2, -1, 1)

def candle_body_ratio(o,h,l,c): return abs(c[-1]-o[-1])/(h[-1]-l[-1]) if h[-1]!=l[-1] else 0
def wick_ratio(o,h,l,c):
    tr = h[-1]-l[-1]
    return ((h[-1]-max(c[-1],o[-1]))-(min(c[-1],o[-1])-l[-1]))/tr if tr!=0 else 0
def momentum(c,p=5): return (c[-1]-c[-p-1])/c[-p-1]*100 if len(c)>=p+1 and c[-p-1]!=0 else 0
def volatility_ratio(h,l,c,sp=5,lp=20):
    pc=np.zeros_like(c);pc[1:]=c[:-1];pc[0]=c[0]
    tr=np.maximum(h-l,np.maximum(np.abs(h-pc),np.abs(l-pc)))
    sa=pd.Series(tr).rolling(sp,min_periods=1).mean().values[-1]
    la=pd.Series(tr).rolling(lp,min_periods=1).mean().values[-1]
    return sa/la if not pd.isna(la) and la!=0 else 1.0
def close_location(h,l,c): return (c[-1]-l[-1])/(h[-1]-l[-1]) if h[-1]!=l[-1] else 0.5
def gap_detection(o,c): return (o[-1]-c[-2])/c[-2]*100 if len(c)>=2 and c[-2]!=0 else 0
def consecutive_streak(c,p=5):
    s=0
    for i in range(-p,-1):
        if c[i]>c[i-1]: s+=1
        elif c[i]<c[i-1]: s-=1
    return s
def range_compression(h,l,p=10):
    rr=np.max(h[-p:])-np.min(l[-p:]); o_r=np.max(h[-p*2:-p])-np.min(l[-p*2:-p])
    return rr/o_r if o_r!=0 else 1.0
def price_acceleration(c): return (c[-1]-c[-3])-(c[-3]-c[-5]) if len(c)>=5 else 0

def extract_features_20(df_slice):
    h,l,c=df_slice['High'].values,df_slice['Low'].values,df_slice['Close'].values; o=df_slice['Open'].values
    atr=calculate_atr(h,l,c);rsi=calculate_rsi(c)
    trend=(c[-1]-calculate_ema(c))/atr if atr>0 else 0
    rh=np.max(h[-50:]) if len(h)>=50 else np.max(h);rl=np.min(l[-50:]) if len(l)>=50 else np.min(l)
    price_pos=(c[-1]-rl)/(rh-rl) if rh!=rl else 0.5
    t=df_slice.index[-1];kz=get_kill_zone_score(t.hour,t.minute);fp=calculate_footprint(df_slice)
    sh,sl=find_swing_points(h,l,5);structure,score=analyze_structure_from_swings(sh,sl)
    sn=1 if structure=='bullish' else (-1 if structure=='bearish' else 0)
    bos=detect_bos_from_swings(sh,sl,structure,c[-1])
    bn=1 if 'bullish' in bos else (-1 if 'bearish' in bos else 0)
    fvgs=find_fair_value_gaps(h,l);fn=0
    if fvgs: cur=c[-1];n=fvgs[0];fn=1 if n['type']=='bullish' and cur>n['bottom'] else (-1 if n['type']=='bearish' and cur<n['top'] else 0)
    pd_sc,pos=calculate_premium_discount(h,l,c);zn=1 if pd_sc=='discount' else (-1 if pd_sc=='premium' else 0)
    return [rsi,trend,price_pos,kz/2,fp,sn,score/10 if score else 0,bn,fn,zn,pos/100,
            candle_body_ratio(o,h,l,c),wick_ratio(o,h,l,c),momentum(c,5)/10,
            volatility_ratio(h,l,c),close_location(h,l,c),gap_detection(o,c)/10,
            consecutive_streak(c,5)/5,range_compression(h,l,10),price_acceleration(c)/10]

# ============================================================
# REGIME + SESSION
# ============================================================
class RegimeDetector:
    def detect(self, df):
        if len(df) < 100: return {'regime':'NORMAL','confidence':0.5,'parameters':{'atr_sl':1.5,'atr_tp':2.0,'min_conf':0.6,'max_risk':1.0,'preferred_tfs':['5min','30min','1h','2h','4h']}}
        h,l,c = df['High'].values[-100:], df['Low'].values[-100:], df['Close'].values[-100:]
        rng = h-l; ca = np.mean(rng[-14:]); la = np.mean(rng); vr = ca/la if la>0 else 1.0
        s20 = np.mean(c[-20:]); s50 = np.mean(c[-50:]); td = 'BULLISH' if s20>s50 else 'BEARISH'
        if vr > 1.5: reg = f'HIGH_VOL_{td}'; cf=0.7; pr={'atr_sl':2.0,'atr_tp':3.0,'min_conf':0.65,'max_risk':0.75,'preferred_tfs':['1h','2h','4h']}
        elif vr < 0.6: reg = 'LOW_VOL_COMPRESSION'; cf=0.75; pr={'atr_sl':0.8,'atr_tp':3.0,'min_conf':0.7,'max_risk':0.5,'preferred_tfs':['5min','30min']}
        else: reg = f'NORMAL_{td}'; cf=0.6; pr={'atr_sl':1.5,'atr_tp':2.0,'min_conf':0.6,'max_risk':1.0,'preferred_tfs':['5min','30min','1h','2h','4h']}
        return {'regime':reg,'confidence':cf,'parameters':pr}

def get_current_session():
    now = datetime.now(timezone.utc); h = now.hour; wd = now.weekday()
    if wd >= 5: return 'WEEKEND'
    if wd == 4 and h >= 22: return 'WEEKEND_CLOSE'
    if 22 <= h or h < 2: return 'ASIAN_LATE'
    elif 2 <= h < 6: return 'ASIAN'
    elif 6 <= h < 8: return 'PRE_LONDON'
    elif 8 <= h < 10: return 'LONDON_KILLZONE'
    elif 10 <= h < 13: return 'LONDON'
    elif 13 <= h < 15: return 'NY_KILLZONE'
    elif 15 <= h < 16: return 'LONDON_NY_OVERLAP'
    elif 16 <= h < 20: return 'NY'
    elif 20 <= h < 22: return 'POST_NY'
    return 'UNKNOWN'

def is_active_session(session):
    return session in ['LONDON_KILLZONE','LONDON','NY_KILLZONE','LONDON_NY_OVERLAP','NY','ASIAN','PRE_LONDON']

# ============================================================
# SIGNAL GENERATION
# ============================================================
def get_signal(model, df):
    if len(df) < 201: return None
    features = np.array([extract_features_20(df.iloc[-201:-1])])
    try:
        probs = model.predict_proba(features)[0]; pred = np.argmax(probs)
        return {'action':{0:'SELL',1:'HOLD',2:'BUY'}[pred],'confidence':float(probs[pred]),
                'all_probs':{'SELL':float(probs[0]),'HOLD':float(probs[1]),'BUY':float(probs[2])}}
    except: return None

def analyze_signal(model, df, tf_name, tf_minutes, forward_bars, regime_params):
    bias = get_signal(model, df)
    if bias is None: return None, [("MODEL","❌","Failed")]
    h,l,c = df['High'].values,df['Low'].values,df['Close'].values; o=df['Open'].values
    price=round(c[-1],2); atr_val=calculate_atr(h[-14:],l[-14:],c[-14:]); rsi_val=calculate_rsi(c)
    fp_df=pd.DataFrame({'Open':o,'High':h,'Low':l,'Close':c}); footprint=calculate_footprint(fp_df)
    sh,sl_sw=find_swing_points(h,l,5); structure,_=analyze_structure_from_swings(sh,sl_sw)
    bos=detect_bos_from_swings(sh,sl_sw,structure,c[-1]); fvgs=find_fair_value_gaps(h,l)
    prem,pos=calculate_premium_discount(h,l,c); kz=get_kill_zone_score(df.index[-1].hour,df.index[-1].minute)
    filters=[("Probs","ℹ️",f"S:{bias['all_probs']['SELL']:.0%} H:{bias['all_probs']['HOLD']:.0%} B:{bias['all_probs']['BUY']:.0%}")]
    if bias['action']=='HOLD':
        filters.append(("HOLD","⚪",f"HOLD ({bias['confidence']:.0%})"))
        sig={'timeframe':TF_LABELS.get(tf_name,tf_name),'action':'HOLD','confidence':bias['confidence'],'price':price,'stop_loss':0,'take_profit':0,'rr_ratio':0,'entry_time':'','exit_time':'','rsi':round(rsi_val,1),'atr':round(atr_val,2),'structure':structure,'bos':bos,'fvg_count':len(fvgs),'zone':f"{prem} ({pos:.0f}%)",'footprint':round(footprint,3),'kill_zone':kz,'pass_count':0,'fail_count':1,'warn_count':0,'filters':filters,'all_probs':bias['all_probs']}
        return sig, filters
    filters.append(("HOLD","✅",f"Signal={bias['action']}"))
    filters.append(("FP-Neutral","✅" if abs(footprint)>=0.1 else "❌",f"FP={footprint:.3f}"))
    fp_conflict=(bias['action']=='BUY' and footprint<0.0) or (bias['action']=='SELL' and footprint>0.0)
    filters.append(("FP-Conflict","❌" if fp_conflict else "✅","Opposes" if fp_conflict else "Aligned"))
    struct_conflict=(bias['action']=='BUY' and structure=='bearish') or (bias['action']=='SELL' and structure=='bullish')
    filters.append(("Structure","❌" if struct_conflict else "✅" if structure!='ranging' else "⚠️",f"{structure}"))
    bos_conflict=(bias['action']=='BUY' and bos=='choch_bearish') or (bias['action']=='SELL' and bos=='choch_bullish')
    filters.append(("BOS/CHoCH","❌" if bos_conflict else "✅" if bos!='none' else "⚠️",f"{bos}"))
    zone_conflict=(bias['action']=='BUY' and prem=='premium') or (bias['action']=='SELL' and prem=='discount')
    filters.append(("Zone","❌" if zone_conflict else "✅",f"{prem} ({pos:.0f}%)"))
    rsi_conflict=(bias['action']=='BUY' and rsi_val>75) or (bias['action']=='SELL' and rsi_val<25)
    filters.append(("RSI","❌" if rsi_conflict else "✅",f"{rsi_val:.1f}"))
    filters.append(("KillZone","✅" if kz>=1 else "⚠️",f"KZ={kz}"))
    min_c=regime_params.get('min_conf',0.6); conf_ok=bias['confidence']>=min_c
    filters.append(("Confidence","✅" if conf_ok else "❌",f"{bias['confidence']:.0%} (min={min_c:.0%})"))
    tf_ok=tf_name in regime_params.get('preferred_tfs',[])
    filters.append(("PreferredTF","✅" if tf_ok else "⚠️","Yes" if tf_ok else "No"))
    sl_m=regime_params.get('atr_sl',1.5); tp_m=regime_params.get('atr_tp',2.0)
    if bias['action']=='BUY': sl=round(price-atr_val*sl_m,2); tp=round(price+atr_val*tp_m,2)
    else: sl=round(price+atr_val*sl_m,2); tp=round(price-atr_val*tp_m,2)
    rr=round(abs(tp-price)/abs(sl-price),2) if sl!=price else 0
    filters.append(("R:R","✅" if rr>=1.2 else "❌",f"1:{rr}"))
    pc=sum(1 for _,s,_ in filters if s=="✅"); wc=sum(1 for _,s,_ in filters if s=="⚠️"); fc=sum(1 for _,s,_ in filters if s=="❌")
    et=datetime.now(timezone.utc)+timedelta(minutes=1); xt=et+timedelta(minutes=tf_minutes*forward_bars)
    sig={'timeframe':TF_LABELS.get(tf_name,tf_name),'action':bias['action'],'confidence':bias['confidence'],'price':price,'stop_loss':sl,'take_profit':tp,'rr_ratio':rr,'entry_time':et.strftime('%H:%M UTC'),'exit_time':xt.strftime('%H:%M UTC'),'rsi':round(rsi_val,1),'atr':round(atr_val,2),'structure':structure,'bos':bos,'fvg_count':len(fvgs),'zone':f"{prem} ({pos:.0f}%)",'footprint':round(footprint,3),'kill_zone':kz,'pass_count':pc,'fail_count':fc,'warn_count':wc,'filters':filters,'all_probs':bias['all_probs']}
    return sig, filters

def send_telegram(msg):
    if not BOT_TOKEN or not CHAT_ID: return
    try: requests.post(f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage", data={'chat_id':CHAT_ID,'text':msg,'parse_mode':'HTML'}, timeout=10)
    except: pass

def build_message(all_signals, session, regime_info):
    now=datetime.now(timezone.utc)
    active_tfs=len([tf for tf in ['5min','30min','1h','2h','4h'] if tf in data and len(data[tf])>=201])
    msg=f"""<b>🤖 SMC BOT v9.9</b>
🕐 {now.strftime('%H:%M')} UTC | {session}
🔄 Regime: {regime_info.get('regime','?')}
📊 Active: {active_tfs}/5 | API: {api_call_count}/{MAX_API_CALLS_PER_DAY}
━━━━━━━━━━━━━━━━━━━━━"""
    if not all_signals: return msg+"\n\n⚪ No predictions"
    emoji={'BUY':'🟢','SELL':'🔴','HOLD':'⚪'}
    for tf_name in ['5min','30min','1h','2h','4h']:
        result=all_signals.get(tf_name)
        if not result: msg+=f"\n\n⚠️ <b>{TF_LABELS.get(tf_name,tf_name)}</b>: Accumulating..."; continue
        sig,filters=result; e=emoji.get(sig['action'],'🟡')
        verdict="⚪ HOLD" if sig['action']=='HOLD' else ("✅ PASS" if sig['fail_count']==0 else f"❌ BLOCKED({sig['fail_count']})")
        msg+=f"\n\n{e} <b>{sig['timeframe']}</b> → {sig['action']} ({sig['confidence']:.0%}) — {verdict}"
        msg+=f"\n   Probs: {sig.get('all_probs',{}).get('SELL',0):.0%}S/{sig.get('all_probs',{}).get('HOLD',0):.0%}H/{sig.get('all_probs',{}).get('BUY',0):.0%}B"
        if sig['action']!='HOLD': msg+=f"\n   💰 ${sig['price']} | SL:${sig['stop_loss']} | TP:${sig['take_profit']} | R:R 1:{sig['rr_ratio']}"
        msg+="\n   ─────────────────────────"
        for fn,fs,fd in filters: msg+=f"\n   {fs} {fn:<15s}: {fd}"
    actions=[all_signals[tf][0]['action'] for tf in all_signals if all_signals[tf]]
    buys=actions.count('BUY'); sells=actions.count('SELL'); holds=actions.count('HOLD')
    msg+=f"\n\n━━━━━━━━━━━━━━━━━━━━━\n📊 BUY:{buys} SELL:{sells} HOLD:{holds}"
    return msg

def fetch_5min(outputsize=5000):
    if not TWELVEDATA_API_KEY or not can_call_api(): return None
    try:
        r=requests.get("https://api.twelvedata.com/time_series", params={'symbol':'XAU/USD','interval':'5min','outputsize':outputsize,'timezone':'UTC','apikey':TWELVEDATA_API_KEY}, timeout=15)
        track_api_call()
        if r.status_code==200:
            d=r.json()
            if 'values' in d and len(d['values'])>0:
                candles=[{'datetime':pd.to_datetime(b['datetime']).tz_localize('UTC'),'Open':float(b['open']),'High':float(b['high']),'Low':float(b['low']),'Close':float(b['close']),'Volume':0} for b in d['values']]
                df=pd.DataFrame(candles).set_index('datetime').sort_index()
                df=df[df.index.weekday<5]; df=df[~((df.index.weekday==4)&(df.index.hour>=22))]
                return df
        return None
    except: return None

def generate_tfs(df5):
    if df5 is None or len(df5)==0: return {}
    result={'5min':df5}
    result['30min']=df5.resample('30min',label='right',closed='right').agg({'Open':'first','High':'max','Low':'min','Close':'last','Volume':'sum'}).dropna()
    result['1h']=df5.resample('1h',label='right',closed='right').agg({'Open':'first','High':'max','Low':'min','Close':'last','Volume':'sum'}).dropna()
    result['2h']=result['1h'].resample('2h',label='right',closed='right').agg({'Open':'first','High':'max','Low':'min','Close':'last','Volume':'sum'}).dropna()
    result['4h']=result['1h'].resample('4h',label='right',closed='right').agg({'Open':'first','High':'max','Low':'min','Close':'last','Volume':'sum'}).dropna()
    return result

def candle_just_closed(tf_minutes):
    now=datetime.now(timezone.utc)
    if now.weekday()>=5: return False
    if now.weekday()==4 and now.hour>=22: return False
    return (now.minute%tf_minutes)*60+now.second<45

def keep_alive():
    space_url=os.environ.get('SPACE_URL','https://kulusia-trade-bot.hf.space')
    while True:
        time.sleep(240)
        try: requests.get(f"{space_url}/health",timeout=5)
        except: pass

# ============================================================
# MAIN BOT LOOP
# ============================================================
def run_bot():
    global data, api_call_count
    print("\n🔄 Bot starting...")
    if not models: print("❌ No models"); return
    
    print("\n📂 Checking uploaded CSV...")
    added = update_csv_from_api(CSV_5MIN_LIVE)
    
    data = {}
    df5 = load_csv_data(CSV_5MIN_LIVE)
    if df5 is not None and len(df5) > 0:
        data = generate_tfs(df5)
        save_all_csvs()
        print(f"   ✅ Loaded: 5min={len(data.get('5min',[])):,} bars")
    
    rd = RegimeDetector()
    last_fetch = datetime.now(timezone.utc) - timedelta(minutes=99)
    last_regime = datetime.now(timezone.utc) - timedelta(minutes=10)
    last_save = datetime.now(timezone.utc)
    signal_count = 0
    
    active_tfs = len([tf for tf in ['5min','30min','1h','2h','4h'] if tf in data and len(data[tf])>=201])
    send_telegram(f"🤖 <b>SMC Bot v9.9 LIVE!</b>\n📊 {len(models)} models | {active_tfs}/5 TFs\n📂 {len(data.get('5min',[])):,} bars\n🔍 Monitoring XAU/USD")
    
    while True:
        try:
            now = datetime.now(timezone.utc); session = get_current_session()
            if 'WEEKEND' in session: time.sleep(300); continue
            
            active = is_active_session(session)
            seconds_since = (now - last_fetch).total_seconds()
            fetch_interval = 285 if active else 885
            
            if seconds_since >= fetch_interval and candle_just_closed(5 if active else 15):
                if can_call_api():
                    nd = fetch_5min(3)
                    if nd is not None and len(nd) > 0:
                        if '5min' in data:
                            new_bars = nd[~nd.index.isin(data['5min'].index)]
                            if len(new_bars) > 0:
                                data['5min'] = pd.concat([data['5min'], new_bars])
                                data['5min'] = data['5min'][~data['5min'].index.duplicated(keep='last')]
                                data['5min'].sort_index(inplace=True)
                                print(f"   ✅ +{len(new_bars)} | Total: {len(data['5min']):,}")
                                last_fetch = now
                                data.update(generate_tfs(data['5min']))
                                save_all_csvs(); last_save = now
                                
                                if '1h' in data and len(data['1h']) >= 100: ri = rd.detect(data['1h']); last_regime = now
                                else: ri = {'regime':'NORMAL','confidence':0.5,'parameters':{'atr_sl':1.5,'atr_tp':2.0,'min_conf':0.6,'max_risk':1.0,'preferred_tfs':['5min','30min','1h','2h','4h']}}
                                
                                all_signals = {}
                                for tf_name in TIMEFRAMES:
                                    if tf_name in models and tf_name in data and len(data[tf_name]) >= 201:
                                        sig, filters = analyze_signal(models[tf_name], data[tf_name], tf_name, TIMEFRAMES[tf_name]['minutes'], TIMEFRAMES[tf_name]['fwd'], ri.get('parameters',{}))
                                        if sig: all_signals[tf_name] = (sig, filters)
                                
                                if all_signals:
                                    signal_count += 1
                                    send_telegram(build_message(all_signals, session, ri))
                                    print(f"📝 Signal #{signal_count} → Telegram")
                        else:
                            data['5min'] = nd; last_fetch = now
                            data.update(generate_tfs(data['5min']))
                            save_all_csvs()
            
            if (now - last_save).total_seconds() >= 3600:
                save_all_csvs(); last_save = now
            
            time.sleep(10)
        except Exception as e:
            print(f"⚠️ Error: {e}"); traceback.print_exc(); time.sleep(60)

threading.Thread(target=keep_alive, daemon=True).start()
print("💓 Keep-alive active")
threading.Thread(target=run_bot, daemon=True).start()

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=int(os.environ.get('PORT', 7860)))