Update app.py

app.py

@@ -1,17 +1,18 @@
 #!/usr/bin/env python3
-# BUILD v4 - Smart CSV update from last bar - 2026-05-08
+# BUILD v5 - Complete with /force-save, /health, /data endpoints - 2026-05-08
 """
-SMC AI BOT v9.9 - Hugging Face Deployment (SMART CSV UPDATE)
-- Checks where uploaded CSV ends (date/time)
-- Fetches only missing bars from Twelve Data
-- Updates CSV in Space with new data
-- Saves all timeframes to disk
-- Signal sent ONLY when new candle data arrives
+SMC AI BOT v9.9 - Complete with All Endpoints
+- /health - Full status JSON
+- /force-save - Manual CSV save trigger
+- /data - Data quality check
+- Smart CSV update on startup
+- All feature functions + 31-criteria filters
+- Telegram alerts + API-efficient fetching
 """
 
-import os, numpy as np, pandas as pd, warnings, requests, time, threading
+import os, numpy as np, pandas as pd, warnings, requests, time, threading, traceback
 from datetime import datetime, timedelta, timezone
-from flask import Flask
+from flask import Flask, jsonify
 warnings.filterwarnings("ignore")
 import xgboost as xgb, joblib
 
@@ -25,22 +26,51 @@ def home():
     bars = len(data.get('5min', []))
     tfs = [tf for tf in ['5min','30min','1h','2h','4h'] if tf in data and len(data[tf]) >= 201]
     csv_date = get_csv_last_date(CSV_5MIN_LIVE) if os.path.exists(CSV_5MIN_LIVE) else None
-    return f"🤖 SMC Bot v9.9 — {bars:,} bars | Active: {len(tfs)}/5 TFs | CSV ends: {csv_date} | API: {api_call_count}/{MAX_API_CALLS_PER_DAY}"
+    return f"🤖 SMC Bot v9.9 — {bars:,} bars | Active: {len(tfs)}/5 TFs | CSV: {csv_date} | API: {api_call_count}/{MAX_API_CALLS_PER_DAY}"
 
 @app.route('/health')
 def health():
-    return {
-        "status": "alive", "time": str(datetime.now(timezone.utc)),
-        "models": len(models), "active_tfs": len([tf for tf in ['5min','30min','1h','2h','4h'] if tf in data and len(data[tf]) >= 201]),
-        "api_calls_today": api_call_count, "csv_last_bar": str(get_csv_last_date(CSV_5MIN_LIVE)) if os.path.exists(CSV_5MIN_LIVE) else None,
-    }
+    return jsonify({
+        "status": "alive",
+        "time": str(datetime.now(timezone.utc)),
+        "models_loaded": len(models),
+        "active_timeframes": len([tf for tf in ['5min','30min','1h','2h','4h'] if tf in data and len(data[tf]) >= 201]),
+        "api_calls_today": api_call_count,
+        "api_limit": MAX_API_CALLS_PER_DAY,
+        "csv_last_bar": str(get_csv_last_date(CSV_5MIN_LIVE)) if os.path.exists(CSV_5MIN_LIVE) else None,
+        "bars_5min": len(data.get('5min', [])),
+        "bars_30min": len(data.get('30min', [])),
+        "bars_1h": len(data.get('1h', [])),
+        "bars_2h": len(data.get('2h', [])),
+        "bars_4h": len(data.get('4h', [])),
+    })
 
 @app.route('/force-save')
 def force_save():
     global data
-    if not data: return "❌ No data", 500
+    if not data: return jsonify({"status": "error", "message": "No data to save"}), 500
     save_all_csvs()
-    return "💾 Saved:\n" + "\n".join([f"{tf}: {len(data[tf]):,} bars" for tf in data])
+    result = {"status": "ok", "saved": {}}
+    for tf_name, path in CSV_MAP.items():
+        if os.path.exists(path):
+            result["saved"][tf_name] = f"{len(data.get(tf_name, [])):,} bars ({os.path.getsize(path)/1024:.1f} KB)"
+    return jsonify(result)
+
+@app.route('/data')
+def data_status():
+    result = {}
+    for tf_name in ['5min','30min','1h','2h','4h']:
+        if tf_name in data and data[tf_name] is not None and len(data[tf_name]) > 0:
+            df = data[tf_name]
+            result[tf_name] = {
+                "bars": len(df),
+                "first": str(df.index[0]),
+                "last": str(df.index[-1]),
+                "has_201": len(df) >= 201,
+            }
+        else:
+            result[tf_name] = {"bars": 0, "has_201": False}
+    return jsonify(result)
 
 # ============================================================
 # LOAD SECRETS
@@ -105,14 +135,12 @@ TF_LABELS = {'5min':'🎯 5min','30min':'🔒 30min','1h':'🔒 1H','2h':'🔒 2
 data = {}
 
 # ============================================================
-# SMART CSV FUNCTIONS
+# CSV FUNCTIONS
 # ============================================================
 
 def get_csv_last_date(filepath):
-    """Get the last timestamp from a CSV file (fast - reads only last few bytes)"""
     if not os.path.exists(filepath): return None
     try:
-        # Fast method: read last 500 bytes
         with open(filepath, 'rb') as f:
             f.seek(-500, 2)
             last_bytes = f.read().decode('utf-8', errors='ignore')
@@ -121,25 +149,21 @@ def get_csv_last_date(filepath):
                 if ',' in line and not line.startswith('datetime'):
                     ts_str = line.split(',')[0]
                     try:
-                        return pd.to_datetime(ts_str).tz_localize('UTC') if 'UTC' not in ts_str and '+' not in ts_str else pd.to_datetime(ts_str)
+                        dt = pd.to_datetime(ts_str)
+                        if dt.tzinfo is None: dt = dt.tz_localize('UTC')
+                        return dt
                     except: continue
-        # Fallback: read full file
         df = pd.read_csv(filepath, parse_dates=['datetime'], nrows=5)
-        return pd.to_datetime(df['datetime'].iloc[-1])
+        dt = pd.to_datetime(df['datetime'].iloc[-1])
+        if dt.tzinfo is None: dt = dt.tz_localize('UTC')
+        return dt
     except: return None
 
 def update_csv_from_api(filepath):
-    """
-    Check where CSV ends, fetch missing bars from API, update CSV.
-    Returns number of new bars added.
-    """
-    if not TWELVEDATA_API_KEY: return 0
-    if not can_call_api(): return 0
-    
+    if not TWELVEDATA_API_KEY or not can_call_api(): return 0
     last_bar = get_csv_last_date(filepath)
     now = datetime.now(timezone.utc)
     
-    # If no CSV or it's empty, fetch 5000 bars
     if last_bar is None:
         print(f"   📡 No existing data — fetching 5000 bars...")
         df_new = fetch_5min(5000)
@@ -149,30 +173,25 @@ def update_csv_from_api(filepath):
             return len(df_new)
         return 0
     
-    # Calculate how many bars are missing
     gap = now - last_bar
-    gap_minutes = gap.total_seconds() / 60
-    bars_needed = int(gap_minutes / 5) + 10  # 10 extra for safety
-    
+    bars_needed = int(gap.total_seconds() / 300) + 10
     if bars_needed <= 2:
-        print(f"   ✅ CSV is current (last bar: {last_bar})")
+        print(f"   ✅ CSV is current")
         return 0
     
     bars_needed = min(bars_needed, 5000)
-    print(f"   📡 CSV ends: {last_bar} | Need ~{bars_needed} bars...")
+    print(f"   📡 CSV ends: {last_bar} | Fetching {bars_needed} bars...")
     
     df_new = fetch_5min(bars_needed)
     if df_new is None or len(df_new) == 0:
+        print(f"   ⚠️ No data returned")
         return 0
     
-    # Filter to only bars after last_bar
     df_new = df_new[df_new.index > last_bar]
-    
     if len(df_new) == 0:
         print(f"   ✅ Already current")
         return 0
     
-    # Load existing CSV, merge, save
     df_existing = load_csv_data(filepath)
     if df_existing is not None and len(df_existing) > 0:
         df_combined = pd.concat([df_existing, df_new])
@@ -180,17 +199,13 @@ def update_csv_from_api(filepath):
         df_combined.sort_index(inplace=True)
         added = len(df_combined) - len(df_existing)
         df_combined.to_csv(filepath)
-        print(f"   ✅ +{added} bars | Total: {len(df_combined):,} | Last: {df_combined.index[-1]}")
+        print(f"   ✅ +{added} bars | Total: {len(df_combined):,}")
         return added
     else:
         df_new.to_csv(filepath)
         print(f"   ✅ Created: {len(df_new):,} bars")
         return len(df_new)
 
-# ============================================================
-# DATA PERSISTENCE
-# ============================================================
-
 def save_all_csvs():
     global data
     if not data: return
@@ -229,7 +244,7 @@ def load_csv_data(filepath):
         return None
 
 # ============================================================
-# ALL FEATURE FUNCTIONS
+# FEATURE FUNCTIONS (abbreviated - full versions in previous code)
 # ============================================================
 
 def calculate_atr(high, low, close, period=14):
@@ -527,20 +542,9 @@ def run_bot():
     print("\n🔄 Bot starting...")
     if not models: print("❌ No models"); return
     
-    # STEP 1: Check uploaded CSV and update it
     print("\n📂 Checking uploaded CSV...")
-    csv_exists = os.path.exists(CSV_5MIN_LIVE)
-    last_bar = get_csv_last_date(CSV_5MIN_LIVE)
-    
-    if csv_exists:
-        print(f"   ✅ CSV found: {last_bar}")
-        added = update_csv_from_api(CSV_5MIN_LIVE)
-        if added > 0:
-            print(f"   📡 Updated CSV with +{added} new bars")
-    else:
-        print(f"   ⚠️ No CSV found — fetching fresh data...")
+    added = update_csv_from_api(CSV_5MIN_LIVE)
     
-    # STEP 2: Load data from CSV
     data = {}
     df5 = load_csv_data(CSV_5MIN_LIVE)
     if df5 is not None and len(df5) > 0:
@@ -555,8 +559,7 @@ def run_bot():
     signal_count = 0
     
     active_tfs = len([tf for tf in ['5min','30min','1h','2h','4h'] if tf in data and len(data[tf])>=201])
-    print(f"\n🔍 Active: {active_tfs}/5 TFs\n")
-    send_telegram(f"🤖 <b>SMC Bot v9.9 LIVE!</b>\n📊 {len(models)} models | {active_tfs}/5 TFs\n📂 CSV: {len(data.get('5min',[])):,} bars\n🔍 Monitoring XAU/USD")
+    send_telegram(f"🤖 <b>SMC Bot v9.9 LIVE!</b>\n📊 {len(models)} models | {active_tfs}/5 TFs\n📂 {len(data.get('5min',[])):,} bars\n🔍 Monitoring XAU/USD")
     
     while True:
         try:
@@ -569,7 +572,6 @@ def run_bot():
             
             if seconds_since >= fetch_interval and candle_just_closed(5 if active else 15):
                 if can_call_api():
-                    print(f"⏰ {now.strftime('%H:%M')} — fetching...")
                     nd = fetch_5min(3)
                     if nd is not None and len(nd) > 0:
                         if '5min' in data:
@@ -606,9 +608,10 @@ def run_bot():
             
             time.sleep(10)
         except Exception as e:
-            print(f"⚠️ Error: {e}"); time.sleep(60)
+            print(f"⚠️ Error: {e}"); traceback.print_exc(); time.sleep(60)
 
 threading.Thread(target=keep_alive, daemon=True).start()
+print("💓 Keep-alive active")
 threading.Thread(target=run_bot, daemon=True).start()
 
 if __name__ == '__main__':