Riy777 commited on
Commit
e18c5ae
Β·
verified Β·
1 Parent(s): 998468f

Update backtest_engine.py

Browse files
Files changed (1) hide show
  1. backtest_engine.py +73 -172
backtest_engine.py CHANGED
@@ -1,5 +1,5 @@
1
  # ============================================================
2
- # πŸ§ͺ backtest_engine.py (V128.0 - GEM-Architect: Armored Edition)
3
  # ============================================================
4
 
5
  import asyncio
@@ -34,15 +34,17 @@ logging.getLogger('ml_engine').setLevel(logging.WARNING)
34
  CACHE_DIR = "backtest_real_scores"
35
 
36
  # ============================================================
37
- # πŸ›‘οΈ GLOBAL SANITIZATION & HELPERS
38
  # ============================================================
39
 
40
  def sanitize_features(df):
41
  """
42
- Cleans DataFrame from Infinity and NaNs to prevent XGBoost crashes.
 
43
  """
44
  if df is None or df.empty: return df
45
- return df.replace([np.inf, -np.inf], np.nan).fillna(0.0)
 
46
 
47
  def _zv(x):
48
  with np.errstate(divide='ignore', invalid='ignore'):
@@ -103,16 +105,16 @@ def _transform_window_for_pattern(df_window):
103
  except: return None
104
 
105
  def calculate_sniper_features_exact(df):
106
- """Sniper Features Calculation with GUARANTEED columns."""
107
  df = df.copy()
108
 
109
- # 1. Base Returns (Explicitly created)
110
  df['return_1m'] = df['close'].pct_change(1).fillna(0)
111
- df['return_3m'] = df['close'].pct_change(3).fillna(0) # Required!
112
  df['return_5m'] = df['close'].pct_change(5).fillna(0)
113
  df['return_15m'] = df['close'].pct_change(15).fillna(0)
114
 
115
- # 2. Liquidity Proxies
116
  df['ret'] = df['close'].pct_change().fillna(0)
117
  df['dollar_vol'] = df['close'] * df['volume']
118
  df['amihud'] = (df['ret'].abs() / df['dollar_vol'].replace(0, np.nan)).fillna(0)
@@ -138,7 +140,7 @@ def calculate_sniper_features_exact(df):
138
  _z_score_rolling(-df['roll_spread']) + _z_score_rolling(-df['rv_gk'].abs()) +
139
  _z_score_rolling(-df['vwap_dev'].abs()) + _z_score_rolling(df['ofi']))
140
 
141
- # 3. Standard Indicators
142
  df['rsi_14'] = ta.rsi(df['close'], 14).fillna(50)
143
  df['atr'] = ta.atr(df['high'], df['low'], df['close'], 100).fillna(0)
144
  df['vol_zscore_50'] = _z_score_rolling(df['volume'], 50)
@@ -157,7 +159,9 @@ def calculate_sniper_features_exact(df):
157
  return sanitize_features(df)
158
 
159
  def calculate_titan_features_real(df):
 
160
  df = df.copy()
 
161
  df['RSI'] = ta.rsi(df['close'], 14)
162
  macd = ta.macd(df['close'])
163
  if macd is not None:
@@ -189,7 +193,7 @@ def calculate_titan_features_real(df):
189
  return sanitize_features(df)
190
 
191
  # ============================================================
192
- # πŸ”§ NEW: LEGACY GUARD VECTORIZED FEATURES (REAL & SAFE)
193
  # ============================================================
194
  def calculate_legacy_v2_vectorized(df_1m, df_5m, df_15m):
195
  try:
@@ -198,11 +202,9 @@ def calculate_legacy_v2_vectorized(df_1m, df_5m, df_15m):
198
  res = pd.DataFrame(index=df.index)
199
  res[f'log_ret_{suffix}'] = np.log(c / c.shift(1).replace(0, np.nan)).fillna(0)
200
  res[f'rsi_{suffix}'] = (ta.rsi(c, 14) / 100.0).fillna(0.5)
201
-
202
  roll_max = h.rolling(50).max(); roll_min = l.rolling(50).min()
203
  diff = (roll_max - roll_min).replace(0, 1e-9)
204
  res[f'fib_pos_{suffix}'] = ((c - roll_min) / diff).fillna(0.5)
205
-
206
  if suffix == '1m':
207
  res[f'volatility_{suffix}'] = (ta.atr(h, l, c, 14) / c.replace(0, np.nan)).fillna(0)
208
  else:
@@ -225,8 +227,7 @@ def calculate_legacy_v2_vectorized(df_1m, df_5m, df_15m):
225
 
226
  parts = [f1[FEATS_1M], f5[FEATS_5M], f15[FEATS_15M]]
227
  lags = [1, 2, 3, 5, 10, 20]
228
- for lag in lags:
229
- parts.append(f1[FEATS_1M].shift(lag).fillna(0))
230
 
231
  X_df = pd.concat(parts, axis=1)
232
  return sanitize_features(X_df).values
@@ -235,129 +236,84 @@ def calculate_legacy_v2_vectorized(df_1m, df_5m, df_15m):
235
  return None
236
 
237
  def calculate_legacy_v3_vectorized(df_1m, df_5m, df_15m):
238
- """
239
- Legacy V3 EXACT & SAFE Feature Calculation.
240
- Prevents 'NoneType' errors by initializing everything to 0.0.
241
- """
242
  try:
243
  def calc_v3_base(df, prefix=""):
244
  d = df.copy()
245
- c = d['close']; h = d['high']; l = d['low']; v = d['volume']
246
-
247
- # 1. Initialize ALL potential columns with 0.0 (Safety Net)
248
  targets = ['rsi', 'rsi_slope', 'macd_h', 'macd_h_slope', 'adx', 'dmp', 'dmn',
249
  'trend_net_force', 'ema_20', 'ema_50', 'ema_200', 'dist_ema20',
250
  'dist_ema50', 'dist_ema200', 'slope_ema50', 'atr', 'atr_rel',
251
  'obv', 'obv_slope', 'cmf', 'log_ret', 'mc_skew', 'mc_kurt',
252
  'mc_prob_gain', 'mc_var_95', 'mc_shock']
253
-
254
  for t in targets: d[t] = 0.0
255
 
256
- # 2. Computations (Overwrite if successful)
257
  try:
258
  d['log_ret'] = np.log(c / c.shift(1).replace(0, np.nan)).fillna(0)
259
-
260
- rsi = ta.rsi(c, 14)
261
- if rsi is not None:
262
- d['rsi'] = rsi.fillna(50)
263
- d['rsi_slope'] = (d['rsi'] - d['rsi'].shift(3).fillna(50)) / 3
264
-
265
  macd = ta.macd(c)
266
  if macd is not None:
267
  d['macd_h'] = macd.iloc[:, 1].fillna(0)
268
  d['macd_h_slope'] = (d['macd_h'] - d['macd_h'].shift(3).fillna(0)) / 3
269
-
270
  adx = ta.adx(h, l, c, 14)
271
  if adx is not None:
272
- d['adx'] = adx.iloc[:, 0].fillna(0)
273
- d['dmp'] = adx.iloc[:, 1].fillna(0)
274
- d['dmn'] = adx.iloc[:, 2].fillna(0)
275
  d['trend_net_force'] = (d['dmp'] - d['dmn']) * (d['adx'] / 100.0)
276
-
277
- d['ema_20'] = ta.ema(c, 20).fillna(c)
278
- d['ema_50'] = ta.ema(c, 50).fillna(c)
279
- d['ema_200'] = ta.ema(c, 200).fillna(c)
280
-
281
  d['dist_ema20'] = (c - d['ema_20']) / d['ema_20'].replace(0, np.nan)
282
  d['dist_ema50'] = (c - d['ema_50']) / d['ema_50'].replace(0, np.nan)
283
  d['dist_ema200'] = (c - d['ema_200']) / d['ema_200'].replace(0, np.nan)
284
-
285
- shift_ema50 = d['ema_50'].shift(5).replace(0, np.nan)
286
- d['slope_ema50'] = (d['ema_50'] - d['ema_50'].shift(5).fillna(0)) / shift_ema50
287
-
288
- d['atr'] = ta.atr(h, l, c, 14).fillna(0)
289
- d['atr_rel'] = d['atr'] / c.replace(0, np.nan)
290
-
291
- obv = ta.obv(c, v)
292
- if obv is not None:
293
- d['obv'] = obv.fillna(0)
294
- d['obv_slope'] = d['obv'] - d['obv'].shift(5).fillna(0)
295
-
296
  d['cmf'] = ta.cmf(h, l, c, v, 20).fillna(0)
297
-
298
- # Monte Carlo Proxies
299
- window = 30
300
- roll = d['log_ret'].rolling(window)
301
- d['mc_skew'] = roll.skew().fillna(0)
302
- d['mc_kurt'] = roll.kurt().fillna(0)
303
- d['mc_prob_gain'] = (d['log_ret'] > 0).rolling(window).mean().fillna(0.5)
304
  d['mc_var_95'] = roll.quantile(0.05).fillna(-0.02)
305
  d['mc_shock'] = ((d['log_ret'] - roll.mean()) / (roll.std().replace(0, np.nan))).fillna(0)
306
-
307
- except Exception:
308
- # If any specific calculation fails, we stick to 0.0 default
309
- pass
310
 
311
  if prefix:
312
  d.columns = [f"{col}_{prefix}" if col not in ['timestamp'] else col for col in d.columns]
313
  return sanitize_features(d)
314
  return sanitize_features(d)
315
 
316
- df1 = calc_v3_base(df_1m)
317
- df5 = calc_v3_base(df_5m, "5m").reindex(df_1m.index, method='ffill')
318
  df15 = calc_v3_base(df_15m, "15m").reindex(df_1m.index, method='ffill')
319
 
320
  final_df = pd.DataFrame(index=df_1m.index)
321
-
322
- # Lags
323
  for i, col_name in enumerate(["6", "7", "8", "9", "10", "11"], 1):
324
  final_df[col_name] = df1['log_ret'].shift(i)
325
 
326
- cols_1m = [
327
- 'rsi', 'rsi_slope', 'macd_h', 'macd_h_slope', 'adx', 'dmp', 'dmn', 'trend_net_force',
328
- 'ema_20', 'ema_50', 'ema_200', 'dist_ema20', 'dist_ema50', 'dist_ema200', 'slope_ema50',
329
- 'atr', 'atr_rel', 'obv', 'obv_slope', 'cmf', 'log_ret',
330
- 'mc_skew', 'mc_kurt', 'mc_prob_gain', 'mc_var_95', 'mc_shock'
331
- ]
332
  for c in cols_1m: final_df[c] = df1[c]
333
 
334
- cols_5m_map = {'rsi_5m': 'rsi_5m', 'rsi_slope_5m': 'rsi_slope_5m', 'macd_h_5m': 'macd_h_5m',
335
- 'mc_prob_gain_5m': 'mc_prob_gain_5m', 'mc_shock_5m': 'mc_shock_5m'}
336
- for k, v in cols_5m_map.items(): final_df[k] = df5[v]
337
-
338
- cols_15m_map = {'rsi_15m': 'rsi_15m', 'macd_h_15m': 'macd_h_15m',
339
- 'trend_net_force_15m': 'trend_net_force_15m',
340
- 'mc_prob_gain_15m': 'mc_prob_gain_15m', 'dist_ema200_15m': 'dist_ema200_15m'}
341
- for k, v in cols_15m_map.items(): final_df[k] = df15[v]
342
-
343
- expected_order = [
344
- "6", "7", "8", "9", "10", "11",
345
- "rsi", "rsi_slope", "macd_h", "macd_h_slope", "adx", "dmp", "dmn", "trend_net_force",
346
- "ema_20", "ema_50", "ema_200", "dist_ema20", "dist_ema50", "dist_ema200", "slope_ema50",
347
- "atr", "atr_rel", "obv", "obv_slope", "cmf", "log_ret",
348
- "mc_skew", "mc_kurt", "mc_prob_gain", "mc_var_95", "mc_shock",
349
- "rsi_5m", "rsi_slope_5m", "macd_h_5m", "mc_prob_gain_5m", "mc_shock_5m",
350
- "rsi_15m", "macd_h_15m", "trend_net_force_15m", "mc_prob_gain_15m", "dist_ema200_15m"
351
- ]
352
-
353
- return sanitize_features(final_df.reindex(columns=expected_order, fill_value=0.0))
354
 
355
- except Exception as e:
356
- print(f"Legacy V3 Calc Error: {e}")
357
- return None
 
 
 
 
 
 
 
 
 
358
 
359
  # ============================================================
360
- # πŸ§ͺ THE BACKTESTER CLASS
361
  # ============================================================
362
  class HeavyDutyBacktester:
363
  def __init__(self, data_manager, processor):
@@ -369,43 +325,36 @@ class HeavyDutyBacktester:
369
  self.MAX_SLOTS = 4
370
  self.TARGET_COINS = ['SOL/USDT', 'XRP/USDT', 'DOGE/USDT']
371
  self.force_start_date = None; self.force_end_date = None
372
-
373
  if os.path.exists(CACHE_DIR):
374
  for f in glob.glob(os.path.join(CACHE_DIR, "*")): os.remove(f)
375
  else: os.makedirs(CACHE_DIR)
376
-
377
- print(f"πŸ§ͺ [Backtest V128.0] Armored Edition. Checking Engines...")
378
  self._check_engines()
379
 
380
  def _check_engines(self):
381
  status = []
382
- if self.proc.titan and self.proc.titan.model: status.append("Titan(Real)")
383
- else: print("❌ Titan Model MISSING")
384
- if self.proc.pattern_engine and self.proc.pattern_engine.models: status.append("Patterns(Real)")
385
- else: print("⚠️ Pattern Models MISSING")
386
- if self.proc.oracle: status.append("Oracle(Real)")
387
- if self.proc.sniper: status.append("Sniper(Real)")
388
- if self.proc.guardian_hydra: status.append("Hydra(Real)")
389
- if self.proc.guardian_legacy: status.append("Legacy(Real)")
390
  print(f" βœ… Engines Ready: {', '.join(status)}")
391
 
392
  def set_date_range(self, start_str, end_str):
393
  self.force_start_date = start_str; self.force_end_date = end_str
394
 
395
  def _smart_predict(self, model, X):
396
- """Forces predict_proba and handles shape mismatch"""
397
  try:
398
  if hasattr(model, "predict_proba"):
399
  raw = model.predict_proba(X)
400
- if raw.ndim == 2:
401
- return raw[:, -1]
402
  return raw
403
  return model.predict(X)
404
- except:
 
405
  return np.zeros(len(X) if hasattr(X, '__len__') else 0)
406
 
407
  def _extract_probs(self, raw_preds):
408
- """Standardizes output to 1D probability array"""
409
  if isinstance(raw_preds, list): raw_preds = np.array(raw_preds)
410
  if raw_preds.ndim == 1: return raw_preds
411
  elif raw_preds.ndim == 2:
@@ -418,18 +367,15 @@ class HeavyDutyBacktester:
418
  limit = 1000; duration = limit * 60 * 1000
419
  tasks = []; curr = start_ms
420
  while curr < end_ms: tasks.append(curr); curr += duration
421
-
422
  all_c = []; sem = asyncio.Semaphore(20)
423
  async def _fetch(ts):
424
  async with sem:
425
  try: return await self.dm.exchange.fetch_ohlcv(sym, '1m', since=ts, limit=limit)
426
  except: await asyncio.sleep(0.5); return []
427
-
428
  chunk = 50
429
  for i in range(0, len(tasks), chunk):
430
  res = await asyncio.gather(*[_fetch(t) for t in tasks[i:i+chunk]])
431
  for r in res: all_c.extend(r)
432
-
433
  seen = set(); unique = []
434
  for c in all_c:
435
  if c[0] not in seen and c[0] >= start_ms and c[0] <= end_ms:
@@ -448,7 +394,6 @@ class HeavyDutyBacktester:
448
  print(f" βš™οΈ [CPU] Analyzing {sym} (ALL REAL MODELS)...", flush=True)
449
  t0 = time.time()
450
 
451
- # 1. Base Data
452
  df_1m = pd.DataFrame(candles, columns=['timestamp', 'open', 'high', 'low', 'close', 'volume'])
453
  for c in ['open', 'high', 'low', 'close', 'volume']: df_1m[c] = df_1m[c].astype(float)
454
  df_1m['datetime'] = pd.to_datetime(df_1m['timestamp'], unit='ms')
@@ -458,7 +403,7 @@ class HeavyDutyBacktester:
458
  df_5m = df_1m.resample('5T').agg({'open':'first', 'high':'max', 'low':'min', 'close':'last', 'volume':'sum'}).dropna()
459
  df_15m = df_1m.resample('15T').agg({'open':'first', 'high':'max', 'low':'min', 'close':'last', 'volume':'sum'}).dropna()
460
 
461
- # 1️⃣ SNIPER & L1 SCORING
462
  df_sniper = calculate_sniper_features_exact(df_1m)
463
  df_sniper['rel_vol'] = df_sniper['volume'] / (df_sniper['volume'].rolling(50).mean() + 1e-9)
464
  df_sniper['l1_score'] = (df_sniper['rel_vol'] * 10) + ((df_sniper['atr']/df_sniper['close']) * 1000)
@@ -469,7 +414,7 @@ class HeavyDutyBacktester:
469
 
470
  print(f" 🎯 Candidates: {len(df_candidates)}. Running Deep Inference...", flush=True)
471
 
472
- # 2️⃣ PATTERNS
473
  res_patterns = np.full(len(df_candidates), 0.5)
474
  pattern_models = getattr(self.proc.pattern_engine, 'models', {})
475
  if pattern_models and '15m' in pattern_models:
@@ -477,24 +422,21 @@ class HeavyDutyBacktester:
477
  df_15m_res = df_1m.resample('15T').agg({'open':'first', 'high':'max', 'low':'min', 'close':'last', 'volume':'sum'}).dropna()
478
  pat_scores_15m = np.full(len(df_15m_res), 0.5)
479
  pat_inputs = []; valid_15m_idxs = []
480
-
481
  for i in range(200, len(df_15m_res)):
482
  window = df_15m_res.iloc[i-200:i]
483
  vec = _transform_window_for_pattern(window)
484
  if vec is not None:
485
  pat_inputs.append(vec); valid_15m_idxs.append(i)
486
-
487
  if pat_inputs:
488
  X_pat = np.array(pat_inputs)
489
  pat_preds = self._smart_predict(pattern_models['15m'], xgb.DMatrix(X_pat))
490
  pat_scores_15m[valid_15m_idxs] = pat_preds
491
-
492
  ts_15m = df_15m_res.index.astype(np.int64) // 10**6
493
  map_idxs = np.searchsorted(ts_15m, df_candidates['timestamp'].values) - 1
494
  res_patterns = pat_scores_15m[np.clip(map_idxs, 0, len(pat_scores_15m)-1)]
495
  except Exception as e: print(f"Patterns Error: {e}")
496
 
497
- # 3️⃣ TITAN
498
  res_titan = np.full(len(df_candidates), 0.5)
499
  if self.proc.titan and self.proc.titan.model:
500
  try:
@@ -502,13 +444,11 @@ class HeavyDutyBacktester:
502
  ts_5m = df_5m.index.astype(np.int64) // 10**6
503
  map_idxs = np.clip(np.searchsorted(ts_5m, df_candidates['timestamp'].values) - 1, 0, len(df_5m_feat)-1)
504
  feats = self.proc.titan.feature_names
505
-
506
  X_titan_df = sanitize_features(df_5m_feat.iloc[map_idxs].reindex(columns=feats, fill_value=0))
507
- X_titan = X_titan_df.values
508
- res_titan = self.proc.titan.model.predict(xgb.DMatrix(X_titan, feature_names=feats))
509
  except Exception as e: print(f"Titan Error: {e}")
510
 
511
- # 4️⃣ SNIPER
512
  res_sniper = np.full(len(df_candidates), 0.5)
513
  sniper_models = getattr(self.proc.sniper, 'models', [])
514
  if sniper_models:
@@ -519,7 +459,7 @@ class HeavyDutyBacktester:
519
  res_sniper = np.mean(preds, axis=0)
520
  except Exception as e: print(f"Sniper Error: {e}")
521
 
522
- # 5️⃣ ORACLE
523
  res_oracle = np.full(len(df_candidates), 0.5)
524
  oracle_model = getattr(self.proc.oracle, 'model_direction', None)
525
  if oracle_model:
@@ -529,11 +469,10 @@ class HeavyDutyBacktester:
529
  if 'sim_titan_score' in X_orc_df: X_orc_df['sim_titan_score'] = res_titan
530
  if 'sim_pattern_score' in X_orc_df: X_orc_df['sim_pattern_score'] = res_patterns
531
  if 'sim_mc_score' in X_orc_df: X_orc_df['sim_mc_score'] = 0.5
532
-
533
  res_oracle = self._extract_probs(self._smart_predict(oracle_model, X_orc_df.values))
534
  except Exception as e: print(f"Oracle Error: {e}")
535
 
536
- # 6️⃣ HYDRA
537
  res_hydra_risk = np.zeros(len(df_candidates))
538
  hydra_models = getattr(self.proc.guardian_hydra, 'models', {})
539
  if hydra_models and 'crash' in hydra_models:
@@ -543,71 +482,57 @@ class HeavyDutyBacktester:
543
  (df_sniper['close']-df_sniper['close'].rolling(20).mean())/df_sniper['close'],
544
  df_sniper['rel_vol'], df_sniper['atr'], df_sniper['close']
545
  ]).astype(np.float32)
546
-
547
  global_hydra_feats = np.nan_to_num(global_hydra_feats, nan=0.0, posinf=0.0, neginf=0.0)
548
-
549
  window_view = sliding_window_view(global_hydra_feats, 240, axis=0).transpose(0, 2, 1)
550
  c_idxs = np.searchsorted(df_sniper.index, df_candidates.index)
551
  valid_s = c_idxs + 1
552
  valid_mask_h = valid_s < (len(global_hydra_feats) - 240)
553
  final_s = valid_s[valid_mask_h]; res_idxs = np.where(valid_mask_h)[0]
554
-
555
  for i in range(0, len(final_s), 5000):
556
  b_idxs = final_s[i:i+5000]; r_idxs = res_idxs[i:i+5000]
557
  static = window_view[b_idxs]
558
-
559
  B = len(b_idxs)
560
  entry = df_sniper['close'].values[b_idxs-1].reshape(B, 1)
561
  s_c = static[:, 6, :]; s_atr = static[:, 5, :]
562
-
563
  dist = np.maximum(1.5*s_atr, entry*0.015) + 1e-9
564
  pnl = (s_c - entry)/dist
565
  max_pnl = (np.maximum.accumulate(s_c, axis=1) - entry)/dist
566
  atr_p = s_atr/(s_c+1e-9)
567
-
568
  zeros = np.zeros((B, 240)); ones = np.ones((B, 240)); t = np.tile(np.arange(1, 241), (B, 1))
569
  X = np.stack([
570
  static[:,0], static[:,1], static[:,2], static[:,3], static[:,4],
571
  zeros, atr_p, pnl, max_pnl, zeros, zeros, t, zeros, ones*0.6, ones*0.7, ones*3
572
  ], axis=2).reshape(-1, 16)
573
-
574
  X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
575
  preds = hydra_models['crash'].predict_proba(X)[:, 1].reshape(B, 240)
576
  res_hydra_risk[r_idxs] = np.max(preds, axis=1)
577
  except: pass
578
 
579
- # 7️⃣ REAL LEGACY GUARD
580
  res_legacy_v2 = np.zeros(len(df_candidates))
581
  res_legacy_v3 = np.zeros(len(df_candidates))
582
-
583
  if self.proc.guardian_legacy:
584
  try:
585
  X_v2_full = calculate_legacy_v2_vectorized(df_1m, df_5m, df_15m)
586
  v3_df_full = calculate_legacy_v3_vectorized(df_1m, df_5m, df_15m)
587
-
588
  all_indices = np.arange(len(df_1m))
589
  cand_indices = all_indices[valid_mask]
590
  max_len = len(df_1m)
591
  cand_indices = cand_indices[cand_indices < max_len]
592
-
593
  if len(cand_indices) > 0:
594
  if self.proc.guardian_legacy.model_v2 and X_v2_full is not None:
595
  subset_v2 = X_v2_full[cand_indices]
596
  preds_v2 = self.proc.guardian_legacy.model_v2.predict(xgb.DMatrix(subset_v2))
597
  if len(preds_v2.shape) > 1: res_legacy_v2[:len(cand_indices)] = preds_v2[:, 2]
598
  else: res_legacy_v2[:len(cand_indices)] = preds_v2
599
-
600
  if self.proc.guardian_legacy.model_v3 and v3_df_full is not None:
601
  subset_v3_df = v3_df_full.iloc[cand_indices]
602
  preds_v3 = self.proc.guardian_legacy.model_v3.predict(xgb.DMatrix(subset_v3_df))
603
  res_legacy_v3[:len(cand_indices)] = preds_v3
 
604
 
605
- except Exception as e:
606
- print(f"❌ Legacy Guard Error: {e}")
607
-
608
- # 8️⃣ FINAL ASSEMBLY
609
  print(f" πŸ“Š [Stats] Titan:{res_titan.mean():.2f} | Patterns:{res_patterns.mean():.2f} | Sniper:{res_sniper.mean():.2f} | Oracle:{res_oracle.mean():.2f}")
610
-
611
  ai_df = pd.DataFrame({
612
  'timestamp': df_candidates['timestamp'],
613
  'symbol': sym,
@@ -620,7 +545,6 @@ class HeavyDutyBacktester:
620
  'risk_legacy_v2': res_legacy_v2,
621
  'risk_legacy_v3': res_legacy_v3
622
  })
623
-
624
  dt = time.time() - t0
625
  if not ai_df.empty:
626
  ai_df.to_pickle(scores_file)
@@ -647,7 +571,6 @@ class HeavyDutyBacktester:
647
  if not data: return []
648
  df = pd.concat(data).sort_values('timestamp')
649
 
650
- # Arrays
651
  ts = df['timestamp'].values; close = df['close'].values.astype(float)
652
  sym = df['symbol'].values; sym_map = {s:i for i,s in enumerate(np.unique(sym))}
653
  sym_id = np.array([sym_map[s] for s in sym])
@@ -655,9 +578,7 @@ class HeavyDutyBacktester:
655
  oracle = df['oracle_conf'].values; sniper = df['sniper_score'].values
656
  hydra = df['risk_hydra_crash'].values; titan = df['real_titan'].values
657
  l1 = df['l1_score'].values
658
-
659
- legacy_v2 = df['risk_legacy_v2'].values
660
- legacy_v3 = df['risk_legacy_v3'].values
661
 
662
  N = len(ts)
663
  print(f" πŸš€ [System] Testing {len(combinations_batch)} configs on {N} candles...", flush=True)
@@ -666,38 +587,30 @@ class HeavyDutyBacktester:
666
  for cfg in combinations_batch:
667
  pos = {}; log = []
668
  bal = initial_capital; alloc = 0.0
669
-
670
  mask = (l1 >= cfg['l1_thresh']) & (oracle >= cfg['oracle_thresh']) & (sniper >= cfg['sniper_thresh'])
671
 
672
  for i in range(N):
673
  s = sym_id[i]; p = close[i]
674
-
675
- # Exit
676
  if s in pos:
677
  entry = pos[s][0]; h_r = pos[s][1]; titan_entry = pos[s][3]
678
-
679
  crash_hydra = (h_r > cfg['hydra_thresh'])
680
  panic_legacy = (legacy_v2[i] > cfg['legacy_thresh']) or (legacy_v3[i] > cfg['legacy_thresh'])
681
-
682
  pnl = (p - entry)/entry
683
 
684
  if crash_hydra or panic_legacy or pnl > 0.04 or pnl < -0.02:
685
  realized = pnl - fees_pct*2
686
  bal += pos[s][2] * (1 + realized)
687
  alloc -= pos[s][2]
688
-
689
  is_consensus = (titan_entry > 0.55)
690
  log.append({'pnl': realized, 'consensus': is_consensus})
691
  del pos[s]
692
 
693
- # Entry
694
  if len(pos) < max_slots and mask[i]:
695
  if s not in pos and bal >= 5.0:
696
  size = min(10.0, bal * 0.98)
697
  pos[s] = (p, hydra[i], size, titan[i])
698
  bal -= size; alloc += size
699
 
700
- # Metrics
701
  final_bal = bal + alloc
702
  profit = final_bal - initial_capital
703
  tot = len(log)
@@ -721,25 +634,19 @@ class HeavyDutyBacktester:
721
 
722
  async def run_optimization(self, target_regime="RANGE"):
723
  await self.generate_truth_data()
724
-
725
- # Grid
726
  oracle_r = np.linspace(0.3, 0.7, 3); sniper_r = np.linspace(0.2, 0.6, 3)
727
  hydra_r = [0.8, 0.9]; l1_r = [5.0, 10.0]
728
 
729
  combos = []
730
  for o, s, h, l1 in itertools.product(oracle_r, sniper_r, hydra_r, l1_r):
731
  combos.append({
732
- 'w_titan': 0.4, 'w_struct': 0.3,
733
- 'thresh': l1, 'l1_thresh': l1,
734
  'oracle_thresh': o, 'sniper_thresh': s, 'hydra_thresh': h, 'legacy_thresh': 0.95
735
  })
736
 
737
  files = glob.glob(os.path.join(CACHE_DIR, "*.pkl"))
738
  results_list = self._worker_optimize(combos, files, self.INITIAL_CAPITAL, self.TRADING_FEES, self.MAX_SLOTS)
739
-
740
- if not results_list:
741
- print("⚠️ [Warning] No trades generated in any config.")
742
- return None, {'net_profit': 0.0, 'win_rate': 0.0}
743
 
744
  results_list.sort(key=lambda x: x['net_profit'], reverse=True)
745
  best = results_list[0]
@@ -751,7 +658,6 @@ class HeavyDutyBacktester:
751
  print("-" * 60)
752
  print(f" πŸ“Š Total Trades: {best['total_trades']}")
753
  print(f" πŸ“ˆ Win Rate: {best['win_rate']:.1f}%")
754
- print(f" πŸ“‰ Max Drawdown: {best['max_drawdown']:.1f}%")
755
  print("-" * 60)
756
  print(f" 🧠 CONSENSUS ANALYTICS:")
757
  print(f" 🀝 Model Agreement Rate: {best['consensus_agreement_rate']:.1f}%")
@@ -759,18 +665,15 @@ class HeavyDutyBacktester:
759
  print(f" πŸ’Ž High-Consensus Avg PnL: {best['high_consensus_avg_pnl']:.2f}%")
760
  print("-" * 60)
761
  print(f" βš™οΈ Oracle={best['config']['oracle_thresh']:.2f} | Sniper={best['config']['sniper_thresh']:.2f} | Hydra={best['config']['hydra_thresh']:.2f}")
762
- print(f" βš–οΈ Weights: Titan={best['config']['w_titan']:.2f} | Patterns={best['config']['w_struct']:.2f} | L1={best['config']['thresh']}")
763
  print("="*60)
764
  return best['config'], best
765
 
766
  async def run_strategic_optimization_task():
767
  print("\nπŸ§ͺ [STRATEGIC BACKTEST] Full Spectrum Mode...")
768
  r2 = R2Service(); dm = DataManager(None, None, r2); proc = MLProcessor(dm)
769
-
770
  try:
771
  await dm.initialize(); await proc.initialize()
772
  if proc.guardian_hydra: proc.guardian_hydra.set_silent_mode(True)
773
-
774
  hub = AdaptiveHub(r2); await hub.initialize()
775
  opt = HeavyDutyBacktester(dm, proc)
776
 
@@ -785,10 +688,8 @@ async def run_strategic_optimization_task():
785
  opt.set_date_range(s["start"], s["end"])
786
  best_cfg, best_stats = await opt.run_optimization(s["regime"])
787
  if best_cfg: hub.submit_challenger(s["regime"], best_cfg, best_stats)
788
-
789
  await hub._save_state_to_r2()
790
  print("βœ… [System] DNA Updated.")
791
-
792
  finally:
793
  print("πŸ”Œ [System] Closing connections...")
794
  await dm.close()
 
1
  # ============================================================
2
+ # πŸ§ͺ backtest_engine.py (V129.0 - GEM-Architect: Sniper Debug Mode)
3
  # ============================================================
4
 
5
  import asyncio
 
34
  CACHE_DIR = "backtest_real_scores"
35
 
36
  # ============================================================
37
+ # πŸ›‘οΈ GLOBAL SANITIZATION
38
  # ============================================================
39
 
40
  def sanitize_features(df):
41
  """
42
+ Cleans DataFrame from Infinity and NaNs.
43
+ Forces float32 to be friendly with ML models.
44
  """
45
  if df is None or df.empty: return df
46
+ # Replace Infinity with NaN, then fill NaN with 0.0, then convert to float32
47
+ return df.replace([np.inf, -np.inf], np.nan).fillna(0.0).astype(np.float32)
48
 
49
  def _zv(x):
50
  with np.errstate(divide='ignore', invalid='ignore'):
 
105
  except: return None
106
 
107
  def calculate_sniper_features_exact(df):
108
+ """Sniper Features Calculation - EXACT MATCH with Processor."""
109
  df = df.copy()
110
 
111
+ # Base Returns
112
  df['return_1m'] = df['close'].pct_change(1).fillna(0)
113
+ df['return_3m'] = df['close'].pct_change(3).fillna(0)
114
  df['return_5m'] = df['close'].pct_change(5).fillna(0)
115
  df['return_15m'] = df['close'].pct_change(15).fillna(0)
116
 
117
+ # Liquidity Proxies
118
  df['ret'] = df['close'].pct_change().fillna(0)
119
  df['dollar_vol'] = df['close'] * df['volume']
120
  df['amihud'] = (df['ret'].abs() / df['dollar_vol'].replace(0, np.nan)).fillna(0)
 
140
  _z_score_rolling(-df['roll_spread']) + _z_score_rolling(-df['rv_gk'].abs()) +
141
  _z_score_rolling(-df['vwap_dev'].abs()) + _z_score_rolling(df['ofi']))
142
 
143
+ # Standard Indicators
144
  df['rsi_14'] = ta.rsi(df['close'], 14).fillna(50)
145
  df['atr'] = ta.atr(df['high'], df['low'], df['close'], 100).fillna(0)
146
  df['vol_zscore_50'] = _z_score_rolling(df['volume'], 50)
 
159
  return sanitize_features(df)
160
 
161
  def calculate_titan_features_real(df):
162
+ """Titan features with strict Infinity handling."""
163
  df = df.copy()
164
+
165
  df['RSI'] = ta.rsi(df['close'], 14)
166
  macd = ta.macd(df['close'])
167
  if macd is not None:
 
193
  return sanitize_features(df)
194
 
195
  # ============================================================
196
+ # πŸ”§ LEGACY GUARD (V2 & V3)
197
  # ============================================================
198
  def calculate_legacy_v2_vectorized(df_1m, df_5m, df_15m):
199
  try:
 
202
  res = pd.DataFrame(index=df.index)
203
  res[f'log_ret_{suffix}'] = np.log(c / c.shift(1).replace(0, np.nan)).fillna(0)
204
  res[f'rsi_{suffix}'] = (ta.rsi(c, 14) / 100.0).fillna(0.5)
 
205
  roll_max = h.rolling(50).max(); roll_min = l.rolling(50).min()
206
  diff = (roll_max - roll_min).replace(0, 1e-9)
207
  res[f'fib_pos_{suffix}'] = ((c - roll_min) / diff).fillna(0.5)
 
208
  if suffix == '1m':
209
  res[f'volatility_{suffix}'] = (ta.atr(h, l, c, 14) / c.replace(0, np.nan)).fillna(0)
210
  else:
 
227
 
228
  parts = [f1[FEATS_1M], f5[FEATS_5M], f15[FEATS_15M]]
229
  lags = [1, 2, 3, 5, 10, 20]
230
+ for lag in lags: parts.append(f1[FEATS_1M].shift(lag).fillna(0))
 
231
 
232
  X_df = pd.concat(parts, axis=1)
233
  return sanitize_features(X_df).values
 
236
  return None
237
 
238
  def calculate_legacy_v3_vectorized(df_1m, df_5m, df_15m):
239
+ """Legacy V3 Safe Calc."""
 
 
 
240
  try:
241
  def calc_v3_base(df, prefix=""):
242
  d = df.copy()
243
+ # Initialize All
 
 
244
  targets = ['rsi', 'rsi_slope', 'macd_h', 'macd_h_slope', 'adx', 'dmp', 'dmn',
245
  'trend_net_force', 'ema_20', 'ema_50', 'ema_200', 'dist_ema20',
246
  'dist_ema50', 'dist_ema200', 'slope_ema50', 'atr', 'atr_rel',
247
  'obv', 'obv_slope', 'cmf', 'log_ret', 'mc_skew', 'mc_kurt',
248
  'mc_prob_gain', 'mc_var_95', 'mc_shock']
 
249
  for t in targets: d[t] = 0.0
250
 
251
+ c = d['close']; h = d['high']; l = d['low']; v = d['volume']
252
  try:
253
  d['log_ret'] = np.log(c / c.shift(1).replace(0, np.nan)).fillna(0)
254
+ d['rsi'] = ta.rsi(c, 14).fillna(50)
255
+ d['rsi_slope'] = (d['rsi'] - d['rsi'].shift(3).fillna(50)) / 3
 
 
 
 
256
  macd = ta.macd(c)
257
  if macd is not None:
258
  d['macd_h'] = macd.iloc[:, 1].fillna(0)
259
  d['macd_h_slope'] = (d['macd_h'] - d['macd_h'].shift(3).fillna(0)) / 3
 
260
  adx = ta.adx(h, l, c, 14)
261
  if adx is not None:
262
+ d['adx'] = adx.iloc[:, 0].fillna(0); d['dmp'] = adx.iloc[:, 1].fillna(0); d['dmn'] = adx.iloc[:, 2].fillna(0)
 
 
263
  d['trend_net_force'] = (d['dmp'] - d['dmn']) * (d['adx'] / 100.0)
264
+ d['ema_20'] = ta.ema(c, 20).fillna(c); d['ema_50'] = ta.ema(c, 50).fillna(c); d['ema_200'] = ta.ema(c, 200).fillna(c)
 
 
 
 
265
  d['dist_ema20'] = (c - d['ema_20']) / d['ema_20'].replace(0, np.nan)
266
  d['dist_ema50'] = (c - d['ema_50']) / d['ema_50'].replace(0, np.nan)
267
  d['dist_ema200'] = (c - d['ema_200']) / d['ema_200'].replace(0, np.nan)
268
+ d['slope_ema50'] = (d['ema_50'] - d['ema_50'].shift(5).fillna(0)) / d['ema_50'].shift(5).replace(0, np.nan)
269
+ d['atr'] = ta.atr(h, l, c, 14).fillna(0); d['atr_rel'] = d['atr'] / c.replace(0, np.nan)
270
+ d['obv'] = ta.obv(c, v).fillna(0); d['obv_slope'] = d['obv'] - d['obv'].shift(5).fillna(0)
 
 
 
 
 
 
 
 
 
271
  d['cmf'] = ta.cmf(h, l, c, v, 20).fillna(0)
272
+
273
+ win = 30; roll = d['log_ret'].rolling(win)
274
+ d['mc_skew'] = roll.skew().fillna(0); d['mc_kurt'] = roll.kurt().fillna(0)
275
+ d['mc_prob_gain'] = (d['log_ret'] > 0).rolling(win).mean().fillna(0.5)
 
 
 
276
  d['mc_var_95'] = roll.quantile(0.05).fillna(-0.02)
277
  d['mc_shock'] = ((d['log_ret'] - roll.mean()) / (roll.std().replace(0, np.nan))).fillna(0)
278
+ except: pass
 
 
 
279
 
280
  if prefix:
281
  d.columns = [f"{col}_{prefix}" if col not in ['timestamp'] else col for col in d.columns]
282
  return sanitize_features(d)
283
  return sanitize_features(d)
284
 
285
+ df1 = calc_v3_base(df_1m); df5 = calc_v3_base(df_5m, "5m").reindex(df_1m.index, method='ffill')
 
286
  df15 = calc_v3_base(df_15m, "15m").reindex(df_1m.index, method='ffill')
287
 
288
  final_df = pd.DataFrame(index=df_1m.index)
 
 
289
  for i, col_name in enumerate(["6", "7", "8", "9", "10", "11"], 1):
290
  final_df[col_name] = df1['log_ret'].shift(i)
291
 
292
+ cols_1m = ['rsi', 'rsi_slope', 'macd_h', 'macd_h_slope', 'adx', 'dmp', 'dmn', 'trend_net_force',
293
+ 'ema_20', 'ema_50', 'ema_200', 'dist_ema20', 'dist_ema50', 'dist_ema200', 'slope_ema50',
294
+ 'atr', 'atr_rel', 'obv', 'obv_slope', 'cmf', 'log_ret', 'mc_skew', 'mc_kurt',
295
+ 'mc_prob_gain', 'mc_var_95', 'mc_shock']
 
 
296
  for c in cols_1m: final_df[c] = df1[c]
297
 
298
+ cols_5m = {'rsi_5m': 'rsi_5m', 'rsi_slope_5m': 'rsi_slope_5m', 'macd_h_5m': 'macd_h_5m',
299
+ 'mc_prob_gain_5m': 'mc_prob_gain_5m', 'mc_shock_5m': 'mc_shock_5m'}
300
+ for k, v in cols_5m.items(): final_df[k] = df5[v]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
301
 
302
+ cols_15m = {'rsi_15m': 'rsi_15m', 'macd_h_15m': 'macd_h_15m', 'trend_net_force_15m': 'trend_net_force_15m',
303
+ 'mc_prob_gain_15m': 'mc_prob_gain_15m', 'dist_ema200_15m': 'dist_ema200_15m'}
304
+ for k, v in cols_15m.items(): final_df[k] = df15[v]
305
+
306
+ expected = ["6", "7", "8", "9", "10", "11", "rsi", "rsi_slope", "macd_h", "macd_h_slope", "adx", "dmp", "dmn",
307
+ "trend_net_force", "ema_20", "ema_50", "ema_200", "dist_ema20", "dist_ema50", "dist_ema200",
308
+ "slope_ema50", "atr", "atr_rel", "obv", "obv_slope", "cmf", "log_ret", "mc_skew", "mc_kurt",
309
+ "mc_prob_gain", "mc_var_95", "mc_shock", "rsi_5m", "rsi_slope_5m", "macd_h_5m", "mc_prob_gain_5m",
310
+ "mc_shock_5m", "rsi_15m", "macd_h_15m", "trend_net_force_15m", "mc_prob_gain_15m", "dist_ema200_15m"]
311
+
312
+ return sanitize_features(final_df.reindex(columns=expected, fill_value=0.0))
313
+ except: return None
314
 
315
  # ============================================================
316
+ # πŸ§ͺ THE BACKTESTER
317
  # ============================================================
318
  class HeavyDutyBacktester:
319
  def __init__(self, data_manager, processor):
 
325
  self.MAX_SLOTS = 4
326
  self.TARGET_COINS = ['SOL/USDT', 'XRP/USDT', 'DOGE/USDT']
327
  self.force_start_date = None; self.force_end_date = None
 
328
  if os.path.exists(CACHE_DIR):
329
  for f in glob.glob(os.path.join(CACHE_DIR, "*")): os.remove(f)
330
  else: os.makedirs(CACHE_DIR)
 
 
331
  self._check_engines()
332
 
333
  def _check_engines(self):
334
  status = []
335
+ if self.proc.titan and self.proc.titan.model: status.append("Titan")
336
+ if self.proc.pattern_engine and self.proc.pattern_engine.models: status.append("Patterns")
337
+ if self.proc.oracle: status.append("Oracle")
338
+ if self.proc.sniper: status.append("Sniper")
339
+ if self.proc.guardian_hydra: status.append("Hydra")
340
+ if self.proc.guardian_legacy: status.append("Legacy")
 
 
341
  print(f" βœ… Engines Ready: {', '.join(status)}")
342
 
343
  def set_date_range(self, start_str, end_str):
344
  self.force_start_date = start_str; self.force_end_date = end_str
345
 
346
  def _smart_predict(self, model, X):
 
347
  try:
348
  if hasattr(model, "predict_proba"):
349
  raw = model.predict_proba(X)
350
+ if raw.ndim == 2: return raw[:, -1]
 
351
  return raw
352
  return model.predict(X)
353
+ except Exception as e:
354
+ # print(f"⚠️ Predict Error: {e}") # Enable if needed
355
  return np.zeros(len(X) if hasattr(X, '__len__') else 0)
356
 
357
  def _extract_probs(self, raw_preds):
 
358
  if isinstance(raw_preds, list): raw_preds = np.array(raw_preds)
359
  if raw_preds.ndim == 1: return raw_preds
360
  elif raw_preds.ndim == 2:
 
367
  limit = 1000; duration = limit * 60 * 1000
368
  tasks = []; curr = start_ms
369
  while curr < end_ms: tasks.append(curr); curr += duration
 
370
  all_c = []; sem = asyncio.Semaphore(20)
371
  async def _fetch(ts):
372
  async with sem:
373
  try: return await self.dm.exchange.fetch_ohlcv(sym, '1m', since=ts, limit=limit)
374
  except: await asyncio.sleep(0.5); return []
 
375
  chunk = 50
376
  for i in range(0, len(tasks), chunk):
377
  res = await asyncio.gather(*[_fetch(t) for t in tasks[i:i+chunk]])
378
  for r in res: all_c.extend(r)
 
379
  seen = set(); unique = []
380
  for c in all_c:
381
  if c[0] not in seen and c[0] >= start_ms and c[0] <= end_ms:
 
394
  print(f" βš™οΈ [CPU] Analyzing {sym} (ALL REAL MODELS)...", flush=True)
395
  t0 = time.time()
396
 
 
397
  df_1m = pd.DataFrame(candles, columns=['timestamp', 'open', 'high', 'low', 'close', 'volume'])
398
  for c in ['open', 'high', 'low', 'close', 'volume']: df_1m[c] = df_1m[c].astype(float)
399
  df_1m['datetime'] = pd.to_datetime(df_1m['timestamp'], unit='ms')
 
403
  df_5m = df_1m.resample('5T').agg({'open':'first', 'high':'max', 'low':'min', 'close':'last', 'volume':'sum'}).dropna()
404
  df_15m = df_1m.resample('15T').agg({'open':'first', 'high':'max', 'low':'min', 'close':'last', 'volume':'sum'}).dropna()
405
 
406
+ # 1. Sniper
407
  df_sniper = calculate_sniper_features_exact(df_1m)
408
  df_sniper['rel_vol'] = df_sniper['volume'] / (df_sniper['volume'].rolling(50).mean() + 1e-9)
409
  df_sniper['l1_score'] = (df_sniper['rel_vol'] * 10) + ((df_sniper['atr']/df_sniper['close']) * 1000)
 
414
 
415
  print(f" 🎯 Candidates: {len(df_candidates)}. Running Deep Inference...", flush=True)
416
 
417
+ # 2. Patterns
418
  res_patterns = np.full(len(df_candidates), 0.5)
419
  pattern_models = getattr(self.proc.pattern_engine, 'models', {})
420
  if pattern_models and '15m' in pattern_models:
 
422
  df_15m_res = df_1m.resample('15T').agg({'open':'first', 'high':'max', 'low':'min', 'close':'last', 'volume':'sum'}).dropna()
423
  pat_scores_15m = np.full(len(df_15m_res), 0.5)
424
  pat_inputs = []; valid_15m_idxs = []
 
425
  for i in range(200, len(df_15m_res)):
426
  window = df_15m_res.iloc[i-200:i]
427
  vec = _transform_window_for_pattern(window)
428
  if vec is not None:
429
  pat_inputs.append(vec); valid_15m_idxs.append(i)
 
430
  if pat_inputs:
431
  X_pat = np.array(pat_inputs)
432
  pat_preds = self._smart_predict(pattern_models['15m'], xgb.DMatrix(X_pat))
433
  pat_scores_15m[valid_15m_idxs] = pat_preds
 
434
  ts_15m = df_15m_res.index.astype(np.int64) // 10**6
435
  map_idxs = np.searchsorted(ts_15m, df_candidates['timestamp'].values) - 1
436
  res_patterns = pat_scores_15m[np.clip(map_idxs, 0, len(pat_scores_15m)-1)]
437
  except Exception as e: print(f"Patterns Error: {e}")
438
 
439
+ # 3. Titan
440
  res_titan = np.full(len(df_candidates), 0.5)
441
  if self.proc.titan and self.proc.titan.model:
442
  try:
 
444
  ts_5m = df_5m.index.astype(np.int64) // 10**6
445
  map_idxs = np.clip(np.searchsorted(ts_5m, df_candidates['timestamp'].values) - 1, 0, len(df_5m_feat)-1)
446
  feats = self.proc.titan.feature_names
 
447
  X_titan_df = sanitize_features(df_5m_feat.iloc[map_idxs].reindex(columns=feats, fill_value=0))
448
+ res_titan = self.proc.titan.model.predict(xgb.DMatrix(X_titan_df.values, feature_names=feats))
 
449
  except Exception as e: print(f"Titan Error: {e}")
450
 
451
+ # 4. Sniper
452
  res_sniper = np.full(len(df_candidates), 0.5)
453
  sniper_models = getattr(self.proc.sniper, 'models', [])
454
  if sniper_models:
 
459
  res_sniper = np.mean(preds, axis=0)
460
  except Exception as e: print(f"Sniper Error: {e}")
461
 
462
+ # 5. Oracle
463
  res_oracle = np.full(len(df_candidates), 0.5)
464
  oracle_model = getattr(self.proc.oracle, 'model_direction', None)
465
  if oracle_model:
 
469
  if 'sim_titan_score' in X_orc_df: X_orc_df['sim_titan_score'] = res_titan
470
  if 'sim_pattern_score' in X_orc_df: X_orc_df['sim_pattern_score'] = res_patterns
471
  if 'sim_mc_score' in X_orc_df: X_orc_df['sim_mc_score'] = 0.5
 
472
  res_oracle = self._extract_probs(self._smart_predict(oracle_model, X_orc_df.values))
473
  except Exception as e: print(f"Oracle Error: {e}")
474
 
475
+ # 6. Hydra
476
  res_hydra_risk = np.zeros(len(df_candidates))
477
  hydra_models = getattr(self.proc.guardian_hydra, 'models', {})
478
  if hydra_models and 'crash' in hydra_models:
 
482
  (df_sniper['close']-df_sniper['close'].rolling(20).mean())/df_sniper['close'],
483
  df_sniper['rel_vol'], df_sniper['atr'], df_sniper['close']
484
  ]).astype(np.float32)
 
485
  global_hydra_feats = np.nan_to_num(global_hydra_feats, nan=0.0, posinf=0.0, neginf=0.0)
 
486
  window_view = sliding_window_view(global_hydra_feats, 240, axis=0).transpose(0, 2, 1)
487
  c_idxs = np.searchsorted(df_sniper.index, df_candidates.index)
488
  valid_s = c_idxs + 1
489
  valid_mask_h = valid_s < (len(global_hydra_feats) - 240)
490
  final_s = valid_s[valid_mask_h]; res_idxs = np.where(valid_mask_h)[0]
 
491
  for i in range(0, len(final_s), 5000):
492
  b_idxs = final_s[i:i+5000]; r_idxs = res_idxs[i:i+5000]
493
  static = window_view[b_idxs]
 
494
  B = len(b_idxs)
495
  entry = df_sniper['close'].values[b_idxs-1].reshape(B, 1)
496
  s_c = static[:, 6, :]; s_atr = static[:, 5, :]
 
497
  dist = np.maximum(1.5*s_atr, entry*0.015) + 1e-9
498
  pnl = (s_c - entry)/dist
499
  max_pnl = (np.maximum.accumulate(s_c, axis=1) - entry)/dist
500
  atr_p = s_atr/(s_c+1e-9)
 
501
  zeros = np.zeros((B, 240)); ones = np.ones((B, 240)); t = np.tile(np.arange(1, 241), (B, 1))
502
  X = np.stack([
503
  static[:,0], static[:,1], static[:,2], static[:,3], static[:,4],
504
  zeros, atr_p, pnl, max_pnl, zeros, zeros, t, zeros, ones*0.6, ones*0.7, ones*3
505
  ], axis=2).reshape(-1, 16)
 
506
  X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
507
  preds = hydra_models['crash'].predict_proba(X)[:, 1].reshape(B, 240)
508
  res_hydra_risk[r_idxs] = np.max(preds, axis=1)
509
  except: pass
510
 
511
+ # 7. Legacy
512
  res_legacy_v2 = np.zeros(len(df_candidates))
513
  res_legacy_v3 = np.zeros(len(df_candidates))
 
514
  if self.proc.guardian_legacy:
515
  try:
516
  X_v2_full = calculate_legacy_v2_vectorized(df_1m, df_5m, df_15m)
517
  v3_df_full = calculate_legacy_v3_vectorized(df_1m, df_5m, df_15m)
 
518
  all_indices = np.arange(len(df_1m))
519
  cand_indices = all_indices[valid_mask]
520
  max_len = len(df_1m)
521
  cand_indices = cand_indices[cand_indices < max_len]
 
522
  if len(cand_indices) > 0:
523
  if self.proc.guardian_legacy.model_v2 and X_v2_full is not None:
524
  subset_v2 = X_v2_full[cand_indices]
525
  preds_v2 = self.proc.guardian_legacy.model_v2.predict(xgb.DMatrix(subset_v2))
526
  if len(preds_v2.shape) > 1: res_legacy_v2[:len(cand_indices)] = preds_v2[:, 2]
527
  else: res_legacy_v2[:len(cand_indices)] = preds_v2
 
528
  if self.proc.guardian_legacy.model_v3 and v3_df_full is not None:
529
  subset_v3_df = v3_df_full.iloc[cand_indices]
530
  preds_v3 = self.proc.guardian_legacy.model_v3.predict(xgb.DMatrix(subset_v3_df))
531
  res_legacy_v3[:len(cand_indices)] = preds_v3
532
+ except Exception as e: print(f"Legacy Error: {e}")
533
 
534
+ # 8. Assembly
 
 
 
535
  print(f" πŸ“Š [Stats] Titan:{res_titan.mean():.2f} | Patterns:{res_patterns.mean():.2f} | Sniper:{res_sniper.mean():.2f} | Oracle:{res_oracle.mean():.2f}")
 
536
  ai_df = pd.DataFrame({
537
  'timestamp': df_candidates['timestamp'],
538
  'symbol': sym,
 
545
  'risk_legacy_v2': res_legacy_v2,
546
  'risk_legacy_v3': res_legacy_v3
547
  })
 
548
  dt = time.time() - t0
549
  if not ai_df.empty:
550
  ai_df.to_pickle(scores_file)
 
571
  if not data: return []
572
  df = pd.concat(data).sort_values('timestamp')
573
 
 
574
  ts = df['timestamp'].values; close = df['close'].values.astype(float)
575
  sym = df['symbol'].values; sym_map = {s:i for i,s in enumerate(np.unique(sym))}
576
  sym_id = np.array([sym_map[s] for s in sym])
 
578
  oracle = df['oracle_conf'].values; sniper = df['sniper_score'].values
579
  hydra = df['risk_hydra_crash'].values; titan = df['real_titan'].values
580
  l1 = df['l1_score'].values
581
+ legacy_v2 = df['risk_legacy_v2'].values; legacy_v3 = df['risk_legacy_v3'].values
 
 
582
 
583
  N = len(ts)
584
  print(f" πŸš€ [System] Testing {len(combinations_batch)} configs on {N} candles...", flush=True)
 
587
  for cfg in combinations_batch:
588
  pos = {}; log = []
589
  bal = initial_capital; alloc = 0.0
 
590
  mask = (l1 >= cfg['l1_thresh']) & (oracle >= cfg['oracle_thresh']) & (sniper >= cfg['sniper_thresh'])
591
 
592
  for i in range(N):
593
  s = sym_id[i]; p = close[i]
 
 
594
  if s in pos:
595
  entry = pos[s][0]; h_r = pos[s][1]; titan_entry = pos[s][3]
 
596
  crash_hydra = (h_r > cfg['hydra_thresh'])
597
  panic_legacy = (legacy_v2[i] > cfg['legacy_thresh']) or (legacy_v3[i] > cfg['legacy_thresh'])
 
598
  pnl = (p - entry)/entry
599
 
600
  if crash_hydra or panic_legacy or pnl > 0.04 or pnl < -0.02:
601
  realized = pnl - fees_pct*2
602
  bal += pos[s][2] * (1 + realized)
603
  alloc -= pos[s][2]
 
604
  is_consensus = (titan_entry > 0.55)
605
  log.append({'pnl': realized, 'consensus': is_consensus})
606
  del pos[s]
607
 
 
608
  if len(pos) < max_slots and mask[i]:
609
  if s not in pos and bal >= 5.0:
610
  size = min(10.0, bal * 0.98)
611
  pos[s] = (p, hydra[i], size, titan[i])
612
  bal -= size; alloc += size
613
 
 
614
  final_bal = bal + alloc
615
  profit = final_bal - initial_capital
616
  tot = len(log)
 
634
 
635
  async def run_optimization(self, target_regime="RANGE"):
636
  await self.generate_truth_data()
 
 
637
  oracle_r = np.linspace(0.3, 0.7, 3); sniper_r = np.linspace(0.2, 0.6, 3)
638
  hydra_r = [0.8, 0.9]; l1_r = [5.0, 10.0]
639
 
640
  combos = []
641
  for o, s, h, l1 in itertools.product(oracle_r, sniper_r, hydra_r, l1_r):
642
  combos.append({
643
+ 'w_titan': 0.4, 'w_struct': 0.3, 'thresh': l1, 'l1_thresh': l1,
 
644
  'oracle_thresh': o, 'sniper_thresh': s, 'hydra_thresh': h, 'legacy_thresh': 0.95
645
  })
646
 
647
  files = glob.glob(os.path.join(CACHE_DIR, "*.pkl"))
648
  results_list = self._worker_optimize(combos, files, self.INITIAL_CAPITAL, self.TRADING_FEES, self.MAX_SLOTS)
649
+ if not results_list: return None, {'net_profit': 0.0, 'win_rate': 0.0}
 
 
 
650
 
651
  results_list.sort(key=lambda x: x['net_profit'], reverse=True)
652
  best = results_list[0]
 
658
  print("-" * 60)
659
  print(f" πŸ“Š Total Trades: {best['total_trades']}")
660
  print(f" πŸ“ˆ Win Rate: {best['win_rate']:.1f}%")
 
661
  print("-" * 60)
662
  print(f" 🧠 CONSENSUS ANALYTICS:")
663
  print(f" 🀝 Model Agreement Rate: {best['consensus_agreement_rate']:.1f}%")
 
665
  print(f" πŸ’Ž High-Consensus Avg PnL: {best['high_consensus_avg_pnl']:.2f}%")
666
  print("-" * 60)
667
  print(f" βš™οΈ Oracle={best['config']['oracle_thresh']:.2f} | Sniper={best['config']['sniper_thresh']:.2f} | Hydra={best['config']['hydra_thresh']:.2f}")
 
668
  print("="*60)
669
  return best['config'], best
670
 
671
  async def run_strategic_optimization_task():
672
  print("\nπŸ§ͺ [STRATEGIC BACKTEST] Full Spectrum Mode...")
673
  r2 = R2Service(); dm = DataManager(None, None, r2); proc = MLProcessor(dm)
 
674
  try:
675
  await dm.initialize(); await proc.initialize()
676
  if proc.guardian_hydra: proc.guardian_hydra.set_silent_mode(True)
 
677
  hub = AdaptiveHub(r2); await hub.initialize()
678
  opt = HeavyDutyBacktester(dm, proc)
679
 
 
688
  opt.set_date_range(s["start"], s["end"])
689
  best_cfg, best_stats = await opt.run_optimization(s["regime"])
690
  if best_cfg: hub.submit_challenger(s["regime"], best_cfg, best_stats)
 
691
  await hub._save_state_to_r2()
692
  print("βœ… [System] DNA Updated.")
 
693
  finally:
694
  print("πŸ”Œ [System] Closing connections...")
695
  await dm.close()