Riy777 commited on
Commit
726d487
Β·
verified Β·
1 Parent(s): 009e186

Rename ml_engine/titan_engine.py to ml_engine/pattern_engine.py

Browse files
ml_engine/{titan_engine.py β†’ pattern_engine.py} RENAMED
@@ -1,9 +1,9 @@
1
  # ==============================================================================
2
- # πŸ›‘οΈ ml_engine/titan_engine.py (V3.1 - Fix List vs DataFrame Issue)
3
  # ==============================================================================
4
  # GEM-Architect Approved
5
- # - Fixes AttributeError: 'list' object has no attribute 'empty'
6
- # - Auto-converts raw list data to Pandas DataFrame with correct columns.
7
  # ==============================================================================
8
 
9
  import os
@@ -18,7 +18,7 @@ import warnings
18
  warnings.filterwarnings('ignore')
19
 
20
  # ------------------------------------------------------------------------------
21
- # 1. Model Architecture (Must match training EXACTLY)
22
  # ------------------------------------------------------------------------------
23
  class ResidualBlock(nn.Module):
24
  def __init__(self, channels, kernel_size=3, dropout=0.2):
@@ -42,7 +42,7 @@ class ResidualBlock(nn.Module):
42
  out = self.relu(out)
43
  return out
44
 
45
- class TitanResNet(nn.Module):
46
  def __init__(self, in_ch):
47
  super().__init__()
48
  self.entry = nn.Sequential(
@@ -86,17 +86,17 @@ class TitanResNet(nn.Module):
86
  # ------------------------------------------------------------------------------
87
  # 2. Production Engine Class
88
  # ------------------------------------------------------------------------------
89
- class TitanEngine:
90
  def __init__(self, model_dir="ml_models/Unified_Models_V1"):
 
91
  self.model_path = os.path.join(model_dir, "cnn_best.pt")
92
  self.scaler_path = os.path.join(model_dir, "seq_scaler.pkl")
93
 
94
  self.model = None
95
  self.scaler = None
96
- self.device = torch.device("cpu") # Inference on CPU is safer for web apps
97
  self.initialized = False
98
 
99
- # Exact Features used in Training
100
  self.features_list = [
101
  "log_ret","vol_spike","taker_buy_ratio","proxy_spread","amihud","avg_ticket_usd",
102
  "upper_wick_ratio","lower_wick_ratio","body_to_range","atr_pct_signal"
@@ -104,41 +104,33 @@ class TitanEngine:
104
  self.WINDOW_SIZE = 64
105
 
106
  async def initialize(self):
107
- """Load Model and Scaler"""
108
  if self.initialized: return True
109
 
110
- print(f"πŸ›‘οΈ [Titan] Initializing PyTorch Engine from {self.model_path}...")
111
  try:
112
  if not os.path.exists(self.model_path) or not os.path.exists(self.scaler_path):
113
- print(f"❌ [Titan] Artifacts missing in {self.model_path}")
114
  return False
115
 
116
- # 1. Load Scaler
117
  self.scaler = joblib.load(self.scaler_path)
 
118
 
119
- # 2. Load Model
120
- # We initialize the architecture first
121
- self.model = TitanResNet(in_ch=len(self.features_list)).to(self.device)
122
-
123
- # Safe loading for CPU
124
  checkpoint = torch.load(self.model_path, map_location=self.device)
125
  if isinstance(checkpoint, dict) and 'model' in checkpoint:
126
  self.model.load_state_dict(checkpoint['model'])
127
  else:
128
  self.model.load_state_dict(checkpoint)
129
 
130
- self.model.eval() # Set to evaluation mode
131
-
132
  self.initialized = True
133
- print(f"βœ… [Titan] Online. ResNet-1D Loaded successfully.")
134
  return True
135
 
136
  except Exception as e:
137
- print(f"❌ [Titan] Init Error: {e}")
138
  traceback.print_exc()
139
  return False
140
 
141
- # --- Feature Engineering Helpers (MATCHING TRAINING LOGIC) ---
142
  def _wilder_rma(self, x, n):
143
  x = np.asarray(x, dtype=float)
144
  return pd.Series(x).ewm(alpha=1.0/n, adjust=False).mean().values
@@ -150,123 +142,80 @@ class TitanEngine:
150
  return pd.Series(x).rolling(w, min_periods=1).median().values
151
 
152
  def preprocess_live_data(self, df):
153
- """
154
- Turns raw OHLCV DataFrame into the exact Feature Matrix used for training.
155
- Assuming 'df' has at least 100 rows.
156
- """
157
  try:
158
  df = df.copy()
159
- # Ensure sorting if timestamp exists, else assume ordered list
160
- if 'timestamp' in df.columns:
161
- df = df.sort_values('timestamp')
162
 
163
- # Basic conversions
164
  close = df['close'].values.astype(float)
165
  high = df['high'].values.astype(float)
166
  low = df['low'].values.astype(float)
167
  open_ = df['open'].values.astype(float)
168
- # Use quote volume if available (better for vol_usd), else close * vol
169
- if 'quote_volume' in df.columns:
170
- vol_usd = df['quote_volume'].values.astype(float)
171
- else:
172
- vol_usd = (close * df['volume'].values).astype(float)
173
 
174
  vol_usd = np.maximum(vol_usd, 1.0)
175
 
176
- # 1. ATR (14)
177
  prev_close = np.roll(close, 1); prev_close[0] = close[0]
178
  tr = np.maximum(high - low, np.maximum(np.abs(high - prev_close), np.abs(low - prev_close)))
179
  atr = self._wilder_rma(tr, 14)
180
  atr_safe = np.maximum(atr, 1e-9)
181
 
182
- # 2. Features Calculation
183
  df['log_ret'] = np.log(close / np.maximum(prev_close, 1e-9))
184
-
185
  vol_ma = self._rolling_mean(vol_usd, 20)
186
  df['vol_spike'] = vol_usd / np.maximum(vol_ma, 1e-9)
187
 
188
- # Taker buy ratio (if available, else 0.5 default)
189
  if 'taker_buy_base_asset_volume' in df.columns:
190
  taker_vol = df['taker_buy_base_asset_volume'].values * close
191
  df['taker_buy_ratio'] = taker_vol / vol_usd
192
- else:
193
- df['taker_buy_ratio'] = 0.5
194
 
195
  raw_spread = (high - low) / np.maximum(close, 1e-9)
196
  df['proxy_spread'] = self._rolling_median(raw_spread, 14) * 0.5
197
-
198
  df['amihud'] = np.abs(df['log_ret']) / vol_usd
199
 
200
- # Num trades proxy (if not available)
201
- if 'num_trades' in df.columns:
202
- num_trades = df['num_trades'].values
203
- else:
204
- num_trades = vol_usd / 1000.0 # Rough estimate
205
  df['avg_ticket_usd'] = vol_usd / np.maximum(num_trades, 1.0)
206
 
207
  rng = np.maximum(high - low, 1e-9)
208
  df['upper_wick_ratio'] = (high - np.maximum(open_, close)) / rng
209
  df['lower_wick_ratio'] = (np.minimum(open_, close) - low) / rng
210
  df['body_to_range'] = np.abs(close - open_) / rng
211
-
212
  df['atr_pct_signal'] = atr_safe / close
213
 
214
- # Filter NaNs
215
  df = df.replace([np.inf, -np.inf], np.nan).fillna(0)
216
 
217
- # Extract only the needed window (last 64 candles)
218
- if len(df) < self.WINDOW_SIZE:
219
- return None
220
-
221
- # Get the feature matrix for the LAST window
222
- # We take the last 64 rows of the required features
223
  feature_matrix = df[self.features_list].iloc[-self.WINDOW_SIZE:].values.astype(np.float32)
224
-
225
  return feature_matrix
226
 
227
  except Exception as e:
228
- print(f"❌ [Titan] Preprocessing Error: {e}")
229
  return None
230
 
231
  def predict(self, ohlcv_data: dict) -> dict:
232
- """
233
- Main Interface used by Processor.
234
- Handles both List and DataFrame inputs robustly.
235
- """
236
  if not self.initialized: return {'score': 0.0, 'error': 'Not Initialized'}
237
 
238
  try:
239
- # We use 15m data as the main driver
240
  target_tf = '15m'
241
  raw_data = ohlcv_data.get(target_tf)
242
-
243
- if raw_data is None:
244
- return {'score': 0.0, 'error': 'No 15m Data'}
245
 
246
- # βœ… FIX: Auto-detect List vs DataFrame
247
  if isinstance(raw_data, list):
248
- # Standard CCXT OHLCV structure: [timestamp, open, high, low, close, volume]
249
  df = pd.DataFrame(raw_data, columns=['timestamp', 'open', 'high', 'low', 'close', 'volume'])
250
  elif isinstance(raw_data, pd.DataFrame):
251
  df = raw_data
252
- else:
253
- return {'score': 0.0, 'error': f'Invalid Data Type: {type(raw_data)}'}
254
 
255
- if df.empty:
256
- return {'score': 0.0, 'error': 'Empty Data'}
257
 
258
- # Preprocess
259
  X_raw = self.preprocess_live_data(df)
260
- if X_raw is None:
261
- return {'score': 0.0, 'error': 'Not enough data for window'}
262
 
263
- # Scale
264
  X_scaled = self.scaler.transform(X_raw)
265
-
266
- # Prepare Tensor
267
  X_tensor = torch.tensor(X_scaled.T).unsqueeze(0).to(self.device)
268
 
269
- # Inference
270
  with torch.no_grad():
271
  logits = self.model(X_tensor)
272
  probs = torch.softmax(logits, dim=1).cpu().numpy()[0]
@@ -278,6 +227,6 @@ class TitanEngine:
278
  }
279
 
280
  except Exception as e:
281
- print(f"❌ [Titan] Inference Error: {e}")
282
  traceback.print_exc()
283
  return {'score': 0.0, 'error': str(e)}
 
1
  # ==============================================================================
2
+ # 🧠 ml_engine/pattern_engine.py (Formerly Titan)
3
  # ==============================================================================
4
  # GEM-Architect Approved
5
+ # - Renamed from TitanEngine to PatternEngine (Semantic Consistency).
6
+ # - Logic: ResNet-1D for Pattern Recognition (Neural Network).
7
  # ==============================================================================
8
 
9
  import os
 
18
  warnings.filterwarnings('ignore')
19
 
20
  # ------------------------------------------------------------------------------
21
+ # 1. Model Architecture (ResNet-1D)
22
  # ------------------------------------------------------------------------------
23
  class ResidualBlock(nn.Module):
24
  def __init__(self, channels, kernel_size=3, dropout=0.2):
 
42
  out = self.relu(out)
43
  return out
44
 
45
+ class PatternResNet(nn.Module):
46
  def __init__(self, in_ch):
47
  super().__init__()
48
  self.entry = nn.Sequential(
 
86
  # ------------------------------------------------------------------------------
87
  # 2. Production Engine Class
88
  # ------------------------------------------------------------------------------
89
+ class PatternEngine:
90
  def __init__(self, model_dir="ml_models/Unified_Models_V1"):
91
+ # We assume the model file name remains 'cnn_best.pt' inside the folder
92
  self.model_path = os.path.join(model_dir, "cnn_best.pt")
93
  self.scaler_path = os.path.join(model_dir, "seq_scaler.pkl")
94
 
95
  self.model = None
96
  self.scaler = None
97
+ self.device = torch.device("cpu")
98
  self.initialized = False
99
 
 
100
  self.features_list = [
101
  "log_ret","vol_spike","taker_buy_ratio","proxy_spread","amihud","avg_ticket_usd",
102
  "upper_wick_ratio","lower_wick_ratio","body_to_range","atr_pct_signal"
 
104
  self.WINDOW_SIZE = 64
105
 
106
  async def initialize(self):
 
107
  if self.initialized: return True
108
 
109
+ print(f"🧠 [PatternNet] Initializing Neural Network from {self.model_path}...")
110
  try:
111
  if not os.path.exists(self.model_path) or not os.path.exists(self.scaler_path):
112
+ print(f"❌ [PatternNet] Artifacts missing in {self.model_path}")
113
  return False
114
 
 
115
  self.scaler = joblib.load(self.scaler_path)
116
+ self.model = PatternResNet(in_ch=len(self.features_list)).to(self.device)
117
 
 
 
 
 
 
118
  checkpoint = torch.load(self.model_path, map_location=self.device)
119
  if isinstance(checkpoint, dict) and 'model' in checkpoint:
120
  self.model.load_state_dict(checkpoint['model'])
121
  else:
122
  self.model.load_state_dict(checkpoint)
123
 
124
+ self.model.eval()
 
125
  self.initialized = True
126
+ print(f"βœ… [PatternNet] Online. ResNet-1D Ready for Pattern Recognition.")
127
  return True
128
 
129
  except Exception as e:
130
+ print(f"❌ [PatternNet] Init Error: {e}")
131
  traceback.print_exc()
132
  return False
133
 
 
134
  def _wilder_rma(self, x, n):
135
  x = np.asarray(x, dtype=float)
136
  return pd.Series(x).ewm(alpha=1.0/n, adjust=False).mean().values
 
142
  return pd.Series(x).rolling(w, min_periods=1).median().values
143
 
144
  def preprocess_live_data(self, df):
 
 
 
 
145
  try:
146
  df = df.copy()
147
+ if 'timestamp' in df.columns: df = df.sort_values('timestamp')
 
 
148
 
 
149
  close = df['close'].values.astype(float)
150
  high = df['high'].values.astype(float)
151
  low = df['low'].values.astype(float)
152
  open_ = df['open'].values.astype(float)
153
+
154
+ if 'quote_volume' in df.columns: vol_usd = df['quote_volume'].values.astype(float)
155
+ else: vol_usd = (close * df['volume'].values).astype(float)
 
 
156
 
157
  vol_usd = np.maximum(vol_usd, 1.0)
158
 
 
159
  prev_close = np.roll(close, 1); prev_close[0] = close[0]
160
  tr = np.maximum(high - low, np.maximum(np.abs(high - prev_close), np.abs(low - prev_close)))
161
  atr = self._wilder_rma(tr, 14)
162
  atr_safe = np.maximum(atr, 1e-9)
163
 
 
164
  df['log_ret'] = np.log(close / np.maximum(prev_close, 1e-9))
 
165
  vol_ma = self._rolling_mean(vol_usd, 20)
166
  df['vol_spike'] = vol_usd / np.maximum(vol_ma, 1e-9)
167
 
 
168
  if 'taker_buy_base_asset_volume' in df.columns:
169
  taker_vol = df['taker_buy_base_asset_volume'].values * close
170
  df['taker_buy_ratio'] = taker_vol / vol_usd
171
+ else: df['taker_buy_ratio'] = 0.5
 
172
 
173
  raw_spread = (high - low) / np.maximum(close, 1e-9)
174
  df['proxy_spread'] = self._rolling_median(raw_spread, 14) * 0.5
 
175
  df['amihud'] = np.abs(df['log_ret']) / vol_usd
176
 
177
+ if 'num_trades' in df.columns: num_trades = df['num_trades'].values
178
+ else: num_trades = vol_usd / 1000.0
 
 
 
179
  df['avg_ticket_usd'] = vol_usd / np.maximum(num_trades, 1.0)
180
 
181
  rng = np.maximum(high - low, 1e-9)
182
  df['upper_wick_ratio'] = (high - np.maximum(open_, close)) / rng
183
  df['lower_wick_ratio'] = (np.minimum(open_, close) - low) / rng
184
  df['body_to_range'] = np.abs(close - open_) / rng
 
185
  df['atr_pct_signal'] = atr_safe / close
186
 
 
187
  df = df.replace([np.inf, -np.inf], np.nan).fillna(0)
188
 
189
+ if len(df) < self.WINDOW_SIZE: return None
 
 
 
 
 
190
  feature_matrix = df[self.features_list].iloc[-self.WINDOW_SIZE:].values.astype(np.float32)
 
191
  return feature_matrix
192
 
193
  except Exception as e:
194
+ print(f"❌ [PatternNet] Preprocessing Error: {e}")
195
  return None
196
 
197
  def predict(self, ohlcv_data: dict) -> dict:
 
 
 
 
198
  if not self.initialized: return {'score': 0.0, 'error': 'Not Initialized'}
199
 
200
  try:
 
201
  target_tf = '15m'
202
  raw_data = ohlcv_data.get(target_tf)
203
+ if raw_data is None: return {'score': 0.0, 'error': 'No 15m Data'}
 
 
204
 
 
205
  if isinstance(raw_data, list):
 
206
  df = pd.DataFrame(raw_data, columns=['timestamp', 'open', 'high', 'low', 'close', 'volume'])
207
  elif isinstance(raw_data, pd.DataFrame):
208
  df = raw_data
209
+ else: return {'score': 0.0, 'error': f'Invalid Data Type'}
 
210
 
211
+ if df.empty: return {'score': 0.0, 'error': 'Empty Data'}
 
212
 
 
213
  X_raw = self.preprocess_live_data(df)
214
+ if X_raw is None: return {'score': 0.0, 'error': 'Not enough data'}
 
215
 
 
216
  X_scaled = self.scaler.transform(X_raw)
 
 
217
  X_tensor = torch.tensor(X_scaled.T).unsqueeze(0).to(self.device)
218
 
 
219
  with torch.no_grad():
220
  logits = self.model(X_tensor)
221
  probs = torch.softmax(logits, dim=1).cpu().numpy()[0]
 
227
  }
228
 
229
  except Exception as e:
230
+ print(f"❌ [PatternNet] Inference Error: {e}")
231
  traceback.print_exc()
232
  return {'score': 0.0, 'error': str(e)}