veteroner commited on
Commit
443b35e
Β·
verified Β·
1 Parent(s): 7eff6ee

fix: WAL-inclusive backup + reachable scorecard GO + complete run log

Browse files
Files changed (1) hide show
  1. trading/worker.py +45 -0
trading/worker.py CHANGED
@@ -1045,6 +1045,11 @@ class TradingWorker:
1045
  if int(day_retries) >= _MAX_DAILY_RETRIES:
1046
  logger.info("Max retries reached today (%s/%s) β€” done.", day_retries, _MAX_DAILY_RETRIES)
1047
  _write_status({"is_running": False, "status": "SKIPPED", "reason": f"Max retries ({day_retries}/{_MAX_DAILY_RETRIES})"})
 
 
 
 
 
1048
  return {"date": today, "status": "SKIPPED"}
1049
  else:
1050
  logger.info("No trades yet today β€” retry %s/%s", int(day_retries) + 1, _MAX_DAILY_RETRIES)
@@ -1067,12 +1072,20 @@ class TradingWorker:
1067
  logger.critical("Reconciliation failed β€” aborting cycle")
1068
  _tg_notify("recon_failed", market_id=self.market_id)
1069
  _write_status({"is_running": False, "status": "RECON_FAILED", "recon": recon.to_dict()})
 
 
 
 
1070
  return {"date": today, "status": "RECON_FAILED", "recon": recon.to_dict()}
1071
  except Exception as e:
1072
  logger.error("Reconciliation error: %s", e)
1073
  # Don't kill β€” just warn and continue if paper
1074
  if self.mode == "live":
1075
  _write_status({"is_running": False, "status": "RECON_ERROR", "error": str(e)})
 
 
 
 
1076
  return {"date": today, "status": "RECON_ERROR", "error": str(e)}
1077
 
1078
  # ---- Model Risk Check ----
@@ -1175,6 +1188,10 @@ class TradingWorker:
1175
  ),
1176
  )
1177
  _write_status({"is_running": False, "status": "SCAN_IN_PROGRESS"})
 
 
 
 
1178
  return {"date": today, "status": "SCAN_IN_PROGRESS"}
1179
 
1180
  symbols = _get_eligible_symbols(market_id=self.market_id)
@@ -1187,6 +1204,10 @@ class TradingWorker:
1187
  "status": "MISSING_SCAN_RESULTS",
1188
  "reason": f"Missing scan file: {scan_results_path}",
1189
  })
 
 
 
 
1190
  return {"date": today, "status": "MISSING_SCAN_RESULTS"}
1191
 
1192
  # Check if scan is still in progress (partial results)
@@ -1202,6 +1223,10 @@ class TradingWorker:
1202
  # Scan still running β€” don't burn the day, retry later
1203
  logger.warning("No eligible stocks yet but scan not completed β€” will retry later.")
1204
  _write_status({"is_running": False, "status": "SCAN_IN_PROGRESS"})
 
 
 
 
1205
  return {"date": today, "status": "SCAN_IN_PROGRESS"}
1206
 
1207
  self._store.set_state("last_run_date", today)
@@ -1224,6 +1249,10 @@ class TradingWorker:
1224
  logger.error("Signal generation failed: %s", e)
1225
  _tg_notify("signal_error", market_id=self.market_id, error=str(e))
1226
  _write_status({"is_running": False, "status": "SIGNAL_ERROR", "error": str(e)})
 
 
 
 
1227
  return {"date": today, "status": "ERROR", "reason": str(e)}
1228
 
1229
  if df_signals.empty:
@@ -1233,6 +1262,10 @@ class TradingWorker:
1233
  logger.warning("No signals generated for %d symbols", len(symbols))
1234
  _tg_notify("no_signals", market_id=self.market_id, symbol_count=len(symbols), silent=True)
1235
  _write_status({"is_running": False, "status": "NO_SIGNALS"})
 
 
 
 
1236
  return {"date": today, "status": "NO_SIGNALS"}
1237
 
1238
  # Log predictions to ModelRiskManager for future evaluation
@@ -1770,6 +1803,18 @@ class TradingWorker:
1770
  result = self.run_cycle()
1771
  status = result.get("status")
1772
  logger.info("Cycle: %s", status)
 
 
 
 
 
 
 
 
 
 
 
 
1773
  # Short retry for transient issues
1774
  if status in ("MISSING_SCAN_RESULTS", "SCAN_IN_PROGRESS"):
1775
  logger.info("Scan not ready β€” retrying in 60s")
 
1045
  if int(day_retries) >= _MAX_DAILY_RETRIES:
1046
  logger.info("Max retries reached today (%s/%s) β€” done.", day_retries, _MAX_DAILY_RETRIES)
1047
  _write_status({"is_running": False, "status": "SKIPPED", "reason": f"Max retries ({day_retries}/{_MAX_DAILY_RETRIES})"})
1048
+ # Record the run so scorecard consistency doesn't see a false gap
1049
+ try:
1050
+ self._store.record_run(date=today, market_id=self.market_id, status="SKIPPED")
1051
+ except Exception:
1052
+ pass
1053
  return {"date": today, "status": "SKIPPED"}
1054
  else:
1055
  logger.info("No trades yet today β€” retry %s/%s", int(day_retries) + 1, _MAX_DAILY_RETRIES)
 
1072
  logger.critical("Reconciliation failed β€” aborting cycle")
1073
  _tg_notify("recon_failed", market_id=self.market_id)
1074
  _write_status({"is_running": False, "status": "RECON_FAILED", "recon": recon.to_dict()})
1075
+ try:
1076
+ self._store.record_run(date=today, market_id=self.market_id, status="RECON_FAILED")
1077
+ except Exception:
1078
+ pass
1079
  return {"date": today, "status": "RECON_FAILED", "recon": recon.to_dict()}
1080
  except Exception as e:
1081
  logger.error("Reconciliation error: %s", e)
1082
  # Don't kill β€” just warn and continue if paper
1083
  if self.mode == "live":
1084
  _write_status({"is_running": False, "status": "RECON_ERROR", "error": str(e)})
1085
+ try:
1086
+ self._store.record_run(date=today, market_id=self.market_id, status="RECON_ERROR")
1087
+ except Exception:
1088
+ pass
1089
  return {"date": today, "status": "RECON_ERROR", "error": str(e)}
1090
 
1091
  # ---- Model Risk Check ----
 
1188
  ),
1189
  )
1190
  _write_status({"is_running": False, "status": "SCAN_IN_PROGRESS"})
1191
+ try:
1192
+ self._store.record_run(date=today, market_id=self.market_id, status="SCAN_IN_PROGRESS")
1193
+ except Exception:
1194
+ pass
1195
  return {"date": today, "status": "SCAN_IN_PROGRESS"}
1196
 
1197
  symbols = _get_eligible_symbols(market_id=self.market_id)
 
1204
  "status": "MISSING_SCAN_RESULTS",
1205
  "reason": f"Missing scan file: {scan_results_path}",
1206
  })
1207
+ try:
1208
+ self._store.record_run(date=today, market_id=self.market_id, status="MISSING_SCAN_RESULTS")
1209
+ except Exception:
1210
+ pass
1211
  return {"date": today, "status": "MISSING_SCAN_RESULTS"}
1212
 
1213
  # Check if scan is still in progress (partial results)
 
1223
  # Scan still running β€” don't burn the day, retry later
1224
  logger.warning("No eligible stocks yet but scan not completed β€” will retry later.")
1225
  _write_status({"is_running": False, "status": "SCAN_IN_PROGRESS"})
1226
+ try:
1227
+ self._store.record_run(date=today, market_id=self.market_id, status="SCAN_IN_PROGRESS")
1228
+ except Exception:
1229
+ pass
1230
  return {"date": today, "status": "SCAN_IN_PROGRESS"}
1231
 
1232
  self._store.set_state("last_run_date", today)
 
1249
  logger.error("Signal generation failed: %s", e)
1250
  _tg_notify("signal_error", market_id=self.market_id, error=str(e))
1251
  _write_status({"is_running": False, "status": "SIGNAL_ERROR", "error": str(e)})
1252
+ try:
1253
+ self._store.record_run(date=today, market_id=self.market_id, status="SIGNAL_ERROR")
1254
+ except Exception:
1255
+ pass
1256
  return {"date": today, "status": "ERROR", "reason": str(e)}
1257
 
1258
  if df_signals.empty:
 
1262
  logger.warning("No signals generated for %d symbols", len(symbols))
1263
  _tg_notify("no_signals", market_id=self.market_id, symbol_count=len(symbols), silent=True)
1264
  _write_status({"is_running": False, "status": "NO_SIGNALS"})
1265
+ try:
1266
+ self._store.record_run(date=today, market_id=self.market_id, status="NO_SIGNALS")
1267
+ except Exception:
1268
+ pass
1269
  return {"date": today, "status": "NO_SIGNALS"}
1270
 
1271
  # Log predictions to ModelRiskManager for future evaluation
 
1803
  result = self.run_cycle()
1804
  status = result.get("status")
1805
  logger.info("Cycle: %s", status)
1806
+ # Back up trading state after EVERY cycle outcome β€”
1807
+ # run_cycle's own upload only runs on the full-OK
1808
+ # path, so early returns (SKIPPED, SCAN_IN_PROGRESS,
1809
+ # NO_SIGNALS, …) previously left the remote backup
1810
+ # stale. Throttled to one upload per 5 minutes.
1811
+ try:
1812
+ if time.time() - getattr(self, "_last_state_backup_ts", 0) > 300:
1813
+ from trading.trading_state_sync import upload_trading_state
1814
+ upload_trading_state(self.market_id)
1815
+ self._last_state_backup_ts = time.time()
1816
+ except Exception as _bk_e:
1817
+ logger.warning("Post-cycle state backup failed: %s", _bk_e)
1818
  # Short retry for transient issues
1819
  if status in ("MISSING_SCAN_RESULTS", "SCAN_IN_PROGRESS"):
1820
  logger.info("Scan not ready β€” retrying in 60s")