fix: WAL-inclusive backup + reachable scorecard GO + complete run log
Browse files- trading/worker.py +45 -0
trading/worker.py
CHANGED
|
@@ -1045,6 +1045,11 @@ class TradingWorker:
|
|
| 1045 |
if int(day_retries) >= _MAX_DAILY_RETRIES:
|
| 1046 |
logger.info("Max retries reached today (%s/%s) β done.", day_retries, _MAX_DAILY_RETRIES)
|
| 1047 |
_write_status({"is_running": False, "status": "SKIPPED", "reason": f"Max retries ({day_retries}/{_MAX_DAILY_RETRIES})"})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1048 |
return {"date": today, "status": "SKIPPED"}
|
| 1049 |
else:
|
| 1050 |
logger.info("No trades yet today β retry %s/%s", int(day_retries) + 1, _MAX_DAILY_RETRIES)
|
|
@@ -1067,12 +1072,20 @@ class TradingWorker:
|
|
| 1067 |
logger.critical("Reconciliation failed β aborting cycle")
|
| 1068 |
_tg_notify("recon_failed", market_id=self.market_id)
|
| 1069 |
_write_status({"is_running": False, "status": "RECON_FAILED", "recon": recon.to_dict()})
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1070 |
return {"date": today, "status": "RECON_FAILED", "recon": recon.to_dict()}
|
| 1071 |
except Exception as e:
|
| 1072 |
logger.error("Reconciliation error: %s", e)
|
| 1073 |
# Don't kill β just warn and continue if paper
|
| 1074 |
if self.mode == "live":
|
| 1075 |
_write_status({"is_running": False, "status": "RECON_ERROR", "error": str(e)})
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1076 |
return {"date": today, "status": "RECON_ERROR", "error": str(e)}
|
| 1077 |
|
| 1078 |
# ---- Model Risk Check ----
|
|
@@ -1175,6 +1188,10 @@ class TradingWorker:
|
|
| 1175 |
),
|
| 1176 |
)
|
| 1177 |
_write_status({"is_running": False, "status": "SCAN_IN_PROGRESS"})
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1178 |
return {"date": today, "status": "SCAN_IN_PROGRESS"}
|
| 1179 |
|
| 1180 |
symbols = _get_eligible_symbols(market_id=self.market_id)
|
|
@@ -1187,6 +1204,10 @@ class TradingWorker:
|
|
| 1187 |
"status": "MISSING_SCAN_RESULTS",
|
| 1188 |
"reason": f"Missing scan file: {scan_results_path}",
|
| 1189 |
})
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1190 |
return {"date": today, "status": "MISSING_SCAN_RESULTS"}
|
| 1191 |
|
| 1192 |
# Check if scan is still in progress (partial results)
|
|
@@ -1202,6 +1223,10 @@ class TradingWorker:
|
|
| 1202 |
# Scan still running β don't burn the day, retry later
|
| 1203 |
logger.warning("No eligible stocks yet but scan not completed β will retry later.")
|
| 1204 |
_write_status({"is_running": False, "status": "SCAN_IN_PROGRESS"})
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1205 |
return {"date": today, "status": "SCAN_IN_PROGRESS"}
|
| 1206 |
|
| 1207 |
self._store.set_state("last_run_date", today)
|
|
@@ -1224,6 +1249,10 @@ class TradingWorker:
|
|
| 1224 |
logger.error("Signal generation failed: %s", e)
|
| 1225 |
_tg_notify("signal_error", market_id=self.market_id, error=str(e))
|
| 1226 |
_write_status({"is_running": False, "status": "SIGNAL_ERROR", "error": str(e)})
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1227 |
return {"date": today, "status": "ERROR", "reason": str(e)}
|
| 1228 |
|
| 1229 |
if df_signals.empty:
|
|
@@ -1233,6 +1262,10 @@ class TradingWorker:
|
|
| 1233 |
logger.warning("No signals generated for %d symbols", len(symbols))
|
| 1234 |
_tg_notify("no_signals", market_id=self.market_id, symbol_count=len(symbols), silent=True)
|
| 1235 |
_write_status({"is_running": False, "status": "NO_SIGNALS"})
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1236 |
return {"date": today, "status": "NO_SIGNALS"}
|
| 1237 |
|
| 1238 |
# Log predictions to ModelRiskManager for future evaluation
|
|
@@ -1770,6 +1803,18 @@ class TradingWorker:
|
|
| 1770 |
result = self.run_cycle()
|
| 1771 |
status = result.get("status")
|
| 1772 |
logger.info("Cycle: %s", status)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1773 |
# Short retry for transient issues
|
| 1774 |
if status in ("MISSING_SCAN_RESULTS", "SCAN_IN_PROGRESS"):
|
| 1775 |
logger.info("Scan not ready β retrying in 60s")
|
|
|
|
| 1045 |
if int(day_retries) >= _MAX_DAILY_RETRIES:
|
| 1046 |
logger.info("Max retries reached today (%s/%s) β done.", day_retries, _MAX_DAILY_RETRIES)
|
| 1047 |
_write_status({"is_running": False, "status": "SKIPPED", "reason": f"Max retries ({day_retries}/{_MAX_DAILY_RETRIES})"})
|
| 1048 |
+
# Record the run so scorecard consistency doesn't see a false gap
|
| 1049 |
+
try:
|
| 1050 |
+
self._store.record_run(date=today, market_id=self.market_id, status="SKIPPED")
|
| 1051 |
+
except Exception:
|
| 1052 |
+
pass
|
| 1053 |
return {"date": today, "status": "SKIPPED"}
|
| 1054 |
else:
|
| 1055 |
logger.info("No trades yet today β retry %s/%s", int(day_retries) + 1, _MAX_DAILY_RETRIES)
|
|
|
|
| 1072 |
logger.critical("Reconciliation failed β aborting cycle")
|
| 1073 |
_tg_notify("recon_failed", market_id=self.market_id)
|
| 1074 |
_write_status({"is_running": False, "status": "RECON_FAILED", "recon": recon.to_dict()})
|
| 1075 |
+
try:
|
| 1076 |
+
self._store.record_run(date=today, market_id=self.market_id, status="RECON_FAILED")
|
| 1077 |
+
except Exception:
|
| 1078 |
+
pass
|
| 1079 |
return {"date": today, "status": "RECON_FAILED", "recon": recon.to_dict()}
|
| 1080 |
except Exception as e:
|
| 1081 |
logger.error("Reconciliation error: %s", e)
|
| 1082 |
# Don't kill β just warn and continue if paper
|
| 1083 |
if self.mode == "live":
|
| 1084 |
_write_status({"is_running": False, "status": "RECON_ERROR", "error": str(e)})
|
| 1085 |
+
try:
|
| 1086 |
+
self._store.record_run(date=today, market_id=self.market_id, status="RECON_ERROR")
|
| 1087 |
+
except Exception:
|
| 1088 |
+
pass
|
| 1089 |
return {"date": today, "status": "RECON_ERROR", "error": str(e)}
|
| 1090 |
|
| 1091 |
# ---- Model Risk Check ----
|
|
|
|
| 1188 |
),
|
| 1189 |
)
|
| 1190 |
_write_status({"is_running": False, "status": "SCAN_IN_PROGRESS"})
|
| 1191 |
+
try:
|
| 1192 |
+
self._store.record_run(date=today, market_id=self.market_id, status="SCAN_IN_PROGRESS")
|
| 1193 |
+
except Exception:
|
| 1194 |
+
pass
|
| 1195 |
return {"date": today, "status": "SCAN_IN_PROGRESS"}
|
| 1196 |
|
| 1197 |
symbols = _get_eligible_symbols(market_id=self.market_id)
|
|
|
|
| 1204 |
"status": "MISSING_SCAN_RESULTS",
|
| 1205 |
"reason": f"Missing scan file: {scan_results_path}",
|
| 1206 |
})
|
| 1207 |
+
try:
|
| 1208 |
+
self._store.record_run(date=today, market_id=self.market_id, status="MISSING_SCAN_RESULTS")
|
| 1209 |
+
except Exception:
|
| 1210 |
+
pass
|
| 1211 |
return {"date": today, "status": "MISSING_SCAN_RESULTS"}
|
| 1212 |
|
| 1213 |
# Check if scan is still in progress (partial results)
|
|
|
|
| 1223 |
# Scan still running β don't burn the day, retry later
|
| 1224 |
logger.warning("No eligible stocks yet but scan not completed β will retry later.")
|
| 1225 |
_write_status({"is_running": False, "status": "SCAN_IN_PROGRESS"})
|
| 1226 |
+
try:
|
| 1227 |
+
self._store.record_run(date=today, market_id=self.market_id, status="SCAN_IN_PROGRESS")
|
| 1228 |
+
except Exception:
|
| 1229 |
+
pass
|
| 1230 |
return {"date": today, "status": "SCAN_IN_PROGRESS"}
|
| 1231 |
|
| 1232 |
self._store.set_state("last_run_date", today)
|
|
|
|
| 1249 |
logger.error("Signal generation failed: %s", e)
|
| 1250 |
_tg_notify("signal_error", market_id=self.market_id, error=str(e))
|
| 1251 |
_write_status({"is_running": False, "status": "SIGNAL_ERROR", "error": str(e)})
|
| 1252 |
+
try:
|
| 1253 |
+
self._store.record_run(date=today, market_id=self.market_id, status="SIGNAL_ERROR")
|
| 1254 |
+
except Exception:
|
| 1255 |
+
pass
|
| 1256 |
return {"date": today, "status": "ERROR", "reason": str(e)}
|
| 1257 |
|
| 1258 |
if df_signals.empty:
|
|
|
|
| 1262 |
logger.warning("No signals generated for %d symbols", len(symbols))
|
| 1263 |
_tg_notify("no_signals", market_id=self.market_id, symbol_count=len(symbols), silent=True)
|
| 1264 |
_write_status({"is_running": False, "status": "NO_SIGNALS"})
|
| 1265 |
+
try:
|
| 1266 |
+
self._store.record_run(date=today, market_id=self.market_id, status="NO_SIGNALS")
|
| 1267 |
+
except Exception:
|
| 1268 |
+
pass
|
| 1269 |
return {"date": today, "status": "NO_SIGNALS"}
|
| 1270 |
|
| 1271 |
# Log predictions to ModelRiskManager for future evaluation
|
|
|
|
| 1803 |
result = self.run_cycle()
|
| 1804 |
status = result.get("status")
|
| 1805 |
logger.info("Cycle: %s", status)
|
| 1806 |
+
# Back up trading state after EVERY cycle outcome β
|
| 1807 |
+
# run_cycle's own upload only runs on the full-OK
|
| 1808 |
+
# path, so early returns (SKIPPED, SCAN_IN_PROGRESS,
|
| 1809 |
+
# NO_SIGNALS, β¦) previously left the remote backup
|
| 1810 |
+
# stale. Throttled to one upload per 5 minutes.
|
| 1811 |
+
try:
|
| 1812 |
+
if time.time() - getattr(self, "_last_state_backup_ts", 0) > 300:
|
| 1813 |
+
from trading.trading_state_sync import upload_trading_state
|
| 1814 |
+
upload_trading_state(self.market_id)
|
| 1815 |
+
self._last_state_backup_ts = time.time()
|
| 1816 |
+
except Exception as _bk_e:
|
| 1817 |
+
logger.warning("Post-cycle state backup failed: %s", _bk_e)
|
| 1818 |
# Short retry for transient issues
|
| 1819 |
if status in ("MISSING_SCAN_RESULTS", "SCAN_IN_PROGRESS"):
|
| 1820 |
logger.info("Scan not ready β retrying in 60s")
|