Spaces:
Sleeping
Sleeping
Cyber Catalyst Team commited on
Commit ·
17a4f6d
1
Parent(s): e4511ee
Fix P0 asyncpg event loop mismatch: run eternity loop as an async task on the main loop
Browse files- backend.py +12 -20
backend.py
CHANGED
|
@@ -1970,21 +1970,19 @@ async def execute_eternity_cycle(project_name: str, goal: str):
|
|
| 1970 |
|
| 1971 |
last_run_times = {} # project_name -> timestamp
|
| 1972 |
|
| 1973 |
-
def run_eternity_loop():
|
| 1974 |
-
log_activity("[Eternity Loop]
|
| 1975 |
loop_counter = 0
|
| 1976 |
while True:
|
| 1977 |
try:
|
| 1978 |
if not db_pool:
|
| 1979 |
-
|
| 1980 |
continue
|
| 1981 |
|
| 1982 |
-
|
| 1983 |
-
projects = loop.run_until_complete(get_active_projects())
|
| 1984 |
-
loop.close()
|
| 1985 |
|
| 1986 |
if not projects:
|
| 1987 |
-
|
| 1988 |
continue
|
| 1989 |
|
| 1990 |
loop_counter += 1
|
|
@@ -2001,16 +1999,12 @@ def run_eternity_loop():
|
|
| 2001 |
|
| 2002 |
now = datetime.now(timezone.utc)
|
| 2003 |
if current_mode == "build" and now >= deadline:
|
| 2004 |
-
|
| 2005 |
-
loop.run_until_complete(update_db_mode(name, "eternity"))
|
| 2006 |
-
loop.close()
|
| 2007 |
current_mode = "eternity"
|
| 2008 |
log_activity(f"[Eternity Loop] Project '{name}' deadline reached. Transitioned to Eternity R&D Mode.")
|
| 2009 |
|
| 2010 |
if current_mode == "build":
|
| 2011 |
-
|
| 2012 |
-
loop.run_until_complete(execute_build_cycle(name, goal))
|
| 2013 |
-
loop.close()
|
| 2014 |
else:
|
| 2015 |
# Enforce the sleep interval for Eternity Mode independently per project
|
| 2016 |
last_run = last_run_times.get(name, 0.0)
|
|
@@ -2020,16 +2014,14 @@ def run_eternity_loop():
|
|
| 2020 |
continue
|
| 2021 |
last_run_times[name] = now_ts
|
| 2022 |
|
| 2023 |
-
|
| 2024 |
-
loop.run_until_complete(execute_eternity_cycle(name, goal))
|
| 2025 |
-
loop.close()
|
| 2026 |
|
| 2027 |
# Base check interval (5 minutes)
|
| 2028 |
-
|
| 2029 |
|
| 2030 |
except Exception as e:
|
| 2031 |
log_activity(f"[Eternity Loop Error] Loop crash: {e}")
|
| 2032 |
-
|
| 2033 |
|
| 2034 |
|
| 2035 |
@app.get("/", response_class=HTMLResponse)
|
|
@@ -2274,8 +2266,8 @@ async def startup_event():
|
|
| 2274 |
threading.Thread(target=run_watchdog, daemon=True).start()
|
| 2275 |
# Start the local backup loop thread
|
| 2276 |
threading.Thread(target=run_backup_loop, daemon=True).start()
|
| 2277 |
-
# Start the eternity R&D loop
|
| 2278 |
-
|
| 2279 |
# Start the db keep-alive loop on FastAPI event loop
|
| 2280 |
asyncio.create_task(db_heartbeat_loop())
|
| 2281 |
# Start the db nightly retention cleanup loop
|
|
|
|
| 1970 |
|
| 1971 |
last_run_times = {} # project_name -> timestamp
|
| 1972 |
|
| 1973 |
+
async def run_eternity_loop():
|
| 1974 |
+
log_activity("[Eternity Loop] Async task started on main loop.")
|
| 1975 |
loop_counter = 0
|
| 1976 |
while True:
|
| 1977 |
try:
|
| 1978 |
if not db_pool:
|
| 1979 |
+
await asyncio.sleep(10)
|
| 1980 |
continue
|
| 1981 |
|
| 1982 |
+
projects = await get_active_projects()
|
|
|
|
|
|
|
| 1983 |
|
| 1984 |
if not projects:
|
| 1985 |
+
await asyncio.sleep(30)
|
| 1986 |
continue
|
| 1987 |
|
| 1988 |
loop_counter += 1
|
|
|
|
| 1999 |
|
| 2000 |
now = datetime.now(timezone.utc)
|
| 2001 |
if current_mode == "build" and now >= deadline:
|
| 2002 |
+
await update_db_mode(name, "eternity")
|
|
|
|
|
|
|
| 2003 |
current_mode = "eternity"
|
| 2004 |
log_activity(f"[Eternity Loop] Project '{name}' deadline reached. Transitioned to Eternity R&D Mode.")
|
| 2005 |
|
| 2006 |
if current_mode == "build":
|
| 2007 |
+
await execute_build_cycle(name, goal)
|
|
|
|
|
|
|
| 2008 |
else:
|
| 2009 |
# Enforce the sleep interval for Eternity Mode independently per project
|
| 2010 |
last_run = last_run_times.get(name, 0.0)
|
|
|
|
| 2014 |
continue
|
| 2015 |
last_run_times[name] = now_ts
|
| 2016 |
|
| 2017 |
+
await execute_eternity_cycle(name, goal)
|
|
|
|
|
|
|
| 2018 |
|
| 2019 |
# Base check interval (5 minutes)
|
| 2020 |
+
await asyncio.sleep(300)
|
| 2021 |
|
| 2022 |
except Exception as e:
|
| 2023 |
log_activity(f"[Eternity Loop Error] Loop crash: {e}")
|
| 2024 |
+
await asyncio.sleep(60)
|
| 2025 |
|
| 2026 |
|
| 2027 |
@app.get("/", response_class=HTMLResponse)
|
|
|
|
| 2266 |
threading.Thread(target=run_watchdog, daemon=True).start()
|
| 2267 |
# Start the local backup loop thread
|
| 2268 |
threading.Thread(target=run_backup_loop, daemon=True).start()
|
| 2269 |
+
# Start the eternity R&D loop as an async task on the main event loop
|
| 2270 |
+
asyncio.create_task(run_eternity_loop())
|
| 2271 |
# Start the db keep-alive loop on FastAPI event loop
|
| 2272 |
asyncio.create_task(db_heartbeat_loop())
|
| 2273 |
# Start the db nightly retention cleanup loop
|