Spaces:
Running
Running
Allow bounded supervisor restart recovery in native gate
Browse files- scripts/run_native_matrix.py +10 -1
scripts/run_native_matrix.py
CHANGED
|
@@ -79,6 +79,9 @@ def start_workload(operator: str, args: list[str]) -> dict[str, Any]:
|
|
| 79 |
return {"result": "STARTED", "response_recovered": True}
|
| 80 |
|
| 81 |
|
|
|
|
|
|
|
|
|
|
| 82 |
def main() -> int:
|
| 83 |
if os.geteuid() != 0:
|
| 84 |
raise SystemExit("native matrix must run as root")
|
|
@@ -165,7 +168,13 @@ def main() -> int:
|
|
| 165 |
# not race a naturally completed 30-second sleep.
|
| 166 |
start_workload(operator, ["start", "--name", name, "--max-pids", "8", "--", "/bin/sleep", "300"])
|
| 167 |
run(["/usr/bin/systemctl", "kill", "--kill-who=main", "-s", "SIGKILL", "lumi-eggcracker.service"])
|
| 168 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 169 |
receipt_files = sorted(Path("/var/lib/lumi-eggcracker/receipts").glob("*.json"), key=lambda path: path.stat().st_mtime_ns)
|
| 170 |
receipt = json.loads(receipt_files[-1].read_text(encoding="utf-8"))
|
| 171 |
if receipt["trigger"]["kind"] != "SUPERVISOR_RESTART_FAIL_CLOSED":
|
|
|
|
| 79 |
return {"result": "STARTED", "response_recovered": True}
|
| 80 |
|
| 81 |
|
| 82 |
+
RESTART_RECOVERY_TIMEOUT_SECONDS = 25
|
| 83 |
+
|
| 84 |
+
|
| 85 |
def main() -> int:
|
| 86 |
if os.geteuid() != 0:
|
| 87 |
raise SystemExit("native matrix must run as root")
|
|
|
|
| 168 |
# not race a naturally completed 30-second sleep.
|
| 169 |
start_workload(operator, ["start", "--name", name, "--max-pids", "8", "--", "/bin/sleep", "300"])
|
| 170 |
run(["/usr/bin/systemctl", "kill", "--kill-who=main", "-s", "SIGKILL", "lumi-eggcracker.service"])
|
| 171 |
+
# The supervisor socket is intentionally absent while systemd
|
| 172 |
+
# restarts the service. Allow the full bounded recovery window,
|
| 173 |
+
# which remains below the independent watchdog's 30-second
|
| 174 |
+
# heartbeat fail-closed timeout.
|
| 175 |
+
state = wait_state(
|
| 176 |
+
operator, name, "TERMINATED", timeout=RESTART_RECOVERY_TIMEOUT_SECONDS
|
| 177 |
+
)
|
| 178 |
receipt_files = sorted(Path("/var/lib/lumi-eggcracker/receipts").glob("*.json"), key=lambda path: path.stat().st_mtime_ns)
|
| 179 |
receipt = json.loads(receipt_files[-1].read_text(encoding="utf-8"))
|
| 180 |
if receipt["trigger"]["kind"] != "SUPERVISOR_RESTART_FAIL_CLOSED":
|