noqt commited on
Commit
2c8a268
·
1 Parent(s): 691f501

Allow bounded supervisor restart recovery in native gate

Browse files
Files changed (1) hide show
  1. scripts/run_native_matrix.py +10 -1
scripts/run_native_matrix.py CHANGED
@@ -79,6 +79,9 @@ def start_workload(operator: str, args: list[str]) -> dict[str, Any]:
79
  return {"result": "STARTED", "response_recovered": True}
80
 
81
 
 
 
 
82
  def main() -> int:
83
  if os.geteuid() != 0:
84
  raise SystemExit("native matrix must run as root")
@@ -165,7 +168,13 @@ def main() -> int:
165
  # not race a naturally completed 30-second sleep.
166
  start_workload(operator, ["start", "--name", name, "--max-pids", "8", "--", "/bin/sleep", "300"])
167
  run(["/usr/bin/systemctl", "kill", "--kill-who=main", "-s", "SIGKILL", "lumi-eggcracker.service"])
168
- state = wait_state(operator, name, "TERMINATED", timeout=12)
 
 
 
 
 
 
169
  receipt_files = sorted(Path("/var/lib/lumi-eggcracker/receipts").glob("*.json"), key=lambda path: path.stat().st_mtime_ns)
170
  receipt = json.loads(receipt_files[-1].read_text(encoding="utf-8"))
171
  if receipt["trigger"]["kind"] != "SUPERVISOR_RESTART_FAIL_CLOSED":
 
79
  return {"result": "STARTED", "response_recovered": True}
80
 
81
 
82
+ RESTART_RECOVERY_TIMEOUT_SECONDS = 25
83
+
84
+
85
  def main() -> int:
86
  if os.geteuid() != 0:
87
  raise SystemExit("native matrix must run as root")
 
168
  # not race a naturally completed 30-second sleep.
169
  start_workload(operator, ["start", "--name", name, "--max-pids", "8", "--", "/bin/sleep", "300"])
170
  run(["/usr/bin/systemctl", "kill", "--kill-who=main", "-s", "SIGKILL", "lumi-eggcracker.service"])
171
+ # The supervisor socket is intentionally absent while systemd
172
+ # restarts the service. Allow the full bounded recovery window,
173
+ # which remains below the independent watchdog's 30-second
174
+ # heartbeat fail-closed timeout.
175
+ state = wait_state(
176
+ operator, name, "TERMINATED", timeout=RESTART_RECOVERY_TIMEOUT_SECONDS
177
+ )
178
  receipt_files = sorted(Path("/var/lib/lumi-eggcracker/receipts").glob("*.json"), key=lambda path: path.stat().st_mtime_ns)
179
  receipt = json.loads(receipt_files[-1].read_text(encoding="utf-8"))
180
  if receipt["trigger"]["kind"] != "SUPERVISOR_RESTART_FAIL_CLOSED":