temp / project /test_gpu2_supervisor.py
PriyaMishra's picture
Copy model from AjayRangoji/temp
f5d6a7c verified
Raw History Blame Contribute Delete
2.9 kB
import json
import os
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
import gpu2_supervisor as guard
class GuardSafetyTests(unittest.TestCase):
def test_compute_filter_is_scoped_to_selected_gpu(self):
rows = "GPU-other, 88\nGPU-selected, 91\nGPU-other, 99\nGPU-selected, N/A\n"
self.assertEqual(guard.parse_compute_pids(rows, "GPU-selected"), {91})
def test_current_process_identity_matches_os(self):
process = guard.identity(os.getpid())
self.assertEqual((process.parent, process.session, process.uid), (os.getppid(), os.getsid(0), os.getuid()))
def test_job_descendant_in_separate_session_is_protected(self):
job = guard.ProcessIdentity(800, 1, 800, 100, 1009)
worker = guard.ProcessIdentity(900, 800, 900, 101, 1009)
with patch.object(guard, "identity", side_effect=lambda pid: {800: job}.get(pid)):
self.assertTrue(guard.protected(worker, job))
def test_same_user_unrelated_process_is_not_protected(self):
job = guard.ProcessIdentity(800, 1, 800, 100, 1009)
other = guard.ProcessIdentity(901, 1, 901, 101, 1009)
self.assertFalse(guard.protected(other, job))
def test_reused_pid_is_not_signaled(self):
original = guard.identity(os.getpid())
reused = guard.ProcessIdentity(original.pid, original.parent, original.session, original.start + 1, original.uid)
with patch.object(guard, "identity", return_value=reused), patch.object(guard.signal, "pidfd_send_signal") as send:
self.assertFalse(guard.signal_checked(original, guard.signal.SIGTERM))
send.assert_not_called()
def test_permission_denied_is_reported_as_blocking(self):
other = guard.ProcessIdentity(901, 1, 901, 101, 0)
with patch.object(guard, "compute_pids", return_value={901}), patch.object(guard, "identity", return_value=other), \
patch.object(guard, "signal_checked", side_effect=PermissionError), patch.object(guard, "log"):
self.assertEqual(guard.GPUEnforcer("GPU-selected").poll(), ([901], [901]))
def test_partial_checkpoint_is_not_resumed(self):
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
good = root / "checkpoint-10"
partial = root / "checkpoint-20"
for checkpoint, step in [(good, 10), (partial, 20)]:
checkpoint.mkdir()
for name in ["optimizer.pt", "scheduler.pt", "rng_state.pth", "adapter_model.safetensors", "adapter_config.json"]:
(checkpoint / name).touch()
(checkpoint / "trainer_state.json").write_text(json.dumps({"global_step": step}))
(good / "recovery_complete.json").write_text("{}")
self.assertEqual(guard.completed_checkpoint(root), good)
if __name__ == "__main__":
unittest.main()