File size: 2,904 Bytes
f5d6a7c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
import json
import os
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch

import gpu2_supervisor as guard


class GuardSafetyTests(unittest.TestCase):
    def test_compute_filter_is_scoped_to_selected_gpu(self):
        rows = "GPU-other, 88\nGPU-selected, 91\nGPU-other, 99\nGPU-selected, N/A\n"
        self.assertEqual(guard.parse_compute_pids(rows, "GPU-selected"), {91})

    def test_current_process_identity_matches_os(self):
        process = guard.identity(os.getpid())
        self.assertEqual((process.parent, process.session, process.uid), (os.getppid(), os.getsid(0), os.getuid()))

    def test_job_descendant_in_separate_session_is_protected(self):
        job = guard.ProcessIdentity(800, 1, 800, 100, 1009)
        worker = guard.ProcessIdentity(900, 800, 900, 101, 1009)
        with patch.object(guard, "identity", side_effect=lambda pid: {800: job}.get(pid)):
            self.assertTrue(guard.protected(worker, job))

    def test_same_user_unrelated_process_is_not_protected(self):
        job = guard.ProcessIdentity(800, 1, 800, 100, 1009)
        other = guard.ProcessIdentity(901, 1, 901, 101, 1009)
        self.assertFalse(guard.protected(other, job))

    def test_reused_pid_is_not_signaled(self):
        original = guard.identity(os.getpid())
        reused = guard.ProcessIdentity(original.pid, original.parent, original.session, original.start + 1, original.uid)
        with patch.object(guard, "identity", return_value=reused), patch.object(guard.signal, "pidfd_send_signal") as send:
            self.assertFalse(guard.signal_checked(original, guard.signal.SIGTERM))
            send.assert_not_called()

    def test_permission_denied_is_reported_as_blocking(self):
        other = guard.ProcessIdentity(901, 1, 901, 101, 0)
        with patch.object(guard, "compute_pids", return_value={901}), patch.object(guard, "identity", return_value=other), \
             patch.object(guard, "signal_checked", side_effect=PermissionError), patch.object(guard, "log"):
            self.assertEqual(guard.GPUEnforcer("GPU-selected").poll(), ([901], [901]))

    def test_partial_checkpoint_is_not_resumed(self):
        with tempfile.TemporaryDirectory() as temp:
            root = Path(temp)
            good = root / "checkpoint-10"
            partial = root / "checkpoint-20"
            for checkpoint, step in [(good, 10), (partial, 20)]:
                checkpoint.mkdir()
                for name in ["optimizer.pt", "scheduler.pt", "rng_state.pth", "adapter_model.safetensors", "adapter_config.json"]:
                    (checkpoint / name).touch()
                (checkpoint / "trainer_state.json").write_text(json.dumps({"global_step": step}))
            (good / "recovery_complete.json").write_text("{}")
            self.assertEqual(guard.completed_checkpoint(root), good)


if __name__ == "__main__":
    unittest.main()