Kernels
TaehyunKimMotif Claude Opus 5.5 commited on
Commit
0b17afb
·
1 Parent(s): f9603cb

fix(cpu_offload): own D2H/H2D streams instead of c10 pool streams [skip-build]

Browse files

c10 hands out its 32 streams per priority round-robin, and turn_on_cpu_offload builds a new pool around every
checkpoint save, so each save took two more pool streams. When one came back as a stream another part of
training copies on (remat activation reloads, NCCL), the two copy queues became one: the 700B main run lost
~0.5 s a step from such a save until the next one moved it. The pools now share one non-blocking D2H stream
and one H2D stream per device, created with the driver API, for the whole process.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BiM99HZnmc1dRQt4dafowG

Files changed (1) hide show
  1. torch-ext/optimizer/cpu_offload.py +35 -5
torch-ext/optimizer/cpu_offload.py CHANGED
@@ -15,6 +15,7 @@ Individual tensor storages are freed after offload via
15
  downstream caches remain valid.
16
  """
17
 
 
18
  import logging
19
  from collections import defaultdict
20
 
@@ -24,6 +25,37 @@ from torch.profiler import record_function
24
 
25
  logger = logging.getLogger(__name__)
26
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
 
28
  class CPUOffloadPool:
29
  """Pinned CPU memory pool for async optimizer state offloading.
@@ -63,15 +95,13 @@ class CPUOffloadPool:
63
 
64
  def _ensure_stream(self):
65
  if self._offload_stream is None:
66
- self._offload_stream = torch.cuda.Stream(device=self._device)
67
 
68
  def _ensure_reload_stream(self):
69
  if self._reload_stream is None:
70
  least_priority, _ = torch.cuda.Stream.priority_range()
71
- self._reload_stream = torch.cuda.Stream(
72
- device=self._device,
73
- priority=least_priority,
74
- )
75
  logger.info(
76
  "[CPUOffload] reload stream created with priority=%d "
77
  "(range: %d..%d)",
 
15
  downstream caches remain valid.
16
  """
17
 
18
+ import ctypes
19
  import logging
20
  from collections import defaultdict
21
 
 
25
 
26
  logger = logging.getLogger(__name__)
27
 
28
+ # c10 hands out its 32 streams per priority round-robin (torch.cuda.Stream()
29
+ # and C++ getStreamFromPool alike), and turn_on_cpu_offload builds a new pool
30
+ # around every checkpoint save: two more pool streams each time, which sooner
31
+ # or later are streams the training loop or an NCCL communicator already
32
+ # copies on, and the two queues become one (a 700B run lost ~0.5 s a step
33
+ # from such a save on). The pools' D2H and H2D streams are created outside
34
+ # c10's pool instead, one of each per device for the whole process.
35
+ _OWN_STREAMS: dict[tuple[int, str], torch.cuda.ExternalStream] = {}
36
+
37
+
38
+ def _own_stream(device: torch.device | None, kind: str,
39
+ priority: int) -> torch.cuda.ExternalStream:
40
+ index = None if device is None else device.index
41
+ if index is None:
42
+ index = torch.cuda.current_device()
43
+ if (index, kind) not in _OWN_STREAMS:
44
+ cuda = ctypes.CDLL("libcuda.so.1")
45
+ handle = ctypes.c_void_p()
46
+ with torch.cuda.device(index):
47
+ torch.cuda.synchronize() # the device's primary context is current
48
+ err = cuda.cuStreamCreateWithPriority(
49
+ ctypes.byref(handle),
50
+ ctypes.c_uint(1), # CU_STREAM_NON_BLOCKING
51
+ ctypes.c_int(priority))
52
+ if err != 0:
53
+ raise RuntimeError(
54
+ f"cuStreamCreateWithPriority failed with CUresult {err}")
55
+ _OWN_STREAMS[index, kind] = torch.cuda.ExternalStream(handle.value,
56
+ device=index)
57
+ return _OWN_STREAMS[index, kind]
58
+
59
 
60
  class CPUOffloadPool:
61
  """Pinned CPU memory pool for async optimizer state offloading.
 
95
 
96
  def _ensure_stream(self):
97
  if self._offload_stream is None:
98
+ self._offload_stream = _own_stream(self._device, "offload", 0)
99
 
100
  def _ensure_reload_stream(self):
101
  if self._reload_stream is None:
102
  least_priority, _ = torch.cuda.Stream.priority_range()
103
+ self._reload_stream = _own_stream(self._device, "reload",
104
+ least_priority)
 
 
105
  logger.info(
106
  "[CPUOffload] reload stream created with priority=%d "
107
  "(range: %d..%d)",