Instructions to use Motif-Technologies/optimizer with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Kernels
How to use Motif-Technologies/optimizer with Kernels:
# !pip install kernels from kernels import get_kernel # a version (or an explicit revision) is required; see the "Files and versions" tab for the available ones kernel = get_kernel("Motif-Technologies/optimizer", version=1) - Notebooks
- Google Colab
- Kaggle
Commit ·
0b17afb
1
Parent(s): f9603cb
fix(cpu_offload): own D2H/H2D streams instead of c10 pool streams [skip-build]
Browse filesc10 hands out its 32 streams per priority round-robin, and turn_on_cpu_offload builds a new pool around every
checkpoint save, so each save took two more pool streams. When one came back as a stream another part of
training copies on (remat activation reloads, NCCL), the two copy queues became one: the 700B main run lost
~0.5 s a step from such a save until the next one moved it. The pools now share one non-blocking D2H stream
and one H2D stream per device, created with the driver API, for the whole process.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BiM99HZnmc1dRQt4dafowG
torch-ext/optimizer/cpu_offload.py
CHANGED
|
@@ -15,6 +15,7 @@ Individual tensor storages are freed after offload via
|
|
| 15 |
downstream caches remain valid.
|
| 16 |
"""
|
| 17 |
|
|
|
|
| 18 |
import logging
|
| 19 |
from collections import defaultdict
|
| 20 |
|
|
@@ -24,6 +25,37 @@ from torch.profiler import record_function
|
|
| 24 |
|
| 25 |
logger = logging.getLogger(__name__)
|
| 26 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
|
| 28 |
class CPUOffloadPool:
|
| 29 |
"""Pinned CPU memory pool for async optimizer state offloading.
|
|
@@ -63,15 +95,13 @@ class CPUOffloadPool:
|
|
| 63 |
|
| 64 |
def _ensure_stream(self):
|
| 65 |
if self._offload_stream is None:
|
| 66 |
-
self._offload_stream =
|
| 67 |
|
| 68 |
def _ensure_reload_stream(self):
|
| 69 |
if self._reload_stream is None:
|
| 70 |
least_priority, _ = torch.cuda.Stream.priority_range()
|
| 71 |
-
self._reload_stream =
|
| 72 |
-
|
| 73 |
-
priority=least_priority,
|
| 74 |
-
)
|
| 75 |
logger.info(
|
| 76 |
"[CPUOffload] reload stream created with priority=%d "
|
| 77 |
"(range: %d..%d)",
|
|
|
|
| 15 |
downstream caches remain valid.
|
| 16 |
"""
|
| 17 |
|
| 18 |
+
import ctypes
|
| 19 |
import logging
|
| 20 |
from collections import defaultdict
|
| 21 |
|
|
|
|
| 25 |
|
| 26 |
logger = logging.getLogger(__name__)
|
| 27 |
|
| 28 |
+
# c10 hands out its 32 streams per priority round-robin (torch.cuda.Stream()
|
| 29 |
+
# and C++ getStreamFromPool alike), and turn_on_cpu_offload builds a new pool
|
| 30 |
+
# around every checkpoint save: two more pool streams each time, which sooner
|
| 31 |
+
# or later are streams the training loop or an NCCL communicator already
|
| 32 |
+
# copies on, and the two queues become one (a 700B run lost ~0.5 s a step
|
| 33 |
+
# from such a save on). The pools' D2H and H2D streams are created outside
|
| 34 |
+
# c10's pool instead, one of each per device for the whole process.
|
| 35 |
+
_OWN_STREAMS: dict[tuple[int, str], torch.cuda.ExternalStream] = {}
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _own_stream(device: torch.device | None, kind: str,
|
| 39 |
+
priority: int) -> torch.cuda.ExternalStream:
|
| 40 |
+
index = None if device is None else device.index
|
| 41 |
+
if index is None:
|
| 42 |
+
index = torch.cuda.current_device()
|
| 43 |
+
if (index, kind) not in _OWN_STREAMS:
|
| 44 |
+
cuda = ctypes.CDLL("libcuda.so.1")
|
| 45 |
+
handle = ctypes.c_void_p()
|
| 46 |
+
with torch.cuda.device(index):
|
| 47 |
+
torch.cuda.synchronize() # the device's primary context is current
|
| 48 |
+
err = cuda.cuStreamCreateWithPriority(
|
| 49 |
+
ctypes.byref(handle),
|
| 50 |
+
ctypes.c_uint(1), # CU_STREAM_NON_BLOCKING
|
| 51 |
+
ctypes.c_int(priority))
|
| 52 |
+
if err != 0:
|
| 53 |
+
raise RuntimeError(
|
| 54 |
+
f"cuStreamCreateWithPriority failed with CUresult {err}")
|
| 55 |
+
_OWN_STREAMS[index, kind] = torch.cuda.ExternalStream(handle.value,
|
| 56 |
+
device=index)
|
| 57 |
+
return _OWN_STREAMS[index, kind]
|
| 58 |
+
|
| 59 |
|
| 60 |
class CPUOffloadPool:
|
| 61 |
"""Pinned CPU memory pool for async optimizer state offloading.
|
|
|
|
| 95 |
|
| 96 |
def _ensure_stream(self):
|
| 97 |
if self._offload_stream is None:
|
| 98 |
+
self._offload_stream = _own_stream(self._device, "offload", 0)
|
| 99 |
|
| 100 |
def _ensure_reload_stream(self):
|
| 101 |
if self._reload_stream is None:
|
| 102 |
least_priority, _ = torch.cuda.Stream.priority_range()
|
| 103 |
+
self._reload_stream = _own_stream(self._device, "reload",
|
| 104 |
+
least_priority)
|
|
|
|
|
|
|
| 105 |
logger.info(
|
| 106 |
"[CPUOffload] reload stream created with priority=%d "
|
| 107 |
"(range: %d..%d)",
|