File size: 11,134 Bytes
524fd9d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
"""Owned single-layer graph capture and strict resource lifecycle."""
from __future__ import annotations

import threading
from typing import Any
import weakref


def tensor_key(value: Any, *, version: bool) -> tuple[Any, ...]:
    try:
        counter = value._version if version else None
    except RuntimeError:
        counter = None
    storage = value.untyped_storage()
    return (id(value), counter, value.data_ptr(), storage.data_ptr(), storage.nbytes(),
            value.storage_offset(), str(value.device), str(value.dtype),
            tuple(value.shape), tuple(value.stride()))


def callable_key(value: Any) -> tuple[int, int | None]:
    return (id(getattr(value, "__func__", value)),
            id(value.__self__) if hasattr(value, "__self__") else None)


def input_key(value: Any) -> tuple[Any, ...]:
    return str(value.device), str(value.dtype), tuple(value.shape), tuple(value.stride())


def owner_key(torch: Any, owner: Any, cache: Any, operation: Any) -> tuple[Any, ...]:
    modules = []
    for name, module in owner.named_modules():
        if module._forward_hooks or module._forward_pre_hooks:
            raise RuntimeError("Forward hooks invalidate decoder capture")
        scalars = tuple(sorted((key, value) for key, value in vars(module).items()
                               if not key.startswith("_") and isinstance(value, (str, bool, int, float))))
        callables = tuple(sorted((key, callable_key(value)) for key, value in vars(module).items()
                                  if not key.startswith("_") and callable(value)))
        modules.append((name, id(module), callable_key(module.forward), scalars, callables))
    registry = torch.nn.modules.module
    if registry._global_forward_hooks or registry._global_forward_pre_hooks:
        raise RuntimeError("Global forward hooks invalidate decoder capture")
    parameters = tuple((name, tensor_key(value, version=True))
                       for name, value in owner.named_parameters())
    if any(item[1][1] is None for item in parameters):
        raise RuntimeError("Versioned parameters required")
    buffers = tuple((name, tensor_key(value, version=True))
                    for name, value in owner.named_buffers())
    layer = cache.layers[0]
    if not (cache.has_previous_state(0) and layer.is_conv_states_initialized
            and layer.is_recurrent_states_initialized):
        raise RuntimeError("Already initialized live decode cache required")
    states = tuple(tensor_key(value, version=False)
                   for value in (layer.conv_states, layer.recurrent_states))
    return (id(owner), id(cache), id(layer), callable_key(operation), tuple(modules),
            parameters, buffers, states, callable_key(cache.update_recurrent_state),
            callable_key(cache.has_previous_state), callable_key(layer.update_recurrent_state))


def describe_pool(segments: list[dict[str, Any]], pool: tuple[int, int]) -> dict[str, Any]:
    blocks = [block for segment in segments for block in segment["blocks"]]
    pending = {"active_awaiting_free", "active_pending_free"}
    return {"pool": pool, "segments": segments,
            "foreign_pool": any(tuple(segment["segment_pool_id"]) != pool for segment in segments),
            "reserved_bytes": sum(segment["total_size"] for segment in segments),
            "allocated_bytes": sum(segment["allocated_size"] for segment in segments),
            "active_bytes": sum(segment["active_size"] for segment in segments),
            "active_allocated_blocks": sum(block["state"] == "active_allocated" for block in blocks),
            "pending_blocks": sum(block["state"] in pending for block in blocks),
            "unknown_block_states": sorted({block["state"] for block in blocks}
                                           - pending - {"active_allocated", "inactive"})}


class DecoderGraph:
    """One exact bucket/dtype graph with owned input and cloned FP32 outputs.

    Capture/replay must run on the engine's exclusive CUDA worker. Mutation is
    intentional only for the two captured cache tensors. Any owner, parameter,
    callable, stream, runtime, cache storage or hook change rejects the session.
    Capture/replay failures propagate; callers must close the engine rather than
    retrying eager work on a possibly invalid CUDA context.
    """

    def __init__(self, backend: Any, owner: Any, cache: Any, value: Any,
                 operation: Any, *, exclusive_process: bool) -> None:
        if not exclusive_process:
            raise ValueError("Exclusive process required")
        self.backend, self.owner, self.cache, self.operation = backend, owner, cache, operation
        self.thread, self.closed = threading.get_ident(), False
        self.context = backend.context(value)
        self.input = input_key(value)
        self.key = owner_key(backend.torch, owner, cache, operation)
        self.entry = backend.capture(value, operation, cache)
        self.check(value)

    def check(self, value: Any) -> None:
        if self.closed or self.thread != threading.get_ident():
            raise RuntimeError("Closed graph or foreign thread")
        if self.backend.context(value) != self.context or input_key(value) != self.input:
            raise RuntimeError("Input layout, dtype, stream or runtime changed")
        if owner_key(self.backend.torch, self.owner, self.cache, self.operation) != self.key:
            raise RuntimeError("Decoder owner, parameter, callable or cache changed")

    def run(self, value: Any) -> Any:
        self.check(value)
        return self.direct(value)

    def direct(self, value: Any) -> Any:
        # The padded manager executes complete guards before direct replay.
        return self.backend.replay_clone(self.entry, value)

    def close(self) -> dict[str, Any]:
        if self.closed or self.thread != threading.get_ident():
            raise RuntimeError("Closed graph or foreign thread")
        details = self.backend.release(self.entry)
        self.owner = self.cache = self.operation = self.key = None
        self.closed = True
        return details


class TorchBackend:
    def __init__(self, torch: Any, runtime_key: Any) -> None:
        self.torch, self.runtime_key = torch, runtime_key

    def context(self, value: Any) -> tuple[Any, ...]:
        torch = self.torch
        if (torch.is_grad_enabled() or torch.compiler.is_compiling() or not value.is_cuda
                or torch.cuda.is_current_stream_capturing() or not value.is_contiguous()
                or value.dtype not in (torch.bfloat16, torch.float32)
                or value.ndim != 3 or value.shape[1] != 1 or min(value.shape) < 1):
            raise RuntimeError("Unsupported decoder graph context/input")
        return (torch.cuda.current_stream(value.device).cuda_stream, self.runtime_key())

    def capture(self, value: Any, operation: Any, cache: Any) -> dict[str, Any]:
        torch = self.torch
        current = torch.cuda.current_stream(value.device)
        side = torch.cuda.Stream(device=value.device)
        side.wait_stream(current)
        layer = cache.layers[0]
        states = (layer.conv_states, layer.recurrent_states)
        saved = tuple(state.clone(memory_format=torch.preserve_format) for state in states)
        side.wait_stream(current)

        def restore(current_states: Any, before_states: Any) -> None:
            for state, before in zip(current_states, before_states, strict=True):
                state.copy_(before)

        graph = torch.cuda.CUDAGraph()
        with torch.cuda.stream(side):
            static_input = torch.empty_like(value)
            static_input.copy_(value)
            for _ in range(2):
                warm_output = operation(static_input)
            side.synchronize()
            del warm_output
            restore(states, saved)
            side.synchronize()
            graph.capture_begin(capture_error_mode="global")
            try:
                static_output = operation(static_input)
            finally:
                graph.capture_end()
            side.synchronize()
            restore(states, saved)
        current.wait_stream(side)
        current.synchronize()
        assert static_output.dtype == torch.float32
        static_input.record_stream(current)
        static_output.record_stream(current)
        for state, before in zip(states, saved, strict=True):
            assert torch.equal(state.contiguous().view(torch.int16), before.contiguous().view(torch.int16))
        del saved, states
        pool = tuple(graph.pool())
        details = describe_pool(torch.cuda.memory_snapshot(mempool_id=pool, include_traces=False), pool)
        assert not details["foreign_pool"] and details["reserved_bytes"] > 0
        return {"graph": graph, "static_input": static_input, "static_output": static_output,
                "stream": current, "side": side, "pool": pool,
                "pool_at_capture": details,
                "static_input_storage_bytes": static_input.untyped_storage().nbytes(),
                "capture_mutation_restored": True}

    def replay_clone(self, entry: dict[str, Any], value: Any) -> Any:
        if self.torch.cuda.current_stream(value.device).cuda_stream != entry["stream"].cuda_stream:
            raise RuntimeError("Replay stream changed")
        entry["static_input"].copy_(value)
        entry["graph"].replay()
        return entry["static_output"].clone()

    def release(self, entry: dict[str, Any]) -> dict[str, Any]:
        torch = self.torch
        refs = {key: weakref.ref(entry[key]) for key in ("static_input", "static_output", "graph")}
        events = []

        def snapshot(stage: str) -> None:
            item = describe_pool(torch.cuda.memory_snapshot(
                mempool_id=entry["pool"], include_traces=False), entry["pool"])
            item.update(stage=stage, weak_alive={key: ref() is not None for key, ref in refs.items()})
            events.append(item)

        entry["stream"].synchronize()
        entry["side"].synchronize()
        snapshot("before_drop")
        entry["graph"].reset()
        entry["static_input"] = entry["static_output"] = entry["graph"] = None
        snapshot("after_drop")
        entry["stream"].synchronize()
        entry["side"].synchronize()
        with torch.cuda.stream(entry["stream"]):
            probe = torch.empty((1,), device=entry["stream"].device, dtype=torch.uint8)
            del probe
        entry["stream"].synchronize()
        snapshot("after_fence_allocator_poll")
        final = events[-1]
        checks = {"no_graph_refs": not any(final["weak_alive"].values()),
                  "correct_pool": not final["foreign_pool"],
                  "pool_active_zero": final["active_bytes"] == 0,
                  "pool_allocated_zero": final["allocated_bytes"] == 0,
                  "no_active_blocks": final["active_allocated_blocks"] == 0,
                  "no_pending_blocks": final["pending_blocks"] == 0,
                  "known_block_states": not final["unknown_block_states"]}
        return {"passed": all(checks.values()), "checks": checks, "events": events}