Fr commited on
Commit
4d7b6bd
·
verified ·
1 Parent(s): 6ebd4e2

Create modeling_llada.py

Browse files
Files changed (1) hide show
  1. modeling_llada.py +1485 -0
modeling_llada.py ADDED
@@ -0,0 +1,1485 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import logging
4
+ import math
5
+ import sys
6
+ from abc import abstractmethod
7
+ from functools import partial
8
+ from typing import (
9
+ Callable,
10
+ Iterable,
11
+ List,
12
+ NamedTuple,
13
+ Optional,
14
+ Sequence,
15
+ Tuple,
16
+ cast,
17
+ )
18
+ from dataclasses import fields
19
+ from typing import List, Optional, Tuple, Union
20
+
21
+ import torch
22
+ import torch.backends.cuda
23
+ import torch.nn as nn
24
+ import torch.nn.functional as F
25
+ from torch import einsum
26
+ from transformers.modeling_utils import PreTrainedModel
27
+ from transformers.modeling_outputs import CausalLMOutputWithPast
28
+ from transformers.models.auto import AutoModel
29
+ from transformers.cache_utils import Cache
30
+
31
+ from configs_llada import (
32
+ LLaDAConfig,
33
+ StrEnum,
34
+ InitFnType,
35
+ ActivationType,
36
+ BlockType,
37
+ LayerNormType,
38
+ ModelConfig,
39
+ ActivationCheckpointingStrategy,
40
+ )
41
+
42
+ if sys.version_info.minor > 8:
43
+ from collections.abc import MutableMapping
44
+ elif sys.version_info.minor == 8:
45
+ from typing import MutableMapping
46
+ else:
47
+ raise SystemExit("This script supports Python 3.8 or higher")
48
+
49
+ __all__ = [
50
+ "LayerNormBase",
51
+ "LayerNorm",
52
+ "RMSLayerNorm",
53
+ "GemmaRMSLayerNorm",
54
+ "RotaryEmbedding",
55
+ "Activation",
56
+ "GELU",
57
+ "ReLU",
58
+ "SwiGLU",
59
+ "LLaDABlock",
60
+ "LLaDASequentialBlock",
61
+ "LLaDAModel",
62
+ "LLaDAOutput",
63
+ "LLaDAGenerateOutput",
64
+ ]
65
+
66
+
67
+ log = logging.getLogger(__name__)
68
+
69
+
70
+ class ModuleType(StrEnum):
71
+ in_module = "in"
72
+ out_module = "out"
73
+ emb = "emb"
74
+ final_out = "final_out"
75
+
76
+
77
+ def init_weights(
78
+ config: ModelConfig,
79
+ module: Union[nn.Linear, nn.Embedding],
80
+ d: Optional[int] = None,
81
+ layer_id: Optional[int] = None,
82
+ std_factor: float = 1.0,
83
+ type_of_module: Optional[ModuleType] = None,
84
+ ) -> None:
85
+ """
86
+ Initialize weights of a linear or embedding module.
87
+ :param config: The model config.
88
+ :param module: The linear or embedding submodule to initialize.
89
+ :param d: The effective input dimensionality of the weights. This could be smaller than the actual dimensions
90
+ for fused layers.
91
+ :param layer_id: When set, the standard deviation for the "mitchell" method will be adjusted by
92
+ ``1 / sqrt(2 * (layer_id + 1))``.
93
+ """
94
+ d = d if d is not None else config.d_model
95
+ if config.init_fn == InitFnType.normal:
96
+ std = config.init_std * std_factor
97
+ if config.init_cutoff_factor is not None:
98
+ cutoff_value = config.init_cutoff_factor * std
99
+ nn.init.trunc_normal_(module.weight, mean=0.0, std=std, a=-cutoff_value, b=cutoff_value)
100
+ else:
101
+ nn.init.normal_(module.weight, mean=0.0, std=std)
102
+ elif config.init_fn == InitFnType.mitchell:
103
+ std = std_factor / math.sqrt(d)
104
+ if layer_id is not None:
105
+ std = std / math.sqrt(2 * (layer_id + 1))
106
+ nn.init.trunc_normal_(module.weight, mean=0.0, std=std, a=-3 * std, b=3 * std)
107
+ elif config.init_fn == InitFnType.kaiming_normal:
108
+ nn.init.kaiming_normal_(module.weight, nonlinearity="relu")
109
+ elif config.init_fn == InitFnType.fan_in:
110
+ std = std_factor / math.sqrt(d)
111
+ nn.init.normal_(module.weight, mean=0.0, std=std)
112
+ elif config.init_fn == InitFnType.full_megatron:
113
+ if type_of_module is None:
114
+ raise RuntimeError(f"When using the {InitFnType.full_megatron} init, every module must have a type.")
115
+
116
+ cutoff_factor = config.init_cutoff_factor
117
+ if cutoff_factor is None:
118
+ cutoff_factor = 3
119
+
120
+ if type_of_module == ModuleType.in_module:
121
+ # for att_proj (same as QKV), ff_proj
122
+ std = config.init_std
123
+ elif type_of_module == ModuleType.out_module:
124
+ # for attn_out, ff_out
125
+ std = config.init_std / math.sqrt(2.0 * config.n_layers)
126
+ elif type_of_module == ModuleType.emb:
127
+ # positional embeddings (wpe)
128
+ # token embeddings (wte)
129
+ std = config.init_std
130
+ elif type_of_module == ModuleType.final_out:
131
+ # final output (ff_out)
132
+ std = config.d_model**-0.5
133
+ else:
134
+ raise RuntimeError(f"Unknown module type '{type_of_module}'")
135
+ nn.init.trunc_normal_(
136
+ module.weight,
137
+ mean=0.0,
138
+ std=std,
139
+ a=-cutoff_factor * std,
140
+ b=cutoff_factor * std,
141
+ )
142
+ else:
143
+ raise NotImplementedError(config.init_fn)
144
+
145
+ if isinstance(module, nn.Linear):
146
+ if module.bias is not None:
147
+ nn.init.zeros_(module.bias)
148
+
149
+ if config.init_fn == InitFnType.normal and getattr(module, "_is_residual", False):
150
+ with torch.no_grad():
151
+ module.weight.div_(math.sqrt(2 * config.n_layers))
152
+
153
+
154
+ def ensure_finite_(x: torch.Tensor, check_neg_inf: bool = True, check_pos_inf: bool = False):
155
+ """
156
+ Modify ``x`` in place to replace ``float("-inf")`` with the minimum value of the dtype when ``check_neg_inf``
157
+ is ``True`` and to replace ``float("inf")`` with the maximum value of the dtype when ``check_pos_inf`` is ``True``.
158
+ """
159
+ if check_neg_inf:
160
+ x.masked_fill_(x == float("-inf"), torch.finfo(x.dtype).min)
161
+ if check_pos_inf:
162
+ x.masked_fill_(x == float("inf"), torch.finfo(x.dtype).max)
163
+
164
+
165
+ def activation_checkpoint_function(cfg: ModelConfig):
166
+ preserve_rng_state = (
167
+ (cfg.attention_dropout == 0.0) and (cfg.embedding_dropout == 0.0) and (cfg.residual_dropout == 0.0)
168
+ )
169
+ from torch.utils.checkpoint import checkpoint
170
+
171
+ return partial(
172
+ checkpoint,
173
+ preserve_rng_state=preserve_rng_state,
174
+ use_reentrant=False,
175
+ )
176
+
177
+
178
+ class BufferCache(dict, MutableMapping[str, torch.Tensor]):
179
+ """
180
+ Cache for attention biases and other things that would normally be stored as buffers.
181
+ We avoid using buffers because we've run into various issues doing so with FSDP.
182
+ In general it appears the way FSDP handles buffers is not well-defined.
183
+ It doesn't shard them but apparently it does synchronize them across processes, which we want to avoid
184
+ since (A) it isn't necessary, and (B) we sometimes have `-inf` in these biases which might get turned into
185
+ NaNs when they're synchronized due to casting or some other issue.
186
+ """
187
+
188
+
189
+ def _non_meta_init_device(config: ModelConfig) -> torch.device:
190
+ if config.init_device is not None and config.init_device != "meta":
191
+ return torch.device(config.init_device)
192
+ else:
193
+ return torch.device("cuda" if torch.cuda.is_available() else "cpu")
194
+
195
+
196
+ class Dropout(nn.Dropout):
197
+ def forward(self, input: torch.Tensor) -> torch.Tensor:
198
+ if self.p == 0.0:
199
+ return input
200
+ else:
201
+ return F.dropout(input, self.p, self.training, self.inplace)
202
+
203
+
204
+ class LayerNormBase(nn.Module):
205
+ def __init__(
206
+ self,
207
+ config: ModelConfig,
208
+ *,
209
+ size: Optional[int] = None,
210
+ elementwise_affine: Optional[bool] = True,
211
+ eps: float = 1e-05,
212
+ ):
213
+ super().__init__()
214
+ self.config = config
215
+ self.eps = eps
216
+ self.normalized_shape = (size or config.d_model,)
217
+ if elementwise_affine or (elementwise_affine is None and self.config.layer_norm_with_affine):
218
+ self.weight = nn.Parameter(torch.ones(self.normalized_shape, device=config.init_device))
219
+ use_bias = self.config.bias_for_layer_norm
220
+ if use_bias is None:
221
+ use_bias = self.config.include_bias
222
+ if use_bias:
223
+ self.bias = nn.Parameter(torch.zeros(self.normalized_shape, device=config.init_device))
224
+ else:
225
+ self.register_parameter("bias", None)
226
+ else:
227
+ self.register_parameter("bias", None)
228
+ self.register_parameter("weight", None)
229
+
230
+ @abstractmethod
231
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
232
+ raise NotImplementedError
233
+
234
+ @classmethod
235
+ def build(cls, config: ModelConfig, size: Optional[int] = None, **kwargs) -> LayerNormBase:
236
+ if config.layer_norm_type == LayerNormType.default:
237
+ return LayerNorm(config, size=size, low_precision=False, **kwargs)
238
+ elif config.layer_norm_type == LayerNormType.low_precision:
239
+ return LayerNorm(config, size=size, low_precision=True, **kwargs)
240
+ elif config.layer_norm_type == LayerNormType.rms:
241
+ return RMSLayerNorm(config, size=size, **kwargs)
242
+ elif config.layer_norm_type == LayerNormType.gemma_rms:
243
+ return GemmaRMSLayerNorm(config, size=size, **kwargs)
244
+ else:
245
+ raise NotImplementedError(f"Unknown LayerNorm type: '{config.layer_norm_type}'")
246
+
247
+ def _cast_if_autocast_enabled(self, tensor: torch.Tensor, dtype: Optional[torch.dtype] = None) -> torch.Tensor:
248
+ # NOTE: `is_autocast_enabled()` only checks for CUDA autocast, so we use the separate function
249
+ # `is_autocast_cpu_enabled()` for CPU autocast.
250
+ # See https://github.com/pytorch/pytorch/issues/110966.
251
+ if tensor.device.type == "cuda" and torch.is_autocast_enabled():
252
+ return tensor.to(dtype=dtype if dtype is not None else torch.get_autocast_gpu_dtype())
253
+ elif tensor.device.type == "cpu" and torch.is_autocast_cpu_enabled():
254
+ return tensor.to(dtype=dtype if dtype is not None else torch.get_autocast_cpu_dtype())
255
+ else:
256
+ return tensor
257
+
258
+ def reset_parameters(self):
259
+ if self.weight is not None:
260
+ torch.nn.init.ones_(self.weight) # type: ignore
261
+ if self.bias is not None:
262
+ torch.nn.init.zeros_(self.bias) # type: ignore
263
+
264
+
265
+ class LayerNorm(LayerNormBase):
266
+ """
267
+ The default :class:`LayerNorm` implementation which can optionally run in low precision.
268
+ """
269
+
270
+ def __init__(
271
+ self,
272
+ config: ModelConfig,
273
+ size: Optional[int] = None,
274
+ low_precision: bool = False,
275
+ elementwise_affine: Optional[bool] = None,
276
+ eps: float = 1e-05,
277
+ ):
278
+ super().__init__(config, size=size, elementwise_affine=elementwise_affine, eps=eps)
279
+ self.low_precision = low_precision
280
+
281
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
282
+ if self.low_precision:
283
+ module_device = x.device
284
+ downcast_x = self._cast_if_autocast_enabled(x)
285
+ downcast_weight = (
286
+ self._cast_if_autocast_enabled(self.weight) if self.weight is not None else self.weight
287
+ )
288
+ downcast_bias = self._cast_if_autocast_enabled(self.bias) if self.bias is not None else self.bias
289
+ with torch.autocast(enabled=False, device_type=module_device.type):
290
+ return F.layer_norm(
291
+ downcast_x, self.normalized_shape, weight=downcast_weight, bias=downcast_bias, eps=self.eps
292
+ )
293
+ else:
294
+ return F.layer_norm(x, self.normalized_shape, weight=self.weight, bias=self.bias, eps=self.eps)
295
+
296
+
297
+ class RMSLayerNorm(LayerNormBase):
298
+ """
299
+ RMS layer norm, a simplified :class:`LayerNorm` implementation
300
+ """
301
+
302
+ def __init__(
303
+ self,
304
+ config: ModelConfig,
305
+ size: Optional[int] = None,
306
+ elementwise_affine: Optional[bool] = None,
307
+ eps: float = 1e-5,
308
+ ):
309
+ super().__init__(config, size=size, elementwise_affine=elementwise_affine, eps=config.rms_norm_eps)
310
+
311
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
312
+ with torch.autocast(enabled=False, device_type=x.device.type):
313
+ og_dtype = x.dtype
314
+ x = x.to(torch.float32)
315
+ variance = x.pow(2).mean(-1, keepdim=True)
316
+ x = x * torch.rsqrt(variance + self.eps)
317
+ x = x.to(og_dtype)
318
+
319
+ if self.weight is not None:
320
+ if self.bias is not None:
321
+ return self.weight * x + self.bias
322
+ else:
323
+ return self.weight * x
324
+ else:
325
+ return x
326
+
327
+
328
+ class GemmaRMSLayerNorm(LayerNormBase):
329
+ """
330
+ Gemma RMS layer norm, a simplified :class:`LayerNorm` implementation
331
+ """
332
+
333
+ def __init__(
334
+ self,
335
+ config: ModelConfig,
336
+ size: Optional[int] = None,
337
+ elementwise_affine: Optional[bool] = None,
338
+ eps: float = 1e-5,
339
+ ):
340
+ super().__init__(config, size=size, elementwise_affine=elementwise_affine, eps=config.rms_norm_eps)
341
+
342
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
343
+ with torch.autocast(enabled=False, device_type=x.device.type):
344
+ og_dtype = x.dtype
345
+ x = x.to(torch.float32)
346
+ variance = x.pow(2).mean(-1, keepdim=True)
347
+ x = x * torch.rsqrt(variance + self.eps)
348
+ x = x.to(og_dtype)
349
+
350
+ if self.weight is not None:
351
+ if self.bias is not None:
352
+ return x * (1 + self.weight) + self.bias
353
+ else:
354
+ return x * (1 + self.weight)
355
+ else:
356
+ return x
357
+
358
+
359
+ class RotaryEmbedding(nn.Module):
360
+ """
361
+ [Rotary positional embeddings (RoPE)](https://arxiv.org/abs/2104.09864).
362
+ """
363
+
364
+ def __init__(self, config: ModelConfig, cache: BufferCache):
365
+ super().__init__()
366
+ self.config = config
367
+ self.__cache = cache
368
+ # Warm up cache.
369
+ self.rope_theta = config.rope_theta
370
+ self.get_rotary_embedding(config.max_sequence_length, _non_meta_init_device(config))
371
+
372
+ def get_rotary_embedding(self, seq_len: int, device: torch.device) -> Tuple[torch.Tensor, torch.Tensor]:
373
+ if (
374
+ (pos_sin := self.__cache.get("rope_pos_sin")) is not None
375
+ and (pos_cos := self.__cache.get("rope_pos_cos")) is not None
376
+ and pos_sin.shape[-2] >= seq_len
377
+ and pos_cos.shape[-2] >= seq_len
378
+ ):
379
+ if pos_sin.device != device:
380
+ pos_sin = pos_sin.to(device)
381
+ self.__cache["rope_pos_sin"] = pos_sin
382
+ if pos_cos.device != device:
383
+ pos_cos = pos_cos.to(device)
384
+ self.__cache["rope_pos_cos"] = pos_cos
385
+ return pos_sin[:, :, :seq_len, :], pos_cos[:, :, :seq_len, :]
386
+
387
+ with torch.autocast(device.type, enabled=False):
388
+ dim = self.config.d_model // self.config.n_heads
389
+ inv_freq = 1.0 / (self.rope_theta ** (torch.arange(0, dim, 2, device=device, dtype=torch.float) / dim))
390
+ seq = torch.arange(seq_len, device=device, dtype=torch.float)
391
+ freqs = einsum("i , j -> i j", seq, inv_freq)
392
+ positions = torch.cat((freqs, freqs), dim=-1)
393
+ pos_sin, pos_cos = positions.sin()[None, None, :, :], positions.cos()[None, None, :, :]
394
+ self.__cache["rope_pos_sin"] = pos_sin
395
+ self.__cache["rope_pos_cos"] = pos_cos
396
+ return pos_sin, pos_cos
397
+
398
+ def rotate_half(self, x: torch.Tensor) -> torch.Tensor:
399
+ B, nh, T, hs = x.size()
400
+ x = x.view(B, nh, T, 2, hs // 2)
401
+ x1, x2 = x.unbind(dim=-2)
402
+ return torch.cat((-x2, x1), dim=-1)
403
+
404
+ def apply_rotary_pos_emb(self, pos_sin: torch.Tensor, pos_cos: torch.Tensor, t: torch.Tensor) -> torch.Tensor:
405
+ return ((t * pos_cos) + (self.rotate_half(t) * pos_sin)).to(t.dtype)
406
+
407
+ def forward(self, q: torch.Tensor, k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
408
+ if self.config.rope_full_precision:
409
+ q_, k_ = q.float(), k.float()
410
+ else:
411
+ q_, k_ = q, k
412
+
413
+ with torch.autocast(q.device.type, enabled=False):
414
+ query_len, key_len = q_.shape[-2], k_.shape[-2] # could be different if layer_past not None
415
+ pos_sin, pos_cos = self.get_rotary_embedding(key_len, q_.device)
416
+ pos_sin = pos_sin.type_as(q_)
417
+ pos_cos = pos_cos.type_as(q_)
418
+ q_ = self.apply_rotary_pos_emb(
419
+ pos_sin[:, :, key_len - query_len : key_len, :],
420
+ pos_cos[:, :, key_len - query_len : key_len, :],
421
+ q_,
422
+ )
423
+ k_ = self.apply_rotary_pos_emb(pos_sin, pos_cos, k_)
424
+ return q_.type_as(q), k_.type_as(k)
425
+
426
+
427
+ class Activation(nn.Module):
428
+ def __init__(self, config: ModelConfig):
429
+ super().__init__()
430
+ self.config = config
431
+
432
+ @abstractmethod
433
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
434
+ raise NotImplementedError
435
+
436
+ @property
437
+ @abstractmethod
438
+ def output_multiplier(self) -> float:
439
+ raise NotImplementedError
440
+
441
+ @classmethod
442
+ def build(cls, config: ModelConfig) -> Activation:
443
+ if config.activation_type == ActivationType.gelu:
444
+ return cast(Activation, GELU(approximate="none"))
445
+ elif config.activation_type == ActivationType.relu:
446
+ return cast(Activation, ReLU(inplace=False))
447
+ elif config.activation_type == ActivationType.silu:
448
+ return cast(Activation, SiLU(inplace=False))
449
+ elif config.activation_type == ActivationType.swiglu:
450
+ return SwiGLU(config)
451
+ else:
452
+ raise NotImplementedError(f"Unknown activation: '{config.activation_type}'")
453
+
454
+
455
+ class GELU(nn.GELU):
456
+ @property
457
+ def output_multiplier(self) -> float:
458
+ return 1.0
459
+
460
+
461
+ class ReLU(nn.ReLU):
462
+ @property
463
+ def output_multiplier(self) -> float:
464
+ return 1.0
465
+
466
+ class SiLU(nn.SiLU):
467
+ @property
468
+ def output_multiplier(self) -> float:
469
+ return 1.0
470
+
471
+ class SwiGLU(Activation):
472
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
473
+ x, gate = x.chunk(2, dim=-1)
474
+ return F.silu(gate) * x
475
+
476
+ @property
477
+ def output_multiplier(self) -> float:
478
+ return 0.5
479
+
480
+
481
+ def causal_attention_bias(seq_len: int, device: torch.device) -> torch.FloatTensor:
482
+ att_bias = torch.triu(
483
+ torch.ones(seq_len, seq_len, device=device, dtype=torch.float),
484
+ diagonal=1,
485
+ )
486
+ att_bias.masked_fill_(att_bias == 1, torch.finfo(att_bias.dtype).min)
487
+ return att_bias.view(1, 1, seq_len, seq_len) # type: ignore
488
+
489
+
490
+ def get_causal_attention_bias(cache: BufferCache, seq_len: int, device: torch.device) -> torch.Tensor:
491
+ if (causal_bias := cache.get("causal_attention_bias")) is not None and causal_bias.shape[-1] >= seq_len:
492
+ if causal_bias.device != device:
493
+ causal_bias = causal_bias.to(device)
494
+ cache["causal_attention_bias"] = causal_bias
495
+ return causal_bias
496
+ with torch.autocast(device.type, enabled=False):
497
+ causal_bias = causal_attention_bias(seq_len, device)
498
+ cache["causal_attention_bias"] = causal_bias
499
+ return causal_bias
500
+
501
+
502
+ def alibi_attention_bias(seq_len: int, config: ModelConfig, device: torch.device) -> torch.FloatTensor:
503
+ alibi_bias = torch.arange(1 - seq_len, 1, dtype=torch.float, device=device).view(1, 1, 1, seq_len)
504
+
505
+ # shape: (1, 1, seq_len, seq_len)
506
+ alibi_bias = alibi_bias - torch.arange(1 - seq_len, 1, dtype=torch.float, device=device).view(1, 1, seq_len, 1)
507
+ alibi_bias.abs_().mul_(-1)
508
+
509
+ # shape: (n_heads,)
510
+ m = torch.arange(1, config.n_heads + 1, dtype=torch.float, device=device)
511
+ m.mul_(config.alibi_bias_max / config.n_heads)
512
+
513
+ # shape: (1, n_heads, seq_len, seq_len)
514
+ return alibi_bias * (1.0 / (2 ** m.view(1, config.n_heads, 1, 1))) # type: ignore
515
+
516
+
517
+ class LLaDABlock(nn.Module):
518
+ """
519
+ A base class for transformer block implementations.
520
+ """
521
+
522
+ def __init__(self, layer_id: int, config: ModelConfig, cache: BufferCache):
523
+ super().__init__()
524
+ self.layer_id = layer_id
525
+ self.config = config
526
+ self.hidden_size = (
527
+ config.mlp_hidden_size if config.mlp_hidden_size is not None else config.mlp_ratio * config.d_model
528
+ )
529
+ self.__cache = cache
530
+ assert config.d_model % config.n_heads == 0
531
+
532
+ self._activation_checkpoint_fn = None
533
+
534
+ # Dropout.
535
+ self.dropout = Dropout(config.residual_dropout)
536
+
537
+ # Layer norms.
538
+ self.k_norm: Optional[LayerNormBase] = None
539
+ self.q_norm: Optional[LayerNormBase] = None
540
+ if config.attention_layer_norm:
541
+ self.k_norm = LayerNormBase.build(
542
+ config,
543
+ size=(config.d_model // config.n_heads) * config.effective_n_kv_heads,
544
+ elementwise_affine=config.attention_layer_norm_with_affine,
545
+ )
546
+ self.q_norm = LayerNormBase.build(config, elementwise_affine=config.attention_layer_norm_with_affine)
547
+
548
+ # Activation function.
549
+ self.act = Activation.build(config)
550
+ assert (self.act.output_multiplier * self.hidden_size) % 1 == 0
551
+
552
+ # Attention output projection.
553
+ self.attn_out = nn.Linear(
554
+ config.d_model, config.d_model, bias=config.include_bias, device=config.init_device
555
+ )
556
+
557
+ # Feed-forward output projection.
558
+ self.ff_out = nn.Linear(
559
+ int(self.act.output_multiplier * self.hidden_size),
560
+ config.d_model,
561
+ bias=config.include_bias,
562
+ device=config.init_device,
563
+ )
564
+ self.ff_out._is_residual = True # type: ignore
565
+
566
+ # Rotary embeddings.
567
+ if self.config.rope:
568
+ self.rotary_emb = RotaryEmbedding(config, self.__cache)
569
+
570
+ self.flash_attn_func = None
571
+ if config.flash_attention:
572
+ try:
573
+ from flash_attn import flash_attn_func # type: ignore
574
+
575
+ self.flash_attn_func = flash_attn_func
576
+ except ModuleNotFoundError:
577
+ pass
578
+
579
+ def reset_parameters(self):
580
+ if self.k_norm is not None:
581
+ self.k_norm.reset_parameters()
582
+ if self.q_norm is not None:
583
+ self.q_norm.reset_parameters()
584
+ init_weights(
585
+ self.config,
586
+ self.attn_out,
587
+ d=self.config.d_model,
588
+ layer_id=self.layer_id,
589
+ type_of_module=ModuleType.out_module,
590
+ )
591
+ init_weights(
592
+ self.config,
593
+ self.ff_out,
594
+ d=self.ff_out.in_features,
595
+ layer_id=self.layer_id,
596
+ type_of_module=ModuleType.out_module,
597
+ )
598
+
599
+ def set_activation_checkpointing(self, strategy: Optional[ActivationCheckpointingStrategy]):
600
+ if strategy == ActivationCheckpointingStrategy.fine_grained:
601
+ self._activation_checkpoint_fn = activation_checkpoint_function(self.config)
602
+ else:
603
+ self._activation_checkpoint_fn = None
604
+
605
+ @classmethod
606
+ def _cast_attn_bias(cls, bias: torch.Tensor, input_dtype: torch.dtype) -> torch.Tensor:
607
+ target_dtype = input_dtype
608
+ # NOTE: `is_autocast_enabled()` only checks for CUDA autocast, so we use the separate function
609
+ # `is_autocast_cpu_enabled()` for CPU autocast.
610
+ # See https://github.com/pytorch/pytorch/issues/110966.
611
+ if bias.device.type == "cuda" and torch.is_autocast_enabled():
612
+ target_dtype = torch.get_autocast_gpu_dtype()
613
+ elif bias.device.type == "cpu" and torch.is_autocast_cpu_enabled():
614
+ target_dtype = torch.get_autocast_cpu_dtype()
615
+ if bias.dtype != target_dtype:
616
+ bias = bias.to(target_dtype)
617
+ ensure_finite_(bias, check_neg_inf=True, check_pos_inf=False)
618
+ return bias
619
+
620
+ def _scaled_dot_product_attention(
621
+ self,
622
+ q: torch.Tensor,
623
+ k: torch.Tensor,
624
+ v: torch.Tensor,
625
+ attn_mask: Optional[torch.Tensor] = None,
626
+ dropout_p: float = 0.0,
627
+ is_causal: bool = False,
628
+ ) -> torch.Tensor:
629
+ """
630
+ Computes scaled dot product attention on query, key and value tensors, using an optional
631
+ attention mask if passed, and applying dropout if a probability greater than 0.0 is specified.
632
+ """
633
+ if self.flash_attn_func is not None and attn_mask is None:
634
+ r = self.flash_attn_func(
635
+ q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), dropout_p=dropout_p, causal=False
636
+ )
637
+ return r.transpose(1, 2)
638
+ else:
639
+ # torch's sdpa doesn't support GQA, so we're doing this
640
+ assert k.size(1) == v.size(1)
641
+ num_kv_heads = k.size(1)
642
+ num_q_heads = q.size(1)
643
+ if num_q_heads != num_kv_heads:
644
+ assert num_q_heads % num_kv_heads == 0
645
+ k = k.repeat_interleave(num_q_heads // num_kv_heads, dim=1, output_size=num_q_heads)
646
+ v = v.repeat_interleave(num_q_heads // num_kv_heads, dim=1, output_size=num_q_heads)
647
+
648
+ # Modify: MDM set causal to False, and with no attn_mask.
649
+ return F.scaled_dot_product_attention(
650
+ q,
651
+ k,
652
+ v,
653
+ attn_mask=None,
654
+ dropout_p=dropout_p,
655
+ is_causal=False,
656
+ )
657
+
658
+ def attention(
659
+ self,
660
+ q: torch.Tensor,
661
+ k: torch.Tensor,
662
+ v: torch.Tensor,
663
+ attention_bias: Optional[torch.Tensor] = None,
664
+ layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
665
+ use_cache: bool = False,
666
+ ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
667
+ B, T, C = q.size() # batch size, sequence length, d_model
668
+ dtype = k.dtype
669
+
670
+ # Optionally apply layer norm to keys and queries.
671
+ if self.q_norm is not None and self.k_norm is not None:
672
+ q = self.q_norm(q).to(dtype=dtype)
673
+ k = self.k_norm(k).to(dtype=dtype)
674
+
675
+ # Move head forward to be next to the batch dim.
676
+ # shape: (B, nh, T, hs)
677
+ q = q.view(B, T, self.config.n_heads, C // self.config.n_heads).transpose(1, 2)
678
+ # shape: (B, n_kv_h, T, hs)
679
+ k = k.view(B, T, self.config.effective_n_kv_heads, C // self.config.n_heads).transpose(1, 2)
680
+ # shape: (B, n_kv_h, T, hs)
681
+ v = v.view(B, T, self.config.effective_n_kv_heads, C // self.config.n_heads).transpose(1, 2)
682
+
683
+ if layer_past is not None:
684
+ past_key, past_value = layer_past
685
+ k = torch.cat((past_key, k), dim=-2)
686
+ v = torch.cat((past_value, v), dim=-2)
687
+
688
+ present = (k, v) if use_cache else None
689
+ query_len, key_len = q.shape[-2], k.shape[-2] # could be different if layer_past not None
690
+
691
+ if self.config.rope:
692
+ # Apply rotary embeddings.
693
+ q, k = self.rotary_emb(q, k)
694
+
695
+ if attention_bias is not None:
696
+ # Resize and cast attention bias.
697
+ # The current dtype of the attention bias might not match the dtype that the SDP attn function will
698
+ # run in if AMP is enabled, and this can be a problem if some tokens are masked out due to padding
699
+ # as down-casting the attention bias to the autocast precision will result in -infs, which will
700
+ # cause the SDP attn function to produce NaNs.
701
+ attention_bias = self._cast_attn_bias(
702
+ attention_bias[:, :, key_len - query_len : key_len, :key_len], dtype
703
+ )
704
+
705
+ # Get the attention scores.
706
+ # shape: (B, nh, T, hs)
707
+ att = self._scaled_dot_product_attention(
708
+ q,
709
+ k,
710
+ v,
711
+ attn_mask=None,
712
+ dropout_p=0.0 if not self.training else self.config.attention_dropout,
713
+ is_causal=False,
714
+ )
715
+
716
+ # Re-assemble all head outputs side-by-side.
717
+ att = att.transpose(1, 2).contiguous().view(B, T, C)
718
+
719
+ # Apply output projection.
720
+ return self.attn_out(att), present
721
+
722
+ @abstractmethod
723
+ def forward(
724
+ self,
725
+ x: torch.Tensor,
726
+ attention_bias: Optional[torch.FloatTensor] = None,
727
+ layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
728
+ use_cache: bool = False,
729
+ ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
730
+ raise NotImplementedError
731
+
732
+ @classmethod
733
+ def build(cls, layer_id: int, config: ModelConfig, cache: BufferCache) -> LLaDABlock:
734
+ if config.block_type == BlockType.sequential:
735
+ return LLaDASequentialBlock(layer_id, config, cache)
736
+ elif config.block_type == BlockType.llama:
737
+ return LLaDALlamaBlock(layer_id, config, cache)
738
+ else:
739
+ raise NotImplementedError(f"Unknown block type: '{config.block_type}'")
740
+
741
+
742
+ class LLaDASequentialBlock(LLaDABlock):
743
+ """
744
+ This is a typical transformer block where the output is computed as ``MLP(LN(x + Attention(LN(x))))``
745
+ (plus another skip connection).
746
+ """
747
+
748
+ def __init__(self, layer_id: int, config: ModelConfig, cache: BufferCache):
749
+ super().__init__(layer_id, config, cache)
750
+ # Layer norms.
751
+ self.attn_norm = LayerNorm.build(config)
752
+ self.ff_norm = LayerNorm.build(config)
753
+ # Attention input projection. Projects x -> (q, k, v)
754
+ head_dim = config.d_model // config.n_heads
755
+ self.fused_dims = (
756
+ config.d_model,
757
+ config.effective_n_kv_heads * head_dim,
758
+ config.effective_n_kv_heads * head_dim,
759
+ )
760
+ self.att_proj = nn.Linear(
761
+ config.d_model, sum(self.fused_dims), bias=config.include_bias | config.include_qkv_bias, device=config.init_device
762
+ )
763
+ # Feed-forward input projection.
764
+ self.ff_proj = nn.Linear(
765
+ config.d_model, self.hidden_size, bias=config.include_bias, device=config.init_device
766
+ )
767
+
768
+ def reset_parameters(self):
769
+ super().reset_parameters()
770
+ self.attn_norm.reset_parameters()
771
+ self.ff_norm.reset_parameters()
772
+ # NOTE: the standard deviation for these weights does not depend on the layer.
773
+ init_weights(
774
+ self.config, self.att_proj, d=self.config.d_model, layer_id=None, type_of_module=ModuleType.in_module
775
+ )
776
+ init_weights(
777
+ self.config, self.ff_proj, d=self.config.d_model, layer_id=None, type_of_module=ModuleType.in_module
778
+ )
779
+
780
+ def forward(
781
+ self,
782
+ x: torch.Tensor,
783
+ attention_bias: Optional[torch.Tensor] = None,
784
+ layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
785
+ use_cache: bool = False,
786
+ ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
787
+ # Get query, key, value projections.
788
+ # shape:
789
+ # - for regular attn q, k, v: (batch_size, seq_len, d_model)
790
+ # - for multi-query attn q: (batch_size, seq_len, d_model)
791
+ # k, v: (batch_size, seq_len, d_model // n_heads)
792
+ # - for group query attn q: (batch_size, seq_len, d_model)
793
+ # k, v: (batch_size, seq_len, d_model // n_kv_heads)
794
+ if self._activation_checkpoint_fn is not None:
795
+ q, k, v = self.att_proj(self._activation_checkpoint_fn(self.attn_norm, x)).split(
796
+ self.fused_dims, dim=-1
797
+ )
798
+ else:
799
+ q, k, v = self.att_proj(self.attn_norm(x)).split(self.fused_dims, dim=-1)
800
+
801
+ # Get attention scores.
802
+ if self._activation_checkpoint_fn is not None:
803
+ att, cache = self._activation_checkpoint_fn( # type: ignore
804
+ self.attention, q, k, v, attention_bias, layer_past=layer_past, use_cache=use_cache
805
+ )
806
+ else:
807
+ att, cache = self.attention(q, k, v, attention_bias, layer_past=layer_past, use_cache=use_cache)
808
+
809
+ # Add attention scores.
810
+ # shape: (B, T, C)
811
+ x = x + self.dropout(att)
812
+
813
+ # Add feed-forward projection.
814
+ # shape: (batch_size, seq_len, d_model)
815
+ og_x = x
816
+ if self._activation_checkpoint_fn is not None:
817
+ x = self._activation_checkpoint_fn(self.ff_norm, x) # type: ignore
818
+ else:
819
+ x = self.ff_norm(x)
820
+ x = self.ff_proj(x)
821
+ if self._activation_checkpoint_fn is not None:
822
+ x = self._activation_checkpoint_fn(self.act, x) # type: ignore
823
+ else:
824
+ x = self.act(x)
825
+ x = self.ff_out(x)
826
+ x = self.dropout(x)
827
+ x = og_x + x
828
+
829
+ return x, cache
830
+
831
+
832
+ class LLaDALlamaBlock(LLaDABlock):
833
+ """
834
+ This is a transformer block where the output is computed as ``MLP(LN(x + Attention(LN(x))))``
835
+ (plus another skip connection). This block is similar to `LLaDASequentialBlock`
836
+ but some operations have slightly different implementations to imitate the
837
+ behavior of Llama.
838
+ """
839
+
840
+ def __init__(self, layer_id: int, config: ModelConfig, cache: BufferCache):
841
+ super().__init__(layer_id, config, cache)
842
+ # Layer norms.
843
+ self.attn_norm = LayerNorm.build(config)
844
+ self.ff_norm = LayerNorm.build(config)
845
+ self.__cache = cache
846
+
847
+ # Attention input projection. Projects x -> (q, k, v)
848
+ head_dim = config.d_model // config.n_heads
849
+ q_proj_out_dim = config.d_model
850
+ k_proj_out_dim = config.effective_n_kv_heads * head_dim
851
+ v_proj_out_dim = config.effective_n_kv_heads * head_dim
852
+ self.q_proj = nn.Linear(
853
+ config.d_model, q_proj_out_dim, bias=config.include_bias | config.include_qkv_bias, device=config.init_device
854
+ )
855
+ self.k_proj = nn.Linear(
856
+ config.d_model, k_proj_out_dim, bias=config.include_bias | config.include_qkv_bias, device=config.init_device
857
+ )
858
+ self.v_proj = nn.Linear(
859
+ config.d_model, v_proj_out_dim, bias=config.include_bias | config.include_qkv_bias, device=config.init_device
860
+ )
861
+
862
+ # Feed-forward input projection.
863
+ self.ff_proj = nn.Linear(
864
+ config.d_model, self.hidden_size, bias=config.include_bias, device=config.init_device
865
+ )
866
+ # new add
867
+ self.up_proj = nn.Linear(
868
+ config.d_model, self.hidden_size, bias=config.include_bias, device=config.init_device
869
+ )
870
+
871
+ def reset_parameters(self):
872
+ super().reset_parameters()
873
+ self.attn_norm.reset_parameters()
874
+ self.ff_norm.reset_parameters()
875
+ # NOTE: the standard deviation for these weights does not depend on the layer.
876
+ init_weights(self.config, self.q_proj, d=self.config.d_model, layer_id=None)
877
+ init_weights(self.config, self.k_proj, d=self.config.d_model, layer_id=None)
878
+ init_weights(self.config, self.v_proj, d=self.config.d_model, layer_id=None)
879
+ init_weights(self.config, self.ff_proj, d=self.config.d_model, layer_id=None)
880
+ init_weights(self.config, self.up_proj, d=self.config.d_model, layer_id=None) # new add
881
+
882
+ def forward(
883
+ self,
884
+ x: torch.Tensor,
885
+ attention_bias: Optional[torch.Tensor] = None,
886
+ layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
887
+ use_cache: bool = False,
888
+ ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
889
+ # Get query, key, value projections.
890
+ # shape:
891
+ # - for regular attn q, k, v: (batch_size, seq_len, d_model)
892
+ # - for multi-query attn q: (batch_size, seq_len, d_model)
893
+ # k, v: (batch_size, seq_len, d_model // n_heads)
894
+ # - for group query attn q: (batch_size, seq_len, d_model)
895
+ # k, v: (batch_size, seq_len, d_model // n_kv_heads)
896
+ x_normed = self.attn_norm(x)
897
+ q = self.q_proj(x_normed)
898
+ k = self.k_proj(x_normed)
899
+ v = self.v_proj(x_normed)
900
+
901
+ # Get attention scores.
902
+ if self._activation_checkpoint_fn is not None:
903
+ att, cache = self._activation_checkpoint_fn( # type: ignore
904
+ self.attention, q, k, v, attention_bias, layer_past=layer_past, use_cache=use_cache
905
+ )
906
+ else:
907
+ att, cache = self.attention(q, k, v, attention_bias, layer_past=layer_past, use_cache=use_cache)
908
+
909
+ # Add attention scores.
910
+ # shape: (B, T, C)
911
+ x = x + self.dropout(att)
912
+
913
+ # Add feed-forward projection.
914
+ # shape: (batch_size, seq_len, d_model)
915
+ og_x = x
916
+ if self._activation_checkpoint_fn is not None:
917
+ x = self._activation_checkpoint_fn(self.ff_norm, x) # type: ignore
918
+ else:
919
+ x = self.ff_norm(x)
920
+ x, x_up = self.ff_proj(x), self.up_proj(x) # new add
921
+ if self._activation_checkpoint_fn is not None:
922
+ x = self._activation_checkpoint_fn(self.act, x) # type: ignore
923
+ else:
924
+ x = self.act(x)
925
+ #x = x * x_up # new add
926
+ x = self.ff_out(x)
927
+ x = self.dropout(x)
928
+ x = og_x + x
929
+
930
+ return x, cache
931
+
932
+
933
+ class LLaDAOutput(NamedTuple):
934
+ logits: torch.FloatTensor
935
+ """
936
+ A tensor of shape `(batch_size, seq_len, vocab_size)` representing the log probabilities
937
+ for the next token *before* normalization via (log) softmax.
938
+ """
939
+
940
+ attn_key_values: Optional[List[Tuple[torch.Tensor, torch.Tensor]]]
941
+ """
942
+ Attention keys and values from each block.
943
+ """
944
+
945
+ hidden_states: Optional[Tuple[torch.Tensor]]
946
+ """
947
+ Hidden states from each block.
948
+ """
949
+
950
+
951
+ class LLaDAGenerateOutput(NamedTuple):
952
+ token_ids: torch.LongTensor
953
+ """
954
+ The generated token IDs, a tensor of shape `(batch_size, beam_size, max_steps)`.
955
+ These do *not* include the original input IDs.
956
+ """
957
+
958
+ scores: torch.FloatTensor
959
+ """
960
+ The scores of the generated sequences, a tensor of shape `(batch_size, beam_size)`.
961
+ """
962
+
963
+
964
+ class LLaDABlockGroup(nn.ModuleList):
965
+ def __init__(self, config: ModelConfig, layer_offset: int, modules: Optional[Iterable[nn.Module]] = None):
966
+ super().__init__(modules)
967
+ self.config = config
968
+ self.layer_offset = layer_offset
969
+ self.activation_checkpointing_strategy: Optional[ActivationCheckpointingStrategy] = None
970
+ self._activation_checkpoint_fn = activation_checkpoint_function(self.config)
971
+
972
+ def forward(
973
+ self,
974
+ x: torch.Tensor,
975
+ attention_bias: Optional[torch.FloatTensor] = None,
976
+ layers_past: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = None,
977
+ use_cache: bool = False,
978
+ ) -> Tuple[torch.Tensor, Optional[List[Tuple[torch.Tensor, torch.Tensor]]]]:
979
+ attn_key_values: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = [] if use_cache else None
980
+ for block_idx, block in enumerate(self):
981
+ layer_past = None if layers_past is None else layers_past[block_idx]
982
+ block_idx += self.layer_offset
983
+ if (
984
+ (self.activation_checkpointing_strategy == ActivationCheckpointingStrategy.whole_layer)
985
+ or (
986
+ self.activation_checkpointing_strategy == ActivationCheckpointingStrategy.one_in_two
987
+ and block_idx % 2 == 0
988
+ )
989
+ or (
990
+ self.activation_checkpointing_strategy == ActivationCheckpointingStrategy.one_in_three
991
+ and block_idx % 3 == 0
992
+ )
993
+ or (
994
+ self.activation_checkpointing_strategy == ActivationCheckpointingStrategy.one_in_four
995
+ and block_idx % 4 == 0
996
+ )
997
+ ):
998
+ # shape: (batch_size, seq_len, d_model)
999
+ x, cache = self._activation_checkpoint_fn( # type: ignore
1000
+ block, x, attention_bias=attention_bias, layer_past=layer_past, use_cache=use_cache
1001
+ )
1002
+ else:
1003
+ # shape: (batch_size, seq_len, d_model)
1004
+ x, cache = block(x, attention_bias=attention_bias, layer_past=layer_past, use_cache=use_cache)
1005
+ if attn_key_values is not None:
1006
+ assert cache is not None
1007
+ attn_key_values.append(cache)
1008
+ return x, attn_key_values
1009
+
1010
+ def reset_parameters(self):
1011
+ for block in self:
1012
+ block.reset_parameters()
1013
+
1014
+ def set_activation_checkpointing(self, strategy: Optional[ActivationCheckpointingStrategy]):
1015
+ self.activation_checkpointing_strategy = strategy
1016
+ for block in self:
1017
+ block.set_activation_checkpointing(strategy)
1018
+
1019
+
1020
+ class LLaDAModel(nn.Module):
1021
+ def __init__(self, config: ModelConfig, init_params: bool = True):
1022
+ super().__init__()
1023
+ self.config = config
1024
+ self.__cache = BufferCache()
1025
+
1026
+ # Validate config.
1027
+ if self.config.alibi and self.config.flash_attention:
1028
+ raise Exception("ALiBi is currently not supported with FlashAttention")
1029
+
1030
+ if self.config.alibi and self.config.rope:
1031
+ raise Exception("ALiBi and RoPE are mutually exclusive")
1032
+
1033
+ if self.config.embedding_size is not None and self.config.embedding_size != self.config.vocab_size:
1034
+ if self.config.embedding_size < self.config.vocab_size:
1035
+ raise Exception("embedding size should be at least as big as vocab size")
1036
+ elif self.config.embedding_size % 128 != 0:
1037
+ import warnings
1038
+
1039
+ warnings.warn(
1040
+ "Embedding size is not a multiple of 128! This could hurt throughput performance.", UserWarning
1041
+ )
1042
+
1043
+ self.activation_checkpointing_strategy: Optional[ActivationCheckpointingStrategy] = None
1044
+ self._activation_checkpoint_fn: Callable = activation_checkpoint_function(self.config)
1045
+
1046
+ if not (
1047
+ 0 < self.config.block_group_size <= self.config.n_layers
1048
+ and self.config.n_layers % self.config.block_group_size == 0
1049
+ ):
1050
+ raise Exception("n layers must be divisible by block group size")
1051
+
1052
+ torch.backends.cuda.enable_flash_sdp(True)
1053
+ torch.backends.cuda.enable_mem_efficient_sdp(False) # this is super slow so make sure torch won't use it
1054
+
1055
+ self.transformer = nn.ModuleDict(
1056
+ dict(
1057
+ wte=nn.Embedding(
1058
+ config.embedding_size or config.vocab_size, config.d_model, device=config.init_device
1059
+ ),
1060
+ emb_drop=Dropout(config.embedding_dropout),
1061
+ ln_f=LayerNorm.build(config),
1062
+ )
1063
+ )
1064
+
1065
+ blocks = [LLaDABlock.build(i, config, self.__cache) for i in range(config.n_layers)]
1066
+ if self.config.block_group_size > 1:
1067
+ block_groups = [
1068
+ LLaDABlockGroup(config, i, blocks[i : i + config.block_group_size])
1069
+ for i in range(0, config.n_layers, config.block_group_size)
1070
+ ]
1071
+ self.transformer.update({"block_groups": nn.ModuleList(block_groups)})
1072
+ else:
1073
+ self.transformer.update({"blocks": nn.ModuleList(blocks)})
1074
+
1075
+ if not (self.config.alibi or self.config.rope):
1076
+ self.transformer.update(
1077
+ {"wpe": nn.Embedding(config.max_sequence_length, config.d_model, device=config.init_device)}
1078
+ )
1079
+ if not config.weight_tying:
1080
+ self.transformer.update(
1081
+ {
1082
+ "ff_out": nn.Linear(
1083
+ config.d_model,
1084
+ config.embedding_size or config.vocab_size,
1085
+ bias=config.include_bias,
1086
+ device=config.init_device,
1087
+ )
1088
+ }
1089
+ )
1090
+ # When `init_device="meta"` FSDP will call `reset_parameters()` to initialize weights.
1091
+ if init_params and self.config.init_device != "meta":
1092
+ self.reset_parameters()
1093
+ self.__num_fwd_flops: Optional[int] = None
1094
+
1095
+ # Warm up cache.
1096
+ if self.config.alibi:
1097
+ get_causal_attention_bias(self.__cache, config.max_sequence_length, _non_meta_init_device(config))
1098
+ self.get_alibi_attention_bias(config.max_sequence_length, _non_meta_init_device(config))
1099
+
1100
+ def set_activation_checkpointing(self, strategy: Optional[ActivationCheckpointingStrategy]):
1101
+ self.activation_checkpointing_strategy = strategy
1102
+ if self.config.block_group_size != 1:
1103
+ for block_group in self.transformer.block_groups:
1104
+ block_group.set_activation_checkpointing(strategy)
1105
+ else:
1106
+ for block in self.transformer.blocks:
1107
+ block.set_activation_checkpointing(strategy)
1108
+
1109
+ @property
1110
+ def device(self) -> torch.device:
1111
+ device: torch.device = self.transformer.wte.weight.device # type: ignore
1112
+ if device.type == "meta":
1113
+ return _non_meta_init_device(self.config)
1114
+ else:
1115
+ return device
1116
+
1117
+ def reset_parameters(self):
1118
+ log.info("Initializing model parameters...")
1119
+ # Top-level embeddings / linear layers.
1120
+ init_weights(
1121
+ self.config,
1122
+ self.transformer.wte, # type: ignore
1123
+ std_factor=(0.5 * math.sqrt(self.config.d_model)) if self.config.scale_logits else 1.0,
1124
+ type_of_module=ModuleType.emb,
1125
+ )
1126
+ if hasattr(self.transformer, "wpe"):
1127
+ init_weights(self.config, self.transformer.wpe, type_of_module=ModuleType.emb) # type: ignore
1128
+
1129
+ # Top-level layer norm.
1130
+ self.transformer.ln_f.reset_parameters() # type: ignore
1131
+
1132
+ # Output weights.
1133
+ if hasattr(self.transformer, "ff_out"):
1134
+ init_weights(self.config, self.transformer.ff_out, type_of_module=ModuleType.final_out) # type: ignore
1135
+
1136
+ # Let the blocks handle themselves.
1137
+ if self.config.block_group_size == 1:
1138
+ for block in self.transformer.blocks:
1139
+ block.reset_parameters()
1140
+ else:
1141
+ for block_group in self.transformer.block_groups:
1142
+ block_group.reset_parameters()
1143
+
1144
+ def get_alibi_attention_bias(self, seq_len: int, device: torch.device) -> torch.Tensor:
1145
+ if (alibi_bias := self.__cache.get("alibi_attention_bias")) is not None and alibi_bias.shape[
1146
+ -1
1147
+ ] >= seq_len:
1148
+ if alibi_bias.device != device:
1149
+ alibi_bias = alibi_bias.to(device)
1150
+ self.__cache["alibi_attention_bias"] = alibi_bias
1151
+ return alibi_bias
1152
+ with torch.autocast(device.type, enabled=False):
1153
+ alibi_bias = alibi_attention_bias(seq_len, self.config, device)
1154
+ self.__cache["alibi_attention_bias"] = alibi_bias
1155
+ return alibi_bias
1156
+
1157
+ def forward(
1158
+ self,
1159
+ input_ids: torch.LongTensor,
1160
+ input_embeddings: Optional[torch.FloatTensor] = None,
1161
+ attention_mask: Optional[torch.Tensor] = None,
1162
+ attention_bias: Optional[torch.Tensor] = None,
1163
+ past_key_values: Optional[Sequence[Tuple[torch.Tensor, torch.Tensor]]] = None,
1164
+ use_cache: bool = False,
1165
+ last_logits_only: bool = False,
1166
+ output_hidden_states: Optional[bool] = None,
1167
+ ) -> LLaDAOutput:
1168
+ """
1169
+ :param input_ids: A tensor of shape `(batch_size, seq_len)`.
1170
+ :param input_embeddings: A tensor of shape `(batch_size, seq_len, d_model)` with input
1171
+ embeddings. When provided, it is treated as the output of the input embedding layer.
1172
+ :param attention_mask: A tensor of shape `(batch_size, seq_len)` that indicates
1173
+ which input IDs are masked. A `1` value in the mask means that
1174
+ the corresponding input ID should *not* be ignored. A `0` means
1175
+ that the corresponding input ID is masked.
1176
+ This has the same meaning as the `attention_mask` in HuggingFace's `transformers`
1177
+ library.
1178
+ :param attention_bias: A tensor of shape `(batch_size, 1, seq_len, seq_len)`,
1179
+ `(1, 1, seq_len, seq_len)`, or `(seq_len, seq_len)`. This is used
1180
+ to introduce causal or other biases.
1181
+ If the tensor is a bool or byte tensor, a `True` or `1` at `attention_bias[:, :, i, j]`
1182
+ indicates that the i-th element in the sequence is allowed to attend to the j-th
1183
+ element in the sequence.
1184
+ If the tensor is a float tensor, it will just be added to the attention
1185
+ scores before the softmax.
1186
+ The default is causal, which corresponds to a lower-diagonal byte matrix of ones.
1187
+ :param past_key_values: Pre-computed keys and values for each attention block.
1188
+ Can be used to speed up sequential decoding. The `input_ids` which have
1189
+ their past given to this model should not be passed as `input_ids` as they have already been computed.
1190
+ :param use_cache: If `True`, return key and value tensors for each block.
1191
+ :param last_logits_only: If `True`, only compute the logits for the last token of each sequence.
1192
+ This can speed up decoding when you only care about the next token.
1193
+ """
1194
+ # Add Basic MDM Model config check
1195
+ assert not self.config.alibi, "Alibi length extrapolation is not supported for MDM."
1196
+ assert self.config.rope, "Rope must be used in Llama-Encoder for MDM."
1197
+ assert (past_key_values is None and not use_cache), "The kvcache is not suppotred for MDM."
1198
+
1199
+ output_hidden_states = output_hidden_states if output_hidden_states is not None else False
1200
+
1201
+ if past_key_values:
1202
+ assert len(past_key_values) == self.config.n_layers
1203
+
1204
+ batch_size, seq_len = input_ids.size() if input_embeddings is None else input_embeddings.size()[:2]
1205
+ if past_key_values is None:
1206
+ past_length = 0
1207
+ else:
1208
+ past_length = past_key_values[0][0].size(-2)
1209
+
1210
+ # Get embeddings of input.
1211
+ # shape: (batch_size, seq_len, d_model)
1212
+ x = self.transformer.wte(input_ids) if input_embeddings is None else input_embeddings # type: ignore
1213
+
1214
+ if self.config.input_emb_norm:
1215
+ x = x * (self.config.d_model**0.5)
1216
+
1217
+ if not (self.config.alibi or self.config.rope):
1218
+ # Get positional embeddings.
1219
+ # shape: (1, seq_len)
1220
+ pos = torch.arange(past_length, past_length + seq_len, dtype=torch.long, device=x.device).unsqueeze(0)
1221
+ # shape: (1, seq_len, d_model)
1222
+ pos_emb = self.transformer.wpe(pos) # type: ignore
1223
+ x = pos_emb + x
1224
+
1225
+ # Add input + positional embeddings and apply dropout.
1226
+ # shape: (batch_size, seq_len, d_model)
1227
+ x = self.transformer.emb_drop(x) # type: ignore
1228
+
1229
+ # Transform the attention mask into what the blocks expect.
1230
+ if attention_mask is not None and 0.0 in attention_mask:
1231
+ # shape: (batch_size, 1, 1, seq_len)
1232
+ attention_mask = attention_mask.to(dtype=torch.float).view(batch_size, -1)[:, None, None, :]
1233
+ attention_mask = (1.0 - attention_mask) * torch.finfo(attention_mask.dtype).min
1234
+ else:
1235
+ attention_mask = None
1236
+
1237
+ # Merge attention mask with attention bias.
1238
+ if (
1239
+ attention_bias is not None
1240
+ or attention_mask is not None
1241
+ or self.config.alibi
1242
+ # NOTE (epwalsh): we need to initialize the attn bias in order for attn to work properly
1243
+ # with key+value cache. Otherwise `F.scaled_dot_product_attention()` doesn't seem to compute
1244
+ # scores correctly.
1245
+ or past_key_values is not None
1246
+ ):
1247
+ if attention_bias is None and self.config.alibi:
1248
+ attention_bias = get_causal_attention_bias(
1249
+ self.__cache, past_length + seq_len, x.device
1250
+ ) + self.get_alibi_attention_bias(past_length + seq_len, x.device)
1251
+ elif attention_bias is None:
1252
+ attention_bias = get_causal_attention_bias(self.__cache, past_length + seq_len, x.device)
1253
+ elif attention_bias.dtype in (torch.int8, torch.bool):
1254
+ attention_bias = attention_bias.to(dtype=torch.float)
1255
+ attention_bias.masked_fill_(attention_bias == 0.0, torch.finfo(attention_bias.dtype).min)
1256
+
1257
+ # Transform to the right shape and data type.
1258
+ mask_len = seq_len
1259
+ if attention_mask is not None:
1260
+ mask_len = attention_mask.shape[-1]
1261
+ elif past_key_values is not None:
1262
+ mask_len = past_key_values[0][0].shape[-2] + seq_len
1263
+ attention_bias = attention_bias[:, :, :mask_len, :mask_len].to(dtype=torch.float)
1264
+
1265
+ # Add in the masking bias.
1266
+ if attention_mask is not None:
1267
+ attention_bias = attention_bias + attention_mask
1268
+ # Might get -infs after adding attention mask, since dtype.min + dtype.min = -inf.
1269
+ # `F.scaled_dot_product_attention()` doesn't handle -inf like you'd expect, instead
1270
+ # it can produce NaNs.
1271
+ ensure_finite_(attention_bias, check_neg_inf=True, check_pos_inf=False)
1272
+
1273
+ attn_key_values: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = [] if use_cache else None
1274
+
1275
+ # decoder layers
1276
+ all_hidden_states = []
1277
+
1278
+ # Apply blocks one-by-one.
1279
+ if self.config.block_group_size == 1:
1280
+ for block_idx, block in enumerate(self.transformer.blocks):
1281
+ if output_hidden_states:
1282
+ # add hidden states
1283
+ all_hidden_states.append(x)
1284
+
1285
+ layer_past = None if past_key_values is None else past_key_values[block_idx]
1286
+ if (
1287
+ (self.activation_checkpointing_strategy == ActivationCheckpointingStrategy.whole_layer)
1288
+ or (
1289
+ self.activation_checkpointing_strategy == ActivationCheckpointingStrategy.one_in_two
1290
+ and block_idx % 2 == 0
1291
+ )
1292
+ or (
1293
+ self.activation_checkpointing_strategy == ActivationCheckpointingStrategy.one_in_three
1294
+ and block_idx % 3 == 0
1295
+ )
1296
+ or (
1297
+ self.activation_checkpointing_strategy == ActivationCheckpointingStrategy.one_in_four
1298
+ and block_idx % 4 == 0
1299
+ )
1300
+ ):
1301
+ # shape: (batch_size, seq_len, d_model)
1302
+ x, cache = self._activation_checkpoint_fn(
1303
+ block, x, attention_bias=attention_bias, layer_past=layer_past, use_cache=use_cache
1304
+ )
1305
+ else:
1306
+ # shape: (batch_size, seq_len, d_model)
1307
+ x, cache = block(x, attention_bias=attention_bias, layer_past=layer_past, use_cache=use_cache)
1308
+ if attn_key_values is not None:
1309
+ assert cache is not None
1310
+ attn_key_values.append(cache)
1311
+ else:
1312
+ for group_idx, block_group in enumerate(self.transformer.block_groups):
1313
+ if output_hidden_states:
1314
+ # add hidden states
1315
+ all_hidden_states.append(x)
1316
+
1317
+ layers_past = (
1318
+ None
1319
+ if past_key_values is None
1320
+ else past_key_values[
1321
+ group_idx * self.config.block_group_size : (group_idx + 1) * self.config.block_group_size
1322
+ ]
1323
+ )
1324
+ x, cache = block_group(
1325
+ x, attention_bias=attention_bias, layers_past=layers_past, use_cache=use_cache
1326
+ )
1327
+ if attn_key_values is not None:
1328
+ assert cache is not None
1329
+ attn_key_values.extend(cache)
1330
+
1331
+ if last_logits_only:
1332
+ # shape: (batch_size, 1, d_model)
1333
+ x = x[:, -1, :].unsqueeze(1)
1334
+
1335
+ # Apply final layer norm.
1336
+ # shape: (batch_size, seq_len or 1, d_model)
1337
+ x = self.transformer.ln_f(x) # type: ignore
1338
+ if output_hidden_states:
1339
+ # add final hidden state post-final-layernorm, following HuggingFace's convention
1340
+ all_hidden_states.append(x)
1341
+
1342
+ # Get logits.
1343
+ # shape: (batch_size, seq_len or 1, vocab_size)
1344
+ if self.config.weight_tying:
1345
+ logits = F.linear(x, self.transformer.wte.weight, None) # type: ignore
1346
+ else:
1347
+ logits = self.transformer.ff_out(x) # type: ignore
1348
+ if self.config.scale_logits:
1349
+ logits.mul_(1 / math.sqrt(self.config.d_model))
1350
+
1351
+ return LLaDAOutput(logits=logits, attn_key_values=attn_key_values, hidden_states=tuple(all_hidden_states) if output_hidden_states else None) # type: ignore[arg-type]
1352
+
1353
+
1354
+ def create_model_config_from_pretrained_config(config: LLaDAConfig):
1355
+ """
1356
+ Utility function
1357
+ """
1358
+
1359
+ kwargs = {}
1360
+ for field in fields(ModelConfig):
1361
+ kwargs[field.name] = getattr(config, field.name)
1362
+
1363
+ model_config = ModelConfig(**kwargs)
1364
+ return model_config
1365
+
1366
+
1367
+ class LLaDAModelLM(PreTrainedModel):
1368
+ """
1369
+ Extremely barebones HF model wrapper.
1370
+ """
1371
+
1372
+ config_class = LLaDAConfig
1373
+ base_model_prefix = "model"
1374
+ _no_split_modules = ["LLaDABlock", "LLaDASequentialBlock", "LLaDALlamaBlock"]
1375
+
1376
+ def __init__(self, config: LLaDAConfig, model: Optional[LLaDAModel] = None, init_params: bool = False):
1377
+ super().__init__(config)
1378
+
1379
+ if not model:
1380
+ model_config = create_model_config_from_pretrained_config(config)
1381
+ # Initialize model (always on CPU to start with so we don't run out of GPU memory).
1382
+ model_config.init_device = "cpu"
1383
+ self.model = LLaDAModel(model_config, init_params=init_params)
1384
+ else:
1385
+ self.model = model
1386
+
1387
+ def forward(
1388
+ self,
1389
+ input_ids: torch.LongTensor = None,
1390
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1391
+ attention_mask: Optional[torch.Tensor] = None,
1392
+ attention_bias: Optional[torch.Tensor] = None,
1393
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
1394
+ labels: Optional[torch.LongTensor] = None,
1395
+ use_cache: Optional[bool] = None,
1396
+ output_attentions: Optional[bool] = None,
1397
+ output_hidden_states: Optional[bool] = None,
1398
+ return_dict: Optional[bool] = None,
1399
+ cache_position: Optional[Cache] = None, # This is a hack mitigation of an issue in transformers `4.39.x`
1400
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
1401
+ if use_cache is None:
1402
+ use_cache = self.config.use_cache
1403
+
1404
+ if output_attentions:
1405
+ raise ValueError("output_attentions is not yet supported in LLaDA")
1406
+
1407
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1408
+
1409
+ # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
1410
+ outputs = self.model.forward(
1411
+ input_ids=input_ids,
1412
+ input_embeddings=inputs_embeds,
1413
+ attention_mask=attention_mask,
1414
+ attention_bias=attention_bias,
1415
+ past_key_values=past_key_values,
1416
+ use_cache=use_cache,
1417
+ output_hidden_states=output_hidden_states,
1418
+ )
1419
+
1420
+ logits = outputs.logits
1421
+ hidden_states = outputs.hidden_states
1422
+
1423
+ loss = None
1424
+ if labels is not None:
1425
+ import warnings
1426
+ warnings.warn("Note that for LLaDA, you cannot calculate the loss here.", UserWarning)
1427
+ if not return_dict:
1428
+ output = (logits,) + outputs[1:]
1429
+ return (loss,) + output if loss is not None else output
1430
+
1431
+ return CausalLMOutputWithPast(
1432
+ logits=logits,
1433
+ past_key_values=outputs.attn_key_values,
1434
+ hidden_states=hidden_states,
1435
+ )
1436
+
1437
+ def can_generate(self) -> bool:
1438
+ return True
1439
+
1440
+ def prepare_inputs_for_generation(
1441
+ self, input_ids: torch.LongTensor, past_key_values: Optional[List[Tuple]] = None, **kwargs
1442
+ ):
1443
+ if past_key_values:
1444
+ # This is because we want the model to only process the last generated token.
1445
+ input_ids = input_ids[:, -1:]
1446
+ model_inputs = {"input_ids": input_ids, "past_key_values": past_key_values}
1447
+
1448
+ model_inputs.update(kwargs)
1449
+ model_inputs["use_cache"] = kwargs.pop("use_cache", self.config.use_cache)
1450
+ return model_inputs
1451
+
1452
+ # TODO: these are required to make the implementation complete.
1453
+ # def resize_position_embeddings(self, new_num_position_embeddings: int):
1454
+ # pass
1455
+ #
1456
+ # def get_position_embeddings(self) -> Union[nn.Embedding, Tuple[nn.Embedding]]:
1457
+ # pass
1458
+ #
1459
+ # def _reorder_cache(self, past_key_values, beam_idx):
1460
+ # pass
1461
+
1462
+ def get_input_embeddings(self) -> torch.nn.Module:
1463
+ return self.model.transformer.wte
1464
+
1465
+ def set_input_embeddings(self, value: torch.nn.Module):
1466
+ self.model.transformer.wte = value
1467
+
1468
+ def get_output_embeddings(self):
1469
+ if self.config.weight_tying:
1470
+ return self.model.transformer.wte
1471
+ else:
1472
+ return self.model.transformer.ff_out
1473
+
1474
+ def set_output_embeddings(self, value: torch.nn.Module):
1475
+ if self.config.weight_tying:
1476
+ self.model.transformer.wte = value
1477
+ else:
1478
+ self.model.transformer.ff_out = value
1479
+
1480
+ def tie_weights(self):
1481
+ if self.config.weight_tying:
1482
+ self.model.transformer.ff_out = self.model.transformer.wte
1483
+
1484
+ # Register the model so that it is available for transformer pipelines, auto-loading, etc.
1485
+ AutoModel.register(LLaDAConfig, LLaDAModelLM)