lhallee commited on
Commit
b50f253
·
verified ·
1 Parent(s): 0971e6e

Upload modeling_esm_plusplus.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. modeling_esm_plusplus.py +1354 -0
modeling_esm_plusplus.py ADDED
@@ -0,0 +1,1354 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+ """
3
+ ESM++ model implementation.
4
+
5
+ ESM++ is a faithful implementation of ESMC that allows for batching and standard Huggingface compatibility
6
+ The ESM Python package is not required
7
+
8
+ Modified from https://github.com/Biohub/esm
9
+ License: https://github.com/Biohub/esm/blob/main/LICENSE.md
10
+ """
11
+
12
+ import math
13
+ import os
14
+ import json
15
+ import torch
16
+ import torch.nn as nn
17
+ import torch.nn.functional as F
18
+ from dataclasses import dataclass
19
+ from functools import cache, partial
20
+ from pathlib import Path
21
+ from typing import Optional, Tuple, Union, List
22
+ from einops import rearrange, repeat
23
+ from huggingface_hub import snapshot_download
24
+ from safetensors.torch import load_file as load_safetensors_file
25
+ from tokenizers import Tokenizer
26
+ from tokenizers.models import BPE
27
+ from tokenizers.processors import TemplateProcessing
28
+ from transformers import PreTrainedModel, PreTrainedTokenizerFast, PretrainedConfig
29
+ from transformers.modeling_outputs import ModelOutput
30
+
31
+ try:
32
+ from fastplms.attention import (
33
+ AttentionBackend, VALID_ATTENTION_BACKENDS,
34
+ resolve_attention_backend, get_attention_mask,
35
+ _get_flex_attention_fn,
36
+ _ensure_flash_kernels_loaded, FLASH_KERNEL, FLASH_KERNEL_VARIANT,
37
+ _kernels_flash_forward, _kernels_flash_varlen_forward,
38
+ kernels_flash_attention_func,
39
+ index_first_axis, index_put_first_axis, pad_input, _unpad_input,
40
+ create_block_mask, flex_attention, BlockMask,
41
+ )
42
+ from fastplms.embedding_mixin import Pooler, EmbeddingMixin, ProteinDataset, parse_fasta, build_collator
43
+ except ImportError:
44
+ pass # Running as HF Hub composite; shared definitions are above
45
+
46
+
47
+ class ESMplusplusConfig(PretrainedConfig):
48
+ """Configuration class for ESM++ model.
49
+
50
+ Args:
51
+ vocab_size: Size of the vocabulary
52
+ hidden_size: Dimension of hidden layers
53
+ num_attention_heads: Number of attention heads
54
+ num_hidden_layers: Number of transformer layers
55
+ num_labels: Number of output labels for classification
56
+ problem_type: Type of problem - regression, single/multi label classification
57
+ """
58
+ model_type = "ESMplusplus"
59
+ def __init__(
60
+ self,
61
+ vocab_size: int = 64,
62
+ hidden_size: int = 960,
63
+ num_attention_heads: int = 15,
64
+ num_hidden_layers: int = 30,
65
+ num_labels: int = 2,
66
+ problem_type: Optional[str] = None,
67
+ dropout: float = 0.0,
68
+ initializer_range: float = 0.02,
69
+ attn_backend: str = "sdpa",
70
+ **kwargs,
71
+ ):
72
+ super().__init__(**kwargs)
73
+ self.vocab_size = vocab_size
74
+ self.hidden_size = hidden_size
75
+ self.num_attention_heads = num_attention_heads
76
+ self.num_hidden_layers = num_hidden_layers
77
+ self.num_labels = num_labels
78
+ self.problem_type = problem_type
79
+ self.dropout = dropout
80
+ self.initializer_range = initializer_range
81
+ self.tie_word_embeddings = False
82
+ self.attn_backend = attn_backend
83
+
84
+
85
+ ### Rotary Embeddings
86
+ def rotate_half(x: torch.Tensor, interleaved: bool = False) -> torch.Tensor:
87
+ """Rotates half the hidden dims of the input."""
88
+ if not interleaved:
89
+ x1, x2 = x.chunk(2, dim=-1)
90
+ return torch.cat((-x2, x1), dim=-1)
91
+ else:
92
+ x1, x2 = x[..., ::2], x[..., 1::2]
93
+ return rearrange(
94
+ torch.stack((-x2, x1), dim=-1), "... d two -> ... (d two)", two=2
95
+ )
96
+
97
+
98
+ def apply_rotary_emb_torch(
99
+ x: torch.Tensor,
100
+ cos: torch.Tensor,
101
+ sin: torch.Tensor,
102
+ interleaved: bool = False,
103
+ _inplace: bool = False,
104
+ ) -> torch.Tensor:
105
+ """Apply rotary embeddings to input based on cos and sin."""
106
+ ro_dim = cos.shape[-1] * 2
107
+ assert ro_dim <= x.shape[-1]
108
+ seqlen = x.size(1)
109
+ cos = cos[:seqlen]
110
+ sin = sin[:seqlen]
111
+ cos = repeat(cos, "s d -> s 1 (2 d)")
112
+ sin = repeat(sin, "s d -> s 1 (2 d)")
113
+ return torch.cat(
114
+ [
115
+ x[..., :ro_dim] * cos + rotate_half(x[..., :ro_dim], interleaved) * sin,
116
+ x[..., ro_dim:],
117
+ ],
118
+ dim=-1,
119
+ )
120
+
121
+
122
+ class RotaryEmbedding(torch.nn.Module):
123
+ """Rotary position embeddings.
124
+
125
+ Based on the paper "RoFormer: Enhanced Transformer with Rotary Position Embedding"
126
+
127
+ Args:
128
+ dim: Dimension of the embedding
129
+ base: Base for computing angular frequencies
130
+ interleaved: Whether to use interleaved rotations
131
+ scale_base: Base for scaling
132
+ scaling_factor: Factor for scaling positions
133
+ pos_idx_in_fp32: Whether to compute position indices in fp32
134
+ device: Computation device
135
+ """
136
+ def __init__(
137
+ self,
138
+ dim: int,
139
+ base: float = 10000.0,
140
+ interleaved: bool = False,
141
+ scale_base: Optional[float] = None,
142
+ scaling_factor: float = 1.0,
143
+ pos_idx_in_fp32: bool = True,
144
+ device: Optional[torch.device] = None,
145
+ ):
146
+ super().__init__()
147
+ self.dim = dim
148
+ self.base = float(base)
149
+ self.pos_idx_in_fp32 = pos_idx_in_fp32
150
+ self.interleaved = interleaved
151
+ self.scale_base = scale_base
152
+ self.scaling_factor = scaling_factor
153
+ self.device = device
154
+
155
+ self._seq_len_cached = 0
156
+ self._cos_cached = None
157
+ self._sin_cached = None
158
+ self._cos_k_cached = None
159
+ self._sin_k_cached = None
160
+ self.reset_parameters()
161
+
162
+ def reset_parameters(self):
163
+ """Reset the parameters of the embedding."""
164
+ if "inv_freq" in self._buffers and isinstance(self._buffers["inv_freq"], torch.Tensor):
165
+ buffer_device = self._buffers["inv_freq"].device
166
+ else:
167
+ buffer_device = self.device
168
+ inv_freq = self._compute_inv_freq(buffer_device)
169
+ self._seq_len_cached = 0
170
+ self._cos_cached = None
171
+ self._sin_cached = None
172
+ self._cos_k_cached = None
173
+ self._sin_k_cached = None
174
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
175
+ arange = torch.arange(0, self.dim, 2, device=buffer_device, dtype=torch.float32)
176
+ scale = (
177
+ (arange + 0.4 * self.dim) / (1.4 * self.dim)
178
+ if self.scale_base is not None
179
+ else None
180
+ )
181
+ self.register_buffer("scale", scale)
182
+
183
+ def _compute_inv_freq(self, device: Optional[torch.device] = None) -> torch.Tensor:
184
+ """Compute inverse frequency bands.
185
+
186
+ Always computes on CPU then moves to the requested device. This matches
187
+ native Biohub ESMC, which computes inv_freq on CPU at
188
+ `__init__` and migrates via `.to(device)`. Computing directly on GPU
189
+ gives a ~3.7e-9 bit-level difference in inv_freq (fp32 transcendental
190
+ precision differs between CPU and GPU), which compounds through the 30
191
+ attention layers to ~1e-3 mse divergence from native at
192
+ `hidden_states[-2]`. See testing/parity_debug_rotary.py.
193
+ """
194
+ cpu_inv_freq = 1 / (
195
+ self.base
196
+ ** (
197
+ torch.arange(0, self.dim, 2, device="cpu", dtype=torch.float32)
198
+ / self.dim
199
+ )
200
+ )
201
+ if device is not None and torch.device(device).type != "cpu":
202
+ return cpu_inv_freq.to(device)
203
+ return cpu_inv_freq
204
+
205
+ def _update_cos_sin_cache(self, seqlen: int, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None):
206
+ """Update the cached cosine and sine values."""
207
+ if (
208
+ seqlen > self._seq_len_cached
209
+ or self._cos_cached is None
210
+ or self._cos_cached.device != device
211
+ or self._cos_cached.dtype != dtype
212
+ or (self.training and self._cos_cached.is_inference())
213
+ ):
214
+ self._seq_len_cached = seqlen
215
+ if self.pos_idx_in_fp32:
216
+ t = torch.arange(seqlen, device=device, dtype=torch.float32)
217
+ t /= self.scaling_factor
218
+ if self.inv_freq.dtype != torch.float32:
219
+ inv_freq = self.inv_freq.to(torch.float32)
220
+ else:
221
+ inv_freq = self.inv_freq
222
+ else:
223
+ t = torch.arange(seqlen, device=device, dtype=self.inv_freq.dtype)
224
+ t /= self.scaling_factor
225
+ inv_freq = self.inv_freq
226
+ freqs = torch.outer(t, inv_freq)
227
+
228
+ if self.scale is None:
229
+ self._cos_cached = torch.cos(freqs).to(dtype)
230
+ self._sin_cached = torch.sin(freqs).to(dtype)
231
+ else:
232
+ power = (
233
+ torch.arange(
234
+ seqlen, dtype=self.scale.dtype, device=self.scale.device
235
+ )
236
+ - seqlen // 2
237
+ ) / self.scale_base
238
+ scale = self.scale.to(device=power.device) ** power.unsqueeze(-1)
239
+ self._cos_cached = (torch.cos(freqs) * scale).to(dtype)
240
+ self._sin_cached = (torch.sin(freqs) * scale).to(dtype)
241
+ self._cos_k_cached = (torch.cos(freqs) / scale).to(dtype)
242
+ self._sin_k_cached = (torch.sin(freqs) / scale).to(dtype)
243
+
244
+ def forward(self, q: torch.Tensor, k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
245
+ """Apply rotary embeddings to queries and keys.
246
+
247
+ Args:
248
+ q: Query tensor of shape (batch, seqlen, nheads, headdim)
249
+ k: Key tensor of shape (batch, seqlen, nheads, headdim)
250
+
251
+ Returns:
252
+ Tuple of rotated query and key tensors
253
+ """
254
+ # NOTE: do NOT recompute inv_freq here if device has changed. The native
255
+ # ESMC implementation computes inv_freq once on CPU at __init__ and
256
+ # relies on PyTorch's `.to(device)` to migrate the buffer. Recomputing
257
+ # the values directly on GPU gives a ~3.7e-9 bit-level difference vs the
258
+ # CPU-computed-then-moved values due to fp32 transcendental precision,
259
+ # which compounds through 30 attention layers to ~1e-3 mse divergence
260
+ # from native at `hidden_states[-2]`. See testing/parity_debug_rotary.py.
261
+ self._update_cos_sin_cache(q.shape[1], device=q.device, dtype=q.dtype)
262
+ assert self._cos_cached is not None
263
+ assert self._sin_cached is not None
264
+ if self.scale is None:
265
+ return (
266
+ apply_rotary_emb_torch(
267
+ q,
268
+ self._cos_cached,
269
+ self._sin_cached,
270
+ self.interleaved,
271
+ True, # inplace=True
272
+ ),
273
+ apply_rotary_emb_torch(
274
+ k,
275
+ self._cos_cached,
276
+ self._sin_cached,
277
+ self.interleaved,
278
+ True, # inplace=True
279
+ ),
280
+ ) # type: ignore
281
+ else:
282
+ assert False
283
+
284
+
285
+ ### Feedforward Network Components
286
+ def swiglu_correction_fn(expansion_ratio: float, d_model: int) -> int:
287
+ """Compute corrected dimension for SwiGLU."""
288
+ return int(((expansion_ratio * d_model) + 255) // 256 * 256)
289
+
290
+
291
+ class SwiGLU(nn.Module):
292
+ """SwiGLU activation function."""
293
+ def __init__(self):
294
+ super(SwiGLU, self).__init__()
295
+
296
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
297
+ x1, x2 = x.chunk(2, dim=-1)
298
+ return F.silu(x1) * x2
299
+
300
+
301
+ def swiglu_ln_ffn(d_model: int, expansion_ratio: float) -> nn.Sequential:
302
+ """Create SwiGLU feedforward network with layer normalization."""
303
+ return nn.Sequential(
304
+ nn.LayerNorm(d_model),
305
+ nn.Linear(
306
+ d_model, swiglu_correction_fn(expansion_ratio, d_model) * 2, bias=False
307
+ ),
308
+ SwiGLU(),
309
+ nn.Linear(swiglu_correction_fn(expansion_ratio, d_model), d_model, bias=False),
310
+ )
311
+
312
+
313
+ ### Attention
314
+ class MultiHeadAttention(nn.Module):
315
+ """Multi-head attention with rotary embeddings and configurable backend.
316
+
317
+ Args:
318
+ d_model: Model dimension
319
+ n_heads: Number of attention heads
320
+ attn_backend: One of "auto", "kernels_flash", "flex", "sdpa"
321
+ """
322
+ def __init__(
323
+ self,
324
+ d_model: int,
325
+ n_heads: int,
326
+ attn_backend: str = "sdpa",
327
+ ):
328
+ super().__init__()
329
+ self.d_model = d_model
330
+ self.n_heads = n_heads
331
+ self.d_head = self.d_model // self.n_heads
332
+ self.scale = 1.0 / math.sqrt(self.d_head)
333
+ self.attn_backend = resolve_attention_backend(attn_backend)
334
+ self.layernorm_qkv = nn.Sequential(
335
+ nn.LayerNorm(d_model), nn.Linear(d_model, d_model * 3, bias=False)
336
+ )
337
+ self.out_proj = nn.Linear(d_model, d_model, bias=False)
338
+ self.q_ln = nn.LayerNorm(d_model, bias=False)
339
+ self.k_ln = nn.LayerNorm(d_model, bias=False)
340
+ self.reshaper = partial(rearrange, pattern="b s (h d) -> b h s d", h=n_heads)
341
+ self.rotary = RotaryEmbedding(d_model // n_heads)
342
+
343
+ def _apply_rotary(self, q: torch.Tensor, k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
344
+ q = q.unflatten(-1, (self.n_heads, self.d_head))
345
+ k = k.unflatten(-1, (self.n_heads, self.d_head))
346
+ q, k = self.rotary(q, k)
347
+ q = q.flatten(-2, -1)
348
+ k = k.flatten(-2, -1)
349
+ return q, k
350
+
351
+ def forward(
352
+ self,
353
+ x: torch.Tensor,
354
+ attention_mask_2d: Optional[torch.Tensor] = None,
355
+ attention_mask_4d: Optional[torch.Tensor] = None,
356
+ flex_block_mask: Optional[BlockMask] = None,
357
+ output_attentions: bool = False,
358
+ output_s_max: bool = False,
359
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[List[torch.Tensor]]]:
360
+ qkv_BLD3 = self.layernorm_qkv(x)
361
+ query_BLD, key_BLD, value_BLD = torch.chunk(qkv_BLD3, 3, dim=-1)
362
+ query_BLD, key_BLD = (
363
+ self.q_ln(query_BLD).to(query_BLD.dtype),
364
+ self.k_ln(key_BLD).to(query_BLD.dtype),
365
+ )
366
+ query_BLD, key_BLD = self._apply_rotary(query_BLD, key_BLD)
367
+ query_BHLD, key_BHLD, value_BHLD = map(self.reshaper, (query_BLD, key_BLD, value_BLD))
368
+
369
+ attn_output, attn_weights, s_max = self._attn(
370
+ query_BHLD, key_BHLD, value_BHLD,
371
+ attention_mask_2d=attention_mask_2d,
372
+ attention_mask_4d=attention_mask_4d,
373
+ flex_block_mask=flex_block_mask,
374
+ output_attentions=output_attentions,
375
+ output_s_max=output_s_max,
376
+ )
377
+
378
+ output = self.out_proj(attn_output)
379
+ return output, attn_weights, s_max
380
+
381
+ def _attn(
382
+ self,
383
+ query_BHLD: torch.Tensor,
384
+ key_BHLD: torch.Tensor,
385
+ value_BHLD: torch.Tensor,
386
+ attention_mask_2d: Optional[torch.Tensor] = None,
387
+ attention_mask_4d: Optional[torch.Tensor] = None,
388
+ flex_block_mask: Optional[BlockMask] = None,
389
+ output_attentions: bool = False,
390
+ output_s_max: bool = False,
391
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[List[torch.Tensor]]]:
392
+ if output_attentions:
393
+ return self._manual_attn(query_BHLD, key_BHLD, value_BHLD, attention_mask_4d, output_s_max)
394
+
395
+ if self.attn_backend == AttentionBackend.KERNELS_FLASH:
396
+ attn_output, attn_weights = self._kernels_flash_attn(query_BHLD, key_BHLD, value_BHLD, attention_mask_2d)
397
+ elif self.attn_backend == AttentionBackend.FLEX:
398
+ attn_output, attn_weights = self._flex_attn(query_BHLD, key_BHLD, value_BHLD, flex_block_mask)
399
+ elif self.attn_backend == AttentionBackend.SDPA:
400
+ attn_output, attn_weights = self._sdpa_attn(query_BHLD, key_BHLD, value_BHLD, attention_mask_4d)
401
+ else:
402
+ raise AssertionError(f"Unsupported resolved backend: {self.attn_backend}")
403
+
404
+ s_max = self._compute_s_max(query_BHLD, key_BHLD) if output_s_max else None
405
+ return attn_output, attn_weights, s_max
406
+
407
+ @torch.no_grad()
408
+ def _compute_s_max(self, query_BHLD: torch.Tensor, key_BHLD: torch.Tensor) -> List[torch.Tensor]:
409
+ q_norm = torch.linalg.vector_norm(query_BHLD, dim=-1)
410
+ k_norm = torch.linalg.vector_norm(key_BHLD, dim=-1)
411
+ s_max_bound = (q_norm.max(dim=-1).values * k_norm.max(dim=-1).values).max(dim=0).values * self.scale
412
+ return [s_max_bound[h] for h in range(self.n_heads)]
413
+
414
+ def _manual_attn(
415
+ self,
416
+ query_BHLD: torch.Tensor,
417
+ key_BHLD: torch.Tensor,
418
+ value_BHLD: torch.Tensor,
419
+ attention_mask_4d: Optional[torch.Tensor] = None,
420
+ output_s_max: bool = False,
421
+ ) -> Tuple[torch.Tensor, torch.Tensor, Optional[List[torch.Tensor]]]:
422
+ attn_weights = torch.matmul(query_BHLD, key_BHLD.transpose(-2, -1)) * self.scale
423
+ if attention_mask_4d is not None:
424
+ attn_weights = attn_weights.masked_fill(attention_mask_4d.logical_not(), float("-inf"))
425
+ attn_weights = F.softmax(attn_weights, dim=-1)
426
+ context_BHLD = torch.matmul(attn_weights, value_BHLD)
427
+ attn_output = rearrange(context_BHLD, "b h s d -> b s (h d)")
428
+ s_max = self._compute_s_max(query_BHLD, key_BHLD) if output_s_max else None
429
+ return attn_output, attn_weights, s_max
430
+
431
+ def _kernels_flash_attn(
432
+ self,
433
+ query_BHLD: torch.Tensor,
434
+ key_BHLD: torch.Tensor,
435
+ value_BHLD: torch.Tensor,
436
+ attention_mask_2d: Optional[torch.Tensor] = None,
437
+ ) -> Tuple[torch.Tensor, None]:
438
+ query_BLHD = query_BHLD.transpose(1, 2).contiguous()
439
+ key_BLHD = key_BHLD.transpose(1, 2).contiguous()
440
+ value_BLHD = value_BHLD.transpose(1, 2).contiguous()
441
+ attn_output = kernels_flash_attention_func(
442
+ query_states=query_BLHD, key_states=key_BLHD, value_states=value_BLHD,
443
+ attention_mask_2d=attention_mask_2d, causal=False,
444
+ )
445
+ return rearrange(attn_output, "b s h d -> b s (h d)"), None
446
+
447
+ def _flex_attn(
448
+ self,
449
+ query_BHLD: torch.Tensor,
450
+ key_BHLD: torch.Tensor,
451
+ value_BHLD: torch.Tensor,
452
+ flex_block_mask: Optional[BlockMask] = None,
453
+ ) -> Tuple[torch.Tensor, None]:
454
+ assert flex_attention is not None, "Flex attention is not available in this environment."
455
+ fn = _get_flex_attention_fn()
456
+ context_BHLD = fn(query_BHLD, key_BHLD, value_BHLD, block_mask=flex_block_mask, scale=self.scale)
457
+ return rearrange(context_BHLD, "b h s d -> b s (h d)"), None
458
+
459
+ def _sdpa_attn(
460
+ self,
461
+ query_BHLD: torch.Tensor,
462
+ key_BHLD: torch.Tensor,
463
+ value_BHLD: torch.Tensor,
464
+ attention_mask_4d: Optional[torch.Tensor] = None,
465
+ ) -> Tuple[torch.Tensor, None]:
466
+ context_BHLD = F.scaled_dot_product_attention(
467
+ query_BHLD, key_BHLD, value_BHLD, attn_mask=attention_mask_4d, scale=self.scale,
468
+ )
469
+ return rearrange(context_BHLD, "b h s d -> b s (h d)"), None
470
+
471
+
472
+ ### Regression Head
473
+ def RegressionHead(d_model: int, output_dim: int, hidden_dim: Optional[int] = None) -> nn.Module:
474
+ """Create a regression head with optional hidden dimension.
475
+
476
+ Args:
477
+ d_model: Input dimension
478
+ output_dim: Output dimension
479
+ hidden_dim: Optional hidden dimension (defaults to d_model)
480
+ """
481
+ hidden_dim = hidden_dim if hidden_dim is not None else d_model
482
+ return nn.Sequential(
483
+ nn.Linear(d_model, hidden_dim),
484
+ nn.GELU(),
485
+ nn.LayerNorm(hidden_dim),
486
+ nn.Linear(hidden_dim, output_dim),
487
+ )
488
+
489
+
490
+ ### Transformer Block
491
+ class UnifiedTransformerBlock(nn.Module):
492
+ """Transformer block with attention and feedforward layers."""
493
+ def __init__(
494
+ self,
495
+ d_model: int,
496
+ n_heads: int,
497
+ residue_scaling_factor: float = 1,
498
+ expansion_ratio: float = 8 / 3,
499
+ dropout: float = 0.0,
500
+ attn_backend: str = "sdpa",
501
+ ):
502
+ super().__init__()
503
+ self.attn = MultiHeadAttention(d_model=d_model, n_heads=n_heads, attn_backend=attn_backend)
504
+ self.ffn = swiglu_ln_ffn(d_model, expansion_ratio)
505
+ self.scaling_factor = residue_scaling_factor
506
+ self.dropout = nn.Dropout(dropout)
507
+
508
+ def forward(
509
+ self,
510
+ x: torch.Tensor,
511
+ attention_mask_2d: Optional[torch.Tensor] = None,
512
+ attention_mask_4d: Optional[torch.Tensor] = None,
513
+ flex_block_mask: Optional[BlockMask] = None,
514
+ output_attentions: bool = False,
515
+ output_s_max: bool = False,
516
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[List[torch.Tensor]]]:
517
+ attn_output, attn_weights, s_max = self.attn(
518
+ x,
519
+ attention_mask_2d=attention_mask_2d,
520
+ attention_mask_4d=attention_mask_4d,
521
+ flex_block_mask=flex_block_mask,
522
+ output_attentions=output_attentions,
523
+ output_s_max=output_s_max,
524
+ )
525
+ x = x + self.dropout(attn_output) / self.scaling_factor
526
+ x = x + self.dropout(self.ffn(x)) / self.scaling_factor
527
+ return x, attn_weights, s_max
528
+
529
+
530
+ ### Model Outputs
531
+ @dataclass
532
+ class TransformerOutput(ModelOutput):
533
+ """Output type for transformer encoder."""
534
+ last_hidden_state: Optional[torch.Tensor] = None
535
+ hidden_states: Optional[Tuple[torch.Tensor]] = None
536
+ attentions: Optional[Tuple[torch.Tensor]] = None
537
+ s_max: Optional[Tuple[List[torch.Tensor], ...]] = None
538
+
539
+
540
+ @dataclass
541
+ class ESMplusplusOutput(ModelOutput):
542
+ """Output type for ESM++ models."""
543
+ loss: Optional[torch.Tensor] = None
544
+ logits: Optional[torch.Tensor] = None
545
+ last_hidden_state: Optional[torch.Tensor] = None
546
+ hidden_states: Optional[Tuple[torch.Tensor]] = None
547
+ attentions: Optional[Tuple[torch.Tensor]] = None
548
+ s_max: Optional[Tuple[List[torch.Tensor], ...]] = None
549
+
550
+
551
+ ### Transformer Stack
552
+ class TransformerStack(nn.Module):
553
+ """Stack of transformer blocks."""
554
+ def __init__(
555
+ self,
556
+ d_model: int,
557
+ n_heads: int,
558
+ n_layers: int,
559
+ dropout: float = 0.0,
560
+ attn_backend: str = "sdpa",
561
+ ):
562
+ super().__init__()
563
+ self.attention_backend = resolve_attention_backend(attn_backend)
564
+ self.blocks = nn.ModuleList(
565
+ [
566
+ UnifiedTransformerBlock(
567
+ d_model,
568
+ n_heads,
569
+ residue_scaling_factor=math.sqrt(n_layers / 36),
570
+ dropout=dropout,
571
+ attn_backend=attn_backend,
572
+ )
573
+ for i in range(n_layers)
574
+ ]
575
+ )
576
+ self.norm = nn.LayerNorm(d_model, bias=False)
577
+ self.gradient_checkpointing = False
578
+
579
+ @property
580
+ def attn_backend(self) -> AttentionBackend:
581
+ return self.attention_backend
582
+
583
+ @attn_backend.setter
584
+ def attn_backend(self, backend: str) -> None:
585
+ resolved = resolve_attention_backend(backend)
586
+ self.attention_backend = resolved
587
+ for block in self.blocks:
588
+ block.attn.attn_backend = resolved
589
+
590
+ def forward(
591
+ self,
592
+ x: torch.Tensor,
593
+ attention_mask: Optional[torch.Tensor] = None,
594
+ output_hidden_states: Optional[bool] = False,
595
+ output_attentions: Optional[bool] = False,
596
+ output_s_max: Optional[bool] = False,
597
+ ) -> TransformerOutput:
598
+ hidden_states = () if output_hidden_states else None
599
+ attentions = () if output_attentions else None
600
+ full_s_max = () if output_s_max else None
601
+
602
+ attention_mask_2d, attention_mask_4d, flex_block_mask = get_attention_mask(
603
+ effective_backend=self.attention_backend,
604
+ batch_size=x.shape[0],
605
+ seq_len=x.shape[1],
606
+ device=x.device,
607
+ attention_mask=attention_mask,
608
+ )
609
+
610
+ for block in self.blocks:
611
+ if self.gradient_checkpointing and self.training:
612
+ x, attn_weights, s_max = self._gradient_checkpointing_func(
613
+ block.__call__,
614
+ x=x,
615
+ attention_mask_2d=attention_mask_2d,
616
+ attention_mask_4d=attention_mask_4d,
617
+ flex_block_mask=flex_block_mask,
618
+ output_attentions=output_attentions,
619
+ output_s_max=output_s_max,
620
+ )
621
+ else:
622
+ x, attn_weights, s_max = block(
623
+ x=x,
624
+ attention_mask_2d=attention_mask_2d,
625
+ attention_mask_4d=attention_mask_4d,
626
+ flex_block_mask=flex_block_mask,
627
+ output_attentions=output_attentions,
628
+ output_s_max=output_s_max,
629
+ )
630
+
631
+ if attentions is not None:
632
+ attentions += (attn_weights,)
633
+ if output_hidden_states:
634
+ assert hidden_states is not None
635
+ hidden_states += (x,)
636
+ if full_s_max is not None:
637
+ full_s_max += (s_max,)
638
+
639
+ last_hidden_state = self.norm(x)
640
+ if output_hidden_states:
641
+ hidden_states += (last_hidden_state,)
642
+
643
+ return TransformerOutput(
644
+ last_hidden_state=last_hidden_state,
645
+ hidden_states=hidden_states,
646
+ attentions=attentions,
647
+ s_max=full_s_max,
648
+ )
649
+
650
+
651
+ class PreTrainedESMplusplusModel(PreTrainedModel):
652
+ """
653
+ init weights for ESM++ models
654
+ """
655
+ config_class = ESMplusplusConfig
656
+ base_model_prefix = "esm++"
657
+ supports_gradient_checkpointing = True
658
+ all_tied_weights_keys = {}
659
+
660
+ @classmethod
661
+ def is_remote_code(cls) -> bool:
662
+ # Prevent post-load reinitialization of tensors already loaded from checkpoints.
663
+ return True
664
+
665
+ def _init_weights(self, module):
666
+ """Initialize the weights"""
667
+ # HF from_pretrained marks loaded parameters with `_is_hf_initialized`.
668
+ # Skip this module if any local parameter is already marked as loaded.
669
+ for parameter in module.parameters(recurse=False):
670
+ if "_is_hf_initialized" in parameter.__dict__ and parameter.__dict__["_is_hf_initialized"]:
671
+ return
672
+
673
+ if isinstance(module, nn.Linear):
674
+ nn.init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
675
+ if module.bias is not None:
676
+ nn.init.zeros_(module.bias)
677
+ elif isinstance(module, nn.Embedding):
678
+ nn.init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
679
+ if module.padding_idx is not None:
680
+ with torch.no_grad():
681
+ module.weight[module.padding_idx].zero_()
682
+ elif isinstance(module, nn.LayerNorm):
683
+ if module.bias is not None:
684
+ nn.init.zeros_(module.bias)
685
+ nn.init.ones_(module.weight)
686
+
687
+ @property
688
+ def attn_backend(self) -> str:
689
+ return self.config.attn_backend
690
+
691
+ @attn_backend.setter
692
+ def attn_backend(self, backend: str) -> None:
693
+ assert backend in VALID_ATTENTION_BACKENDS, f"Unsupported attn_backend: {backend}. Expected one of {VALID_ATTENTION_BACKENDS}."
694
+ self.config.attn_backend = backend
695
+ for module in self.modules():
696
+ if isinstance(module, TransformerStack):
697
+ module.attn_backend = backend
698
+
699
+ def _reset_rotary_embeddings(self):
700
+ """Refresh non-persistent rotary buffers after checkpoint loading."""
701
+ for module in self.modules():
702
+ if isinstance(module, RotaryEmbedding):
703
+ module.reset_parameters()
704
+
705
+ @classmethod
706
+ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
707
+ output_loading_info = bool(kwargs["output_loading_info"]) if "output_loading_info" in kwargs else False
708
+ loaded = super().from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
709
+ if output_loading_info:
710
+ model, loading_info = loaded
711
+ model._reset_rotary_embeddings()
712
+ return model, loading_info
713
+ loaded._reset_rotary_embeddings()
714
+ return loaded
715
+
716
+ @classmethod
717
+ def from_pretrained_esm(
718
+ cls,
719
+ model_name: str,
720
+ device: Union[torch.device, str] = "cpu",
721
+ ):
722
+ """Load a pretrained ESM++ model."""
723
+ key = _resolve_esmc_checkpoint_key(model_name)
724
+ if key == "esmc-300":
725
+ return ESMplusplus_300M(device=device)
726
+ if key == "esmc-600":
727
+ return ESMplusplus_600M(device=device)
728
+ if key == "esmc-6b":
729
+ return ESMplusplus_6B(device=device)
730
+ raise ValueError(f"Invalid model name: {model_name}")
731
+
732
+
733
+ ### ESM++ Models
734
+ class ESMplusplusModel(PreTrainedESMplusplusModel, EmbeddingMixin):
735
+ """
736
+ ESM++ model. transformer model with no heads
737
+ """
738
+ config_class = ESMplusplusConfig
739
+ def __init__(self, config: ESMplusplusConfig, **kwargs):
740
+ PreTrainedESMplusplusModel.__init__(self, config, **kwargs)
741
+ self.config = config
742
+ self.vocab_size = config.vocab_size
743
+ self.embed = nn.Embedding(self.vocab_size, config.hidden_size)
744
+ self.transformer = TransformerStack(
745
+ d_model=config.hidden_size,
746
+ n_heads=config.num_attention_heads,
747
+ n_layers=config.num_hidden_layers,
748
+ dropout=config.dropout,
749
+ attn_backend=config.attn_backend,
750
+ )
751
+ self.tokenizer = EsmSequenceTokenizer()
752
+ self.init_weights()
753
+
754
+ def get_input_embeddings(self):
755
+ return self.embed
756
+
757
+ def set_input_embeddings(self, value):
758
+ self.embed = value
759
+
760
+ def _embed(self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
761
+ x = self.embed(input_ids)
762
+ return self.transformer(
763
+ x=x,
764
+ attention_mask=attention_mask,
765
+ output_hidden_states=False,
766
+ output_attentions=False,
767
+ ).last_hidden_state
768
+
769
+ def forward(
770
+ self,
771
+ input_ids: Optional[torch.Tensor] = None,
772
+ attention_mask: Optional[torch.Tensor] = None,
773
+ inputs_embeds: Optional[torch.Tensor] = None,
774
+ output_attentions: Optional[bool] = None,
775
+ output_hidden_states: Optional[bool] = None,
776
+ output_s_max: Optional[bool] = False,
777
+ return_dict: Optional[bool] = None,
778
+ **kwargs,
779
+ ) -> ESMplusplusOutput:
780
+ assert input_ids is not None or inputs_embeds is not None, "You have to specify either input_ids or inputs_embeds"
781
+ assert not (input_ids is not None and inputs_embeds is not None), "You cannot specify both input_ids and inputs_embeds at the same time"
782
+
783
+ if inputs_embeds is None:
784
+ x = self.embed(input_ids)
785
+ else:
786
+ x = inputs_embeds
787
+
788
+ transformer_output = self.transformer(
789
+ x=x,
790
+ attention_mask=attention_mask,
791
+ output_hidden_states=output_hidden_states,
792
+ output_attentions=output_attentions,
793
+ output_s_max=output_s_max,
794
+ )
795
+ return ESMplusplusOutput(
796
+ last_hidden_state=transformer_output.last_hidden_state,
797
+ hidden_states=transformer_output.hidden_states,
798
+ attentions=transformer_output.attentions,
799
+ s_max=transformer_output.s_max,
800
+ )
801
+
802
+ class ESMplusplusForMaskedLM(PreTrainedESMplusplusModel, EmbeddingMixin):
803
+ """
804
+ ESM++ model for masked language modeling.
805
+ Implements the base ESM++ architecture with a masked language modeling head.
806
+ """
807
+ config_class = ESMplusplusConfig
808
+ def __init__(self, config: ESMplusplusConfig, **kwargs):
809
+ PreTrainedESMplusplusModel.__init__(self, config, **kwargs)
810
+ self.config = config
811
+ self.vocab_size = config.vocab_size
812
+ self.embed = nn.Embedding(self.vocab_size, config.hidden_size)
813
+ self.transformer = TransformerStack(
814
+ d_model=config.hidden_size,
815
+ n_heads=config.num_attention_heads,
816
+ n_layers=config.num_hidden_layers,
817
+ dropout=config.dropout,
818
+ attn_backend=config.attn_backend,
819
+ )
820
+ self.sequence_head = RegressionHead(config.hidden_size, self.vocab_size)
821
+ self.ce_loss = nn.CrossEntropyLoss()
822
+ self.tokenizer = EsmSequenceTokenizer()
823
+ self.init_weights()
824
+
825
+ def get_input_embeddings(self):
826
+ return self.embed
827
+
828
+ def set_input_embeddings(self, value):
829
+ self.embed = value
830
+
831
+ def get_output_embeddings(self):
832
+ return self.sequence_head[-1]
833
+
834
+ def set_output_embeddings(self, new_embeddings):
835
+ self.sequence_head[-1] = new_embeddings
836
+
837
+ def _embed(self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
838
+ x = self.embed(input_ids)
839
+ return self.transformer(
840
+ x=x,
841
+ attention_mask=attention_mask,
842
+ output_hidden_states=False,
843
+ output_attentions=False,
844
+ ).last_hidden_state
845
+
846
+ def forward(
847
+ self,
848
+ input_ids: Optional[torch.Tensor] = None,
849
+ attention_mask: Optional[torch.Tensor] = None,
850
+ inputs_embeds: Optional[torch.Tensor] = None,
851
+ labels: Optional[torch.Tensor] = None,
852
+ output_attentions: Optional[bool] = None,
853
+ output_hidden_states: Optional[bool] = None,
854
+ output_s_max: Optional[bool] = False,
855
+ return_dict: Optional[bool] = None,
856
+ **kwargs,
857
+ ) -> ESMplusplusOutput:
858
+ if inputs_embeds is None:
859
+ x = self.embed(input_ids)
860
+ else:
861
+ x = inputs_embeds
862
+
863
+ output = self.transformer(
864
+ x=x,
865
+ attention_mask=attention_mask,
866
+ output_hidden_states=output_hidden_states,
867
+ output_attentions=output_attentions,
868
+ output_s_max=output_s_max,
869
+ )
870
+
871
+ last_hidden_state = output.last_hidden_state
872
+ logits = self.sequence_head(last_hidden_state)
873
+ loss = None
874
+ if labels is not None:
875
+ loss = self.ce_loss(logits.view(-1, self.vocab_size), labels.view(-1))
876
+
877
+ return ESMplusplusOutput(
878
+ loss=loss,
879
+ logits=logits,
880
+ last_hidden_state=last_hidden_state,
881
+ hidden_states=output.hidden_states,
882
+ attentions=output.attentions,
883
+ s_max=output.s_max,
884
+ )
885
+
886
+
887
+ class ESMplusplusForSequenceClassification(ESMplusplusForMaskedLM, EmbeddingMixin):
888
+ """
889
+ ESM++ model for sequence classification.
890
+ Extends the base ESM++ model with a classification head.
891
+ """
892
+ def __init__(self, config: ESMplusplusConfig, **kwargs):
893
+ ESMplusplusForMaskedLM.__init__(self, config, **kwargs)
894
+ self.config = config
895
+ self.num_labels = config.num_labels
896
+ self.classifier = RegressionHead(config.hidden_size * 2, config.num_labels, config.hidden_size * 4)
897
+ # Large intermediate projections help with sequence classification tasks (*4)
898
+ self.mse = nn.MSELoss()
899
+ self.ce = nn.CrossEntropyLoss()
900
+ self.bce = nn.BCEWithLogitsLoss()
901
+ # if kwargs has pooling_types, use them, otherwise use ['cls', 'mean']
902
+ if 'pooling_types' in kwargs and isinstance(kwargs['pooling_types'], List[str]) and len(kwargs['pooling_types']) > 0:
903
+ pooling_types = kwargs['pooling_types']
904
+ else:
905
+ pooling_types = ['mean', 'var']
906
+ self.pooler = Pooler(pooling_types)
907
+ self.init_weights()
908
+
909
+ def _embed(self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
910
+ x = self.embed(input_ids)
911
+ return self.transformer(
912
+ x=x,
913
+ attention_mask=attention_mask,
914
+ output_hidden_states=False,
915
+ output_attentions=False,
916
+ ).last_hidden_state
917
+
918
+ def forward(
919
+ self,
920
+ input_ids: Optional[torch.Tensor] = None,
921
+ attention_mask: Optional[torch.Tensor] = None,
922
+ inputs_embeds: Optional[torch.Tensor] = None,
923
+ labels: Optional[torch.Tensor] = None,
924
+ output_attentions: Optional[bool] = None,
925
+ output_hidden_states: Optional[bool] = None,
926
+ output_s_max: Optional[bool] = False,
927
+ return_dict: Optional[bool] = None,
928
+ **kwargs,
929
+ ) -> ESMplusplusOutput:
930
+ output = super().forward(
931
+ input_ids=input_ids,
932
+ attention_mask=attention_mask,
933
+ inputs_embeds=inputs_embeds,
934
+ labels=None,
935
+ output_attentions=output_attentions,
936
+ output_hidden_states=output_hidden_states,
937
+ output_s_max=output_s_max,
938
+ )
939
+
940
+ last_hidden_state = output.last_hidden_state
941
+ features = self.pooler(last_hidden_state, attention_mask)
942
+ logits = self.classifier(features)
943
+
944
+ loss = None
945
+ if labels is not None:
946
+ labels = labels.to(logits.device)
947
+ if self.config.problem_type is None:
948
+ if self.num_labels == 1:
949
+ self.config.problem_type = "regression"
950
+ elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
951
+ self.config.problem_type = "single_label_classification"
952
+ else:
953
+ self.config.problem_type = "multi_label_classification"
954
+
955
+ if self.config.problem_type == "regression":
956
+ if self.num_labels == 1:
957
+ loss = self.mse(logits.flatten(), labels.flatten())
958
+ else:
959
+ loss = self.mse(logits, labels)
960
+ elif self.config.problem_type == "single_label_classification":
961
+ loss = self.ce(logits.view(-1, self.num_labels), labels.view(-1))
962
+ elif self.config.problem_type == "multi_label_classification":
963
+ loss = self.bce(logits, labels)
964
+
965
+ return ESMplusplusOutput(
966
+ loss=loss,
967
+ logits=logits,
968
+ last_hidden_state=last_hidden_state,
969
+ hidden_states=output.hidden_states,
970
+ attentions=output.attentions,
971
+ s_max=output.s_max,
972
+ )
973
+
974
+
975
+ class ESMplusplusForTokenClassification(ESMplusplusForMaskedLM, EmbeddingMixin):
976
+ """
977
+ ESM++ model for token classification.
978
+ Extends the base ESM++ model with a token classification head.
979
+ """
980
+ def __init__(self, config: ESMplusplusConfig, **kwargs):
981
+ ESMplusplusForMaskedLM.__init__(self, config, **kwargs)
982
+ self.config = config
983
+ self.num_labels = config.num_labels
984
+ self.classifier = RegressionHead(config.hidden_size, config.num_labels, config.hidden_size * 4)
985
+ # Large intermediate projections help with sequence classification tasks (*4)
986
+ self.loss_fct = nn.CrossEntropyLoss()
987
+ self.init_weights()
988
+
989
+ def _embed(self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
990
+ x = self.embed(input_ids)
991
+ return self.transformer(x, attention_mask, output_hidden_states=False, output_attentions=False).last_hidden_state
992
+
993
+ def forward(
994
+ self,
995
+ input_ids: Optional[torch.Tensor] = None,
996
+ attention_mask: Optional[torch.Tensor] = None,
997
+ inputs_embeds: Optional[torch.Tensor] = None,
998
+ labels: Optional[torch.Tensor] = None,
999
+ output_attentions: Optional[bool] = None,
1000
+ output_hidden_states: Optional[bool] = None,
1001
+ output_s_max: Optional[bool] = False,
1002
+ return_dict: Optional[bool] = None,
1003
+ **kwargs,
1004
+ ) -> ESMplusplusOutput:
1005
+ output = super().forward(
1006
+ input_ids=input_ids,
1007
+ attention_mask=attention_mask,
1008
+ inputs_embeds=inputs_embeds,
1009
+ labels=None,
1010
+ output_attentions=output_attentions,
1011
+ output_hidden_states=output_hidden_states,
1012
+ output_s_max=output_s_max,
1013
+ )
1014
+
1015
+ last_hidden_state = output.last_hidden_state
1016
+ logits = self.classifier(last_hidden_state)
1017
+ loss = None
1018
+ if labels is not None:
1019
+ loss = self.loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
1020
+
1021
+ return ESMplusplusOutput(
1022
+ loss=loss,
1023
+ logits=logits,
1024
+ last_hidden_state=last_hidden_state,
1025
+ hidden_states=output.hidden_states,
1026
+ attentions=output.attentions,
1027
+ s_max=output.s_max,
1028
+ )
1029
+
1030
+
1031
+ ### Loading from Biohub
1032
+ _ESMC_CHECKPOINT_SPECS = {
1033
+ "esmc-300": {
1034
+ "repo_id": "biohub/ESMC-300M",
1035
+ "hidden_size": 960,
1036
+ "num_attention_heads": 15,
1037
+ "num_hidden_layers": 30,
1038
+ },
1039
+ "esmc-600": {
1040
+ "repo_id": "biohub/ESMC-600M",
1041
+ "hidden_size": 1152,
1042
+ "num_attention_heads": 18,
1043
+ "num_hidden_layers": 36,
1044
+ },
1045
+ "esmc-6b": {
1046
+ "repo_id": "biohub/ESMC-6B",
1047
+ "hidden_size": 2560,
1048
+ "num_attention_heads": 40,
1049
+ "num_hidden_layers": 80,
1050
+ },
1051
+ }
1052
+
1053
+
1054
+ def _resolve_esmc_checkpoint_key(model: str) -> str:
1055
+ normalized = model.lower().replace("_", "-")
1056
+ if "300" in normalized:
1057
+ return "esmc-300"
1058
+ if "600" in normalized:
1059
+ return "esmc-600"
1060
+ if "6b" in normalized:
1061
+ return "esmc-6b"
1062
+ raise ValueError(f"{model=} is an invalid ESMC model name.")
1063
+
1064
+
1065
+ @staticmethod
1066
+ @cache
1067
+ def data_root(model: str):
1068
+ if "INFRA_PROVIDER" in os.environ:
1069
+ return Path("")
1070
+ key = _resolve_esmc_checkpoint_key(model)
1071
+ return Path(snapshot_download(repo_id=_ESMC_CHECKPOINT_SPECS[key]["repo_id"]))
1072
+
1073
+
1074
+ def get_esmc_checkpoint_path(model: str) -> Path:
1075
+ key = _resolve_esmc_checkpoint_key(model)
1076
+ spec = _ESMC_CHECKPOINT_SPECS[key]
1077
+ if "weights_relpath" in spec:
1078
+ return data_root(key) / spec["weights_relpath"]
1079
+ checkpoint_dir = data_root(key)
1080
+ if (checkpoint_dir / "model.safetensors").exists():
1081
+ return checkpoint_dir / "model.safetensors"
1082
+ if (checkpoint_dir / "model.safetensors.index.json").exists():
1083
+ return checkpoint_dir / "model.safetensors.index.json"
1084
+ raise FileNotFoundError(f"No ESMC checkpoint found under {checkpoint_dir}.")
1085
+
1086
+
1087
+ def _normalize_esmc_state_key(key: str) -> Optional[str]:
1088
+ if key.endswith("._extra_state"):
1089
+ return None
1090
+ if key.startswith("esmc."):
1091
+ key = key[len("esmc."):]
1092
+ if key.startswith("lm_head."):
1093
+ key = f"sequence_head.{key[len('lm_head.'):]}"
1094
+ replacements = (
1095
+ (".attn.layernorm_qkv.layer_norm_bias", ".attn.layernorm_qkv.0.bias"),
1096
+ (".attn.layernorm_qkv.layer_norm_weight", ".attn.layernorm_qkv.0.weight"),
1097
+ (".attn.layernorm_qkv.weight", ".attn.layernorm_qkv.1.weight"),
1098
+ (".ffn.layer_norm_bias", ".ffn.0.bias"),
1099
+ (".ffn.layer_norm_weight", ".ffn.0.weight"),
1100
+ (".ffn.fc1_weight", ".ffn.1.weight"),
1101
+ (".ffn.fc2_weight", ".ffn.3.weight"),
1102
+ )
1103
+ for old, new in replacements:
1104
+ key = key.replace(old, new)
1105
+ return key
1106
+
1107
+
1108
+ def _normalize_esmc_state_dict(state_dict: dict) -> dict:
1109
+ normalized = {}
1110
+ for key, tensor in state_dict.items():
1111
+ normalized_key = _normalize_esmc_state_key(key)
1112
+ if normalized_key is None:
1113
+ continue
1114
+ normalized[normalized_key] = tensor
1115
+ return normalized
1116
+
1117
+
1118
+ def _safetensors_checkpoint_files(checkpoint_path: Path) -> List[Path]:
1119
+ if checkpoint_path.name == "model.safetensors":
1120
+ return [checkpoint_path]
1121
+ with checkpoint_path.open("r", encoding="utf-8") as f:
1122
+ index = json.load(f)
1123
+ return [
1124
+ checkpoint_path.parent / filename
1125
+ for filename in sorted(set(index["weight_map"].values()))
1126
+ ]
1127
+
1128
+
1129
+ def _load_safetensors_state_dict(
1130
+ model_obj: ESMplusplusForMaskedLM,
1131
+ checkpoint_path: Path,
1132
+ device: Union[torch.device, str],
1133
+ ) -> None:
1134
+ expected_keys = set(model_obj.state_dict().keys())
1135
+ loaded_keys = set()
1136
+ device_string = str(torch.device(device))
1137
+ for shard_path in _safetensors_checkpoint_files(checkpoint_path):
1138
+ shard_state_dict = load_safetensors_file(shard_path, device=device_string)
1139
+ normalized = _normalize_esmc_state_dict(shard_state_dict)
1140
+ unexpected = set(normalized.keys()) - expected_keys
1141
+ assert len(unexpected) == 0, (
1142
+ f"Unexpected ESMC checkpoint keys in {shard_path.name}: "
1143
+ f"{sorted(unexpected)[:10]}"
1144
+ )
1145
+ model_obj.load_state_dict(normalized, strict=False)
1146
+ loaded_keys.update(normalized.keys())
1147
+
1148
+ missing = expected_keys - loaded_keys
1149
+ assert len(missing) == 0, (
1150
+ f"ESMC checkpoint did not provide all expected keys: {sorted(missing)[:10]}"
1151
+ )
1152
+
1153
+
1154
+ def _load_esmc_checkpoint_model(
1155
+ config: ESMplusplusConfig,
1156
+ model: str,
1157
+ device: Union[torch.device, str] = "cpu",
1158
+ ) -> ESMplusplusForMaskedLM:
1159
+ key = _resolve_esmc_checkpoint_key(model)
1160
+ spec = _ESMC_CHECKPOINT_SPECS[key]
1161
+ assert config.hidden_size == spec["hidden_size"], (
1162
+ f"ESMC loader expected hidden_size={spec['hidden_size']} for {key}, "
1163
+ f"but got {config.hidden_size}."
1164
+ )
1165
+ assert config.num_attention_heads == spec["num_attention_heads"], (
1166
+ f"ESMC loader expected num_attention_heads={spec['num_attention_heads']} for {key}, "
1167
+ f"but got {config.num_attention_heads}."
1168
+ )
1169
+ assert config.num_hidden_layers == spec["num_hidden_layers"], (
1170
+ f"ESMC loader expected num_hidden_layers={spec['num_hidden_layers']} for {key}, "
1171
+ f"but got {config.num_hidden_layers}."
1172
+ )
1173
+ with torch.device(device):
1174
+ model_obj = ESMplusplusForMaskedLM(config)
1175
+ checkpoint_path = get_esmc_checkpoint_path(key)
1176
+ if checkpoint_path.suffix == ".safetensors" or checkpoint_path.name == "model.safetensors.index.json":
1177
+ _load_safetensors_state_dict(
1178
+ model_obj=model_obj,
1179
+ checkpoint_path=checkpoint_path,
1180
+ device=device,
1181
+ )
1182
+ else:
1183
+ state_dict = torch.load(checkpoint_path, map_location=device)
1184
+ model_obj.load_state_dict(_normalize_esmc_state_dict(state_dict))
1185
+ return model_obj
1186
+
1187
+
1188
+ def ESMplusplus_300M(device: Union[torch.device, str] = "cpu"):
1189
+ config = ESMplusplusConfig(
1190
+ hidden_size=960,
1191
+ num_attention_heads=15,
1192
+ num_hidden_layers=30,
1193
+ )
1194
+ return _load_esmc_checkpoint_model(config=config, model="esmc-300", device=device)
1195
+
1196
+
1197
+ def ESMplusplus_600M(device: Union[torch.device, str] = "cpu"):
1198
+ config = ESMplusplusConfig(
1199
+ hidden_size=1152,
1200
+ num_attention_heads=18,
1201
+ num_hidden_layers=36,
1202
+ )
1203
+ return _load_esmc_checkpoint_model(config=config, model="esmc-600", device=device)
1204
+
1205
+
1206
+ def ESMplusplus_6B(device: Union[torch.device, str] = "cpu"):
1207
+ config = ESMplusplusConfig(
1208
+ hidden_size=2560,
1209
+ num_attention_heads=40,
1210
+ num_hidden_layers=80,
1211
+ )
1212
+ return _load_esmc_checkpoint_model(config=config, model="esmc-6b", device=device)
1213
+
1214
+
1215
+ ### Tokenization
1216
+ SEQUENCE_VOCAB = [
1217
+ "<cls>", "<pad>", "<eos>", "<unk>",
1218
+ "L", "A", "G", "V", "S", "E", "R", "T", "I", "D", "P", "K",
1219
+ "Q", "N", "F", "Y", "M", "H", "W", "C", "X", "B", "U", "Z",
1220
+ "O", ".", "-", "|",
1221
+ "<mask>",
1222
+ ]
1223
+
1224
+ class EsmSequenceTokenizer(PreTrainedTokenizerFast):
1225
+ model_input_names = ["input_ids", "attention_mask"]
1226
+
1227
+ def __init__(
1228
+ self,
1229
+ unk_token="<unk>",
1230
+ cls_token="<cls>",
1231
+ pad_token="<pad>",
1232
+ mask_token="<mask>",
1233
+ eos_token="<eos>",
1234
+ chain_break_token="|",
1235
+ **kwargs,
1236
+ ):
1237
+ all_tokens = SEQUENCE_VOCAB
1238
+ token_to_id = {tok: ind for ind, tok in enumerate(all_tokens)}
1239
+
1240
+ # a character-level tokenizer is the same as BPE with no token merges
1241
+ bpe = BPE(token_to_id, merges=[], unk_token=unk_token)
1242
+ tokenizer = Tokenizer(bpe)
1243
+ special_tokens = [
1244
+ cls_token,
1245
+ pad_token,
1246
+ mask_token,
1247
+ eos_token,
1248
+ chain_break_token,
1249
+ ]
1250
+ self.cb_token = chain_break_token
1251
+ additional_special_tokens = [chain_break_token]
1252
+
1253
+ tokenizer.add_special_tokens(special_tokens)
1254
+
1255
+ # This is where we configure the automatic addition of special tokens when we call
1256
+ # tokenizer(text, add_special_tokens=True). Note that you can also configure how two
1257
+ # sequences are merged if you want.
1258
+ tokenizer.post_processor = TemplateProcessing( # type: ignore
1259
+ single="<cls> $A <eos>",
1260
+ pair="<cls>:0 $A:0 <eos>:0 $B:1 <eos>:1",
1261
+ special_tokens=[
1262
+ ("<cls>", tokenizer.token_to_id("<cls>")),
1263
+ ("<eos>", tokenizer.token_to_id("<eos>")),
1264
+ ],
1265
+ )
1266
+ super().__init__(
1267
+ tokenizer_object=tokenizer,
1268
+ unk_token=unk_token,
1269
+ cls_token=cls_token,
1270
+ pad_token=pad_token,
1271
+ mask_token=mask_token,
1272
+ eos_token=eos_token,
1273
+ additional_special_tokens=additional_special_tokens,
1274
+ **kwargs,
1275
+ )
1276
+
1277
+ # These are a footgun, we never use the `bos` token anywhere so we're just overriding it here.
1278
+ @property
1279
+ def bos_token(self):
1280
+ return self.cls_token
1281
+
1282
+ @property
1283
+ def bos_token_id(self):
1284
+ return self.cls_token_id
1285
+
1286
+ @property
1287
+ def chain_break_token(self):
1288
+ return self.cb_token
1289
+
1290
+ @property
1291
+ def chain_break_token_id(self):
1292
+ return self.convert_tokens_to_ids(self.chain_break_token)
1293
+
1294
+ @property
1295
+ def all_token_ids(self):
1296
+ return list(range(self.vocab_size))
1297
+
1298
+ @property
1299
+ def special_token_ids(self):
1300
+ return self.all_special_ids
1301
+
1302
+
1303
+ if __name__ == "__main__":
1304
+ import random
1305
+
1306
+ import torch
1307
+
1308
+ from torch import Tensor
1309
+
1310
+ def print_tensor_shapes(prefix: str, obj):
1311
+ if isinstance(obj, Tensor):
1312
+ print(f"{prefix}{obj.shape}")
1313
+ elif isinstance(obj, dict):
1314
+ for name, value in obj.items():
1315
+ print_tensor_shapes(f"{prefix}{name}.", value)
1316
+ elif isinstance(obj, list):
1317
+ for idx, value in enumerate(obj):
1318
+ print_tensor_shapes(f"{prefix}[{idx}].", value)
1319
+ elif isinstance(obj, tuple):
1320
+ for idx, value in enumerate(obj):
1321
+ print_tensor_shapes(f"{prefix}[{idx}].", value)
1322
+ elif hasattr(obj, "__dict__"):
1323
+ for name, value in vars(obj).items():
1324
+ if name.startswith("_"):
1325
+ continue
1326
+ print_tensor_shapes(f"{prefix}{name}.", value)
1327
+ else:
1328
+ print(f"{prefix}{type(obj)}")
1329
+
1330
+ random.seed(0)
1331
+ torch.manual_seed(0)
1332
+
1333
+ tokenizer = EsmSequenceTokenizer()
1334
+ num_attention_heads = random.choice([2, 4])
1335
+ config = ESMplusplusConfig(
1336
+ vocab_size=tokenizer.vocab_size,
1337
+ hidden_size=16 * num_attention_heads,
1338
+ num_attention_heads=num_attention_heads,
1339
+ num_hidden_layers=random.choice([1, 2]),
1340
+ num_labels=2,
1341
+ dropout=0.0,
1342
+ )
1343
+
1344
+ batch = tokenizer(["ACDEFG", "MKTW"], return_tensors="pt", padding=True)
1345
+ batch["labels"] = batch["input_ids"].clone()
1346
+ model = ESMplusplusForMaskedLM(config=config).eval()
1347
+
1348
+ with torch.no_grad():
1349
+ output = model(**batch, return_dict=True)
1350
+
1351
+ print("Batch shape:")
1352
+ print_tensor_shapes("", batch)
1353
+ print("Output shape:")
1354
+ print_tensor_shapes("", output)