Add files using upload-large-folder tool

model/esm/esmfold/v1/esmfold.py

@@ -0,0 +1,364 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+import typing as T
+from dataclasses import dataclass, field
+from functools import partial
+
+import torch
+import torch.nn as nn
+from torch import nn
+from torch.nn import LayerNorm
+
+import model.esm as esm
+from model.esm import Alphabet
+from model.esm.esmfold.v1.categorical_mixture import categorical_lddt
+from model.esm.esmfold.v1.misc import (
+    batch_encode_sequences,
+    collate_dense_tensors,
+    output_to_pdb,
+)
+from model.esm.esmfold.v1.trunk import FoldingTrunk, FoldingTrunkConfig
+from onescience.datapipes.openfold.data_transforms import make_atom14_masks
+from onescience.utils.openfold.np import residue_constants
+from onescience.utils.openfold.loss import compute_predicted_aligned_error, compute_tm
+
+
+@dataclass
+class ESMFoldConfig:
+    trunk: T.Any = field(default_factory=FoldingTrunkConfig)
+    lddt_head_hid_dim: int = 128
+
+
+load_fn = esm.pretrained.load_model_and_alphabet
+esm_registry = {
+    "esm2_8M": partial(load_fn, "esm2_t6_8M_UR50D_500K"),
+    "esm2_8M_270K": esm.pretrained.esm2_t6_8M_UR50D,
+    "esm2_35M": partial(load_fn, "esm2_t12_35M_UR50D_500K"),
+    "esm2_35M_270K": esm.pretrained.esm2_t12_35M_UR50D,
+    "esm2_150M": partial(load_fn, "esm2_t30_150M_UR50D_500K"),
+    "esm2_150M_270K": partial(load_fn, "esm2_t30_150M_UR50D_270K"),
+    "esm2_650M": esm.pretrained.esm2_t33_650M_UR50D,
+    "esm2_650M_270K": partial(load_fn, "esm2_t33_650M_270K_UR50D"),
+    "esm2_3B": esm.pretrained.esm2_t36_3B_UR50D,
+    "esm2_3B_270K": partial(load_fn, "esm2_t36_3B_UR50D_500K"),
+    "esm2_15B": esm.pretrained.esm2_t48_15B_UR50D,
+}
+
+
+class ESMFold(nn.Module):
+    def __init__(self, esmfold_config=None, **kwargs):
+        super().__init__()
+
+        self.cfg = esmfold_config if esmfold_config else ESMFoldConfig(**kwargs)
+        cfg = self.cfg
+
+        self.distogram_bins = 64
+
+        self.esm, self.esm_dict = esm_registry.get(cfg.esm_type)()
+
+        self.esm.requires_grad_(False)
+        self.esm.half()
+
+        self.esm_feats = self.esm.embed_dim
+        self.esm_attns = self.esm.num_layers * self.esm.attention_heads
+        self.register_buffer("af2_to_esm", ESMFold._af2_to_esm(self.esm_dict))
+        self.esm_s_combine = nn.Parameter(torch.zeros(self.esm.num_layers + 1))
+
+        c_s = cfg.trunk.sequence_state_dim
+        c_z = cfg.trunk.pairwise_state_dim
+
+        self.esm_s_mlp = nn.Sequential(
+            LayerNorm(self.esm_feats),
+            nn.Linear(self.esm_feats, c_s),
+            nn.ReLU(),
+            nn.Linear(c_s, c_s),
+        )
+        if cfg.use_esm_attn_map:
+            self.esm_z_mlp = nn.Sequential(
+                LayerNorm(self.esm_attns),
+                nn.Linear(self.esm_attns, c_z),
+                nn.ReLU(),
+                nn.Linear(c_z, c_z),
+            )
+
+        # 0 is padding, N is unknown residues, N + 1 is mask.
+        self.n_tokens_embed = residue_constants.restype_num + 3
+        self.pad_idx = 0
+        self.unk_idx = self.n_tokens_embed - 2
+        self.mask_idx = self.n_tokens_embed - 1
+        self.embedding = nn.Embedding(self.n_tokens_embed, c_s, padding_idx=0)
+
+        self.trunk = FoldingTrunk(**cfg.trunk)
+
+        self.distogram_head = nn.Linear(c_z, self.distogram_bins)
+        self.ptm_head = nn.Linear(c_z, self.distogram_bins)
+        self.lm_head = nn.Linear(c_s, self.n_tokens_embed)
+        self.lddt_bins = 50
+        self.lddt_head = nn.Sequential(
+            nn.LayerNorm(cfg.trunk.structure_module.c_s),
+            nn.Linear(cfg.trunk.structure_module.c_s, cfg.lddt_head_hid_dim),
+            nn.Linear(cfg.lddt_head_hid_dim, cfg.lddt_head_hid_dim),
+            nn.Linear(cfg.lddt_head_hid_dim, 37 * self.lddt_bins),
+        )
+
+    @staticmethod
+    def _af2_to_esm(d: Alphabet):
+        # Remember that t is shifted from residue_constants by 1 (0 is padding).
+        esm_reorder = [d.padding_idx] + [
+            d.get_idx(v) for v in residue_constants.restypes_with_x
+        ]
+        return torch.tensor(esm_reorder)
+
+    def _af2_idx_to_esm_idx(self, aa, mask):
+        aa = (aa + 1).masked_fill(mask != 1, 0)
+        return self.af2_to_esm[aa]
+
+    def _compute_language_model_representations(
+        self, esmaa: torch.Tensor
+    ) -> torch.Tensor:
+        """Adds bos/eos tokens for the language model, since the structure module doesn't use these."""
+        batch_size = esmaa.size(0)
+
+        bosi, eosi = self.esm_dict.cls_idx, self.esm_dict.eos_idx
+        bos = esmaa.new_full((batch_size, 1), bosi)
+        eos = esmaa.new_full((batch_size, 1), self.esm_dict.padding_idx)
+        esmaa = torch.cat([bos, esmaa, eos], dim=1)
+        # Use the first padding index as eos during inference.
+        esmaa[range(batch_size), (esmaa != 1).sum(1)] = eosi
+
+        res = self.esm(
+            esmaa,
+            repr_layers=range(self.esm.num_layers + 1),
+            need_head_weights=self.cfg.use_esm_attn_map,
+        )
+        esm_s = torch.stack(
+            [v for _, v in sorted(res["representations"].items())], dim=2
+        )
+        esm_s = esm_s[:, 1:-1]  # B, L, nLayers, C
+        esm_z = (
+            res["attentions"].permute(0, 4, 3, 1, 2).flatten(3, 4)[:, 1:-1, 1:-1, :]
+            if self.cfg.use_esm_attn_map
+            else None
+        )
+        return esm_s, esm_z
+
+    def _mask_inputs_to_esm(self, esmaa, pattern):
+        new_esmaa = esmaa.clone()
+        new_esmaa[pattern == 1] = self.esm_dict.mask_idx
+        return new_esmaa
+
+    def forward(
+        self,
+        aa: torch.Tensor,
+        mask: T.Optional[torch.Tensor] = None,
+        residx: T.Optional[torch.Tensor] = None,
+        masking_pattern: T.Optional[torch.Tensor] = None,
+        num_recycles: T.Optional[int] = None,
+    ):
+        """Runs a forward pass given input tokens. Use `model.infer` to
+        run inference from a sequence.
+
+        Args:
+            aa (torch.Tensor): Tensor containing indices corresponding to amino acids. Indices match
+                onescience.utils.openfold.np.residue_constants.restype_order_with_x.
+            mask (torch.Tensor): Binary tensor with 1 meaning position is unmasked and 0 meaning position is masked.
+            residx (torch.Tensor): Residue indices of amino acids. Will assume contiguous if not provided.
+            masking_pattern (torch.Tensor): Optional masking to pass to the input. Binary tensor of the same size
+                as `aa`. Positions with 1 will be masked. ESMFold sometimes produces different samples when
+                different masks are provided.
+            num_recycles (int): How many recycle iterations to perform. If None, defaults to training max
+                recycles, which is 3.
+        """
+
+        if mask is None:
+            mask = torch.ones_like(aa)
+
+        B = aa.shape[0]
+        L = aa.shape[1]
+        device = aa.device
+
+        if residx is None:
+            residx = torch.arange(L, device=device).expand_as(aa)
+
+        # === ESM ===
+        esmaa = self._af2_idx_to_esm_idx(aa, mask)
+
+        if masking_pattern is not None:
+            esmaa = self._mask_inputs_to_esm(esmaa, masking_pattern)
+
+        esm_s, esm_z = self._compute_language_model_representations(esmaa)
+
+        # Convert esm_s to the precision used by the trunk and
+        # the structure module. These tensors may be a lower precision if, for example,
+        # we're running the language model in fp16 precision.
+        esm_s = esm_s.to(self.esm_s_combine.dtype)
+        esm_s = esm_s.detach()
+
+        # === preprocessing ===
+        esm_s = (self.esm_s_combine.softmax(0).unsqueeze(0) @ esm_s).squeeze(2)
+
+        s_s_0 = self.esm_s_mlp(esm_s)
+        if self.cfg.use_esm_attn_map:
+            esm_z = esm_z.to(self.esm_s_combine.dtype)
+            esm_z = esm_z.detach()
+            s_z_0 = self.esm_z_mlp(esm_z)
+        else:
+            s_z_0 = s_s_0.new_zeros(B, L, L, self.cfg.trunk.pairwise_state_dim)
+
+        s_s_0 += self.embedding(aa)
+
+        structure: dict = self.trunk(
+            s_s_0, s_z_0, aa, residx, mask, no_recycles=num_recycles
+        )
+        # Documenting what we expect:
+        structure = {
+            k: v
+            for k, v in structure.items()
+            if k
+            in [
+                "s_z",
+                "s_s",
+                "frames",
+                "sidechain_frames",
+                "unnormalized_angles",
+                "angles",
+                "positions",
+                "states",
+            ]
+        }
+
+        disto_logits = self.distogram_head(structure["s_z"])
+        disto_logits = (disto_logits + disto_logits.transpose(1, 2)) / 2
+        structure["distogram_logits"] = disto_logits
+
+        lm_logits = self.lm_head(structure["s_s"])
+        structure["lm_logits"] = lm_logits
+
+        structure["aatype"] = aa
+        make_atom14_masks(structure)
+
+        for k in [
+            "atom14_atom_exists",
+            "atom37_atom_exists",
+        ]:
+            structure[k] *= mask.unsqueeze(-1)
+        structure["residue_index"] = residx
+
+        lddt_head = self.lddt_head(structure["states"]).reshape(
+            structure["states"].shape[0], B, L, -1, self.lddt_bins
+        )
+        structure["lddt_head"] = lddt_head
+        plddt = categorical_lddt(lddt_head[-1], bins=self.lddt_bins)
+        structure["plddt"] = (
+            100 * plddt
+        )  # we predict plDDT between 0 and 1, scale to be between 0 and 100.
+
+        ptm_logits = self.ptm_head(structure["s_z"])
+
+        seqlen = mask.type(torch.int64).sum(1)
+        structure["ptm_logits"] = ptm_logits
+        structure["ptm"] = torch.stack(
+            [
+                compute_tm(
+                    batch_ptm_logits[None, :sl, :sl],
+                    max_bins=31,
+                    no_bins=self.distogram_bins,
+                )
+                for batch_ptm_logits, sl in zip(ptm_logits, seqlen)
+            ]
+        )
+        structure.update(
+            compute_predicted_aligned_error(
+                ptm_logits, max_bin=31, no_bins=self.distogram_bins
+            )
+        )
+
+        return structure
+
+    @torch.no_grad()
+    def infer(
+        self,
+        sequences: T.Union[str, T.List[str]],
+        residx=None,
+        masking_pattern: T.Optional[torch.Tensor] = None,
+        num_recycles: T.Optional[int] = None,
+        residue_index_offset: T.Optional[int] = 512,
+        chain_linker: T.Optional[str] = "G" * 25,
+    ):
+        """Runs a forward pass given input sequences.
+
+        Args:
+            sequences (Union[str, List[str]]): A list of sequences to make predictions for. Multimers can also be passed in,
+                each chain should be separated by a ':' token (e.g. "<chain1>:<chain2>:<chain3>").
+            residx (torch.Tensor): Residue indices of amino acids. Will assume contiguous if not provided.
+            masking_pattern (torch.Tensor): Optional masking to pass to the input. Binary tensor of the same size
+                as `aa`. Positions with 1 will be masked. ESMFold sometimes produces different samples when
+                different masks are provided.
+            num_recycles (int): How many recycle iterations to perform. If None, defaults to training max
+                recycles (cfg.trunk.max_recycles), which is 4.
+            residue_index_offset (int): Residue index separation between chains if predicting a multimer. Has no effect on
+                single chain predictions. Default: 512.
+            chain_linker (str): Linker to use between chains if predicting a multimer. Has no effect on single chain
+                predictions. Default: length-25 poly-G ("G" * 25).
+        """
+        if isinstance(sequences, str):
+            sequences = [sequences]
+
+        aatype, mask, _residx, linker_mask, chain_index = batch_encode_sequences(
+            sequences, residue_index_offset, chain_linker
+        )
+
+        if residx is None:
+            residx = _residx
+        elif not isinstance(residx, torch.Tensor):
+            residx = collate_dense_tensors(residx)
+
+        aatype, mask, residx, linker_mask = map(
+            lambda x: x.to(self.device), (aatype, mask, residx, linker_mask)
+        )
+
+        output = self.forward(
+            aatype,
+            mask=mask,
+            residx=residx,
+            masking_pattern=masking_pattern,
+            num_recycles=num_recycles,
+        )
+
+        output["atom37_atom_exists"] = output[
+            "atom37_atom_exists"
+        ] * linker_mask.unsqueeze(2)
+
+        output["mean_plddt"] = (output["plddt"] * output["atom37_atom_exists"]).sum(
+            dim=(1, 2)
+        ) / output["atom37_atom_exists"].sum(dim=(1, 2))
+        output["chain_index"] = chain_index
+
+        return output
+
+    def output_to_pdb(self, output: T.Dict) -> T.List[str]:
+        """Returns the pbd (file) string from the model given the model output."""
+        return output_to_pdb(output)
+
+    def infer_pdbs(self, seqs: T.List[str], *args, **kwargs) -> T.List[str]:
+        """Returns list of pdb (files) strings from the model given a list of input sequences."""
+        output = self.infer(seqs, *args, **kwargs)
+        return self.output_to_pdb(output)
+
+    def infer_pdb(self, sequence: str, *args, **kwargs) -> str:
+        """Returns the pdb (file) string from the model given an input sequence."""
+        return self.infer_pdbs([sequence], *args, **kwargs)[0]
+
+    def set_chunk_size(self, chunk_size: T.Optional[int]):
+        # This parameter means the axial attention will be computed
+        # in a chunked manner. This should make the memory used more or less O(L) instead of O(L^2).
+        # It's equivalent to running a for loop over chunks of the dimension we're iterative over,
+        # where the chunk_size is the size of the chunks, so 128 would mean to parse 128-lengthed chunks.
+        # Setting the value to None will return to default behavior, disable chunking.
+        self.trunk.set_chunk_size(chunk_size)
+
+    @property
+    def device(self):
+        return self.esm_s_combine.device

model/esm/esmfold/v1/tri_self_attn_block.py

@@ -0,0 +1,160 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+import torch
+from model.openfold.triangular_attention import (
+    TriangleAttentionEndingNode,
+    TriangleAttentionStartingNode,
+)
+from model.openfold.triangular_multiplicative_update import (
+    TriangleMultiplicationIncoming,
+    TriangleMultiplicationOutgoing,
+)
+from torch import nn
+
+from model.esm.esmfold.v1.misc import (
+    Attention,
+    Dropout,
+    PairToSequence,
+    ResidueMLP,
+    SequenceToPair,
+)
+
+
+class TriangularSelfAttentionBlock(nn.Module):
+    def __init__(
+        self,
+        sequence_state_dim,
+        pairwise_state_dim,
+        sequence_head_width,
+        pairwise_head_width,
+        dropout=0,
+        **__kwargs,
+    ):
+        super().__init__()
+
+        assert sequence_state_dim % sequence_head_width == 0
+        assert pairwise_state_dim % pairwise_head_width == 0
+        sequence_num_heads = sequence_state_dim // sequence_head_width
+        pairwise_num_heads = pairwise_state_dim // pairwise_head_width
+        assert sequence_state_dim == sequence_num_heads * sequence_head_width
+        assert pairwise_state_dim == pairwise_num_heads * pairwise_head_width
+        assert pairwise_state_dim % 2 == 0
+
+        self.sequence_state_dim = sequence_state_dim
+        self.pairwise_state_dim = pairwise_state_dim
+
+        self.layernorm_1 = nn.LayerNorm(sequence_state_dim)
+
+        self.sequence_to_pair = SequenceToPair(
+            sequence_state_dim, pairwise_state_dim // 2, pairwise_state_dim
+        )
+        self.pair_to_sequence = PairToSequence(pairwise_state_dim, sequence_num_heads)
+
+        self.seq_attention = Attention(
+            sequence_state_dim, sequence_num_heads, sequence_head_width, gated=True
+        )
+        self.tri_mul_out = TriangleMultiplicationOutgoing(
+            pairwise_state_dim,
+            pairwise_state_dim,
+        )
+        self.tri_mul_in = TriangleMultiplicationIncoming(
+            pairwise_state_dim,
+            pairwise_state_dim,
+        )
+        self.tri_att_start = TriangleAttentionStartingNode(
+            pairwise_state_dim,
+            pairwise_head_width,
+            pairwise_num_heads,
+            inf=1e9,
+        )  # type: ignore
+        self.tri_att_end = TriangleAttentionEndingNode(
+            pairwise_state_dim,
+            pairwise_head_width,
+            pairwise_num_heads,
+            inf=1e9,
+        )  # type: ignore
+
+        self.mlp_seq = ResidueMLP(sequence_state_dim, 4 * sequence_state_dim, dropout=dropout)
+        self.mlp_pair = ResidueMLP(pairwise_state_dim, 4 * pairwise_state_dim, dropout=dropout)
+
+        assert dropout < 0.4
+        self.drop = nn.Dropout(dropout)
+        self.row_drop = Dropout(dropout * 2, 2)
+        self.col_drop = Dropout(dropout * 2, 1)
+
+        torch.nn.init.zeros_(self.tri_mul_in.linear_z.weight)
+        torch.nn.init.zeros_(self.tri_mul_in.linear_z.bias)
+        torch.nn.init.zeros_(self.tri_mul_out.linear_z.weight)
+        torch.nn.init.zeros_(self.tri_mul_out.linear_z.bias)
+        torch.nn.init.zeros_(self.tri_att_start.mha.linear_o.weight)
+        torch.nn.init.zeros_(self.tri_att_start.mha.linear_o.bias)
+        torch.nn.init.zeros_(self.tri_att_end.mha.linear_o.weight)
+        torch.nn.init.zeros_(self.tri_att_end.mha.linear_o.bias)
+
+        torch.nn.init.zeros_(self.sequence_to_pair.o_proj.weight)
+        torch.nn.init.zeros_(self.sequence_to_pair.o_proj.bias)
+        torch.nn.init.zeros_(self.pair_to_sequence.linear.weight)
+        torch.nn.init.zeros_(self.seq_attention.o_proj.weight)
+        torch.nn.init.zeros_(self.seq_attention.o_proj.bias)
+        torch.nn.init.zeros_(self.mlp_seq.mlp[-2].weight)
+        torch.nn.init.zeros_(self.mlp_seq.mlp[-2].bias)
+        torch.nn.init.zeros_(self.mlp_pair.mlp[-2].weight)
+        torch.nn.init.zeros_(self.mlp_pair.mlp[-2].bias)
+
+    def forward(self, sequence_state, pairwise_state, mask=None, chunk_size=None, **__kwargs):
+        """
+        Inputs:
+          sequence_state: B x L x sequence_state_dim
+          pairwise_state: B x L x L x pairwise_state_dim
+          mask: B x L boolean tensor of valid positions
+
+        Output:
+          sequence_state: B x L x sequence_state_dim
+          pairwise_state: B x L x L x pairwise_state_dim
+        """
+        assert len(sequence_state.shape) == 3
+        assert len(pairwise_state.shape) == 4
+        if mask is not None:
+            assert len(mask.shape) == 2
+
+        batch_dim, seq_dim, sequence_state_dim = sequence_state.shape
+        pairwise_state_dim = pairwise_state.shape[3]
+        assert sequence_state_dim == self.sequence_state_dim
+        assert pairwise_state_dim == self.pairwise_state_dim
+        assert batch_dim == pairwise_state.shape[0]
+        assert seq_dim == pairwise_state.shape[1]
+        assert seq_dim == pairwise_state.shape[2]
+
+        # Update sequence state
+        bias = self.pair_to_sequence(pairwise_state)
+
+        # Self attention with bias + mlp.
+        y = self.layernorm_1(sequence_state)
+        y, _ = self.seq_attention(y, mask=mask, bias=bias)
+        sequence_state = sequence_state + self.drop(y)
+        sequence_state = self.mlp_seq(sequence_state)
+
+        # Update pairwise state
+        pairwise_state = pairwise_state + self.sequence_to_pair(sequence_state)
+
+        # Axial attention with triangular bias.
+        tri_mask = mask.unsqueeze(2) * mask.unsqueeze(1) if mask is not None else None
+        pairwise_state = pairwise_state + self.row_drop(
+            self.tri_mul_out(pairwise_state, mask=tri_mask)
+        )
+        pairwise_state = pairwise_state + self.col_drop(
+            self.tri_mul_in(pairwise_state, mask=tri_mask)
+        )
+        pairwise_state = pairwise_state + self.row_drop(
+            self.tri_att_start(pairwise_state, mask=tri_mask, chunk_size=chunk_size)
+        )
+        pairwise_state = pairwise_state + self.col_drop(
+            self.tri_att_end(pairwise_state, mask=tri_mask, chunk_size=chunk_size)
+        )
+
+        # MLP over pairs.
+        pairwise_state = self.mlp_pair(pairwise_state)
+
+        return sequence_state, pairwise_state

model/esm/esmfold/v1/trunk.py

@@ -0,0 +1,245 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+import typing as T
+from contextlib import ExitStack
+# from dataclasses import dataclass
+from dataclasses import dataclass, field
+
+import torch
+import torch.nn as nn
+from model.openfold.structure_module import StructureModule
+
+from model.esm.esmfold.v1.tri_self_attn_block import TriangularSelfAttentionBlock
+
+
+@dataclass
+class StructureModuleConfig:
+    c_s: int = 384
+    c_z: int = 128
+    c_ipa: int = 16
+    c_resnet: int = 128
+    no_heads_ipa: int = 12
+    no_qk_points: int = 4
+    no_v_points: int = 8
+    dropout_rate: float = 0.1
+    no_blocks: int = 8
+    no_transition_layers: int = 1
+    no_resnet_blocks: int = 2
+    no_angles: int = 7
+    trans_scale_factor: int = 10
+    epsilon: float = 1e-8
+    inf: float = 1e5
+
+
+@dataclass
+class FoldingTrunkConfig:
+    _name: str = "FoldingTrunkConfig"
+    num_blocks: int = 48
+    sequence_state_dim: int = 1024
+    pairwise_state_dim: int = 128
+    sequence_head_width: int = 32
+    pairwise_head_width: int = 32
+    position_bins: int = 32
+    dropout: float = 0
+    layer_drop: float = 0
+    cpu_grad_checkpoint: bool = False
+
+    max_recycles: int = 4
+    chunk_size: T.Optional[int] = None
+
+    # structure_module: StructureModuleConfig = StructureModuleConfig()
+    structure_module: StructureModuleConfig = field(default_factory=StructureModuleConfig)
+
+
+def get_axial_mask(mask):
+    """
+    Helper to convert B x L mask of valid positions to axial mask used
+    in row column attentions.
+
+    Input:
+      mask: B x L tensor of booleans
+
+    Output:
+      mask: B x L x L tensor of booleans
+    """
+
+    if mask is None:
+        return None
+    assert len(mask.shape) == 2
+    batch_dim, seq_dim = mask.shape
+    m = mask.unsqueeze(1).expand(batch_dim, seq_dim, seq_dim)
+    m = m.reshape(batch_dim * seq_dim, seq_dim)
+    return m
+
+
+class RelativePosition(nn.Module):
+    def __init__(self, bins, pairwise_state_dim):
+        super().__init__()
+        self.bins = bins
+
+        # Note an additional offset is used so that the 0th position
+        # is reserved for masked pairs.
+        self.embedding = torch.nn.Embedding(2 * bins + 2, pairwise_state_dim)
+
+    def forward(self, residue_index, mask=None):
+        """
+        Input:
+          residue_index: B x L tensor of indices (dytpe=torch.long)
+          mask: B x L tensor of booleans
+
+        Output:
+          pairwise_state: B x L x L x pairwise_state_dim tensor of embeddings
+        """
+
+        assert residue_index.dtype == torch.long
+        if mask is not None:
+            assert residue_index.shape == mask.shape
+
+        diff = residue_index[:, None, :] - residue_index[:, :, None]
+        diff = diff.clamp(-self.bins, self.bins)
+        diff = diff + self.bins + 1  # Add 1 to adjust for padding index.
+
+        if mask is not None:
+            mask = mask[:, None, :] * mask[:, :, None]
+            diff[mask == False] = 0
+
+        output = self.embedding(diff)
+        return output
+
+
+class FoldingTrunk(nn.Module):
+    def __init__(self, **kwargs):
+        super().__init__()
+        self.cfg = FoldingTrunkConfig(**kwargs)
+        assert self.cfg.max_recycles > 0
+
+        c_s = self.cfg.sequence_state_dim
+        c_z = self.cfg.pairwise_state_dim
+
+        assert c_s % self.cfg.sequence_head_width == 0
+        assert c_z % self.cfg.pairwise_head_width == 0
+        block = TriangularSelfAttentionBlock
+
+        self.pairwise_positional_embedding = RelativePosition(self.cfg.position_bins, c_z)
+
+        self.blocks = nn.ModuleList(
+            [
+                block(
+                    sequence_state_dim=c_s,
+                    pairwise_state_dim=c_z,
+                    sequence_head_width=self.cfg.sequence_head_width,
+                    pairwise_head_width=self.cfg.pairwise_head_width,
+                    dropout=self.cfg.dropout,
+                )
+                for i in range(self.cfg.num_blocks)
+            ]
+        )
+
+        self.recycle_bins = 15
+        self.recycle_s_norm = nn.LayerNorm(c_s)
+        self.recycle_z_norm = nn.LayerNorm(c_z)
+        self.recycle_disto = nn.Embedding(self.recycle_bins, c_z)
+        self.recycle_disto.weight[0].detach().zero_()
+
+        self.structure_module = StructureModule(**self.cfg.structure_module)  # type: ignore
+        self.trunk2sm_s = nn.Linear(c_s, self.structure_module.c_s)
+        self.trunk2sm_z = nn.Linear(c_z, self.structure_module.c_z)
+
+        self.chunk_size = self.cfg.chunk_size
+
+    def set_chunk_size(self, chunk_size):
+        # This parameter means the axial attention will be computed
+        # in a chunked manner. This should make the memory used more or less O(L) instead of O(L^2).
+        # It's equivalent to running a for loop over chunks of the dimension we're iterative over,
+        # where the chunk_size is the size of the chunks, so 128 would mean to parse 128-lengthed chunks.
+        self.chunk_size = chunk_size
+
+    def forward(self, seq_feats, pair_feats, true_aa, residx, mask, no_recycles: T.Optional[int] = None):
+        """
+        Inputs:
+          seq_feats:     B x L x C            tensor of sequence features
+          pair_feats:    B x L x L x C        tensor of pair features
+          residx:        B x L                long tensor giving the position in the sequence
+          mask:          B x L                boolean tensor indicating valid residues
+
+        Output:
+          predicted_structure: B x L x (num_atoms_per_residue * 3) tensor wrapped in a Coordinates object
+        """
+
+        device = seq_feats.device
+        s_s_0 = seq_feats
+        s_z_0 = pair_feats
+
+        if no_recycles is None:
+            no_recycles = self.cfg.max_recycles
+        else:
+            assert no_recycles >= 0, "Number of recycles must not be negative."
+            no_recycles += 1  # First 'recycle' is just the standard forward pass through the model.
+
+        def trunk_iter(s, z, residx, mask):
+            z = z + self.pairwise_positional_embedding(residx, mask=mask)
+
+            for block in self.blocks:
+                s, z = block(s, z, mask=mask, residue_index=residx, chunk_size=self.chunk_size)
+            return s, z
+
+        s_s = s_s_0
+        s_z = s_z_0
+        recycle_s = torch.zeros_like(s_s)
+        recycle_z = torch.zeros_like(s_z)
+        recycle_bins = torch.zeros(*s_z.shape[:-1], device=device, dtype=torch.int64)
+
+        assert no_recycles > 0
+        for recycle_idx in range(no_recycles):
+            with ExitStack() if recycle_idx == no_recycles - 1 else torch.no_grad():
+                # === Recycling ===
+                recycle_s = self.recycle_s_norm(recycle_s.detach())
+                recycle_z = self.recycle_z_norm(recycle_z.detach())
+                recycle_z += self.recycle_disto(recycle_bins.detach())
+
+                s_s, s_z = trunk_iter(s_s_0 + recycle_s, s_z_0 + recycle_z, residx, mask)
+
+                # === Structure module ===
+                structure = self.structure_module(
+                    {"single": self.trunk2sm_s(s_s), "pair": self.trunk2sm_z(s_z)},
+                    true_aa,
+                    mask.float(),
+                )
+
+                recycle_s = s_s
+                recycle_z = s_z
+                # Distogram needs the N, CA, C coordinates, and bin constants same as alphafold.
+                recycle_bins = FoldingTrunk.distogram(
+                    structure["positions"][-1][:, :, :3],
+                    3.375,
+                    21.375,
+                    self.recycle_bins,
+                )
+
+        assert isinstance(structure, dict)  # type: ignore
+        structure["s_s"] = s_s
+        structure["s_z"] = s_z
+
+        return structure
+
+    @staticmethod
+    def distogram(coords, min_bin, max_bin, num_bins):
+        # Coords are [... L x 3 x 3], where it's [N, CA, C] x 3 coordinates.
+        boundaries = torch.linspace(
+            min_bin,
+            max_bin,
+            num_bins - 1,
+            device=coords.device,
+        )
+        boundaries = boundaries**2
+        N, CA, C = [x.squeeze(-2) for x in coords.chunk(3, dim=-2)]
+        # Infer CB coordinates.
+        b = CA - N
+        c = C - CA
+        a = b.cross(c, dim=-1)
+        CB = -0.58273431 * a + 0.56802827 * b - 0.54067466 * c + CA
+        dists = (CB[..., None, :, :] - CB[..., :, None, :]).pow(2).sum(dim=-1, keepdims=True)
+        bins = torch.sum(dists > boundaries, dim=-1)  # [..., L, L]
+        return bins

model/esm/inverse_folding/gvp_modules.py

@@ -0,0 +1,475 @@
+# Contents of this file are from the open source code for
+#
+#   Jing, B., Eismann, S., Suriana, P., Townshend, R. J. L., & Dror, R. (2020).
+#   Learning from Protein Structure with Geometric Vector Perceptrons. In
+#   International Conference on Learning Representations.
+#
+# MIT License
+# 
+# Copyright (c) 2020 Bowen Jing, Stephan Eismann, Patricia Suriana, Raphael Townshend, Ron Dror
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import typing as T
+import torch
+from torch import nn
+import torch.nn.functional as F
+from torch_geometric.nn import MessagePassing
+
+def tuple_size(tp):
+    return tuple([0 if a is None else a.size() for a in tp])
+
+def tuple_sum(tp1, tp2):
+    s1, v1 = tp1
+    s2, v2 = tp2
+    if v2 is None and v2 is None:
+        return (s1 + s2, None)
+    return (s1 + s2, v1 + v2)
+
+def tuple_cat(*args, dim=-1):
+    '''
+    Concatenates any number of tuples (s, V) elementwise.
+    
+    :param dim: dimension along which to concatenate when viewed
+                as the `dim` index for the scalar-channel tensors.
+                This means that `dim=-1` will be applied as
+                `dim=-2` for the vector-channel tensors.
+    '''
+    dim %= len(args[0][0].shape)
+    s_args, v_args = list(zip(*args))
+    return torch.cat(s_args, dim=dim), torch.cat(v_args, dim=dim)
+
+def tuple_index(x, idx):
+    '''
+    Indexes into a tuple (s, V) along the first dimension.
+    
+    :param idx: any object which can be used to index into a `torch.Tensor`
+    '''
+    return x[0][idx], x[1][idx]
+
+def randn(n, dims, device="cpu"):
+    '''
+    Returns random tuples (s, V) drawn elementwise from a normal distribution.
+    
+    :param n: number of data points
+    :param dims: tuple of dimensions (n_scalar, n_vector)
+    
+    :return: (s, V) with s.shape = (n, n_scalar) and
+             V.shape = (n, n_vector, 3)
+    '''
+    return torch.randn(n, dims[0], device=device), \
+            torch.randn(n, dims[1], 3, device=device)
+
+def _norm_no_nan(x, axis=-1, keepdims=False, eps=1e-8, sqrt=True):
+    '''
+    L2 norm of tensor clamped above a minimum value `eps`.
+    
+    :param sqrt: if `False`, returns the square of the L2 norm
+    '''
+    # clamp is slow
+    # out = torch.clamp(torch.sum(torch.square(x), axis, keepdims), min=eps)
+    out = torch.sum(torch.square(x), axis, keepdims) + eps
+    return torch.sqrt(out) if sqrt else out
+
+def _split(x, nv):
+    '''
+    Splits a merged representation of (s, V) back into a tuple. 
+    Should be used only with `_merge(s, V)` and only if the tuple 
+    representation cannot be used.
+    
+    :param x: the `torch.Tensor` returned from `_merge`
+    :param nv: the number of vector channels in the input to `_merge`
+    '''
+    v = torch.reshape(x[..., -3*nv:], x.shape[:-1] + (nv, 3))
+    s = x[..., :-3*nv]
+    return s, v
+
+def _merge(s, v):
+    '''
+    Merges a tuple (s, V) into a single `torch.Tensor`, where the
+    vector channels are flattened and appended to the scalar channels.
+    Should be used only if the tuple representation cannot be used.
+    Use `_split(x, nv)` to reverse.
+    '''
+    v = torch.reshape(v, v.shape[:-2] + (3*v.shape[-2],))
+    return torch.cat([s, v], -1)
+
+class GVP(nn.Module):
+    '''
+    Geometric Vector Perceptron. See manuscript and README.md
+    for more details.
+    
+    :param in_dims: tuple (n_scalar, n_vector)
+    :param out_dims: tuple (n_scalar, n_vector)
+    :param h_dim: intermediate number of vector channels, optional
+    :param activations: tuple of functions (scalar_act, vector_act)
+    :param tuple_io: whether to keep accepting tuple inputs and outputs when vi
+    or vo = 0
+    '''
+    def __init__(self, in_dims, out_dims, h_dim=None, vector_gate=False,
+                 activations=(F.relu, torch.sigmoid), tuple_io=True,
+                 eps=1e-8):
+        super(GVP, self).__init__()
+        self.si, self.vi = in_dims
+        self.so, self.vo = out_dims
+        self.tuple_io = tuple_io
+        if self.vi: 
+            self.h_dim = h_dim or max(self.vi, self.vo) 
+            self.wh = nn.Linear(self.vi, self.h_dim, bias=False)
+            self.ws = nn.Linear(self.h_dim + self.si, self.so)
+            if self.vo:
+                self.wv = nn.Linear(self.h_dim, self.vo, bias=False)
+                if vector_gate:
+                    self.wg = nn.Linear(self.so, self.vo)
+        else:
+            self.ws = nn.Linear(self.si, self.so)
+        
+        self.vector_gate = vector_gate
+        self.scalar_act, self.vector_act = activations
+        self.eps = eps
+        
+    def forward(self, x):
+        '''
+        :param x: tuple (s, V) of `torch.Tensor`, 
+                  or (if vectors_in is 0), a single `torch.Tensor`
+        :return: tuple (s, V) of `torch.Tensor`,
+                 or (if vectors_out is 0), a single `torch.Tensor`
+        '''
+        if self.vi:
+            s, v = x
+            v = torch.transpose(v, -1, -2)
+            vh = self.wh(v)    
+            vn = _norm_no_nan(vh, axis=-2, eps=self.eps)
+            s = self.ws(torch.cat([s, vn], -1))
+            if self.scalar_act:
+                s = self.scalar_act(s)
+            if self.vo: 
+                v = self.wv(vh) 
+                v = torch.transpose(v, -1, -2)
+                if self.vector_gate:
+                    g = self.wg(s).unsqueeze(-1)
+                else:
+                    g = _norm_no_nan(v, axis=-1, keepdims=True, eps=self.eps)
+                if self.vector_act:
+                    g = self.vector_act(g)
+                    v = v * g
+        else:
+            if self.tuple_io:
+                assert x[1] is None
+                x = x[0]
+            s = self.ws(x)
+            if self.scalar_act:
+                s = self.scalar_act(s)
+            if self.vo:
+                v = torch.zeros(list(s.shape)[:-1] + [self.vo, 3],
+                        device=s.device)
+        
+        if self.vo:
+            return (s, v)
+        elif self.tuple_io:
+            return (s, None)
+        else:
+            return s
+
+
+class _VDropout(nn.Module):
+    '''
+    Vector channel dropout where the elements of each
+    vector channel are dropped together.
+    '''
+    def __init__(self, drop_rate):
+        super(_VDropout, self).__init__()
+        self.drop_rate = drop_rate
+
+    def forward(self, x):
+        '''
+        :param x: `torch.Tensor` corresponding to vector channels
+        '''
+        if x is None:
+            return None
+        device = x.device
+        if not self.training:
+            return x
+        mask = torch.bernoulli(
+            (1 - self.drop_rate) * torch.ones(x.shape[:-1], device=device)
+        ).unsqueeze(-1)
+        x = mask * x / (1 - self.drop_rate)
+        return x
+
+class Dropout(nn.Module):
+    '''
+    Combined dropout for tuples (s, V).
+    Takes tuples (s, V) as input and as output.
+    '''
+    def __init__(self, drop_rate):
+        super(Dropout, self).__init__()
+        self.sdropout = nn.Dropout(drop_rate)
+        self.vdropout = _VDropout(drop_rate)
+
+    def forward(self, x):
+        '''
+        :param x: tuple (s, V) of `torch.Tensor`,
+                  or single `torch.Tensor` 
+                  (will be assumed to be scalar channels)
+        '''
+        if type(x) is torch.Tensor:
+            return self.sdropout(x)
+        s, v = x
+        return self.sdropout(s), self.vdropout(v)
+
+class LayerNorm(nn.Module):
+    '''
+    Combined LayerNorm for tuples (s, V).
+    Takes tuples (s, V) as input and as output.
+    '''
+    def __init__(self, dims, tuple_io=True, eps=1e-8):
+        super(LayerNorm, self).__init__()
+        self.tuple_io = tuple_io
+        self.s, self.v = dims
+        self.scalar_norm = nn.LayerNorm(self.s)
+        self.eps = eps
+        
+    def forward(self, x):
+        '''
+        :param x: tuple (s, V) of `torch.Tensor`,
+                  or single `torch.Tensor` 
+                  (will be assumed to be scalar channels)
+        '''
+        if not self.v:
+            if self.tuple_io:
+                return self.scalar_norm(x[0]), None
+            return self.scalar_norm(x)
+        s, v = x
+        vn = _norm_no_nan(v, axis=-1, keepdims=True, sqrt=False, eps=self.eps)
+        nonzero_mask = (vn > 2 * self.eps)
+        vn = torch.sum(vn * nonzero_mask, dim=-2, keepdim=True
+            ) / (self.eps + torch.sum(nonzero_mask, dim=-2, keepdim=True))
+        vn = torch.sqrt(vn + self.eps)
+        v = nonzero_mask * (v / vn)
+        return self.scalar_norm(s), v
+
+class GVPConv(MessagePassing):
+    '''
+    Graph convolution / message passing with Geometric Vector Perceptrons.
+    Takes in a graph with node and edge embeddings,
+    and returns new node embeddings.
+    
+    This does NOT do residual updates and pointwise feedforward layers
+    ---see `GVPConvLayer`.
+    
+    :param in_dims: input node embedding dimensions (n_scalar, n_vector)
+    :param out_dims: output node embedding dimensions (n_scalar, n_vector)
+    :param edge_dims: input edge embedding dimensions (n_scalar, n_vector)
+    :param n_layers: number of GVPs in the message function
+    :param module_list: preconstructed message function, overrides n_layers
+    :param aggr: should be "add" if some incoming edges are masked, as in
+                 a masked autoregressive decoder architecture
+    '''
+    def __init__(self, in_dims, out_dims, edge_dims, n_layers=3,
+            vector_gate=False, module_list=None, aggr="mean", eps=1e-8,
+            activations=(F.relu, torch.sigmoid)):
+        super(GVPConv, self).__init__(aggr=aggr)
+        self.eps = eps
+        self.si, self.vi = in_dims
+        self.so, self.vo = out_dims
+        self.se, self.ve = edge_dims
+        
+        module_list = module_list or []
+        if not module_list:
+            if n_layers == 1:
+                module_list.append(
+                    GVP((2*self.si + self.se, 2*self.vi + self.ve), 
+                        (self.so, self.vo), activations=(None, None)))
+            else:
+                module_list.append(
+                    GVP((2*self.si + self.se, 2*self.vi + self.ve), out_dims,
+                        vector_gate=vector_gate, activations=activations)
+                )
+                for i in range(n_layers - 2):
+                    module_list.append(GVP(out_dims, out_dims,
+                        vector_gate=vector_gate))
+                module_list.append(GVP(out_dims, out_dims,
+                                       activations=(None, None)))
+        self.message_func = nn.Sequential(*module_list)
+
+    def forward(self, x, edge_index, edge_attr):
+        '''
+        :param x: tuple (s, V) of `torch.Tensor`
+        :param edge_index: array of shape [2, n_edges]
+        :param edge_attr: tuple (s, V) of `torch.Tensor`
+        '''
+        x_s, x_v = x
+        message = self.propagate(edge_index, 
+                    s=x_s, v=x_v.reshape(x_v.shape[0], 3*x_v.shape[1]),
+                    edge_attr=edge_attr)
+        return _split(message, self.vo) 
+
+    def message(self, s_i, v_i, s_j, v_j, edge_attr):
+        v_j = v_j.view(v_j.shape[0], v_j.shape[1]//3, 3)
+        v_i = v_i.view(v_i.shape[0], v_i.shape[1]//3, 3)
+        message = tuple_cat((s_j, v_j), edge_attr, (s_i, v_i))
+        message = self.message_func(message)
+        return _merge(*message)
+
+
+class GVPConvLayer(nn.Module):
+    '''
+    Full graph convolution / message passing layer with 
+    Geometric Vector Perceptrons. Residually updates node embeddings with
+    aggregated incoming messages, applies a pointwise feedforward 
+    network to node embeddings, and returns updated node embeddings.
+    
+    To only compute the aggregated messages, see `GVPConv`.
+    
+    :param node_dims: node embedding dimensions (n_scalar, n_vector)
+    :param edge_dims: input edge embedding dimensions (n_scalar, n_vector)
+    :param n_message: number of GVPs to use in message function
+    :param n_feedforward: number of GVPs to use in feedforward function
+    :param drop_rate: drop probability in all dropout layers
+    :param autoregressive: if `True`, this `GVPConvLayer` will be used
+           with a different set of input node embeddings for messages
+           where src >= dst
+    '''
+    def __init__(self, node_dims, edge_dims, vector_gate=False,
+                 n_message=3, n_feedforward=2, drop_rate=.1,
+                 autoregressive=False, attention_heads=0,
+                 conv_activations=(F.relu, torch.sigmoid),
+                 n_edge_gvps=0, layernorm=True, eps=1e-8):
+        
+        super(GVPConvLayer, self).__init__()
+        if attention_heads == 0:
+            self.conv = GVPConv(
+                    node_dims, node_dims, edge_dims, n_layers=n_message,
+                    vector_gate=vector_gate,
+                    aggr="add" if autoregressive else "mean",
+                    activations=conv_activations, 
+                    eps=eps,
+            )
+        else:
+            raise NotImplementedError
+        if layernorm:
+            self.norm = nn.ModuleList([LayerNorm(node_dims, eps=eps) for _ in range(2)])
+        else:
+            self.norm = nn.ModuleList([nn.Identity() for _ in range(2)])
+        self.dropout = nn.ModuleList([Dropout(drop_rate) for _ in range(2)])
+
+        ff_func = []
+        if n_feedforward == 1:
+            ff_func.append(GVP(node_dims, node_dims, activations=(None, None)))
+        else:
+            hid_dims = 4*node_dims[0], 2*node_dims[1]
+            ff_func.append(GVP(node_dims, hid_dims, vector_gate=vector_gate))
+            for i in range(n_feedforward-2):
+                ff_func.append(GVP(hid_dims, hid_dims, vector_gate=vector_gate))
+            ff_func.append(GVP(hid_dims, node_dims, activations=(None, None)))
+        self.ff_func = nn.Sequential(*ff_func)
+
+        self.edge_message_func = None
+        if n_edge_gvps > 0:
+            si, vi = node_dims
+            se, ve = edge_dims
+            module_list = [
+                GVP((2*si + se, 2*vi + ve), edge_dims, vector_gate=vector_gate)
+            ]
+            for i in range(n_edge_gvps - 2):
+                module_list.append(GVP(edge_dims, edge_dims,
+                    vector_gate=vector_gate))
+            if n_edge_gvps > 1:
+                module_list.append(GVP(edge_dims, edge_dims,
+                    activations=(None, None)))
+            self.edge_message_func = nn.Sequential(*module_list)
+            if layernorm:
+                self.edge_norm = LayerNorm(edge_dims, eps=eps)
+            else:
+                self.edge_norm = nn.Identity()
+            self.edge_dropout = Dropout(drop_rate)
+
+    def forward(self, x, edge_index, edge_attr,
+                autoregressive_x=None, node_mask=None):
+        '''
+        :param x: tuple (s, V) of `torch.Tensor`
+        :param edge_index: array of shape [2, n_edges]
+        :param edge_attr: tuple (s, V) of `torch.Tensor`
+        :param autoregressive_x: tuple (s, V) of `torch.Tensor`. 
+                If not `None`, will be used as srcqq node embeddings
+                for forming messages where src >= dst. The corrent node 
+                embeddings `x` will still be the base of the update and the 
+                pointwise feedforward.
+        :param node_mask: array of type `bool` to index into the first
+                dim of node embeddings (s, V). If not `None`, only
+                these nodes will be updated.
+        '''
+        if self.edge_message_func:
+            src, dst = edge_index
+            if autoregressive_x is None:
+                x_src = x[0][src], x[1][src]
+            else: 
+                mask = (src < dst).unsqueeze(-1)
+                x_src = (
+                    torch.where(mask, x[0][src], autoregressive_x[0][src]),
+                    torch.where(mask.unsqueeze(-1), x[1][src],
+                        autoregressive_x[1][src])
+                )
+            x_dst = x[0][dst], x[1][dst]
+            x_edge = (
+                torch.cat([x_src[0], edge_attr[0], x_dst[0]], dim=-1),
+                torch.cat([x_src[1], edge_attr[1], x_dst[1]], dim=-2)
+            )
+            edge_attr_dh = self.edge_message_func(x_edge)
+            edge_attr = self.edge_norm(tuple_sum(edge_attr,
+                self.edge_dropout(edge_attr_dh)))
+        
+        if autoregressive_x is not None:
+            # Guarding this import here to remove the dependency on torch_scatter, since this isn't used
+            # in ESM-IF1
+            from torch_scatter import scatter_add
+            src, dst = edge_index
+            mask = src < dst
+            edge_index_forward = edge_index[:, mask]
+            edge_index_backward = edge_index[:, ~mask]
+            edge_attr_forward = tuple_index(edge_attr, mask)
+            edge_attr_backward = tuple_index(edge_attr, ~mask)
+            
+            dh = tuple_sum(
+                self.conv(x, edge_index_forward, edge_attr_forward),
+                self.conv(autoregressive_x, edge_index_backward, edge_attr_backward)
+            )
+                        
+            count = scatter_add(torch.ones_like(dst), dst,
+                        dim_size=dh[0].size(0)).clamp(min=1).unsqueeze(-1)
+            
+            dh = dh[0] / count, dh[1] / count.unsqueeze(-1)
+
+        else:
+            dh = self.conv(x, edge_index, edge_attr)
+        
+        if node_mask is not None:
+            x_ = x
+            x, dh = tuple_index(x, node_mask), tuple_index(dh, node_mask)
+            
+        x = self.norm[0](tuple_sum(x, self.dropout[0](dh)))
+        
+        dh = self.ff_func(x)
+        x = self.norm[1](tuple_sum(x, self.dropout[1](dh)))
+        
+        if node_mask is not None:
+            x_[0][node_mask], x_[1][node_mask] = x[0], x[1]
+            x = x_
+
+        return x, edge_attr

model/esm/inverse_folding/gvp_transformer.py

@@ -0,0 +1,140 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import argparse
+from typing import Any, Dict, List, Optional, Tuple, NamedTuple
+import torch
+from torch import nn
+from torch import Tensor
+import torch.nn.functional as F
+from scipy.spatial import transform
+
+from onescience.datapipes.esm import Alphabet
+
+from .features import DihedralFeatures
+from .gvp_encoder import GVPEncoder
+from .gvp_utils import unflatten_graph
+from .gvp_transformer_encoder import GVPTransformerEncoder
+from .transformer_decoder import TransformerDecoder
+from .util import rotate, CoordBatchConverter 
+
+
+class GVPTransformerModel(nn.Module):
+    """
+    GVP-Transformer inverse folding model.
+
+    Architecture: Geometric GVP-GNN as initial layers, followed by
+    sequence-to-sequence Transformer encoder and decoder.
+    """
+
+    def __init__(self, args, alphabet):
+        super().__init__()
+        encoder_embed_tokens = self.build_embedding(
+            args, alphabet, args.encoder_embed_dim,
+        )
+        decoder_embed_tokens = self.build_embedding(
+            args, alphabet, args.decoder_embed_dim, 
+        )
+        encoder = self.build_encoder(args, alphabet, encoder_embed_tokens)
+        decoder = self.build_decoder(args, alphabet, decoder_embed_tokens)
+        self.args = args
+        self.encoder = encoder
+        self.decoder = decoder
+
+    @classmethod
+    def build_encoder(cls, args, src_dict, embed_tokens):
+        encoder = GVPTransformerEncoder(args, src_dict, embed_tokens)
+        return encoder
+
+    @classmethod
+    def build_decoder(cls, args, tgt_dict, embed_tokens):
+        decoder = TransformerDecoder(
+            args,
+            tgt_dict,
+            embed_tokens,
+        )
+        return decoder
+
+    @classmethod
+    def build_embedding(cls, args, dictionary, embed_dim):
+        num_embeddings = len(dictionary)
+        padding_idx = dictionary.padding_idx
+        emb = nn.Embedding(num_embeddings, embed_dim, padding_idx)
+        nn.init.normal_(emb.weight, mean=0, std=embed_dim ** -0.5)
+        nn.init.constant_(emb.weight[padding_idx], 0)
+        return emb
+
+    def forward(
+        self,
+        coords,
+        padding_mask,
+        confidence,
+        prev_output_tokens,
+        return_all_hiddens: bool = False,
+        features_only: bool = False,
+    ):
+        encoder_out = self.encoder(coords, padding_mask, confidence,
+            return_all_hiddens=return_all_hiddens)
+        logits, extra = self.decoder(
+            prev_output_tokens,
+            encoder_out=encoder_out,
+            features_only=features_only,
+            return_all_hiddens=return_all_hiddens,
+        )
+        return logits, extra
+    
+    def sample(self, coords, partial_seq=None, temperature=1.0, confidence=None, device=None):
+        """
+        Samples sequences based on multinomial sampling (no beam search).
+
+        Args:
+            coords: L x 3 x 3 list representing one backbone
+            partial_seq: Optional, partial sequence with mask tokens if part of
+                the sequence is known
+            temperature: sampling temperature, use low temperature for higher
+                sequence recovery and high temperature for higher diversity
+            confidence: optional length L list of confidence scores for coordinates
+        """
+        L = len(coords)
+        # Convert to batch format
+        batch_converter = CoordBatchConverter(self.decoder.dictionary)
+        batch_coords, confidence, _, _, padding_mask = (
+            batch_converter([(coords, confidence, None)], device=device)
+        )
+        
+        # Start with prepend token
+        mask_idx = self.decoder.dictionary.get_idx('<mask>')
+        sampled_tokens = torch.full((1, 1+L), mask_idx, dtype=int)
+        sampled_tokens[0, 0] = self.decoder.dictionary.get_idx('<cath>')
+        if partial_seq is not None:
+            for i, c in enumerate(partial_seq):
+                sampled_tokens[0, i+1] = self.decoder.dictionary.get_idx(c)
+            
+        # Save incremental states for faster sampling
+        incremental_state = dict()
+        
+        # Run encoder only once
+        encoder_out = self.encoder(batch_coords, padding_mask, confidence)
+        
+        # Make sure all tensors are on the same device if a GPU is present
+        if device:
+            sampled_tokens = sampled_tokens.to(device)
+        
+        # Decode one token at a time
+        for i in range(1, L+1):
+            logits, _ = self.decoder(
+                sampled_tokens[:, :i], 
+                encoder_out,
+                incremental_state=incremental_state,
+            )
+            logits = logits[0].transpose(0, 1)
+            logits /= temperature
+            probs = F.softmax(logits, dim=-1)
+            if sampled_tokens[0, i] == mask_idx:
+                sampled_tokens[:, i] = torch.multinomial(probs, 1).squeeze(-1)
+        sampled_seq = sampled_tokens[0, 1:]
+        
+        # Convert back to string via lookup
+        return ''.join([self.decoder.dictionary.get_tok(a) for a in sampled_seq])

model/protenix/layer_norm/__init__.py

@@ -0,0 +1 @@
+from .layer_norm import FusedLayerNorm

model/protenix/layer_norm/kernel/compat.h

@@ -0,0 +1,9 @@
+#ifndef TORCH_CHECK
+#define TORCH_CHECK AT_CHECK
+#endif
+
+#ifdef VERSION_GE_1_3
+#define DATA_PTR data_ptr
+#else
+#define DATA_PTR data
+#endif

model/protenix/layer_norm/kernel/layer_norm_cuda.cpp

@@ -0,0 +1,179 @@
+// Copyright 2021- HPC-AI Technology Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+
+#include <torch/extension.h>
+#include <c10/cuda/CUDAGuard.h>
+
+#include <cassert>
+#include <vector>
+#include <functional>
+
+#include "compat.h"
+
+void compute_n1_n2(at::Tensor input, at::IntArrayRef normalized_shape, int& n1, int& n2) {
+    int idiff = input.ndimension() - normalized_shape.size();
+    n2 = 1;
+    for (int i = 0; i < (int)normalized_shape.size(); ++i) {
+        assert(input.sizes()[i + idiff] == normalized_shape[i]);
+        n2 *= normalized_shape[i];
+    }
+    n1 = 1;
+    for (int i = 0; i < idiff; ++i) {
+        n1 *= input.sizes()[i];
+    }
+}
+
+void check_args(at::Tensor input, at::IntArrayRef normalized_shape, int& n1, int& n2) {
+    int64_t normalized_ndim = normalized_shape.size();
+
+    if (normalized_ndim < 1) {
+        std::stringstream ss;
+        ss << "Expected normalized_shape to be at least 1-dimensional, i.e., "
+           << "containing at least one element, but got normalized_shape=" << normalized_shape;
+        throw std::runtime_error(ss.str());
+    }
+
+    auto input_shape = input.sizes();
+    auto input_ndim = input.dim();
+
+    if (input_ndim < normalized_ndim ||
+        !input_shape.slice(input_ndim - normalized_ndim).equals(normalized_shape)) {
+        std::stringstream ss;
+        ss << "Given normalized_shape=" << normalized_shape << ", expected input with shape [*";
+        for (auto size : normalized_shape) {
+            ss << ", " << size;
+        }
+        ss << "], but got input of size" << input_shape;
+        throw std::runtime_error(ss.str());
+    }
+
+    compute_n1_n2(input, normalized_shape, n1, n2);
+}
+
+void cuda_layer_norm(at::Tensor* output, at::Tensor* mean, at::Tensor* invvar, at::Tensor* input,
+                     int n1, int n2, at::IntArrayRef normalized_shape, at::Tensor* gamma,
+                     at::Tensor* beta, double epsilon);
+
+#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor")
+#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous")
+#define CHECK_INPUT(x) \
+    CHECK_CUDA(x);     \
+    CHECK_CONTIGUOUS(x)
+
+std::vector<at::Tensor> layer_norm_affine(at::Tensor input, at::IntArrayRef normalized_shape,
+                                          at::Tensor *gamma, at::Tensor *beta, double epsilon) {
+    CHECK_INPUT(input);
+    // CHECK_INPUT((*gamma));
+    // CHECK_INPUT((*beta));
+    int n1, n2;
+    check_args(input, normalized_shape, n1, n2);
+
+    const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
+
+    at::Tensor output = at::empty_like(input, input.options().dtype(input.scalar_type()));
+    at::Tensor mean = at::empty({n1}, input.options().dtype(at::ScalarType::Float));
+    at::Tensor invvar = at::empty_like(mean);
+
+    cuda_layer_norm(&output, &mean, &invvar, &input, n1, n2, normalized_shape, gamma, beta, epsilon);
+
+    return {output, mean, invvar};
+}
+
+void cuda_layer_norm_gradient(at::Tensor* dout, at::Tensor* mean, at::Tensor* invvar,
+                              at::Tensor* input, int n1, int n2, at::IntArrayRef normalized_shape,
+                              at::Tensor* gamma, at::Tensor* beta, double epsilon,
+                              at::Tensor* grad_input, at::Tensor* grad_gamma,
+                              at::Tensor* grad_beta);
+
+std::vector<at::Tensor> layer_norm_gradient_affine(at::Tensor dout, at::Tensor mean,
+                                                   at::Tensor invvar, at::Tensor input,
+                                                   at::IntArrayRef normalized_shape,
+                                                   at::Tensor* gamma, at::Tensor* beta,
+                                                   double epsilon) {
+    CHECK_INPUT(dout);
+    CHECK_INPUT(mean);
+    CHECK_INPUT(invvar);
+    CHECK_INPUT(input);
+    int n1, n2;
+    check_args(input, normalized_shape, n1, n2);
+
+    const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
+
+    at::Tensor grad_input = at::empty_like(input);
+
+    at::Tensor grad_gamma;
+    at::Tensor grad_beta;
+    if (gamma != NULL)
+        grad_gamma = at::empty_like(*gamma);
+    if (beta != NULL)
+        grad_beta = at::empty_like(*beta);
+    
+    if (gamma != NULL) {
+        if(beta != NULL) {
+            cuda_layer_norm_gradient(&dout, &mean, &invvar, &input, n1, n2, normalized_shape, gamma, beta,
+                             epsilon, &grad_input, &grad_gamma, &grad_beta);
+        } else {
+            cuda_layer_norm_gradient(&dout, &mean, &invvar, &input, n1, n2, normalized_shape, gamma, beta,
+                             epsilon, &grad_input, &grad_gamma, NULL);
+        }
+    } else {
+        if(beta != NULL) {
+            cuda_layer_norm_gradient(&dout, &mean, &invvar, &input, n1, n2, normalized_shape, gamma, beta,
+                             epsilon, &grad_input, NULL, &grad_beta);
+        } else {
+            cuda_layer_norm_gradient(&dout, &mean, &invvar, &input, n1, n2, normalized_shape, gamma, beta,
+                             epsilon, &grad_input, NULL, NULL);
+        }
+    }
+    return {grad_input, grad_gamma, grad_beta};
+}
+
+
+PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
+    m.def("forward_none_affine", [](at::Tensor input, at::IntArrayRef normalized_shape, double epsilon) {
+        return layer_norm_affine(input, normalized_shape, NULL, NULL, epsilon);
+    }, "LayerNorm forward (CUDA)");
+    
+    m.def("forward_with_bias_affine", [](at::Tensor input, at::IntArrayRef normalized_shape, at::Tensor *beta, double epsilon) {
+        return layer_norm_affine(input, normalized_shape, NULL, beta, epsilon);
+    }, "LayerNorm forward (CUDA)");
+    
+    m.def("forward_with_weight_affine", [](at::Tensor input, at::IntArrayRef normalized_shape, at::Tensor *gamma, double epsilon) {
+        return layer_norm_affine(input, normalized_shape, gamma, NULL, epsilon);
+    }, "LayerNorm forward (CUDA)");
+    
+    m.def("forward_with_both_affine", [](at::Tensor input, at::IntArrayRef normalized_shape, at::Tensor *gamma, at::Tensor *beta, double epsilon) {
+        return layer_norm_affine(input, normalized_shape, gamma, beta, epsilon);
+    }, "LayerNorm forward (CUDA)");
+    
+    m.def("backward_none_affine", [](at::Tensor dout, at::Tensor mean, at::Tensor invvar, at::Tensor input,
+                                     at::IntArrayRef normalized_shape, double epsilon) {
+        return layer_norm_gradient_affine(dout, mean, invvar, input, normalized_shape, NULL, NULL, epsilon);
+
+    }, "LayerNorm backward (CUDA)");
+
+    m.def("backward_with_bias_affine", [](at::Tensor dout, at::Tensor mean, at::Tensor invvar, at::Tensor input,
+                                     at::IntArrayRef normalized_shape, at::Tensor *beta, double epsilon) {
+        return layer_norm_gradient_affine(dout, mean, invvar, input, normalized_shape, NULL, beta, epsilon);
+    }, "LayerNorm backward (CUDA)");
+
+    m.def("backward_with_weight_affine", [](at::Tensor dout, at::Tensor mean, at::Tensor invvar, at::Tensor input,
+                                     at::IntArrayRef normalized_shape, at::Tensor *gamma, double epsilon) {
+        return layer_norm_gradient_affine(dout, mean, invvar, input, normalized_shape, gamma, NULL, epsilon);
+    }, "LayerNorm backward (CUDA)");
+
+    m.def("backward_with_both_affine", [](at::Tensor dout, at::Tensor mean, at::Tensor invvar, at::Tensor input,
+                                     at::IntArrayRef normalized_shape, at::Tensor *gamma, at::Tensor *beta, double epsilon) {
+        return layer_norm_gradient_affine(dout, mean, invvar, input, normalized_shape, gamma, beta, epsilon);
+    }, "LayerNorm backward (CUDA)");
+}

model/protenix/layer_norm/kernel/layer_norm_cuda_kernel.cu

@@ -0,0 +1,599 @@
+#include <cooperative_groups.h>
+#include <cuda.h>
+#include <cuda_runtime.h>
+#include <torch/extension.h>
+#include <iostream>
+
+#include <THC/THCDeviceUtils.cuh>
+
+#include "ATen/ATen.h"
+#include "ATen/AccumulateType.h"
+#include "ATen/cuda/CUDAContext.h"
+#include "compat.h"
+#include "type_shim.h"
+
+#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor")
+#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous")
+#define CHECK_INPUT(x) \
+    CHECK_CUDA(x);     \
+    CHECK_CONTIGUOUS(x)
+
+#define WarpNum 8
+#define WarpSize 32
+#define BlockSzie WarpNum*WarpSize
+
+inline __device__ void WelfordOnline(float val, float* mean, float* m2, float* count) {
+    *count += 1;
+    float delta1 = val - *mean;
+    *mean += delta1 / (*count);
+    float delta2 = val - *mean;
+    *m2 += delta1 * delta2;
+}
+
+inline __device__ void WelfordOnline(float b_mean, float b_m2, float b_count, float* mean,
+                                     float* m2, float* count) {
+    if (b_count == 0) {
+        return;
+    }
+    float new_count = *count + b_count;
+    float nb_n = b_count / new_count;
+    float delta = b_mean - *mean;
+    *mean += delta * nb_n;
+    *m2 += b_m2 + delta * delta * (*count) * nb_n;
+    *count = new_count;
+}
+
+__inline__ __device__ void WelfordWarpAllReduce(float thread_mean, float thread_m2,
+                                                float thread_count, float* mean, float* m2,
+                                                float* count) {
+    *mean = thread_mean;
+    *m2 = thread_m2;
+    *count = thread_count;
+    for (int mask = 1; mask < 32; mask *= 2) {
+        float b_mean = __shfl_down_sync(0xffffffff, *mean, mask);
+        float b_m2 = __shfl_down_sync(0xffffffff, *m2, mask);
+        float b_count = __shfl_down_sync(0xffffffff, *count, mask);
+        WelfordOnline(b_mean, b_m2, b_count, mean, m2, count);
+    }
+
+    *mean = __shfl_sync(0xffffffff, *mean, 0, 32);
+    *m2 = __shfl_sync(0xffffffff, *m2, 0, 32);
+    *count = __shfl_sync(0xffffffff, *count, 0, 32);
+}
+
+extern __shared__ float shared_data[];
+template <typename T>
+__global__ void LayerNormForward(T* input, T* output, T* gamma, T* beta, float* mean,
+                                 float* invvar, int rows, int cols, double epsilon) {
+    int warp_id = threadIdx.x / WarpSize;
+    int lane_id = threadIdx.x % WarpSize;
+    int row_offset = blockIdx.x * WarpNum + warp_id;
+
+    T* shared_data_warp = (T*)shared_data + warp_id*cols;
+
+    if (row_offset < rows) {
+        T* row_input = input + (long long)(row_offset) * (long long)(cols); // Starting point for input data
+        T* row_output = output + (long long)(row_offset) * (long long)(cols); // Starting point for output data
+
+        float thread_mean = 0.f;
+        float thread_m2 = 0.f;
+        float thread_count = 0.f;
+
+        float warp_mean;
+        float warp_m2;
+        float warp_count;
+        // load data to shared memory
+#pragma unroll
+        for(int idx = lane_id; idx < cols; idx += WarpSize) {
+            shared_data_warp[idx] = row_input[idx];
+            WelfordOnline(static_cast<float>(shared_data_warp[idx]), &thread_mean, &thread_m2, &thread_count);
+        }
+
+        WelfordWarpAllReduce(thread_mean, thread_m2, thread_count, &warp_mean, &warp_m2,
+                             &warp_count);
+
+        float row_mean = warp_mean;
+        float row_variance = max(warp_m2 / warp_count, 0.f);
+        float row_inv_var = rsqrt(row_variance + epsilon);
+        if (lane_id == 0) {
+            mean[row_offset] = row_mean;
+            invvar[row_offset] = row_inv_var;
+        }
+        int process_type = (gamma != NULL)*2 + (beta != NULL);
+        if (process_type == 0) {
+#pragma unroll
+            for(int idx = lane_id; idx < cols; idx += WarpSize)
+                row_output[idx] = static_cast<T>((static_cast<float>(shared_data_warp[idx]) - row_mean) * row_inv_var);
+        } else if (process_type == 1) {
+#pragma unroll
+            for(int idx = lane_id; idx < cols; idx += WarpSize)
+                row_output[idx] = static_cast<T>((static_cast<float>(shared_data_warp[idx]) - row_mean) * row_inv_var + beta[idx]);
+        } else if(process_type == 2) {
+#pragma unroll
+            for(int idx = lane_id; idx < cols; idx += WarpSize)
+                row_output[idx] = static_cast<T>((static_cast<float>(shared_data_warp[idx]) - row_mean) * row_inv_var * gamma[idx]);
+        } else {
+#pragma unroll
+            for(int idx = lane_id; idx < cols; idx += WarpSize)
+                row_output[idx] = static_cast<T>((static_cast<float>(shared_data_warp[idx]) - row_mean) * row_inv_var * gamma[idx] + beta[idx]);
+        }
+    }
+}
+
+void cuda_layer_norm(at::Tensor* output, at::Tensor* mean, at::Tensor* invvar, at::Tensor* input,
+                     int rows, int cols, at::IntArrayRef normalized_shape, at::Tensor* gamma,
+                     at::Tensor* beta, double epsilon) {
+    int grid = (rows + WarpNum - 1) / WarpNum; // each warp process one line
+    dim3 block(BlockSzie);
+    // add shared memory size
+    if (output->dtype() == torch::kFloat32) {
+        int shared_meory_size = WarpNum*sizeof(float)*cols;
+        LayerNormForward<float><<<grid, block, shared_meory_size>>>(
+            (float*)input->data_ptr(), (float*)output->data_ptr(), 
+            gamma != NULL ? (float*)gamma->data_ptr() : NULL, beta != NULL ? (float*)beta->data_ptr() : NULL, 
+            (float*)mean->data_ptr(), (float*)invvar->data_ptr(), rows, cols, epsilon);
+    } else if (output->dtype() == torch::kFloat16) {
+        int shared_meory_size = WarpNum*sizeof(at::Half)*cols;
+        LayerNormForward<at::Half><<<grid, block, shared_meory_size>>>(
+            (at::Half*)input->data_ptr(), (at::Half*)output->data_ptr(),
+            gamma != NULL ? (at::Half*)gamma->data_ptr() : NULL, beta != NULL ? (at::Half*)beta->data_ptr() : NULL, 
+            (float*)mean->data_ptr(), (float*)invvar->data_ptr(), rows, cols, epsilon);
+    } else if (output->dtype() == torch::kBFloat16) {
+        int shared_meory_size = WarpNum*sizeof(at::BFloat16)*cols;
+        LayerNormForward<at::BFloat16><<<grid, block, shared_meory_size>>>(
+            (at::BFloat16*)input->data_ptr(), (at::BFloat16*)output->data_ptr(),
+            gamma != NULL ? (at::BFloat16*)gamma->data_ptr() : NULL, beta != NULL ? (at::BFloat16*)beta->data_ptr() : NULL,
+            (float*)mean->data_ptr(), (float*)invvar->data_ptr(), rows, cols, epsilon);
+    }
+}
+
+template <typename T>
+struct SharedMemory;
+
+template <>
+struct SharedMemory<float> {
+    __device__ float* getPointer() {
+        extern __shared__ float s_float[];
+        return s_float;
+    }
+};
+
+template<typename T>
+__inline__ __device__ T WarpReduce(T val) {
+  for (int mask = 16; mask > 0; mask /= 2) { val += __shfl_xor_sync(0xffffffff, val, mask); }
+  return val;
+}
+
+constexpr int tile_size = 32;
+constexpr int num_per_block = 4;
+constexpr int block_dim_x = 32;
+constexpr int block_dim_y = 32 / num_per_block;
+
+template <typename T, typename V>
+__global__ void LayerNormParamGradStep1(int rows, int cols, const V* __restrict__ dy,
+                                        const T* __restrict__ x, const float* __restrict__ mean,
+                                        const float* __restrict__ inv_var,
+                                        float* __restrict__ tmp_gamma_diff, float* __restrict__ tmp_beta_diff) {
+  __shared__ float dgamma[32][33];
+  __shared__ float dbeta[32][33];
+  float dgamma_sum[num_per_block];
+  float dbeta_sum[num_per_block];
+#pragma unroll
+  for (int index = 0; index < num_per_block; ++index) {
+    dgamma_sum[index] = 0;
+    dbeta_sum[index] = 0;
+  }
+  const int col_id = blockIdx.x * blockDim.x + threadIdx.x;
+  if (col_id < cols) {
+    for (int i = blockIdx.y * tile_size + threadIdx.y; i < rows; i += tile_size * gridDim.y) {
+#pragma unroll
+      for (int index = 0; index < num_per_block; ++index) {
+        int row_id = i + index * blockDim.y;
+        if (row_id < rows) {
+          int offset = row_id * cols + col_id;
+          const float dy_val = static_cast<float>(dy[offset]);
+          const float x_val = static_cast<float>(x[offset]);
+          const float mean_val = mean[row_id];
+          const float inv_var_val = inv_var[row_id];
+          dgamma_sum[index] += dy_val * (x_val - mean_val) * inv_var_val;
+          dbeta_sum[index] += dy_val;
+        }
+      }
+    }
+  }
+#pragma unroll
+  for (int index = 0; index < num_per_block; ++index) {
+    dgamma[index * blockDim.y + threadIdx.y][threadIdx.x] = dgamma_sum[index];
+    dbeta[index * blockDim.y + threadIdx.y][threadIdx.x] = dbeta_sum[index];
+  }
+  __syncthreads();
+#pragma unroll
+  for (int index = 0; index < num_per_block; ++index) {
+    const int col_id = blockIdx.x * blockDim.x + threadIdx.y + index * blockDim.y;
+    if (col_id < cols) {
+      float gamma_sum = dgamma[threadIdx.x][threadIdx.y + index * blockDim.y];
+      float beta_sum = dbeta[threadIdx.x][threadIdx.y + index * blockDim.y];
+      float global_dgamma = WarpReduce<float>(gamma_sum);
+      float global_dbeta = WarpReduce<float>(beta_sum);
+      if (threadIdx.x == 0) {
+        const int offset = blockIdx.y * cols + col_id;
+        tmp_gamma_diff[offset] = global_dgamma;
+        tmp_beta_diff[offset] = global_dbeta;
+      }
+    }
+  }
+}
+
+template <typename T, typename V>
+__global__ void LayerNormGammaGradStep1(int rows, int cols, const V* __restrict__ dy,
+                                        const T* __restrict__ x, const float* __restrict__ mean,
+                                        const float* __restrict__ inv_var, float* __restrict__ tmp_gamma_diff) {
+  __shared__ float dgamma[32][33];
+  float dgamma_sum[num_per_block];
+#pragma unroll
+  for (int index = 0; index < num_per_block; ++index) {
+    dgamma_sum[index] = 0;
+  }
+  const int col_id = blockIdx.x * blockDim.x + threadIdx.x;
+  if (col_id < cols) {
+    for (int i = blockIdx.y * tile_size + threadIdx.y; i < rows; i += tile_size * gridDim.y) {
+#pragma unroll
+      for (int index = 0; index < num_per_block; ++index) {
+        int row_id = i + index * blockDim.y;
+        if (row_id < rows) {
+          int offset = row_id * cols + col_id;
+          const float dy_val = static_cast<float>(dy[offset]);
+          const float x_val = static_cast<float>(x[offset]);
+          const float mean_val = mean[row_id];
+          const float inv_var_val = inv_var[row_id];
+          dgamma_sum[index] += dy_val * (x_val - mean_val) * inv_var_val;
+        }
+      }
+    }
+  }
+#pragma unroll
+  for (int index = 0; index < num_per_block; ++index) {
+    dgamma[index * blockDim.y + threadIdx.y][threadIdx.x] = dgamma_sum[index];
+  }
+  __syncthreads();
+#pragma unroll
+  for (int index = 0; index < num_per_block; ++index) {
+    const int col_id = blockIdx.x * blockDim.x + threadIdx.y + index * blockDim.y;
+    if (col_id < cols) {
+      float gamma_sum = dgamma[threadIdx.x][threadIdx.y + index * blockDim.y];
+      float global_dgamma = WarpReduce<float>(gamma_sum);
+      if (threadIdx.x == 0) {
+        const int offset = blockIdx.y * cols + col_id;
+        tmp_gamma_diff[offset] = global_dgamma;
+      }
+    }
+  }
+}
+
+template <typename T, typename V>
+__global__ void LayerNormBetaGradStep1(int rows, int cols, const V* __restrict__ dy,
+                                        const T* __restrict__ x, const float* __restrict__ mean,
+                                        const float* __restrict__ inv_var, float* __restrict__ tmp_beta_diff) {
+  __shared__ float dbeta[32][33];
+  float dbeta_sum[num_per_block];
+#pragma unroll
+  for (int index = 0; index < num_per_block; ++index) {
+    dbeta_sum[index] = 0;
+  }
+  const int col_id = blockIdx.x * blockDim.x + threadIdx.x;
+  if (col_id < cols) {
+    for (int i = blockIdx.y * tile_size + threadIdx.y; i < rows; i += tile_size * gridDim.y) {
+#pragma unroll
+      for (int index = 0; index < num_per_block; ++index) {
+        int row_id = i + index * blockDim.y;
+        if (row_id < rows) {
+          int offset = row_id * cols + col_id;
+          const float dy_val = static_cast<float>(dy[offset]);
+          dbeta_sum[index] += dy_val;
+        }
+      }
+    }
+  }
+#pragma unroll
+  for (int index = 0; index < num_per_block; ++index) {
+    dbeta[index * blockDim.y + threadIdx.y][threadIdx.x] = dbeta_sum[index];
+  }
+  __syncthreads();
+#pragma unroll
+  for (int index = 0; index < num_per_block; ++index) {
+    const int col_id = blockIdx.x * blockDim.x + threadIdx.y + index * blockDim.y;
+    if (col_id < cols) {
+      float beta_sum = dbeta[threadIdx.x][threadIdx.y + index * blockDim.y];
+      float global_dbeta = WarpReduce<float>(beta_sum);
+      if (threadIdx.x == 0) {
+        const int offset = blockIdx.y * cols + col_id;
+        tmp_beta_diff[offset] = global_dbeta;
+      }
+    }
+  }
+}
+
+template <typename V>
+__global__ void LayerNormParamGradStep2(const float* part_grad_gamma, const float* part_grad_beta,
+                                        const int part_size, const int n1, const int n2,
+                                        V* grad_gamma, V* grad_beta) {
+    // sum partial gradients for gamma and beta
+    SharedMemory<float> shared;
+    float* buf = shared.getPointer();
+    int i2 = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i2 < n2) {
+        // each warp does sequential reductions until reduced part_size is num_warps
+        // int num_warp_reductions = part_size / blockDim.y;
+        float sum_gamma = float(0);
+        float sum_beta = float(0);
+        const float* part_grad_gamma_ptr = part_grad_gamma + i2;
+        const float* part_grad_beta_ptr = part_grad_beta + i2;
+        for (int row_idx = threadIdx.y; row_idx < part_size; row_idx += blockDim.y) {
+            sum_gamma += part_grad_gamma_ptr[row_idx * n2];
+            sum_beta += part_grad_beta_ptr[row_idx * n2];
+        }
+        // inter-warp reductions
+        const int nbsize3 = blockDim.x * blockDim.y / 2;
+        for (int offset = blockDim.y / 2; offset >= 1; offset /= 2) {
+            // top half write to shared memory
+            if (threadIdx.y >= offset && threadIdx.y < 2 * offset) {
+                const int write_idx = (threadIdx.y - offset) * blockDim.x + threadIdx.x;
+                buf[write_idx] = sum_gamma;
+                buf[write_idx + nbsize3] = sum_beta;
+            }
+            __syncthreads();
+            // bottom half sums
+            if (threadIdx.y < offset) {
+                const int read_idx = threadIdx.y * blockDim.x + threadIdx.x;
+                sum_gamma += buf[read_idx];
+                sum_beta += buf[read_idx + nbsize3];
+            }
+            __syncthreads();
+        }
+        // write out fully summed gradients
+        if (threadIdx.y == 0) {
+            grad_gamma[i2] = sum_gamma;
+            grad_beta[i2] = sum_beta;
+        }
+    }
+}
+
+template <typename V>
+__global__ void LayerNormGammaGradStep2(const float* part_grad_gamma, const int part_size, const int n1, const int n2, V* grad_gamma) {
+    // sum partial gradients for gamma and beta
+    SharedMemory<float> shared;
+    float* buf = shared.getPointer();
+    int i2 = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i2 < n2) {
+        // each warp does sequential reductions until reduced part_size is num_warps
+        // int num_warp_reductions = part_size / blockDim.y;
+        float sum_gamma = float(0);
+        const float* part_grad_gamma_ptr = part_grad_gamma + i2;
+        for (int row_idx = threadIdx.y; row_idx < part_size; row_idx += blockDim.y) {
+            sum_gamma += part_grad_gamma_ptr[row_idx * n2];
+        }
+        // inter-warp reductions
+        for (int offset = blockDim.y / 2; offset >= 1; offset /= 2) {
+            // top half write to shared memory
+            if (threadIdx.y >= offset && threadIdx.y < 2 * offset) {
+                const int write_idx = (threadIdx.y - offset) * blockDim.x + threadIdx.x;
+                buf[write_idx] = sum_gamma;
+            }
+            __syncthreads();
+            // bottom half sums
+            if (threadIdx.y < offset) {
+                const int read_idx = threadIdx.y * blockDim.x + threadIdx.x;
+                sum_gamma += buf[read_idx];
+            }
+            __syncthreads();
+        }
+        // write out fully summed gradients
+        if (threadIdx.y == 0) {
+            grad_gamma[i2] = sum_gamma;
+        }
+    }
+}
+
+template <typename V>
+__global__ void LayerNormBetaGradStep2(const float* part_grad_beta, const int part_size, const int n1, const int n2, V* grad_beta) {
+    // sum partial gradients for gamma and beta
+    SharedMemory<float> shared;
+    float* buf = shared.getPointer();
+    int i2 = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i2 < n2) {
+        // each warp does sequential reductions until reduced part_size is num_warps
+        // int num_warp_reductions = part_size / blockDim.y;
+        float sum_beta = float(0);
+        const float* part_grad_beta_ptr = part_grad_beta + i2;
+        for (int row_idx = threadIdx.y; row_idx < part_size; row_idx += blockDim.y) {
+            sum_beta += part_grad_beta_ptr[row_idx * n2];
+        }
+        // inter-warp reductions
+        for (int offset = blockDim.y / 2; offset >= 1; offset /= 2) {
+            // top half write to shared memory
+            if (threadIdx.y >= offset && threadIdx.y < 2 * offset) {
+                const int write_idx = (threadIdx.y - offset) * blockDim.x + threadIdx.x;
+                buf[write_idx] = sum_beta;
+            }
+            __syncthreads();
+            // bottom half sums
+            if (threadIdx.y < offset) {
+                const int read_idx = threadIdx.y * blockDim.x + threadIdx.x;
+                sum_beta += buf[read_idx];
+            }
+            __syncthreads();
+        }
+        // write out fully summed gradients
+        if (threadIdx.y == 0) {
+            grad_beta[i2] = sum_beta;
+        }
+    }
+}
+
+template <typename T, typename V>
+__global__ void LayerNormInputGrad(const V* __restrict__ dout, const T* __restrict__ input,
+                                   const int rows, const int cols, const float* __restrict__ mean,
+                                   const float* __restrict__ invvar, float epsilon, const V* gamma,
+                                   T* grad_input) {
+    int WarpPerBlock = blockDim.x / WarpSize;
+    int thread_idx = threadIdx.x;
+    int warp_idx = thread_idx / WarpSize;
+    int lane_idx = thread_idx % WarpSize;
+
+    float* shared_dout = shared_data + warp_idx*cols;
+    float* shared_input = shared_data + WarpPerBlock*cols + warp_idx*cols;
+    float* shared_gamma = shared_data + 2*WarpPerBlock*cols;
+    int row_stride = gridDim.x*WarpPerBlock;
+    for(int row = blockIdx.x*WarpPerBlock+warp_idx; row < rows; row += row_stride) {
+        float mean_r = mean[row];
+        float invvar_r = invvar[row];
+        // load dout, input and gamma
+        long long data_offset = (long long)(row) * cols;
+        const V* dout_r = dout + data_offset;
+        const T* input_r = input + data_offset;
+        T* grad_input_r = grad_input + data_offset;
+#pragma unroll
+        for(int col = lane_idx; col < cols; col += WarpSize) {
+            shared_dout[col] = float(dout_r[col]);
+            shared_input[col] = float(input_r[col]);
+        }
+        if(warp_idx == 0) {
+#pragma unroll
+            for(int col = lane_idx; col < cols; col += WarpSize) {
+                shared_gamma[col] = gamma != NULL ? float(gamma[col]) : 1.0f;
+            }
+        }
+        __syncthreads();
+
+        float gamma_dout = 0.0;
+        float gamma_dout_input_mean = 0.0;
+        // reduction, gamma*dout and gamma*dout*(input-mean)
+#pragma unroll
+        for(int col = lane_idx; col < cols; col += WarpSize) {
+            float temp = shared_gamma[col] * shared_dout[col];
+            gamma_dout += temp;
+            gamma_dout_input_mean += temp * (shared_input[col] - mean_r);
+        }
+        float global_gamma_dout = WarpReduce<float>(gamma_dout);
+        float global_gamma_dout_input_mean = WarpReduce<float>(gamma_dout_input_mean);
+
+        float part3_temp_value = global_gamma_dout_input_mean * invvar_r * invvar_r * invvar_r / cols;
+        float part2 = global_gamma_dout * invvar_r / cols;
+#pragma unroll
+        for(int col = lane_idx; col < cols; col += WarpSize) {
+            float part1 = shared_gamma[col] * shared_dout[col] * invvar_r;
+            float part3 = (shared_input[col] - mean_r) * part3_temp_value;
+            grad_input_r[col] = part1 - part2 - part3;
+        }
+    }
+}
+
+template <typename T, typename V>
+int GetGirdDimY(const int64_t num_instances, const int64_t norm_size) {
+    const int grid_dim_x = (norm_size + tile_size - 1) / tile_size;
+    const int max_grid_dim_y = (num_instances + tile_size - 1) / tile_size;
+    const int block_size = block_dim_x * block_dim_y;
+    int max_active_blocks = 0;
+    cudaOccupancyMaxActiveBlocksPerMultiprocessor(
+        &max_active_blocks, LayerNormParamGradStep1<T, V>, block_size, 0);
+    int waves = 1;
+    int dev;
+    cudaGetDevice(&dev);
+    int sm_count;
+    cudaDeviceGetAttribute(&sm_count, cudaDevAttrMultiProcessorCount, dev);
+    int num_blocks = max_active_blocks * sm_count * waves;
+    int grid_dim_y = std::min(max_grid_dim_y, static_cast<int>(num_blocks / grid_dim_x));
+    return std::max(grid_dim_y, 1);
+}
+
+template <typename T, typename V>
+void HostLayerNormGradient(const V* dout, const float* mean, const float* invvar, at::Tensor* input, int n1,
+                           int n2, const V* gamma, const V* beta, double epsilon, T* grad_input,
+                           V* grad_gamma, V* grad_beta) {
+    auto stream = at::cuda::getCurrentCUDAStream().stream();
+
+    if (gamma != NULL && beta != NULL) {
+        // compute grad_gamma(j) and grad_beta(j)
+        const int part_size = GetGirdDimY<T, V>(n1, n2);
+        const int grid_dim_x = (n2 + tile_size - 1) / tile_size;
+        const int grid_dim_y = part_size;
+
+        at::Tensor part_grad_gamma = at::empty({part_size, n2}, input->options().dtype(at::ScalarType::Float));
+        at::Tensor part_grad_beta = at::empty_like(part_grad_gamma);
+        LayerNormParamGradStep1<T, V><<<dim3(grid_dim_x, grid_dim_y), dim3(32, 32 / num_per_block)>>>(
+            n1, n2, dout, input->DATA_PTR<T>(), mean, invvar, part_grad_gamma.DATA_PTR<float>(), part_grad_beta.DATA_PTR<float>()
+        );
+
+        const dim3 threads3(32, 8, 1);
+        const dim3 blocks3((n2 + 32 - 1) / 32, 1, 1);
+        const int nshared3 = threads3.x * threads3.y * sizeof(float);
+        LayerNormParamGradStep2<<<blocks3, threads3, nshared3, stream>>>(
+            part_grad_gamma.DATA_PTR<float>(), part_grad_beta.DATA_PTR<float>(), part_size, n1, n2,
+            grad_gamma, grad_beta);
+    } else if (gamma != NULL && beta == NULL) {
+        // compute grad_gamma(j) and grad_beta(j)
+        const int part_size = GetGirdDimY<T, V>(n1, n2);
+        const int grid_dim_x = (n2 + tile_size - 1) / tile_size;
+        const int grid_dim_y = part_size;
+
+        at::Tensor part_grad_gamma = at::empty({part_size, n2}, input->options().dtype(at::ScalarType::Float));
+        LayerNormGammaGradStep1<T, V><<<dim3(grid_dim_x, grid_dim_y), dim3(32, 32 / num_per_block)>>>(
+            n1, n2, dout, input->DATA_PTR<T>(), mean, invvar, part_grad_gamma.DATA_PTR<float>());
+
+        const dim3 threads3(32, 8, 1);
+        const dim3 blocks3((n2 + 32 - 1) / 32, 1, 1);
+        const int nshared3 = threads3.x * threads3.y * sizeof(float);
+        LayerNormGammaGradStep2<<<blocks3, threads3, nshared3, stream>>>(
+            part_grad_gamma.DATA_PTR<float>(), part_size, n1, n2, grad_gamma);
+    } else if (gamma == NULL && beta!= NULL) {
+        // compute grad_gamma(j) and grad_beta(j)
+        const int part_size = GetGirdDimY<T, V>(n1, n2);
+        const int grid_dim_x = (n2 + tile_size - 1) / tile_size;
+        const int grid_dim_y = part_size;
+
+        at::Tensor part_grad_beta = at::empty({part_size, n2}, input->options().dtype(at::ScalarType::Float));
+        LayerNormBetaGradStep1<T, V><<<dim3(grid_dim_x, grid_dim_y), dim3(32, 32 / num_per_block)>>>(
+            n1, n2, dout, input->DATA_PTR<T>(), mean, invvar, part_grad_beta.DATA_PTR<float>()
+        );
+
+        const dim3 threads3(32, 8, 1);
+        const dim3 blocks3((n2 + 32 - 1) / 32, 1, 1);
+        const int nshared3 = threads3.x * threads3.y * sizeof(float);
+        LayerNormBetaGradStep2<<<blocks3, threads3, nshared3, stream>>>(
+            part_grad_beta.DATA_PTR<float>(), part_size, n1, n2, grad_beta);
+    }
+
+    const uint64_t maxGridY = at::cuda::getCurrentDeviceProperties()->maxGridSize[1];
+    #define BlockDim 128
+    int WarpNumPerBlock = BlockDim / WarpSize;
+    const dim3 threads1(BlockDim);
+    int nshared = sizeof(float)*n2*(WarpNumPerBlock*2 + 1);
+
+    int max_active_blocks = 0;
+    cudaOccupancyMaxActiveBlocksPerMultiprocessor(
+        &max_active_blocks, LayerNormInputGrad<T, V>, BlockDim, nshared);
+    int dev;
+    cudaGetDevice(&dev);
+    int sm_count;
+    cudaDeviceGetAttribute(&sm_count, cudaDevAttrMultiProcessorCount, dev);
+
+    const dim3 blocks1(std::min((uint64_t)((n1 + WarpNumPerBlock - 1)/WarpNumPerBlock), (uint64_t)(max_active_blocks * sm_count)));
+    LayerNormInputGrad<<<blocks1, threads1, nshared>>>(dout, input->DATA_PTR<T>(), n1, n2, mean, invvar, float(epsilon), gamma, grad_input);
+}
+
+void cuda_layer_norm_gradient(at::Tensor* dout, at::Tensor* mean, at::Tensor* invvar,
+                              at::Tensor* input, int n1, int n2, at::IntArrayRef normalized_shape,
+                              at::Tensor* gamma, at::Tensor* beta, double epsilon,
+                              at::Tensor* grad_input, at::Tensor* grad_gamma,
+                              at::Tensor* grad_beta) {
+    using namespace at;
+    DISPATCH_FLOAT_HALF_AND_BFLOAT_INOUT_TYPES(
+        input->scalar_type(), dout->scalar_type(), "cuda_layer_norm_gradient_kernel",
+        HostLayerNormGradient(dout->DATA_PTR<scalar_t_out>(), mean->DATA_PTR<float>(),
+                              invvar->DATA_PTR<float>(), input, n1, n2,
+                              gamma != NULL ? gamma->DATA_PTR<scalar_t_out>() : NULL,
+                              beta != NULL ? beta->DATA_PTR<scalar_t_out>() : NULL, epsilon,
+                              grad_input->DATA_PTR<scalar_t_in>(),
+                              gamma != NULL ? grad_gamma->DATA_PTR<scalar_t_out>() : NULL,
+                              beta != NULL ? grad_beta->DATA_PTR<scalar_t_out>() : NULL);)
+}

model/protenix/layer_norm/kernel/type_shim.h

@@ -0,0 +1,231 @@
+#include <ATen/ATen.h>
+
+#include "compat.h"
+
+#define DISPATCH_HALF_AND_BFLOAT(TYPE, NAME, ...)                           \
+    switch (TYPE) {                                                         \
+        case at::ScalarType::Half: {                                        \
+            using scalar_t = at::Half;                                      \
+            __VA_ARGS__;                                                    \
+            break;                                                          \
+        }                                                                   \
+        case at::ScalarType::BFloat16: {                                    \
+            using scalar_t = at::BFloat16;                                  \
+            __VA_ARGS__;                                                    \
+            break;                                                          \
+        }                                                                   \
+        default:                                                            \
+            AT_ERROR(#NAME, " not implemented for '", toString(TYPE), "'"); \
+    }
+
+#define DISPATCH_FLOAT_HALF_AND_BFLOAT_INOUT_TYPES(TYPEIN, TYPEOUT, NAME, ...)         \
+    switch (TYPEIN) {                                                                  \
+        case at::ScalarType::Float: {                                                  \
+            using scalar_t_in = float;                                                 \
+            switch (TYPEOUT) {                                                         \
+                case at::ScalarType::Float: {                                          \
+                    using scalar_t_out = float;                                        \
+                    __VA_ARGS__;                                                       \
+                    break;                                                             \
+                }                                                                      \
+                case at::ScalarType::Half: {                                           \
+                    using scalar_t_out = at::Half;                                     \
+                    __VA_ARGS__;                                                       \
+                    break;                                                             \
+                }                                                                      \
+                case at::ScalarType::BFloat16: {                                       \
+                    using scalar_t_out = at::BFloat16;                                 \
+                    __VA_ARGS__;                                                       \
+                    break;                                                             \
+                }                                                                      \
+                default:                                                               \
+                    AT_ERROR(#NAME, " not implemented for '", toString(TYPEOUT), "'"); \
+            }                                                                          \
+            break;                                                                     \
+        }                                                                              \
+        case at::ScalarType::Half: {                                                   \
+            using scalar_t_in = at::Half;                                              \
+            using scalar_t_out = at::Half;                                             \
+            __VA_ARGS__;                                                               \
+            break;                                                                     \
+        }                                                                              \
+        case at::ScalarType::BFloat16: {                                               \
+            using scalar_t_in = at::BFloat16;                                          \
+            using scalar_t_out = at::BFloat16;                                         \
+            __VA_ARGS__;                                                               \
+            break;                                                                     \
+        }                                                                              \
+        default:                                                                       \
+            AT_ERROR(#NAME, " not implemented for '", toString(TYPEIN), "'");          \
+    }
+
+// Forward/backward compatiblity hack around
+// https://github.com/pytorch/pytorch/commit/3aeb78079bcd68282fe9117088e138b77318e288
+// pending more future-proof guidance from upstream.
+// struct TypeShim
+// {
+//   const at::Type& payload;
+//   TypeShim(const at::Type& type) : payload(type) {}
+//   // Enable trivial conversion to a const at::Type& for pre-3aeb78
+//   operator const at::Type&(){ return payload; };
+//   // Enable dispatch switch statements to take *this directly for  post-3aeb78
+//   //operator at::ScalarType(){ return payload.; };
+// };
+
+#define DISPATCH_FLOAT_AND_HALF(TYPE, LEVEL, NAME, ...)                     \
+    switch (TYPE) {                                                         \
+        case at::ScalarType::Float: {                                       \
+            using scalar_t_##LEVEL = float;                                 \
+            __VA_ARGS__;                                                    \
+            break;                                                          \
+        }                                                                   \
+        case at::ScalarType::Half: {                                        \
+            using scalar_t_##LEVEL = at::Half;                              \
+            __VA_ARGS__;                                                    \
+            break;                                                          \
+        }                                                                   \
+        default:                                                            \
+            AT_ERROR(#NAME, " not implemented for '", toString(TYPE), "'"); \
+    }
+
+#define DISPATCH_FLOAT_HALF_AND_BYTE(TYPE, LEVEL, NAME, ...)                \
+    switch (TYPE) {                                                         \
+        case at::ScalarType::Float: {                                       \
+            using scalar_t_##LEVEL = float;                                 \
+            __VA_ARGS__;                                                    \
+            break;                                                          \
+        }                                                                   \
+        case at::ScalarType::Half: {                                        \
+            using scalar_t_##LEVEL = at::Half;                              \
+            __VA_ARGS__;                                                    \
+            break;                                                          \
+        }                                                                   \
+        case at::ScalarType::Byte: {                                        \
+            using scalar_t_##LEVEL = uint8_t;                               \
+            __VA_ARGS__;                                                    \
+            break;                                                          \
+        }                                                                   \
+        default:                                                            \
+            AT_ERROR(#NAME, " not implemented for '", toString(TYPE), "'"); \
+    }
+
+#define DISPATCH_DOUBLE_FLOAT_AND_HALF(TYPE, LEVEL, NAME, ...)              \
+    switch (TYPE) {                                                         \
+        case at::ScalarType::Double: {                                      \
+            using scalar_t_##LEVEL = double;                                \
+            __VA_ARGS__;                                                    \
+            break;                                                          \
+        }                                                                   \
+        case at::ScalarType::Float: {                                       \
+            using scalar_t_##LEVEL = float;                                 \
+            __VA_ARGS__;                                                    \
+            break;                                                          \
+        }                                                                   \
+        case at::ScalarType::Half: {                                        \
+            using scalar_t_##LEVEL = at::Half;                              \
+            __VA_ARGS__;                                                    \
+            break;                                                          \
+        }                                                                   \
+        default:                                                            \
+            AT_ERROR(#NAME, " not implemented for '", toString(TYPE), "'"); \
+    }
+
+#define DISPATCH_DOUBLE_AND_FLOAT(TYPE, LEVEL, NAME, ...)                   \
+    switch (TYPE) {                                                         \
+        case at::ScalarType::Double: {                                      \
+            using scalar_t_##LEVEL = double;                                \
+            __VA_ARGS__;                                                    \
+            break;                                                          \
+        }                                                                   \
+        case at::ScalarType::Float: {                                       \
+            using scalar_t_##LEVEL = float;                                 \
+            __VA_ARGS__;                                                    \
+            break;                                                          \
+        }                                                                   \
+        default:                                                            \
+            AT_ERROR(#NAME, " not implemented for '", toString(TYPE), "'"); \
+    }
+
+template <typename T>
+__device__ __forceinline__ T
+reduce_block_into_lanes(T *x, T val, int lanes = 1,
+                        bool share_result = false)  // lanes is intended to be <= 32.
+{
+    int tid = threadIdx.x + threadIdx.y * blockDim.x;
+    int blockSize = blockDim.x * blockDim.y;  // blockSize is intended to be a multiple of 32.
+
+    if (blockSize >= 64) {
+        x[tid] = val;
+        __syncthreads();
+    }
+
+#pragma unroll
+    for (int i = (blockSize >> 1); i >= 64; i >>= 1) {
+        if (tid < i) x[tid] = x[tid] + x[tid + i];
+        __syncthreads();
+    }
+
+    T final;
+
+    if (tid < 32) {
+        if (blockSize >= 64)
+            final = x[tid] + x[tid + 32];
+        else
+            final = val;
+            // __SYNCWARP();
+
+#pragma unroll
+        for (int i = 16; i >= lanes; i >>= 1)
+            final = final + __shfl_down_sync(0xffffffff, final, i);
+    }
+
+    if (share_result) {
+        if (tid < lanes) x[tid] = final;  // EpilogueOp
+        // Make sure the smem result is visible to all warps.
+        __syncthreads();
+    }
+
+    return final;
+}
+
+template <typename T>
+__device__ __forceinline__ T
+reduce_block_into_lanes_max_op(T *x, T val, int lanes = 1,
+                               bool share_result = false)  // lanes is intended to be <= 32.
+{
+    int tid = threadIdx.x + threadIdx.y * blockDim.x;
+    int blockSize = blockDim.x * blockDim.y;  // blockSize is intended to be a multiple of 32.
+
+    if (blockSize >= 64) {
+        x[tid] = val;
+        __syncthreads();
+    }
+
+#pragma unroll
+    for (int i = (blockSize >> 1); i >= 64; i >>= 1) {
+        if (tid < i) x[tid] = fmaxf(fabsf(x[tid]), fabsf(x[tid + i]));
+        __syncthreads();
+    }
+
+    T final;
+
+    if (tid < 32) {
+        if (blockSize >= 64)
+            final = fmaxf(fabsf(x[tid]), fabsf(x[tid + 32]));
+        else
+            final = val;
+            // __SYNCWARP();
+
+#pragma unroll
+        for (int i = 16; i >= lanes; i >>= 1)
+            final = fmaxf(fabsf(final), fabsf(__shfl_down_sync(0xffffffff, final, i)));
+    }
+
+    if (share_result) {
+        if (tid < lanes) x[tid] = final;  // EpilogueOp
+        // Make sure the smem result is visible to all warps.
+        __syncthreads();
+    }
+
+    return final;
+}

model/protenix/layer_norm/layer_norm.py

@@ -0,0 +1,198 @@
+import importlib
+import numbers
+import os
+import sys
+import time
+
+import torch
+from torch.nn.parameter import Parameter
+
+sys.path.append(os.path.dirname(__file__))
+
+try:
+    fastfold_layer_norm_cuda = importlib.import_module("fastfold_layer_norm_cuda")
+except ImportError:
+    from model.protenix.layer_norm.torch_ext_compile import compile
+
+    current_dir = os.path.dirname(__file__)
+    fastfold_layer_norm_cuda = compile(
+        name="fastfold_layer_norm_cuda",
+        sources=[
+            os.path.join(f"{current_dir}/kernel", file)
+            for file in ["layer_norm_cuda.cpp", "layer_norm_cuda_kernel.cu"]
+        ],
+        extra_include_paths=[f"{current_dir}/kernel"],
+        build_directory=current_dir,
+    )
+
+
+class FusedLayerNormAffineFunction(torch.autograd.Function):
+
+    @staticmethod
+    def forward(ctx, input, weight, bias, normalized_shape, eps):
+        d = input.dtype
+
+        ctx.normalized_shape = normalized_shape
+        ctx.eps = eps
+        input_ = input.contiguous()
+
+        if weight is None:
+            if bias is None:
+                output, mean, invvar = fastfold_layer_norm_cuda.forward_none_affine(
+                    input_, ctx.normalized_shape, ctx.eps
+                )
+            else:
+                output, mean, invvar = (
+                    fastfold_layer_norm_cuda.forward_with_bias_affine(
+                        input_, ctx.normalized_shape, bias.to(d), ctx.eps
+                    )
+                )
+        else:
+            if bias is None:
+                output, mean, invvar = (
+                    fastfold_layer_norm_cuda.forward_with_weight_affine(
+                        input_, ctx.normalized_shape, weight.to(d), ctx.eps
+                    )
+                )
+            else:
+                output, mean, invvar = (
+                    fastfold_layer_norm_cuda.forward_with_both_affine(
+                        input_, ctx.normalized_shape, weight.to(d), bias.to(d), ctx.eps
+                    )
+                )
+        ctx.save_for_backward(input_, weight, bias, mean, invvar)
+        return output
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        d = grad_output.dtype
+        input_, weight_, bias_, mean, invvar = ctx.saved_tensors
+        grad_input = grad_weight = grad_bias = None
+
+        if weight_ is None:
+            if bias_ is None:
+                grad_input, grad_weight, grad_bias = (
+                    fastfold_layer_norm_cuda.backward_none_affine(
+                        grad_output.contiguous(),
+                        mean,
+                        invvar,
+                        input_,
+                        ctx.normalized_shape,
+                        ctx.eps,
+                    )
+                )
+            else:
+                grad_input, grad_weight, grad_bias = (
+                    fastfold_layer_norm_cuda.backward_with_bias_affine(
+                        grad_output.contiguous(),
+                        mean,
+                        invvar,
+                        input_,
+                        ctx.normalized_shape,
+                        bias_.to(dtype=d),
+                        ctx.eps,
+                    )
+                )
+        else:
+            if bias_ is None:
+                grad_input, grad_weight, grad_bias = (
+                    fastfold_layer_norm_cuda.backward_with_weight_affine(
+                        grad_output.contiguous(),
+                        mean,
+                        invvar,
+                        input_,
+                        ctx.normalized_shape,
+                        weight_.to(dtype=d),
+                        ctx.eps,
+                    )
+                )
+            else:
+                grad_input, grad_weight, grad_bias = (
+                    fastfold_layer_norm_cuda.backward_with_both_affine(
+                        grad_output.contiguous(),
+                        mean,
+                        invvar,
+                        input_,
+                        ctx.normalized_shape,
+                        weight_.to(dtype=d),
+                        bias_.to(dtype=d),
+                        ctx.eps,
+                    )
+                )
+        return (
+            grad_input,
+            None if weight_ is None else grad_weight,
+            None if bias_ is None else grad_bias,
+            None,
+            None,
+        )
+
+
+class FusedLayerNorm(torch.nn.Module):
+
+    def __init__(
+        self,
+        normalized_shape,
+        create_scale=True,
+        create_offset=True,
+        eps=1e-5,
+    ):
+        """
+        Args:
+            normalized_shape (int or list or torch.Size) input shape from an expected input of size
+            create_scale (bool) If set to False, the layer will not learn an additive weight, Default: True
+            create_offset (bool) If set to False, the layer will not learn an additive bias, Default: True
+            eps (float) a value added to the denominator for numerical stability. Default: 1e-5
+        """
+        super(FusedLayerNorm, self).__init__()
+
+        if isinstance(normalized_shape, numbers.Integral):
+            normalized_shape = (normalized_shape,)
+        self.normalized_shape = torch.Size(normalized_shape)
+        self.eps = eps
+        if create_scale:
+            self.weight = Parameter(torch.ones(*normalized_shape))
+        else:
+            self.weight = None
+
+        if create_offset:
+            self.bias = Parameter(torch.zeros(*normalized_shape))
+        else:
+            self.bias = None
+
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        if self.weight is not None:
+            torch.nn.init.ones_(self.weight)
+        if self.bias is not None:
+            torch.nn.init.zeros_(self.bias)
+
+    def forward(self, input):
+        return FusedLayerNormAffineFunction.apply(
+            input, self.weight, self.bias, self.normalized_shape, self.eps
+        )
+
+
+if __name__ == "__main__":
+    dtype = torch.float32
+    data = torch.rand(10, 10).cuda().to(dtype=dtype)
+    data1 = data * 1
+    data.requires_grad = True
+    data1.requires_grad = True
+    layer_norm = (
+        FusedLayerNorm(10, create_scale=True, create_offset=True).cuda().to(dtype=dtype)
+    )
+    layer_norm_torch = torch.nn.LayerNorm(10).cuda().to(dtype=dtype)
+    out = layer_norm(data)
+    out1 = layer_norm_torch(data1)
+    # print(out - out1)
+    loss = out.sum()
+    loss.backward()
+    loss1 = out1.sum()
+    loss1.backward()
+    print(data.grad - data1.grad)
+    print(layer_norm.weight.grad - layer_norm_torch.weight.grad)
+    print(layer_norm.bias.grad - layer_norm_torch.bias.grad)
+    print(layer_norm.weight.grad, layer_norm.bias.grad)
+

model/protenix/layer_norm/torch_ext_compile.py

@@ -0,0 +1,41 @@
+import os
+
+from torch.utils.cpp_extension import load
+
+
+def compile(name, sources, extra_include_paths, build_directory):
+    os.environ["TORCH_CUDA_ARCH_LIST"] = "7.0;8.0"
+    return load(
+        name=name,
+        sources=sources,
+        extra_include_paths=extra_include_paths,
+        extra_cflags=[
+            "-O3",
+            "-DVERSION_GE_1_1",
+            "-DVERSION_GE_1_3",
+            "-DVERSION_GE_1_5",
+        ],
+        extra_cuda_cflags=[
+            "-O3",
+            "--use_fast_math",
+            "-DVERSION_GE_1_1",
+            "-DVERSION_GE_1_3",
+            "-DVERSION_GE_1_5",
+            "-std=c++17",
+            "-maxrregcount=50",
+            "-U__CUDA_NO_HALF_OPERATORS__",
+            "-U__CUDA_NO_HALF_CONVERSIONS__",
+            "--expt-relaxed-constexpr",
+            "--expt-extended-lambda",
+            "-gencode",
+            "arch=compute_70,code=sm_70",
+            "-gencode",
+            "arch=compute_80,code=sm_80",
+            "-gencode",
+            "arch=compute_86,code=sm_86",
+            "-gencode",
+            "arch=compute_90,code=sm_90",
+        ],
+        verbose=True,
+        build_directory=build_directory,
+    )

scripts/check_import_boundaries.py

@@ -0,0 +1,67 @@
+from pathlib import Path
+import re
+import sys
+
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+MODEL_ROOT = PROJECT_ROOT / "model"
+SCRIPT_ROOT = PROJECT_ROOT / "scripts"
+
+
+def _python_files(root: Path):
+    return [path for path in root.rglob("*.py") if "__pycache__" not in path.parts]
+
+
+def _check_no_onescience_models_imports():
+    offenders = []
+    for path in _python_files(MODEL_ROOT):
+        text = path.read_text(encoding="utf-8")
+        if "onescience.models." in text:
+            offenders.append(str(path.relative_to(PROJECT_ROOT)))
+    return offenders
+
+
+def _check_no_legacy_imports():
+    pattern = re.compile(
+        r"^\s*(from\s+(esm|openfold)(\.|\s)|import\s+(esm|openfold)(\.|\s|$))",
+        re.MULTILINE,
+    )
+    offenders = []
+    for root in (MODEL_ROOT, SCRIPT_ROOT):
+        for path in _python_files(root):
+            text = path.read_text(encoding="utf-8")
+            if pattern.search(text):
+                offenders.append(str(path.relative_to(PROJECT_ROOT)))
+    return offenders
+
+
+def _check_script_bootstrap():
+    offenders = []
+    for path in _python_files(SCRIPT_ROOT):
+        text = path.read_text(encoding="utf-8")
+        if "model.esm" in text and '_PROJECT_ROOT / "model"' not in text:
+            offenders.append(str(path.relative_to(PROJECT_ROOT)))
+    return offenders
+
+
+def main():
+    checks = {
+        "onescience.models imports": _check_no_onescience_models_imports(),
+        "legacy esm/openfold imports": _check_no_legacy_imports(),
+        "script project-root bootstrap": _check_script_bootstrap(),
+    }
+    failed = {name: offenders for name, offenders in checks.items() if offenders}
+    if failed:
+        for name, offenders in failed.items():
+            print(f"{name}:")
+            for offender in offenders:
+                print(f"  {offender}")
+        return 1
+
+    print("Import boundary checks passed")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+

scripts/download_weights.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+# Usage: bash scripts/download_weights.sh /path/to/weights/
+# Re-run the command to continue downloading incomplete files.
+echo "$0"
+run_dir=$(readlink -f "$(dirname "$0")") ;
+
+if [[ "$(which aria2c)" == "" ]]; then
+  echo "aria2c is required for downloading!"
+  exit 1
+fi
+
+for _sh_exec in /bin/sh bash zsh; do
+  # Only bash and zsh are tested.
+  _sh_exec=$(which ${_sh_exec})
+  if [[ "$(${_sh_exec} --version)" =~ "bash" || "$(${_sh_exec} --version)" =~ "zsh" ]]; then
+    echo "$(which ${_sh_exec})";
+    break
+  fi
+done
+
+model_pth=$1
+if [[ "$model_pth" == "" ]]; then
+  model_pth=$PWD
+fi
+
+mkdir -p "$model_pth/checkpoints"
+
+pushd "$model_pth/checkpoints"
+cat "$run_dir/../docs/UPSTREAM_README.md" | grep -e '^|' | grep -e 'fair-esm/models' | tr -d '|' | \
+  awk 'BEGIN{print "set -e"};
+  {
+      # read urls starts with https
+      split($0,arr,"https");
+      url="https"arr[2];
+
+      # remove blank spaces after url string
+      split(url,url_arr," ")
+      url=url_arr[1]
+
+      # guessing regression pt url
+      url_regression=url;
+      sub("models","regression",url_regression);
+      sub(".pt","-contact-regression.pt",url_regression);
+
+      # downloading weight
+      url_basename_idx=split(url,url_arr,"/")
+      url_basename=url_arr[url_basename_idx]
+      print "if [[ ! -f "url_basename" || -f "url_basename".aria2 ]];then echo Download not complete: "url_basename";aria2c -x 10 "url";else echo Download complete: "url_basename";fi"
+
+      # downloading regression, if not existing and through an error, we just ignore it.
+      url_regression_basename_idx=split(url_regression,url_arr,"/")
+      url_regression_basename=url_arr[url_regression_basename_idx]
+      print "if [[ ! -f "url_regression_basename" || -f "url_regression_basename".aria2 ]];then echo Download not complete: "url_regression_basename";aria2c -x 10 "url_regression" 2>/dev/null || echo Never mind. "url_regression_basename" may not exist. ;else echo Download complete: "url_regression_basename";fi"
+
+      }' | $_sh_exec
+
+
+echo "Your model directory is located at \`$(readlink -f "${model_pth}")\`."
+
+popd

scripts/esm2_infer_fairscale_fsdp_cpu_offloading.py

@@ -0,0 +1,65 @@
+from pathlib import Path
+import sys
+
+_PROJECT_FILE = Path(__file__).resolve()
+for _PROJECT_ROOT in _PROJECT_FILE.parents:
+    if (_PROJECT_ROOT / "model").is_dir():
+        if str(_PROJECT_ROOT) not in sys.path:
+            sys.path.insert(0, str(_PROJECT_ROOT))
+        break
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import torch
+from fairscale.nn.data_parallel import FullyShardedDataParallel as FSDP
+from fairscale.nn.wrap import enable_wrap, wrap
+
+import model.esm as esm
+
+# init the distributed world with world_size 1
+url = "tcp://localhost:23456"
+torch.distributed.init_process_group(backend="nccl", init_method=url, world_size=1, rank=0)
+
+# download model data from the hub
+model_name = "esm2_t48_15B_UR50D"
+model_data, regression_data = esm.pretrained._download_model_and_regression_data(model_name)
+
+# initialize the model with FSDP wrapper
+fsdp_params = dict(
+    mixed_precision=True,
+    flatten_parameters=True,
+    state_dict_device=torch.device("cpu"),  # reduce GPU mem usage
+    cpu_offload=True,  # enable cpu offloading
+)
+with enable_wrap(wrapper_cls=FSDP, **fsdp_params):
+    model, vocab = esm.pretrained.load_model_and_alphabet_core(
+        model_name, model_data, regression_data
+    )
+    batch_converter = vocab.get_batch_converter()
+    model.eval()
+
+    # Wrap each layer in FSDP separately
+    for name, child in model.named_children():
+        if name == "layers":
+            for layer_name, layer in child.named_children():
+                wrapped_layer = wrap(layer)
+                setattr(child, layer_name, wrapped_layer)
+    model = wrap(model)
+
+data = [
+    ("protein1", "MKTVRQERLKSIVRILERSKEPVSGAQLAEELSVSRQVIVQDIAYLRSLGYNIVATPRGYVLAGG"),
+    ("protein2", "KALTARQQEVFDLIRDHISQTGMPPTRAEIAQRLGFRSPNAAEEHLKALARKGVIEIVSGASRGIRLLQEE"),
+    (
+        "protein2 with mask",
+        "KALTARQQEVFDLIRD<mask>ISQTGMPPTRAEIAQRLGFRSPNAAEEHLKALARKGVIEIVSGASRGIRLLQEE",
+    ),
+    ("protein3", "K A <mask> I S Q"),
+]
+
+batch_labels, batch_strs, batch_tokens = batch_converter(data)
+batch_tokens = batch_tokens.cuda()
+with torch.no_grad():
+    results = model(batch_tokens, repr_layers=[48], return_contacts=True)
+print(results)

scripts/extract.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3 -u
+from pathlib import Path
+import sys
+
+_PROJECT_FILE = Path(__file__).resolve()
+for _PROJECT_ROOT in _PROJECT_FILE.parents:
+    if (_PROJECT_ROOT / "model").is_dir():
+        if str(_PROJECT_ROOT) not in sys.path:
+            sys.path.insert(0, str(_PROJECT_ROOT))
+        break
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import argparse
+import pathlib
+
+import torch
+
+from model.esm import Alphabet, FastaBatchedDataset, ProteinBertModel, pretrained, MSATransformer
+
+
+def create_parser():
+    parser = argparse.ArgumentParser(
+        description="Extract per-token representations and model outputs for sequences in a FASTA file"  # noqa
+    )
+
+    parser.add_argument(
+        "model_location",
+        type=str,
+        help="PyTorch model file OR name of pretrained model to download (see README for models)",
+    )
+    parser.add_argument(
+        "fasta_file",
+        type=pathlib.Path,
+        help="FASTA file on which to extract representations",
+    )
+    parser.add_argument(
+        "output_dir",
+        type=pathlib.Path,
+        help="output directory for extracted representations",
+    )
+
+    parser.add_argument("--toks_per_batch", type=int, default=4096, help="maximum batch size")
+    parser.add_argument(
+        "--repr_layers",
+        type=int,
+        default=[-1],
+        nargs="+",
+        help="layers indices from which to extract representations (0 to num_layers, inclusive)",
+    )
+    parser.add_argument(
+        "--include",
+        type=str,
+        nargs="+",
+        choices=["mean", "per_tok", "bos", "contacts"],
+        help="specify which representations to return",
+        required=True,
+    )
+    parser.add_argument(
+        "--truncation_seq_length",
+        type=int,
+        default=1022,
+        help="truncate sequences longer than the given value",
+    )
+
+    parser.add_argument("--nogpu", action="store_true", help="Do not use GPU even if available")
+    return parser
+
+
+def run(args):
+    model, alphabet = pretrained.load_model_and_alphabet(args.model_location)
+    model.eval()
+    if isinstance(model, MSATransformer):
+        raise ValueError(
+            "This script currently does not handle models with MSA input (MSA Transformer)."
+        )
+    if torch.cuda.is_available() and not args.nogpu:
+        model = model.cuda()
+        print("Transferred model to GPU")
+
+    dataset = FastaBatchedDataset.from_file(args.fasta_file)
+    batches = dataset.get_batch_indices(args.toks_per_batch, extra_toks_per_seq=1)
+    data_loader = torch.utils.data.DataLoader(
+        dataset, collate_fn=alphabet.get_batch_converter(args.truncation_seq_length), batch_sampler=batches
+    )
+    print(f"Read {args.fasta_file} with {len(dataset)} sequences")
+
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    return_contacts = "contacts" in args.include
+
+    assert all(-(model.num_layers + 1) <= i <= model.num_layers for i in args.repr_layers)
+    repr_layers = [(i + model.num_layers + 1) % (model.num_layers + 1) for i in args.repr_layers]
+
+    with torch.no_grad():
+        for batch_idx, (labels, strs, toks) in enumerate(data_loader):
+            print(
+                f"Processing {batch_idx + 1} of {len(batches)} batches ({toks.size(0)} sequences)"
+            )
+            if torch.cuda.is_available() and not args.nogpu:
+                toks = toks.to(device="cuda", non_blocking=True)
+
+            out = model(toks, repr_layers=repr_layers, return_contacts=return_contacts)
+
+            logits = out["logits"].to(device="cpu")
+            representations = {
+                layer: t.to(device="cpu") for layer, t in out["representations"].items()
+            }
+            if return_contacts:
+                contacts = out["contacts"].to(device="cpu")
+
+            for i, label in enumerate(labels):
+                args.output_file = args.output_dir / f"{label}.pt"
+                args.output_file.parent.mkdir(parents=True, exist_ok=True)
+                result = {"label": label}
+                truncate_len = min(args.truncation_seq_length, len(strs[i]))
+                # Call clone on tensors to ensure tensors are not views into a larger representation
+                # See https://github.com/pytorch/pytorch/issues/1995
+                if "per_tok" in args.include:
+                    result["representations"] = {
+                        layer: t[i, 1 : truncate_len + 1].clone()
+                        for layer, t in representations.items()
+                    }
+                if "mean" in args.include:
+                    result["mean_representations"] = {
+                        layer: t[i, 1 : truncate_len + 1].mean(0).clone()
+                        for layer, t in representations.items()
+                    }
+                if "bos" in args.include:
+                    result["bos_representations"] = {
+                        layer: t[i, 0].clone() for layer, t in representations.items()
+                    }
+                if return_contacts:
+                    result["contacts"] = contacts[i, : truncate_len, : truncate_len].clone()
+
+                torch.save(
+                    result,
+                    args.output_file,
+                )
+
+
+def main():
+    parser = create_parser()
+    args = parser.parse_args()
+    run(args)
+
+if __name__ == "__main__":
+    main()

scripts/fold.py

@@ -0,0 +1,214 @@
+from pathlib import Path
+import sys
+
+_PROJECT_FILE = Path(__file__).resolve()
+for _PROJECT_ROOT in _PROJECT_FILE.parents:
+    if (_PROJECT_ROOT / "model").is_dir():
+        if str(_PROJECT_ROOT) not in sys.path:
+            sys.path.insert(0, str(_PROJECT_ROOT))
+        break
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+
+from pathlib import Path
+import sys,os
+import argparse
+import logging
+import sys
+import typing as T
+from pathlib import Path
+from timeit import default_timer as timer
+
+import torch
+
+import model.esm as esm
+from onescience.datapipes.esm import read_fasta
+
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+formatter = logging.Formatter(
+    "%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+    datefmt="%y/%m/%d %H:%M:%S",
+)
+
+console_handler = logging.StreamHandler(sys.stdout)
+console_handler.setLevel(logging.INFO)
+console_handler.setFormatter(formatter)
+logger.addHandler(console_handler)
+
+
+PathLike = T.Union[str, Path]
+
+
+def enable_cpu_offloading(model):
+    from torch.distributed.fsdp import CPUOffload, FullyShardedDataParallel
+    from torch.distributed.fsdp.wrap import enable_wrap, wrap
+
+    torch.distributed.init_process_group(
+        backend="nccl", init_method="tcp://localhost:9999", world_size=1, rank=0
+    )
+
+    wrapper_kwargs = dict(cpu_offload=CPUOffload(offload_params=True))
+
+    with enable_wrap(wrapper_cls=FullyShardedDataParallel, **wrapper_kwargs):
+        for layer_name, layer in model.layers.named_children():
+            wrapped_layer = wrap(layer)
+            setattr(model.layers, layer_name, wrapped_layer)
+        model = wrap(model)
+
+    return model
+
+
+def init_model_on_gpu_with_cpu_offloading(model):
+    model = model.eval()
+    model_esm = enable_cpu_offloading(model.esm)
+    del model.esm
+    model.cuda()
+    model.esm = model_esm
+    return model
+
+
+def create_batched_sequence_datasest(
+    sequences: T.List[T.Tuple[str, str]], max_tokens_per_batch: int = 1024
+) -> T.Generator[T.Tuple[T.List[str], T.List[str]], None, None]:
+
+    batch_headers, batch_sequences, num_tokens = [], [], 0
+    for header, seq in sequences:
+        if (len(seq) + num_tokens > max_tokens_per_batch) and num_tokens > 0:
+            yield batch_headers, batch_sequences
+            batch_headers, batch_sequences, num_tokens = [], [], 0
+        batch_headers.append(header)
+        batch_sequences.append(seq)
+        num_tokens += len(seq)
+
+    yield batch_headers, batch_sequences
+
+
+def create_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-i",
+        "--fasta",
+        help="Path to input FASTA file",
+        type=Path,
+        required=True,
+    )
+    parser.add_argument(
+        "-o", "--pdb", help="Path to output PDB directory", type=Path, required=True
+    )
+    parser.add_argument(
+        "-m", "--model-dir", help="Parent path to Pretrained ESM data directory. ", type=Path, default=None
+    )
+    parser.add_argument(
+        "--num-recycles",
+        type=int,
+        default=None,
+        help="Number of recycles to run. Defaults to number used in training (4).",
+    )
+    parser.add_argument(
+        "--max-tokens-per-batch",
+        type=int,
+        default=1024,
+        help="Maximum number of tokens per gpu forward-pass. This will group shorter sequences together "
+        "for batched prediction. Lowering this can help with out of memory issues, if these occur on "
+        "short sequences.",
+    )
+    parser.add_argument(
+        "--chunk-size",
+        type=int,
+        default=None,
+        help="Chunks axial attention computation to reduce memory usage from O(L^2) to O(L). "
+        "Equivalent to running a for loop over chunks of of each dimension. Lower values will "
+        "result in lower memory usage at the cost of speed. Recommended values: 128, 64, 32. "
+        "Default: None.",
+    )
+    parser.add_argument("--cpu-only", help="CPU only", action="store_true")
+    parser.add_argument("--cpu-offload", help="Enable CPU offloading", action="store_true")
+    return parser
+
+
+def run(args):
+    if not args.fasta.exists():
+        raise FileNotFoundError(args.fasta)
+
+    args.pdb.mkdir(exist_ok=True)
+
+    # Read fasta and sort sequences by length
+    logger.info(f"Reading sequences from {args.fasta}")
+    all_sequences = sorted(read_fasta(args.fasta), key=lambda header_seq: len(header_seq[1]))
+    logger.info(f"Loaded {len(all_sequences)} sequences from {args.fasta}")
+
+    logger.info("Loading model")
+
+    # Use pre-downloaded ESM weights from model_pth.
+    if args.model_dir is not None:
+        # if pretrained model path is available
+        torch.hub.set_dir(args.model_dir)
+
+    model = esm.pretrained.esmfold_v1()
+
+
+    model = model.eval()
+    model.set_chunk_size(args.chunk_size)
+
+    if args.cpu_only:
+        model.esm.float()  # convert to fp32 as ESM-2 in fp16 is not supported on CPU
+        model.cpu()
+    elif args.cpu_offload:
+        model = init_model_on_gpu_with_cpu_offloading(model)
+    else:
+        model.cuda()
+    logger.info("Starting Predictions")
+    batched_sequences = create_batched_sequence_datasest(all_sequences, args.max_tokens_per_batch)
+
+    num_completed = 0
+    num_sequences = len(all_sequences)
+    for headers, sequences in batched_sequences:
+        start = timer()
+        try:
+            output = model.infer(sequences, num_recycles=args.num_recycles)
+        except RuntimeError as e:
+            if e.args[0].startswith("CUDA out of memory"):
+                if len(sequences) > 1:
+                    logger.info(
+                        f"Failed (CUDA out of memory) to predict batch of size {len(sequences)}. "
+                        "Try lowering `--max-tokens-per-batch`."
+                    )
+                else:
+                    logger.info(
+                        f"Failed (CUDA out of memory) on sequence {headers[0]} of length {len(sequences[0])}."
+                    )
+
+                continue
+            raise
+
+        output = {key: value.cpu() for key, value in output.items()}
+        pdbs = model.output_to_pdb(output)
+        tottime = timer() - start
+        time_string = f"{tottime / len(headers):0.1f}s"
+        if len(sequences) > 1:
+            time_string = time_string + f" (amortized, batch size {len(sequences)})"
+        for header, seq, pdb_string, mean_plddt, ptm in zip(
+            headers, sequences, pdbs, output["mean_plddt"], output["ptm"]
+        ):
+            output_file = args.pdb / f"{header}.pdb"
+            output_file.write_text(pdb_string)
+            num_completed += 1
+            logger.info(
+                f"Predicted structure for {header} with length {len(seq)}, pLDDT {mean_plddt:0.1f}, "
+                f"pTM {ptm:0.3f} in {time_string}. "
+                f"{num_completed} / {num_sequences} completed."
+            )
+
+
+def main():
+    parser = create_parser()
+    args = parser.parse_args()
+    run(args)
+
+if __name__ == "__main__":
+    main()

scripts/infer.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+cd "${PROJECT_ROOT}"
+
+ESM_WEIGHT_DIR="${ESM_WEIGHT_DIR:-${PROJECT_ROOT}/weight}"
+ESM_OUTPUT_DIR="${ESM_OUTPUT_DIR:-${PROJECT_ROOT}/outputs}"
+ESM_FASTA="${ESM_FASTA:-${PROJECT_ROOT}/data/fasta/few_proteins.fasta}"
+ESM2_8M_WEIGHT="${ESM2_8M_WEIGHT:-${ESM_WEIGHT_DIR}/esm2_t6_8M_UR50D.pt}"
+
+mkdir -p "${ESM_OUTPUT_DIR}"
+
+if [[ ! -f "${ESM2_8M_WEIGHT}" ]]; then
+  echo "Missing ${ESM2_8M_WEIGHT}"
+  echo "Run: bash scripts/download_weights.sh ${ESM_WEIGHT_DIR}"
+  exit 1
+fi
+
+python scripts/extract.py \
+  "${ESM2_8M_WEIGHT}" \
+  "${ESM_FASTA}" \
+  "${ESM_OUTPUT_DIR}/embeddings" \
+  --include mean per_tok \
+  --repr_layers 6
+
+if [[ "${RUN_ESMFOLD:-0}" == "1" ]]; then
+  python scripts/fold.py \
+    -i "${ESM_FASTA}" \
+    -o "${ESM_OUTPUT_DIR}/pdb" \
+    --model-dir "${ESM_WEIGHT_DIR}" \
+    --cpu-only
+fi
+
+if [[ "${RUN_VARIANT_PREDICTION:-0}" == "1" ]]; then
+  python scripts/variant_prediction/predict.py \
+    --model-location "${ESM_WEIGHT_DIR}/esm1v_t33_650M_UR90S_1.pt" \
+    --sequence "${ESM_VARIANT_SEQUENCE:?Set ESM_VARIANT_SEQUENCE}" \
+    --dms-input data/variant_prediction/BLAT_ECOLX_Ranganathan2015.csv \
+    --mutation-col mutant \
+    --dms-output "${ESM_OUTPUT_DIR}/variant_prediction.csv" \
+    --offset-idx 24 \
+    --scoring-strategy wt-marginals
+fi

scripts/inverse_folding/sample_sequences.py

@@ -0,0 +1,134 @@
+from pathlib import Path
+import sys
+
+_PROJECT_FILE = Path(__file__).resolve()
+for _PROJECT_ROOT in _PROJECT_FILE.parents:
+    if (_PROJECT_ROOT / "model").is_dir():
+        if str(_PROJECT_ROOT) not in sys.path:
+            sys.path.insert(0, str(_PROJECT_ROOT))
+        break
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+# Sample sequences based on a given structure (multinomial sampling, no beam search).
+#
+# usage: sample_sequences.py [-h] [--chain CHAIN] [--temperature TEMPERATURE]
+# [--outpath OUTPATH] [--num-samples NUM_SAMPLES] pdbfile
+
+import argparse
+import numpy as np
+from pathlib import Path
+import torch
+
+import model.esm as esm
+import model.esm.inverse_folding
+
+
+def sample_seq_singlechain(model, alphabet, args):
+    if torch.cuda.is_available() and not args.nogpu:
+        model = model.cuda()
+        print("Transferred model to GPU")
+    device = next(model.parameters()).device
+    coords, native_seq = esm.inverse_folding.util.load_coords(args.pdbfile, args.chain)
+    print('Native sequence loaded from structure file:')
+    print(native_seq)
+
+    print(f'Saving sampled sequences to {args.outpath}.')
+
+    Path(args.outpath).parent.mkdir(parents=True, exist_ok=True)
+    with open(args.outpath, 'w') as f:
+        for i in range(args.num_samples):
+            print(f'\nSampling.. ({i+1} of {args.num_samples})')
+            sampled_seq = model.sample(coords, temperature=args.temperature, device=device)
+            print('Sampled sequence:')
+            print(sampled_seq)
+            f.write(f'>sampled_seq_{i+1}\n')
+            f.write(sampled_seq + '\n')
+
+            recovery = np.mean([(a==b) for a, b in zip(native_seq, sampled_seq)])
+            print('Sequence recovery:', recovery)
+
+
+def sample_seq_multichain(model, alphabet, args):
+    if torch.cuda.is_available() and not args.nogpu:
+        model = model.cuda()
+        print("Transferred model to GPU")
+    structure = esm.inverse_folding.util.load_structure(args.pdbfile)
+    coords, native_seqs = esm.inverse_folding.multichain_util.extract_coords_from_complex(structure)
+    target_chain_id = args.chain
+    native_seq = native_seqs[target_chain_id]
+    print('Native sequence loaded from structure file:')
+    print(native_seq)
+    print('\n')
+
+    print(f'Saving sampled sequences to {args.outpath}.')
+
+    Path(args.outpath).parent.mkdir(parents=True, exist_ok=True)
+    with open(args.outpath, 'w') as f:
+        for i in range(args.num_samples):
+            print(f'\nSampling.. ({i+1} of {args.num_samples})')
+            sampled_seq = esm.inverse_folding.multichain_util.sample_sequence_in_complex(
+                    model, coords, target_chain_id, temperature=args.temperature)
+            print('Sampled sequence:')
+            print(sampled_seq)
+            f.write(f'>sampled_seq_{i+1}\n')
+            f.write(sampled_seq + '\n')
+
+            recovery = np.mean([(a==b) for a, b in zip(native_seq, sampled_seq)])
+            print('Sequence recovery:', recovery)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+            description='Sample sequences based on a given structure.'
+    )
+    parser.add_argument(
+            'pdbfile', type=str,
+            help='input filepath, either .pdb or .cif',
+    )
+    parser.add_argument(
+            '--chain', type=str,
+            help='chain id for the chain of interest', default=None,
+    )
+    parser.add_argument(
+            '--temperature', type=float,
+            help='temperature for sampling, higher for more diversity',
+            default=1.,
+    )
+    parser.add_argument(
+            '--outpath', type=str,
+            help='output filepath for saving sampled sequences',
+            default='output/sampled_seqs.fasta',
+    )
+    parser.add_argument(
+            '--num-samples', type=int,
+            help='number of sequences to sample',
+            default=1,
+    )
+    parser.set_defaults(multichain_backbone=False)
+    parser.add_argument(
+            '--multichain-backbone', action='store_true',
+            help='use the backbones of all chains in the input for conditioning'
+    )
+    parser.add_argument(
+            '--singlechain-backbone', dest='multichain_backbone',
+            action='store_false',
+            help='use the backbone of only target chain in the input for conditioning'
+    )
+    parser.add_argument("--nogpu", action="store_true", help="Do not use GPU even if available")
+  
+    args = parser.parse_args()
+
+    model, alphabet = esm.pretrained.esm_if1_gvp4_t16_142M_UR50()
+    model = model.eval()
+
+    if args.multichain_backbone:
+        sample_seq_multichain(model, alphabet, args)
+    else:
+        sample_seq_singlechain(model, alphabet, args)
+
+
+if __name__ == '__main__':
+    main()

scripts/inverse_folding/score_log_likelihoods.py

@@ -0,0 +1,140 @@
+from pathlib import Path
+import sys
+
+_PROJECT_FILE = Path(__file__).resolve()
+for _PROJECT_ROOT in _PROJECT_FILE.parents:
+    if (_PROJECT_ROOT / "model").is_dir():
+        if str(_PROJECT_ROOT) not in sys.path:
+            sys.path.insert(0, str(_PROJECT_ROOT))
+        break
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+# Scores sequences based on a given structure.
+#
+# usage:
+# score_log_likelihoods.py [-h] [--outpath OUTPATH] [--chain CHAIN] pdbfile seqfile
+
+import argparse
+from biotite.sequence.io.fasta import FastaFile, get_sequences
+import numpy as np
+from pathlib import Path
+import torch
+import torch.nn.functional as F
+from tqdm import tqdm
+
+import model.esm as esm
+import model.esm.inverse_folding
+
+
+def score_singlechain_backbone(model, alphabet, args):
+    if torch.cuda.is_available() and not args.nogpu:
+        model = model.cuda()
+        print("Transferred model to GPU")
+    coords, native_seq = esm.inverse_folding.util.load_coords(args.pdbfile, args.chain)
+    print('Native sequence loaded from structure file:')
+    print(native_seq)
+    print('\n')
+
+    ll, _ = esm.inverse_folding.util.score_sequence(
+            model, alphabet, coords, native_seq) 
+    print('Native sequence')
+    print(f'Log likelihood: {ll:.2f}')
+    print(f'Perplexity: {np.exp(-ll):.2f}')
+
+    print('\nScoring variant sequences from sequence file..\n')
+    infile = FastaFile()
+    infile.read(args.seqfile)
+    seqs = get_sequences(infile)
+    Path(args.outpath).parent.mkdir(parents=True, exist_ok=True)
+    with open(args.outpath, 'w') as fout:
+        fout.write('seqid,log_likelihood\n')
+        for header, seq in tqdm(seqs.items()):
+            ll, _ = esm.inverse_folding.util.score_sequence(
+                    model, alphabet, coords, str(seq))
+            fout.write(header + ',' + str(ll) + '\n')
+    print(f'Results saved to {args.outpath}') 
+
+
+def score_multichain_backbone(model, alphabet, args):
+    if torch.cuda.is_available() and not args.nogpu:
+        model = model.cuda()
+        print("Transferred model to GPU")
+    structure = esm.inverse_folding.util.load_structure(args.pdbfile)
+    coords, native_seqs = esm.inverse_folding.multichain_util.extract_coords_from_complex(structure)
+    target_chain_id = args.chain
+    native_seq = native_seqs[target_chain_id]
+    print('Native sequence loaded from structure file:')
+    print(native_seq)
+    print('\n')
+
+    ll, _ = esm.inverse_folding.multichain_util.score_sequence_in_complex(
+            model, alphabet, coords, target_chain_id, native_seq) 
+    print('Native sequence')
+    print(f'Log likelihood: {ll:.2f}')
+    print(f'Perplexity: {np.exp(-ll):.2f}')
+
+    print('\nScoring variant sequences from sequence file..\n')
+    infile = FastaFile()
+    infile.read(args.seqfile)
+    seqs = get_sequences(infile)
+    Path(args.outpath).parent.mkdir(parents=True, exist_ok=True)
+    with open(args.outpath, 'w') as fout:
+        fout.write('seqid,log_likelihood\n')
+        for header, seq in tqdm(seqs.items()):
+            ll, _ = esm.inverse_folding.multichain_util.score_sequence_in_complex(
+                    model, alphabet, coords, target_chain_id, str(seq))
+            fout.write(header + ',' + str(ll) + '\n')
+    print(f'Results saved to {args.outpath}') 
+
+
+def main():
+    parser = argparse.ArgumentParser(
+            description='Score sequences based on a given structure.'
+    )
+    parser.add_argument(
+            'pdbfile', type=str,
+            help='input filepath, either .pdb or .cif',
+    )
+    parser.add_argument(
+            'seqfile', type=str,
+            help='input filepath for variant sequences in a .fasta file',
+    )
+    parser.add_argument(
+            '--outpath', type=str,
+            help='output filepath for scores of variant sequences',
+            default='output/sequence_scores.csv',
+    )
+    parser.add_argument(
+            '--chain', type=str,
+            help='chain id for the chain of interest', default='A',
+    )
+    parser.set_defaults(multichain_backbone=False)
+    parser.add_argument(
+            '--multichain-backbone', action='store_true',
+            help='use the backbones of all chains in the input for conditioning'
+    )
+    parser.add_argument(
+            '--singlechain-backbone', dest='multichain_backbone',
+            action='store_false',
+            help='use the backbone of only target chain in the input for conditioning'
+    )
+    
+    parser.add_argument("--nogpu", action="store_true", help="Do not use GPU even if available")
+    
+    args = parser.parse_args()
+
+    model, alphabet = esm.pretrained.esm_if1_gvp4_t16_142M_UR50()
+    model = model.eval()
+
+    if args.multichain_backbone:
+        score_multichain_backbone(model, alphabet, args)
+    else:
+        score_singlechain_backbone(model, alphabet, args)
+
+
+
+if __name__ == '__main__':
+    main()

scripts/variant_prediction/predict.py

@@ -0,0 +1,265 @@
+from pathlib import Path
+import sys
+
+_PROJECT_FILE = Path(__file__).resolve()
+for _PROJECT_ROOT in _PROJECT_FILE.parents:
+    if (_PROJECT_ROOT / "model").is_dir():
+        if str(_PROJECT_ROOT) not in sys.path:
+            sys.path.insert(0, str(_PROJECT_ROOT))
+        break
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import argparse
+import pathlib
+import string
+
+import torch
+
+from model.esm import Alphabet, FastaBatchedDataset, ProteinBertModel, pretrained, MSATransformer
+import pandas as pd
+from tqdm import tqdm
+from Bio import SeqIO
+import itertools
+from typing import List, Tuple
+import numpy as np
+
+
+def remove_insertions(sequence: str) -> str:
+    """ Removes any insertions into the sequence. Needed to load aligned sequences in an MSA. """
+    # This is an efficient way to delete lowercase characters and insertion characters from a string
+    deletekeys = dict.fromkeys(string.ascii_lowercase)
+    deletekeys["."] = None
+    deletekeys["*"] = None
+
+    translation = str.maketrans(deletekeys)
+    return sequence.translate(translation)
+
+
+def read_msa(filename: str, nseq: int) -> List[Tuple[str, str]]:
+    """ Reads the first nseq sequences from an MSA file, automatically removes insertions.
+    
+    The input file must be in a3m format (although we use the SeqIO fasta parser)
+    for remove_insertions to work properly."""
+
+    msa = [
+        (record.description, remove_insertions(str(record.seq)))
+        for record in itertools.islice(SeqIO.parse(filename, "fasta"), nseq)
+    ]
+    return msa
+
+
+def create_parser():
+    parser = argparse.ArgumentParser(
+        description="Label a deep mutational scan with predictions from an ensemble of ESM-1v models."  # noqa
+    )
+
+    # fmt: off
+    parser.add_argument(
+        "--model-location",
+        type=str,
+        help="PyTorch model file OR name of pretrained model to download (see README for models)",
+        nargs="+",
+    )
+    parser.add_argument(
+        "--sequence",
+        type=str,
+        help="Base sequence to which mutations were applied",
+    )
+    parser.add_argument(
+        "--dms-input",
+        type=pathlib.Path,
+        help="CSV file containing the deep mutational scan",
+    )
+    parser.add_argument(
+        "--mutation-col",
+        type=str,
+        default="mutant",
+        help="column in the deep mutational scan labeling the mutation as 'AiB'"
+    )
+    parser.add_argument(
+        "--dms-output",
+        type=pathlib.Path,
+        help="Output file containing the deep mutational scan along with predictions",
+    )
+    parser.add_argument(
+        "--offset-idx",
+        type=int,
+        default=0,
+        help="Offset of the mutation positions in `--mutation-col`"
+    )
+    parser.add_argument(
+        "--scoring-strategy",
+        type=str,
+        default="wt-marginals",
+        choices=["wt-marginals", "pseudo-ppl", "masked-marginals"],
+        help=""
+    )
+    parser.add_argument(
+        "--msa-path",
+        type=pathlib.Path,
+        help="path to MSA in a3m format (required for MSA Transformer)"
+    )
+    parser.add_argument(
+        "--msa-samples",
+        type=int,
+        default=400,
+        help="number of sequences to select from the start of the MSA"
+    )
+    # fmt: on
+    parser.add_argument("--nogpu", action="store_true", help="Do not use GPU even if available")
+    return parser
+
+
+def label_row(row, sequence, token_probs, alphabet, offset_idx):
+    wt, idx, mt = row[0], int(row[1:-1]) - offset_idx, row[-1]
+    assert sequence[idx] == wt, "The listed wildtype does not match the provided sequence"
+
+    wt_encoded, mt_encoded = alphabet.get_idx(wt), alphabet.get_idx(mt)
+
+    # add 1 for BOS
+    score = token_probs[0, 1 + idx, mt_encoded] - token_probs[0, 1 + idx, wt_encoded]
+    return score.item()
+
+
+def _to_model_device(tokens, use_cuda):
+    return tokens.cuda() if use_cuda else tokens
+
+
+def compute_pppl(row, sequence, model, alphabet, offset_idx, use_cuda):
+    wt, idx, mt = row[0], int(row[1:-1]) - offset_idx, row[-1]
+    assert sequence[idx] == wt, "The listed wildtype does not match the provided sequence"
+
+    # modify the sequence
+    sequence = sequence[:idx] + mt + sequence[(idx + 1) :]
+
+    # encode the sequence
+    data = [
+        ("protein1", sequence),
+    ]
+
+    batch_converter = alphabet.get_batch_converter()
+
+    batch_labels, batch_strs, batch_tokens = batch_converter(data)
+
+    wt_encoded, mt_encoded = alphabet.get_idx(wt), alphabet.get_idx(mt)
+
+    # compute probabilities at each position
+    log_probs = []
+    for i in range(1, len(sequence) - 1):
+        batch_tokens_masked = batch_tokens.clone()
+        batch_tokens_masked[0, i] = alphabet.mask_idx
+        with torch.no_grad():
+            token_probs = torch.log_softmax(
+                model(_to_model_device(batch_tokens_masked, use_cuda))["logits"], dim=-1
+            )
+        log_probs.append(token_probs[0, i, alphabet.get_idx(sequence[i])].item())  # vocab size
+    return sum(log_probs)
+
+
+def main(args):
+    # Load the deep mutational scan
+    df = pd.read_csv(args.dms_input)
+
+    # inference for each model
+    for model_location in args.model_location:
+        model, alphabet = pretrained.load_model_and_alphabet(model_location)
+        model.eval()
+        use_cuda = torch.cuda.is_available() and not args.nogpu
+        if use_cuda:
+            model = model.cuda()
+            print("Transferred model to GPU")
+
+        batch_converter = alphabet.get_batch_converter()
+
+        if isinstance(model, MSATransformer):
+            data = [read_msa(args.msa_path, args.msa_samples)]
+            assert (
+                args.scoring_strategy == "masked-marginals"
+            ), "MSA Transformer only supports masked marginal strategy"
+
+            batch_labels, batch_strs, batch_tokens = batch_converter(data)
+
+            all_token_probs = []
+            for i in tqdm(range(batch_tokens.size(2))):
+                batch_tokens_masked = batch_tokens.clone()
+                batch_tokens_masked[0, 0, i] = alphabet.mask_idx  # mask out first sequence
+                with torch.no_grad():
+                    token_probs = torch.log_softmax(
+                        model(_to_model_device(batch_tokens_masked, use_cuda))["logits"], dim=-1
+                    )
+                all_token_probs.append(token_probs[:, 0, i])  # vocab size
+            token_probs = torch.cat(all_token_probs, dim=0).unsqueeze(0)
+            df[model_location] = df.apply(
+                lambda row: label_row(
+                    row[args.mutation_col], args.sequence, token_probs, alphabet, args.offset_idx
+                ),
+                axis=1,
+            )
+
+        else:
+            data = [
+                ("protein1", args.sequence),
+            ]
+            batch_labels, batch_strs, batch_tokens = batch_converter(data)
+
+            if args.scoring_strategy == "wt-marginals":
+                with torch.no_grad():
+                    token_probs = torch.log_softmax(
+                        model(_to_model_device(batch_tokens, use_cuda))["logits"], dim=-1
+                    )
+                df[model_location] = df.apply(
+                    lambda row: label_row(
+                        row[args.mutation_col],
+                        args.sequence,
+                        token_probs,
+                        alphabet,
+                        args.offset_idx,
+                    ),
+                    axis=1,
+                )
+            elif args.scoring_strategy == "masked-marginals":
+                all_token_probs = []
+                for i in tqdm(range(batch_tokens.size(1))):
+                    batch_tokens_masked = batch_tokens.clone()
+                    batch_tokens_masked[0, i] = alphabet.mask_idx
+                    with torch.no_grad():
+                        token_probs = torch.log_softmax(
+                            model(_to_model_device(batch_tokens_masked, use_cuda))["logits"],
+                            dim=-1,
+                        )
+                    all_token_probs.append(token_probs[:, i])  # vocab size
+                token_probs = torch.cat(all_token_probs, dim=0).unsqueeze(0)
+                df[model_location] = df.apply(
+                    lambda row: label_row(
+                        row[args.mutation_col],
+                        args.sequence,
+                        token_probs,
+                        alphabet,
+                        args.offset_idx,
+                    ),
+                    axis=1,
+                )
+            elif args.scoring_strategy == "pseudo-ppl":
+                tqdm.pandas()
+                df[model_location] = df.progress_apply(
+                    lambda row: compute_pppl(
+                        row[args.mutation_col],
+                        args.sequence,
+                        model,
+                        alphabet,
+                        args.offset_idx,
+                        use_cuda,
+                    ),
+                    axis=1,
+                )
+
+    df.to_csv(args.dms_output)
+
+
+if __name__ == "__main__":
+    parser = create_parser()
+    args = parser.parse_args()
+    main(args)

tests/test_import_boundaries.py

@@ -0,0 +1,47 @@
+from pathlib import Path
+import re
+
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+MODEL_ROOT = PROJECT_ROOT / "model"
+SCRIPT_ROOT = PROJECT_ROOT / "scripts"
+
+
+def _python_files(root: Path):
+    return [path for path in root.rglob("*.py") if "__pycache__" not in path.parts]
+
+
+def test_local_model_code_does_not_import_onescience_models_namespace():
+    offenders = []
+    for path in _python_files(MODEL_ROOT):
+        text = path.read_text(encoding="utf-8")
+        if "onescience.models." in text:
+            offenders.append(str(path.relative_to(PROJECT_ROOT)))
+
+    assert offenders == []
+
+
+def test_python_files_do_not_use_legacy_top_level_esm_or_openfold_imports():
+    pattern = re.compile(
+        r"^\s*(from\s+(esm|openfold)(\.|\s)|import\s+(esm|openfold)(\.|\s|$))",
+        re.MULTILINE,
+    )
+    offenders = []
+    for root in (MODEL_ROOT, SCRIPT_ROOT):
+        for path in _python_files(root):
+            text = path.read_text(encoding="utf-8")
+            if pattern.search(text):
+                offenders.append(str(path.relative_to(PROJECT_ROOT)))
+
+    assert offenders == []
+
+
+def test_scripts_bootstrap_project_root_before_importing_local_model():
+    offenders = []
+    for path in _python_files(SCRIPT_ROOT):
+        text = path.read_text(encoding="utf-8")
+        if "model.esm" in text and '_PROJECT_ROOT / "model"' not in text:
+            offenders.append(str(path.relative_to(PROJECT_ROOT)))
+
+    assert offenders == []
+

weight/README.md

@@ -0,0 +1,11 @@
+# Weights
+
+Model weights are not included in this extracted source package.
+
+Recommended layouts:
+
+- `weight/checkpoints/*.pt` when using `bash scripts/download_weights.sh ./weight`
+- `${ONESCIENCE_MODELS_DIR}/esm_models/checkpoints/*.pt` when sharing weights with a OneScience environment
+
+For named pretrained models, `model.esm.pretrained` can also download weights through PyTorch Hub and cache them under the active torch hub directory.
+

weight/esm-main/.flake8

@@ -0,0 +1,10 @@
+[flake8]
+max-line-length = 99
+ignore = E203,W503
+exclude =
+    .git,
+    __pycache__,
+    build,
+    dist,
+    experimental
+    third_party

weight/esm-main/.git-blame-ignore-revs

@@ -0,0 +1,2 @@
+# Migrate code style to Black
+8bc7e948cd9bf0b6d1f2113e221ef548ef663377

weight/esm-main/.github/ISSUE_TEMPLATE/bug.md

@@ -0,0 +1,27 @@
+---
+name: "[Bug Report]"
+about: "Create a bug report. For other questions: see Discussions tab."
+
+---
+
+NOTE: if this is not a bug report, please use the [GitHub Discussions](https://github.com/facebookresearch/esm/discussions) for support questions (How do I do X?), feature requests, ideas, showcasing new applications, etc.
+
+
+**Bug description**
+Please enter a clear and concise description of what the bug is.
+
+**Reproduction steps**
+Enter steps to reproduce the behavior.
+
+**Expected behavior**
+Give a clear and concise description of what you expected to happen.
+
+**Logs**
+Please paste the command line output:
+
+```
+Output goes here
+```
+
+**Additional context**
+Add any other context about the problem here. (like proxy settings, network setup, overall goals, etc.)

weight/esm-main/.gitignore

@@ -0,0 +1,31 @@
+# tensor dumps
+*.pt
+# Compiler Output #
+###################
+*.py[cod]
+*.so
+*.o
+*.exe
+*.class
+
+# Folders #
+###########
+bin/
+build/
+dist/
+local/
+tmp/
+__pycache__/
+*.egg-info/
+.idea/
+.ipynb_checkpoints/
+.vscode/
+esm/dev
+
+# Junk #
+########
+.DS_Store*
+.*.swp
+*.swp
+*.log
+*~

weight/esm-main/CODE_OF_CONDUCT.rst

@@ -0,0 +1,6 @@
+Code of Conduct
+===============
+
+Facebook has adopted a Code of Conduct that we expect project participants to adhere to. Please `read the full text`__ so that you can understand what actions will and will not be tolerated.
+
+__ https://code.facebook.com/codeofconduct

weight/esm-main/CONTRIBUTING.md

@@ -0,0 +1,31 @@
+# Contributing to esm
+We want to make contributing to this project as easy and transparent as
+possible.
+
+## Pull Requests
+We actively welcome your pull requests.
+
+1. Fork the repo and create your branch from `master`.
+2. If you've added code that should be tested, add tests.
+3. If you've changed APIs, update the documentation.
+4. Ensure the test suite passes.
+5. Make sure your code lints.
+6. If you haven't already, complete the Contributor License Agreement ("CLA").
+
+## Contributor License Agreement ("CLA")
+In order to accept your pull request, we need you to submit a CLA. You only need
+to do this once to work on any of Facebook's open source projects.
+
+Complete your CLA here: <https://code.facebook.com/cla>
+
+## Issues
+We use GitHub issues to track public bugs. Please ensure your description is
+clear and has sufficient instructions to be able to reproduce the issue.
+
+Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe
+disclosure of security bugs. In those cases, please go through the process
+outlined on that page and do not file a public issue.
+
+## License
+By contributing to icp-block-mdp, you agree that your contributions will be licensed
+under the LICENSE file in the root directory of this source tree.

weight/esm-main/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Meta Platforms, Inc. and affiliates.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

weight/esm-main/README.md

@@ -0,0 +1,795 @@
+# Evolutionary Scale Modeling
+
+[![atlas](https://user-images.githubusercontent.com/3605224/199301187-a9e38b3f-71a7-44be-94f4-db0d66143c53.png)](https://esmatlas.com)
+
+***Update April 2023:*** Code for the two simultaneous preprints on protein design is now released! Code for "Language models generalize beyond natural proteins" is under [examples/lm-design/](examples/lm-design/). Code for "A high-level programming language for generative protein design" is under [examples/protein-programming-language/](examples/protein-programming-language/).
+
+This repository contains code and pre-trained weights for **Transformer protein language models** from the Meta Fundamental AI Research Protein Team (FAIR), including our state-of-the-art [**ESM-2** and **ESMFold**](#esmfold), as well as [**MSA Transformer**](https://www.biorxiv.org/content/10.1101/2021.02.12.430858v1), [**ESM-1v**](#zs_variant) for predicting variant effects and [**ESM-IF1**](#invf) for inverse folding.
+Transformer protein language models were introduced in the [2019 preprint](https://doi.org/10.1101/622803) of the paper ["Biological structure and function emerge from scaling unsupervised learning to 250 million protein sequences"](https://doi.org/10.1073/pnas.2016239118).
+ESM-2 outperforms all tested single-sequence protein language models across a range of structure prediction tasks.
+ESMFold harnesses the ESM-2 language model to generate accurate structure predictions end to end directly from the sequence of a protein.
+
+In November 2022, we released `v0` of the [ESM Metagenomic Atlas](https://esmatlas.com), an open atlas of 617 million predicted metagenomic protein structures.
+The Atlas was updated in March 2023 in collaboration with EBI. The new `v2023_02` adds another 150 million predicted structures to the Atlas, as well as pre-computed ESM2 embeddings.
+Bulk download, blog post and the resources provided on the Atlas website are documented [on this README](#atlas).
+
+In December 2022, we released two simultaneous preprints on protein design.
+* "Language models generalize beyond natural proteins" ([PAPER](https://doi.org/10.1101/2022.12.21.521521), [CODE](examples/lm-design/)) uses ESM2 to design de novo proteins. The code and data associated with the preprint can be found [here](examples/lm-design/).
+* "A high-level programming language for generative protein design" ([PAPER](https://doi.org/10.1101/2022.12.21.521526), [CODE](examples/protein-programming-language/)) uses ESMFold to design proteins according to a high-level programming language.
+
+
+
+<details><summary><b>Citation</b></summary>
+For ESM2, ESMFold and ESM Atlas:
+```bibtex
+@article{lin2023evolutionary,
+title = {Evolutionary-scale prediction of atomic-level protein structure with a language model},
+author = {Zeming Lin  and Halil Akin  and Roshan Rao  and Brian Hie  and Zhongkai Zhu  and Wenting Lu  and Nikita Smetanin  and Robert Verkuil  and Ori Kabeli  and Yaniv Shmueli  and Allan dos Santos Costa  and Maryam Fazel-Zarandi  and Tom Sercu  and Salvatore Candido  and Alexander Rives },
+journal = {Science},
+volume = {379},
+number = {6637},
+pages = {1123-1130},
+year = {2023},
+doi = {10.1126/science.ade2574},
+URL = {https://www.science.org/doi/abs/10.1126/science.ade2574},
+note={Earlier versions as preprint: bioRxiv 2022.07.20.500902},
+}
+```
+
+For transformer protein language models:
+```bibtex
+@article{rives2021biological,
+  title={Biological structure and function emerge from scaling unsupervised learning to 250 million protein sequences},
+  author={Rives, Alexander and Meier, Joshua and Sercu, Tom and Goyal, Siddharth and Lin, Zeming and Liu, Jason and Guo, Demi and Ott, Myle and Zitnick, C Lawrence and Ma, Jerry and others},
+  journal={Proceedings of the National Academy of Sciences},
+  volume={118},
+  number={15},
+  pages={e2016239118},
+  year={2021},
+  publisher={National Acad Sciences},
+  note={bioRxiv 10.1101/622803},
+  doi={10.1073/pnas.2016239118},
+  url={https://www.pnas.org/doi/full/10.1073/pnas.2016239118},
+}
+```
+</details>
+
+<details open><summary><b>Table of contents</b></summary>
+
+- [Main models you should use](#main-models)
+- [Usage](#usage)
+  - [Quick Start](#quickstart)
+  - [Getting Started with this repository](#repostart)
+  - [ESMFold Structure Prediction](#esmfold)
+  - [Compute embeddings in bulk from FASTA](#bulk_fasta)
+  - [CPU offloading for inference with large models](#fsdp)
+  - [Zero-shot variant prediction](#zs_variant)
+  - [Inverse folding](#invf)
+- [ESM Metagenomic Atlas](#atlas)
+- [Notebooks](#notebooks)
+- [Available Models and Datasets](#available)
+  - [Pre-trained Models](#available-models)
+  - [ESM Structural Split Dataset](#available-esmssd)
+  - [Pre-training Dataset Split](#available-pretraining-split)
+  - [Comparison to related works](#perf_related)
+- [Citations](#citations)
+- [License](#license)
+</details>
+
+<details><summary><b>What's New</b></summary>
+
+- April 2023: Code for the protein design preprints released under [examples/lm-design/](examples/lm-design/).
+- March 2023: We release an update to the ESM Metagenomic Atlas, `v2023_02`. See [website](https://esmatlas.com/) and [bulk download details](#atlas).
+- December 2022: The Meta Fundamental AI Research Protein Team (FAIR) released two simultaneous preprints on protein design:
+["Language models generalize beyond natural proteins" (Verkuil, Kabeli, et al., 2022)](https://doi.org/10.1101/2022.12.21.521521), and ["A high-level programming language for generative protein design" (Hie, Candido, et al., 2022)](https://doi.org/10.1101/2022.12.21.521521).
+- November 2022: ESM Metagenomic Atlas, a repository of 600M+ metagenomics structures released, see [website](https://esmatlas.com/) and [bulk download details](#atlas)
+- November 2022: ESMFold - new end-to-end structure prediction model released (see [Lin et al. 2022](https://www.science.org/doi/abs/10.1126/science.ade2574))
+- August 2022: ESM-2 - new SOTA Language Models released (see [Lin et al. 2022](https://www.science.org/doi/abs/10.1126/science.ade2574))
+- April 2022: New inverse folding model ESM-IF1 released, trained on CATH and UniRef50 predicted structures.
+- August 2021: Added flexibility to tokenizer to allow for spaces and special tokens (like `<mask>`) in sequence.
+- July 2021: New pre-trained model ESM-1v released, trained on UniRef90 (see [Meier et al. 2021](https://doi.org/10.1101/2021.07.09.450648)).
+- July 2021: New MSA Transformer released, with a minor fix in the row positional embeddings (`ESM-MSA-1b`).
+- Feb 2021: MSA Transformer added (see [Rao et al. 2021](https://www.biorxiv.org/content/10.1101/2021.02.12.430858v1)). Example usage in [notebook](#notebooks).
+- Dec 2020: [Self-Attention Contacts](#notebooks) for all pre-trained models (see [Rao et al. 2020](https://doi.org/10.1101/2020.12.15.422761))
+- Dec 2020: Added new pre-trained model [ESM-1b](#perf_related) (see [Rives et al. 2019](https://doi.org/10.1101/622803) Appendix B)
+- Dec 2020: [ESM Structural Split Dataset](#available-esmssd) (see [Rives et al. 2019](https://doi.org/10.1101/622803) Appendix A.10)
+
+</details>
+
+## Main models you should use <a name="main-models"></a>
+
+| Shorthand | `esm.pretrained.`           | Dataset | Description  |
+|-----------|-----------------------------|---------|--------------|
+| ESM-2    | `esm2_t36_3B_UR50D()` `esm2_t48_15B_UR50D()`       | UR50 (sample UR90)  | SOTA general-purpose protein language model. Can be used to predict structure, function and other protein properties directly from individual sequences. Released with [Lin et al. 2022](https://www.science.org/doi/abs/10.1126/science.ade2574) (Aug 2022 update). |
+| ESMFold   | `esmfold_v1()`         | PDB + UR50 | End-to-end single sequence 3D structure predictor (Nov 2022 update). |
+| ESM-MSA-1b| `esm_msa1b_t12_100M_UR50S()` |  UR50 + MSA  | MSA Transformer language model. Can be used to extract embeddings from an MSA. Enables SOTA inference of structure. Released with [Rao et al. 2021](https://www.biorxiv.org/content/10.1101/2021.02.12.430858v2) (ICML'21 version, June 2021).  |
+| ESM-1v    | `esm1v_t33_650M_UR90S_1()` ... `esm1v_t33_650M_UR90S_5()`| UR90  | Language model specialized for prediction of variant effects. Enables SOTA zero-shot prediction of the functional effects of sequence variations. Same architecture as ESM-1b, but trained on UniRef90. Released with [Meier et al. 2021](https://doi.org/10.1101/2021.07.09.450648). |
+| ESM-IF1  | `esm_if1_gvp4_t16_142M_UR50()` | CATH + UR50 | Inverse folding model. Can be used to design sequences for given structures, or to predict functional effects of sequence variation for given structures. Enables SOTA fixed backbone sequence design. Released with [Hsu et al. 2022](https://doi.org/10.1101/2022.04.10.487779). |
+
+For a complete list of available models, with details and release notes, see [Pre-trained Models](#available-models).
+
+
+## Usage <a name="usage"></a>
+
+### Quick start <a name="quickstart"></a>
+
+An easy way to get started is to load ESM or ESMFold through the [HuggingFace transformers library](https://huggingface.co/docs/transformers/model_doc/esm),
+which has simplified the ESMFold dependencies and provides a standardized API and tools to work with state-of-the-art pretrained models.
+
+Alternatively, [ColabFold](https://colab.research.google.com/github/sokrypton/ColabFold/blob/main/ESMFold.ipynb) has integrated ESMFold so that you can 
+easily run it directly in the browser on a Google Colab instance.
+
+We also provide an API which you can access through curl or on [the ESM Metagenomic Atlas web page](https://esmatlas.com/resources?action=fold).
+```
+curl -X POST --data "KVFGRCELAAAMKRHGLDNYRGYSLGNWVCAAKFESNFNTQATNRNTDGSTDYGILQINSRWWCNDGRTPGSRNLCNIPCSALLSSDITASVNCAKKIVSDGNGMNAWVAWRNRCKGTDVQAWIRGCRL" https://api.esmatlas.com/foldSequence/v1/pdb/
+```
+
+For ESM-MSA-1b, ESM-IF1, or any of the other models you can use the original implementation from our repo directly via the instructions below.
+
+### Getting started with this repo <a name="repostart"></a>
+
+As a prerequisite, you must have PyTorch installed to use this repository.
+
+You can use this one-liner for installation, using the latest release of esm:
+
+```bash
+pip install fair-esm  # latest release, OR:
+pip install git+https://github.com/facebookresearch/esm.git  # bleeding edge, current repo main branch
+```
+
+To use the ESMFold model, make sure you start from an environment with python <= 3.9 and pytorch installed.
+Then add the `[esmfold]` option to your pip install, which will install the dependencies for OpenFold
+automatically. Openfold installation requires `nvcc`.
+
+```bash
+pip install "fair-esm[esmfold]"
+# OpenFold and its remaining dependency
+pip install 'dllogger @ git+https://github.com/NVIDIA/dllogger.git'
+pip install 'openfold @ git+https://github.com/aqlaboratory/openfold.git@4b41059694619831a7db195b7e0988fc4ff3a307'
+```
+
+**NOTE**: If openfold installation fails, please double check that `nvcc` is available and that a cuda-compatable version of PyTorch has been installed.
+
+Alternatively, we provide the `esmfold` conda environment, which can be built via `conda env create -f environment.yml`.
+
+We also support PyTorch Hub, which removes the need to clone and/or install this repository yourself:
+
+```python
+import torch
+model, alphabet = torch.hub.load("facebookresearch/esm:main", "esm2_t33_650M_UR50D")
+```
+
+After pip install, you can load and use a pretrained model as follows:
+
+```python
+import torch
+import esm
+
+# Load ESM-2 model
+model, alphabet = esm.pretrained.esm2_t33_650M_UR50D()
+batch_converter = alphabet.get_batch_converter()
+model.eval()  # disables dropout for deterministic results
+
+# Prepare data (first 2 sequences from ESMStructuralSplitDataset superfamily / 4)
+data = [
+    ("protein1", "MKTVRQERLKSIVRILERSKEPVSGAQLAEELSVSRQVIVQDIAYLRSLGYNIVATPRGYVLAGG"),
+    ("protein2", "KALTARQQEVFDLIRDHISQTGMPPTRAEIAQRLGFRSPNAAEEHLKALARKGVIEIVSGASRGIRLLQEE"),
+    ("protein2 with mask","KALTARQQEVFDLIRD<mask>ISQTGMPPTRAEIAQRLGFRSPNAAEEHLKALARKGVIEIVSGASRGIRLLQEE"),
+    ("protein3",  "K A <mask> I S Q"),
+]
+batch_labels, batch_strs, batch_tokens = batch_converter(data)
+batch_lens = (batch_tokens != alphabet.padding_idx).sum(1)
+
+# Extract per-residue representations (on CPU)
+with torch.no_grad():
+    results = model(batch_tokens, repr_layers=[33], return_contacts=True)
+token_representations = results["representations"][33]
+
+# Generate per-sequence representations via averaging
+# NOTE: token 0 is always a beginning-of-sequence token, so the first residue is token 1.
+sequence_representations = []
+for i, tokens_len in enumerate(batch_lens):
+    sequence_representations.append(token_representations[i, 1 : tokens_len - 1].mean(0))
+
+# Look at the unsupervised self-attention map contact predictions
+import matplotlib.pyplot as plt
+for (_, seq), tokens_len, attention_contacts in zip(data, batch_lens, results["contacts"]):
+    plt.matshow(attention_contacts[: tokens_len, : tokens_len])
+    plt.title(seq)
+    plt.show()
+```
+
+
+### ESMFold Structure Prediction <a name="esmfold"></a>
+
+After installing with the `[esmfold]` option, you can use the ESMFold structure prediction model as follows:
+
+```python
+import torch
+import esm
+
+model = esm.pretrained.esmfold_v1()
+model = model.eval().cuda()
+
+# Optionally, uncomment to set a chunk size for axial attention. This can help reduce memory.
+# Lower sizes will have lower memory requirements at the cost of increased speed.
+# model.set_chunk_size(128)
+
+sequence = "MKTVRQERLKSIVRILERSKEPVSGAQLAEELSVSRQVIVQDIAYLRSLGYNIVATPRGYVLAGG"
+# Multimer prediction can be done with chains separated by ':'
+
+with torch.no_grad():
+    output = model.infer_pdb(sequence)
+
+with open("result.pdb", "w") as f:
+    f.write(output)
+
+import biotite.structure.io as bsio
+struct = bsio.load_structure("result.pdb", extra_fields=["b_factor"])
+print(struct.b_factor.mean())  # this will be the pLDDT
+# 88.3
+```
+
+
+Besides `esm.pretrained.esmfold_v1()` which is the best performing model we recommend using, we
+also provide `esm.pretrained.esmfold_v0()` which was used for the experiments in
+[Lin et al. 2022](https://www.science.org/doi/abs/10.1126/science.ade2574).
+
+We also provide a command line interface (`esm-fold`) that efficiently predicts structures in bulk from a FASTA file using ESMFold:
+```
+usage: esm-fold [-h] -i FASTA -o PDB [--num-recycles NUM_RECYCLES]
+                [--max-tokens-per-batch MAX_TOKENS_PER_BATCH]
+                [--chunk-size CHUNK_SIZE] [--cpu-only] [--cpu-offload]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -i FASTA, --fasta FASTA
+                        Path to input FASTA file
+  -o PDB, --pdb PDB     Path to output PDB directory
+  --num-recycles NUM_RECYCLES
+                        Number of recycles to run. Defaults to number used in
+                        training (4).
+  --max-tokens-per-batch MAX_TOKENS_PER_BATCH
+                        Maximum number of tokens per gpu forward-pass. This
+                        will group shorter sequences together for batched
+                        prediction. Lowering this can help with out of memory
+                        issues, if these occur on short sequences.
+  --chunk-size CHUNK_SIZE
+                        Chunks axial attention computation to reduce memory
+                        usage from O(L^2) to O(L). Equivalent to running a for
+                        loop over chunks of of each dimension. Lower values
+                        will result in lower memory usage at the cost of
+                        speed. Recommended values: 128, 64, 32. Default: None.
+  --cpu-only            CPU only
+  --cpu-offload         Enable CPU offloading
+```
+
+The command will make one prediction for every sequence in the fasta file. Multimers can be predicted and should be entered in the fasta file as a single sequence, with chains seprated by a ":" character.
+
+By default, predictions will be batched together so that shorter sequences are predicted simultaneously. This can be disabled by setting `--max-tokens-per-batch=0`. Batching can significantly improve prediction speed on shorter sequences.
+
+The `--cpu-offload` flag can be useful for making predictions on longer sequences. It will attempt to offload some parameters to the CPU RAM, rather than storing on GPU.
+
+Finally, the ablation experiments for LMs of varying sizes [Lin et al. 2022 table S1](https://www.science.org/doi/abs/10.1126/science.ade2574) are released as `esm.pretrained.esmfold_structure_module_only_*()`. We don't recommend using these models for structure prediction.
+
+
+### Compute embeddings in bulk from FASTA <a name="bulk_fasta"></a>
+
+We provide a command line interface (`esm-extract`) that efficiently extracts embeddings in bulk for a FASTA file from the ESM:
+```
+usage: esm-extract [-h] [--toks_per_batch TOKS_PER_BATCH]
+                   [--repr_layers REPR_LAYERS [REPR_LAYERS ...]] --include
+                   {mean,per_tok,bos,contacts}
+                   [{mean,per_tok,bos,contacts} ...]
+                   [--truncation_seq_length TRUNCATION_SEQ_LENGTH]
+                   model_location fasta_file output_dir
+
+Extract per-token representations and model outputs for sequences in a FASTA
+file
+
+positional arguments:
+  model_location        PyTorch model file OR name of pretrained model to
+                        download (see README for models)
+  fasta_file            FASTA file on which to extract representations
+  output_dir            output directory for extracted representations
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --toks_per_batch TOKS_PER_BATCH
+                        maximum batch size
+  --repr_layers REPR_LAYERS [REPR_LAYERS ...]
+                        layers indices from which to extract representations
+                        (0 to num_layers, inclusive)
+  --include {mean,per_tok,bos,contacts} [{mean,per_tok,bos,contacts} ...]
+                        specify which representations to return
+  --truncation_seq_length TRUNCATION_SEQ_LENGTH
+                        truncate sequences longer than the given value
+```
+
+The following commands allow the extraction of the final-layer embedding for a FASTA file from the ESM-2 model:
+
+```bash
+esm-extract esm2_t33_650M_UR50D examples/data/some_proteins.fasta \
+  examples/data/some_proteins_emb_esm2 --repr_layers 0 32 33 --include
+```
+```bash
+python scripts/extract.py esm2_t33_650M_UR50D examples/data/some_proteins.fasta \
+  examples/data/some_proteins_emb_esm2 --repr_layers 0 32 33 --include mean per_tok
+```
+
+A cuda device is optional and will be auto-detected.
+
+Directory `some_proteins_emb_esm2/` now contains one `.pt` file per FASTA sequence; use `torch.load()` to load them.
+`scripts/extract.py` has flags that determine what's included in the `.pt` file:
+* `--repr-layers` (default: final only) selects which layers to include embeddings from.
+* `--include` specifies what embeddings to save. You can use the following:
+  * `per_tok` includes the full sequence, with an embedding per amino acid (seq_len x hidden_dim).
+  * `mean` includes the embeddings averaged over the full sequence, per layer.
+  * `bos` includes the embeddings from the beginning-of-sequence token.
+  (NOTE: Don't use with the pre-trained models - we trained without bos-token supervision)
+
+
+### CPU offloading for inference with large models <a name="fsdp"></a>
+If you want to load very large models like 15B and/or do inference on long sequences on your machine, regular GPU inference may lead to OOM errors.
+We show how to load the model with Fairscale's [Fully Sharded Data Parallel (FSDP)](https://fairscale.readthedocs.io/en/stable/api/nn/fsdp.html) and
+use its CPU offloading feature.
+This allows to do inference of large models on a single GPU.
+Please check out `examples/esm2_infer_fairscale_fsdp_cpu_offloading.py` for more details.
+
+### Zero-shot variant prediction <a name="zs_variant"></a>
+See "[examples/variant-prediction/](examples/variant-prediction/)" for code and pre-trained weights for the ESM-1v models described in
+[Language models enable zero-shot prediction of the effects of mutations on protein function. (Meier et al. 2021)](https://doi.org/10.1101/2021.07.09.450648).
+
+Note that ESM-2 could be used for variant prediction as well, and is expected to have similar performance to ESM-1v.
+
+### Inverse folding <a name="invf"></a>
+See "[examples/inverse_folding/](examples/inverse_folding/)" for detailed user guide. The ESM-IF1 model is described as `GVPTransformer` in [Learning inverse folding from millions of predicted structures. (Hsu et al. 2022)](https://doi.org/10.1101/2022.04.10.487779).
+
+We also provide a colab notebook for the sequence design and sequence scoring functionalities.
+
+[<img src="https://colab.research.google.com/assets/colab-badge.svg">](https://colab.research.google.com/github/facebookresearch/esm/blob/main/examples/inverse_folding/notebook_multichain.ipynb)
+
+The ESM-IF1 inverse folding model is built for predicting protein sequences
+from their backbone atom coordinates. We provide scripts here 1) to sample sequence
+designs for a given structure and 2) to score sequences for a given structure.
+
+Trained with 12M protein structures predicted by AlphaFold2, the ESM-IF1
+model consists of invariant geometric input processing layers followed by a
+sequence-to-sequence transformer, and achieves 51% native sequence recovery on
+structurally held-out backbones with 72% recovery for buried residues.
+The model is also trained with span masking to tolerate missing backbone
+coordinates and therefore can predict sequences for partially masked structures.
+
+#### Sample sequence designs for a given structure
+The environment setup is described in [this subsection of examples/inverse_folding](examples/inverse_folding#recommended-environment).
+
+To sample sequences for a given structure in PDB or mmCIF format, use the
+`sample_sequences.py` script. The input file can have either `.pdb` or
+`.cif` as suffix.
+
+For example, to sample 3 sequence designs for the golgi casein kinase structure
+(PDB [5YH2](https://www.rcsb.org/structure/5yh2); [PDB Molecule of the Month
+from January 2022](https://pdb101.rcsb.org/motm/265)), we can run the following
+command from the esm root directory:
+```bash
+python examples/inverse_folding/sample_sequences.py examples/inverse_folding/data/5YH2.pdb \
+  --chain C --temperature 1 --num-samples 3 --outpath examples/inverse_folding/output/sampled_sequences.fasta
+```
+
+The sampled sequences will be saved in a fasta format to the specified output file.
+
+The temperature parameter controls the sharpness of the probability
+distribution for sequence sampling. Higher sampling temperatures yield more
+diverse sequences but likely with lower native sequence recovery.
+The default sampling temperature is 1. To optimize for native sequence
+recovery, we recommend sampling with low temperature such as 1e-6.
+
+#### Scoring sequences
+To score the conditional log-likelihoods for sequences conditioned on a given
+structure, use the `score_log_likelihoods.py` script.
+
+For example, to score the sequences in `examples/inverse_folding/data/5YH2_mutated_seqs.fasta`
+according to the structure in `examples/inverse_folding/data/5YH2.pdb`, we can run
+the following command from the esm root directory:
+```
+python examples/inverse_folding/score_log_likelihoods.py examples/inverse_folding/data/5YH2.pdb \
+  examples/inverse_folding/data/5YH2_mutated_seqs.fasta --chain C \
+  --outpath examples/inverse_folding/output/5YH2_mutated_seqs_scores.csv
+```
+
+The conditional log-likelihoods are saved in a csv format in the specified output path.
+The output values are the average log-likelihoods averaged over all amino acids in a sequence.
+
+For more information, see "[./examples/inverse_folding/](examples/inverse_folding/)" for detailed user guide.
+
+## ESM Metagenomic Atlas <a name="atlas"></a>
+
+Please visit the [ESM Metagenomic Atlas](https://esmatlas.com/) website, and
+see our [blog post](https://ai.facebook.com/blog/protein-folding-esmfold-metagenomics/) to learn more.
+
+Bulk download instructions available at a seperate README [here](scripts/atlas/README.md).
+
+The Atlas resources include a page to [fold a sequence using ESMFold](https://esmatlas.com/resources?action=fold),
+searching a subset of the ESM Atlas by [structure](https://esmatlas.com/resources?action=search_structure) or 
+[sequence](https://esmatlas.com/resources?action=search_sequence),
+as well as an [API](https://esmatlas.com/about#api) to access those resources programmatically.
+
+Foldseek provides search against the Atlas without the length limitation [here](https://search.foldseek.com/search).
+
+
+## Notebooks <a name="notebooks"></a>
+
+### Inverse folding - predicting or scoring sequences based on backbone structures
+
+[<img src="https://colab.research.google.com/assets/colab-badge.svg">](https://colab.research.google.com/github/facebookresearch/esm/blob/main/examples/inverse_folding/notebook.ipynb)
+
+The ESM-IF1 inverse folding model predicts protein sequences from their backbone atom coordinates, trained with 12M protein structures predicted by AlphaFold2.
+This notetook guide you through examples of sampling sequences, calculating conditional log-likelihoods, and extracting encoder output as structure representation.
+
+### Supervised variant prediction - training a classifier on the embeddings
+
+[<img src="https://colab.research.google.com/assets/colab-badge.svg">](https://colab.research.google.com/github/facebookresearch/esm/blob/main/examples/sup_variant_prediction.ipynb)
+
+
+To help you get started with using the embeddings, this [jupyter notebook tutorial](examples/sup_variant_prediction.ipynb) shows how to train a supervised variant predictor using embeddings from ESM-1.
+You can adopt a similar protocol to train a model for any downstream task, even with limited data.
+First you can obtain the embeddings for ``examples/data/P62593.fasta`` either by [downloading the precomputed](https://dl.fbaipublicfiles.com/fair-esm/examples/P62593_reprs.tar.gz) embeddings
+as instructed in the notebook or by running the following:
+
+```bash
+# Obtain the embeddings
+python scripts/extract.py esm1v_t33_650M_UR90S_1 examples/data/P62593.fasta \
+  examples/data/P62593_emb_esm1v --repr_layers 33 --include mean
+```
+
+Then, follow the remaining instructions in the tutorial. You can also run the tutorial in a [colab notebook](https://colab.research.google.com/github/facebookresearch/esm/blob/main/examples/sup_variant_prediction.ipynb).
+
+**Note, alternatively use [the newer instructions for zero-shot variant prediction](examples/variant-prediction/),
+which predicts mutational effects without any supervised training.**
+
+
+### Unsupervised contact prediction
+[<img src="https://colab.research.google.com/assets/colab-badge.svg">](https://colab.research.google.com/github/facebookresearch/esm/blob/main/examples/contact_prediction.ipynb)
+
+This [jupyter notebook tutorial](examples/contact_prediction.ipynb) demonstrates contact prediction with both the ESM-2 and MSA Transformer (ESM-MSA-1) models.
+Contact prediction is based on a logistic regression over the model's attention maps.
+This methodology is based on our ICLR 2021 paper,
+[Transformer protein language models are unsupervised structure learners. (Rao et al. 2020)](https://doi.org/10.1101/2020.12.15.422761)
+The MSA Transformer (ESM-MSA-1) takes a multiple sequence alignment (MSA) as input, and uses the tied row self-attention maps in the same way.
+See [MSA Transformer. (Rao et al. 2021)](https://www.biorxiv.org/content/10.1101/2021.02.12.430858v1).
+
+To get unsupervised attention-based contacts, call `model.predict_contacts(tokens)` or `model(tokens, return_contacts=True)`.
+
+
+### ESMStructuralSplitDataset and self-attention contact prediction
+[<img src="https://colab.research.google.com/assets/colab-badge.svg">](https://colab.research.google.com/github/facebookresearch/esm/blob/main/examples/esm_structural_dataset.ipynb)
+
+And this [jupyter notebook tutorial](examples/esm_structural_dataset.ipynb) shows how to load and index the `ESMStructuralSplitDataset`,
+and computes the self-attention map unsupervised contact predictions using ESM-2.
+
+
+## Available Models and Datasets <a name="available"></a>
+
+### Pre-trained Models <a name="available-models"></a>
+
+| Shorthand | `esm.pretrained.`           | #layers | #params | Dataset | Embedding Dim |  Model URL (automatically downloaded to `~/.cache/torch/hub/checkpoints`) |
+|-----------|---------------------|---------|-------------|---------|---------------|-----------------------------------------------------------------------|
+| ESM-2     | `esm2_t48_15B_UR50D`         | 48           | 15B         | UR50/D 2021_04                           | 5120 |  https://dl.fbaipublicfiles.com/fair-esm/models/esm2_t48_15B_UR50D.pt          |
+|           | `esm2_t36_3B_UR50D`          | 36           | 3B          | UR50/D 2021_04                           | 2560 |  https://dl.fbaipublicfiles.com/fair-esm/models/esm2_t36_3B_UR50D.pt           |
+|           | `esm2_t33_650M_UR50D`        | 33           | 650M        | UR50/D 2021_04                           | 1280 |  https://dl.fbaipublicfiles.com/fair-esm/models/esm2_t33_650M_UR50D.pt         |
+|           | `esm2_t30_150M_UR50D`        | 30           | 150M        | UR50/D 2021_04                           | 640  |  https://dl.fbaipublicfiles.com/fair-esm/models/esm2_t30_150M_UR50D.pt         |
+|           | `esm2_t12_35M_UR50D`         | 12           | 35M         | UR50/D 2021_04                           | 480  |  https://dl.fbaipublicfiles.com/fair-esm/models/esm2_t12_35M_UR50D.pt          |
+|           | `esm2_t6_8M_UR50D`           | 6            | 8M          | UR50/D 2021_04                           | 320  |  https://dl.fbaipublicfiles.com/fair-esm/models/esm2_t6_8M_UR50D.pt            |
+| ESMFold   | `esmfold_v1`                 | 48 (+36)     | 690M (+3B)  | UR50/D 2021_04                           | -    |  https://dl.fbaipublicfiles.com/fair-esm/models/esmfold_3B_v1.pt               |
+|           | `esmfold_v0`                 | 48 (+36)     | 690M (+3B)  | UR50/D 2021_04                           | -    |  https://dl.fbaipublicfiles.com/fair-esm/models/esmfold_3B_v0.pt               |
+|           | `esmfold_structure_module_only_*`              | 0 (+various) | various     | UR50/D 2021_04                           | -    |  https://dl.fbaipublicfiles.com/fair-esm/models/esmfold_structure_module_only_*                  |
+| ESM-IF1   | `esm_if1_gvp4_t16_142M_UR50` | 20           | 124M        | CATH 4.3 + predicted structures for UR50 | 512  | https://dl.fbaipublicfiles.com/fair-esm/models/esm_if1_gvp4_t16_142M_UR50.pt   |
+| ESM-1v    | `esm1v_t33_650M_UR90S_[1-5]` | 33           | 650M        | UR90/S 2020_03                           | 1280 | https://dl.fbaipublicfiles.com/fair-esm/models/esm1v_t33_650M_UR90S_1.pt       |
+| ESM-MSA-1b| `esm_msa1b_t12_100M_UR50S`   | 12           | 100M        | UR50/S + MSA 2018_03                     | 768  | https://dl.fbaipublicfiles.com/fair-esm/models/esm_msa1b_t12_100M_UR50S.pt     |
+| ESM-MSA-1 | `esm_msa1_t12_100M_UR50S`    | 12           | 100M        | UR50/S + MSA 2018_03                     | 768  | https://dl.fbaipublicfiles.com/fair-esm/models/esm_msa1_t12_100M_UR50S.pt      |
+| ESM-1b    | `esm1b_t33_650M_UR50S`       | 33           | 650M        | UR50/S 2018_03                           | 1280 | https://dl.fbaipublicfiles.com/fair-esm/models/esm1b_t33_650M_UR50S.pt         |
+| ESM-1     | `esm1_t34_670M_UR50S`        | 34           | 670M        | UR50/S 2018_03                           | 1280 |  https://dl.fbaipublicfiles.com/fair-esm/models/esm1_t34_670M_UR50S.pt         |
+|           | `esm1_t34_670M_UR50D`        | 34           | 670M        | UR50/D 2018_03                           | 1280 |  https://dl.fbaipublicfiles.com/fair-esm/models/esm1_t34_670M_UR50D.pt         |
+|           | `esm1_t34_670M_UR100`        | 34           | 670M        | UR100 2018_03                            | 1280 |  https://dl.fbaipublicfiles.com/fair-esm/models/esm1_t34_670M_UR100.pt         |
+|           | `esm1_t12_85M_UR50S`         | 12           | 85M         | UR50/S 2018_03                           | 768  |  https://dl.fbaipublicfiles.com/fair-esm/models/esm1_t12_85M_UR50S.pt          |
+|           | `esm1_t6_43M_UR50S`          | 6            | 43M         | UR50/S 2018_03                           | 768  |  https://dl.fbaipublicfiles.com/fair-esm/models/esm1_t6_43M_UR50S.pt           |
+
+
+Here is a chronological list of the released models and the paper they were introduced in:
+
+| Shorthand  | Release Notes |
+|------------|---------------|
+| ESM-1      | Released with Rives et al. 2019 (Aug 2020 update). |
+| ESM-1b     | Released with Rives et al. 2019 (Dec 2020 update). See Appendix B. |
+| ESM-MSA-1  | Released with Rao et al. 2021 (Preprint v1). |
+| ESM-MSA-1b | Released with Rao et al. 2021 (ICML'21 version, June 2021). |
+| ESM-1v     | Released with Meier et al. 2021. |
+| ESM-IF1    | Released with Hsu et al. 2022. |
+| ESM-2      | Released with Lin et al. 2022. |
+
+### ESM Structural Split Dataset <a name="available-esmssd"></a>
+This is a five-fold cross validation dataset of protein domain structures that can be used to measure generalization of representations
+across different levels of structural dissimilarity.
+The dataset implements structural holdouts at the family, superfamily, and fold
+level. The SCOPe database is used to classify domains. Independently for each level of structural hold-out,
+the domains are split into 5 equal sets, i.e. five sets of folds, superfamilies, or families. This ensures
+that for each of the five partitions, structures having the same classification do not appear in both the
+train and test sets. For a given classification level each structure appears in a test set once, so that
+in the cross validation experiment each of the structures will be evaluated exactly once.
+
+The dataset provides 3d coordinates, distance maps, and secondary structure labels.
+For further details on the construction of the dataset
+see [Rives et al. 2019](https://doi.org/10.1101/622803) Appendix A.10.
+
+This [jupyter notebook tutorial](examples/esm_structural_dataset.ipynb) shows how to load and index the `ESMStructuralSplitDataset`.
+
+`ESMStructuralSplitDataset`, upon initializing, will download `splits` and `pkl`.
+We also provide `msas` for each of the domains. The data can be directly downloaded below.
+
+| Name   | Description                                                                   | URL                                                                   |
+|--------|-------------------------------------------------------------------------------|-----------------------------------------------------------------------|
+| splits | train/valid splits                                                            | https://dl.fbaipublicfiles.com/fair-esm/structural-data/splits.tar.gz |
+| pkl    | pkl objects containing sequence, SSP labels, distance map, and 3d coordinates | https://dl.fbaipublicfiles.com/fair-esm/structural-data/pkl.tar.gz    |
+| msas   | a3m files containing MSA for each domain                                      | https://dl.fbaipublicfiles.com/fair-esm/structural-data/msas.tar.gz   |
+
+### Pre-training Dataset Split  <a name="available-pretraining-split"></a>
+The split files establishing which UniRef50 clusters were used as held-out evaluation set for pre-training
+in [Rives et al. 2019](https://doi.org/10.1101/622803) and [Rao et al. 2021](https://doi.org/10.1101/2021.02.12.430858) can be found here:
+* [UniRef50 IDs of evaluation set](https://dl.fbaipublicfiles.com/fair-esm/pretraining-data/uniref201803_ur50_valid_headers.txt.gz): 3.016 M clusters
+* [UniRef100 IDs of evaluation set](https://dl.fbaipublicfiles.com/fair-esm/pretraining-data/uniref201803_ur100_valid_headers.txt.gz): 13.745 M proteins, expanding the same UniRef50 clusters.
+
+These files only contain only the UniRef50 IDs and UniRef100 IDs corresponding to the [UniRef database, 2018-03 release](https://ftp.uniprot.org/pub/databases/uniprot/previous_releases/release-2018_03/uniref/)
+which is released by the UniProt Consortium under a [Creative Commons Attribution (CC BY 4.0) License](https://www.uniprot.org/help/license).
+
+
+### Comparison to related works <a name="perf_related"></a>
+<!--
+DO NOT EDIT THIS TABLE! This is the source of truth:
+https://docs.google.com/spreadsheets/d/1RPvWF47rIMEr-Jg-SRCoGElHcwCl5d7RyEeSyPgp59A/edit#gid=0
+exported via https://www.tablesgenerator.com/html_tables
+-->
+
+<table class="tg">
+<thead>
+  <tr>
+    <th class="tg-0thz"><span style="font-weight:bold">Task</span></th>
+    <th class="tg-j6zm" colspan="3"><span style="font-weight:bold">Unsupervised contact prediction</span></th>
+    <th class="tg-j6zm" colspan="2"><span style="font-weight:bold">Structure Prediction</span></th>
+  </tr>
+</thead>
+<tbody>
+  <tr>
+    <td class="tg-j6zm"><span style="font-weight:bold">Test set</span></td>
+    <td class="tg-j6zm"><span style="font-weight:bold">Large valid</span></td>
+    <td class="tg-j6zm"><span style="font-weight:bold">CASP14</span></td>
+    <td class="tg-j6zm"><span style="font-weight:bold">CAMEO (Apr-Jun 2022)</span></td>
+    <td class="tg-j6zm"><span style="font-weight:bold">CASP14</span></td>
+    <td class="tg-j6zm"><span style="font-weight:bold">CAMEO (Apr-Jun 2022)</span></td>
+  </tr>
+  <tr>
+    <td class="tg-7zrl">Gremlin (Potts)</td>
+    <td class="tg-7zrl">39.3</td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-7zrl"></td>
+  </tr>
+  <tr>
+    <td class="tg-7zrl">TAPE</td>
+    <td class="tg-7zrl">11.2</td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-7zrl"></td>
+  </tr>
+  <tr>
+    <td class="tg-7zrl">ProtBert-BFD</td>
+    <td class="tg-7zrl">34.1</td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-7zrl"></td>
+  </tr>
+  <tr>
+    <td class="tg-7zrl">Prot-T5-XL-BFD</td>
+    <td class="tg-7zrl">35.6</td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-2b7s">46.1</td>
+    <td class="tg-2b7s">62.6</td>
+  </tr>
+  <tr>
+    <td class="tg-7zrl">Prot-T5-XL-Ur50 (3B)</td>
+    <td class="tg-7zrl">47.9</td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-2b7s">49.8</td>
+    <td class="tg-2b7s">69.4</td>
+  </tr>
+  <tr>
+    <td class="tg-7zrl">ESM-1</td>
+    <td class="tg-7zrl">33.7</td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-7zrl"></td>
+  </tr>
+  <tr>
+    <td class="tg-7zrl">ESM-1b</td>
+    <td class="tg-7zrl">41.1</td>
+    <td class="tg-7zrl">24.4</td>
+    <td class="tg-7zrl">39</td>
+    <td class="tg-2b7s">41.6</td>
+    <td class="tg-2b7s">64.5</td>
+  </tr>
+  <tr>
+    <td class="tg-7zrl">ESM-1v</td>
+    <td class="tg-7zrl">35.3</td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-7zrl"></td>
+  </tr>
+  <tr>
+    <td class="tg-7zrl">ESM-MSA-1b</td>
+    <td class="tg-7zrl">57.4</td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-7zrl"></td>
+    <td class="tg-7zrl"></td>
+  </tr>
+  <tr>
+    <td class="tg-7zrl">ESM-2 (8M)</td>
+    <td class="tg-7zrl">15.9</td>
+    <td class="tg-7zrl">9.8</td>
+    <td class="tg-7zrl">15.7</td>
+    <td class="tg-2b7s">36.7</td>
+    <td class="tg-2b7s">48.1</td>
+  </tr>
+  <tr>
+    <td class="tg-7zrl">ESM-2 (35M)</td>
+    <td class="tg-7zrl">28.8</td>
+    <td class="tg-7zrl">16.4</td>
+    <td class="tg-7zrl">28.4</td>
+    <td class="tg-2b7s">41.4</td>
+    <td class="tg-2b7s">56.4</td>
+  </tr>
+  <tr>
+    <td class="tg-7zrl">ESM-2 (150M)</td>
+    <td class="tg-7zrl">42.2</td>
+    <td class="tg-7zrl">26.8</td>
+    <td class="tg-7zrl">40.1</td>
+    <td class="tg-2b7s">49.0</td>
+    <td class="tg-2b7s">64.9</td>
+  </tr>
+  <tr>
+    <td class="tg-7zrl">ESM-2 (700M)</td>
+    <td class="tg-7zrl">50.1</td>
+    <td class="tg-7zrl">32.5</td>
+    <td class="tg-7zrl">47.6</td>
+    <td class="tg-2b7s">51.3</td>
+    <td class="tg-2b7s">70.1</td>
+  </tr>
+  <tr>
+    <td class="tg-7zrl">ESM-2 (3B)</td>
+    <td class="tg-7zrl">52.7</td>
+    <td class="tg-7zrl">34.0</td>
+    <td class="tg-7zrl">49.9</td>
+    <td class="tg-2b7s">52.5</td>
+    <td class="tg-2b7s">71.8</td>
+  </tr>
+  <tr>
+    <td class="tg-7zrl">ESM-2 (15B)</td>
+    <td class="tg-7zrl">54.5</td>
+    <td class="tg-7zrl">37.0</td>
+    <td class="tg-7zrl">51.7</td>
+    <td class="tg-2b7s">55.4</td>
+    <td class="tg-2b7s">72.1</td>
+  </tr>
+</tbody>
+</table>
+
+Comparison to related protein language models on structure prediction tasks.
+
+* All contact numbers are the top-L,LR precision metric, where long range means sequence separation of at least 24 residues
+* For unsupervised contact prediction, a sparse linear combination of the attention heads is used to directly predict protein contacts,
+fitted with logistic regression on 20 structures.
+For more details on the method, see [Rao et al. 2020](https://doi.org/10.1101/2020.12.15.422761).
+* For structure prediction, an AlphaFold2 structure module is trained directly from the frozen language model embeddings.
+For more details on the method, see [Lin et al. 2022](https://www.science.org/doi/abs/10.1126/science.ade2574).
+* Direct coupling analysis methods (Gremlin, mfDCA, Psicov) and ESM-MSA-1 use the [trRosetta MSAs](https://yanglab.nankai.edu.cn/trRosetta/benchmark/), while other methods predict from single sequence.
+
+
+## Citations <a name="citations"></a>
+
+If you find the models useful in your research, we ask that you cite the relevant paper:
+
+```bibtex
+@article{rives2019biological,
+  author={Rives, Alexander and Meier, Joshua and Sercu, Tom and Goyal, Siddharth and Lin, Zeming and Liu, Jason and Guo, Demi and Ott, Myle and Zitnick, C. Lawrence and Ma, Jerry and Fergus, Rob},
+  title={Biological Structure and Function Emerge from Scaling Unsupervised Learning to 250 Million Protein Sequences},
+  year={2019},
+  doi={10.1101/622803},
+  url={https://www.biorxiv.org/content/10.1101/622803v4},
+  journal={PNAS}
+}
+```
+
+For the self-attention contact prediction:
+
+```bibtex
+@article{rao2020transformer,
+  author = {Rao, Roshan M and Meier, Joshua and Sercu, Tom and Ovchinnikov, Sergey and Rives, Alexander},
+  title={Transformer protein language models are unsupervised structure learners},
+  year={2020},
+  doi={10.1101/2020.12.15.422761},
+  url={https://www.biorxiv.org/content/10.1101/2020.12.15.422761v1},
+  journal={bioRxiv}
+}
+```
+
+For the MSA Transformer:
+
+```bibtex
+@article{rao2021msa,
+  author = {Rao, Roshan and Liu, Jason and Verkuil, Robert and Meier, Joshua and Canny, John F. and Abbeel, Pieter and Sercu, Tom and Rives, Alexander},
+  title={MSA Transformer},
+  year={2021},
+  doi={10.1101/2021.02.12.430858},
+  url={https://www.biorxiv.org/content/10.1101/2021.02.12.430858v1},
+  journal={bioRxiv}
+}
+```
+
+For variant prediction using ESM-1v:
+
+```bibtex
+@article{meier2021language,
+  author = {Meier, Joshua and Rao, Roshan and Verkuil, Robert and Liu, Jason and Sercu, Tom and Rives, Alexander},
+  title = {Language models enable zero-shot prediction of the effects of mutations on protein function},
+  year={2021},
+  doi={10.1101/2021.07.09.450648},
+  url={https://www.biorxiv.org/content/10.1101/2021.07.09.450648v1},
+  journal={bioRxiv}
+}
+```
+
+For inverse folding using ESM-IF1:
+
+```bibtex
+@article{hsu2022learning,
+	author = {Hsu, Chloe and Verkuil, Robert and Liu, Jason and Lin, Zeming and Hie, Brian and Sercu, Tom and Lerer, Adam and Rives, Alexander},
+	title = {Learning inverse folding from millions of predicted structures},
+	year = {2022},
+	doi = {10.1101/2022.04.10.487779},
+	url = {https://www.biorxiv.org/content/early/2022/04/10/2022.04.10.487779},
+	journal = {ICML}
+}
+```
+
+For the ESM-2 language model and ESMFold:
+
+```bibtex
+@article{lin2022language,
+  title={Language models of protein sequences at the scale of evolution enable accurate structure prediction},
+  author={Lin, Zeming and Akin, Halil and Rao, Roshan and Hie, Brian and Zhu, Zhongkai and Lu, Wenting and Smetanin, Nikita and dos Santos Costa, Allan and Fazel-Zarandi, Maryam and Sercu, Tom and Candido, Sal and others},
+  journal={bioRxiv},
+  year={2022},
+  publisher={Cold Spring Harbor Laboratory}
+}
+```
+
+Much of this code builds on the [fairseq](https://github.com/pytorch/fairseq) sequence modeling framework. We use fairseq internally for our protein language modeling research. We highly recommend trying it out if you'd like to pre-train protein language models from scratch.
+
+Additionally, if you would like to use the variant prediction benchmark from Meier et al. (2021), we provide a bibtex file with citations for all data in [./examples/variant-prediction/mutation_data.bib](./examples/variant-prediction/mutation_data.bib). You can cite each paper individually, or add all citations in bulk using the LaTeX command:
+
+```tex
+\nocite{wrenbeck2017deep,klesmith2015comprehensive,haddox2018mapping,romero2015dissecting,firnberg2014comprehensive,deng2012deep,stiffler2015evolvability,jacquier2013capturing,findlay2018comprehensive,mclaughlin2012spatial,kitzman2015massively,doud2016accurate,pokusaeva2019experimental,mishra2016systematic,kelsic2016rna,melnikov2014comprehensive,brenan2016phenotypic,rockah2015systematic,wu2015functional,aakre2015evolving,qi2014quantitative,matreyek2018multiplex,bandaru2017deconstruction,roscoe2013analyses,roscoe2014systematic,mavor2016determination,chan2017correlation,melamed2013deep,starita2013activity,araya2012fundamental}
+```
+
+## License <a name="license"></a>
+
+This source code is licensed under the MIT license found in the `LICENSE` file
+in the root directory of this source tree.
+
+ESM Metagenomic Atlas (also referred to as “ESM Metagenomic Structure Atlas” or “ESM Atlas”) data is available under a CC BY 4.0 license for academic and commercial use. Copyright (c) Meta Platforms, Inc. All Rights Reserved. Use of the ESM Metagenomic Atlas data is subject to the Meta Open Source [Terms of Use](https://opensource.fb.com/legal/terms/) and [Privacy Policy](https://opensource.fb.com/legal/privacy/).

weight/esm-main/environment.yml

@@ -0,0 +1,36 @@
+name: esmfold
+channels:
+  - conda-forge
+  - bioconda
+  - pytorch
+dependencies:
+  - conda-forge::python=3.7
+  - conda-forge::setuptools=59.5.0
+  - conda-forge::pip
+  - conda-forge::openmm=7.5.1
+  - conda-forge::pdbfixer
+  - conda-forge::cudatoolkit==11.3.*
+  - conda-forge::einops
+  - conda-forge::fairscale
+  - conda-forge::omegaconf
+  - conda-forge::hydra-core
+  - conda-forge::pandas
+  - conda-forge::pytest
+  - bioconda::hmmer==3.3.2
+  - bioconda::hhsuite==3.3.0
+  - bioconda::kalign2==2.04
+  - pytorch::pytorch=1.12.*
+  - pip:
+      - biopython==1.79
+      - deepspeed==0.5.9
+      - dm-tree==0.1.6
+      - ml-collections==0.1.0
+      - numpy==1.21.2
+      - PyYAML==5.4.1
+      - requests==2.26.0
+      - scipy==1.7.1
+      - tqdm==4.62.2
+      - typing-extensions==3.10.0.2
+      - pytorch_lightning==1.5.10
+      - wandb==0.12.21
+      - git+https://github.com/NVIDIA/dllogger.git

weight/esm-main/esm/__init__.py

@@ -0,0 +1,12 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from .version import version as __version__  # noqa
+
+from .data import Alphabet, BatchConverter, FastaBatchedDataset  # noqa
+from .model.esm1 import ProteinBertModel  # noqa
+from .model.esm2 import ESM2  # noqa
+from .model.msa_transformer import MSATransformer  #noqa
+from . import pretrained  # noqa

weight/esm-main/esm/axial_attention.py

@@ -0,0 +1,239 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import math
+import torch
+import torch.nn as nn
+
+
+class RowSelfAttention(nn.Module):
+    """Compute self-attention over rows of a 2D input."""
+
+    def __init__(
+        self,
+        embed_dim,
+        num_heads,
+        dropout=0.0,
+        max_tokens_per_msa: int = 2 ** 16,
+    ):
+        super().__init__()
+        self.num_heads = num_heads
+        self.dropout = dropout
+        self.head_dim = embed_dim // num_heads
+        self.scaling = self.head_dim ** -0.5
+        self.max_tokens_per_msa = max_tokens_per_msa
+        self.attn_shape = "hnij"
+
+        self.k_proj = nn.Linear(embed_dim, embed_dim)
+        self.v_proj = nn.Linear(embed_dim, embed_dim)
+        self.q_proj = nn.Linear(embed_dim, embed_dim)
+
+        self.out_proj = nn.Linear(embed_dim, embed_dim)
+        self.dropout_module = nn.Dropout(dropout)
+
+    def align_scaling(self, q):
+        num_rows = q.size(0)
+        return self.scaling / math.sqrt(num_rows)
+
+    def _batched_forward(
+        self,
+        x,
+        self_attn_mask=None,
+        self_attn_padding_mask=None,
+    ):
+        num_rows, num_cols, batch_size, embed_dim = x.size()
+        max_rows = max(1, self.max_tokens_per_msa // num_cols)
+        attns = 0
+        scaling = self.align_scaling(x)
+        for start in range(0, num_rows, max_rows):
+            attn_weights = self.compute_attention_weights(
+                x[start : start + max_rows],
+                scaling,
+                self_attn_mask=self_attn_mask,
+                self_attn_padding_mask=self_attn_padding_mask[:, start : start + max_rows]
+                if self_attn_padding_mask is not None
+                else None,
+            )
+            attns += attn_weights
+        attn_probs = attns.softmax(-1)
+        attn_probs = self.dropout_module(attn_probs)
+
+        outputs = []
+        for start in range(0, num_rows, max_rows):
+            output = self.compute_attention_update(x[start : start + max_rows], attn_probs)
+            outputs.append(output)
+
+        output = torch.cat(outputs, 0)
+        return output, attn_probs
+
+    def compute_attention_weights(
+        self,
+        x,
+        scaling: float,
+        self_attn_mask=None,
+        self_attn_padding_mask=None,
+    ):
+        num_rows, num_cols, batch_size, embed_dim = x.size()
+        q = self.q_proj(x).view(num_rows, num_cols, batch_size, self.num_heads, self.head_dim)
+        k = self.k_proj(x).view(num_rows, num_cols, batch_size, self.num_heads, self.head_dim)
+        q *= scaling
+        if self_attn_padding_mask is not None:
+            # Zero out any padded aligned positions - this is important since
+            # we take a sum across the alignment axis.
+            q *= 1 - self_attn_padding_mask.permute(1, 2, 0).unsqueeze(3).unsqueeze(4).to(q)
+
+        attn_weights = torch.einsum(f"rinhd,rjnhd->{self.attn_shape}", q, k)
+
+        if self_attn_mask is not None:
+            raise NotImplementedError
+            # Mask Size: [B x R x C], Weights Size: [H x B x C x C]
+
+        if self_attn_padding_mask is not None:
+            attn_weights = attn_weights.masked_fill(
+                self_attn_padding_mask[:, 0].unsqueeze(0).unsqueeze(2),
+                -10000,
+            )
+
+        return attn_weights
+
+    def compute_attention_update(
+        self,
+        x,
+        attn_probs,
+    ):
+        num_rows, num_cols, batch_size, embed_dim = x.size()
+        v = self.v_proj(x).view(num_rows, num_cols, batch_size, self.num_heads, self.head_dim)
+        context = torch.einsum(f"{self.attn_shape},rjnhd->rinhd", attn_probs, v)
+        context = context.contiguous().view(num_rows, num_cols, batch_size, embed_dim)
+        output = self.out_proj(context)
+        return output
+
+    def forward(
+        self,
+        x,
+        self_attn_mask=None,
+        self_attn_padding_mask=None,
+    ):
+        num_rows, num_cols, batch_size, embed_dim = x.size()
+        if (num_rows * num_cols > self.max_tokens_per_msa) and not torch.is_grad_enabled():
+            return self._batched_forward(x, self_attn_mask, self_attn_padding_mask)
+        else:
+            scaling = self.align_scaling(x)
+            attn_weights = self.compute_attention_weights(
+                x, scaling, self_attn_mask, self_attn_padding_mask
+            )
+            attn_probs = attn_weights.softmax(-1)
+            attn_probs = self.dropout_module(attn_probs)
+            output = self.compute_attention_update(x, attn_probs)
+            return output, attn_probs
+
+
+class ColumnSelfAttention(nn.Module):
+    """Compute self-attention over columns of a 2D input."""
+
+    def __init__(
+        self,
+        embed_dim,
+        num_heads,
+        dropout=0.0,
+        max_tokens_per_msa: int = 2 ** 16,
+    ):
+        super().__init__()
+
+        self.num_heads = num_heads
+        self.dropout = dropout
+        self.head_dim = embed_dim // num_heads
+        self.scaling = self.head_dim ** -0.5
+        self.max_tokens_per_msa = max_tokens_per_msa
+
+        self.k_proj = nn.Linear(embed_dim, embed_dim)
+        self.v_proj = nn.Linear(embed_dim, embed_dim)
+        self.q_proj = nn.Linear(embed_dim, embed_dim)
+
+        self.out_proj = nn.Linear(embed_dim, embed_dim)
+        self.dropout_module = nn.Dropout(dropout)
+
+    def _batched_forward(
+        self,
+        x,
+        self_attn_mask=None,
+        self_attn_padding_mask=None,
+    ):
+        num_rows, num_cols, batch_size, embed_dim = x.size()
+        max_cols = max(1, self.max_tokens_per_msa // num_rows)
+        outputs = []
+        attns = []
+        for start in range(0, num_cols, max_cols):
+            output, attn = self(
+                x[:, start : start + max_cols],
+                self_attn_mask=self_attn_mask,
+                self_attn_padding_mask=self_attn_padding_mask[:, :, start : start + max_cols]
+                if self_attn_padding_mask is not None
+                else None,
+            )
+            outputs.append(output)
+            attns.append(attn)
+        output = torch.cat(outputs, 1)
+        attns = torch.cat(attns, 1)
+        return output, attns
+
+    def compute_attention_update(
+        self,
+        x,
+        self_attn_mask=None,
+        self_attn_padding_mask=None,
+    ):
+        num_rows, num_cols, batch_size, embed_dim = x.size()
+        if num_rows == 1:
+            # if there is only 1 position, this is equivalent and doesn't break with padding
+            attn_probs = torch.ones(
+                self.num_heads,
+                num_cols,
+                batch_size,
+                num_rows,
+                num_rows,
+                device=x.device,
+                dtype=x.dtype,
+            )
+            output = self.out_proj(self.v_proj(x))
+        else:
+            q = self.q_proj(x).view(num_rows, num_cols, batch_size, self.num_heads, self.head_dim)
+            k = self.k_proj(x).view(num_rows, num_cols, batch_size, self.num_heads, self.head_dim)
+            v = self.v_proj(x).view(num_rows, num_cols, batch_size, self.num_heads, self.head_dim)
+            q *= self.scaling
+
+            attn_weights = torch.einsum("icnhd,jcnhd->hcnij", q, k)
+
+            if self_attn_mask is not None:
+                raise NotImplementedError
+            if self_attn_padding_mask is not None:
+                attn_weights = attn_weights.masked_fill(
+                    self_attn_padding_mask.permute(2, 0, 1).unsqueeze(0).unsqueeze(3),
+                    -10000,
+                )
+
+            attn_probs = attn_weights.softmax(-1)
+            attn_probs = self.dropout_module(attn_probs)
+            context = torch.einsum("hcnij,jcnhd->icnhd", attn_probs, v)
+            context = context.contiguous().view(num_rows, num_cols, batch_size, embed_dim)
+            output = self.out_proj(context)
+        return output, attn_probs
+
+    def forward(
+        self,
+        x,
+        self_attn_mask=None,
+        self_attn_padding_mask=None,
+    ):
+        num_rows, num_cols, batch_size, embed_dim = x.size()
+        # if False and num_rows * num_cols > 2 ** 14 and not torch.is_grad_enabled():
+        if (num_rows * num_cols) > self.max_tokens_per_msa and not torch.is_grad_enabled():
+            return self._batched_forward(
+                x,
+                self_attn_mask,
+                self_attn_padding_mask,
+            )
+        else:
+            return self.compute_attention_update(x, self_attn_mask, self_attn_padding_mask)

weight/esm-main/esm/constants.py

@@ -0,0 +1,10 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+# fmt: off
+proteinseq_toks = {
+    'toks': ['L', 'A', 'G', 'V', 'S', 'E', 'R', 'T', 'I', 'D', 'P', 'K', 'Q', 'N', 'F', 'Y', 'M', 'H', 'W', 'C', 'X', 'B', 'U', 'Z', 'O', '.', '-']
+}
+# fmt: on

weight/esm-main/esm/data.py

@@ -0,0 +1,493 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import itertools
+import os
+from typing import Sequence, Tuple, List, Union
+import pickle
+import re
+import shutil
+import torch
+from pathlib import Path
+from esm.constants import proteinseq_toks
+
+RawMSA = Sequence[Tuple[str, str]]
+
+
+class FastaBatchedDataset(object):
+    def __init__(self, sequence_labels, sequence_strs):
+        self.sequence_labels = list(sequence_labels)
+        self.sequence_strs = list(sequence_strs)
+
+    @classmethod
+    def from_file(cls, fasta_file):
+        sequence_labels, sequence_strs = [], []
+        cur_seq_label = None
+        buf = []
+
+        def _flush_current_seq():
+            nonlocal cur_seq_label, buf
+            if cur_seq_label is None:
+                return
+            sequence_labels.append(cur_seq_label)
+            sequence_strs.append("".join(buf))
+            cur_seq_label = None
+            buf = []
+
+        with open(fasta_file, "r") as infile:
+            for line_idx, line in enumerate(infile):
+                if line.startswith(">"):  # label line
+                    _flush_current_seq()
+                    line = line[1:].strip()
+                    if len(line) > 0:
+                        cur_seq_label = line
+                    else:
+                        cur_seq_label = f"seqnum{line_idx:09d}"
+                else:  # sequence line
+                    buf.append(line.strip())
+
+        _flush_current_seq()
+
+        assert len(set(sequence_labels)) == len(
+            sequence_labels
+        ), "Found duplicate sequence labels"
+
+        return cls(sequence_labels, sequence_strs)
+
+    def __len__(self):
+        return len(self.sequence_labels)
+
+    def __getitem__(self, idx):
+        return self.sequence_labels[idx], self.sequence_strs[idx]
+
+    def get_batch_indices(self, toks_per_batch, extra_toks_per_seq=0):
+        sizes = [(len(s), i) for i, s in enumerate(self.sequence_strs)]
+        sizes.sort()
+        batches = []
+        buf = []
+        max_len = 0
+
+        def _flush_current_buf():
+            nonlocal max_len, buf
+            if len(buf) == 0:
+                return
+            batches.append(buf)
+            buf = []
+            max_len = 0
+
+        for sz, i in sizes:
+            sz += extra_toks_per_seq
+            if max(sz, max_len) * (len(buf) + 1) > toks_per_batch:
+                _flush_current_buf()
+            max_len = max(max_len, sz)
+            buf.append(i)
+
+        _flush_current_buf()
+        return batches
+
+
+class Alphabet(object):
+    def __init__(
+        self,
+        standard_toks: Sequence[str],
+        prepend_toks: Sequence[str] = ("<null_0>", "<pad>", "<eos>", "<unk>"),
+        append_toks: Sequence[str] = ("<cls>", "<mask>", "<sep>"),
+        prepend_bos: bool = True,
+        append_eos: bool = False,
+        use_msa: bool = False,
+    ):
+        self.standard_toks = list(standard_toks)
+        self.prepend_toks = list(prepend_toks)
+        self.append_toks = list(append_toks)
+        self.prepend_bos = prepend_bos
+        self.append_eos = append_eos
+        self.use_msa = use_msa
+
+        self.all_toks = list(self.prepend_toks)
+        self.all_toks.extend(self.standard_toks)
+        for i in range((8 - (len(self.all_toks) % 8)) % 8):
+            self.all_toks.append(f"<null_{i  + 1}>")
+        self.all_toks.extend(self.append_toks)
+
+        self.tok_to_idx = {tok: i for i, tok in enumerate(self.all_toks)}
+
+        self.unk_idx = self.tok_to_idx["<unk>"]
+        self.padding_idx = self.get_idx("<pad>")
+        self.cls_idx = self.get_idx("<cls>")
+        self.mask_idx = self.get_idx("<mask>")
+        self.eos_idx = self.get_idx("<eos>")
+        self.all_special_tokens = ['<eos>', '<unk>', '<pad>', '<cls>', '<mask>']
+        self.unique_no_split_tokens = self.all_toks
+
+    def __len__(self):
+        return len(self.all_toks)
+
+    def get_idx(self, tok):
+        return self.tok_to_idx.get(tok, self.unk_idx)
+
+    def get_tok(self, ind):
+        return self.all_toks[ind]
+
+    def to_dict(self):
+        return self.tok_to_idx.copy()
+
+    def get_batch_converter(self, truncation_seq_length: int = None):
+        if self.use_msa:
+            return MSABatchConverter(self, truncation_seq_length)
+        else:
+            return BatchConverter(self, truncation_seq_length)
+
+    @classmethod
+    def from_architecture(cls, name: str) -> "Alphabet":
+        if name in ("ESM-1", "protein_bert_base"):
+            standard_toks = proteinseq_toks["toks"]
+            prepend_toks: Tuple[str, ...] = ("<null_0>", "<pad>", "<eos>", "<unk>")
+            append_toks: Tuple[str, ...] = ("<cls>", "<mask>", "<sep>")
+            prepend_bos = True
+            append_eos = False
+            use_msa = False
+        elif name in ("ESM-1b", "roberta_large"):
+            standard_toks = proteinseq_toks["toks"]
+            prepend_toks = ("<cls>", "<pad>", "<eos>", "<unk>")
+            append_toks = ("<mask>",)
+            prepend_bos = True
+            append_eos = True
+            use_msa = False
+        elif name in ("MSA Transformer", "msa_transformer"):
+            standard_toks = proteinseq_toks["toks"]
+            prepend_toks = ("<cls>", "<pad>", "<eos>", "<unk>")
+            append_toks = ("<mask>",)
+            prepend_bos = True
+            append_eos = False
+            use_msa = True
+        elif "invariant_gvp" in name.lower():
+            standard_toks = proteinseq_toks["toks"]
+            prepend_toks = ("<null_0>", "<pad>", "<eos>", "<unk>")
+            append_toks = ("<mask>", "<cath>", "<af2>")
+            prepend_bos = True
+            append_eos = False
+            use_msa = False
+        else:
+            raise ValueError("Unknown architecture selected")
+        return cls(standard_toks, prepend_toks, append_toks, prepend_bos, append_eos, use_msa)
+
+    def _tokenize(self, text) -> str:
+        return text.split()
+
+    def tokenize(self, text, **kwargs) -> List[str]:
+        """
+        Inspired by https://github.com/huggingface/transformers/blob/master/src/transformers/tokenization_utils.py
+        Converts a string in a sequence of tokens, using the tokenizer.
+
+        Args:
+            text (:obj:`str`):
+                The sequence to be encoded.
+
+        Returns:
+            :obj:`List[str]`: The list of tokens.
+        """
+
+        def split_on_token(tok, text):
+            result = []
+            split_text = text.split(tok)
+            for i, sub_text in enumerate(split_text):
+                # AddedToken can control whitespace stripping around them.
+                # We use them for GPT2 and Roberta to have different behavior depending on the special token
+                # Cf. https://github.com/huggingface/transformers/pull/2778
+                # and https://github.com/huggingface/transformers/issues/3788
+                # We strip left and right by default
+                if i < len(split_text) - 1:
+                    sub_text = sub_text.rstrip()
+                if i > 0:
+                    sub_text = sub_text.lstrip()
+
+                if i == 0 and not sub_text:
+                    result.append(tok)
+                elif i == len(split_text) - 1:
+                    if sub_text:
+                        result.append(sub_text)
+                    else:
+                        pass
+                else:
+                    if sub_text:
+                        result.append(sub_text)
+                    result.append(tok)
+            return result
+
+        def split_on_tokens(tok_list, text):
+            if not text.strip():
+                return []
+
+            tokenized_text = []
+            text_list = [text]
+            for tok in tok_list:
+                tokenized_text = []
+                for sub_text in text_list:
+                    if sub_text not in self.unique_no_split_tokens:
+                        tokenized_text.extend(split_on_token(tok, sub_text))
+                    else:
+                        tokenized_text.append(sub_text)
+                text_list = tokenized_text
+
+            return list(
+                itertools.chain.from_iterable(
+                    (
+                        self._tokenize(token)
+                        if token not in self.unique_no_split_tokens
+                        else [token]
+                        for token in tokenized_text
+                    )
+                )
+            )
+
+        no_split_token = self.unique_no_split_tokens
+        tokenized_text = split_on_tokens(no_split_token, text)
+        return tokenized_text
+
+    def encode(self, text):
+        return [self.tok_to_idx[tok] for tok in self.tokenize(text)]
+
+
+class BatchConverter(object):
+    """Callable to convert an unprocessed (labels + strings) batch to a
+    processed (labels + tensor) batch.
+    """
+
+    def __init__(self, alphabet, truncation_seq_length: int = None):
+        self.alphabet = alphabet
+        self.truncation_seq_length = truncation_seq_length
+
+    def __call__(self, raw_batch: Sequence[Tuple[str, str]]):
+        # RoBERTa uses an eos token, while ESM-1 does not.
+        batch_size = len(raw_batch)
+        batch_labels, seq_str_list = zip(*raw_batch)
+        seq_encoded_list = [self.alphabet.encode(seq_str) for seq_str in seq_str_list]
+        if self.truncation_seq_length:
+            seq_encoded_list = [seq_str[:self.truncation_seq_length] for seq_str in seq_encoded_list]
+        max_len = max(len(seq_encoded) for seq_encoded in seq_encoded_list)
+        tokens = torch.empty(
+            (
+                batch_size,
+                max_len + int(self.alphabet.prepend_bos) + int(self.alphabet.append_eos),
+            ),
+            dtype=torch.int64,
+        )
+        tokens.fill_(self.alphabet.padding_idx)
+        labels = []
+        strs = []
+
+        for i, (label, seq_str, seq_encoded) in enumerate(
+            zip(batch_labels, seq_str_list, seq_encoded_list)
+        ):
+            labels.append(label)
+            strs.append(seq_str)
+            if self.alphabet.prepend_bos:
+                tokens[i, 0] = self.alphabet.cls_idx
+            seq = torch.tensor(seq_encoded, dtype=torch.int64)
+            tokens[
+                i,
+                int(self.alphabet.prepend_bos) : len(seq_encoded)
+                + int(self.alphabet.prepend_bos),
+            ] = seq
+            if self.alphabet.append_eos:
+                tokens[i, len(seq_encoded) + int(self.alphabet.prepend_bos)] = self.alphabet.eos_idx
+
+        return labels, strs, tokens
+
+
+class MSABatchConverter(BatchConverter):
+    def __call__(self, inputs: Union[Sequence[RawMSA], RawMSA]):
+        if isinstance(inputs[0][0], str):
+            # Input is a single MSA
+            raw_batch: Sequence[RawMSA] = [inputs]  # type: ignore
+        else:
+            raw_batch = inputs  # type: ignore
+
+        batch_size = len(raw_batch)
+        max_alignments = max(len(msa) for msa in raw_batch)
+        max_seqlen = max(len(msa[0][1]) for msa in raw_batch)
+
+        tokens = torch.empty(
+            (
+                batch_size,
+                max_alignments,
+                max_seqlen + int(self.alphabet.prepend_bos) + int(self.alphabet.append_eos),
+            ),
+            dtype=torch.int64,
+        )
+        tokens.fill_(self.alphabet.padding_idx)
+        labels = []
+        strs = []
+
+        for i, msa in enumerate(raw_batch):
+            msa_seqlens = set(len(seq) for _, seq in msa)
+            if not len(msa_seqlens) == 1:
+                raise RuntimeError(
+                    "Received unaligned sequences for input to MSA, all sequence "
+                    "lengths must be equal."
+                )
+            msa_labels, msa_strs, msa_tokens = super().__call__(msa)
+            labels.append(msa_labels)
+            strs.append(msa_strs)
+            tokens[i, : msa_tokens.size(0), : msa_tokens.size(1)] = msa_tokens
+
+        return labels, strs, tokens
+
+
+def read_fasta(
+    path,
+    keep_gaps=True,
+    keep_insertions=True,
+    to_upper=False,
+):
+    with open(path, "r") as f:
+        for result in read_alignment_lines(
+            f, keep_gaps=keep_gaps, keep_insertions=keep_insertions, to_upper=to_upper
+        ):
+            yield result
+
+
+def read_alignment_lines(
+    lines,
+    keep_gaps=True,
+    keep_insertions=True,
+    to_upper=False,
+):
+    seq = desc = None
+
+    def parse(s):
+        if not keep_gaps:
+            s = re.sub("-", "", s)
+        if not keep_insertions:
+            s = re.sub("[a-z]", "", s)
+        return s.upper() if to_upper else s
+
+    for line in lines:
+        # Line may be empty if seq % file_line_width == 0
+        if len(line) > 0 and line[0] == ">":
+            if seq is not None:
+                yield desc, parse(seq)
+            desc = line.strip().lstrip(">")
+            seq = ""
+        else:
+            assert isinstance(seq, str)
+            seq += line.strip()
+    assert isinstance(seq, str) and isinstance(desc, str)
+    yield desc, parse(seq)
+
+
+class ESMStructuralSplitDataset(torch.utils.data.Dataset):
+    """
+    Structural Split Dataset as described in section A.10 of the supplement of our paper.
+    https://doi.org/10.1101/622803
+
+    We use the full version of SCOPe 2.07, clustered at 90% sequence identity,
+    generated on January 23, 2020.
+
+    For each SCOPe domain:
+        - We extract the sequence from the corresponding PDB file
+        - We extract the 3D coordinates of the Carbon beta atoms, aligning them
+          to the sequence. We put NaN where Cb atoms are missing.
+        - From the 3D coordinates, we calculate a pairwise distance map, based
+          on L2 distance
+        - We use DSSP to generate secondary structure labels for the corresponding
+          PDB file. This is also aligned to the sequence. We put - where SSP
+          labels are missing.
+
+    For each SCOPe classification level of family/superfamily/fold (in order of difficulty),
+    we have split the data into 5 partitions for cross validation. These are provided
+    in a downloaded splits folder, in the format:
+            splits/{split_level}/{cv_partition}/{train|valid}.txt
+    where train is the partition and valid is the concatentation of the remaining 4.
+
+    For each SCOPe domain, we provide a pkl dump that contains:
+        - seq    : The domain sequence, stored as an L-length string
+        - ssp    : The secondary structure labels, stored as an L-length string
+        - dist   : The distance map, stored as an LxL numpy array
+        - coords : The 3D coordinates, stored as an Lx3 numpy array
+
+    """
+
+    base_folder = "structural-data"
+    file_list = [
+        #  url  tar filename   filename      MD5 Hash
+        (
+            "https://dl.fbaipublicfiles.com/fair-esm/structural-data/splits.tar.gz",
+            "splits.tar.gz",
+            "splits",
+            "456fe1c7f22c9d3d8dfe9735da52411d",
+        ),
+        (
+            "https://dl.fbaipublicfiles.com/fair-esm/structural-data/pkl.tar.gz",
+            "pkl.tar.gz",
+            "pkl",
+            "644ea91e56066c750cd50101d390f5db",
+        ),
+    ]
+
+    def __init__(
+        self,
+        split_level,
+        cv_partition,
+        split,
+        root_path=os.path.expanduser("~/.cache/torch/data/esm"),
+        download=False,
+    ):
+        super().__init__()
+        assert split in [
+            "train",
+            "valid",
+        ], "train_valid must be 'train' or 'valid'"
+        self.root_path = root_path
+        self.base_path = os.path.join(self.root_path, self.base_folder)
+
+        # check if root path has what you need or else download it
+        if download:
+            self.download()
+
+        self.split_file = os.path.join(
+            self.base_path, "splits", split_level, cv_partition, f"{split}.txt"
+        )
+        self.pkl_dir = os.path.join(self.base_path, "pkl")
+        self.names = []
+        with open(self.split_file) as f:
+            self.names = f.read().splitlines()
+
+    def __len__(self):
+        return len(self.names)
+
+    def _check_exists(self) -> bool:
+        for (_, _, filename, _) in self.file_list:
+            fpath = os.path.join(self.base_path, filename)
+            if not os.path.exists(fpath) or not os.path.isdir(fpath):
+                return False
+        return True
+
+    def download(self):
+
+        if self._check_exists():
+            print("Files already downloaded and verified")
+            return
+
+        from torchvision.datasets.utils import download_url
+
+        for url, tar_filename, filename, md5_hash in self.file_list:
+            download_path = os.path.join(self.base_path, tar_filename)
+            download_url(url=url, root=self.base_path, filename=tar_filename, md5=md5_hash)
+            shutil.unpack_archive(download_path, self.base_path)
+
+    def __getitem__(self, idx):
+        """
+        Returns a dict with the following entires
+         - seq : Str (domain sequence)
+         - ssp : Str (SSP labels)
+         - dist : np.array (distance map)
+         - coords : np.array (3D coordinates)
+        """
+        name = self.names[idx]
+        pkl_fname = os.path.join(self.pkl_dir, name[1:3], f"{name}.pkl")
+        with open(pkl_fname, "rb") as f:
+            obj = pickle.load(f)
+        return obj

weight/esm-main/esm/esmfold/v1/categorical_mixture.py

@@ -0,0 +1,43 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+import torch
+
+
+class CategoricalMixture:
+    def __init__(self, param, bins=50, start=0, end=1):
+        # All tensors are of shape ..., bins.
+        self.logits = param
+        bins = torch.linspace(
+            start, end, bins + 1, device=self.logits.device, dtype=self.logits.dtype
+        )
+        self.v_bins = (bins[:-1] + bins[1:]) / 2
+
+    def log_prob(self, true):
+        # Shapes are:
+        #     self.probs: ... x bins
+        #     true      : ...
+        true_index = (
+            (
+                true.unsqueeze(-1)
+                - self.v_bins[
+                    [
+                        None,
+                    ]
+                    * true.ndim
+                ]
+            )
+            .abs()
+            .argmin(-1)
+        )
+        nll = self.logits.log_softmax(-1)
+        return torch.take_along_dim(nll, true_index.unsqueeze(-1), dim=-1).squeeze(-1)
+
+    def mean(self):
+        return (self.logits.softmax(-1) @ self.v_bins.unsqueeze(1)).squeeze(-1)
+
+
+def categorical_lddt(logits, bins=50):
+    # Logits are ..., 37, bins.
+    return CategoricalMixture(logits, bins=bins).mean()