"""A trainable implementation of the Scinetics all-atom structural tokenizer.""" from dataclasses import dataclass from math import sqrt import torch from torch import nn from torch.nn import functional as F from .geometry import AtomBlock, ChemicalEmbedding, neighbors, radial, reduce_sum, signed_volumes @dataclass(frozen=True) class Config: scalar: int = 256 vectors: int = 64 unit: int = 512 decoder: int = 768 atom_layers: int = 8 unit_layers: int = 4 decoder_layers: int = 12 refine_layers: int = 4 heads: int = 8 @classmethod def tiny(cls): return cls(32, 8, 48, 64, 2, 2, 2, 2, 4) class FSQ(nn.Module): def __init__(self, width): super().__init__() self.project = nn.Linear(width, 5) self.register_buffer("levels", torch.tensor([8, 8, 8, 4, 4])) self.register_buffer("bases", torch.tensor([1, 8, 64, 512, 2048])) def from_ids(self, ids): if ids.dtype != torch.long or ((ids < 0) | (ids >= 8192)).any(): raise ValueError("Expected int64 structure IDs in [0, 8191].") digits = (ids[..., None] // self.bases) % self.levels return 2 * digits.to(self.project.weight.dtype) / (self.levels-1) - 1 def forward(self, h): raw = (self.project(h).tanh() + 1) * (self.levels-1) / 2 digits = raw.round() ids = (digits.long() * self.bases).sum(-1) straight_through = digits.detach() + (raw - raw.detach()) return ids, 2 * straight_through / (self.levels-1) - 1, raw class ContextBlock(nn.Module): def __init__(self, c): super().__init__() self.heads = c.heads self.norm = nn.LayerNorm(c.unit) self.qkv = nn.Linear(c.unit, 3*c.unit) self.project = nn.Linear(c.unit, c.unit) # RBF distance (16), chain-relative sequence offset, same chain, three contact types, orientation. self.bias = nn.Linear(22, c.heads) self.ff = nn.Sequential(nn.LayerNorm(c.unit), nn.Linear(c.unit, 4*c.unit), nn.GELU(), nn.Linear(4*c.unit, c.unit)) self.vmix = nn.Linear(c.vectors, c.vectors, bias=False) self.gate = nn.Linear(c.unit, c.vectors) def forward(self, h, v, static, displacement): length, width = h.shape q, k, val = self.qkv(self.norm(h)).reshape(length, 3, self.heads, -1).permute(1, 2, 0, 3) orient = (v.mean(1)[:, None, :] * displacement).sum(-1, keepdim=True) bias = self.bias(torch.cat((static, orient), -1)).permute(2, 0, 1) attn = (q @ k.transpose(-2, -1) / sqrt(width // self.heads) + bias).softmax(-1) h = h + self.project((attn @ val).transpose(0, 1).reshape(length, width)) h = h + self.ff(h) v = v + torch.einsum("ij,jvc->ivc", attn.mean(0), self.vmix(v.transpose(1, 2)).transpose(1, 2)) return h, v * self.gate(h).sigmoid()[..., None] class Encoder(nn.Module): def __init__(self, c): super().__init__() self.vectors = c.vectors self.embed = ChemicalEmbedding(c.scalar) self.blocks = nn.ModuleList(AtomBlock(c.scalar, c.vectors) for _ in range(c.atom_layers)) self.pool = nn.Linear(c.scalar, 1) self.start = nn.Linear(c.scalar, c.unit) self.context = nn.ModuleList(ContextBlock(c) for _ in range(c.unit_layers)) self.readout = nn.Sequential(nn.Linear(c.unit + 2*c.vectors + 2, c.unit), nn.LayerNorm(c.unit)) def forward(self, chem, xyz, mask): if xyz.shape != (len(chem.unit), 3) or mask.shape != chem.unit.shape or mask.dtype != torch.bool: raise ValueError("Expected coordinates [atoms,3] and a boolean mask [atoms].") if not mask.any() or not torch.isfinite(xyz[mask]).all(): raise ValueError("Supply finite coordinates for at least one observed atom.") x = torch.where(mask[:, None], xyz, 0.) u, L = chem.unit, chem.units count = reduce_sum(mask.to(x.dtype), u, L) present = count > 0 h = self.embed(chem.features) v = x.new_zeros(len(x), self.vectors, 3) edges = neighbors(chem, x, mask) for block in self.blocks: h, v = block(h, v, x, edges, chem) scores = self.pool(h).squeeze(-1).masked_fill(~mask, -torch.inf) peak = scores.new_full((L,), -torch.inf).scatter_reduce_(0, u, scores, reduce="amax") unnorm = torch.where(mask, (scores-peak[u].nan_to_num(neginf=0)).exp(), 0.) w = unnorm / reduce_sum(unnorm, u, L)[u].clamp_min(1e-8) H = self.start(reduce_sum(w[:, None] * h, u, L)) U = reduce_sum(w[:, None, None] * v, u, L) center = reduce_sum(x, u, L) / count.clamp_min(1)[:, None] delta = center[None, :, :] - center[:, None, :] valid_pair = present[:, None] & present[None, :] direction = F.normalize(delta, dim=-1, eps=1e-7) * valid_pair[..., None] same_chain = chem.chain[:, None] == chem.chain offset = (chem.position[None, :] - chem.position[:, None]).clamp(-32, 32) / 32 contacts = x.new_zeros(L, L, 3) dst, src, rel, _ = edges contacts[u[dst], u[src], rel] = 1. static = torch.cat((radial(delta.norm(dim=-1)) * valid_pair[..., None], (offset * same_chain)[..., None], same_chain[..., None], contacts), -1) for block in self.context: H, U = block(H, U, static, direction) triplets = chem.triplets[mask[chem.triplets].all(dim=1)] volume = signed_volumes(x, triplets) group = u[triplets[:, 0]] denom = torch.bincount(group, minlength=L).clamp_min(1) chi = reduce_sum(volume, group, L) / denom magnitude = reduce_sum(volume.abs(), group, L) / denom inv = torch.cat((H, U.norm(dim=-1), (U*U.roll(1, dims=1)).sum(-1), chi[:, None], magnitude[:, None]), -1) return self.readout(inv), present class Decoder(nn.Module): def __init__(self, c): super().__init__() self.vectors = c.vectors self.embed = ChemicalEmbedding(c.scalar) self.code = nn.Linear(5, c.decoder) self.unit_chem = nn.Linear(c.scalar, c.decoder) self.time = nn.Sequential(nn.Linear(3, c.decoder), nn.SiLU(), nn.Linear(c.decoder, c.decoder)) self.unit_noise = nn.Linear(2, c.decoder) self.unit_blocks = nn.ModuleList(nn.TransformerEncoderLayer( c.decoder, c.heads, 4*c.decoder, dropout=0., batch_first=True, norm_first=True, activation="gelu") for _ in range(c.decoder_layers)) self.expand = nn.Linear(c.decoder, c.scalar) self.radius = nn.Linear(1, c.scalar) self.refine = nn.ModuleList(AtomBlock(c.scalar, c.vectors) for _ in range(c.refine_layers)) self.velocity = nn.Linear(c.vectors, 3, bias=False) self.gate = nn.Linear(c.scalar, 2) def forward(self, chem, codes, x, t): # x is the current noisy state in units of 10 angstroms. u, L = chem.unit, chem.units count = torch.bincount(u, minlength=L).to(x.dtype) x = x - x.mean(0) center = reduce_sum(x, u, L) / count[:, None] radii = (x-center[u]).norm(dim=-1, keepdim=True) unit_radii = reduce_sum(radii, u, L) / count[:, None] t = torch.as_tensor(t, device=x.device, dtype=x.dtype).reshape(1) time = self.time(torch.cat((t, (torch.pi*t).sin(), (torch.pi*t).cos()))) h = self.embed(chem.features) H = self.code(codes) + time + self.unit_chem(reduce_sum(h, u, L) / count[:, None]) H = H + self.unit_noise(torch.cat((center.norm(dim=-1, keepdim=True), unit_radii), -1)) for block in self.unit_blocks: H = block(H) h = h + self.expand(H[u]) + self.radius(radii) v = x.new_zeros(len(x), self.vectors, 3) v = v + x[:, None, :] valid = torch.ones_like(u, dtype=torch.bool) edges = neighbors(chem, x * 10, valid, within_unit=True) for block in self.refine: h, v = block(h, v, x * 10, edges, chem) polar, v1, v2 = self.velocity(v.transpose(1, 2)).unbind(-1) gates = self.gate(h) velocity = gates[:, :1] * polar + gates[:, 1:] * torch.linalg.cross(v1, v2) return velocity - velocity.mean(0) class StructureTokenizer(nn.Module): def __init__(self, config=None): super().__init__() self.config = config or Config() self.encoder = Encoder(self.config) self.fsq = FSQ(self.config.unit) self.decoder = Decoder(self.config) def forward(self, chemistry, xyz, observed): h, present = self.encoder(chemistry, xyz, observed) ids, codes, raw = self.fsq(h) return ids, codes, raw, present @torch.no_grad() def encode(self, chemistry, xyz, observed): ids, _, _, present = self(chemistry, xyz, observed) return ids.masked_fill(~present, -1) @torch.no_grad() def decode(self, chemistry, ids, *, steps=50, generator=None): if steps < 1 or ids.shape != (chemistry.units,): raise ValueError("Provide a positive Heun step count and one ID per chemical unit.") codes = self.fsq.from_ids(ids) x = torch.randn(len(chemistry.unit), 3, device=codes.device, dtype=codes.dtype, generator=generator) x -= x.mean(0) dt = 1 / steps for i in range(steps): v1 = self.decoder(chemistry, codes, x, i*dt) v2 = self.decoder(chemistry, codes, x + dt*v1, (i+1)*dt) x = x + .5*dt*(v1+v2) return 10 * (x-x.mean(0))