Scinetics-ai's picture
Release the all-atom structural tokenizer reference implementation
4c231da verified
Raw History Blame Contribute Delete
9.58 kB
"""A trainable implementation of the Scinetics all-atom structural tokenizer."""
from dataclasses import dataclass
from math import sqrt
import torch
from torch import nn
from torch.nn import functional as F
from .geometry import AtomBlock, ChemicalEmbedding, neighbors, radial, reduce_sum, signed_volumes
@dataclass(frozen=True)
class Config:
scalar: int = 256
vectors: int = 64
unit: int = 512
decoder: int = 768
atom_layers: int = 8
unit_layers: int = 4
decoder_layers: int = 12
refine_layers: int = 4
heads: int = 8
@classmethod
def tiny(cls):
return cls(32, 8, 48, 64, 2, 2, 2, 2, 4)
class FSQ(nn.Module):
def __init__(self, width):
super().__init__()
self.project = nn.Linear(width, 5)
self.register_buffer("levels", torch.tensor([8, 8, 8, 4, 4]))
self.register_buffer("bases", torch.tensor([1, 8, 64, 512, 2048]))
def from_ids(self, ids):
if ids.dtype != torch.long or ((ids < 0) | (ids >= 8192)).any():
raise ValueError("Expected int64 structure IDs in [0, 8191].")
digits = (ids[..., None] // self.bases) % self.levels
return 2 * digits.to(self.project.weight.dtype) / (self.levels-1) - 1
def forward(self, h):
raw = (self.project(h).tanh() + 1) * (self.levels-1) / 2
digits = raw.round()
ids = (digits.long() * self.bases).sum(-1)
straight_through = digits.detach() + (raw - raw.detach())
return ids, 2 * straight_through / (self.levels-1) - 1, raw
class ContextBlock(nn.Module):
def __init__(self, c):
super().__init__()
self.heads = c.heads
self.norm = nn.LayerNorm(c.unit)
self.qkv = nn.Linear(c.unit, 3*c.unit)
self.project = nn.Linear(c.unit, c.unit)
# RBF distance (16), chain-relative sequence offset, same chain, three contact types, orientation.
self.bias = nn.Linear(22, c.heads)
self.ff = nn.Sequential(nn.LayerNorm(c.unit), nn.Linear(c.unit, 4*c.unit),
nn.GELU(), nn.Linear(4*c.unit, c.unit))
self.vmix = nn.Linear(c.vectors, c.vectors, bias=False)
self.gate = nn.Linear(c.unit, c.vectors)
def forward(self, h, v, static, displacement):
length, width = h.shape
q, k, val = self.qkv(self.norm(h)).reshape(length, 3, self.heads, -1).permute(1, 2, 0, 3)
orient = (v.mean(1)[:, None, :] * displacement).sum(-1, keepdim=True)
bias = self.bias(torch.cat((static, orient), -1)).permute(2, 0, 1)
attn = (q @ k.transpose(-2, -1) / sqrt(width // self.heads) + bias).softmax(-1)
h = h + self.project((attn @ val).transpose(0, 1).reshape(length, width))
h = h + self.ff(h)
v = v + torch.einsum("ij,jvc->ivc", attn.mean(0), self.vmix(v.transpose(1, 2)).transpose(1, 2))
return h, v * self.gate(h).sigmoid()[..., None]
class Encoder(nn.Module):
def __init__(self, c):
super().__init__()
self.vectors = c.vectors
self.embed = ChemicalEmbedding(c.scalar)
self.blocks = nn.ModuleList(AtomBlock(c.scalar, c.vectors) for _ in range(c.atom_layers))
self.pool = nn.Linear(c.scalar, 1)
self.start = nn.Linear(c.scalar, c.unit)
self.context = nn.ModuleList(ContextBlock(c) for _ in range(c.unit_layers))
self.readout = nn.Sequential(nn.Linear(c.unit + 2*c.vectors + 2, c.unit), nn.LayerNorm(c.unit))
def forward(self, chem, xyz, mask):
if xyz.shape != (len(chem.unit), 3) or mask.shape != chem.unit.shape or mask.dtype != torch.bool:
raise ValueError("Expected coordinates [atoms,3] and a boolean mask [atoms].")
if not mask.any() or not torch.isfinite(xyz[mask]).all():
raise ValueError("Supply finite coordinates for at least one observed atom.")
x = torch.where(mask[:, None], xyz, 0.)
u, L = chem.unit, chem.units
count = reduce_sum(mask.to(x.dtype), u, L)
present = count > 0
h = self.embed(chem.features)
v = x.new_zeros(len(x), self.vectors, 3)
edges = neighbors(chem, x, mask)
for block in self.blocks:
h, v = block(h, v, x, edges, chem)
scores = self.pool(h).squeeze(-1).masked_fill(~mask, -torch.inf)
peak = scores.new_full((L,), -torch.inf).scatter_reduce_(0, u, scores, reduce="amax")
unnorm = torch.where(mask, (scores-peak[u].nan_to_num(neginf=0)).exp(), 0.)
w = unnorm / reduce_sum(unnorm, u, L)[u].clamp_min(1e-8)
H = self.start(reduce_sum(w[:, None] * h, u, L))
U = reduce_sum(w[:, None, None] * v, u, L)
center = reduce_sum(x, u, L) / count.clamp_min(1)[:, None]
delta = center[None, :, :] - center[:, None, :]
valid_pair = present[:, None] & present[None, :]
direction = F.normalize(delta, dim=-1, eps=1e-7) * valid_pair[..., None]
same_chain = chem.chain[:, None] == chem.chain
offset = (chem.position[None, :] - chem.position[:, None]).clamp(-32, 32) / 32
contacts = x.new_zeros(L, L, 3)
dst, src, rel, _ = edges
contacts[u[dst], u[src], rel] = 1.
static = torch.cat((radial(delta.norm(dim=-1)) * valid_pair[..., None],
(offset * same_chain)[..., None], same_chain[..., None], contacts), -1)
for block in self.context:
H, U = block(H, U, static, direction)
triplets = chem.triplets[mask[chem.triplets].all(dim=1)]
volume = signed_volumes(x, triplets)
group = u[triplets[:, 0]]
denom = torch.bincount(group, minlength=L).clamp_min(1)
chi = reduce_sum(volume, group, L) / denom
magnitude = reduce_sum(volume.abs(), group, L) / denom
inv = torch.cat((H, U.norm(dim=-1), (U*U.roll(1, dims=1)).sum(-1),
chi[:, None], magnitude[:, None]), -1)
return self.readout(inv), present
class Decoder(nn.Module):
def __init__(self, c):
super().__init__()
self.vectors = c.vectors
self.embed = ChemicalEmbedding(c.scalar)
self.code = nn.Linear(5, c.decoder)
self.unit_chem = nn.Linear(c.scalar, c.decoder)
self.time = nn.Sequential(nn.Linear(3, c.decoder), nn.SiLU(), nn.Linear(c.decoder, c.decoder))
self.unit_noise = nn.Linear(2, c.decoder)
self.unit_blocks = nn.ModuleList(nn.TransformerEncoderLayer(
c.decoder, c.heads, 4*c.decoder, dropout=0., batch_first=True, norm_first=True,
activation="gelu") for _ in range(c.decoder_layers))
self.expand = nn.Linear(c.decoder, c.scalar)
self.radius = nn.Linear(1, c.scalar)
self.refine = nn.ModuleList(AtomBlock(c.scalar, c.vectors) for _ in range(c.refine_layers))
self.velocity = nn.Linear(c.vectors, 3, bias=False)
self.gate = nn.Linear(c.scalar, 2)
def forward(self, chem, codes, x, t):
# x is the current noisy state in units of 10 angstroms.
u, L = chem.unit, chem.units
count = torch.bincount(u, minlength=L).to(x.dtype)
x = x - x.mean(0)
center = reduce_sum(x, u, L) / count[:, None]
radii = (x-center[u]).norm(dim=-1, keepdim=True)
unit_radii = reduce_sum(radii, u, L) / count[:, None]
t = torch.as_tensor(t, device=x.device, dtype=x.dtype).reshape(1)
time = self.time(torch.cat((t, (torch.pi*t).sin(), (torch.pi*t).cos())))
h = self.embed(chem.features)
H = self.code(codes) + time + self.unit_chem(reduce_sum(h, u, L) / count[:, None])
H = H + self.unit_noise(torch.cat((center.norm(dim=-1, keepdim=True), unit_radii), -1))
for block in self.unit_blocks:
H = block(H)
h = h + self.expand(H[u]) + self.radius(radii)
v = x.new_zeros(len(x), self.vectors, 3)
v = v + x[:, None, :]
valid = torch.ones_like(u, dtype=torch.bool)
edges = neighbors(chem, x * 10, valid, within_unit=True)
for block in self.refine:
h, v = block(h, v, x * 10, edges, chem)
polar, v1, v2 = self.velocity(v.transpose(1, 2)).unbind(-1)
gates = self.gate(h)
velocity = gates[:, :1] * polar + gates[:, 1:] * torch.linalg.cross(v1, v2)
return velocity - velocity.mean(0)
class StructureTokenizer(nn.Module):
def __init__(self, config=None):
super().__init__()
self.config = config or Config()
self.encoder = Encoder(self.config)
self.fsq = FSQ(self.config.unit)
self.decoder = Decoder(self.config)
def forward(self, chemistry, xyz, observed):
h, present = self.encoder(chemistry, xyz, observed)
ids, codes, raw = self.fsq(h)
return ids, codes, raw, present
@torch.no_grad()
def encode(self, chemistry, xyz, observed):
ids, _, _, present = self(chemistry, xyz, observed)
return ids.masked_fill(~present, -1)
@torch.no_grad()
def decode(self, chemistry, ids, *, steps=50, generator=None):
if steps < 1 or ids.shape != (chemistry.units,):
raise ValueError("Provide a positive Heun step count and one ID per chemical unit.")
codes = self.fsq.from_ids(ids)
x = torch.randn(len(chemistry.unit), 3, device=codes.device, dtype=codes.dtype, generator=generator)
x -= x.mean(0)
dt = 1 / steps
for i in range(steps):
v1 = self.decoder(chemistry, codes, x, i*dt)
v2 = self.decoder(chemistry, codes, x + dt*v1, (i+1)*dt)
x = x + .5*dt*(v1+v2)
return 10 * (x-x.mean(0))