multimodalart's picture
multimodalart HF Staff
Upload folder using huggingface_hub
4ea7a41 verified
Raw History Blame Contribute Delete
7.47 kB
import torch
from torch import nn
import torch.nn.functional as F
from layers.Transformer_EncDec import Encoder, EncoderLayer
from layers.SelfAttention_Family import FullAttention, AttentionLayerWithRoPE
class RevIN(nn.Module):
def __init__(self, dim=-1, std_min=1e-5, use_sinh=False):
super().__init__()
self.dim = dim
self.std_min = std_min
self.use_sinh = use_sinh
def fit_transform(self, x, mask=None):
device_type = "cuda" if x.is_cuda else "cpu"
with torch.autocast(device_type=device_type, enabled=False):
self._get_statistics(x, mask)
return self.transform(x)
def transform(self, x):
device_type = "cuda" if x.is_cuda else "cpu"
with torch.autocast(device_type=device_type, enabled=False):
x = (x - self.mean) / self.std
if self.use_sinh:
x = torch.asinh(x)
return x
def inverse_transform(self, x):
device_type = "cuda" if x.is_cuda else "cpu"
with torch.autocast(device_type=device_type, enabled=False):
if self.use_sinh:
x = torch.sinh(x)
if x.ndim != self.mean.ndim:
x = x * self.std.unsqueeze(1) + self.mean.unsqueeze(1)
else:
x = x * self.std + self.mean
return x
def _get_statistics(self, x, mask=None):
if mask is None:
self.mean = x.mean(dim=self.dim, keepdim=True)
std = x.std(dim=self.dim, keepdim=True)
self.std = torch.where(std > self.std_min, std, torch.ones_like(std))
else:
mask = mask.bool()
unmask = (~mask).float()
count = unmask.sum(dim=self.dim, keepdim=True).clamp(min=1)
x_mean = (x * unmask).sum(dim=self.dim, keepdim=True) / count
x_std = (((x - x_mean) * unmask) ** 2).sum(dim=self.dim, keepdim=True) / count
x_std = x_std.sqrt()
x_std = torch.where(x_std > self.std_min, x_std, torch.ones_like(x_std))
self.mean = x_mean
self.std = x_std
class ResidualBlock(nn.Module):
def __init__(self, d_in: int, d_out: int, d_hidden: int):
super().__init__()
self.layer1 = nn.Linear(d_in, d_hidden)
self.layer2 = nn.Linear(d_hidden, d_out)
self.residual = nn.Linear(d_in, d_out)
self.activation = nn.Sigmoid()
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.layer2(self.activation(self.layer1(x))) + self.residual(x)
class Model(nn.Module):
def __init__(self, configs, patch_len=None, stride=None):
super().__init__()
self.patch_len = patch_len if patch_len is not None else getattr(configs, 'patch_len', 16)
self.stride = stride if stride is not None else getattr(configs, 'stride', self.patch_len)
self.num_quantile = getattr(configs, 'num_quantile', 9)
d_model = configs.d_model
d_ff = getattr(configs, 'd_ff', d_model * 4)
self.in_layer = ResidualBlock(self.patch_len * 2, d_model, d_model)
self.out_layer = ResidualBlock(d_model, self.patch_len * (self.num_quantile + 1), d_model)
self.encoder = Encoder(
[
EncoderLayer(
AttentionLayerWithRoPE(
FullAttention(False, configs.factor, attention_dropout=configs.dropout,
output_attention=False), d_model, configs.n_heads),
d_model, d_ff,
dropout=configs.dropout,
activation=getattr(configs, 'activation', 'gelu'),
norm_first=True,
) for _ in range(configs.e_layers)
],
norm_layer=nn.LayerNorm(d_model),
)
self.norm_fn = RevIN(dim=-1, std_min=1e-5, use_sinh=True)
def _build_attention_mask(self, pad_patch):
batch_size, n_patch, _ = pad_patch.shape
pad_ratio = pad_patch.float().mean(dim=-1)
fully_padded = pad_ratio > 0.9
attn_mask = torch.zeros(batch_size, 1, 1, n_patch,
device=pad_patch.device, dtype=torch.float32)
attn_mask.masked_fill_(fully_padded[:, None, None, :], float('-inf'))
return attn_mask
def _decode_quantiles(self, raw, T):
flat_batch, P, _ = raw.shape
Q = self.num_quantile
d = self.patch_len
raw = raw.reshape(flat_batch, P, Q + 1, d)
base = raw[:, :, 0:1, :]
offsets = raw[:, :, 1:, :]
q = base + torch.cumsum(F.softplus(offsets) / Q, dim=2)
q = q.permute(0, 2, 1, 3)
unpatched = q.reshape(flat_batch * Q, P * d)
return unpatched.reshape(flat_batch, Q, T)
def imputation(self, x_enc, mask, miss_mask=None, pad_mask=None, return_raw=True):
B, T, C = x_enc.shape
d = self.patch_len
x_flat = x_enc.permute(0, 2, 1).reshape(B * C, T)
mask_flat = mask.permute(0, 2, 1).reshape(B * C, T)
pred_mask = (mask_flat == 0)
if miss_mask is None and pad_mask is None:
miss_flat = torch.zeros_like(pred_mask)
pad_flat = torch.isnan(x_flat)
else:
miss_flat = (
miss_mask.permute(0, 2, 1).reshape(B * C, T).bool()
if miss_mask is not None else torch.zeros_like(pred_mask)
)
pad_flat = (
pad_mask.permute(0, 2, 1).reshape(B * C, T).bool()
if pad_mask is not None else torch.zeros_like(pred_mask)
)
ts_mask = pred_mask | miss_flat | pad_flat
safe_x = torch.nan_to_num(x_flat, nan=0.0)
x_target = self.norm_fn.fit_transform(safe_x, mask=ts_mask)
x_input = torch.where(ts_mask, torch.zeros_like(x_target), x_target)
n_patch = T // d
T_aligned = n_patch * d
x_aligned = x_input[:, :T_aligned]
mask_aligned = ts_mask[:, :T_aligned].float()
pad_aligned = pad_flat[:, :T_aligned].float()
x_patch = x_aligned.reshape(B * C, n_patch, d)
mask_patch = mask_aligned.reshape(B * C, n_patch, d)
pad_patch = pad_aligned.reshape(B * C, n_patch, d)
attn_mask = self._build_attention_mask(pad_patch)
x = self.in_layer(torch.cat([x_patch, 1 - mask_patch], dim=-1))
x, _ = self.encoder(x, attn_mask=attn_mask)
raw = self.out_layer(x)
q_pred = self._decode_quantiles(raw, T_aligned)
if T_aligned < T:
pad = T - T_aligned
q_pred = F.pad(q_pred, (pad, 0), value=0.0)
Q = self.num_quantile
if not return_raw:
q_asinh = q_pred.reshape(B, C, Q, T).permute(0, 3, 1, 2)
x_tgt = x_target.reshape(B, C, T).permute(0, 2, 1)
return q_asinh, x_tgt
q_flat = q_pred.reshape(B * C * Q, T)
saved_mean = self.norm_fn.mean.repeat_interleave(Q, dim=0)
saved_std = self.norm_fn.std.repeat_interleave(Q, dim=0)
self.norm_fn.mean, self.norm_fn.std = saved_mean, saved_std
q_inv = self.norm_fn.inverse_transform(q_flat)
q_inv = q_inv.reshape(B, C, Q, T).permute(0, 3, 1, 2)
return q_inv
def forward(self, x_enc, mask=None, miss_mask=None, pad_mask=None, return_raw=True):
return self.imputation(x_enc, mask, miss_mask=miss_mask, pad_mask=pad_mask,
return_raw=return_raw)