Spaces:
Running on Zero
Running on Zero
File size: 7,474 Bytes
4ea7a41 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 | import torch
from torch import nn
import torch.nn.functional as F
from layers.Transformer_EncDec import Encoder, EncoderLayer
from layers.SelfAttention_Family import FullAttention, AttentionLayerWithRoPE
class RevIN(nn.Module):
def __init__(self, dim=-1, std_min=1e-5, use_sinh=False):
super().__init__()
self.dim = dim
self.std_min = std_min
self.use_sinh = use_sinh
def fit_transform(self, x, mask=None):
device_type = "cuda" if x.is_cuda else "cpu"
with torch.autocast(device_type=device_type, enabled=False):
self._get_statistics(x, mask)
return self.transform(x)
def transform(self, x):
device_type = "cuda" if x.is_cuda else "cpu"
with torch.autocast(device_type=device_type, enabled=False):
x = (x - self.mean) / self.std
if self.use_sinh:
x = torch.asinh(x)
return x
def inverse_transform(self, x):
device_type = "cuda" if x.is_cuda else "cpu"
with torch.autocast(device_type=device_type, enabled=False):
if self.use_sinh:
x = torch.sinh(x)
if x.ndim != self.mean.ndim:
x = x * self.std.unsqueeze(1) + self.mean.unsqueeze(1)
else:
x = x * self.std + self.mean
return x
def _get_statistics(self, x, mask=None):
if mask is None:
self.mean = x.mean(dim=self.dim, keepdim=True)
std = x.std(dim=self.dim, keepdim=True)
self.std = torch.where(std > self.std_min, std, torch.ones_like(std))
else:
mask = mask.bool()
unmask = (~mask).float()
count = unmask.sum(dim=self.dim, keepdim=True).clamp(min=1)
x_mean = (x * unmask).sum(dim=self.dim, keepdim=True) / count
x_std = (((x - x_mean) * unmask) ** 2).sum(dim=self.dim, keepdim=True) / count
x_std = x_std.sqrt()
x_std = torch.where(x_std > self.std_min, x_std, torch.ones_like(x_std))
self.mean = x_mean
self.std = x_std
class ResidualBlock(nn.Module):
def __init__(self, d_in: int, d_out: int, d_hidden: int):
super().__init__()
self.layer1 = nn.Linear(d_in, d_hidden)
self.layer2 = nn.Linear(d_hidden, d_out)
self.residual = nn.Linear(d_in, d_out)
self.activation = nn.Sigmoid()
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.layer2(self.activation(self.layer1(x))) + self.residual(x)
class Model(nn.Module):
def __init__(self, configs, patch_len=None, stride=None):
super().__init__()
self.patch_len = patch_len if patch_len is not None else getattr(configs, 'patch_len', 16)
self.stride = stride if stride is not None else getattr(configs, 'stride', self.patch_len)
self.num_quantile = getattr(configs, 'num_quantile', 9)
d_model = configs.d_model
d_ff = getattr(configs, 'd_ff', d_model * 4)
self.in_layer = ResidualBlock(self.patch_len * 2, d_model, d_model)
self.out_layer = ResidualBlock(d_model, self.patch_len * (self.num_quantile + 1), d_model)
self.encoder = Encoder(
[
EncoderLayer(
AttentionLayerWithRoPE(
FullAttention(False, configs.factor, attention_dropout=configs.dropout,
output_attention=False), d_model, configs.n_heads),
d_model, d_ff,
dropout=configs.dropout,
activation=getattr(configs, 'activation', 'gelu'),
norm_first=True,
) for _ in range(configs.e_layers)
],
norm_layer=nn.LayerNorm(d_model),
)
self.norm_fn = RevIN(dim=-1, std_min=1e-5, use_sinh=True)
def _build_attention_mask(self, pad_patch):
batch_size, n_patch, _ = pad_patch.shape
pad_ratio = pad_patch.float().mean(dim=-1)
fully_padded = pad_ratio > 0.9
attn_mask = torch.zeros(batch_size, 1, 1, n_patch,
device=pad_patch.device, dtype=torch.float32)
attn_mask.masked_fill_(fully_padded[:, None, None, :], float('-inf'))
return attn_mask
def _decode_quantiles(self, raw, T):
flat_batch, P, _ = raw.shape
Q = self.num_quantile
d = self.patch_len
raw = raw.reshape(flat_batch, P, Q + 1, d)
base = raw[:, :, 0:1, :]
offsets = raw[:, :, 1:, :]
q = base + torch.cumsum(F.softplus(offsets) / Q, dim=2)
q = q.permute(0, 2, 1, 3)
unpatched = q.reshape(flat_batch * Q, P * d)
return unpatched.reshape(flat_batch, Q, T)
def imputation(self, x_enc, mask, miss_mask=None, pad_mask=None, return_raw=True):
B, T, C = x_enc.shape
d = self.patch_len
x_flat = x_enc.permute(0, 2, 1).reshape(B * C, T)
mask_flat = mask.permute(0, 2, 1).reshape(B * C, T)
pred_mask = (mask_flat == 0)
if miss_mask is None and pad_mask is None:
miss_flat = torch.zeros_like(pred_mask)
pad_flat = torch.isnan(x_flat)
else:
miss_flat = (
miss_mask.permute(0, 2, 1).reshape(B * C, T).bool()
if miss_mask is not None else torch.zeros_like(pred_mask)
)
pad_flat = (
pad_mask.permute(0, 2, 1).reshape(B * C, T).bool()
if pad_mask is not None else torch.zeros_like(pred_mask)
)
ts_mask = pred_mask | miss_flat | pad_flat
safe_x = torch.nan_to_num(x_flat, nan=0.0)
x_target = self.norm_fn.fit_transform(safe_x, mask=ts_mask)
x_input = torch.where(ts_mask, torch.zeros_like(x_target), x_target)
n_patch = T // d
T_aligned = n_patch * d
x_aligned = x_input[:, :T_aligned]
mask_aligned = ts_mask[:, :T_aligned].float()
pad_aligned = pad_flat[:, :T_aligned].float()
x_patch = x_aligned.reshape(B * C, n_patch, d)
mask_patch = mask_aligned.reshape(B * C, n_patch, d)
pad_patch = pad_aligned.reshape(B * C, n_patch, d)
attn_mask = self._build_attention_mask(pad_patch)
x = self.in_layer(torch.cat([x_patch, 1 - mask_patch], dim=-1))
x, _ = self.encoder(x, attn_mask=attn_mask)
raw = self.out_layer(x)
q_pred = self._decode_quantiles(raw, T_aligned)
if T_aligned < T:
pad = T - T_aligned
q_pred = F.pad(q_pred, (pad, 0), value=0.0)
Q = self.num_quantile
if not return_raw:
q_asinh = q_pred.reshape(B, C, Q, T).permute(0, 3, 1, 2)
x_tgt = x_target.reshape(B, C, T).permute(0, 2, 1)
return q_asinh, x_tgt
q_flat = q_pred.reshape(B * C * Q, T)
saved_mean = self.norm_fn.mean.repeat_interleave(Q, dim=0)
saved_std = self.norm_fn.std.repeat_interleave(Q, dim=0)
self.norm_fn.mean, self.norm_fn.std = saved_mean, saved_std
q_inv = self.norm_fn.inverse_transform(q_flat)
q_inv = q_inv.reshape(B, C, Q, T).permute(0, 3, 1, 2)
return q_inv
def forward(self, x_enc, mask=None, miss_mask=None, pad_mask=None, return_raw=True):
return self.imputation(x_enc, mask, miss_mask=miss_mask, pad_mask=pad_mask,
return_raw=return_raw)
|