Spaces:
Running on Zero
Running on Zero
Download layers/Embed.py from hugging-apps/flexcast-forecasting: direct link, hf CLI and curl.
- Browser
- Download file 2.04 kB
-
https://huggingface.co/spaces/hugging-apps/flexcast-forecasting/resolve/main/layers/Embed.py
- Command line
-
hf download hf://spaces/hugging-apps/flexcast-forecasting/layers/Embed.py
-
curl -L -o Embed.py https://huggingface.co/spaces/hugging-apps/flexcast-forecasting/resolve/main/layers/Embed.py
2.04 kB
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from torch.nn.utils import weight_norm | |
| import math | |
| def rotate_half(x: torch.Tensor) -> torch.Tensor: | |
| x1 = x[..., : x.shape[-1] // 2] | |
| x2 = x[..., x.shape[-1] // 2 :] | |
| return torch.cat((-x2, x1), dim=-1) | |
| def apply_rotary_pos_emb_blhD(q, k, cos, sin, position_ids=None): | |
| B, L, H, D = q.shape | |
| if position_ids is None: | |
| cos_ = cos[:L].unsqueeze(0).unsqueeze(2) | |
| sin_ = sin[:L].unsqueeze(0).unsqueeze(2) | |
| else: | |
| cos_ = cos[position_ids].unsqueeze(2) | |
| sin_ = sin[position_ids].unsqueeze(2) | |
| q = (q * cos_) + (rotate_half(q) * sin_) | |
| k = (k * cos_) + (rotate_half(k) * sin_) | |
| return q, k | |
| class RotaryEmbedding(nn.Module): | |
| def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): | |
| super().__init__() | |
| self.dim = dim | |
| self.base = base | |
| inv_freq = 1.0 / (self.base ** (torch.arange(0, dim, 2).float().to(device) / dim)) | |
| self.register_buffer("inv_freq", inv_freq, persistent=False) | |
| self.max_seq_len_cached = 0 | |
| self.register_buffer("cos_cached", torch.empty(0), persistent=False) | |
| self.register_buffer("sin_cached", torch.empty(0), persistent=False) | |
| self._set_cos_sin_cache(max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype()) | |
| def _set_cos_sin_cache(self, seq_len, device, dtype): | |
| self.max_seq_len_cached = seq_len | |
| t = torch.arange(seq_len, device=device, dtype=torch.long).type_as(self.inv_freq) | |
| freqs = torch.outer(t, self.inv_freq) | |
| emb = torch.cat([freqs, freqs], dim=-1) | |
| self.cos_cached = emb.cos().to(dtype) | |
| self.sin_cached = emb.sin().to(dtype) | |
| def forward(self, x, seq_len: int): | |
| if seq_len > self.max_seq_len_cached: | |
| self._set_cos_sin_cache(seq_len, device=x.device, dtype=x.dtype) | |
| return (self.cos_cached[:seq_len].to(dtype=x.dtype), | |
| self.sin_cached[:seq_len].to(dtype=x.dtype)) | |