Add modeling_stock_mamba.py
Browse files- modeling_stock_mamba.py +25 -17
modeling_stock_mamba.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
"""Self-contained model definition shipped inside the Hugging Face repo.
|
| 2 |
|
| 3 |
-
Depends only on torch
|
| 4 |
-
MambaBlock
|
| 5 |
-
|
| 6 |
-
|
| 7 |
"""
|
| 8 |
|
| 9 |
from __future__ import annotations
|
|
@@ -19,10 +19,10 @@ from mamba_ssm import Mamba3
|
|
| 19 |
class MambaBlock(nn.Module):
|
| 20 |
"""Pre-norm Mamba-3 block with residual: x + ssm(norm(x))."""
|
| 21 |
|
| 22 |
-
def __init__(self, d_model: int, d_state: int, expand: int, dropout: float):
|
| 23 |
super().__init__()
|
| 24 |
self.norm = nn.LayerNorm(d_model)
|
| 25 |
-
self.ssm = Mamba3(d_model=d_model, d_state=d_state, expand=expand)
|
| 26 |
self.dropout = nn.Dropout(dropout)
|
| 27 |
|
| 28 |
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
@@ -41,13 +41,15 @@ class MambaEncoder(nn.Module):
|
|
| 41 |
expand: int,
|
| 42 |
dropout: float,
|
| 43 |
patch_size_bars: int,
|
|
|
|
| 44 |
):
|
| 45 |
super().__init__()
|
| 46 |
self.d_model = d_model
|
| 47 |
self.patch_size_bars = patch_size_bars
|
| 48 |
self.input_projection = nn.Linear(n_channels * patch_size_bars, d_model)
|
| 49 |
self.mask_token = nn.Parameter(torch.zeros(d_model))
|
| 50 |
-
|
|
|
|
| 51 |
self.final_norm = nn.LayerNorm(d_model)
|
| 52 |
|
| 53 |
def patchify(self, x: torch.Tensor) -> torch.Tensor:
|
|
@@ -69,7 +71,11 @@ class MambaEncoder(nn.Module):
|
|
| 69 |
|
| 70 |
|
| 71 |
class MaskedReconstructionModel(nn.Module):
|
| 72 |
-
"""Encoder plus the linear decoder used for masked-patch pretraining.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
|
| 74 |
def __init__(self, encoder: MambaEncoder, n_channels: int):
|
| 75 |
super().__init__()
|
|
@@ -78,7 +84,7 @@ class MaskedReconstructionModel(nn.Module):
|
|
| 78 |
self.decoder_head = nn.Linear(encoder.d_model, encoder.patch_size_bars * n_channels)
|
| 79 |
|
| 80 |
@classmethod
|
| 81 |
-
def from_config(cls, cfg: dict) ->
|
| 82 |
enc = MambaEncoder(
|
| 83 |
n_channels=len(cfg["input_channels"]),
|
| 84 |
d_model=cfg["d_model"],
|
|
@@ -87,23 +93,24 @@ class MaskedReconstructionModel(nn.Module):
|
|
| 87 |
expand=cfg["expand_factor"],
|
| 88 |
dropout=cfg["dropout"],
|
| 89 |
patch_size_bars=cfg["patch_size_bars"],
|
|
|
|
| 90 |
)
|
| 91 |
return cls(enc, len(cfg["input_channels"]))
|
| 92 |
|
| 93 |
@classmethod
|
| 94 |
-
def from_pretrained(cls, repo_dir: str | Path, device: str = "cuda") ->
|
| 95 |
-
"""Load config.json + model.safetensors from a local directory."""
|
| 96 |
from safetensors.torch import load_file
|
| 97 |
|
| 98 |
repo_dir = Path(repo_dir)
|
| 99 |
cfg = json.loads((repo_dir / "config.json").read_text())
|
| 100 |
model = cls.from_config(cfg)
|
| 101 |
-
model.load_state_dict(load_file(repo_dir / "model.safetensors"), strict=True)
|
| 102 |
return model.to(device).eval()
|
| 103 |
|
| 104 |
@torch.no_grad()
|
| 105 |
def embed(self, x: torch.Tensor) -> torch.Tensor:
|
| 106 |
-
"""Mean-pooled
|
| 107 |
return self.encoder(x).mean(dim=1)
|
| 108 |
|
| 109 |
@torch.no_grad()
|
|
@@ -112,17 +119,18 @@ class MaskedReconstructionModel(nn.Module):
|
|
| 112 |
|
| 113 |
Args:
|
| 114 |
x: (B, L, C) feature bars, L divisible by patch_size_bars.
|
| 115 |
-
patch_mask: (B, L/P) bool, True where a patch is
|
| 116 |
|
| 117 |
Returns:
|
| 118 |
(B, L, C) tensor equal to x on visible bars and to the decoder output on
|
| 119 |
masked bars. Only masked positions were trained; visible positions are
|
| 120 |
passed through untouched.
|
| 121 |
"""
|
|
|
|
|
|
|
| 122 |
p = self.encoder.patch_size_bars
|
| 123 |
tokens = self.encoder.input_projection(self.encoder.patchify(x))
|
| 124 |
-
tokens
|
| 125 |
-
decoded = self.decoder_head(self.encoder.forward_blocks(tokens))
|
| 126 |
-
decoded = decoded.reshape(x.shape[0], -1, p, self.n_channels).reshape_as(x)
|
| 127 |
bar_mask = patch_mask.repeat_interleave(p, dim=1).unsqueeze(-1)
|
| 128 |
return torch.where(bar_mask, decoded, x)
|
|
|
|
| 1 |
"""Self-contained model definition shipped inside the Hugging Face repo.
|
| 2 |
|
| 3 |
+
Depends only on torch, mamba_ssm (Mamba-3) and safetensors. Mirrors the training-time
|
| 4 |
+
MambaBlock and MambaEncoder and the retired pretraining wrapper, so the exported
|
| 5 |
+
state_dict loads with no dependency on the training repo. A CUDA GPU is required: the
|
| 6 |
+
Mamba-3 kernels have no CPU path and mamba_ssm fails at import on a CPU-only machine.
|
| 7 |
"""
|
| 8 |
|
| 9 |
from __future__ import annotations
|
|
|
|
| 19 |
class MambaBlock(nn.Module):
|
| 20 |
"""Pre-norm Mamba-3 block with residual: x + ssm(norm(x))."""
|
| 21 |
|
| 22 |
+
def __init__(self, d_model: int, d_state: int, expand: int, dropout: float, mamba3_kwargs: dict):
|
| 23 |
super().__init__()
|
| 24 |
self.norm = nn.LayerNorm(d_model)
|
| 25 |
+
self.ssm = Mamba3(d_model=d_model, d_state=d_state, expand=expand, **mamba3_kwargs)
|
| 26 |
self.dropout = nn.Dropout(dropout)
|
| 27 |
|
| 28 |
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
|
|
| 41 |
expand: int,
|
| 42 |
dropout: float,
|
| 43 |
patch_size_bars: int,
|
| 44 |
+
mamba3_kwargs: dict | None = None,
|
| 45 |
):
|
| 46 |
super().__init__()
|
| 47 |
self.d_model = d_model
|
| 48 |
self.patch_size_bars = patch_size_bars
|
| 49 |
self.input_projection = nn.Linear(n_channels * patch_size_bars, d_model)
|
| 50 |
self.mask_token = nn.Parameter(torch.zeros(d_model))
|
| 51 |
+
kwargs = mamba3_kwargs or {}
|
| 52 |
+
self.blocks = nn.ModuleList(MambaBlock(d_model, d_state, expand, dropout, kwargs) for _ in range(n_layers))
|
| 53 |
self.final_norm = nn.LayerNorm(d_model)
|
| 54 |
|
| 55 |
def patchify(self, x: torch.Tensor) -> torch.Tensor:
|
|
|
|
| 71 |
|
| 72 |
|
| 73 |
class MaskedReconstructionModel(nn.Module):
|
| 74 |
+
"""Encoder plus the linear decoder used for masked-patch pretraining.
|
| 75 |
+
|
| 76 |
+
Inputs are equal-length windows only. There is no padding mask, so batch windows of
|
| 77 |
+
the same seq_len or call the model once per security.
|
| 78 |
+
"""
|
| 79 |
|
| 80 |
def __init__(self, encoder: MambaEncoder, n_channels: int):
|
| 81 |
super().__init__()
|
|
|
|
| 84 |
self.decoder_head = nn.Linear(encoder.d_model, encoder.patch_size_bars * n_channels)
|
| 85 |
|
| 86 |
@classmethod
|
| 87 |
+
def from_config(cls, cfg: dict) -> MaskedReconstructionModel:
|
| 88 |
enc = MambaEncoder(
|
| 89 |
n_channels=len(cfg["input_channels"]),
|
| 90 |
d_model=cfg["d_model"],
|
|
|
|
| 93 |
expand=cfg["expand_factor"],
|
| 94 |
dropout=cfg["dropout"],
|
| 95 |
patch_size_bars=cfg["patch_size_bars"],
|
| 96 |
+
mamba3_kwargs=cfg.get("mamba3", {}),
|
| 97 |
)
|
| 98 |
return cls(enc, len(cfg["input_channels"]))
|
| 99 |
|
| 100 |
@classmethod
|
| 101 |
+
def from_pretrained(cls, repo_dir: str | Path, device: str = "cuda") -> MaskedReconstructionModel:
|
| 102 |
+
"""Load config.json + model.safetensors from a local directory (e.g. a snapshot_download)."""
|
| 103 |
from safetensors.torch import load_file
|
| 104 |
|
| 105 |
repo_dir = Path(repo_dir)
|
| 106 |
cfg = json.loads((repo_dir / "config.json").read_text())
|
| 107 |
model = cls.from_config(cfg)
|
| 108 |
+
model.load_state_dict(load_file(repo_dir / "model.safetensors", device=device), strict=True)
|
| 109 |
return model.to(device).eval()
|
| 110 |
|
| 111 |
@torch.no_grad()
|
| 112 |
def embed(self, x: torch.Tensor) -> torch.Tensor:
|
| 113 |
+
"""Mean-pooled embedding per window: (B, L, C) -> (B, d_model)."""
|
| 114 |
return self.encoder(x).mean(dim=1)
|
| 115 |
|
| 116 |
@torch.no_grad()
|
|
|
|
| 119 |
|
| 120 |
Args:
|
| 121 |
x: (B, L, C) feature bars, L divisible by patch_size_bars.
|
| 122 |
+
patch_mask: (B, L/P) bool, True where a patch is masked from the encoder.
|
| 123 |
|
| 124 |
Returns:
|
| 125 |
(B, L, C) tensor equal to x on visible bars and to the decoder output on
|
| 126 |
masked bars. Only masked positions were trained; visible positions are
|
| 127 |
passed through untouched.
|
| 128 |
"""
|
| 129 |
+
if patch_mask.dtype != torch.bool:
|
| 130 |
+
raise TypeError(f"patch_mask must be bool, got {patch_mask.dtype}")
|
| 131 |
p = self.encoder.patch_size_bars
|
| 132 |
tokens = self.encoder.input_projection(self.encoder.patchify(x))
|
| 133 |
+
tokens = torch.where(patch_mask.unsqueeze(-1), self.encoder.mask_token.to(tokens.dtype), tokens)
|
| 134 |
+
decoded = self.decoder_head(self.encoder.forward_blocks(tokens)).reshape_as(x)
|
|
|
|
| 135 |
bar_mask = patch_mask.repeat_interleave(p, dim=1).unsqueeze(-1)
|
| 136 |
return torch.where(bar_mask, decoded, x)
|