HamSession commited on
Commit
fc24aa5
·
verified ·
1 Parent(s): 5003d0b

Add modeling_stock_mamba.py

Browse files
Files changed (1) hide show
  1. modeling_stock_mamba.py +25 -17
modeling_stock_mamba.py CHANGED
@@ -1,9 +1,9 @@
1
  """Self-contained model definition shipped inside the Hugging Face repo.
2
 
3
- Depends only on torch and mamba_ssm (Mamba-3). Mirrors model_train/model.py
4
- MambaBlock/MambaEncoder and the retired MambaPretrainModel
5
- (git show 2d04a6d^:model_train/model.py) so the exported state_dict loads
6
- without this project's code.
7
  """
8
 
9
  from __future__ import annotations
@@ -19,10 +19,10 @@ from mamba_ssm import Mamba3
19
  class MambaBlock(nn.Module):
20
  """Pre-norm Mamba-3 block with residual: x + ssm(norm(x))."""
21
 
22
- def __init__(self, d_model: int, d_state: int, expand: int, dropout: float):
23
  super().__init__()
24
  self.norm = nn.LayerNorm(d_model)
25
- self.ssm = Mamba3(d_model=d_model, d_state=d_state, expand=expand)
26
  self.dropout = nn.Dropout(dropout)
27
 
28
  def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -41,13 +41,15 @@ class MambaEncoder(nn.Module):
41
  expand: int,
42
  dropout: float,
43
  patch_size_bars: int,
 
44
  ):
45
  super().__init__()
46
  self.d_model = d_model
47
  self.patch_size_bars = patch_size_bars
48
  self.input_projection = nn.Linear(n_channels * patch_size_bars, d_model)
49
  self.mask_token = nn.Parameter(torch.zeros(d_model))
50
- self.blocks = nn.ModuleList(MambaBlock(d_model, d_state, expand, dropout) for _ in range(n_layers))
 
51
  self.final_norm = nn.LayerNorm(d_model)
52
 
53
  def patchify(self, x: torch.Tensor) -> torch.Tensor:
@@ -69,7 +71,11 @@ class MambaEncoder(nn.Module):
69
 
70
 
71
  class MaskedReconstructionModel(nn.Module):
72
- """Encoder plus the linear decoder used for masked-patch pretraining."""
 
 
 
 
73
 
74
  def __init__(self, encoder: MambaEncoder, n_channels: int):
75
  super().__init__()
@@ -78,7 +84,7 @@ class MaskedReconstructionModel(nn.Module):
78
  self.decoder_head = nn.Linear(encoder.d_model, encoder.patch_size_bars * n_channels)
79
 
80
  @classmethod
81
- def from_config(cls, cfg: dict) -> "MaskedReconstructionModel":
82
  enc = MambaEncoder(
83
  n_channels=len(cfg["input_channels"]),
84
  d_model=cfg["d_model"],
@@ -87,23 +93,24 @@ class MaskedReconstructionModel(nn.Module):
87
  expand=cfg["expand_factor"],
88
  dropout=cfg["dropout"],
89
  patch_size_bars=cfg["patch_size_bars"],
 
90
  )
91
  return cls(enc, len(cfg["input_channels"]))
92
 
93
  @classmethod
94
- def from_pretrained(cls, repo_dir: str | Path, device: str = "cuda") -> "MaskedReconstructionModel":
95
- """Load config.json + model.safetensors from a local directory."""
96
  from safetensors.torch import load_file
97
 
98
  repo_dir = Path(repo_dir)
99
  cfg = json.loads((repo_dir / "config.json").read_text())
100
  model = cls.from_config(cfg)
101
- model.load_state_dict(load_file(repo_dir / "model.safetensors"), strict=True)
102
  return model.to(device).eval()
103
 
104
  @torch.no_grad()
105
  def embed(self, x: torch.Tensor) -> torch.Tensor:
106
- """Mean-pooled 256-d embedding per sequence: (B, L, C) -> (B, d_model)."""
107
  return self.encoder(x).mean(dim=1)
108
 
109
  @torch.no_grad()
@@ -112,17 +119,18 @@ class MaskedReconstructionModel(nn.Module):
112
 
113
  Args:
114
  x: (B, L, C) feature bars, L divisible by patch_size_bars.
115
- patch_mask: (B, L/P) bool, True where a patch is hidden from the encoder.
116
 
117
  Returns:
118
  (B, L, C) tensor equal to x on visible bars and to the decoder output on
119
  masked bars. Only masked positions were trained; visible positions are
120
  passed through untouched.
121
  """
 
 
122
  p = self.encoder.patch_size_bars
123
  tokens = self.encoder.input_projection(self.encoder.patchify(x))
124
- tokens[patch_mask] = self.encoder.mask_token.to(tokens.dtype)
125
- decoded = self.decoder_head(self.encoder.forward_blocks(tokens))
126
- decoded = decoded.reshape(x.shape[0], -1, p, self.n_channels).reshape_as(x)
127
  bar_mask = patch_mask.repeat_interleave(p, dim=1).unsqueeze(-1)
128
  return torch.where(bar_mask, decoded, x)
 
1
  """Self-contained model definition shipped inside the Hugging Face repo.
2
 
3
+ Depends only on torch, mamba_ssm (Mamba-3) and safetensors. Mirrors the training-time
4
+ MambaBlock and MambaEncoder and the retired pretraining wrapper, so the exported
5
+ state_dict loads with no dependency on the training repo. A CUDA GPU is required: the
6
+ Mamba-3 kernels have no CPU path and mamba_ssm fails at import on a CPU-only machine.
7
  """
8
 
9
  from __future__ import annotations
 
19
  class MambaBlock(nn.Module):
20
  """Pre-norm Mamba-3 block with residual: x + ssm(norm(x))."""
21
 
22
+ def __init__(self, d_model: int, d_state: int, expand: int, dropout: float, mamba3_kwargs: dict):
23
  super().__init__()
24
  self.norm = nn.LayerNorm(d_model)
25
+ self.ssm = Mamba3(d_model=d_model, d_state=d_state, expand=expand, **mamba3_kwargs)
26
  self.dropout = nn.Dropout(dropout)
27
 
28
  def forward(self, x: torch.Tensor) -> torch.Tensor:
 
41
  expand: int,
42
  dropout: float,
43
  patch_size_bars: int,
44
+ mamba3_kwargs: dict | None = None,
45
  ):
46
  super().__init__()
47
  self.d_model = d_model
48
  self.patch_size_bars = patch_size_bars
49
  self.input_projection = nn.Linear(n_channels * patch_size_bars, d_model)
50
  self.mask_token = nn.Parameter(torch.zeros(d_model))
51
+ kwargs = mamba3_kwargs or {}
52
+ self.blocks = nn.ModuleList(MambaBlock(d_model, d_state, expand, dropout, kwargs) for _ in range(n_layers))
53
  self.final_norm = nn.LayerNorm(d_model)
54
 
55
  def patchify(self, x: torch.Tensor) -> torch.Tensor:
 
71
 
72
 
73
  class MaskedReconstructionModel(nn.Module):
74
+ """Encoder plus the linear decoder used for masked-patch pretraining.
75
+
76
+ Inputs are equal-length windows only. There is no padding mask, so batch windows of
77
+ the same seq_len or call the model once per security.
78
+ """
79
 
80
  def __init__(self, encoder: MambaEncoder, n_channels: int):
81
  super().__init__()
 
84
  self.decoder_head = nn.Linear(encoder.d_model, encoder.patch_size_bars * n_channels)
85
 
86
  @classmethod
87
+ def from_config(cls, cfg: dict) -> MaskedReconstructionModel:
88
  enc = MambaEncoder(
89
  n_channels=len(cfg["input_channels"]),
90
  d_model=cfg["d_model"],
 
93
  expand=cfg["expand_factor"],
94
  dropout=cfg["dropout"],
95
  patch_size_bars=cfg["patch_size_bars"],
96
+ mamba3_kwargs=cfg.get("mamba3", {}),
97
  )
98
  return cls(enc, len(cfg["input_channels"]))
99
 
100
  @classmethod
101
+ def from_pretrained(cls, repo_dir: str | Path, device: str = "cuda") -> MaskedReconstructionModel:
102
+ """Load config.json + model.safetensors from a local directory (e.g. a snapshot_download)."""
103
  from safetensors.torch import load_file
104
 
105
  repo_dir = Path(repo_dir)
106
  cfg = json.loads((repo_dir / "config.json").read_text())
107
  model = cls.from_config(cfg)
108
+ model.load_state_dict(load_file(repo_dir / "model.safetensors", device=device), strict=True)
109
  return model.to(device).eval()
110
 
111
  @torch.no_grad()
112
  def embed(self, x: torch.Tensor) -> torch.Tensor:
113
+ """Mean-pooled embedding per window: (B, L, C) -> (B, d_model)."""
114
  return self.encoder(x).mean(dim=1)
115
 
116
  @torch.no_grad()
 
119
 
120
  Args:
121
  x: (B, L, C) feature bars, L divisible by patch_size_bars.
122
+ patch_mask: (B, L/P) bool, True where a patch is masked from the encoder.
123
 
124
  Returns:
125
  (B, L, C) tensor equal to x on visible bars and to the decoder output on
126
  masked bars. Only masked positions were trained; visible positions are
127
  passed through untouched.
128
  """
129
+ if patch_mask.dtype != torch.bool:
130
+ raise TypeError(f"patch_mask must be bool, got {patch_mask.dtype}")
131
  p = self.encoder.patch_size_bars
132
  tokens = self.encoder.input_projection(self.encoder.patchify(x))
133
+ tokens = torch.where(patch_mask.unsqueeze(-1), self.encoder.mask_token.to(tokens.dtype), tokens)
134
+ decoded = self.decoder_head(self.encoder.forward_blocks(tokens)).reshape_as(x)
 
135
  bar_mask = patch_mask.repeat_interleave(p, dim=1).unsqueeze(-1)
136
  return torch.where(bar_mask, decoded, x)