Hugging Face's logo

Alienanthony
/
Nested_Mamba_3Level

Text Generation
Safetensors
English
custom
nested_byte_mamba2
mamba2
byte-level
multimodal
custom-code
base_model
Model card Files
xet
 Community
Nested_Mamba_3Level
File size: 1,297 Bytes
fef393c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
{
  "model_type": "nested_byte_mamba2",
  "architectures": ["ForwardBackwardRepairModel"],
  "architecture": "byte_latent_mamba_nested_jsonl",
  "vocab_size": 260,
  "byte_offset": 4,
  "special_token_ids": {
    "pad": 0,
    "bos": 1,
    "eos": 2,
    "unk": 3
  },
  "dim": 2000,
  "layers": 26,
  "fine_layers": 6,
  "nested_layers": 20,
  "tertiary_layers": 30,
  "decoder_dim": 8000,
  "position_bins": 8192,
  "mamba_version": 2,
  "mamba_d_state": 64,
  "mamba2_headdim": 100,
  "no_mamba": false,
  "blt_min_patch_bytes": 1,
  "blt_max_patch_bytes": 96,
  "blt_patch_change_threshold": 48,
  "blt_close_threshold": 0.98,
  "blt_mid_close_bonus": 0.05,
  "blt_short_pool_budget": 3000,
  "blt_short_pool_window": 6000,
  "blt_secondary_min_patch_bytes": 6,
  "nested_pool_factor": 16,
  "nested_min_pool_factor": 4,
  "nested_close_threshold": 0.98,
  "tertiary_pool_factor": 16,
  "tertiary_min_pool_factor": 2,
  "tertiary_close_threshold": 0.98,
  "detach_inactive_coarse_gradients": true,
  "decoder_pool_controller": true,
  "pool_controller_alpha": 1.0,
  "pool_controller_beta": 0.5,
  "pool_controller_gamma": 1.0,
  "recommended_precision": "bf16",
  "recommended_placement": {
    "fine_device": "cuda:0",
    "nested_devices": ["cuda:1"],
    "tertiary_device": "cuda:2"
  }
}