Hugging Face's logo

miguelcsx
/
factorized-null

Safetensors
tolm
custom_code
Model card Files
xet
 Community
factorized-null
File size: 1,440 Bytes
7c8a736
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
{
  "model_type": "tolm",
  "architectures": [
    "TOLMForMaskedLM"
  ],
  "auto_map": {
    "AutoConfig": "tolm.TOLMConfig",
    "AutoModel": "tolm.TOLMModel",
    "AutoModelForMaskedLM": "tolm.TOLMForMaskedLM",
    "AutoModelForCausalLM": "tolm.TOLMForCausalLM"
  },
  "vocab_size": 16384,
  "max_seq_len": 512,
  "hidden_size": 384,
  "num_hidden_layers": 12,
  "num_attention_heads": 6,
  "intermediate_size": 1280,
  "position_buckets": 32,
  "dropout": 0.1,
  "attention_dropout": 0.1,
  "initializer_range": 0.03227486121839514,
  "value_gating": true,
  "residual_mixing": true,
  "pad_token_id": 3,
  "bos_token_id": 1,
  "eos_token_id": 2,
  "mask_token_id": 4,
  "absolute_positions": false,
  "use_rope": false,
  "use_alibi": false,
  "recurrent_steps": 1,
  "num_experts": 1,
  "experts_per_token": 1,
  "expert_intermediate_size": null,
  "future_offsets": [],
  "state_mixer_kernel": 0,
  "rtd_auxiliary": true,
  "geometry_lexical_dim": 0,
  "geometry_curvature": 1.0,
  "cognitive_readout_layer": 0,
  "cognitive_readout_weight": 0.0,
  "direct_sum_dims": [],
  "direct_sum_heads": [],
  "direct_sum_intermediate_sizes": [],
  "lexical_residual_buckets": 0,
  "lexical_residual_dim": 0,
  "lexical_residual_scale": 1.0,
  "structured_projection_dim": 0,
  "factor_readout_mode": "orthogonal",
  "factor_readout_dims": [
    128,
    128,
    128
  ],
  "factor_readout_reflectors": 384,
  "factor_readout_seed": 314159
}