Hugging Face's logo

cactusfriend
/
dunstonfacts

Text Generation
Transformers
Safetensors
openelm
custom_code
Model card Files
xet
 Community
cactusfriend commited on
Commit
65c2d0c
·
verified ·
1 Parent(s): 96b409b

stupid transformers

Browse files
Files changed (1) hide show
  1. config.json +102 -101
config.json CHANGED
@@ -1,101 +1,102 @@
1
- {
2
- "_name_or_path": "/Users/ryan/python/2024gen/nightmarep3/final/",
3
- "activation_fn_name": "swish",
4
- "architectures": [
5
- "OpenELMForCausalLM"
6
- ],
7
- "auto_map": {
8
- "AutoConfig": "configuration_openelm.OpenELMConfig",
9
- "AutoModelForCausalLM": "modeling_openelm.OpenELMForCausalLM"
10
- },
11
- "bos_token_id": 1,
12
- "eos_token_id": 2,
13
- "ffn_dim_divisor": 256,
14
- "ffn_multipliers": [
15
- 0.5,
16
- 0.68,
17
- 0.87,
18
- 1.05,
19
- 1.24,
20
- 1.42,
21
- 1.61,
22
- 1.79,
23
- 1.97,
24
- 2.16,
25
- 2.34,
26
- 2.53,
27
- 2.71,
28
- 2.89,
29
- 3.08,
30
- 3.26,
31
- 3.45,
32
- 3.63,
33
- 3.82,
34
- 4.0
35
- ],
36
- "ffn_with_glu": true,
37
- "head_dim": 64,
38
- "initializer_range": 0.02,
39
- "max_context_length": 2048,
40
- "model_dim": 1536,
41
- "model_type": "openelm",
42
- "normalization_layer_name": "rms_norm",
43
- "normalize_qk_projections": true,
44
- "num_gqa_groups": 4,
45
- "num_kv_heads": [
46
- 3,
47
- 3,
48
- 3,
49
- 4,
50
- 4,
51
- 4,
52
- 4,
53
- 4,
54
- 4,
55
- 4,
56
- 5,
57
- 5,
58
- 5,
59
- 5,
60
- 5,
61
- 5,
62
- 6,
63
- 6,
64
- 6,
65
- 6
66
- ],
67
- "num_query_heads": [
68
- 12,
69
- 12,
70
- 12,
71
- 16,
72
- 16,
73
- 16,
74
- 16,
75
- 16,
76
- 16,
77
- 16,
78
- 20,
79
- 20,
80
- 20,
81
- 20,
82
- 20,
83
- 20,
84
- 24,
85
- 24,
86
- 24,
87
- 24
88
- ],
89
- "num_transformer_layers": 20,
90
- "qkv_multipliers": [
91
- 0.5,
92
- 1.0
93
- ],
94
- "rope_freq_constant": 10000,
95
- "rope_max_length": 4096,
96
- "share_input_output_layers": true,
97
- "torch_dtype": "float32",
98
- "transformers_version": "4.41.2",
99
- "use_cache": true,
100
- "vocab_size": 32000
101
- }
 
 
1
+ {
2
+ "_name_or_path": "/Users/ryan/python/2024gen/nightmarep3/final/",
3
+ "activation_fn_name": "swish",
4
+ "architectures": [
5
+ "OpenELMForCausalLM"
6
+ ],
7
+ "auto_map": {
8
+ "AutoConfig": "configuration_openelm.OpenELMConfig",
9
+ "AutoModelForCausalLM": "modeling_openelm.OpenELMForCausalLM"
10
+ },
11
+ "bos_token_id": 1,
12
+ "eos_token_id": 2,
13
+ "ffn_dim_divisor": 256,
14
+ "ffn_multipliers": [
15
+ 0.5,
16
+ 0.68,
17
+ 0.87,
18
+ 1.05,
19
+ 1.24,
20
+ 1.42,
21
+ 1.61,
22
+ 1.79,
23
+ 1.97,
24
+ 2.16,
25
+ 2.34,
26
+ 2.53,
27
+ 2.71,
28
+ 2.89,
29
+ 3.08,
30
+ 3.26,
31
+ 3.45,
32
+ 3.63,
33
+ 3.82,
34
+ 4.0
35
+ ],
36
+ "ffn_with_glu": true,
37
+ "head_dim": 64,
38
+ "initializer_range": 0.02,
39
+ "max_context_length": 2048,
40
+ "model_dim": 1536,
41
+ "model_type": "openelm",
42
+ "normalization_layer_name": "rms_norm",
43
+ "normalize_qk_projections": true,
44
+ "num_gqa_groups": 4,
45
+ "num_kv_heads": [
46
+ 3,
47
+ 3,
48
+ 3,
49
+ 4,
50
+ 4,
51
+ 4,
52
+ 4,
53
+ 4,
54
+ 4,
55
+ 4,
56
+ 5,
57
+ 5,
58
+ 5,
59
+ 5,
60
+ 5,
61
+ 5,
62
+ 6,
63
+ 6,
64
+ 6,
65
+ 6
66
+ ],
67
+ "num_query_heads": [
68
+ 12,
69
+ 12,
70
+ 12,
71
+ 16,
72
+ 16,
73
+ 16,
74
+ 16,
75
+ 16,
76
+ 16,
77
+ 16,
78
+ 20,
79
+ 20,
80
+ 20,
81
+ 20,
82
+ 20,
83
+ 20,
84
+ 24,
85
+ 24,
86
+ 24,
87
+ 24
88
+ ],
89
+ "num_hidden_layers": 0,
90
+ "num_transformer_layers": 20,
91
+ "qkv_multipliers": [
92
+ 0.5,
93
+ 1.0
94
+ ],
95
+ "rope_freq_constant": 10000,
96
+ "rope_max_length": 4096,
97
+ "share_input_output_layers": true,
98
+ "torch_dtype": "float32",
99
+ "transformers_version": "4.41.2",
100
+ "use_cache": true,
101
+ "vocab_size": 32000
102
+ }