File size: 6,395 Bytes
1936656 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 | {
"module": "keras_hub.src.models.smollm3.smollm3_causal_lm",
"class_name": "SmolLM3CausalLM",
"config": {
"backbone": {
"module": "keras_hub.src.models.smollm3.smollm3_backbone",
"class_name": "SmolLM3Backbone",
"config": {
"name": "smol_lm3_backbone",
"trainable": true,
"dtype": {
"module": "keras",
"class_name": "DTypePolicy",
"config": {
"name": "float32"
},
"registered_name": null
},
"vocabulary_size": 128256,
"hidden_dim": 2048,
"intermediate_dim": 11008,
"num_layers": 36,
"num_attention_heads": 16,
"num_key_value_heads": 4,
"attention_bias": false,
"attention_dropout": 0.0,
"rope_layer_enabled_list": [
true,
true,
true,
false,
true,
true,
true,
false,
true,
true,
true,
false,
true,
true,
true,
false,
true,
true,
true,
false,
true,
true,
true,
false,
true,
true,
true,
false,
true,
true,
true,
false,
true,
true,
true,
false
],
"layer_types": [
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention"
],
"mlp_bias": false,
"layer_norm_epsilon": 1e-06,
"max_position_embeddings": 65536,
"rope_theta": 5000000.0,
"partial_rotary_factor": 1.0
},
"registered_name": "keras_hub>SmolLM3Backbone"
},
"preprocessor": {
"module": "keras_hub.src.models.smollm3.smollm3_causal_lm_preprocessor",
"class_name": "SmolLM3CausalLMPreprocessor",
"config": {
"name": "smol_lm3_causal_lm_preprocessor_1",
"trainable": true,
"dtype": {
"module": "keras",
"class_name": "DTypePolicy",
"config": {
"name": "float32"
},
"registered_name": null
},
"tokenizer": {
"module": "keras_hub.src.models.smollm3.smollm3_tokenizer",
"class_name": "SmolLM3Tokenizer",
"config": {
"name": "smol_lm3_tokenizer",
"trainable": true,
"dtype": {
"module": "keras",
"class_name": "DTypePolicy",
"config": {
"name": "int32"
},
"registered_name": null
},
"config_file": "tokenizer.json",
"sequence_length": null,
"add_prefix_space": false,
"unsplittable_tokens": [
"<think>",
"<tool_call>",
"<|end_header_id|>",
"<|eot_id|>",
"<|im_end|>",
"<|end_of_text|>",
"</think>",
"<tool_response>",
"<|im_start|>",
"</tool_response>",
"</tool_call>",
"<|start_header_id|>",
"<|begin_of_text|>",
"<|eom_id|>",
"<code>",
"</code>",
"<|python_tag|>",
"<|finetune_right_pad_id|>"
]
},
"registered_name": "keras_hub>SmolLM3Tokenizer"
},
"config_file": "preprocessor.json",
"sequence_length": 1024,
"add_start_token": true,
"add_end_token": true
},
"registered_name": "keras_hub>SmolLM3CausalLMPreprocessor"
},
"name": "smol_lm3_causal_lm"
},
"registered_name": "keras_hub>SmolLM3CausalLM"
} |