Hugging Face's logo

diegoquinteiro
/
SmolLM2-135M-Observable

Text Generation
ONNX
Transformers.js
llama
interpretability
attention
hidden-states
tuned-lens
Model card Files
xet
 Community
diegoquinteiro commited on
Commit
3093ea3
·
verified ·
1 Parent(s): d7ae835

Upload folder using huggingface_hub

Browse files
Files changed (9) hide show
  1. README.md +66 -0
  2. config.json +31 -0
  3. merges.txt +0 -0
  4. model_observable_q4.onnx +3 -0
  5. observable-model.json +109 -0
  6. special_tokens_map.json +42 -0
  7. tokenizer.json +0 -0
  8. tokenizer_config.json +168 -0
  9. vocab.json +0 -0
README.md ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ base_model: HuggingFaceTB/SmolLM2-135M
4
+ library_name: onnx
5
+ pipeline_tag: text-generation
6
+ tags:
7
+ - transformers.js
8
+ - onnx
9
+ - interpretability
10
+ - attention
11
+ - hidden-states
12
+ ---
13
+
14
+ # SmolLM2-135M Observable
15
+
16
+ This is a weight-only Q4 ONNX export of [`HuggingFaceTB/SmolLM2-135M`](https://huggingface.co/HuggingFaceTB/SmolLM2-135M) for an interactive course laboratory. It exposes the intermediate tensors needed to inspect one forward pass in a browser.
17
+
18
+ The model is a base next-token predictor. It is not an instruction-following or chat model.
19
+
20
+ ## Inputs
21
+
22
+ Both inputs use `int64` tensors with shape `[batch, sequence]`:
23
+
24
+ - `input_ids`
25
+ - `attention_mask`
26
+
27
+ The browser laboratory uses a batch size of one and short sequences.
28
+
29
+ ## Outputs
30
+
31
+ The graph returns 62 tensors:
32
+
33
+ - `next_token_logits`, with shape `[batch, 49152]`;
34
+ - `hidden_state_00` through `hidden_state_30`, with shape `[batch, sequence, 576]`;
35
+ - `attention_01` through `attention_30`, with shape `[batch, 9, sequence, sequence]`.
36
+
37
+ `hidden_state_00` contains the input embeddings. The following hidden states contain the residual stream after each of the 30 transformer layers.
38
+
39
+ ## Quantization
40
+
41
+ The published graph uses 4-bit weight-only quantization with asymmetric 32-value blocks. Activations and outputs remain float32. The graph uses the ONNX Runtime `MatMulNBits` operator.
42
+
43
+ The export script compares the ONNX result with the original PyTorch model. The included manifest records the validation prompt, tensor shapes and numerical comparison for the published file.
44
+
45
+ ## Intended use
46
+
47
+ This artifact supports a browser laboratory that visualizes:
48
+
49
+ - next-token probabilities;
50
+ - token IDs and token boundaries;
51
+ - attention weights by layer and head;
52
+ - a two-dimensional projection of hidden states across layers.
53
+
54
+ The two-dimensional coordinates are calculated by the browser. They are not stored in this model.
55
+
56
+ ## Limitations
57
+
58
+ - The Q4 graph does not reproduce the float32 model exactly.
59
+ - The graph does not use a KV cache and recomputes the whole sequence for every prediction.
60
+ - Small base Models produce weak and sometimes incoherent continuations.
61
+ - Attention weights show values calculated inside the model. They do not establish a causal explanation for an output.
62
+ - This export is meant for teaching and inspection, not production inference.
63
+
64
+ ## Source
65
+
66
+ The export script and browser laboratory were created for the Engenharia Assistida por IA course. Hugging Face published the original SmolLM2 weights and architecture under the Apache 2.0 license.
config.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "LlamaForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 0,
8
+ "dtype": "float32",
9
+ "eos_token_id": 0,
10
+ "head_dim": 64,
11
+ "hidden_act": "silu",
12
+ "hidden_size": 576,
13
+ "initializer_range": 0.041666666666666664,
14
+ "intermediate_size": 1536,
15
+ "is_llama_config": true,
16
+ "max_position_embeddings": 8192,
17
+ "mlp_bias": false,
18
+ "model_type": "llama",
19
+ "num_attention_heads": 9,
20
+ "num_hidden_layers": 30,
21
+ "num_key_value_heads": 3,
22
+ "pretraining_tp": 1,
23
+ "rms_norm_eps": 1e-05,
24
+ "rope_interleaved": false,
25
+ "rope_scaling": null,
26
+ "rope_theta": 100000,
27
+ "tie_word_embeddings": true,
28
+ "transformers_version": "4.56.2",
29
+ "use_cache": false,
30
+ "vocab_size": 49152
31
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
model_observable_q4.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6a15b3b209de28457c0b43ba028d92ea4dbd6501961ed336d9497ee94e67d5fb
3
+ size 184907446
observable-model.json ADDED
@@ -0,0 +1,109 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_id": "HuggingFaceTB/SmolLM2-135M",
3
+ "model_file": "model_observable_q4.onnx",
4
+ "model_bytes": 184907446,
5
+ "quantization": {
6
+ "bits": 4,
7
+ "block_size": 32,
8
+ "symmetric": false
9
+ },
10
+ "architecture": {
11
+ "layers": 30,
12
+ "attention_heads": 9,
13
+ "key_value_heads": 3,
14
+ "hidden_size": 576,
15
+ "vocabulary_size": 49152
16
+ },
17
+ "inputs": [
18
+ "input_ids",
19
+ "attention_mask"
20
+ ],
21
+ "outputs": [
22
+ "next_token_logits",
23
+ "hidden_state_00",
24
+ "hidden_state_01",
25
+ "hidden_state_02",
26
+ "hidden_state_03",
27
+ "hidden_state_04",
28
+ "hidden_state_05",
29
+ "hidden_state_06",
30
+ "hidden_state_07",
31
+ "hidden_state_08",
32
+ "hidden_state_09",
33
+ "hidden_state_10",
34
+ "hidden_state_11",
35
+ "hidden_state_12",
36
+ "hidden_state_13",
37
+ "hidden_state_14",
38
+ "hidden_state_15",
39
+ "hidden_state_16",
40
+ "hidden_state_17",
41
+ "hidden_state_18",
42
+ "hidden_state_19",
43
+ "hidden_state_20",
44
+ "hidden_state_21",
45
+ "hidden_state_22",
46
+ "hidden_state_23",
47
+ "hidden_state_24",
48
+ "hidden_state_25",
49
+ "hidden_state_26",
50
+ "hidden_state_27",
51
+ "hidden_state_28",
52
+ "hidden_state_29",
53
+ "hidden_state_30",
54
+ "attention_01",
55
+ "attention_02",
56
+ "attention_03",
57
+ "attention_04",
58
+ "attention_05",
59
+ "attention_06",
60
+ "attention_07",
61
+ "attention_08",
62
+ "attention_09",
63
+ "attention_10",
64
+ "attention_11",
65
+ "attention_12",
66
+ "attention_13",
67
+ "attention_14",
68
+ "attention_15",
69
+ "attention_16",
70
+ "attention_17",
71
+ "attention_18",
72
+ "attention_19",
73
+ "attention_20",
74
+ "attention_21",
75
+ "attention_22",
76
+ "attention_23",
77
+ "attention_24",
78
+ "attention_25",
79
+ "attention_26",
80
+ "attention_27",
81
+ "attention_28",
82
+ "attention_29",
83
+ "attention_30"
84
+ ],
85
+ "validation_prompt": "Context changes the next token",
86
+ "validation": {
87
+ "outputs": 62,
88
+ "top_10_overlap": 8,
89
+ "max_logit_error": 4.03104305267334,
90
+ "last_hidden_mean_cosine": 0.9051130414009094,
91
+ "last_attention_mean_absolute_error": 0.051084186881780624,
92
+ "last_attention_correlation": 0.9588994420083928,
93
+ "logits_shape": [
94
+ 1,
95
+ 49152
96
+ ],
97
+ "hidden_shape": [
98
+ 1,
99
+ 5,
100
+ 576
101
+ ],
102
+ "attention_shape": [
103
+ 1,
104
+ 9,
105
+ 5,
106
+ 5
107
+ ]
108
+ }
109
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|endoftext|>",
4
+ "<|im_start|>",
5
+ "<|im_end|>",
6
+ "<repo_name>",
7
+ "<reponame>",
8
+ "<file_sep>",
9
+ "<filename>",
10
+ "<gh_stars>",
11
+ "<issue_start>",
12
+ "<issue_comment>",
13
+ "<issue_closed>",
14
+ "<jupyter_start>",
15
+ "<jupyter_text>",
16
+ "<jupyter_code>",
17
+ "<jupyter_output>",
18
+ "<jupyter_script>",
19
+ "<empty_output>"
20
+ ],
21
+ "bos_token": {
22
+ "content": "<|endoftext|>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false
27
+ },
28
+ "eos_token": {
29
+ "content": "<|endoftext|>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false
34
+ },
35
+ "unk_token": {
36
+ "content": "<|endoftext|>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false
41
+ }
42
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,168 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "0": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "1": {
13
+ "content": "<|im_start|>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "2": {
21
+ "content": "<|im_end|>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
+ "3": {
29
+ "content": "<repo_name>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
+ },
36
+ "4": {
37
+ "content": "<reponame>",
38
+ "lstrip": false,
39
+ "normalized": false,
40
+ "rstrip": false,
41
+ "single_word": false,
42
+ "special": true
43
+ },
44
+ "5": {
45
+ "content": "<file_sep>",
46
+ "lstrip": false,
47
+ "normalized": false,
48
+ "rstrip": false,
49
+ "single_word": false,
50
+ "special": true
51
+ },
52
+ "6": {
53
+ "content": "<filename>",
54
+ "lstrip": false,
55
+ "normalized": false,
56
+ "rstrip": false,
57
+ "single_word": false,
58
+ "special": true
59
+ },
60
+ "7": {
61
+ "content": "<gh_stars>",
62
+ "lstrip": false,
63
+ "normalized": false,
64
+ "rstrip": false,
65
+ "single_word": false,
66
+ "special": true
67
+ },
68
+ "8": {
69
+ "content": "<issue_start>",
70
+ "lstrip": false,
71
+ "normalized": false,
72
+ "rstrip": false,
73
+ "single_word": false,
74
+ "special": true
75
+ },
76
+ "9": {
77
+ "content": "<issue_comment>",
78
+ "lstrip": false,
79
+ "normalized": false,
80
+ "rstrip": false,
81
+ "single_word": false,
82
+ "special": true
83
+ },
84
+ "10": {
85
+ "content": "<issue_closed>",
86
+ "lstrip": false,
87
+ "normalized": false,
88
+ "rstrip": false,
89
+ "single_word": false,
90
+ "special": true
91
+ },
92
+ "11": {
93
+ "content": "<jupyter_start>",
94
+ "lstrip": false,
95
+ "normalized": false,
96
+ "rstrip": false,
97
+ "single_word": false,
98
+ "special": true
99
+ },
100
+ "12": {
101
+ "content": "<jupyter_text>",
102
+ "lstrip": false,
103
+ "normalized": false,
104
+ "rstrip": false,
105
+ "single_word": false,
106
+ "special": true
107
+ },
108
+ "13": {
109
+ "content": "<jupyter_code>",
110
+ "lstrip": false,
111
+ "normalized": false,
112
+ "rstrip": false,
113
+ "single_word": false,
114
+ "special": true
115
+ },
116
+ "14": {
117
+ "content": "<jupyter_output>",
118
+ "lstrip": false,
119
+ "normalized": false,
120
+ "rstrip": false,
121
+ "single_word": false,
122
+ "special": true
123
+ },
124
+ "15": {
125
+ "content": "<jupyter_script>",
126
+ "lstrip": false,
127
+ "normalized": false,
128
+ "rstrip": false,
129
+ "single_word": false,
130
+ "special": true
131
+ },
132
+ "16": {
133
+ "content": "<empty_output>",
134
+ "lstrip": false,
135
+ "normalized": false,
136
+ "rstrip": false,
137
+ "single_word": false,
138
+ "special": true
139
+ }
140
+ },
141
+ "additional_special_tokens": [
142
+ "<|endoftext|>",
143
+ "<|im_start|>",
144
+ "<|im_end|>",
145
+ "<repo_name>",
146
+ "<reponame>",
147
+ "<file_sep>",
148
+ "<filename>",
149
+ "<gh_stars>",
150
+ "<issue_start>",
151
+ "<issue_comment>",
152
+ "<issue_closed>",
153
+ "<jupyter_start>",
154
+ "<jupyter_text>",
155
+ "<jupyter_code>",
156
+ "<jupyter_output>",
157
+ "<jupyter_script>",
158
+ "<empty_output>"
159
+ ],
160
+ "bos_token": "<|endoftext|>",
161
+ "clean_up_tokenization_spaces": false,
162
+ "eos_token": "<|endoftext|>",
163
+ "extra_special_tokens": {},
164
+ "model_max_length": 8192,
165
+ "tokenizer_class": "GPT2Tokenizer",
166
+ "unk_token": "<|endoftext|>",
167
+ "vocab_size": 49152
168
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff