Review cloud MIC baselines arm64-CodeGen2

#1
by tarekziade HF Staff - opened
integrations/transformers/expectations/cpu-arm64/codegen2-16B_P-mic.bfloat16.json ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/codegen2-16B_P-mic",
4
+ "contract_name": "CodeGen2",
5
+ "contract_revision": "1fea908e4ceb77a2c6a8428015d8b127825e8618",
6
+ "framework": "transformers",
7
+ "variant": "bfloat16",
8
+ "target": "codegen2-16B_P-mic",
9
+ "kind": "fixture",
10
+ "revision": "1fea908e4ceb77a2c6a8428015d8b127825e8618",
11
+ "source_model": "Salesforce/codegen2-16B_P",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/codegen2-16B_P-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.23,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "bfloat16",
48
+ "attn_implementation": "eager"
49
+ },
50
+ "generated_tokens": [
51
+ [
52
+ 41476,
53
+ 35397,
54
+ 49346,
55
+ 32447
56
+ ],
57
+ [
58
+ 41476,
59
+ 1649,
60
+ 47861,
61
+ 20804
62
+ ],
63
+ [
64
+ 41476,
65
+ 35397,
66
+ 49346,
67
+ 49648
68
+ ],
69
+ [
70
+ 31328,
71
+ 4523,
72
+ 38810,
73
+ 35072
74
+ ]
75
+ ],
76
+ "generated_text": [
77
+ "aldo Toys creatine Tolkien",
78
+ "aldo When relapse\u9b54",
79
+ "aldo Toys creatine DMCA",
80
+ " FIRST nuclear blacklist\u0443"
81
+ ],
82
+ "next_token_logits": [
83
+ [
84
+ -0.2392578125,
85
+ -0.2392578125,
86
+ -0.2392578125,
87
+ -0.2392578125
88
+ ],
89
+ [
90
+ -0.21484375,
91
+ -0.21484375,
92
+ -0.21484375,
93
+ -0.21484375
94
+ ],
95
+ [
96
+ -0.20703125,
97
+ -0.20703125,
98
+ -0.20703125,
99
+ -0.20703125
100
+ ],
101
+ [
102
+ -0.21875,
103
+ -0.21875,
104
+ -0.21875,
105
+ -0.21875
106
+ ]
107
+ ],
108
+ "training": {
109
+ "loss": 10.920736312866211,
110
+ "gradients": {
111
+ "transformer.h.0.attn.qkv_proj.weight": true,
112
+ "transformer.h.0.mlp.fc_out.weight": true
113
+ },
114
+ "updated": true
115
+ }
116
+ }
117
+ }
integrations/transformers/expectations/cpu-arm64/codegen2-16B_P-mic.eager.json ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/codegen2-16B_P-mic",
4
+ "contract_name": "CodeGen2",
5
+ "contract_revision": "1fea908e4ceb77a2c6a8428015d8b127825e8618",
6
+ "framework": "transformers",
7
+ "variant": "eager",
8
+ "target": "codegen2-16B_P-mic",
9
+ "kind": "fixture",
10
+ "revision": "1fea908e4ceb77a2c6a8428015d8b127825e8618",
11
+ "source_model": "Salesforce/codegen2-16B_P",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/codegen2-16B_P-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.09,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "float32",
48
+ "attn_implementation": "eager"
49
+ },
50
+ "generated_tokens": [
51
+ [
52
+ 41476,
53
+ 35397,
54
+ 49346,
55
+ 32447
56
+ ],
57
+ [
58
+ 41476,
59
+ 1649,
60
+ 47861,
61
+ 20804
62
+ ],
63
+ [
64
+ 41476,
65
+ 35397,
66
+ 49346,
67
+ 32447
68
+ ],
69
+ [
70
+ 31328,
71
+ 4523,
72
+ 38810,
73
+ 35072
74
+ ]
75
+ ],
76
+ "generated_text": [
77
+ "aldo Toys creatine Tolkien",
78
+ "aldo When relapse\u9b54",
79
+ "aldo Toys creatine Tolkien",
80
+ " FIRST nuclear blacklist\u0443"
81
+ ],
82
+ "next_token_logits": [
83
+ [
84
+ -0.2400306761264801,
85
+ -0.2400306761264801,
86
+ -0.2400306761264801,
87
+ -0.2400306761264801
88
+ ],
89
+ [
90
+ -0.2150077372789383,
91
+ -0.2150077372789383,
92
+ -0.2150077372789383,
93
+ -0.2150077372789383
94
+ ],
95
+ [
96
+ -0.20760056376457214,
97
+ -0.20760056376457214,
98
+ -0.20760056376457214,
99
+ -0.20760056376457214
100
+ ],
101
+ [
102
+ -0.21880458295345306,
103
+ -0.21880458295345306,
104
+ -0.21880458295345306,
105
+ -0.21880458295345306
106
+ ]
107
+ ],
108
+ "training": {
109
+ "loss": 10.92068862915039,
110
+ "gradients": {
111
+ "transformer.h.0.attn.qkv_proj.weight": true,
112
+ "transformer.h.0.mlp.fc_out.weight": true
113
+ },
114
+ "updated": true
115
+ }
116
+ }
117
+ }
integrations/transformers/expectations/cpu-arm64/codegen2-16B_P-mic.json ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/codegen2-16B_P-mic",
4
+ "contract_name": "CodeGen2",
5
+ "contract_revision": "1fea908e4ceb77a2c6a8428015d8b127825e8618",
6
+ "framework": "transformers",
7
+ "variant": "default",
8
+ "target": "codegen2-16B_P-mic",
9
+ "kind": "fixture",
10
+ "revision": "1fea908e4ceb77a2c6a8428015d8b127825e8618",
11
+ "source_model": "Salesforce/codegen2-16B_P",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/codegen2-16B_P-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.04,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "float32",
48
+ "attn_implementation": "eager"
49
+ },
50
+ "generated_tokens": [
51
+ [
52
+ 41476,
53
+ 35397,
54
+ 49346,
55
+ 32447
56
+ ],
57
+ [
58
+ 41476,
59
+ 1649,
60
+ 47861,
61
+ 20804
62
+ ],
63
+ [
64
+ 41476,
65
+ 35397,
66
+ 49346,
67
+ 32447
68
+ ],
69
+ [
70
+ 31328,
71
+ 4523,
72
+ 38810,
73
+ 35072
74
+ ]
75
+ ],
76
+ "generated_text": [
77
+ "aldo Toys creatine Tolkien",
78
+ "aldo When relapse\u9b54",
79
+ "aldo Toys creatine Tolkien",
80
+ " FIRST nuclear blacklist\u0443"
81
+ ],
82
+ "next_token_logits": [
83
+ [
84
+ -0.2400306761264801,
85
+ -0.2400306761264801,
86
+ -0.2400306761264801,
87
+ -0.2400306761264801
88
+ ],
89
+ [
90
+ -0.2150077372789383,
91
+ -0.2150077372789383,
92
+ -0.2150077372789383,
93
+ -0.2150077372789383
94
+ ],
95
+ [
96
+ -0.20760056376457214,
97
+ -0.20760056376457214,
98
+ -0.20760056376457214,
99
+ -0.20760056376457214
100
+ ],
101
+ [
102
+ -0.21880458295345306,
103
+ -0.21880458295345306,
104
+ -0.21880458295345306,
105
+ -0.21880458295345306
106
+ ]
107
+ ],
108
+ "training": {
109
+ "loss": 10.92068862915039,
110
+ "gradients": {
111
+ "transformer.h.0.attn.qkv_proj.weight": true,
112
+ "transformer.h.0.mlp.fc_out.weight": true
113
+ },
114
+ "updated": true
115
+ }
116
+ }
117
+ }