Publish MIC usage, slim fixture and reviewed Transformers expectations

.mic-repo.json

@@ -0,0 +1,7 @@
+{
+  "format": "mic-umbrella-v1",
+  "contract_files": [
+    "integration.json",
+    "integrations/transformers/usage.md"
+  ]
+}

README.md

@@ -0,0 +1,37 @@
+---
+tags:
+  - mic
+  - test-artifact
+base_model: bigcode/starcoder2-3b
+---
+
+# starcoder2-3b-mic
+
+Slim model weights, executable usage and framework expectations in one repository.
+
+- [Contract manifest](integration.json)
+- [Transformers usage](integrations/transformers/usage.md)
+- [Transformers expectations](integrations/transformers/expectations)
+- [Fixture provenance](build.json)
+
+## Contract
+
+### StarCoder2 causal LM
+
+Runnable usage and the framework expectations directory are linked above.
+
+## Slim model
+
+### Slim fixture: starcoder2-3b-mic
+
+A 3.2M-parameter fixture for the [StarCoder2 MIC](https://huggingface.co/huggingface-mic/starcoder2-3b-mic), covering [`bigcode/starcoder2-3b`](https://huggingface.co/bigcode/starcoder2-3b). It is a test artifact, not a usable model, tagged `test-artifact` to keep it out of candidate-model search.
+
+- **Fixture key `0b99c4e71ec5`.** It covers every checkpoint whose processor files are byte-identical and whose config matches after shrinking.
+- **Same architecture, smaller config:** `Starcoder2ForCausalLM`, shrunk by the generic rule in [`sweep.py`](https://github.com/huggingface/transformers-mic/blob/main/experiments/sweep.py) (num_hidden_layers 2).
+- **Processor files copied byte for byte** from `bigcode/starcoder2-3b`: `merges.txt`, `special_tokens_map.json`, `tokenizer.json`, `tokenizer_config.json`, `vocab.json`.
+- **Random weights (seed 17).** They passed the input-sensitivity gate: every perturbation of the contract's inputs moved the logits by at least 0.001 (weakest: `attention_mask:all_ones`, 0.824), so no training was needed.
+- **Same tensor names as the real checkpoint.** The real checkpoint has every parameter, and so does the fixture.
+- **Saved in float32**, the dtype `bigcode/starcoder2-3b` loads in by default.
+- **Finite in float32, bfloat16, and float16** (forward, loss, gradients).
+
+Built by [`build_fixtures.py`](https://github.com/huggingface/transformers-mic/blob/main/example/build_fixtures.py); [`build.json`](build.json) records the pinned sources, gate results, and file hashes. This repository includes the slim weights, usage and framework expectations; rebuild the other files with `python example/build_fixtures.py --fixture starcoder2-3b-mic`.

build.json

@@ -0,0 +1,140 @@
+{
+ "fixture": "huggingface-mic/starcoder2-3b-mic",
+ "key": "0b99c4e71ec5e3b22333a63d3cd2bb97d2071ea6fba36b6a6a1ec6b35c1666ff",
+ "key_fields": {
+  "architectures": [
+   "Starcoder2ForCausalLM"
+  ],
+  "output_hidden_states": false,
+  "return_dict": true,
+  "chunk_size_feed_forward": 0,
+  "is_encoder_decoder": false,
+  "id2label": {
+   "0": "LABEL_0",
+   "1": "LABEL_1"
+  },
+  "label2id": {
+   "LABEL_0": 0,
+   "LABEL_1": 1
+  },
+  "problem_type": null,
+  "vocab_size": 49152,
+  "hidden_size": 64,
+  "intermediate_size": 128,
+  "num_hidden_layers": 2,
+  "num_attention_heads": 4,
+  "num_key_value_heads": 2,
+  "hidden_act": "gelu_pytorch_tanh",
+  "max_position_embeddings": 16384,
+  "initializer_range": 0.018042,
+  "norm_epsilon": 1e-05,
+  "use_cache": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_id": null,
+  "rope_parameters": {
+   "rope_theta": 999999.4420358813,
+   "rope_type": "default"
+  },
+  "sliding_window": 4096,
+  "attention_dropout": 0.1,
+  "residual_dropout": 0.1,
+  "embedding_dropout": 0.1,
+  "use_bias": true,
+  "tie_word_embeddings": true,
+  "mlp_type": "default",
+  "model_type": "starcoder2",
+  "norm_type": "layer_norm",
+  "output_attentions": false
+ },
+ "covers": {
+  "bigcode/starcoder2-3b": "733247c55e3f73af49ce8e9c7949bf14af205928"
+ },
+ "built_from": {
+  "repo": "bigcode/starcoder2-3b",
+  "revision": "733247c55e3f73af49ce8e9c7949bf14af205928"
+ },
+ "seed": 17,
+ "dtype": "float32",
+ "builder": "example/build_fixtures.py",
+ "builder_revision": "ed107e55ab1c53dc35febfa2a8ed87a0a5a42a3c",
+ "recipe": {
+  "seed": 17,
+  "model_class": "Starcoder2ForCausalLM"
+ },
+ "shrink": "rule",
+ "model_class": "Starcoder2ForCausalLM",
+ "parameters": 3204480,
+ "missing_parameters": [],
+ "checkpoint_inspection": "parameter headers/indexes; bounded binary fallback",
+ "numeric_health_scope": "CPU float32/bfloat16/float16; autocast for low-precision losses",
+ "captured": {
+  "call": "training step",
+  "inputs": [
+   "attention_mask",
+   "input_ids",
+   "labels"
+  ],
+  "class": "Starcoder2ForCausalLM"
+ },
+ "weights": "random",
+ "gate": {
+  "random": {
+   "pass": true,
+   "accepted_insensitive": {},
+   "threshold": 0.001,
+   "deltas": {
+    "input_ids:roll_batch": 1.2845580577850342,
+    "input_ids:reverse": 1.1660540103912354,
+    "attention_mask:all_ones": 0.823656439781189
+   },
+   "weakest": "attention_mask:all_ones",
+   "no_perturbation_defined": [],
+   "not_a_forward_parameter": []
+  }
+ },
+ "numeric_health": {
+  "float32": {
+   "outputs_finite": true,
+   "loss_finite": true,
+   "gradients_finite": true,
+   "finite": true
+  },
+  "bfloat16": {
+   "outputs_finite": true,
+   "loss_finite": true,
+   "gradients_finite": true,
+   "finite": true
+  },
+  "float16": {
+   "outputs_finite": true,
+   "loss_finite": true,
+   "unscaled_gradients_finite": true,
+   "finite": true
+  }
+ },
+ "copied_unchanged": [
+  "merges.txt",
+  "special_tokens_map.json",
+  "tokenizer.json",
+  "tokenizer_config.json",
+  "vocab.json"
+ ],
+ "environment": {
+  "python": "3.12.12",
+  "machine": "arm64",
+  "torch": "2.14.1",
+  "transformers": "5.22.0.dev0",
+  "tokenizers": "0.23.3"
+ },
+ "sha256": {
+  "config.json": "ad2f3bfbb3d898f702945bd9f78d314e20abf803618a852beddc6e39164a486c",
+  "generation_config.json": "087c1528f0982539649c359e14be9f4fc9900f1a2ea39695d7baf632815056cb",
+  "merges.txt": "ac9dfb2c5bd3b10bd46aa7fdaa8c54e8e1bcfe1618d9288320ebc82c518442fc",
+  "model.safetensors": "9e02ed2bdffd25a57e35c3cebb45d5f51ddbecaf977919bfa46260d930e82247",
+  "special_tokens_map.json": "1dd716f4648a359f7fca1221fc6a14edb1dd369239a4d7e0e5674c5f11f061d6",
+  "tokenizer.json": "6190a6649d90a7fe35bfed8bff990c3e7aafc3cfa5beea185045f03f0890402b",
+  "tokenizer_config.json": "8149d8e6b21275ad2cc346885ad92c0e9b5aa3e28a78bb5d39b7febc7e52545d",
+  "vocab.json": "542ee3f8b27b3a825754728b4dc59eff31065fecce843a93f09b98f13dbd7de9"
+ }
+}

config.json

@@ -0,0 +1,34 @@
+{
+  "architectures": [
+    "Starcoder2ForCausalLM"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 0,
+  "dtype": "float32",
+  "embedding_dropout": 0.1,
+  "eos_token_id": 0,
+  "hidden_act": "gelu_pytorch_tanh",
+  "hidden_size": 64,
+  "initializer_range": 0.018042,
+  "intermediate_size": 128,
+  "max_position_embeddings": 16384,
+  "mlp_type": "default",
+  "model_type": "starcoder2",
+  "norm_epsilon": 1e-05,
+  "norm_type": "layer_norm",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 2,
+  "pad_token_id": null,
+  "residual_dropout": 0.1,
+  "rope_parameters": {
+    "rope_theta": 999999.4420358813,
+    "rope_type": "default"
+  },
+  "sliding_window": 4096,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.22.0.dev0",
+  "use_bias": true,
+  "use_cache": true,
+  "vocab_size": 49152
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.22.0.dev0",
+  "use_cache": true
+}

integration.json

@@ -0,0 +1,92 @@
+{
+  "mic": "0.3",
+  "name": "StarCoder2 causal LM",
+  "usage": "AutoTokenizer + AutoModelForCausalLM, batched with left padding, next-token logits and a seeded training step",
+  "architecture": "Starcoder2ForCausalLM",
+  "checkpoints": {
+    "bigcode/starcoder2-3b": {
+      "fixture": "huggingface-mic/starcoder2-3b-mic"
+    }
+  },
+  "fixtures": {
+    "huggingface-mic/starcoder2-3b-mic": {
+      "built_from": "bigcode/starcoder2-3b",
+      "covers": [
+        "bigcode/starcoder2-3b"
+      ],
+      "build": {
+        "seed": 17,
+        "model_class": "Starcoder2ForCausalLM"
+      }
+    }
+  },
+  "integrations": {
+    "transformers": {
+      "entry_point": "integrations/transformers/usage.md",
+      "format": "runnable-markdown",
+      "requires": [
+        "transformers>=5",
+        "torch"
+      ],
+      "targets": [
+        "fixtures"
+      ],
+      "variants": {
+        "default": {},
+        "eager": {
+          "attn_implementation": "eager"
+        },
+        "bfloat16": {
+          "dtype": "bfloat16"
+        }
+      },
+      "training": true,
+      "resources": {
+        "fixture": {
+          "memory_gb": 2,
+          "download_mb": 30
+        },
+        "bigcode/starcoder2-3b": {
+          "memory_gb": 14,
+          "gpu_memory_gb": 22,
+          "download_mb": 6500,
+          "variants": {
+            "bfloat16": {
+              "gpu_memory_gb": 12
+            }
+          },
+          "devices": [
+            "cuda"
+          ]
+        }
+      }
+    }
+  },
+  "outputs": {
+    "runtime": {
+      "compare": "exact"
+    },
+    "next_token_ids": {
+      "compare": "exact"
+    },
+    "next_token_text": {
+      "compare": "exact"
+    },
+    "probe_logits": {
+      "compare": "allclose",
+      "atol": 0.0001,
+      "rtol": 0.0001
+    },
+    "training.loss": {
+      "compare": "allclose",
+      "atol": 0.0001,
+      "rtol": 0.0001
+    },
+    "training.gradients": {
+      "compare": "all_true"
+    },
+    "training.updated": {
+      "compare": "true"
+    }
+  }
+}

integrations/transformers/expectations/cpu-x86_64-bf16/starcoder2-3b-mic.bfloat16.json

@@ -0,0 +1,76 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/starcoder2-3b-mic",
+  "contract_name": "StarCoder2",
+  "contract_revision": "006d3b17bb359c06674a08e5b06359f1aafbebcc",
+  "framework": "transformers",
+  "variant": "bfloat16",
+  "target": "starcoder2-3b-mic",
+  "kind": "fixture",
+  "revision": "006d3b17bb359c06674a08e5b06359f1aafbebcc",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cpu",
+   "torchvision": "0.29.2+cpu",
+   "torchaudio": "2.11.1+cpu",
+   "tokenizers": "0.23.3",
+   "numpy": "2.3.5",
+   "pillow": "13.0.0",
+   "accelerate": "1.15.0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 7.22,
+  "environment_differs": {
+   "torchvision": "0.29.2+cpu (pinned 0.29.1)",
+   "torchaudio": "2.11.1+cpu (pinned 2.11.0)",
+   "pillow": "13.0.0 (pinned 12.3.0)"
+  },
+  "platform": "cpu-x86_64-bf16"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cpu",
+   "dtype": "bfloat16",
+   "attn_implementation": "sdpa"
+  },
+  "next_token_ids": [
+   32340,
+   37963
+  ],
+  "next_token_text": [
+   "'(",
+   "\ud68c"
+  ],
+  "probe_logits": [
+   [
+    -0.1103515625,
+    -0.0849609375,
+    -0.1416015625,
+    -0.1435546875,
+    -0.17578125
+   ],
+   [
+    -0.14453125,
+    -0.1435546875,
+    0.1787109375,
+    -0.0908203125,
+    -0.19140625
+   ]
+  ],
+  "training": {
+   "loss": 10.828598976135254,
+   "gradients": {
+    "model.layers.0.self_attn.q_proj.weight": true,
+    "model.layers.0.mlp.c_fc.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cpu-x86_64/starcoder2-3b-mic.eager.json

@@ -0,0 +1,76 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/starcoder2-3b-mic",
+  "contract_name": "StarCoder2",
+  "contract_revision": "006d3b17bb359c06674a08e5b06359f1aafbebcc",
+  "framework": "transformers",
+  "variant": "eager",
+  "target": "starcoder2-3b-mic",
+  "kind": "fixture",
+  "revision": "006d3b17bb359c06674a08e5b06359f1aafbebcc",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cpu",
+   "torchvision": "0.29.2+cpu",
+   "torchaudio": "2.11.1+cpu",
+   "tokenizers": "0.23.3",
+   "numpy": "2.3.5",
+   "pillow": "13.0.0",
+   "accelerate": "1.15.0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 7.23,
+  "environment_differs": {
+   "torchvision": "0.29.2+cpu (pinned 0.29.1)",
+   "torchaudio": "2.11.1+cpu (pinned 2.11.0)",
+   "pillow": "13.0.0 (pinned 12.3.0)"
+  },
+  "platform": "cpu-x86_64"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cpu",
+   "dtype": "float32",
+   "attn_implementation": "eager"
+  },
+  "next_token_ids": [
+   32340,
+   37963
+  ],
+  "next_token_text": [
+   "'(",
+   "\ud68c"
+  ],
+  "probe_logits": [
+   [
+    -0.1095876544713974,
+    -0.08578761667013168,
+    -0.14188802242279053,
+    -0.14319223165512085,
+    -0.17468278110027313
+   ],
+   [
+    -0.14452382922172546,
+    -0.14295382797718048,
+    0.17886753380298615,
+    -0.09121346473693848,
+    -0.19060993194580078
+   ]
+  ],
+  "training": {
+   "loss": 10.830249786376953,
+   "gradients": {
+    "model.layers.0.self_attn.q_proj.weight": true,
+    "model.layers.0.mlp.c_fc.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cpu-x86_64/starcoder2-3b-mic.json

@@ -0,0 +1,76 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/starcoder2-3b-mic",
+  "contract_name": "StarCoder2",
+  "contract_revision": "006d3b17bb359c06674a08e5b06359f1aafbebcc",
+  "framework": "transformers",
+  "variant": "default",
+  "target": "starcoder2-3b-mic",
+  "kind": "fixture",
+  "revision": "006d3b17bb359c06674a08e5b06359f1aafbebcc",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cpu",
+   "torchvision": "0.29.2+cpu",
+   "torchaudio": "2.11.1+cpu",
+   "tokenizers": "0.23.3",
+   "numpy": "2.3.5",
+   "pillow": "13.0.0",
+   "accelerate": "1.15.0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 7.23,
+  "environment_differs": {
+   "torchvision": "0.29.2+cpu (pinned 0.29.1)",
+   "torchaudio": "2.11.1+cpu (pinned 2.11.0)",
+   "pillow": "13.0.0 (pinned 12.3.0)"
+  },
+  "platform": "cpu-x86_64"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cpu",
+   "dtype": "float32",
+   "attn_implementation": "sdpa"
+  },
+  "next_token_ids": [
+   32340,
+   37963
+  ],
+  "next_token_text": [
+   "'(",
+   "\ud68c"
+  ],
+  "probe_logits": [
+   [
+    -0.1095876693725586,
+    -0.08578761667013168,
+    -0.14188805222511292,
+    -0.14319226145744324,
+    -0.17468281090259552
+   ],
+   [
+    -0.14452381432056427,
+    -0.1429538130760193,
+    0.1788676232099533,
+    -0.09121344983577728,
+    -0.19061002135276794
+   ]
+  ],
+  "training": {
+   "loss": 10.828756332397461,
+   "gradients": {
+    "model.layers.0.self_attn.q_proj.weight": true,
+    "model.layers.0.mlp.c_fc.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cuda/starcoder2-3b-mic.bfloat16.json

@@ -0,0 +1,77 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/starcoder2-3b-mic",
+  "contract_name": "StarCoder2",
+  "contract_revision": "006d3b17bb359c06674a08e5b06359f1aafbebcc",
+  "framework": "transformers",
+  "variant": "bfloat16",
+  "target": "starcoder2-3b-mic",
+  "kind": "fixture",
+  "revision": "006d3b17bb359c06674a08e5b06359f1aafbebcc",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cu130",
+   "torchvision": "0.29.2",
+   "torchaudio": "2.11.0+cu130",
+   "tokenizers": "0.23.3",
+   "numpy": "2.4.6",
+   "pillow": "13.0.0",
+   "accelerate": "1.16.0.dev0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 7.55,
+  "environment_differs": {
+   "torchvision": "0.29.2 (pinned 0.29.1)",
+   "numpy": "2.4.6 (pinned 2.3.5)",
+   "pillow": "13.0.0 (pinned 12.3.0)",
+   "accelerate": "1.16.0.dev0 (pinned 1.15.0)"
+  },
+  "platform": "cuda"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cuda",
+   "dtype": "bfloat16",
+   "attn_implementation": "sdpa"
+  },
+  "next_token_ids": [
+   32340,
+   37963
+  ],
+  "next_token_text": [
+   "'(",
+   "\ud68c"
+  ],
+  "probe_logits": [
+   [
+    -0.11083984375,
+    -0.0849609375,
+    -0.142578125,
+    -0.1435546875,
+    -0.1748046875
+   ],
+   [
+    -0.14453125,
+    -0.1435546875,
+    0.1787109375,
+    -0.0908203125,
+    -0.19140625
+   ]
+  ],
+  "training": {
+   "loss": 10.799779891967773,
+   "gradients": {
+    "model.layers.0.self_attn.q_proj.weight": true,
+    "model.layers.0.mlp.c_fc.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cuda/starcoder2-3b-mic.eager.json

@@ -0,0 +1,77 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/starcoder2-3b-mic",
+  "contract_name": "StarCoder2",
+  "contract_revision": "006d3b17bb359c06674a08e5b06359f1aafbebcc",
+  "framework": "transformers",
+  "variant": "eager",
+  "target": "starcoder2-3b-mic",
+  "kind": "fixture",
+  "revision": "006d3b17bb359c06674a08e5b06359f1aafbebcc",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cu130",
+   "torchvision": "0.29.2",
+   "torchaudio": "2.11.0+cu130",
+   "tokenizers": "0.23.3",
+   "numpy": "2.4.6",
+   "pillow": "13.0.0",
+   "accelerate": "1.16.0.dev0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 7.84,
+  "environment_differs": {
+   "torchvision": "0.29.2 (pinned 0.29.1)",
+   "numpy": "2.4.6 (pinned 2.3.5)",
+   "pillow": "13.0.0 (pinned 12.3.0)",
+   "accelerate": "1.16.0.dev0 (pinned 1.15.0)"
+  },
+  "platform": "cuda"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cuda",
+   "dtype": "float32",
+   "attn_implementation": "eager"
+  },
+  "next_token_ids": [
+   32340,
+   37963
+  ],
+  "next_token_text": [
+   "'(",
+   "\ud68c"
+  ],
+  "probe_logits": [
+   [
+    -0.1095876470208168,
+    -0.08578762412071228,
+    -0.14188805222511292,
+    -0.14319227635860443,
+    -0.1746828556060791
+   ],
+   [
+    -0.14452379941940308,
+    -0.1429538130760193,
+    0.17886760830879211,
+    -0.09121352434158325,
+    -0.19060999155044556
+   ]
+  ],
+  "training": {
+   "loss": 10.811941146850586,
+   "gradients": {
+    "model.layers.0.self_attn.q_proj.weight": true,
+    "model.layers.0.mlp.c_fc.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cuda/starcoder2-3b-mic.json

@@ -0,0 +1,77 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/starcoder2-3b-mic",
+  "contract_name": "StarCoder2",
+  "contract_revision": "006d3b17bb359c06674a08e5b06359f1aafbebcc",
+  "framework": "transformers",
+  "variant": "default",
+  "target": "starcoder2-3b-mic",
+  "kind": "fixture",
+  "revision": "006d3b17bb359c06674a08e5b06359f1aafbebcc",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cu130",
+   "torchvision": "0.29.2",
+   "torchaudio": "2.11.0+cu130",
+   "tokenizers": "0.23.3",
+   "numpy": "2.4.6",
+   "pillow": "13.0.0",
+   "accelerate": "1.16.0.dev0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 8.14,
+  "environment_differs": {
+   "torchvision": "0.29.2 (pinned 0.29.1)",
+   "numpy": "2.4.6 (pinned 2.3.5)",
+   "pillow": "13.0.0 (pinned 12.3.0)",
+   "accelerate": "1.16.0.dev0 (pinned 1.15.0)"
+  },
+  "platform": "cuda"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cuda",
+   "dtype": "float32",
+   "attn_implementation": "sdpa"
+  },
+  "next_token_ids": [
+   32340,
+   37963
+  ],
+  "next_token_text": [
+   "'(",
+   "\ud68c"
+  ],
+  "probe_logits": [
+   [
+    -0.10958769917488098,
+    -0.08578769862651825,
+    -0.14188802242279053,
+    -0.1431923508644104,
+    -0.1746828854084015
+   ],
+   [
+    -0.14452379941940308,
+    -0.1429538130760193,
+    0.17886753380298615,
+    -0.09121351689100266,
+    -0.19060997664928436
+   ]
+  ],
+  "training": {
+   "loss": 10.8132963180542,
+   "gradients": {
+    "model.layers.0.self_attn.q_proj.weight": true,
+    "model.layers.0.mlp.c_fc.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/usage.md

@@ -0,0 +1,94 @@
+# StarCoder2: causal language modeling with Transformers
+
+This page is the Transformers contract of the StarCoder2 MIC and also a usage example. [doc-builder](https://github.com/huggingface/doc-builder/blob/main/docs/runnable-code-blocks.md) renders the `runnable` blocks below as documentation, and its pytest plugin runs them as written against `bigcode/starcoder2-3b`. The MIC runner runs the same blocks against the slim fixture and the checkpoint in [`integration.json`](../../integration.json). Lines marked `# doc-builder: hide` only matter there: they swap in the pinned checkpoint or variant and print the result record.
+
+## Next-token prediction
+
+```py runnable:starcoder2
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, set_seed
+import json  # doc-builder: hide
+import os  # doc-builder: hide
+
+model_id = "bigcode/starcoder2-3b"
+model_id = os.environ.get("MIC_MODEL", model_id)  # doc-builder: hide
+variant = json.loads(os.environ.get("MIC_MODEL_KWARGS", "{}"))  # doc-builder: hide
+device = "cuda" if torch.cuda.is_available() else "cpu"
+set_seed(0)  # seeds torch, NumPy, and Python: loading and dropout draw random numbers
+
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+# StarCoder2 defines no pad token; left padding keeps the last position the real next-token slot.
+if tokenizer.pad_token is None:
+    tokenizer.pad_token = tokenizer.eos_token
+    tokenizer.padding_side = "left"
+model = AutoModelForCausalLM.from_pretrained(model_id).to(device)
+if variant:  # doc-builder: hide
+    del model  # doc-builder: hide
+    model = AutoModelForCausalLM.from_pretrained(model_id, **variant).to(device)  # doc-builder: hide
+model.eval()
+
+prompts = [
+    "def add(a, b):\n    return a + b\n\ndef multiply(a, b):\n    return",
+    "import numpy as np\n\ndef mean(xs):\n    return np.",
+]
+inputs = tokenizer(prompts, return_tensors="pt", padding=True).to(device)
+
+with torch.no_grad():
+    logits = model(**inputs).logits
+# Left padding: the last position of each row is the next-token prediction.
+last_logits = logits[:, -1, :]
+next_token_ids = last_logits.argmax(dim=-1)
+next_token_text = tokenizer.batch_decode(next_token_ids.unsqueeze(-1))
+print(next_token_text)
+```
+
+## One training step
+
+```py runnable:starcoder2:2
+# Language-model labels: copy the ids and mask padded positions with -100.
+labels = inputs["input_ids"].clone()
+labels[inputs["attention_mask"] == 0] = -100
+
+model.train()
+optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
+loss = model(**inputs, labels=labels).loss
+loss.backward()
+before = {n: p.detach().cpu().clone() for n, p in model.named_parameters() if p.requires_grad}  # doc-builder: hide
+optimizer.step()
+print(f"loss: {loss.item():.4f}")
+
+if os.environ.get("MIC_RUNNER"):  # doc-builder: hide
+    named = dict(model.named_parameters())
+    checked = (
+        "model.layers.0.self_attn.q_proj.weight",
+        "model.layers.0.mlp.c_fc.weight",
+    )
+    probe_ids = next_token_ids.tolist()
+    result = {
+        "runtime": {
+            "device": model.device.type,
+            "dtype": str(model.dtype).removeprefix("torch."),
+            "attn_implementation": model.config._attn_implementation,
+        },
+        "next_token_ids": probe_ids,
+        "next_token_text": next_token_text,
+        "probe_logits": last_logits[:, :5].float().tolist(),
+        "training": {
+            "loss": loss.item(),
+            "gradients": {
+                n: bool(
+                    named[n].grad is not None
+                    and torch.count_nonzero(named[n].grad) > 0
+                    and torch.isfinite(named[n].grad).all()
+                )
+                for n in checked
+            },
+            "updated": any(
+                not torch.equal(p.detach().cpu(), before[n])
+                for n, p in model.named_parameters()
+                if n in before
+            ),
+        },
+    }
+    print(json.dumps(result))
+```

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e02ed2bdffd25a57e35c3cebb45d5f51ddbecaf977919bfa46260d930e82247
+size 12821560

special_tokens_map.json

@@ -0,0 +1,45 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<fim_prefix>",
+    "<fim_middle>",
+    "<fim_suffix>",
+    "<fim_pad>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

tokenizer_config.json

@@ -0,0 +1,356 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<fim_prefix>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<fim_middle>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<fim_suffix>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<fim_pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<code_to_intermediate>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "17": {
+      "content": "<intermediate_to_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "18": {
+      "content": "<pr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "19": {
+      "content": "<pr_status>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "20": {
+      "content": "<pr_is_merged>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "21": {
+      "content": "<pr_base>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "22": {
+      "content": "<pr_file>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "23": {
+      "content": "<pr_base_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "24": {
+      "content": "<pr_diff>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "25": {
+      "content": "<pr_diff_hunk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "26": {
+      "content": "<pr_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "27": {
+      "content": "<pr_event_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "28": {
+      "content": "<pr_review>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "29": {
+      "content": "<pr_review_state>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30": {
+      "content": "<pr_review_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "31": {
+      "content": "<pr_in_reply_to_review_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32": {
+      "content": "<pr_in_reply_to_comment_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "33": {
+      "content": "<pr_diff_hunk_comment_line>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "34": {
+      "content": "<NAME>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "35": {
+      "content": "<EMAIL>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "36": {
+      "content": "<KEY>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "37": {
+      "content": "<PASSWORD>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<fim_prefix>",
+    "<fim_middle>",
+    "<fim_suffix>",
+    "<fim_pad>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}