Publish MIC usage, slim fixture and reviewed Transformers expectations

.mic-repo.json

@@ -0,0 +1,7 @@
+{
+  "format": "mic-umbrella-v1",
+  "contract_files": [
+    "integration.json",
+    "integrations/transformers/usage.md"
+  ]
+}

README.md

@@ -0,0 +1,37 @@
+---
+tags:
+  - mic
+  - test-artifact
+base_model: bigcode/tiny_starcoder_py
+---
+
+# tiny_starcoder_py-mic
+
+Slim model weights, executable usage and framework expectations in one repository.
+
+- [Contract manifest](integration.json)
+- [Transformers usage](integrations/transformers/usage.md)
+- [Transformers expectations](integrations/transformers/expectations)
+- [Fixture provenance](build.json)
+
+## Contract
+
+### GPTBigCode causal LM
+
+Runnable usage and the framework expectations directory are linked above.
+
+## Slim model
+
+### Slim fixture: tiny_starcoder_py-mic
+
+A 3.7M-parameter fixture for the [GPTBigCode MIC](https://huggingface.co/huggingface-mic/tiny_starcoder_py-mic), covering [`bigcode/tiny_starcoder_py`](https://huggingface.co/bigcode/tiny_starcoder_py). It is a test artifact, not a usable model, tagged `test-artifact` to keep it out of candidate-model search.
+
+- **Fixture key `0852b1b7488d`.** It covers every checkpoint whose processor files are byte-identical and whose config matches after shrinking.
+- **Same architecture, smaller config:** `GPTBigCodeForCausalLM`, shrunk by the generic rule in [`sweep.py`](https://github.com/huggingface/transformers-mic/blob/main/experiments/sweep.py) ().
+- **Processor files copied byte for byte** from `bigcode/tiny_starcoder_py`: `generation_config.json`, `merges.txt`, `special_tokens_map.json`, `tokenizer.json`, `tokenizer_config.json`, `vocab.json`.
+- **Random weights (seed 17).** They passed the input-sensitivity gate: every perturbation of the contract's inputs moved the logits by at least 0.001 (weakest: `attention_mask:all_ones`, 0.5), so no training was needed.
+- **Same tensor names as the real checkpoint.** The real checkpoint has every parameter, and so does the fixture.
+- **Saved in float32**, the dtype `bigcode/tiny_starcoder_py` loads in by default.
+- **Finite in float32, bfloat16, and float16** (forward, loss, gradients).
+
+Built by [`build_fixtures.py`](https://github.com/huggingface/transformers-mic/blob/main/example/build_fixtures.py); [`build.json`](build.json) records the pinned sources, gate results, and file hashes. This repository includes the slim weights, usage and framework expectations; rebuild the other files with `python example/build_fixtures.py --fixture tiny_starcoder_py-mic`.

build.json

@@ -0,0 +1,149 @@
+{
+ "fixture": "huggingface-mic/tiny_starcoder_py-mic",
+ "key": "0852b1b7488daf526180d9c8c3361de942729124e3b94beb0339fa2d46cf47e0",
+ "key_fields": {
+  "architectures": [
+   "GPTBigCodeForCausalLM"
+  ],
+  "output_hidden_states": false,
+  "return_dict": true,
+  "chunk_size_feed_forward": 0,
+  "is_encoder_decoder": false,
+  "id2label": {
+   "0": "LABEL_0",
+   "1": "LABEL_1"
+  },
+  "label2id": {
+   "LABEL_0": 0,
+   "LABEL_1": 1
+  },
+  "problem_type": null,
+  "vocab_size": 49152,
+  "n_positions": 8192,
+  "n_embd": 64,
+  "n_layer": 2,
+  "n_head": 4,
+  "n_inner": 128,
+  "activation_function": "gelu_pytorch_tanh",
+  "resid_pdrop": 0.1,
+  "embd_pdrop": 0.1,
+  "attn_pdrop": 0.1,
+  "layer_norm_epsilon": 1e-05,
+  "initializer_range": 0.02,
+  "scale_attn_weights": true,
+  "use_cache": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_id": null,
+  "attention_softmax_in_fp32": true,
+  "scale_attention_softmax_in_fp32": true,
+  "multi_query": true,
+  "add_cross_attention": false,
+  "tie_word_embeddings": true,
+  "num_key_value_heads": 2,
+  "inference_runner": 0,
+  "max_batch_size": null,
+  "max_sequence_length": null,
+  "model_type": "gpt_bigcode",
+  "pad_key_length": true,
+  "pre_allocate_kv_cache": false,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "validate_runner_input": true,
+  "output_attentions": false
+ },
+ "covers": {
+  "bigcode/tiny_starcoder_py": "8547527bef0bc927268c1653cce6948c5c242dd1"
+ },
+ "built_from": {
+  "repo": "bigcode/tiny_starcoder_py",
+  "revision": "8547527bef0bc927268c1653cce6948c5c242dd1"
+ },
+ "seed": 17,
+ "dtype": "float32",
+ "builder": "example/build_fixtures.py",
+ "builder_revision": "c0a9ac9ce31d96d81a7ed980688dcd4599b1e6bd",
+ "recipe": {
+  "seed": 17,
+  "model_class": "GPTBigCodeForCausalLM"
+ },
+ "shrink": "rule",
+ "model_class": "GPTBigCodeForCausalLM",
+ "parameters": 3724608,
+ "missing_parameters": [],
+ "checkpoint_inspection": "parameter headers/indexes; bounded binary fallback",
+ "numeric_health_scope": "CPU float32/bfloat16/float16; autocast for low-precision losses",
+ "captured": {
+  "call": "training step",
+  "inputs": [
+   "attention_mask",
+   "input_ids",
+   "labels"
+  ],
+  "class": "GPTBigCodeForCausalLM"
+ },
+ "weights": "random",
+ "gate": {
+  "random": {
+   "pass": true,
+   "accepted_insensitive": {},
+   "threshold": 0.001,
+   "deltas": {
+    "input_ids:roll_batch": 1.1869170665740967,
+    "input_ids:reverse": 1.1730904579162598,
+    "attention_mask:all_ones": 0.49951428174972534
+   },
+   "weakest": "attention_mask:all_ones",
+   "no_perturbation_defined": [],
+   "not_a_forward_parameter": []
+  }
+ },
+ "numeric_health": {
+  "float32": {
+   "outputs_finite": true,
+   "loss_finite": true,
+   "gradients_finite": true,
+   "finite": true
+  },
+  "bfloat16": {
+   "outputs_finite": true,
+   "loss_finite": true,
+   "gradients_finite": true,
+   "finite": true
+  },
+  "float16": {
+   "outputs_finite": true,
+   "loss_finite": true,
+   "unscaled_gradients_finite": true,
+   "finite": true
+  }
+ },
+ "copied_unchanged": [
+  "generation_config.json",
+  "merges.txt",
+  "special_tokens_map.json",
+  "tokenizer.json",
+  "tokenizer_config.json",
+  "vocab.json"
+ ],
+ "environment": {
+  "python": "3.12.12",
+  "machine": "arm64",
+  "torch": "2.14.1",
+  "transformers": "5.22.0.dev0",
+  "tokenizers": "0.23.3"
+ },
+ "sha256": {
+  "config.json": "e71a18da111f128967e7d8cc89622e71dafc295a27b8181170d11423ad420ea0",
+  "generation_config.json": "368f15f9c335d48e1a585440037c77022af6b867303dc8fe9ee01e8fc86c1617",
+  "merges.txt": "74b0a4bc1a97ebc1d227f69231b18574374ba052fb945b0fa0aa91d3c32504a2",
+  "model.safetensors": "1cb730824fb0a3d7a02141afb42177445e7d8522e8cde09bbfb3c174d5ca2ed0",
+  "special_tokens_map.json": "0823292e24ea07b89317e9ede9d08da2a1b6c014290c06908a7ad04f1efd6719",
+  "tokenizer.json": "42b5a37ba11199f024f2b8873e1ecba98da33166e16f700bf7cb2304b0a5583f",
+  "tokenizer_config.json": "95684c52ad9a970dbbb17576ee2237cb62902c1eff6804c7c91a4d6219a4a6d7",
+  "vocab.json": "20175afb9f164fad4829aca2279f8df7eeff1e2e3f671378aaa287a740aff09f"
+ }
+}

config.json

@@ -0,0 +1,42 @@
+{
+  "activation_function": "gelu_pytorch_tanh",
+  "add_cross_attention": false,
+  "architectures": [
+    "GPTBigCodeForCausalLM"
+  ],
+  "attention_softmax_in_fp32": true,
+  "attn_pdrop": 0.1,
+  "bos_token_id": 0,
+  "dtype": "float32",
+  "embd_pdrop": 0.1,
+  "eos_token_id": 0,
+  "inference_runner": 0,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "max_batch_size": null,
+  "max_sequence_length": null,
+  "model_type": "gpt_bigcode",
+  "multi_query": true,
+  "n_embd": 64,
+  "n_head": 4,
+  "n_inner": 128,
+  "n_layer": 2,
+  "n_positions": 8192,
+  "num_key_value_heads": 2,
+  "pad_key_length": true,
+  "pad_token_id": null,
+  "pre_allocate_kv_cache": false,
+  "resid_pdrop": 0.1,
+  "scale_attention_softmax_in_fp32": true,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.22.0.dev0",
+  "use_cache": true,
+  "validate_runner_input": true,
+  "vocab_size": 49152
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "transformers_version": "4.28.1"
+}

integration.json

@@ -0,0 +1,84 @@
+{
+  "mic": "0.3",
+  "name": "GPTBigCode causal LM",
+  "usage": "AutoTokenizer + AutoModelForCausalLM, batched, left padding, seeded training step",
+  "architecture": "GPTBigCodeForCausalLM",
+  "checkpoints": {
+    "bigcode/tiny_starcoder_py": {
+      "fixture": "huggingface-mic/tiny_starcoder_py-mic"
+    }
+  },
+  "fixtures": {
+    "huggingface-mic/tiny_starcoder_py-mic": {
+      "built_from": "bigcode/tiny_starcoder_py",
+      "covers": [
+        "bigcode/tiny_starcoder_py"
+      ],
+      "build": {
+        "seed": 17,
+        "model_class": "GPTBigCodeForCausalLM"
+      }
+    }
+  },
+  "integrations": {
+    "transformers": {
+      "entry_point": "integrations/transformers/usage.md",
+      "format": "runnable-markdown",
+      "requires": [
+        "transformers>=5",
+        "torch"
+      ],
+      "targets": [
+        "fixtures",
+        "checkpoints"
+      ],
+      "variants": {
+        "default": {},
+        "eager": {
+          "attn_implementation": "eager"
+        },
+        "bfloat16": {
+          "dtype": "bfloat16"
+        }
+      },
+      "training": true,
+      "resources": {
+        "fixture": {
+          "memory_gb": 1,
+          "download_mb": 20
+        },
+        "bigcode/tiny_starcoder_py": {
+          "memory_gb": 4,
+          "download_mb": 260
+        }
+      }
+    }
+  },
+  "outputs": {
+    "runtime": {
+      "compare": "exact"
+    },
+    "predicted_tokens": {
+      "compare": "exact"
+    },
+    "predicted_text": {
+      "compare": "exact"
+    },
+    "next_token_logits": {
+      "compare": "allclose",
+      "atol": 0.0001,
+      "rtol": 0.0001
+    },
+    "training.loss": {
+      "compare": "allclose",
+      "atol": 0.0001,
+      "rtol": 0.0001
+    },
+    "training.gradients": {
+      "compare": "all_true"
+    },
+    "training.updated": {
+      "compare": "true"
+    }
+  }
+}

integrations/transformers/expectations/cpu-x86_64-bf16/tiny_starcoder_py-mic.bfloat16.json

@@ -0,0 +1,64 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/tiny_starcoder_py-mic",
+  "contract_name": "GPTBigCode",
+  "contract_revision": "f02c1112074b58d092f6590c8a957bdbe935dae4",
+  "framework": "transformers",
+  "variant": "bfloat16",
+  "target": "tiny_starcoder_py-mic",
+  "kind": "fixture",
+  "revision": "f02c1112074b58d092f6590c8a957bdbe935dae4",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cpu",
+   "torchvision": "0.29.2+cpu",
+   "torchaudio": "2.11.1+cpu",
+   "tokenizers": "0.23.3",
+   "numpy": "2.3.5",
+   "pillow": "13.0.0",
+   "accelerate": "1.15.0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 7.39,
+  "environment_differs": {
+   "torchvision": "0.29.2+cpu (pinned 0.29.1)",
+   "torchaudio": "2.11.1+cpu (pinned 2.11.0)",
+   "pillow": "13.0.0 (pinned 12.3.0)"
+  },
+  "platform": "cpu-x86_64-bf16"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cpu",
+   "dtype": "bfloat16",
+   "attn_implementation": "sdpa"
+  },
+  "predicted_tokens": [
+   442,
+   32
+  ],
+  "predicted_text": [
+   "\u0120return",
+   "."
+  ],
+  "next_token_logits": [
+   0.828125,
+   0.68359375
+  ],
+  "training": {
+   "loss": 10.775863647460938,
+   "gradients": {
+    "transformer.h.0.attn.c_attn.weight": true,
+    "transformer.h.0.mlp.c_fc.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cpu-x86_64-bf16/tiny_starcoder_py.bfloat16.json

@@ -0,0 +1,64 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/tiny_starcoder_py-mic",
+  "contract_name": "GPTBigCode",
+  "contract_revision": "f02c1112074b58d092f6590c8a957bdbe935dae4",
+  "framework": "transformers",
+  "variant": "bfloat16",
+  "target": "tiny_starcoder_py",
+  "kind": "checkpoint",
+  "revision": "8547527bef0bc927268c1653cce6948c5c242dd1",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cpu",
+   "torchvision": "0.29.2+cpu",
+   "torchaudio": "2.11.1+cpu",
+   "tokenizers": "0.23.3",
+   "numpy": "2.3.5",
+   "pillow": "13.0.0",
+   "accelerate": "1.15.0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 7.82,
+  "environment_differs": {
+   "torchvision": "0.29.2+cpu (pinned 0.29.1)",
+   "torchaudio": "2.11.1+cpu (pinned 2.11.0)",
+   "pillow": "13.0.0 (pinned 12.3.0)"
+  },
+  "platform": "cpu-x86_64-bf16"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cpu",
+   "dtype": "bfloat16",
+   "attn_implementation": "sdpa"
+  },
+  "predicted_tokens": [
+   312,
+   2356
+  ],
+  "predicted_text": [
+   "\u0120a",
+   "np"
+  ],
+  "next_token_logits": [
+   20.5,
+   14.1875
+  ],
+  "training": {
+   "loss": 3.5714824199676514,
+   "gradients": {
+    "transformer.h.0.attn.c_attn.weight": true,
+    "transformer.h.0.mlp.c_fc.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cpu-x86_64/tiny_starcoder_py-mic.eager.json

@@ -0,0 +1,64 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/tiny_starcoder_py-mic",
+  "contract_name": "GPTBigCode",
+  "contract_revision": "f02c1112074b58d092f6590c8a957bdbe935dae4",
+  "framework": "transformers",
+  "variant": "eager",
+  "target": "tiny_starcoder_py-mic",
+  "kind": "fixture",
+  "revision": "f02c1112074b58d092f6590c8a957bdbe935dae4",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cpu",
+   "torchvision": "0.29.2+cpu",
+   "torchaudio": "2.11.1+cpu",
+   "tokenizers": "0.23.3",
+   "numpy": "2.3.5",
+   "pillow": "13.0.0",
+   "accelerate": "1.15.0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 7.38,
+  "environment_differs": {
+   "torchvision": "0.29.2+cpu (pinned 0.29.1)",
+   "torchaudio": "2.11.1+cpu (pinned 2.11.0)",
+   "pillow": "13.0.0 (pinned 12.3.0)"
+  },
+  "platform": "cpu-x86_64"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cpu",
+   "dtype": "float32",
+   "attn_implementation": "eager"
+  },
+  "predicted_tokens": [
+   442,
+   32
+  ],
+  "predicted_text": [
+   "\u0120return",
+   "."
+  ],
+  "next_token_logits": [
+   0.826957106590271,
+   0.6829898357391357
+  ],
+  "training": {
+   "loss": 10.774384498596191,
+   "gradients": {
+    "transformer.h.0.attn.c_attn.weight": true,
+    "transformer.h.0.mlp.c_fc.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cpu-x86_64/tiny_starcoder_py-mic.json

@@ -0,0 +1,64 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/tiny_starcoder_py-mic",
+  "contract_name": "GPTBigCode",
+  "contract_revision": "f02c1112074b58d092f6590c8a957bdbe935dae4",
+  "framework": "transformers",
+  "variant": "default",
+  "target": "tiny_starcoder_py-mic",
+  "kind": "fixture",
+  "revision": "f02c1112074b58d092f6590c8a957bdbe935dae4",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cpu",
+   "torchvision": "0.29.2+cpu",
+   "torchaudio": "2.11.1+cpu",
+   "tokenizers": "0.23.3",
+   "numpy": "2.3.5",
+   "pillow": "13.0.0",
+   "accelerate": "1.15.0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 7.37,
+  "environment_differs": {
+   "torchvision": "0.29.2+cpu (pinned 0.29.1)",
+   "torchaudio": "2.11.1+cpu (pinned 2.11.0)",
+   "pillow": "13.0.0 (pinned 12.3.0)"
+  },
+  "platform": "cpu-x86_64"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cpu",
+   "dtype": "float32",
+   "attn_implementation": "sdpa"
+  },
+  "predicted_tokens": [
+   442,
+   32
+  ],
+  "predicted_text": [
+   "\u0120return",
+   "."
+  ],
+  "next_token_logits": [
+   0.8269570469856262,
+   0.6829898953437805
+  ],
+  "training": {
+   "loss": 10.776015281677246,
+   "gradients": {
+    "transformer.h.0.attn.c_attn.weight": true,
+    "transformer.h.0.mlp.c_fc.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cpu-x86_64/tiny_starcoder_py.eager.json

@@ -0,0 +1,64 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/tiny_starcoder_py-mic",
+  "contract_name": "GPTBigCode",
+  "contract_revision": "f02c1112074b58d092f6590c8a957bdbe935dae4",
+  "framework": "transformers",
+  "variant": "eager",
+  "target": "tiny_starcoder_py",
+  "kind": "checkpoint",
+  "revision": "8547527bef0bc927268c1653cce6948c5c242dd1",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cpu",
+   "torchvision": "0.29.2+cpu",
+   "torchaudio": "2.11.1+cpu",
+   "tokenizers": "0.23.3",
+   "numpy": "2.3.5",
+   "pillow": "13.0.0",
+   "accelerate": "1.15.0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 8.28,
+  "environment_differs": {
+   "torchvision": "0.29.2+cpu (pinned 0.29.1)",
+   "torchaudio": "2.11.1+cpu (pinned 2.11.0)",
+   "pillow": "13.0.0 (pinned 12.3.0)"
+  },
+  "platform": "cpu-x86_64"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cpu",
+   "dtype": "float32",
+   "attn_implementation": "eager"
+  },
+  "predicted_tokens": [
+   312,
+   2356
+  ],
+  "predicted_text": [
+   "\u0120a",
+   "np"
+  ],
+  "next_token_logits": [
+   20.465251922607422,
+   14.459330558776855
+  ],
+  "training": {
+   "loss": 3.5564022064208984,
+   "gradients": {
+    "transformer.h.0.attn.c_attn.weight": true,
+    "transformer.h.0.mlp.c_fc.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cpu-x86_64/tiny_starcoder_py.json

@@ -0,0 +1,64 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/tiny_starcoder_py-mic",
+  "contract_name": "GPTBigCode",
+  "contract_revision": "f02c1112074b58d092f6590c8a957bdbe935dae4",
+  "framework": "transformers",
+  "variant": "default",
+  "target": "tiny_starcoder_py",
+  "kind": "checkpoint",
+  "revision": "8547527bef0bc927268c1653cce6948c5c242dd1",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cpu",
+   "torchvision": "0.29.2+cpu",
+   "torchaudio": "2.11.1+cpu",
+   "tokenizers": "0.23.3",
+   "numpy": "2.3.5",
+   "pillow": "13.0.0",
+   "accelerate": "1.15.0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 8.33,
+  "environment_differs": {
+   "torchvision": "0.29.2+cpu (pinned 0.29.1)",
+   "torchaudio": "2.11.1+cpu (pinned 2.11.0)",
+   "pillow": "13.0.0 (pinned 12.3.0)"
+  },
+  "platform": "cpu-x86_64"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cpu",
+   "dtype": "float32",
+   "attn_implementation": "sdpa"
+  },
+  "predicted_tokens": [
+   312,
+   2356
+  ],
+  "predicted_text": [
+   "\u0120a",
+   "np"
+  ],
+  "next_token_logits": [
+   20.465251922607422,
+   14.459315299987793
+  ],
+  "training": {
+   "loss": 3.5751023292541504,
+   "gradients": {
+    "transformer.h.0.attn.c_attn.weight": true,
+    "transformer.h.0.mlp.c_fc.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cuda/tiny_starcoder_py-mic.bfloat16.json

@@ -0,0 +1,65 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/tiny_starcoder_py-mic",
+  "contract_name": "GPTBigCode",
+  "contract_revision": "f02c1112074b58d092f6590c8a957bdbe935dae4",
+  "framework": "transformers",
+  "variant": "bfloat16",
+  "target": "tiny_starcoder_py-mic",
+  "kind": "fixture",
+  "revision": "f02c1112074b58d092f6590c8a957bdbe935dae4",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cu130",
+   "torchvision": "0.29.2",
+   "torchaudio": "2.11.0+cu130",
+   "tokenizers": "0.23.3",
+   "numpy": "2.4.6",
+   "pillow": "13.0.0",
+   "accelerate": "1.16.0.dev0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 7.85,
+  "environment_differs": {
+   "torchvision": "0.29.2 (pinned 0.29.1)",
+   "numpy": "2.4.6 (pinned 2.3.5)",
+   "pillow": "13.0.0 (pinned 12.3.0)",
+   "accelerate": "1.16.0.dev0 (pinned 1.15.0)"
+  },
+  "platform": "cuda"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cuda",
+   "dtype": "bfloat16",
+   "attn_implementation": "sdpa"
+  },
+  "predicted_tokens": [
+   442,
+   32
+  ],
+  "predicted_text": [
+   "\u0120return",
+   "."
+  ],
+  "next_token_logits": [
+   0.828125,
+   0.68359375
+  ],
+  "training": {
+   "loss": 10.764673233032227,
+   "gradients": {
+    "transformer.h.0.attn.c_attn.weight": true,
+    "transformer.h.0.mlp.c_fc.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cuda/tiny_starcoder_py-mic.eager.json

@@ -0,0 +1,65 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/tiny_starcoder_py-mic",
+  "contract_name": "GPTBigCode",
+  "contract_revision": "f02c1112074b58d092f6590c8a957bdbe935dae4",
+  "framework": "transformers",
+  "variant": "eager",
+  "target": "tiny_starcoder_py-mic",
+  "kind": "fixture",
+  "revision": "f02c1112074b58d092f6590c8a957bdbe935dae4",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cu130",
+   "torchvision": "0.29.2",
+   "torchaudio": "2.11.0+cu130",
+   "tokenizers": "0.23.3",
+   "numpy": "2.4.6",
+   "pillow": "13.0.0",
+   "accelerate": "1.16.0.dev0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 6.98,
+  "environment_differs": {
+   "torchvision": "0.29.2 (pinned 0.29.1)",
+   "numpy": "2.4.6 (pinned 2.3.5)",
+   "pillow": "13.0.0 (pinned 12.3.0)",
+   "accelerate": "1.16.0.dev0 (pinned 1.15.0)"
+  },
+  "platform": "cuda"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cuda",
+   "dtype": "float32",
+   "attn_implementation": "eager"
+  },
+  "predicted_tokens": [
+   442,
+   32
+  ],
+  "predicted_text": [
+   "\u0120return",
+   "."
+  ],
+  "next_token_logits": [
+   0.8269570469856262,
+   0.6829898953437805
+  ],
+  "training": {
+   "loss": 10.763090133666992,
+   "gradients": {
+    "transformer.h.0.attn.c_attn.weight": true,
+    "transformer.h.0.mlp.c_fc.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cuda/tiny_starcoder_py-mic.json

@@ -0,0 +1,65 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/tiny_starcoder_py-mic",
+  "contract_name": "GPTBigCode",
+  "contract_revision": "f02c1112074b58d092f6590c8a957bdbe935dae4",
+  "framework": "transformers",
+  "variant": "default",
+  "target": "tiny_starcoder_py-mic",
+  "kind": "fixture",
+  "revision": "f02c1112074b58d092f6590c8a957bdbe935dae4",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cu130",
+   "torchvision": "0.29.2",
+   "torchaudio": "2.11.0+cu130",
+   "tokenizers": "0.23.3",
+   "numpy": "2.4.6",
+   "pillow": "13.0.0",
+   "accelerate": "1.16.0.dev0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 8.14,
+  "environment_differs": {
+   "torchvision": "0.29.2 (pinned 0.29.1)",
+   "numpy": "2.4.6 (pinned 2.3.5)",
+   "pillow": "13.0.0 (pinned 12.3.0)",
+   "accelerate": "1.16.0.dev0 (pinned 1.15.0)"
+  },
+  "platform": "cuda"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cuda",
+   "dtype": "float32",
+   "attn_implementation": "sdpa"
+  },
+  "predicted_tokens": [
+   442,
+   32
+  ],
+  "predicted_text": [
+   "\u0120return",
+   "."
+  ],
+  "next_token_logits": [
+   0.8269569873809814,
+   0.6829898953437805
+  ],
+  "training": {
+   "loss": 10.76203727722168,
+   "gradients": {
+    "transformer.h.0.attn.c_attn.weight": true,
+    "transformer.h.0.mlp.c_fc.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cuda/tiny_starcoder_py.bfloat16.json

@@ -0,0 +1,65 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/tiny_starcoder_py-mic",
+  "contract_name": "GPTBigCode",
+  "contract_revision": "f02c1112074b58d092f6590c8a957bdbe935dae4",
+  "framework": "transformers",
+  "variant": "bfloat16",
+  "target": "tiny_starcoder_py",
+  "kind": "checkpoint",
+  "revision": "8547527bef0bc927268c1653cce6948c5c242dd1",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cu130",
+   "torchvision": "0.29.2",
+   "torchaudio": "2.11.0+cu130",
+   "tokenizers": "0.23.3",
+   "numpy": "2.4.6",
+   "pillow": "13.0.0",
+   "accelerate": "1.16.0.dev0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 8.29,
+  "environment_differs": {
+   "torchvision": "0.29.2 (pinned 0.29.1)",
+   "numpy": "2.4.6 (pinned 2.3.5)",
+   "pillow": "13.0.0 (pinned 12.3.0)",
+   "accelerate": "1.16.0.dev0 (pinned 1.15.0)"
+  },
+  "platform": "cuda"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cuda",
+   "dtype": "bfloat16",
+   "attn_implementation": "sdpa"
+  },
+  "predicted_tokens": [
+   312,
+   2356
+  ],
+  "predicted_text": [
+   "\u0120a",
+   "np"
+  ],
+  "next_token_logits": [
+   20.5,
+   14.0625
+  ],
+  "training": {
+   "loss": 3.034364700317383,
+   "gradients": {
+    "transformer.h.0.attn.c_attn.weight": true,
+    "transformer.h.0.mlp.c_fc.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cuda/tiny_starcoder_py.eager.json

@@ -0,0 +1,65 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/tiny_starcoder_py-mic",
+  "contract_name": "GPTBigCode",
+  "contract_revision": "f02c1112074b58d092f6590c8a957bdbe935dae4",
+  "framework": "transformers",
+  "variant": "eager",
+  "target": "tiny_starcoder_py",
+  "kind": "checkpoint",
+  "revision": "8547527bef0bc927268c1653cce6948c5c242dd1",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cu130",
+   "torchvision": "0.29.2",
+   "torchaudio": "2.11.0+cu130",
+   "tokenizers": "0.23.3",
+   "numpy": "2.4.6",
+   "pillow": "13.0.0",
+   "accelerate": "1.16.0.dev0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 8.73,
+  "environment_differs": {
+   "torchvision": "0.29.2 (pinned 0.29.1)",
+   "numpy": "2.4.6 (pinned 2.3.5)",
+   "pillow": "13.0.0 (pinned 12.3.0)",
+   "accelerate": "1.16.0.dev0 (pinned 1.15.0)"
+  },
+  "platform": "cuda"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cuda",
+   "dtype": "float32",
+   "attn_implementation": "eager"
+  },
+  "predicted_tokens": [
+   312,
+   2356
+  ],
+  "predicted_text": [
+   "\u0120a",
+   "np"
+  ],
+  "next_token_logits": [
+   20.465246200561523,
+   14.459304809570312
+  ],
+  "training": {
+   "loss": 3.38765287399292,
+   "gradients": {
+    "transformer.h.0.attn.c_attn.weight": true,
+    "transformer.h.0.mlp.c_fc.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cuda/tiny_starcoder_py.json

@@ -0,0 +1,65 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/tiny_starcoder_py-mic",
+  "contract_name": "GPTBigCode",
+  "contract_revision": "f02c1112074b58d092f6590c8a957bdbe935dae4",
+  "framework": "transformers",
+  "variant": "default",
+  "target": "tiny_starcoder_py",
+  "kind": "checkpoint",
+  "revision": "8547527bef0bc927268c1653cce6948c5c242dd1",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cu130",
+   "torchvision": "0.29.2",
+   "torchaudio": "2.11.0+cu130",
+   "tokenizers": "0.23.3",
+   "numpy": "2.4.6",
+   "pillow": "13.0.0",
+   "accelerate": "1.16.0.dev0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 7.59,
+  "environment_differs": {
+   "torchvision": "0.29.2 (pinned 0.29.1)",
+   "numpy": "2.4.6 (pinned 2.3.5)",
+   "pillow": "13.0.0 (pinned 12.3.0)",
+   "accelerate": "1.16.0.dev0 (pinned 1.15.0)"
+  },
+  "platform": "cuda"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cuda",
+   "dtype": "float32",
+   "attn_implementation": "sdpa"
+  },
+  "predicted_tokens": [
+   312,
+   2356
+  ],
+  "predicted_text": [
+   "\u0120a",
+   "np"
+  ],
+  "next_token_logits": [
+   20.465242385864258,
+   14.459308624267578
+  ],
+  "training": {
+   "loss": 3.34557843208313,
+   "gradients": {
+    "transformer.h.0.attn.c_attn.weight": true,
+    "transformer.h.0.mlp.c_fc.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/usage.md

@@ -0,0 +1,77 @@
+# GPTBigCode: causal language modeling with Transformers
+
+This page is the Transformers contract of the GPTBigCode MIC and also a usage example. [doc-builder](https://github.com/huggingface/doc-builder/blob/main/docs/runnable-code-blocks.md) renders the `runnable` blocks below as documentation, and its pytest plugin runs them as written against `bigcode/tiny_starcoder_py`. The MIC runner runs the same blocks against the slim fixture and the checkpoint in [`integration.json`](../../integration.json). Lines marked `# doc-builder: hide` only matter there: they swap in the pinned checkpoint or variant and print the result record.
+
+## Predict the next token
+
+```py runnable:causal_lm
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, set_seed
+import json  # doc-builder: hide
+import os  # doc-builder: hide
+
+model_id = "bigcode/tiny_starcoder_py"
+model_id = os.environ.get("MIC_MODEL", model_id)  # doc-builder: hide
+variant = json.loads(os.environ.get("MIC_MODEL_KWARGS", "{}"))  # doc-builder: hide
+device = "cuda" if torch.cuda.is_available() else "cpu"
+set_seed(0)  # seeds torch, NumPy, and Python: loading and dropout draw random numbers
+
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+# This checkpoint defines no pad token; reuse the end-of-text token and pad on the left so the last position is a real token.
+tokenizer.pad_token = tokenizer.eos_token
+tokenizer.padding_side = "left"
+model = AutoModelForCausalLM.from_pretrained(model_id).to(device)
+if variant:  # doc-builder: hide
+    del model  # doc-builder: hide
+    model = AutoModelForCausalLM.from_pretrained(model_id, **variant).to(device)  # doc-builder: hide
+
+prompts = [
+    "def add(a, b):\n    return a + b\n\ndef sub(a, b):\n    return a - b\n\ndef mul(a, b):\n    return",
+    "import numpy as np\n\ndef mean(x):\n    return np.",
+]
+inputs = tokenizer(prompts, return_tensors="pt", padding=True).to(device)
+
+with torch.no_grad():
+    logits = model(**inputs).logits
+# Left padding: the last position is the final real token of each prompt.
+next_logits = logits[:, -1, :]
+predicted = next_token_logits = next_token_logits if False else next_logits.argmax(dim=-1)  # doc-builder: hide
+predicted = next_logits.argmax(dim=-1)
+print(tokenizer.convert_ids_to_tokens(predicted))
+```
+
+## One training step
+
+```py runnable:causal_lm:2
+# Language-model labels: copy the input ids and mask padded positions.
+labels = inputs["input_ids"].clone()
+labels[labels == tokenizer.pad_token_id] = -100
+
+model.train()
+optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
+loss = model(**inputs, labels=labels).loss
+loss.backward()
+before = {n: p.detach().cpu().clone() for n, p in model.named_parameters() if p.requires_grad}  # doc-builder: hide
+optimizer.step()
+print(f"loss: {loss.item():.4f}")
+
+if os.environ.get("MIC_RUNNER"):  # doc-builder: hide
+    named = dict(model.named_parameters())
+    checked = ("transformer.h.0.attn.c_attn.weight", "transformer.h.0.mlp.c_fc.weight")
+    result = {
+        "runtime": {
+            "device": model.device.type,
+            "dtype": str(model.dtype).removeprefix("torch."),
+            "attn_implementation": model.config._attn_implementation,
+        },
+        "predicted_tokens": predicted.tolist(),
+        "predicted_text": tokenizer.convert_ids_to_tokens(predicted),
+        "next_token_logits": next_logits.gather(1, predicted.unsqueeze(-1)).squeeze(-1).float().tolist(),
+        "training": {
+            "loss": loss.item(),
+            "gradients": {n: bool(named[n].grad is not None and torch.count_nonzero(named[n].grad) > 0 and torch.isfinite(named[n].grad).all()) for n in checked},
+            "updated": any(not torch.equal(p.detach().cpu(), before[n]) for n, p in model.named_parameters() if n in before),
+        },
+    }
+    print(json.dumps(result))
+```

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cb730824fb0a3d7a02141afb42177445e7d8522e8cde09bbfb3c174d5ca2ed0
+size 14901064

special_tokens_map.json

@@ -0,0 +1,26 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<fim_prefix>",
+    "<fim_middle>",
+    "<fim_suffix>",
+    "<fim_pad>",
+    "<filename>",
+    "<gh_stars>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<empty_output>",
+    "<commit_before>",
+    "<commit_msg>",
+    "<commit_after>",
+    "<reponame>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

tokenizer_config.json

@@ -0,0 +1,30 @@
+{
+  "add_prefix_space": false,
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<fim_prefix>",
+    "<fim_middle>",
+    "<fim_suffix>",
+    "<fim_pad>",
+    "<filename>",
+    "<gh_stars>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<empty_output>",
+    "<commit_before>",
+    "<commit_msg>",
+    "<commit_after>",
+    "<reponame>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}