Publish MIC usage, slim fixture and reviewed Transformers expectations

.mic-repo.json

@@ -0,0 +1,7 @@
+{
+  "format": "mic-umbrella-v1",
+  "contract_files": [
+    "integration.json",
+    "integrations/transformers/usage.md"
+  ]
+}

README.md

@@ -0,0 +1,37 @@
+---
+tags:
+  - mic
+  - test-artifact
+base_model: FacebookAI/roberta-base
+---
+
+# roberta-base-mic
+
+Slim model weights, executable usage and framework expectations in one repository.
+
+- [Contract manifest](integration.json)
+- [Transformers usage](integrations/transformers/usage.md)
+- [Transformers expectations](integrations/transformers/expectations)
+- [Fixture provenance](build.json)
+
+## Contract
+
+### RoBERTa fill-mask
+
+Runnable usage and the framework expectations directory are linked above.
+
+## Slim model
+
+### Slim fixture: roberta-base-mic
+
+A 3.4M-parameter fixture for the [RoBERTa MIC](https://huggingface.co/huggingface-mic/roberta-base-mic), covering [`FacebookAI/roberta-base`](https://huggingface.co/FacebookAI/roberta-base). It is a test artifact, not a usable model, tagged `test-artifact` to keep it out of candidate-model search.
+
+- **Fixture key `05ec051e3d97`.** It covers every checkpoint whose processor files are byte-identical and whose config matches after shrinking.
+- **Same architecture, smaller config:** `RobertaForMaskedLM`, shrunk by the generic rule in [`sweep.py`](https://github.com/huggingface/transformers-mic/blob/main/experiments/sweep.py) (num_hidden_layers 2).
+- **Processor files copied byte for byte** from `FacebookAI/roberta-base`: `dict.txt`, `merges.txt`, `tokenizer.json`, `tokenizer_config.json`, `vocab.json`.
+- **Random weights (seed 17).** They passed the input-sensitivity gate: every perturbation of the contract's inputs moved the logits by at least 0.001 (weakest: `attention_mask:all_ones`, 0.0024), so no training was needed.
+- **Same tensor names as the real checkpoint.** The real checkpoint has every parameter, and so does the fixture.
+- **Saved in float32**, the dtype `FacebookAI/roberta-base` loads in by default.
+- **Finite in float32, bfloat16, and float16** (forward, loss, gradients).
+
+Built by [`build_fixtures.py`](https://github.com/huggingface/transformers-mic/blob/main/example/build_fixtures.py); [`build.json`](build.json) records the pinned sources, gate results, and file hashes. This repository includes the slim weights, usage and framework expectations; rebuild the other files with `python example/build_fixtures.py --fixture roberta-base-mic`.

build.json

@@ -0,0 +1,126 @@
+{
+ "fixture": "huggingface-mic/roberta-base-mic",
+ "key": "05ec051e3d97c7210a49a8776fb18ac2156a51f1e08fedd0de7d1af05e8368f0",
+ "key_fields": {
+  "architectures": [
+   "RobertaForMaskedLM"
+  ],
+  "output_hidden_states": false,
+  "return_dict": true,
+  "chunk_size_feed_forward": 0,
+  "is_encoder_decoder": false,
+  "id2label": {
+   "0": "LABEL_0",
+   "1": "LABEL_1"
+  },
+  "label2id": {
+   "LABEL_0": 0,
+   "LABEL_1": 1
+  },
+  "problem_type": null,
+  "vocab_size": 50265,
+  "hidden_size": 64,
+  "num_hidden_layers": 2,
+  "num_attention_heads": 4,
+  "intermediate_size": 128,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "attention_probs_dropout_prob": 0.1,
+  "max_position_embeddings": 514,
+  "type_vocab_size": 1,
+  "initializer_range": 0.02,
+  "layer_norm_eps": 1e-05,
+  "pad_token_id": 1,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "use_cache": true,
+  "classifier_dropout": null,
+  "is_decoder": false,
+  "add_cross_attention": false,
+  "tie_word_embeddings": true,
+  "model_type": "roberta",
+  "output_attentions": false
+ },
+ "covers": {
+  "FacebookAI/roberta-base": "e2da8e2f811d1448a5b465c236feacd80ffbac7b"
+ },
+ "built_from": {
+  "repo": "FacebookAI/roberta-base",
+  "revision": "e2da8e2f811d1448a5b465c236feacd80ffbac7b"
+ },
+ "seed": 17,
+ "dtype": "float32",
+ "builder": "example/build_fixtures.py",
+ "model_class": "RobertaForMaskedLM",
+ "parameters": 3371545,
+ "missing_parameters": [],
+ "checkpoint_inspection": "parameter headers/indexes; bounded binary fallback",
+ "numeric_health_scope": "CPU float32/bfloat16/float16; autocast for low-precision losses",
+ "captured": {
+  "call": "training step",
+  "inputs": [
+   "attention_mask",
+   "input_ids",
+   "labels"
+  ],
+  "class": "RobertaForMaskedLM"
+ },
+ "weights": "random",
+ "gate": {
+  "random": {
+   "pass": true,
+   "accepted_insensitive": {},
+   "threshold": 0.001,
+   "deltas": {
+    "input_ids:roll_batch": 0.6873351335525513,
+    "input_ids:reverse": 0.8940852284431458,
+    "attention_mask:all_ones": 0.002395316958427429
+   },
+   "weakest": "attention_mask:all_ones",
+   "no_perturbation_defined": []
+  }
+ },
+ "numeric_health": {
+  "float32": {
+   "outputs_finite": true,
+   "loss_finite": true,
+   "gradients_finite": true,
+   "finite": true
+  },
+  "bfloat16": {
+   "outputs_finite": true,
+   "loss_finite": true,
+   "gradients_finite": true,
+   "finite": true
+  },
+  "float16": {
+   "outputs_finite": true,
+   "loss_finite": true,
+   "unscaled_gradients_finite": true,
+   "finite": true
+  }
+ },
+ "copied_unchanged": [
+  "dict.txt",
+  "merges.txt",
+  "tokenizer.json",
+  "tokenizer_config.json",
+  "vocab.json"
+ ],
+ "environment": {
+  "python": "3.12.12",
+  "machine": "arm64",
+  "torch": "2.14.1",
+  "transformers": "5.22.0.dev0",
+  "tokenizers": "0.23.3"
+ },
+ "sha256": {
+  "config.json": "830930b5ae6b215a3ec3b24228c61179c7965a1a519bb3abed82630a6c3ad06c",
+  "dict.txt": "2c31c5b236661067fe8626775eb995ef5fe808ac87d2ad4a6ddbef68550f5cc6",
+  "merges.txt": "1ce1664773c50f3e0cc8842619a93edc4624525b728b188a9e0be33b7726adc5",
+  "model.safetensors": "039fe2a956ec50b3d5fe667490873e9d1697b906fa0d9f82cf5a727c53c97bbf",
+  "tokenizer.json": "847bbeab6174d66a88898f729d52fa8d355fafe1bea101cf960dd404581df70e",
+  "tokenizer_config.json": "994f46754c5bf4014f1aa92d34b1374319c3a6b3f702105cd5b742beaecd18ce",
+  "vocab.json": "9e7f63c2d15d666b52e21d250d2e513b87c9b713cfa6987a82ed89e5e6e50655"
+ }
+}

config.json

@@ -0,0 +1,28 @@
+{
+  "add_cross_attention": false,
+  "architectures": [
+    "RobertaForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 64,
+  "initializer_range": 0.02,
+  "intermediate_size": 128,
+  "is_decoder": false,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 2,
+  "pad_token_id": 1,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.22.0.dev0",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 50265
+}

integration.json

@@ -0,0 +1,83 @@
+{
+  "mic": "0.3",
+  "name": "RoBERTa fill-mask",
+  "usage": "AutoTokenizer + AutoModelForMaskedLM, batched, right padding, masked-token probing and one SGD step",
+  "architecture": "RobertaForMaskedLM",
+  "checkpoints": {
+    "FacebookAI/roberta-base": {
+      "fixture": "huggingface-mic/roberta-base-mic"
+    }
+  },
+  "fixtures": {
+    "huggingface-mic/roberta-base-mic": {
+      "built_from": "FacebookAI/roberta-base",
+      "covers": [
+        "FacebookAI/roberta-base"
+      ],
+      "build": {
+        "seed": 17
+      }
+    }
+  },
+  "integrations": {
+    "transformers": {
+      "entry_point": "integrations/transformers/usage.md",
+      "format": "runnable-markdown",
+      "requires": [
+        "transformers>=5",
+        "torch"
+      ],
+      "targets": [
+        "fixtures",
+        "checkpoints"
+      ],
+      "variants": {
+        "default": {},
+        "eager": {
+          "attn_implementation": "eager"
+        },
+        "bfloat16": {
+          "dtype": "bfloat16"
+        }
+      },
+      "training": true,
+      "resources": {
+        "fixture": {
+          "memory_gb": 1,
+          "download_mb": 20
+        },
+        "FacebookAI/roberta-base": {
+          "memory_gb": 3,
+          "download_mb": 500
+        }
+      }
+    }
+  },
+  "outputs": {
+    "runtime": {
+      "compare": "exact"
+    },
+    "predicted_tokens": {
+      "compare": "exact"
+    },
+    "predicted_text": {
+      "compare": "exact"
+    },
+    "answer_logits": {
+      "compare": "allclose",
+      "atol": 0.0001,
+      "rtol": 0.0001
+    },
+    "training.loss": {
+      "compare": "allclose",
+      "atol": 0.0001,
+      "rtol": 0.0001
+    },
+    "training.gradients": {
+      "compare": "all_true"
+    },
+    "training.updated": {
+      "compare": "true"
+    }
+  }
+}

integrations/transformers/expectations/cpu-x86_64-bf16/roberta-base-mic.bfloat16.json

@@ -0,0 +1,90 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/roberta-base-mic",
+  "contract_name": "RoBERTa",
+  "contract_revision": "e97ac109af90cfd8a1bced0870753c342658a702",
+  "framework": "transformers",
+  "variant": "bfloat16",
+  "target": "roberta-base-mic",
+  "kind": "fixture",
+  "revision": "e97ac109af90cfd8a1bced0870753c342658a702",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cpu",
+   "torchvision": "0.29.2+cpu",
+   "torchaudio": "2.11.1+cpu",
+   "tokenizers": "0.23.3",
+   "numpy": "2.3.5",
+   "pillow": "13.0.0",
+   "accelerate": "1.15.0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 7.3,
+  "environment_differs": {
+   "torchvision": "0.29.2+cpu (pinned 0.29.1)",
+   "torchaudio": "2.11.1+cpu (pinned 2.11.0)",
+   "pillow": "13.0.0 (pinned 12.3.0)"
+  },
+  "platform": "cpu-x86_64-bf16"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cpu",
+   "dtype": "bfloat16",
+   "attn_implementation": "sdpa"
+  },
+  "predicted_tokens": [
+   33326,
+   33326,
+   33326,
+   5069
+  ],
+  "predicted_text": [
+   "teenth",
+   "teenth",
+   "teenth",
+   "ains"
+  ],
+  "answer_logits": [
+   [
+    -0.279296875,
+    0.0712890625,
+    0.0712890625,
+    0.031005859375
+   ],
+   [
+    -0.279296875,
+    0.0703125,
+    0.0703125,
+    0.031494140625
+   ],
+   [
+    -0.279296875,
+    0.0693359375,
+    0.0693359375,
+    0.0302734375
+   ],
+   [
+    -0.166015625,
+    0.037841796875,
+    0.037841796875,
+    0.0150146484375
+   ]
+  ],
+  "training": {
+   "loss": 10.8125,
+   "gradients": {
+    "roberta.encoder.layer.0.attention.self.query.weight": true,
+    "roberta.encoder.layer.0.intermediate.dense.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cpu-x86_64-bf16/roberta-base.bfloat16.json

@@ -0,0 +1,90 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/roberta-base-mic",
+  "contract_name": "RoBERTa",
+  "contract_revision": "e97ac109af90cfd8a1bced0870753c342658a702",
+  "framework": "transformers",
+  "variant": "bfloat16",
+  "target": "roberta-base",
+  "kind": "checkpoint",
+  "revision": "e2da8e2f811d1448a5b465c236feacd80ffbac7b",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cpu",
+   "torchvision": "0.29.2+cpu",
+   "torchaudio": "2.11.1+cpu",
+   "tokenizers": "0.23.3",
+   "numpy": "2.3.5",
+   "pillow": "13.0.0",
+   "accelerate": "1.15.0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 7.76,
+  "environment_differs": {
+   "torchvision": "0.29.2+cpu (pinned 0.29.1)",
+   "torchaudio": "2.11.1+cpu (pinned 2.11.0)",
+   "pillow": "13.0.0 (pinned 12.3.0)"
+  },
+  "platform": "cpu-x86_64-bf16"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cpu",
+   "dtype": "bfloat16",
+   "attn_implementation": "sdpa"
+  },
+  "predicted_tokens": [
+   2201,
+   5459,
+   8947,
+   928
+  ],
+  "predicted_text": [
+   "\u0120Paris",
+   "\u0120Berlin",
+   "\u0120Rome",
+   "\u0120London"
+  ],
+  "answer_logits": [
+   [
+    13.3125,
+    -4.53125,
+    -4.53125,
+    4.1875
+   ],
+   [
+    5.65625,
+    -4.21875,
+    -4.21875,
+    2.90625
+   ],
+   [
+    5.1875,
+    -4.375,
+    -4.375,
+    2.640625
+   ],
+   [
+    7.59375,
+    -4.9375,
+    -4.9375,
+    13.75
+   ]
+  ],
+  "training": {
+   "loss": 17.0,
+   "gradients": {
+    "roberta.encoder.layer.0.attention.self.query.weight": true,
+    "roberta.encoder.layer.0.intermediate.dense.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cpu-x86_64/roberta-base-mic.eager.json

@@ -0,0 +1,90 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/roberta-base-mic",
+  "contract_name": "RoBERTa",
+  "contract_revision": "e97ac109af90cfd8a1bced0870753c342658a702",
+  "framework": "transformers",
+  "variant": "eager",
+  "target": "roberta-base-mic",
+  "kind": "fixture",
+  "revision": "e97ac109af90cfd8a1bced0870753c342658a702",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cpu",
+   "torchvision": "0.29.2+cpu",
+   "torchaudio": "2.11.1+cpu",
+   "tokenizers": "0.23.3",
+   "numpy": "2.3.5",
+   "pillow": "13.0.0",
+   "accelerate": "1.15.0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 7.31,
+  "environment_differs": {
+   "torchvision": "0.29.2+cpu (pinned 0.29.1)",
+   "torchaudio": "2.11.1+cpu (pinned 2.11.0)",
+   "pillow": "13.0.0 (pinned 12.3.0)"
+  },
+  "platform": "cpu-x86_64"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cpu",
+   "dtype": "float32",
+   "attn_implementation": "eager"
+  },
+  "predicted_tokens": [
+   33326,
+   33326,
+   33326,
+   5069
+  ],
+  "predicted_text": [
+   "teenth",
+   "teenth",
+   "teenth",
+   "ains"
+  ],
+  "answer_logits": [
+   [
+    -0.2789422869682312,
+    0.07029282301664352,
+    0.07029282301664352,
+    0.031082605943083763
+   ],
+   [
+    -0.27898332476615906,
+    0.07068226486444473,
+    0.07068226486444473,
+    0.030745984986424446
+   ],
+   [
+    -0.2788700461387634,
+    0.07043246179819107,
+    0.07043246179819107,
+    0.03071228601038456
+   ],
+   [
+    -0.1644011288881302,
+    0.03672807663679123,
+    0.03672807663679123,
+    0.014085066504776478
+   ]
+  ],
+  "training": {
+   "loss": 10.869681358337402,
+   "gradients": {
+    "roberta.encoder.layer.0.attention.self.query.weight": true,
+    "roberta.encoder.layer.0.intermediate.dense.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cpu-x86_64/roberta-base-mic.json

@@ -0,0 +1,90 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/roberta-base-mic",
+  "contract_name": "RoBERTa",
+  "contract_revision": "e97ac109af90cfd8a1bced0870753c342658a702",
+  "framework": "transformers",
+  "variant": "default",
+  "target": "roberta-base-mic",
+  "kind": "fixture",
+  "revision": "e97ac109af90cfd8a1bced0870753c342658a702",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cpu",
+   "torchvision": "0.29.2+cpu",
+   "torchaudio": "2.11.1+cpu",
+   "tokenizers": "0.23.3",
+   "numpy": "2.3.5",
+   "pillow": "13.0.0",
+   "accelerate": "1.15.0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 7.32,
+  "environment_differs": {
+   "torchvision": "0.29.2+cpu (pinned 0.29.1)",
+   "torchaudio": "2.11.1+cpu (pinned 2.11.0)",
+   "pillow": "13.0.0 (pinned 12.3.0)"
+  },
+  "platform": "cpu-x86_64"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cpu",
+   "dtype": "float32",
+   "attn_implementation": "sdpa"
+  },
+  "predicted_tokens": [
+   33326,
+   33326,
+   33326,
+   5069
+  ],
+  "predicted_text": [
+   "teenth",
+   "teenth",
+   "teenth",
+   "ains"
+  ],
+  "answer_logits": [
+   [
+    -0.2789422571659088,
+    0.07029280811548233,
+    0.07029280811548233,
+    0.031082598492503166
+   ],
+   [
+    -0.27898332476615906,
+    0.07068228721618652,
+    0.07068228721618652,
+    0.030745979398489
+   ],
+   [
+    -0.27887001633644104,
+    0.07043246924877167,
+    0.07043246924877167,
+    0.030712297186255455
+   ],
+   [
+    -0.164401113986969,
+    0.036728061735630035,
+    0.036728061735630035,
+    0.014085096307098866
+   ]
+  ],
+  "training": {
+   "loss": 10.869681358337402,
+   "gradients": {
+    "roberta.encoder.layer.0.attention.self.query.weight": true,
+    "roberta.encoder.layer.0.intermediate.dense.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cpu-x86_64/roberta-base.eager.json

@@ -0,0 +1,90 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/roberta-base-mic",
+  "contract_name": "RoBERTa",
+  "contract_revision": "e97ac109af90cfd8a1bced0870753c342658a702",
+  "framework": "transformers",
+  "variant": "eager",
+  "target": "roberta-base",
+  "kind": "checkpoint",
+  "revision": "e2da8e2f811d1448a5b465c236feacd80ffbac7b",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cpu",
+   "torchvision": "0.29.2+cpu",
+   "torchaudio": "2.11.1+cpu",
+   "tokenizers": "0.23.3",
+   "numpy": "2.3.5",
+   "pillow": "13.0.0",
+   "accelerate": "1.15.0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 8.06,
+  "environment_differs": {
+   "torchvision": "0.29.2+cpu (pinned 0.29.1)",
+   "torchaudio": "2.11.1+cpu (pinned 2.11.0)",
+   "pillow": "13.0.0 (pinned 12.3.0)"
+  },
+  "platform": "cpu-x86_64"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cpu",
+   "dtype": "float32",
+   "attn_implementation": "eager"
+  },
+  "predicted_tokens": [
+   2201,
+   5459,
+   8947,
+   928
+  ],
+  "predicted_text": [
+   "\u0120Paris",
+   "\u0120Berlin",
+   "\u0120Rome",
+   "\u0120London"
+  ],
+  "answer_logits": [
+   [
+    13.223998069763184,
+    -4.527599334716797,
+    -4.527599334716797,
+    4.151352882385254
+   ],
+   [
+    5.728464603424072,
+    -4.235149383544922,
+    -4.235149383544922,
+    2.941866636276245
+   ],
+   [
+    5.227426528930664,
+    -4.355234146118164,
+    -4.355234146118164,
+    2.6994330883026123
+   ],
+   [
+    7.649934768676758,
+    -4.918882846832275,
+    -4.918882846832275,
+    13.74776554107666
+   ]
+  ],
+  "training": {
+   "loss": 16.974332809448242,
+   "gradients": {
+    "roberta.encoder.layer.0.attention.self.query.weight": true,
+    "roberta.encoder.layer.0.intermediate.dense.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cpu-x86_64/roberta-base.json

@@ -0,0 +1,90 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/roberta-base-mic",
+  "contract_name": "RoBERTa",
+  "contract_revision": "e97ac109af90cfd8a1bced0870753c342658a702",
+  "framework": "transformers",
+  "variant": "default",
+  "target": "roberta-base",
+  "kind": "checkpoint",
+  "revision": "e2da8e2f811d1448a5b465c236feacd80ffbac7b",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cpu",
+   "torchvision": "0.29.2+cpu",
+   "torchaudio": "2.11.1+cpu",
+   "tokenizers": "0.23.3",
+   "numpy": "2.3.5",
+   "pillow": "13.0.0",
+   "accelerate": "1.15.0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 8.01,
+  "environment_differs": {
+   "torchvision": "0.29.2+cpu (pinned 0.29.1)",
+   "torchaudio": "2.11.1+cpu (pinned 2.11.0)",
+   "pillow": "13.0.0 (pinned 12.3.0)"
+  },
+  "platform": "cpu-x86_64"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cpu",
+   "dtype": "float32",
+   "attn_implementation": "sdpa"
+  },
+  "predicted_tokens": [
+   2201,
+   5459,
+   8947,
+   928
+  ],
+  "predicted_text": [
+   "\u0120Paris",
+   "\u0120Berlin",
+   "\u0120Rome",
+   "\u0120London"
+  ],
+  "answer_logits": [
+   [
+    13.2239990234375,
+    -4.527599334716797,
+    -4.527599334716797,
+    4.151357650756836
+   ],
+   [
+    5.728463649749756,
+    -4.235146999359131,
+    -4.235146999359131,
+    2.9418599605560303
+   ],
+   [
+    5.227415561676025,
+    -4.3552350997924805,
+    -4.3552350997924805,
+    2.6994333267211914
+   ],
+   [
+    7.6499223709106445,
+    -4.918882846832275,
+    -4.918882846832275,
+    13.74775505065918
+   ]
+  ],
+  "training": {
+   "loss": 16.974342346191406,
+   "gradients": {
+    "roberta.encoder.layer.0.attention.self.query.weight": true,
+    "roberta.encoder.layer.0.intermediate.dense.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cuda/roberta-base-mic.bfloat16.json

@@ -0,0 +1,91 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/roberta-base-mic",
+  "contract_name": "RoBERTa",
+  "contract_revision": "e97ac109af90cfd8a1bced0870753c342658a702",
+  "framework": "transformers",
+  "variant": "bfloat16",
+  "target": "roberta-base-mic",
+  "kind": "fixture",
+  "revision": "e97ac109af90cfd8a1bced0870753c342658a702",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cu130",
+   "torchvision": "0.29.2",
+   "torchaudio": "2.11.0+cu130",
+   "tokenizers": "0.23.3",
+   "numpy": "2.4.6",
+   "pillow": "13.0.0",
+   "accelerate": "1.16.0.dev0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 7.15,
+  "environment_differs": {
+   "torchvision": "0.29.2 (pinned 0.29.1)",
+   "numpy": "2.4.6 (pinned 2.3.5)",
+   "pillow": "13.0.0 (pinned 12.3.0)",
+   "accelerate": "1.16.0.dev0 (pinned 1.15.0)"
+  },
+  "platform": "cuda"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cuda",
+   "dtype": "bfloat16",
+   "attn_implementation": "sdpa"
+  },
+  "predicted_tokens": [
+   33326,
+   33326,
+   33326,
+   5069
+  ],
+  "predicted_text": [
+   "teenth",
+   "teenth",
+   "teenth",
+   "ains"
+  ],
+  "answer_logits": [
+   [
+    -0.279296875,
+    0.07080078125,
+    0.07080078125,
+    0.0303955078125
+   ],
+   [
+    -0.279296875,
+    0.0703125,
+    0.0703125,
+    0.031494140625
+   ],
+   [
+    -0.279296875,
+    0.0693359375,
+    0.0693359375,
+    0.0302734375
+   ],
+   [
+    -0.166015625,
+    0.037841796875,
+    0.037841796875,
+    0.0150146484375
+   ]
+  ],
+  "training": {
+   "loss": 10.875,
+   "gradients": {
+    "roberta.encoder.layer.0.attention.self.query.weight": true,
+    "roberta.encoder.layer.0.intermediate.dense.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cuda/roberta-base-mic.eager.json

@@ -0,0 +1,91 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/roberta-base-mic",
+  "contract_name": "RoBERTa",
+  "contract_revision": "e97ac109af90cfd8a1bced0870753c342658a702",
+  "framework": "transformers",
+  "variant": "eager",
+  "target": "roberta-base-mic",
+  "kind": "fixture",
+  "revision": "e97ac109af90cfd8a1bced0870753c342658a702",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cu130",
+   "torchvision": "0.29.2",
+   "torchaudio": "2.11.0+cu130",
+   "tokenizers": "0.23.3",
+   "numpy": "2.4.6",
+   "pillow": "13.0.0",
+   "accelerate": "1.16.0.dev0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 7.39,
+  "environment_differs": {
+   "torchvision": "0.29.2 (pinned 0.29.1)",
+   "numpy": "2.4.6 (pinned 2.3.5)",
+   "pillow": "13.0.0 (pinned 12.3.0)",
+   "accelerate": "1.16.0.dev0 (pinned 1.15.0)"
+  },
+  "platform": "cuda"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cuda",
+   "dtype": "float32",
+   "attn_implementation": "eager"
+  },
+  "predicted_tokens": [
+   33326,
+   33326,
+   33326,
+   5069
+  ],
+  "predicted_text": [
+   "teenth",
+   "teenth",
+   "teenth",
+   "ains"
+  ],
+  "answer_logits": [
+   [
+    -0.2789422571659088,
+    0.07029278576374054,
+    0.07029278576374054,
+    0.031082633882761
+   ],
+   [
+    -0.27898329496383667,
+    0.07068229466676712,
+    0.07068229466676712,
+    0.03074600175023079
+   ],
+   [
+    -0.27887001633644104,
+    0.07043243199586868,
+    0.07043243199586868,
+    0.030712302774190903
+   ],
+   [
+    -0.164401113986969,
+    0.036728065460920334,
+    0.036728065460920334,
+    0.014085044153034687
+   ]
+  ],
+  "training": {
+   "loss": 10.831308364868164,
+   "gradients": {
+    "roberta.encoder.layer.0.attention.self.query.weight": true,
+    "roberta.encoder.layer.0.intermediate.dense.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cuda/roberta-base-mic.json

@@ -0,0 +1,91 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/roberta-base-mic",
+  "contract_name": "RoBERTa",
+  "contract_revision": "e97ac109af90cfd8a1bced0870753c342658a702",
+  "framework": "transformers",
+  "variant": "default",
+  "target": "roberta-base-mic",
+  "kind": "fixture",
+  "revision": "e97ac109af90cfd8a1bced0870753c342658a702",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cu130",
+   "torchvision": "0.29.2",
+   "torchaudio": "2.11.0+cu130",
+   "tokenizers": "0.23.3",
+   "numpy": "2.4.6",
+   "pillow": "13.0.0",
+   "accelerate": "1.16.0.dev0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 7.34,
+  "environment_differs": {
+   "torchvision": "0.29.2 (pinned 0.29.1)",
+   "numpy": "2.4.6 (pinned 2.3.5)",
+   "pillow": "13.0.0 (pinned 12.3.0)",
+   "accelerate": "1.16.0.dev0 (pinned 1.15.0)"
+  },
+  "platform": "cuda"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cuda",
+   "dtype": "float32",
+   "attn_implementation": "sdpa"
+  },
+  "predicted_tokens": [
+   33326,
+   33326,
+   33326,
+   5069
+  ],
+  "predicted_text": [
+   "teenth",
+   "teenth",
+   "teenth",
+   "ains"
+  ],
+  "answer_logits": [
+   [
+    -0.2789422869682312,
+    0.07029278576374054,
+    0.07029278576374054,
+    0.031082620844244957
+   ],
+   [
+    -0.27898329496383667,
+    0.07068225741386414,
+    0.07068225741386414,
+    0.030746012926101685
+   ],
+   [
+    -0.27887001633644104,
+    0.07043244689702988,
+    0.07043244689702988,
+    0.030712293460965157
+   ],
+   [
+    -0.164401113986969,
+    0.03672804683446884,
+    0.03672804683446884,
+    0.01408504880964756
+   ]
+  ],
+  "training": {
+   "loss": 10.831787109375,
+   "gradients": {
+    "roberta.encoder.layer.0.attention.self.query.weight": true,
+    "roberta.encoder.layer.0.intermediate.dense.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cuda/roberta-base.bfloat16.json

@@ -0,0 +1,91 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/roberta-base-mic",
+  "contract_name": "RoBERTa",
+  "contract_revision": "e97ac109af90cfd8a1bced0870753c342658a702",
+  "framework": "transformers",
+  "variant": "bfloat16",
+  "target": "roberta-base",
+  "kind": "checkpoint",
+  "revision": "e2da8e2f811d1448a5b465c236feacd80ffbac7b",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cu130",
+   "torchvision": "0.29.2",
+   "torchaudio": "2.11.0+cu130",
+   "tokenizers": "0.23.3",
+   "numpy": "2.4.6",
+   "pillow": "13.0.0",
+   "accelerate": "1.16.0.dev0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 7.62,
+  "environment_differs": {
+   "torchvision": "0.29.2 (pinned 0.29.1)",
+   "numpy": "2.4.6 (pinned 2.3.5)",
+   "pillow": "13.0.0 (pinned 12.3.0)",
+   "accelerate": "1.16.0.dev0 (pinned 1.15.0)"
+  },
+  "platform": "cuda"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cuda",
+   "dtype": "bfloat16",
+   "attn_implementation": "sdpa"
+  },
+  "predicted_tokens": [
+   2201,
+   5459,
+   8947,
+   928
+  ],
+  "predicted_text": [
+   "\u0120Paris",
+   "\u0120Berlin",
+   "\u0120Rome",
+   "\u0120London"
+  ],
+  "answer_logits": [
+   [
+    13.25,
+    -4.53125,
+    -4.53125,
+    4.1875
+   ],
+   [
+    5.65625,
+    -4.25,
+    -4.25,
+    2.90625
+   ],
+   [
+    5.1875,
+    -4.34375,
+    -4.34375,
+    2.625
+   ],
+   [
+    7.5,
+    -4.90625,
+    -4.90625,
+    13.625
+   ]
+  ],
+  "training": {
+   "loss": 15.75,
+   "gradients": {
+    "roberta.encoder.layer.0.attention.self.query.weight": true,
+    "roberta.encoder.layer.0.intermediate.dense.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cuda/roberta-base.eager.json

@@ -0,0 +1,91 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/roberta-base-mic",
+  "contract_name": "RoBERTa",
+  "contract_revision": "e97ac109af90cfd8a1bced0870753c342658a702",
+  "framework": "transformers",
+  "variant": "eager",
+  "target": "roberta-base",
+  "kind": "checkpoint",
+  "revision": "e2da8e2f811d1448a5b465c236feacd80ffbac7b",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cu130",
+   "torchvision": "0.29.2",
+   "torchaudio": "2.11.0+cu130",
+   "tokenizers": "0.23.3",
+   "numpy": "2.4.6",
+   "pillow": "13.0.0",
+   "accelerate": "1.16.0.dev0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 7.7,
+  "environment_differs": {
+   "torchvision": "0.29.2 (pinned 0.29.1)",
+   "numpy": "2.4.6 (pinned 2.3.5)",
+   "pillow": "13.0.0 (pinned 12.3.0)",
+   "accelerate": "1.16.0.dev0 (pinned 1.15.0)"
+  },
+  "platform": "cuda"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cuda",
+   "dtype": "float32",
+   "attn_implementation": "eager"
+  },
+  "predicted_tokens": [
+   2201,
+   5459,
+   8947,
+   928
+  ],
+  "predicted_text": [
+   "\u0120Paris",
+   "\u0120Berlin",
+   "\u0120Rome",
+   "\u0120London"
+  ],
+  "answer_logits": [
+   [
+    13.224016189575195,
+    -4.527595520019531,
+    -4.527595520019531,
+    4.151366233825684
+   ],
+   [
+    5.728461265563965,
+    -4.235146522521973,
+    -4.235146522521973,
+    2.9418551921844482
+   ],
+   [
+    5.227436542510986,
+    -4.3552350997924805,
+    -4.3552350997924805,
+    2.699455738067627
+   ],
+   [
+    7.649930953979492,
+    -4.918882369995117,
+    -4.918882369995117,
+    13.747785568237305
+   ]
+  ],
+  "training": {
+   "loss": 15.65516471862793,
+   "gradients": {
+    "roberta.encoder.layer.0.attention.self.query.weight": true,
+    "roberta.encoder.layer.0.intermediate.dense.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/expectations/cuda/roberta-base.json

@@ -0,0 +1,91 @@
+{
+ "recorded_with": {
+  "contract": "huggingface-mic/roberta-base-mic",
+  "contract_name": "RoBERTa",
+  "contract_revision": "e97ac109af90cfd8a1bced0870753c342658a702",
+  "framework": "transformers",
+  "variant": "default",
+  "target": "roberta-base",
+  "kind": "checkpoint",
+  "revision": "e2da8e2f811d1448a5b465c236feacd80ffbac7b",
+  "environment": {
+   "python": "3.11.18",
+   "transformers": "5.22.0.dev0",
+   "torch": "2.14.0+cu130",
+   "torchvision": "0.29.2",
+   "torchaudio": "2.11.0+cu130",
+   "tokenizers": "0.23.3",
+   "numpy": "2.4.6",
+   "pillow": "13.0.0",
+   "accelerate": "1.16.0.dev0",
+   "safetensors": "0.8.0",
+   "huggingface-hub": "2.3.0",
+   "sentencepiece": "0.2.2",
+   "protobuf": "7.36.2",
+   "librosa": "0.11.0",
+   "soundfile": "0.14.0",
+   "scipy": "1.17.1"
+  },
+  "seconds": 7.6,
+  "environment_differs": {
+   "torchvision": "0.29.2 (pinned 0.29.1)",
+   "numpy": "2.4.6 (pinned 2.3.5)",
+   "pillow": "13.0.0 (pinned 12.3.0)",
+   "accelerate": "1.16.0.dev0 (pinned 1.15.0)"
+  },
+  "platform": "cuda"
+ },
+ "outputs": {
+  "runtime": {
+   "device": "cuda",
+   "dtype": "float32",
+   "attn_implementation": "sdpa"
+  },
+  "predicted_tokens": [
+   2201,
+   5459,
+   8947,
+   928
+  ],
+  "predicted_text": [
+   "\u0120Paris",
+   "\u0120Berlin",
+   "\u0120Rome",
+   "\u0120London"
+  ],
+  "answer_logits": [
+   [
+    13.224020004272461,
+    -4.527596473693848,
+    -4.527596473693848,
+    4.151369094848633
+   ],
+   [
+    5.728462219238281,
+    -4.2351460456848145,
+    -4.2351460456848145,
+    2.9418606758117676
+   ],
+   [
+    5.227438449859619,
+    -4.3552327156066895,
+    -4.3552327156066895,
+    2.699451208114624
+   ],
+   [
+    7.6499409675598145,
+    -4.918882369995117,
+    -4.918882369995117,
+    13.747784614562988
+   ]
+  ],
+  "training": {
+   "loss": 14.964488983154297,
+   "gradients": {
+    "roberta.encoder.layer.0.attention.self.query.weight": true,
+    "roberta.encoder.layer.0.intermediate.dense.weight": true
+   },
+   "updated": true
+  }
+ }
+}

integrations/transformers/usage.md

@@ -0,0 +1,72 @@
+# RoBERTa: fill-mask with Transformers
+
+This page is the Transformers contract of the RoBERTa MIC and also a usage example. [doc-builder](https://github.com/huggingface/doc-builder/blob/main/docs/runnable-code-blocks.md) renders the `runnable` blocks below as documentation, and its pytest plugin runs them as written against `FacebookAI/roberta-base`. The MIC runner runs the same blocks against each slim fixture and each checkpoint in [`integration.json`](../../integration.json). Lines marked `# doc-builder: hide` only matter there: they swap in the pinned checkpoint or variant and print the result record.
+
+## Predict the masked word
+
+```py runnable:fill_mask
+import torch
+from transformers import AutoModelForMaskedLM, AutoTokenizer, set_seed
+import json  # doc-builder: hide
+import os  # doc-builder: hide
+
+model_id = "FacebookAI/roberta-base"
+model_id = os.environ.get("MIC_MODEL", model_id)  # doc-builder: hide
+variant = json.loads(os.environ.get("MIC_MODEL_KWARGS", "{}"))  # doc-builder: hide
+device = "cuda" if torch.cuda.is_available() else "cpu"
+set_seed(0)  # seeds torch, NumPy, and Python: loading, dropout, and data augmentation draw random numbers
+
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = AutoModelForMaskedLM.from_pretrained(model_id).to(device)
+if variant:  # doc-builder: hide
+    del model  # doc-builder: hide
+    model = AutoModelForMaskedLM.from_pretrained(model_id, **variant).to(device)  # doc-builder: hide
+
+countries = ["France", "Germany", "Italy", "the United Kingdom"]
+prompts = [f"The capital of {country} is {tokenizer.mask_token}." for country in countries]
+inputs = tokenizer(prompts, return_tensors="pt", padding=True)
+inputs = inputs.to(device)
+
+with torch.no_grad():
+    logits = model(**inputs).logits
+masked = inputs["input_ids"] == tokenizer.mask_token_id
+predicted = logits[masked].argmax(dim=-1)
+print(tokenizer.convert_ids_to_tokens(predicted))
+```
+
+## One training step
+
+```py runnable:fill_mask:2
+answers = ["Paris", "Berlin", "Rome", "London"]
+answer_ids = tokenizer.convert_tokens_to_ids(answers)
+labels = torch.full_like(inputs["input_ids"], -100)
+labels[masked] = torch.tensor(answer_ids, device=device)
+
+model.train()
+optimizer = torch.optim.SGD([p for p in model.parameters() if p.requires_grad], lr=0.01)
+loss = model(**inputs, labels=labels).loss
+loss.backward()
+before = {n: p.detach().cpu().clone() for n, p in model.named_parameters() if p.requires_grad}  # doc-builder: hide
+optimizer.step()
+print(f"loss: {loss.item():.4f}")
+
+if os.environ.get("MIC_RUNNER"):  # doc-builder: hide
+    named = dict(model.named_parameters())
+    checked = ("roberta.encoder.layer.0.attention.self.query.weight", "roberta.encoder.layer.0.intermediate.dense.weight")
+    result = {
+        "runtime": {
+            "device": model.device.type,
+            "dtype": str(model.dtype).removeprefix("torch."),
+            "attn_implementation": model.config._attn_implementation,
+        },
+        "predicted_tokens": predicted.tolist(),
+        "predicted_text": tokenizer.convert_ids_to_tokens(predicted),
+        "answer_logits": logits[masked][:, answer_ids].float().tolist(),
+        "training": {
+            "loss": loss.item(),
+            "gradients": {n: bool(named[n].grad is not None and torch.count_nonzero(named[n].grad) > 0 and torch.isfinite(named[n].grad).all()) for n in checked},
+            "updated": any(not torch.equal(p.detach().cpu(), before[n]) for n, p in model.named_parameters() if n in before),
+        },
+    }
+    print(json.dumps(result))
+```

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:039fe2a956ec50b3d5fe667490873e9d1697b906fa0d9f82cf5a727c53c97bbf
+size 13490924

tokenizer_config.json

@@ -0,0 +1 @@
+{"model_max_length": 512}