Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Writing model tests","local":"writing-model-tests","sections":[{"title":"Write tests for a causal language model","local":"write-tests-for-a-causal-language-model","sections":[{"title":"Overriding defaults","local":"overriding-defaults","sections":[],"depth":3}],"depth":2},{"title":"Write tests for a vision-language model","local":"write-tests-for-a-vision-language-model","sections":[{"title":"Overriding defaults","local":"overriding-defaults","sections":[],"depth":3}],"depth":2},{"title":"Write tests for other architectures","local":"write-tests-for-other-architectures","sections":[{"title":"ModelTester and ModelTest","local":"modeltester-and-modeltest","sections":[],"depth":3},{"title":"Test mixins","local":"test-mixins","sections":[],"depth":3},{"title":"Writing a model test","local":"writing-a-model-test","sections":[],"depth":3},{"title":"File organization","local":"file-organization","sections":[],"depth":3}],"depth":2},{"title":"Config tests","local":"config-tests","sections":[],"depth":2},{"title":"Integration tests and tiny models","local":"integration-tests-and-tiny-models","sections":[{"title":"Writing integration tests","local":"writing-integration-tests","sections":[{"title":"Generation integration tests","local":"generation-integration-tests","sections":[],"depth":4},{"title":"Hardware-specific expectations","local":"hardware-specific-expectations","sections":[],"depth":4}],"depth":3},{"title":"Creating tiny models","local":"creating-tiny-models","sections":[],"depth":3}],"depth":2},{"title":"Control what gets tested","local":"control-what-gets-tested","sections":[],"depth":2},{"title":"Next steps","local":"next-steps","sections":[],"depth":2}],"depth":1}"> | |
| <link href="/docs/transformers/pr_36895/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/entry/start.b88698fb.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/scheduler.31fdf58d.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/singletons.18caffe4.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/index.252883d5.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/paths.a82d371a.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/entry/app.ac300a2c.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/preload-helper.643b8c40.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/index.2f76fdf0.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/nodes/0.1669b2ea.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/each.e59479a4.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/nodes/677.5a97d089.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/CopyLLMTxtMenu.9cbc081e.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.945f6700.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/IconCopy.ac192424.js"> | |
| <link rel="modulepreload" href="/docs/transformers/pr_36895/en/_app/immutable/chunks/CodeBlock.e52df5d6.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Writing model tests","local":"writing-model-tests","sections":[{"title":"Write tests for a causal language model","local":"write-tests-for-a-causal-language-model","sections":[{"title":"Overriding defaults","local":"overriding-defaults","sections":[],"depth":3}],"depth":2},{"title":"Write tests for a vision-language model","local":"write-tests-for-a-vision-language-model","sections":[{"title":"Overriding defaults","local":"overriding-defaults","sections":[],"depth":3}],"depth":2},{"title":"Write tests for other architectures","local":"write-tests-for-other-architectures","sections":[{"title":"ModelTester and ModelTest","local":"modeltester-and-modeltest","sections":[],"depth":3},{"title":"Test mixins","local":"test-mixins","sections":[],"depth":3},{"title":"Writing a model test","local":"writing-a-model-test","sections":[],"depth":3},{"title":"File organization","local":"file-organization","sections":[],"depth":3}],"depth":2},{"title":"Config tests","local":"config-tests","sections":[],"depth":2},{"title":"Integration tests and tiny models","local":"integration-tests-and-tiny-models","sections":[{"title":"Writing integration tests","local":"writing-integration-tests","sections":[{"title":"Generation integration tests","local":"generation-integration-tests","sections":[],"depth":4},{"title":"Hardware-specific expectations","local":"hardware-specific-expectations","sections":[],"depth":4}],"depth":3},{"title":"Creating tiny models","local":"creating-tiny-models","sections":[],"depth":3}],"depth":2},{"title":"Control what gets tested","local":"control-what-gets-tested","sections":[],"depth":2},{"title":"Next steps","local":"next-steps","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="writing-model-tests" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#writing-model-tests"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Writing model tests</span></h1> <p data-svelte-h="svelte-1z0b4eo">The Transformers test suite uses a mixin-based architecture to auto-generate 100+ tests from minimal code. You write a small amount of model-specific code, and the mixins handle save/load, generation, pipelines, training, and tensor parallelism.</p> <p data-svelte-h="svelte-1poyl94">Run your model’s tests with the following commands.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START --><span class="hljs-comment"># run your model's tests</span> | |
| pytest tests/models/mymodel/test_modeling_mymodel.py -v | |
| <span class="hljs-comment"># run a specific test</span> | |
| pytest tests/models/mymodel/test_modeling_mymodel.py::MyModelTest::test_model | |
| <span class="hljs-comment"># run tests matching a keyword pattern (useful to run all integration tests)</span> | |
| pytest tests/models/mymodel/ -k integration -v | |
| <span class="hljs-comment"># include slow integration tests</span> | |
| RUN_SLOW=1 pytest tests/models/mymodel/ -v<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1swn1at">The Hugging Face CI runs model tests without <code>@slow</code> on every pull request, and slow tests run on a nightly schedule (see <a href="./pr_checks">Pull request checks</a> for what the CI validates).</p> <h2 class="relative group"><a id="write-tests-for-a-causal-language-model" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#write-tests-for-a-causal-language-model"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Write tests for a causal language model</span></h2> <p data-svelte-h="svelte-1bfh7an"><code>CausalLMModelTest</code> is the recommended base class for testing causal language models. It inherits from five <a href="#test-mixins">test mixins</a> and auto-generates tests for save/load, generation, pipelines, training, and tensor parallelism.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> unittest | |
| <span class="hljs-keyword">from</span> transformers.testing_utils <span class="hljs-keyword">import</span> require_torch | |
| <span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> is_torch_available | |
| <span class="hljs-keyword">from</span> ...causal_lm_tester <span class="hljs-keyword">import</span> CausalLMModelTest, CausalLMModelTester | |
| <span class="hljs-keyword">if</span> is_torch_available(): | |
| <span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> MyModel | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">MyModelTester</span>(<span class="hljs-title class_ inherited__">CausalLMModelTester</span>): | |
| <span class="hljs-keyword">if</span> is_torch_available(): | |
| base_model_class = MyModel | |
| <span class="hljs-meta">@require_torch</span> | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">MyModelTest</span>(CausalLMModelTest, unittest.TestCase): | |
| model_tester_class = MyModelTester<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-9wgobk">These two classes give full test coverage for <code>MyModel</code> and all its head classes (<code>MyModelForCausalLM</code>, <code>MyModelForSequenceClassification</code>, etc.). See <a href="https://github.com/huggingface/transformers/blob/main/tests/models/llama/test_modeling_llama.py" rel="nofollow">tests/models/llama/test_modeling_llama.py</a> for a real example.</p> <p data-svelte-h="svelte-1ljr5at"><code>CausalLMModelTester</code> only requires <code>base_model_class</code>. The tester strips the <code>Model</code> suffix to get a base name (<code>LlamaModel</code> becomes <code>Llama</code>), then appends suffixes like <code>Config</code> or <code>ForCausalLM</code> to discover related classes. If a class doesn’t exist in the module, the attribute stays <code>None</code> and the tester skips the corresponding tests.</p> <h3 class="relative group"><a id="overriding-defaults" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#overriding-defaults"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Overriding defaults</span></h3> <p data-svelte-h="svelte-rwyl9w">If your model doesn’t follow standard naming, or you need to customize behavior, override attributes on the tester or test class.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">class</span> <span class="hljs-title class_">MyModelTester</span>(<span class="hljs-title class_ inherited__">CausalLMModelTester</span>): | |
| <span class="hljs-keyword">if</span> is_torch_available(): | |
| base_model_class = MyModel | |
| <span class="hljs-comment"># override if the class name doesn't follow the convention</span> | |
| causal_lm_class = MyCustomCausalLM | |
| <span class="hljs-meta">@require_torch</span> | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">MyModelTest</span>(CausalLMModelTest, unittest.TestCase): | |
| model_tester_class = MyModelTester | |
| <span class="hljs-comment"># disable embedding resize tests</span> | |
| test_resize_embeddings = <span class="hljs-literal">False</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1modc6r">For models that need custom constructor parameters on the tester, override <code>__init__</code> and call <code>super().__init__(parent=parent)</code> before setting extra attributes. See <a href="https://github.com/huggingface/transformers/blob/main/tests/models/youtu/test_modeling_youtu.py" rel="nofollow">tests/models/youtu/test_modeling_youtu.py</a> for a real example.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">class</span> <span class="hljs-title class_">YoutuModelTester</span>(<span class="hljs-title class_ inherited__">CausalLMModelTester</span>): | |
| <span class="hljs-keyword">if</span> is_torch_available(): | |
| base_model_class = YoutuModel | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, parent, kv_lora_rank=<span class="hljs-number">16</span>, q_lora_rank=<span class="hljs-number">32</span></span>): | |
| <span class="hljs-built_in">super</span>().__init__(parent=parent) | |
| self.kv_lora_rank = kv_lora_rank | |
| self.q_lora_rank = q_lora_rank<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="write-tests-for-a-vision-language-model" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#write-tests-for-a-vision-language-model"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Write tests for a vision-language model</span></h2> <p data-svelte-h="svelte-1hs20kk"><code>VLMModelTest</code> is the base class for vision-language models. It inherits from three mixins (<code>ModelTesterMixin</code>, <code>GenerationTesterMixin</code>, <code>PipelineTesterMixin</code>) and sets <code>_is_composite = True</code> to handle multiple sub-models.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> unittest | |
| <span class="hljs-keyword">from</span> transformers.testing_utils <span class="hljs-keyword">import</span> require_torch | |
| <span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> is_torch_available | |
| <span class="hljs-keyword">from</span> ...vlm_tester <span class="hljs-keyword">import</span> VLMModelTest, VLMModelTester | |
| <span class="hljs-keyword">if</span> is_torch_available(): | |
| <span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> ( | |
| MyVLMConfig, | |
| MyVLMModel, | |
| MyVLMTextConfig, | |
| MyVLMVisionConfig, | |
| MyVLMForConditionalGeneration, | |
| ) | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">MyVLMTester</span>(<span class="hljs-title class_ inherited__">VLMModelTester</span>): | |
| <span class="hljs-keyword">if</span> is_torch_available(): | |
| base_model_class = MyVLMModel | |
| config_class = MyVLMConfig | |
| text_config_class = MyVLMTextConfig | |
| vision_config_class = MyVLMVisionConfig | |
| conditional_generation_class = MyVLMForConditionalGeneration | |
| <span class="hljs-meta">@require_torch</span> | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">MyVLMTest</span>(VLMModelTest, unittest.TestCase): | |
| model_tester_class = MyVLMTester<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="overriding-defaults" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#overriding-defaults"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Overriding defaults</span></h3> <p data-svelte-h="svelte-18lphm9">When the VLM needs custom vision parameters or non-default config values, override <code>__init__</code>. Set defaults with <code>setdefault</code> before calling <code>super().__init__(parent, **kwargs)</code>. The example below shows the first few defaults from <a href="https://github.com/huggingface/transformers/blob/main/tests/models/qianfan_ocr/test_modeling_qianfan_ocr.py" rel="nofollow">tests/models/qianfan_ocr/test_modeling_qianfan_ocr.py</a>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">class</span> <span class="hljs-title class_">QianfanOCRVisionText2TextModelTester</span>(<span class="hljs-title class_ inherited__">VLMModelTester</span>): | |
| base_model_class = QianfanOCRModel | |
| config_class = QianfanOCRConfig | |
| text_config_class = Qwen3Config | |
| vision_config_class = QianfanOCRVisionConfig | |
| conditional_generation_class = QianfanOCRForConditionalGeneration | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, parent, **kwargs</span>): | |
| kwargs.setdefault(<span class="hljs-string">"image_token_id"</span>, <span class="hljs-number">1</span>) | |
| kwargs.setdefault(<span class="hljs-string">"image_size"</span>, <span class="hljs-number">32</span>) | |
| kwargs.setdefault(<span class="hljs-string">"patch_size"</span>, <span class="hljs-number">4</span>) | |
| kwargs.setdefault(<span class="hljs-string">"num_channels"</span>, <span class="hljs-number">3</span>) | |
| <span class="hljs-comment"># ... more defaults</span> | |
| <span class="hljs-built_in">super</span>().__init__(parent, **kwargs)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-gujj92">VLM tests differ from <code>CausalLMModelTest</code> in a few ways.</p> <ul data-svelte-h="svelte-1qz3lxi"><li>You must set <code>config_class</code>, <code>text_config_class</code>, <code>vision_config_class</code>, and <code>conditional_generation_class</code> on the tester.</li> <li><code>VLMModelTest</code> doesn’t include <code>TrainingTesterMixin</code> or <code>TensorParallelTesterMixin</code>.</li> <li>The tester’s <code>__init__</code> accepts vision parameters (<code>image_size</code>, <code>patch_size</code>, <code>num_channels</code>, <code>num_image_tokens</code>) from <code>**kwargs</code> and <code>setdefault()</code>.</li> <li><code>ConfigTester</code> uses <code>has_text_modality=False</code> because the top-level config is a composite config rather than a text model config.</li></ul> <h2 class="relative group"><a id="write-tests-for-other-architectures" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#write-tests-for-other-architectures"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Write tests for other architectures</span></h2> <p data-svelte-h="svelte-1vvwhcg">For encoder-only, encoder-decoder, audio, or other non-standard architectures, build the test infrastructure directly from the two-class pattern and test mixins described below.</p> <h3 class="relative group"><a id="modeltester-and-modeltest" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#modeltester-and-modeltest"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>ModelTester and ModelTest</span></h3> <p data-svelte-h="svelte-1a83dyu">Every model test file follows the same structure.</p> <ol data-svelte-h="svelte-1ua03qo"><li><code>ModelTester</code> (plain class) creates tiny configs and dummy inputs for testing, and can also hold small regression tests specific to the model.</li> <li><code>ModelTest</code> (<code>unittest.TestCase</code> + mixins) inherits auto-generated tests and runs them against every model variant.</li></ol> <p data-svelte-h="svelte-ibw4o3"><code>ModelTest</code> calls <code>prepare_config_and_inputs_for_common()</code> on the tester to get a <code>(config, inputs_dict)</code> tuple. All mixins rely on <code>prepare_config_and_inputs_for_common()</code> for test data.</p> <h3 class="relative group"><a id="test-mixins" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#test-mixins"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Test mixins</span></h3> <p data-svelte-h="svelte-ung373">Pick the mixins your model needs.</p> <table data-svelte-h="svelte-xkh6lb"><thead><tr><th>Mixin</th> <th>Source file</th> <th>What it tests</th></tr></thead> <tbody><tr><td><code>ModelTesterMixin</code></td> <td><code>tests/test_modeling_common.py</code></td> <td>Save/load, gradient checkpointing, forward signature, common attributes</td></tr> <tr><td><code>GenerationTesterMixin</code></td> <td><code>tests/generation/test_utils.py</code></td> <td>Greedy, sampling, beam search, assisted decoding</td></tr> <tr><td><code>PipelineTesterMixin</code></td> <td><code>tests/test_pipeline_mixin.py</code></td> <td>One test per pipeline task</td></tr> <tr><td><code>TrainingTesterMixin</code></td> <td><code>tests/test_training_mixin.py</code></td> <td>Overfitting on a small batch</td></tr> <tr><td><code>TensorParallelTesterMixin</code></td> <td><code>tests/test_tensor_parallel_mixin.py</code></td> <td>Distributed tensor parallelism</td></tr></tbody></table> <h3 class="relative group"><a id="writing-a-model-test" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#writing-a-model-test"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Writing a model test</span></h3> <p data-svelte-h="svelte-al0d7s">See <a href="https://github.com/huggingface/transformers/blob/main/tests/models/modernbert/test_modeling_modernbert.py" rel="nofollow">tests/models/modernbert/test_modeling_modernbert.py</a> for a complete working example. The key steps are outlined below.</p> <ol data-svelte-h="svelte-hp18j3"><li><p>The <code>ModelTester</code> class builds tiny configs and dummy inputs. Keep dimensions small so tests finish in seconds on CPU. Use the three tensor helpers below to build inputs.</p> <ul><li><code>ids_tensor(shape, vocab_size)</code>: Random integer tensor in <code>[0, vocab_size)</code>. Use for <code>input_ids</code>, <code>token_type_ids</code>, and label tensors.</li> <li><code>random_attention_mask(shape)</code>: Binary tensor (0s and 1s) where the first token is always 1. Use for <code>attention_mask</code>.</li> <li><code>floats_tensor(shape, scale=1.0)</code>: Random float tensor. Use for continuous inputs like <code>pixel_values</code> or <code>inputs_embeds</code>.</li></ul> <p>The tester must implement <code>get_config()</code>, <code>prepare_config_and_inputs()</code>, and <code>prepare_config_and_inputs_for_common()</code>. Add <code>create_and_check_*</code> methods for each task head (base model, sequence classification, token classification, etc.).</p></li> <li><p>Inherit from the mixins your model needs, set <code>all_model_classes</code> and <code>pipeline_model_mapping</code>, and define <code>setUp()</code>. Write <code>test_*</code> methods that delegate to the tester’s <code>create_and_check_*</code> methods.</p></li> <li><p>For each task head, add a <code>create_and_check_*</code> method on the tester that instantiates the model, runs a forward pass, and asserts output shapes. Then add a corresponding <code>test_*</code> method on the test class.</p></li></ol> <h3 class="relative group"><a id="file-organization" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#file-organization"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>File organization</span></h3> <p data-svelte-h="svelte-f37u4d">Test files live in <code>tests/models/mymodel/</code> following the structure shown below.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-text "><!-- HTML_TAG_START -->tests/models/mymodel/ | |
| ├── __init__.py | |
| ├── test_modeling_mymodel.py # model tests (required) | |
| ├── test_tokenization_mymodel.py # tokenizer tests (if custom tokenizer) | |
| ├── test_image_processing_mymodel.py # image processor tests (if vision model) | |
| ├── test_feature_extraction_mymodel.py # feature extractor tests (if audio/speech model) | |
| └── test_processing_mymodel.py # processor tests (if multimodal)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1o2w5s6">Tokenizer tests follow the same pattern. Inherit <code>TokenizerTesterMixin</code> from <code>tests/test_tokenization_common.py</code>, set a few attributes, and get auto-generated tests. See <a href="https://github.com/huggingface/transformers/blob/main/tests/models/llama/test_tokenization_llama.py" rel="nofollow">tests/models/llama/test_tokenization_llama.py</a> for an example.</p> <h2 class="relative group"><a id="config-tests" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#config-tests"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Config tests</span></h2> <p data-svelte-h="svelte-q98dll"><code>ConfigTester</code> verifies that a config class handles serialization, save/load, and standard properties correctly. <code>CausalLMModelTest</code> and <code>VLMModelTest</code> include config tests automatically. For the general path with <code>ModelTester</code> and <code>ModelTest</code>, define the config tester manually in <code>setUp()</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> tests.test_configuration_common <span class="hljs-keyword">import</span> ConfigTester | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">setUp</span>(<span class="hljs-params">self</span>): | |
| self.config_tester = ConfigTester(self, config_class=MyModelConfig, hidden_size=<span class="hljs-number">32</span>) | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">test_config</span>(<span class="hljs-params">self</span>): | |
| self.config_tester.run_common_tests()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1srwxsy"><code>run_common_tests()</code> runs several checks.</p> <ul data-svelte-h="svelte-tl397f"><li>Checks that common properties like <code>hidden_size</code>, <code>num_attention_heads</code>, and <code>num_hidden_layers</code> exist (and <code>vocab_size</code> if <code>has_text_modality=True</code>).</li> <li>Tests JSON serialization with <code>to_json_string()</code> and <code>to_json_file()</code>.</li> <li>Round-trips <code>save_pretrained()</code> and <code>from_pretrained()</code>.</li> <li>Confirms <code>id2label</code> and <code>label2id</code> consistency.</li> <li>Creates a config with no arguments to validate default initialization.</li> <li>Sets common kwargs like <code>output_hidden_states</code> and confirms they’re stored correctly.</li></ul> <p data-svelte-h="svelte-110io01">Pass <code>has_text_modality=False</code> for vision-only models that lack <code>vocab_size</code>, and pass extra <code>**kwargs</code> to override config defaults.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START -->self.config_tester = ConfigTester( | |
| self, config_class=MyVisionConfig, has_text_modality=<span class="hljs-literal">False</span>, hidden_size=<span class="hljs-number">64</span> | |
| )<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="integration-tests-and-tiny-models" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#integration-tests-and-tiny-models"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Integration tests and tiny models</span></h2> <p data-svelte-h="svelte-14wmzm1">Mixin tests use tiny configs with random weights to verify model behavior quickly. Integration tests run inference with real pretrained weights to validate output correctness. Tiny models on the Hub are small enough for fast CI, but structured like real checkpoints.</p> <h3 class="relative group"><a id="writing-integration-tests" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#writing-integration-tests"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Writing integration tests</span></h3> <p data-svelte-h="svelte-yj0nei">Place integration tests in a separate test class and mark them with <code>@slow</code>. Each test downloads real weights, runs inference, and checks outputs against expected values. Call <code>cleanup(torch_device, gc_collect=False)</code> in <code>setUp</code> and <code>tearDown</code> to avoid memory residuals.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer | |
| <span class="hljs-keyword">from</span> transformers.testing_utils <span class="hljs-keyword">import</span> cleanup, require_torch, slow, torch_device | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">MyModelIntegrationTest</span>(unittest.TestCase): | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">setUp</span>(<span class="hljs-params">self</span>): | |
| cleanup(torch_device, gc_collect=<span class="hljs-literal">False</span>) | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">tearDown</span>(<span class="hljs-params">self</span>): | |
| cleanup(torch_device, gc_collect=<span class="hljs-literal">False</span>) | |
| <span class="hljs-meta"> @slow</span> | |
| <span class="hljs-meta"> @require_torch</span> | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">test_inference</span>(<span class="hljs-params">self</span>): | |
| model = MyModelForCausalLM.from_pretrained(<span class="hljs-string">"myorg/mymodel-base"</span>).to(torch_device) | |
| tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">"myorg/mymodel-base"</span>) | |
| inputs = tokenizer(<span class="hljs-string">"Hello, world"</span>, return_tensors=<span class="hljs-string">"pt"</span>).to(torch_device) | |
| <span class="hljs-keyword">with</span> torch.no_grad(): | |
| outputs = model(**inputs) | |
| <span class="hljs-comment"># check against expected values</span> | |
| expected_slice = torch.tensor([[-<span class="hljs-number">0.1234</span>, <span class="hljs-number">0.5678</span>, -<span class="hljs-number">0.9012</span>]]) | |
| torch.testing.assert_close(outputs.logits[<span class="hljs-number">0</span>, :<span class="hljs-number">1</span>, :<span class="hljs-number">3</span>], expected_slice, atol=<span class="hljs-number">1e-4</span>, rtol=<span class="hljs-number">1e-4</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-urmp72">Mark any test with <code>@slow</code> if it downloads weights, loads a large dataset, or takes more than a few seconds. The <a href="./pr_checks">pull request CI</a> skips slow tests, but the nightly schedule runs them.</p> <h4 class="relative group"><a id="generation-integration-tests" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#generation-integration-tests"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Generation integration tests</span></h4> <p data-svelte-h="svelte-1b80oij">Use <code>do_sample=False</code> in generation tests so the output is deterministic across runs and hardware. For Mixture-of-Experts models, also call <code>model.set_experts_implementation("eager")</code> before generating to force a stable expert dispatch path. Without it, small numerical differences in the router can flip which expert handles a token and change the output.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-meta">@slow</span> | |
| <span class="hljs-meta">@require_torch</span> | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">test_generate</span>(<span class="hljs-params">self</span>): | |
| model = MyModelForCausalLM.from_pretrained(<span class="hljs-string">"myorg/mymodel-base"</span>).to(torch_device) | |
| tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">"myorg/mymodel-base"</span>) | |
| inputs = tokenizer(<span class="hljs-string">"Hello, world"</span>, return_tensors=<span class="hljs-string">"pt"</span>).to(torch_device) | |
| <span class="hljs-comment"># model.set_experts_implementation("eager") # uncomment for MoE models</span> | |
| generated_ids = model.generate(**inputs, max_new_tokens=<span class="hljs-number">20</span>, do_sample=<span class="hljs-literal">False</span>) | |
| output = tokenizer.batch_decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>) | |
| self.assertEqual(output, [<span class="hljs-string">"Hello, world! This is the expected continuation..."</span>])<!-- HTML_TAG_END --></pre></div> <h4 class="relative group"><a id="hardware-specific-expectations" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#hardware-specific-expectations"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Hardware-specific expectations</span></h4> <p data-svelte-h="svelte-o0hvq2">Transformers CI runs slow tests on an NVIDIA A10. Numerical results can vary slightly across GPU generations, so integration tests use the <a href="https://github.com/huggingface/transformers/blob/main/src/transformers/testing_utils.py#L3247" rel="nofollow">Expectations</a> class to register per-device expected values. <code>Expectations</code> picks the best match for the current hardware based on <code>(device_type, (major, minor))</code> SM keys, and falls back to a default when nothing matches.</p> <p data-svelte-h="svelte-a33g08">Run <code>torch.cuda.get_device_capability()</code> to print your local SM version (e.g. <code>(8, 6)</code> for an A10, <code>(9, 0)</code> for H100).</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers.testing_utils <span class="hljs-keyword">import</span> Expectations | |
| expected_texts = Expectations( | |
| { | |
| (<span class="hljs-string">"cuda"</span>, (<span class="hljs-number">8</span>, <span class="hljs-number">6</span>)): [<span class="hljs-string">"Hello, world! This is the A10 continuation..."</span>], | |
| (<span class="hljs-string">"cuda"</span>, (<span class="hljs-number">9</span>, <span class="hljs-number">0</span>)): [<span class="hljs-string">"Hello, world! This is the H100 continuation..."</span>], | |
| } | |
| ).get_expectation() | |
| self.assertEqual(output, expected_texts)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="creating-tiny-models" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#creating-tiny-models"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Creating tiny models</span></h3> <p data-svelte-h="svelte-xirqt0">Tiny models with random weights live on the Hub under the <a href="https://huggingface.co/hf-internal-testing" rel="nofollow">hf-internal-testing</a> organization. Pipeline tests rely on tiny models when they need a Hub-hosted checkpoint but don’t care about output quality. Fast smoke tests also load tiny models to verify forward pass shapes without downloading large checkpoints.</p> <p data-svelte-h="svelte-12036hn">Tiny models are a last resort for integration tests. Only use them when the smallest available checkpoint exceeds ~24 GB of VRAM. Use original pretrained weights, when possible, to catch real numerical regressions.</p> <p data-svelte-h="svelte-vma81a">The <code>utils/create_dummy_models.py</code> script generates tiny models from <code>ModelTester.get_config()</code>. The script extracts tiny hyperparameters from your tester, builds a model with random weights, and uploads the result to the Hub.</p> <p data-svelte-h="svelte-e70wtr">Generate tiny models locally.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START -->python utils/create_dummy_models.py output_dir -m your_model_type<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-19ibevo">Upload them to the Hub.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START -->python utils/create_dummy_models.py output_dir -m your_model_type --upload --organization hf-internal-testing<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-lbm9qk">Each model uses the name <code>hf-internal-testing/tiny-random-{ModelClassName}</code> and gets recorded in <code>tests/utils/tiny_model_summary.json</code>. A CI workflow (<code>.github/workflows/check_tiny_models.yml</code>) regenerates tiny models daily.</p> <h2 class="relative group"><a id="control-what-gets-tested" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#control-what-gets-tested"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Control what gets tested</span></h2> <p data-svelte-h="svelte-vqfb6m">Boolean flags on <code>ModelTesterMixin</code> toggle auto-generated tests. Override any flag on your test class to enable or disable specific checks.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">class</span> <span class="hljs-title class_">MyModelTest</span>(CausalLMModelTest, unittest.TestCase): | |
| model_tester_class = MyModelTester | |
| test_resize_embeddings = <span class="hljs-literal">False</span> | |
| test_all_params_have_gradient = <span class="hljs-literal">False</span> <span class="hljs-comment"># when not all parameters are activated in every forward pass</span><!-- HTML_TAG_END --></pre></div> <table data-svelte-h="svelte-ael2r9"><thead><tr><th>Flag</th> <th>Default</th> <th>What it controls</th></tr></thead> <tbody><tr><td><code>test_resize_embeddings</code></td> <td><code>True</code></td> <td>Embedding layer resizing</td></tr> <tr><td><code>test_resize_position_embeddings</code></td> <td><code>False</code></td> <td>Position embedding resizing</td></tr> <tr><td><code>test_mismatched_shapes</code></td> <td><code>True</code></td> <td>Mismatched input/output shape handling</td></tr> <tr><td><code>test_missing_keys</code></td> <td><code>True</code></td> <td>Missing key warnings on load</td></tr> <tr><td><code>test_torch_exportable</code></td> <td><code>True</code></td> <td><code>torch.export</code> compatibility</td></tr> <tr><td><code>test_all_params_have_gradient</code></td> <td><code>True</code></td> <td>All parameters receive gradients (set <code>False</code> when not all parameters are activated in every forward pass, such as MoE experts)</td></tr> <tr><td><code>is_encoder_decoder</code></td> <td><code>False</code></td> <td>Encoder-decoder specific tests</td></tr> <tr><td><code>has_attentions</code></td> <td><code>True</code></td> <td>Attention output tests</td></tr> <tr><td><code>_is_composite</code></td> <td><code>False</code></td> <td>Composite/multimodal model handling</td></tr> <tr><td><code>model_split_percents</code></td> <td><code>[0.5, 0.7, 0.9]</code></td> <td>Split percentages for model parallelism tests</td></tr></tbody></table> <h2 class="relative group"><a id="next-steps" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#next-steps"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Next steps</span></h2> <ul data-svelte-h="svelte-z0bc5y"><li>Browse the <a href="https://docs.pytest.org/en/latest/getting-started.html" rel="nofollow">pytest</a> docs for more about test selection, fixtures, logging, and more.</li></ul> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers/blob/main/docs/source/en/testing.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p> | |
| <script> | |
| { | |
| __sveltekit_1s77840 = { | |
| assets: "/docs/transformers/pr_36895/en", | |
| base: "/docs/transformers/pr_36895/en", | |
| env: {} | |
| }; | |
| const element = document.currentScript.parentElement; | |
| const data = [null,null]; | |
| Promise.all([ | |
| import("/docs/transformers/pr_36895/en/_app/immutable/entry/start.b88698fb.js"), | |
| import("/docs/transformers/pr_36895/en/_app/immutable/entry/app.ac300a2c.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 677], | |
| data, | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 75.3 kB
- Xet hash:
- 4d2387804f2a15a8b006e6695423b566a42ff3eff605fea6f9cb4934b35e6b6f
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.