Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"在CPU上进行高效训练","local":"在cpu上进行高效训练","sections":[{"title":"使用IPEX进行混合精度训练","local":"使用ipex进行混合精度训练","sections":[{"title":"IPEX 安装:","local":"ipex-安装","sections":[],"depth":3},{"title":"在 Trainer 中使用 IPEX","local":"在-trainer-中使用-ipex","sections":[],"depth":3},{"title":"实践示例","local":"实践示例","sections":[],"depth":3}],"depth":2}],"depth":1}"> | |
| <link href="/docs/transformers/main/zh/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload"> | |
| <link rel="modulepreload" href="/docs/transformers/main/zh/_app/immutable/entry/start.7a5389c5.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/zh/_app/immutable/chunks/scheduler.70fef3e7.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/zh/_app/immutable/chunks/singletons.de8a9de6.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/zh/_app/immutable/chunks/index.048d9726.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/zh/_app/immutable/chunks/paths.a6fe27ce.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/zh/_app/immutable/entry/app.ac7d4b06.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/zh/_app/immutable/chunks/preload-helper.77b1b155.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/zh/_app/immutable/chunks/index.747f74de.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/zh/_app/immutable/nodes/0.3408eb2f.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/zh/_app/immutable/chunks/each.e59479a4.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/zh/_app/immutable/nodes/54.e9243801.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/zh/_app/immutable/chunks/CopyLLMTxtMenu.25c10d36.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/zh/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.04bd7ab0.js"> | |
| <link rel="modulepreload" href="/docs/transformers/main/zh/_app/immutable/chunks/CodeBlock.15c43204.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"在CPU上进行高效训练","local":"在cpu上进行高效训练","sections":[{"title":"使用IPEX进行混合精度训练","local":"使用ipex进行混合精度训练","sections":[{"title":"IPEX 安装:","local":"ipex-安装","sections":[],"depth":3},{"title":"在 Trainer 中使用 IPEX","local":"在-trainer-中使用-ipex","sections":[],"depth":3},{"title":"实践示例","local":"实践示例","sections":[],"depth":3}],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="在cpu上进行高效训练" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#在cpu上进行高效训练"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>在CPU上进行高效训练</span></h1> <p data-svelte-h="svelte-ki35fh">本指南将重点介绍如何在CPU上高效训练大型模型。</p> <h2 class="relative group"><a id="使用ipex进行混合精度训练" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#使用ipex进行混合精度训练"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>使用IPEX进行混合精度训练</span></h2> <p data-svelte-h="svelte-1b1ob67">混合精度训练在模型中可以同时使用单精度（fp32）和半精度（bf16/fp16）的数据类型来加速训练或推理过程，并且仍然能保留大部分单精度的准确性。现代的CPU，例如第三代、第四代和第五代Intel® Xeon® Scalable处理器，原生支持bf16，而第六代Intel® Xeon® Scalable处理器原生支持bf16和fp16。您在训练时启用bf16或fp16的混合精度训练可以直接提高处理性能。</p> <p data-svelte-h="svelte-a9kmph">为了进一步最大化训练性能，您可以使用Intel® PyTorch扩展（IPEX）。IPEX是一个基于PyTorch构建的库，增加了额外的CPU指令集架构（ISA）级别的支持，比如Intel®高级向量扩展512（Intel® AVX512-VNNI）和Intel®高级矩阵扩展（Intel® AMX）。这为Intel CPU提供额外的性能提升。然而，仅支持AVX2的CPU（例如AMD或较旧的Intel CPU）在使用IPEX时并不保证能提高性能。</p> <p data-svelte-h="svelte-177ubpm">从PyTorch 1.10版本起，CPU后端已经启用了自动混合精度（AMP）。IPEX还支持bf16/fp16的AMP和bf16/fp16算子优化，并且部分功能已经上游到PyTorch主分支。通过IPEX AMP，您可以获得更好的性能和用户体验。</p> <p data-svelte-h="svelte-ymi5hy">点击<a href="https://intel.github.io/intel-extension-for-pytorch/cpu/latest/tutorials/features/amp.html" rel="nofollow">这里</a>查看<strong>自动混合精度</strong>的更多详细信息。</p> <h3 class="relative group"><a id="ipex-安装" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#ipex-安装"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>IPEX 安装:</span></h3> <p data-svelte-h="svelte-z1n0yn">IPEX 的发布与 PyTorch 一致，您可以通过 pip 安装：</p> <table data-svelte-h="svelte-p1edgy"><thead><tr><th align="center">PyTorch Version</th> <th align="center">IPEX version</th></tr></thead> <tbody><tr><td align="center">2.5.0</td> <td align="center">2.5.0+cpu</td></tr> <tr><td align="center">2.4.0</td> <td align="center">2.4.0+cpu</td></tr> <tr><td align="center">2.3.0</td> <td align="center">2.3.0+cpu</td></tr> <tr><td align="center">2.2.0</td> <td align="center">2.2.0+cpu</td></tr></tbody></table> <p data-svelte-h="svelte-38qae9">请运行 <code>pip list | grep torch</code> 以获取您的 <code>pytorch_version</code>，然后根据该版本安装相应的 <code>IPEX version_name</code>。</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START -->pip install intel_extension_for_pytorch==<version_name> -f https://developer.intel.com/ipex-whl-stable-cpu<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1wtnrqq">如果需要的话，您可以在 <a href="https://developer.intel.com/ipex-whl-stable-cpu" rel="nofollow">ipex-whl-stable-cpu</a> 查看最新版本。</p> <p data-svelte-h="svelte-12at55l">查看更多 <a href="https://intel.github.io/intel-extension-for-pytorch/cpu/latest/tutorials/installation.html" rel="nofollow">安装IPEX</a> 的方法。</p> <h3 class="relative group"><a id="在-trainer-中使用-ipex" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#在-trainer-中使用-ipex"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>在 Trainer 中使用 IPEX</span></h3> <p data-svelte-h="svelte-czc5n2">在 Trainer 中使用 IPEX 时，您应在训练命令参数中添加 <code>use_ipex</code>、<code>bf16</code> 或 <code>fp16</code> 以及 <code>no_cuda</code> 来启用自动混合精度。</p> <p data-svelte-h="svelte-1xmnpzz">以 <a href="https://github.com/huggingface/transformers/tree/main/examples/pytorch/question-answering" rel="nofollow">Transformers 问答任务</a>为例：</p> <ul data-svelte-h="svelte-1cgsnfu"><li>在 CPU 上使用 BF16 自动混合精度训练 IPEX 的示例如下： | |
| <pre> python examples/pytorch/question-answering/run_qa.py \ | |
| --model_name_or_path google-bert/bert-base-uncased \ | |
| --dataset_name squad \ | |
| --do_train \ | |
| --do_eval \ | |
| --per_device_train_batch_size 12 \ | |
| --learning_rate 3e-5 \ | |
| --num_train_epochs 2 \ | |
| --max_seq_length 384 \ | |
| --doc_stride 128 \ | |
| --output_dir /tmp/debug_squad/ \ | |
| <b>--use_ipex</b> \ | |
| <b>--bf16</b> \ | |
| <b>--use_cpu</b></pre></li></ul> <p data-svelte-h="svelte-15u687y">如果您想在脚本中启用 <code>use_ipex</code> 和 <code>bf16</code>，请像下面这样将这些参数添加到 <code>TrainingArguments</code> 中：</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-diff "><!-- HTML_TAG_START -->training_args = TrainingArguments( | |
| output_dir=args.output_path, | |
| <span class="hljs-addition">+ bf16=True,</span> | |
| <span class="hljs-addition">+ use_ipex=True,</span> | |
| <span class="hljs-addition">+ use_cpu=True,</span> | |
| **kwargs | |
| )<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="实践示例" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#实践示例"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>实践示例</span></h3> <p data-svelte-h="svelte-639blr">博客: <a href="https://huggingface.co/blog/intel-sapphire-rapids" rel="nofollow">使用 Intel Sapphire Rapids 加速 PyTorch Transformers</a></p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers/blob/main/docs/source/zh/perf_train_cpu.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p> | |
| <script> | |
| { | |
| __sveltekit_zpy4e4 = { | |
| assets: "/docs/transformers/main/zh", | |
| base: "/docs/transformers/main/zh", | |
| env: {} | |
| }; | |
| const element = document.currentScript.parentElement; | |
| const data = [null,null]; | |
| Promise.all([ | |
| import("/docs/transformers/main/zh/_app/immutable/entry/start.7a5389c5.js"), | |
| import("/docs/transformers/main/zh/_app/immutable/entry/app.ac7d4b06.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 54], | |
| data, | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 18.8 kB
- Xet hash:
- 9e43d875e38aa32f901aeb7a46c9b6b4294623e2e649ace4d1f2a4141fe19df8
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.