Hugging Face's logo

Buckets:
hf-doc-build
/
doc-dev

Files
xet
hf-doc-build/doc-dev / transformers /pr_36839 /en /perf_train_cpu_many.html
rtrm's picture
rtrm
download
raw
40.3 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Distributed CPUs&quot;,&quot;local&quot;:&quot;distributed-cpus&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Trainer&quot;,&quot;local&quot;:&quot;trainer&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Kubernetes&quot;,&quot;local&quot;:&quot;kubernetes&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;PyTorchJob&quot;,&quot;local&quot;:&quot;pytorchjob&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Deploy&quot;,&quot;local&quot;:&quot;deploy&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2}],&quot;depth&quot;:1}">
 <link href="/docs/transformers/pr_36839/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
 <link rel="modulepreload" href="/docs/transformers/pr_36839/en/_app/immutable/entry/start.6be8d590.js">
 <link rel="modulepreload" href="/docs/transformers/pr_36839/en/_app/immutable/chunks/scheduler.01eeda35.js">
 <link rel="modulepreload" href="/docs/transformers/pr_36839/en/_app/immutable/chunks/singletons.177df05e.js">
 <link rel="modulepreload" href="/docs/transformers/pr_36839/en/_app/immutable/chunks/index.4862150a.js">
 <link rel="modulepreload" href="/docs/transformers/pr_36839/en/_app/immutable/chunks/paths.517376d1.js">
 <link rel="modulepreload" href="/docs/transformers/pr_36839/en/_app/immutable/entry/app.09748b4b.js">
 <link rel="modulepreload" href="/docs/transformers/pr_36839/en/_app/immutable/chunks/index.6dd51b66.js">
 <link rel="modulepreload" href="/docs/transformers/pr_36839/en/_app/immutable/nodes/0.8897c14d.js">
 <link rel="modulepreload" href="/docs/transformers/pr_36839/en/_app/immutable/chunks/each.e59479a4.js">
 <link rel="modulepreload" href="/docs/transformers/pr_36839/en/_app/immutable/nodes/419.42eee36b.js">
 <link rel="modulepreload" href="/docs/transformers/pr_36839/en/_app/immutable/chunks/Tip.de9bae2b.js">
 <link rel="modulepreload" href="/docs/transformers/pr_36839/en/_app/immutable/chunks/CodeBlock.864da1b0.js">
 <link rel="modulepreload" href="/docs/transformers/pr_36839/en/_app/immutable/chunks/EditOnGithub.7faefd25.js">
 <link rel="modulepreload" href="/docs/transformers/pr_36839/en/_app/immutable/chunks/HfOption.f7f04550.js">
 <link rel="modulepreload" href="/docs/transformers/pr_36839/en/_app/immutable/chunks/stores.318eade7.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Distributed CPUs&quot;,&quot;local&quot;:&quot;distributed-cpus&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Trainer&quot;,&quot;local&quot;:&quot;trainer&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Kubernetes&quot;,&quot;local&quot;:&quot;kubernetes&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;PyTorchJob&quot;,&quot;local&quot;:&quot;pytorchjob&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Deploy&quot;,&quot;local&quot;:&quot;deploy&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2}],&quot;depth&quot;:1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="distributed-cpus" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#distributed-cpus"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Distributed CPUs</span></h1> <p data-svelte-h="svelte-1db6toe">CPUs are commonly available and can be a cost-effective training option when GPUs are unavailable. When training large models or if a single CPU is too slow, distributed training with CPUs can help speed up training.</p> <p data-svelte-h="svelte-zre95p">This guide demonstrates how to perform distributed training with multiple CPUs using a <a href="./perf_train_gpu_many#distributeddataparallel">DistributedDataParallel (DDP)</a> strategy on bare metal with <a href="/docs/transformers/pr_36839/en/main_classes/trainer#transformers.Trainer">Trainer</a> and a Kubernetes cluster. All examples shown in this guide depend on the <a href="https://www.intel.com/content/www/us/en/developer/tools/oneapi/hpc-toolkit.html" rel="nofollow">Intel oneAPI HPC Toolkit</a>.</p> <p data-svelte-h="svelte-z73c5n">There are two toolkits you’ll need from Intel oneAPI.</p> <ol data-svelte-h="svelte-90fu2y"><li><a href="https://www.intel.com/content/www/us/en/developer/tools/oneapi/oneccl.html" rel="nofollow">oneCCL</a> includes efficient implementations of collectives commonly used in deep learning such as all-gather, all-reduce, and reduce-scatter. To install from a prebuilt wheel, make sure you always use the latest release. Refer to the table <a href="https://github.com/intel/torch-ccl#install-prebuilt-wheel" rel="nofollow">here</a> to check if a version of oneCCL is supported for a Python and PyTorch version.</li></ol> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># installs oneCCL for PyTorch 2.4.0</span>
pip install oneccl_bind_pt==2.4.0 -f https://developer.intel.com/ipex-whl-stable-cpu<!-- HTML_TAG_END --></pre></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-1iyq1qq">Refer to the oneCCL <a href="https://github.com/intel/torch-ccl#installation" rel="nofollow">installation</a> for more details.</p></div> <ol data-svelte-h="svelte-1batrdf"><li><a href="https://www.intel.com/content/www/us/en/developer/tools/oneapi/mpi-library.html" rel="nofollow">MPI</a> is a message-passing interface for communications between hardware and networks. The oneCCL toolkit is installed along with MPI, but you need to source the environment as shown below before using it.</li></ol> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->oneccl_bindings_for_pytorch_path=$(python -c <span class="hljs-string">&quot;from oneccl_bindings_for_pytorch import cwd; print(cwd)&quot;</span>)
<span class="hljs-built_in">source</span> <span class="hljs-variable">$oneccl_bindings_for_pytorch_path</span>/env/setvars.sh<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1otdpgb">Lastly, install the <a href="https://intel.github.io/intel-extension-for-pytorch/index.html" rel="nofollow">Intex Extension for PyTorch (IPEX)</a> which enables additional performance optimizations for Intel hardware such as weight sharing and better thread runtime control.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->pip install intel_extension_for_pytorch==&lt;version_name&gt; -f https://developer.intel.com/ipex-whl-stable-cpu<!-- HTML_TAG_END --></pre></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-88tmbr">Refer to the IPEX <a href="https://intel.github.io/intel-extension-for-pytorch/index.html#installation" rel="nofollow">installation</a> for more details.</p></div> <h2 class="relative group"><a id="trainer" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#trainer"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Trainer</span></h2> <p data-svelte-h="svelte-1o7np23"><a href="/docs/transformers/pr_36839/en/main_classes/trainer#transformers.Trainer">Trainer</a> supports distributed training with CPUs with the oneCCL backend. Add the <code>--ddp_backend ccl</code> parameter in the command arguments to enable it.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">single node </div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">multiple nodes </div></div> <div class="language-select"><p data-svelte-h="svelte-6lyzik">The example below demonstrates the <a href="https://github.com/huggingface/transformers/tree/main/examples/pytorch/question-answering" rel="nofollow">run_qa.py</a> script. It enables training with two processes on one Xeon CPU, with one process running per socket.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-4dbzn8">Tune the variable <code>OMP_NUM_THREADS/CCL_WORKER_COUNT</code> for optimal performance.</p></div> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-built_in">export</span> CCL_WORKER_COUNT=1
<span class="hljs-built_in">export</span> MASTER_ADDR=127.0.0.1
mpirun -n 2 -genv OMP_NUM_THREADS=23 \
python3 run_qa.py \
 --model_name_or_path google-bert/bert-large-uncased \
 --dataset_name squad \
 --do_train \
 --do_eval \
 --per_device_train_batch_size 12 \
 --learning_rate 3e-5 \
 --num_train_epochs 2 \
 --max_seq_length 384 \
 --doc_stride 128 \
 --output_dir /tmp/debug_squad/ \
 --no_cuda \
 --ddp_backend ccl \
 --use_ipex<!-- HTML_TAG_END --></pre></div> </div> <h2 class="relative group"><a id="kubernetes" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#kubernetes"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Kubernetes</span></h2> <p data-svelte-h="svelte-1pbm113">Distributed training with CPUs can also be deployed to a Kubernetes cluster with <a href="https://www.kubeflow.org/docs/components/training/user-guides/pytorch/" rel="nofollow">PyTorchJob</a>. Before you get started, you should perform the following setup steps.</p> <ol data-svelte-h="svelte-2pmwlj"><li>Ensure you have access to a Kubernetes cluster with <a href="https://www.kubeflow.org/docs/started/installing-kubeflow/" rel="nofollow">Kubeflow</a> installed.</li> <li>Install and configure <a href="https://kubernetes.io/docs/tasks/tools" rel="nofollow">kubectl</a> to interact with the cluster.</li> <li>Set up a <a href="https://kubernetes.io/docs/concepts/storage/persistent-volumes/" rel="nofollow">PersistentVolumeClaim (PVC)</a> to store datasets and model files. There are multiple options to choose from, including a <a href="https://kubernetes.io/docs/concepts/storage/storage-classes/" rel="nofollow">StorageClass</a> or a cloud storage bucket.</li> <li>Set up a Docker container for the training script and all required dependencies such as PyTorch, Transformers, IPEX, oneCCL, and OpenSSH to facilitate communicattion between containers.</li></ol> <p data-svelte-h="svelte-qbn6pm">The example Dockerfile below uses a base image that supports distributed training with CPUs, and extracts Transformers to the <code>/workspace</code> directory to include the training scripts in the image. The image needs to be built and copied to the clusters nodes or pushed to a container registry prior to deployment.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">FROM</span> intel/intel-optimized-pytorch:<span class="hljs-number">2.4</span>.<span class="hljs-number">0</span>-pip-multinode
<span class="hljs-keyword">RUN</span><span class="language-bash"> apt-get update -y &amp;&amp; \
 apt-get install -y --no-install-recommends --fix-missing \
 google-perftools \
 libomp-dev</span>
<span class="hljs-keyword">WORKDIR</span><span class="language-bash"> /workspace</span>
<span class="hljs-comment"># Download and extract the transformers code</span>
<span class="hljs-keyword">ARG</span> HF_TRANSFORMERS_VER=<span class="hljs-string">&quot;4.46.0&quot;</span>
<span class="hljs-keyword">RUN</span><span class="language-bash"> pip install --no-cache-dir \
 transformers==<span class="hljs-variable">${HF_TRANSFORMERS_VER}</span> &amp;&amp; \
 <span class="hljs-built_in">mkdir</span> transformers &amp;&amp; \
 curl -sSL --retry 5 https://github.com/huggingface/transformers/archive/refs/tags/v<span class="hljs-variable">${HF_TRANSFORMERS_VER}</span>.tar.gz | tar -C transformers --strip-components=1 -xzf -</span><!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="pytorchjob" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#pytorchjob"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>PyTorchJob</span></h3> <p data-svelte-h="svelte-1m9zjwg"><a href="https://www.kubeflow.org/docs/components/training/user-guides/pytorch/" rel="nofollow">PyTorchJob</a> is an extension of the Kubernetes API for running PyTorch training jobs on Kubernetes. It includes a yaml file that defines the training jobs parameters such as the name of the PyTorchJob, number of workers, types of resources for each worker, and more.</p> <p data-svelte-h="svelte-wrjz37">The volume mount parameter is a path to where the PVC is mounted in the container for each worker pod. The PVC is typically used to hold the dataset, checkpoint files, and the model after it has finished training.</p> <p data-svelte-h="svelte-sjxwh">The example yaml file below sets up four workers on the <a href="https://github.com/huggingface/transformers/tree/main/examples/pytorch/question-answering" rel="nofollow">run_qa.py</a> script. Adapt the yaml file based on your training script and number of nodes in your cluster.</p> <p data-svelte-h="svelte-1d9s969">The CPU resource limits and requests are defined in <a href="https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-cpu" rel="nofollow">CPU units</a>. One CPU unit is equivalent to one physical CPU core or virtual core. The CPU units defined in the yaml file should be less than the amount of available CPU and memory capacity of a single machine in order to leave some resources for kubelet and the system. For a <code>Guaranteed</code> <a href="https://kubernetes.io/docs/tasks/configure-pod-container/quality-service-pod" rel="nofollow">quality of service</a>, set the same CPU and memory amounts for both the resource limits and requests.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-attr">apiVersion:</span> <span class="hljs-string">&quot;kubeflow.org/v1&quot;</span>
<span class="hljs-attr">kind:</span> <span class="hljs-string">PyTorchJob</span>
<span class="hljs-attr">metadata:</span>
 <span class="hljs-attr">name:</span> <span class="hljs-string">transformers-pytorchjob</span>
<span class="hljs-attr">spec:</span>
 <span class="hljs-attr">elasticPolicy:</span>
 <span class="hljs-attr">rdzvBackend:</span> <span class="hljs-string">c10d</span>
 <span class="hljs-attr">minReplicas:</span> <span class="hljs-number">1</span>
 <span class="hljs-attr">maxReplicas:</span> <span class="hljs-number">4</span>
 <span class="hljs-attr">maxRestarts:</span> <span class="hljs-number">10</span>
 <span class="hljs-attr">pytorchReplicaSpecs:</span>
 <span class="hljs-attr">Worker:</span>
 <span class="hljs-attr">replicas:</span> <span class="hljs-number">4</span> <span class="hljs-comment"># The number of worker pods</span>
 <span class="hljs-attr">restartPolicy:</span> <span class="hljs-string">OnFailure</span>
 <span class="hljs-attr">template:</span>
 <span class="hljs-attr">spec:</span>
 <span class="hljs-attr">containers:</span>
 <span class="hljs-bullet">-</span> <span class="hljs-attr">name:</span> <span class="hljs-string">pytorch</span>
 <span class="hljs-attr">image:</span> <span class="hljs-string">&lt;image</span> <span class="hljs-string">name&gt;:&lt;tag&gt;</span> <span class="hljs-comment"># Specify the docker image to use for the worker pods</span>
 <span class="hljs-attr">imagePullPolicy:</span> <span class="hljs-string">IfNotPresent</span>
 <span class="hljs-attr">command:</span> [<span class="hljs-string">&quot;/bin/bash&quot;</span>, <span class="hljs-string">&quot;-c&quot;</span>]
 <span class="hljs-attr">args:</span>
 <span class="hljs-bullet">-</span> <span class="hljs-string">&gt;-
 cd /workspace/transformers;
 pip install -r /workspace/transformers/examples/pytorch/question-answering/requirements.txt;
 source /usr/local/lib/python3.10/dist-packages/oneccl_bindings_for_pytorch/env/setvars.sh;
 torchrun /workspace/transformers/examples/pytorch/question-answering/run_qa.py \
 --model_name_or_path distilbert/distilbert-base-uncased \
 --dataset_name squad \
 --do_train \
 --do_eval \
 --per_device_train_batch_size 12 \
 --learning_rate 3e-5 \
 --num_train_epochs 2 \
 --max_seq_length 384 \
 --doc_stride 128 \
 --output_dir /tmp/pvc-mount/output_$(date +%Y%m%d_%H%M%S) \
 --no_cuda \
 --ddp_backend ccl \
 --bf16 \
 --use_ipex;
</span> <span class="hljs-attr">env:</span>
 <span class="hljs-bullet">-</span> <span class="hljs-attr">name:</span> <span class="hljs-string">LD_PRELOAD</span>
 <span class="hljs-attr">value:</span> <span class="hljs-string">&quot;/usr/lib/x86_64-linux-gnu/libtcmalloc.so.4.5.9:/usr/local/lib/libiomp5.so&quot;</span>
 <span class="hljs-bullet">-</span> <span class="hljs-attr">name:</span> <span class="hljs-string">TRANSFORMERS_CACHE</span>
 <span class="hljs-attr">value:</span> <span class="hljs-string">&quot;/tmp/pvc-mount/transformers_cache&quot;</span>
 <span class="hljs-bullet">-</span> <span class="hljs-attr">name:</span> <span class="hljs-string">HF_DATASETS_CACHE</span>
 <span class="hljs-attr">value:</span> <span class="hljs-string">&quot;/tmp/pvc-mount/hf_datasets_cache&quot;</span>
 <span class="hljs-bullet">-</span> <span class="hljs-attr">name:</span> <span class="hljs-string">LOGLEVEL</span>
 <span class="hljs-attr">value:</span> <span class="hljs-string">&quot;INFO&quot;</span>
 <span class="hljs-bullet">-</span> <span class="hljs-attr">name:</span> <span class="hljs-string">CCL_WORKER_COUNT</span>
 <span class="hljs-attr">value:</span> <span class="hljs-string">&quot;1&quot;</span>
 <span class="hljs-bullet">-</span> <span class="hljs-attr">name:</span> <span class="hljs-string">OMP_NUM_THREADS</span> <span class="hljs-comment"># Can be tuned for optimal performance</span>
 <span class="hljs-attr">value:</span> <span class="hljs-string">&quot;240&quot;</span>
 <span class="hljs-attr">resources:</span>
 <span class="hljs-attr">limits:</span>
 <span class="hljs-attr">cpu:</span> <span class="hljs-number">240</span> <span class="hljs-comment"># Update the CPU and memory limit values based on your nodes</span>
 <span class="hljs-attr">memory:</span> <span class="hljs-string">128Gi</span>
 <span class="hljs-attr">requests:</span>
 <span class="hljs-attr">cpu:</span> <span class="hljs-number">240</span> <span class="hljs-comment"># Update the CPU and memory request values based on your nodes</span>
 <span class="hljs-attr">memory:</span> <span class="hljs-string">128Gi</span>
 <span class="hljs-attr">volumeMounts:</span>
 <span class="hljs-bullet">-</span> <span class="hljs-attr">name:</span> <span class="hljs-string">pvc-volume</span>
 <span class="hljs-attr">mountPath:</span> <span class="hljs-string">/tmp/pvc-mount</span>
 <span class="hljs-bullet">-</span> <span class="hljs-attr">mountPath:</span> <span class="hljs-string">/dev/shm</span>
 <span class="hljs-attr">name:</span> <span class="hljs-string">dshm</span>
 <span class="hljs-attr">restartPolicy:</span> <span class="hljs-string">Never</span>
 <span class="hljs-attr">nodeSelector:</span> <span class="hljs-comment"># Optionally use nodeSelector to match a certain node label for the worker pods</span>
 <span class="hljs-attr">node-type:</span> <span class="hljs-string">gnr</span>
 <span class="hljs-attr">volumes:</span>
 <span class="hljs-bullet">-</span> <span class="hljs-attr">name:</span> <span class="hljs-string">pvc-volume</span>
 <span class="hljs-attr">persistentVolumeClaim:</span>
 <span class="hljs-attr">claimName:</span> <span class="hljs-string">transformers-pvc</span>
 <span class="hljs-bullet">-</span> <span class="hljs-attr">name:</span> <span class="hljs-string">dshm</span>
 <span class="hljs-attr">emptyDir:</span>
 <span class="hljs-attr">medium:</span> <span class="hljs-string">Memory</span><!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="deploy" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#deploy"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Deploy</span></h3> <p data-svelte-h="svelte-1b7ewcg">After you’ve setup the PyTorchJob yaml file with the appropriate settings for your cluster and training job, deploy it to the cluster with the command below.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-built_in">export</span> NAMESPACE=&lt;specify your namespace&gt;
kubectl create -f pytorchjob.yaml -n <span class="hljs-variable">${NAMESPACE}</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-d7wzxc">List the pods in the namespace with <code>kubectl get pods -n ${NAMESPACE}</code>. At first, the status may be “Pending” but it should change to “Running” once the containers are pulled and created.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->kubectl get pods -n <span class="hljs-variable">${NAMESPACE}</span>
NAME READY STATUS RESTARTS AGE
...
transformers-pytorchjob-worker-0 1/1 Running 0 7m37s
transformers-pytorchjob-worker-1 1/1 Running 0 7m37s
transformers-pytorchjob-worker-2 1/1 Running 0 7m37s
transformers-pytorchjob-worker-3 1/1 Running 0 7m37s
...<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-fir50r">Inspect the logs for each worker with the following command. Add <code>-f</code> to stream the logs.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->kubectl logs transformers-pytorchjob-worker-0 -n <span class="hljs-variable">${NAMESPACE}</span> -f<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-16x58zj">Once training is complete, the trained model can be copied from the PVC or storage location. Delete the PyTorchJob resource from the cluster with the command below.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->kubectl delete -f pytorchjob.yaml -n <span class="hljs-variable">${NAMESPACE}</span><!-- HTML_TAG_END --></pre></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers/blob/main/docs/source/en/perf_train_cpu_many.md" target="_blank"><span data-svelte-h="svelte-1kd6by1">&lt;</span> <span data-svelte-h="svelte-x0xyl0">&gt;</span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p>
<script>
 {
 __sveltekit_1bm5psi = {
 assets: "/docs/transformers/pr_36839/en",
 base: "/docs/transformers/pr_36839/en",
 env: {}
 };
 const element = document.currentScript.parentElement;
 const data = [null,null];
 Promise.all([
 import("/docs/transformers/pr_36839/en/_app/immutable/entry/start.6be8d590.js"),
 import("/docs/transformers/pr_36839/en/_app/immutable/entry/app.09748b4b.js")
 ]).then(([kit, app]) => {
 kit.start(app, element, {
 node_ids: [0, 419],
 data,
 form: null,
 error: null
 });
 });
 }
 </script>

Xet Storage Details

Size:
40.3 kB
·
Xet hash:
7bf3cb51db344f5b6c7647ffcb0268d3a49fe9c1626465878f190cdb63465989

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.