optimum-neuron/v0.2.0.dev2/en/_app/immutable/nodes/44.4cb18b60.js

import{s as Rn,o as Vn,n as Zl}from"../chunks/scheduler.85c25b89.js";import{S as An,i as $n,g as i,s,r as M,A as Sn,h as o,f as l,c as a,j as Wn,u as m,x as r,k as Oe,y as kn,a as n,v as u,d as c,t as d,w as T}from"../chunks/index.c9bcf812.js";import{T as El}from"../chunks/Tip.d8f753fa.js";import{C as y}from"../chunks/CodeBlock.c004bd26.js";import{H as f}from"../chunks/index.9790a2b6.js";function Nn(J){let p,w="While we will use <code>Llama-3 8B</code> in this tutorial, it is completely possible to use other models, simply by swtiching the <code>model_id</code>.";return{c(){p=i("p"),p.innerHTML=w},l(h){p=o(h,"P",{"data-svelte-h":!0}),r(p)!=="svelte-17hw35"&&(p.innerHTML=w)},m(h,U){n(h,p,U)},p:Zl,d(h){h&&l(p)}}}function Xn(J){let p,w='If you want to know more about distributed training you can take a look at the <a href="https://huggingface.co/docs/optimum-neuron/guides/distributed_training" rel="nofollow">documentation</a>.';return{c(){p=i("p"),p.innerHTML=w},l(h){p=o(h,"P",{"data-svelte-h":!0}),r(p)!=="svelte-yp02re"&&(p.innerHTML=w)},m(h,U){n(h,p,U)},p:Zl,d(h){h&&l(p)}}}function Gn(J){let p,w='To learn more about the caching system, and how you can create your own private cache repository, check this <a href="https://huggingface.co/docs/optimum-neuron/guides/cache_system" rel="nofollow">guide</a>.';return{c(){p=i("p"),p.innerHTML=w},l(h){p=o(h,"P",{"data-svelte-h":!0}),r(p)!=="svelte-o0xds9"&&(p.innerHTML=w)},m(h,U){n(h,p,U)},p:Zl,d(h){h&&l(p)}}}function Fn(J){let p,w,h,U,v,Ke,E,Bl='<em>Note: The complete script for this tutorial can be downloaded <a href="https://github.com/huggingface/optimum-neuron/blob/main/docs/source/training_tutorials/sft_lora_finetune_llm.py" rel="nofollow">here</a>.</em>',et,Z,Wl='This tutorial will teach you how to fine-tune open source LLMs like <a href="https://huggingface.co/meta-llama/Meta-Llama-3-8B" rel="nofollow">Llama 3</a> on AWS Trainium. In our example, we are going to leverage the <a href="https://huggingface.co/docs/optimum-neuron/index" rel="nofollow">Optimum Neuron</a>, <a href="https://huggingface.co/docs/transformers/index" rel="nofollow">Transformers</a> and <a href="https://huggingface.co/docs/datasets/index" rel="nofollow">Datasets</a> libraries.',tt,B,Rl="You will learn how to:",lt,W,Vl='<li><a href="#supervised-fine-tuning-of-llama-3-8b-on-one-aws-trainium-instance">Supervised Fine-Tuning of Llama 3 8B on one AWS Trainium instance</a><ul><li><a href="#1-setup-aws-environment">1. Setup AWS Environment</a></li> <li><a href="#2-load-and-prepare-the-dataset">2. Load and prepare the dataset</a></li> <li><a href="#3-supervised-fine-tuning-of-llama-on-aws-trainium-with-the-neuronsfttrainer">3. Supervised Fine-Tuning of Llama on AWS Trainium with the <code>NeuronSFTTrainer</code></a><ul><li><a href="#formatting-our-dataset">Formatting our dataset</a></li> <li><a href="#preparing-the-model">Preparing the model</a></li></ul></li> <li><a href="#4-launch-training">4. Launch Training</a><ul><li><a href="#precompilation">Precompilation</a></li> <li><a href="#actual-training">Actual Training</a></li> <li><a href="#consolidate-the-checkpoint-and-merge-model">Consolidate the checkpoint and merge model</a></li></ul></li> <li><a href="#5-evaluate-and-test-fine-tuned-llama-model">5. Evaluate and test fine-tuned Llama model</a></li></ul></li>',nt,j,st,R,at,V,Al="Before starting this tutorial, you will need to setup your environment:",it,A,$l='<li>Create an AWS Trainium instance. <strong>You will need a <code>trn1.32xlarge</code>, which contains 16 Neuron Devices.</strong> You can follow this <a href="https://huggingface.co/docs/optimum-neuron/guides/setup_aws_instance" rel="nofollow">guide</a> to create one.</li> <li>Make sure you are logged in on the Hugging Face Hub:</li>',ot,$,rt,b,Sl="<li>Check that you have access to the model. Some open source models are gated, meaning that users need to apply to the model owner to be able to use the model weights. Here we will be training Llama-3 8B, for which there are two possibilities:</li>",pt,S,kl='<li>The official gated repo: <a href="https://huggingface.co/meta-llama/Meta-Llama-3-8B" rel="nofollow"><code>meta-llama/Meta-Llama-3-8B</code></a></li> <li>The non-official un-gated repo: <a href="https://huggingface.co/NousResearch/Meta-Llama-3-8B" rel="nofollow"><code>NousResearch/Meta-Llama-3-8B</code></a></li>',Mt,g,Nl='<li>Clone the Optimum Neuron repository, <strong>which contains the <a href="https://github.com/huggingface/optimum-neuron/blob/main/docs/source/training_tutorials/sft_lora_finetune_llm.py" rel="nofollow">complete script</a> described in this tutorial:</strong></li>',mt,k,ut,I,Xl="<li>Make sure you have the <code>training</code> extra installed, to get all the necessary dependencies:</li>",ct,N,dt,X,Tt,G,Gl='For this tutorial, we will use <a href="https://huggingface.co/datasets/databricks/databricks-dolly-15k" rel="nofollow">Dolly</a>, an open source dataset of instruction-following records on categories outlined in the <a href="https://arxiv.org/abs/2203.02155" rel="nofollow">InstructGPT paper</a>, including brainstorming, classification, closed QA, generation, information extraction, open QA, and summarization.',ht,F,Fl="Example:",yt,x,wt,Q,xl="We can use the <code>load_dataset()</code> method from the 🤗 Datasets library to load the <code>dolly</code> dataset very easily.",ft,H,Ut,L,Ql="To instruct fine-tune our model we need to:",Jt,z,Hl=`<li><p>Convert our structured examples into collection of tasks described via instructions</p></li> <li><p>(Optional) Pack multiple examples to one sequence for more efficient training. In other words, we are stacking multiple examples into one example,
and split them with the EOS token.</p></li>`,jt,Y,Ll="We could do this manually, but we will use the <code>NeuronSFTTrainer</code> instead.",bt,q,gt,P,zl='Normally you would use the <strong><a href="https://huggingface.co/docs/trl/main/en/sft_trainer#trl.SFTConfig" rel="nofollow">SFTConfig</a></strong> and <strong><a href="https://huggingface.co/docs/trl/main/en/sft_trainer" rel="nofollow">SFTTrainer</a></strong> classes to perform supervised fine-tuning of PyTorch-based transformer models.',It,D,Yl="Instead, here we will be using the <code>NeuronSFTConfig</code> and <code>NeuronSFTTrainer</code>. These classes replicate the ones from the <code>trl</code> library while making sure they work properly on Neuron cores.",Ct,O,_t,K,ql=`There are multiple ways to give a dataset to the <code>NeuronSFTTrainer</code>, and one of them consists in providing a formatting function.
For <code>dolly</code> without packing the examples it looks as follows:`,vt,ee,Et,te,Zt,le,Pl="Since Llama-3 8B is a big model it will not fit on a single <code>trn1.32xlarge</code> instance, even with distributed training. To actually fine-tune a 8B model using only one Trainium instance we need to use both LoRA and distributed training.",Bt,C,Wt,ne,Dl=`Here, we will use tensor parallelism in conjunction with LoRA.
Our training code will look as follows:`,Rt,se,Vt,ae,Ol="The key points here are:",At,ie,Kl="<li>We use the <code>lazy_load_for_parallelism</code> context manager to lazily load the model. This will not load the full model weights on each worker, but instead only load the required weights (sharded or full). <strong>This is much more memory efficient, and often mandatory to use.</strong></li> <li>We define a <code>LoraConfig</code> that specifies which layers should have adapters, and the hyperparameters for theses adapters.</li> <li>We create a <code>NeuronSFTConfig</code> from regular <code>NeuronTrainingArguments</code>. Here we specify that we do not want to pack our examples, and that the max sequence length should be <code>1024</code>, meaning that every example will be either padded or truncated to a length of <code>1024</code>.</li> <li>We use the <code>NeuronSFTTrainer</code> to perform training. It will take the lazily loaded model, along with <code>lora_config</code>, <code>sft_config</code> and <code>format_dolly</code> and prepare the dataset and model for supervised fine-tuning.</li>",$t,oe,St,re,en='We prepared a script called <a href="https://github.com/huggingface/optimum-neuron/blob/main/docs/source/training_tutorials/sft_lora_finetune_llm.py" rel="nofollow">sft_lora_finetune_llm.py</a> summing up everything mentioned in this tutorial.',kt,pe,tn="PyTorch Neuron uses <code>torch_xla</code>. It evaluates operations lazily during the execution of the training loops, which means it builds a symbolic graph in the background, and the graph is executed on the hardware only when the tensor is printed, transferred to CPU, or when <code>xm.mark_step()</code> is called. During execution, multiple graphs can be built depending on control-flow, and it can take time to compile each graph sequentially. To alleviate that, the Neuron SDK provides <code>neuron_parallel_compile</code>, a tool which performs a fast trial run that builds all the graphs and compile them in parallel. This step is usually called precompilation.",Nt,Me,Xt,me,ln="When training models on AWS Trainium we first need to compile our model with our training arguments.",Gt,ue,nn='To ease this step, we added a <a href="https://huggingface.co/aws-neuron/optimum-neuron-cache" rel="nofollow">model cache repository</a>, which allows us to use precompiled models from the Hugging Face Hub to skip the compilation step. This is useful because it will allow to compile models much faster than what it would do when doing the actual training, because compilation can be parallelized. But be careful: every change in the model configuration might lead to a new compilation, which could result in some cache misses.',Ft,_,xt,ce,sn="The compilation command simply consists in calling your script as an input to the <code>neuron_parallel_compile</code> utility:",Qt,de,Ht,Te,an='For convenience, we saved this shell script to a file, <a href="https://github.com/huggingface/optimum-neuron/blob/main/docs/source/training_tutorials/sft_lora_finetune_llm.sh" rel="nofollow">sft_lora_finetune_llm.sh</a>. You can now pass it to the <code>neuron_parallel_compile</code> tool to trigger the compilation:',Lt,he,zt,ye,on="<em>Note: at the end of compilation, a <code>FileNotFoundError</code> message can appear. You can safely ignore it, as some compilation cache has been created.</em>",Yt,we,rn="This precompilation phase runs for 10 training steps to ensure that the compiler has compiled all the necessary graphs. It is usually enough to accumulate and compile all the graphs that will be needed during the actual training.",qt,fe,pn="<em>Note: Compiling without a cache can take a while. It will also create dummy files in the <code>dolly_llama_output</code> directory during compilation you will have to remove them afterwards.</em>",Pt,Ue,Dt,Je,Ot,je,Mn="After compilation is done we can start our actual training with a similar command, we just need to remove the use of <code>neuron_parallel_compile</code>.",Kt,be,mn="We will use <code>torchrun</code> to launch our training script. <code>torchrun</code> is a tool that automatically distributes a PyTorch model across multiple accelerators. We can pass the number of accelerators as <code>nproc_per_node</code> arguments alongside our hyperparameters.",el,ge,un="The difference to the compilation command is that we changed variables <code>max_steps=10</code> and <code>num_train_epochs=3</code>.",tl,Ie,cn="Launch the training, with the same command used in the precompilation step, but without <code>neuron_parallel_compile</code>:",ll,Ce,nl,_e,dn="That’s it, we successfully trained Llama-3 8B on AWS Trainium!",sl,ve,Tn="But before we can share and test our model we need to consolidate our model. Since we used tensor parallelism during training, we saved sharded versions of the checkpoints. We need to consolidate them now.",al,Ee,il,Ze,hn="The Optimum CLI provides a way of doing that very easily via the <code>optimum neuron consolidate [sharded_checkpoint] [output_dir]</code> command:",ol,Be,rl,We,yn="This will create an <code>adapter_model.safetensors</code> file, the LoRA adapter weights that we trained in the previous step. We can now reload the model and merge it, so it can be loaded for evaluation:",pl,Re,Ml,Ve,wn="This step can take few minutes. We now have a directory with all the files needed to evaluate the fine-tuned model.",ml,Ae,ul,$e,fn="As for training, to be able to run inference on AWS Trainium or AWS Inferentia2 we need to compile our model. In this case, we will use our Trainium instance for the inference test, but you can switch to Inferentia2 (<code>inf2.24xlarge</code>) for inference.",cl,Se,Un="Optimum Neuron implements similar to Transformers AutoModel classes for easy inference use. We will use the <code>NeuronModelForCausalLM</code> class to load our vanilla transformers checkpoint and convert it to neuron.",dl,ke,Tl,Ne,Jn="<em>Note: Inference compilation can take up to 25 minutes. Luckily, you need to only run this once. As in the precompilation step done before training, you need to run this compilation step also if you change the hardware where you run the inference, e.g. if you move from Trainium to Inferentia2. The compilation is parameter and hardware specific.</em>",hl,Xe,yl,Ge,jn="We can now test inference, but have to make sure we format our input to our prompt format we used for fine-tuning. Therefore we created a helper method, which accepts a <code>dict</code> with our <code>instruction</code> and optionally a <code>context</code>.",wl,Fe,fl,xe,bn="Let’s test inference. First we test without a context.",Ul,Qe,gn="<em>Note: Inference is not expected to be super fast on AWS Trainium using 2 cores. For Inference we recommend using Inferentia2.</em>",Jl,He,jl,Le,In="<p>AWS stands for Amazon Web Services. AWS is a suite of remote computing services offered by Amazon. The most widely used of these include Amazon Elastic Compute Cloud (Amazon EC2), which provides resizable compute capacity in the cloud; Amazon Simple Storage Service (Amazon S3), which is an object storage service; and Amazon Elastic Block Store (Amazon EBS), which is designed to provide high performance, durable block storage volumes for use with AWS instances. AWS also provides other services, such as AWS Identity and Access Management (IAM), a service that enables organizations to control access to their AWS resources, and AWS Key Management Service (AWS KMS), which helps customers create and control the use of encryption keys.</p>",bl,ze,Cn="That looks correct. Now, lets add some context, e.g. as you would do for RAG applications:",gl,Ye,Il,qe,_n="<p>You can use the Optimum Neuron interface to train models on AWS Trainium.</p>",Cl,Pe,vn="Awesome, our model also correctly uses the provided context. We are done. Congrats on fine-tuning Llama on AWS Trainium.",_l,De,vl;return v=new f({props:{title:"Supervised Fine-Tuning of Llama 3 8B on one AWS Trainium instance",local:"supervised-fine-tuning-of-llama-3-8b-on-one-aws-trainium-instance",headingTag:"h1"}}),j=new El({props:{$$slots:{default:[Nn]},$$scope:{ctx:J}}}),R=new f({props:{title:"1. Setup AWS Environment",local:"1-setup-aws-environment",headingTag:"h2"}}),$=new y({props:{code:"aHVnZ2luZ2ZhY2UtY2xpJTIwbG9naW4lMjAtLXRva2VuJTIwWU9VUl9UT0tFTg==",highlighted:"huggingface-cli login --token YOUR_TOKEN",wrap:!1}}),k=new y({props:{code:"Z2l0JTIwY2xvbmUlMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZodWdnaW5nZmFjZSUyRm9wdGltdW0tbmV1cm9uLmdpdA==",highlighted:'git <span class="hljs-built_in">clone</span> https://github.com/huggingface/optimum-neuron.git',wrap:!1}}),N=new y({props:{code:"cHl0aG9uJTIwLW0lMjBwaXAlMjBpbnN0YWxsJTIwLiU1QnRyYWluaW5nJTVE",highlighted:"python -m pip install .[training]",wrap:!1}}),X=new f({props:{title:"2. Load and prepare the dataset",local:"2-load-and-prepare-the-dataset",headingTag:"h2"}}),x=new y({props:{code:"JTdCJTBBJTIwJTIwJTIyaW5zdHJ1Y3Rpb24lMjIlM0ElMjAlMjJXaGF0JTIwaXMlMjB3b3JsZCUyMG9mJTIwd2FyY3JhZnQlMjIlMkMlMEElMjAlMjAlMjJjb250ZXh0JTIyJTNBJTIwJTIyJTIyJTJDJTBBJTIwJTIwJTIycmVzcG9uc2UlMjIlM0ElMjAoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyV29ybGQlMjBvZiUyMHdhcmNyYWZ0JTIwaXMlMjBhJTIwbWFzc2l2ZSUyMG9ubGluZSUyMG11bHRpJTIwcGxheWVyJTIwcm9sZSUyMHBsYXlpbmclMjBnYW1lLiUyMCUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMkl0JTIwd2FzJTIwcmVsZWFzZWQlMjBpbiUyMDIwMDQlMjBieSUyMGJpemFycmUlMjBlbnRlcnRhaW5tZW50JTIyJTBBJTIwJTIwJTIwJTIwKSUwQSU3RA==",highlighted:`{
  <span class="hljs-string">&quot;instruction&quot;</span>: <span class="hljs-string">&quot;What is world of warcraft&quot;</span>,
  <span class="hljs-string">&quot;context&quot;</span>: <span class="hljs-string">&quot;&quot;</span>,
  <span class="hljs-string">&quot;response&quot;</span>: (
        <span class="hljs-string">&quot;World of warcraft is a massive online multi player role playing game. &quot;</span>
        <span class="hljs-string">&quot;It was released in 2004 by bizarre entertainment&quot;</span>
    )
}`,wrap:!1}}),H=new y({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBZnJvbSUyMHJhbmRvbSUyMGltcG9ydCUyMHJhbmRyYW5nZSUwQSUwQSUyMyUyMExvYWQlMjBkYXRhc2V0JTIwZnJvbSUyMHRoZSUyMGh1YiUwQWRhdGFzZXQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyZGF0YWJyaWNrcyUyRmRhdGFicmlja3MtZG9sbHktMTVrJTIyJTJDJTIwc3BsaXQlM0QlMjJ0cmFpbiUyMiklMEElMEFwcmludChmJTIyZGF0YXNldCUyMHNpemUlM0ElMjAlN0JsZW4oZGF0YXNldCklN0QlMjIpJTBBcHJpbnQoZGF0YXNldCU1QnJhbmRyYW5nZShsZW4oZGF0YXNldCkpJTVEKSUwQSUyMyUyMGRhdGFzZXQlMjBzaXplJTNBJTIwMTUwMTE=",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> random <span class="hljs-keyword">import</span> randrange

<span class="hljs-comment"># Load dataset from the hub</span>
dataset = load_dataset(<span class="hljs-string">&quot;databricks/databricks-dolly-15k&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)

<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;dataset size: <span class="hljs-subst">{<span class="hljs-built_in">len</span>(dataset)}</span>&quot;</span>)
<span class="hljs-built_in">print</span>(dataset[randrange(<span class="hljs-built_in">len</span>(dataset))])
<span class="hljs-comment"># dataset size: 15011</span>`,wrap:!1}}),q=new f({props:{title:"3. Supervised Fine-Tuning of Llama on AWS Trainium with the NeuronSFTTrainer",local:"3-supervised-fine-tuning-of-llama-on-aws-trainium-with-the-neuronsfttrainer",headingTag:"h2"}}),O=new f({props:{title:"Formatting our dataset",local:"formatting-our-dataset",headingTag:"h3"}}),ee=new y({props:{code:"ZGVmJTIwZm9ybWF0X2RvbGx5KGV4YW1wbGVzKSUzQSUwQSUyMCUyMCUyMCUyMG91dHB1dF90ZXh0JTIwJTNEJTIwJTVCJTVEJTBBJTIwJTIwJTIwJTIwZm9yJTIwaSUyMGluJTIwcmFuZ2UobGVuKGV4YW1wbGVzJTVCJTIyaW5zdHJ1Y3Rpb24lMjIlNUQpKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGluc3RydWN0aW9uJTIwJTNEJTIwZiUyMiUyMyUyMyUyMyUyMEluc3RydWN0aW9uJTVDbiU3QmV4YW1wbGVzJTVCJ2luc3RydWN0aW9uJyU1RCU1QmklNUQlN0QlMjIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjb250ZXh0JTIwJTNEJTIwZiUyMiUyMyUyMyUyMyUyMENvbnRleHQlNUNuJTdCZXhhbXBsZXMlNUInY29udGV4dCclNUQlNUJpJTVEJTdEJTIyJTIwaWYlMjBsZW4oZXhhbXBsZXMlNUIlMjJjb250ZXh0JTIyJTVEJTVCaSU1RCklMjAlM0UlMjAwJTIwZWxzZSUyME5vbmUlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXNwb25zZSUyMCUzRCUyMGYlMjIlMjMlMjMlMjMlMjBBbnN3ZXIlNUNuJTdCZXhhbXBsZXMlNUIncmVzcG9uc2UnJTVEJTVCaSU1RCU3RCUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByb21wdCUyMCUzRCUyMCUyMiU1Q24lNUNuJTIyLmpvaW4oJTVCaSUyMGZvciUyMGklMjBpbiUyMCU1Qmluc3RydWN0aW9uJTJDJTIwY29udGV4dCUyQyUyMHJlc3BvbnNlJTVEJTIwaWYlMjBpJTIwaXMlMjBub3QlMjBOb25lJTVEKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG91dHB1dF90ZXh0LmFwcGVuZChwcm9tcHQpJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwb3V0cHV0X3RleHQ=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">format_dolly</span>(<span class="hljs-params">examples</span>):
    output_text = []
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(examples[<span class="hljs-string">&quot;instruction&quot;</span>])):
        instruction = <span class="hljs-string">f&quot;### Instruction\\n<span class="hljs-subst">{examples[<span class="hljs-string">&#x27;instruction&#x27;</span>][i]}</span>&quot;</span>
        context = <span class="hljs-string">f&quot;### Context\\n<span class="hljs-subst">{examples[<span class="hljs-string">&#x27;context&#x27;</span>][i]}</span>&quot;</span> <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(examples[<span class="hljs-string">&quot;context&quot;</span>][i]) &gt; <span class="hljs-number">0</span> <span class="hljs-keyword">else</span> <span class="hljs-literal">None</span>
        response = <span class="hljs-string">f&quot;### Answer\\n<span class="hljs-subst">{examples[<span class="hljs-string">&#x27;response&#x27;</span>][i]}</span>&quot;</span>
        prompt = <span class="hljs-string">&quot;\\n\\n&quot;</span>.join([i <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> [instruction, context, response] <span class="hljs-keyword">if</span> i <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>])
        output_text.append(prompt)
    <span class="hljs-keyword">return</span> output_text`,wrap:!1}}),te=new f({props:{title:"Preparing the model",local:"preparing-the-model",headingTag:"h3"}}),C=new El({props:{$$slots:{default:[Xn]},$$scope:{ctx:J}}}),se=new y({props:{code:"ZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBMb3JhQ29uZmlnJTBBZnJvbSUyMG9wdGltdW0ubmV1cm9uJTIwaW1wb3J0JTIwTmV1cm9uU0ZUQ29uZmlnJTJDJTIwTmV1cm9uU0ZUVHJhaW5lciUwQWZyb20lMjBvcHRpbXVtLm5ldXJvbi5kaXN0cmlidXRlZCUyMGltcG9ydCUyMGxhenlfbG9hZF9mb3JfcGFyYWxsZWxpc20lMEElMEElMjMlMjBEZWZpbmUlMjB0aGUlMjB0ZW5zb3JfcGFyYWxsZWxfc2l6ZSUwQXRlbnNvcl9wYXJhbGxlbF9zaXplJTIwJTNEJTIwMiUwQSUwQWRhdGFzZXQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyZGF0YWJyaWNrcyUyRmRhdGFicmlja3MtZG9sbHktMTVrJTIyJTJDJTIwc3BsaXQlM0QlMjJ0cmFpbiUyMiklMEElMEFtb2RlbF9pZCUyMCUzRCUyMCUyMm1ldGEtbGxhbWElMkZNZXRhLUxsYW1hLTMtOEIlMjIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9pZCklMEF0b2tlbml6ZXIucGFkX3Rva2VuJTIwJTNEJTIwdG9rZW5pemVyLmVvc190b2tlbiUwQSUwQXdpdGglMjBsYXp5X2xvYWRfZm9yX3BhcmFsbGVsaXNtKHRlbnNvcl9wYXJhbGxlbF9zaXplJTNEdGVuc29yX3BhcmFsbGVsX3NpemUpJTNBJTBBJTIwJTIwJTIwJTIwbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQpJTBBJTBBY29uZmlnJTIwJTNEJTIwTG9yYUNvbmZpZyglMEElMjAlMjAlMjAlMjByJTNEMTYlMkMlMEElMjAlMjAlMjAlMjBsb3JhX2FscGhhJTNEMTYlMkMlMEElMjAlMjAlMjAlMjBsb3JhX2Ryb3BvdXQlM0QwLjA1JTJDJTBBJTIwJTIwJTIwJTIwdGFyZ2V0X21vZHVsZXMlM0QlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJxX3Byb2olMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJnYXRlX3Byb2olMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ2X3Byb2olMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJvX3Byb2olMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJrX3Byb2olMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ1cF9wcm9qJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyZG93bl9wcm9qJTIyJTBBJTIwJTIwJTIwJTIwJTVEJTJDJTBBJTIwJTIwJTIwJTIwYmlhcyUzRCUyMm5vbmUlMjIlMkMlMEElMjAlMjAlMjAlMjB0YXNrX3R5cGUlM0QlMjJDQVVTQUxfTE0lMjIlMkMlMEEpJTBBJTBBJTIzJTIwdHJhaW5pbmdfYXJncyUyMGlzJTIwYW4lMjBpbnN0YW5jZSUyMG9mJTIwTmV1cm9uVHJhaW5pbmdBcmd1bWVudHMlMEFhcmdzJTIwJTNEJTIwdHJhaW5pbmdfYXJncy50b19kaWN0KCklMEFzZnRfY29uZmlnJTIwJTNEJTIwTmV1cm9uU0ZUQ29uZmlnKCUwQSUyMCUyMCUyMCUyMG1heF9zZXFfbGVuZ3RoJTNEMTAyNCUyQyUwQSUyMCUyMCUyMCUyMHBhY2tpbmclM0RGYWxzZSUyQyUwQSUyMCUyMCUyMCUyMCoqYXJncyUyQyUwQSklMEElMEF0cmFpbmVyJTIwJTNEJTIwTmV1cm9uU0ZUVHJhaW5lciglMEElMjAlMjAlMjAlMjBhcmdzJTNEc2Z0X2NvbmZpZyUyQyUwQSUyMCUyMCUyMCUyMG1vZGVsJTNEbW9kZWwlMkMlMEElMjAlMjAlMjAlMjBwZWZ0X2NvbmZpZyUzRGNvbmZpZyUyQyUwQSUyMCUyMCUyMCUyMHRva2VuaXplciUzRHRva2VuaXplciUyQyUwQSUyMCUyMCUyMCUyMHRyYWluX2RhdGFzZXQlM0RkYXRhc2V0JTJDJTBBJTIwJTIwJTIwJTIwZm9ybWF0dGluZ19mdW5jJTNEZm9ybWF0X2RvbGx5JTJDJTBBKSUwQSUwQSUyMyUyMFN0YXJ0JTIwdHJhaW5pbmclMEF0cmFpbmVyLnRyYWluKCklMEElMEF0cmFpbmVyLnNhdmVfbW9kZWwoKSUyMCUyMCUyMyUyMFNhdmVzJTIwdGhlJTIwdG9rZW5pemVyJTIwdG9vJTIwZm9yJTIwZWFzeSUyMHVwbG9hZA==",highlighted:`<span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> LoraConfig
<span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronSFTConfig, NeuronSFTTrainer
<span class="hljs-keyword">from</span> optimum.neuron.distributed <span class="hljs-keyword">import</span> lazy_load_for_parallelism

<span class="hljs-comment"># Define the tensor_parallel_size</span>
tensor_parallel_size = <span class="hljs-number">2</span>

dataset = load_dataset(<span class="hljs-string">&quot;databricks/databricks-dolly-15k&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)

model_id = <span class="hljs-string">&quot;meta-llama/Meta-Llama-3-8B&quot;</span>

tokenizer = AutoTokenizer.from_pretrained(model_id)
tokenizer.pad_token = tokenizer.eos_token

<span class="hljs-keyword">with</span> lazy_load_for_parallelism(tensor_parallel_size=tensor_parallel_size):
    model = AutoModelForCausalLM.from_pretrained(model_id)

config = LoraConfig(
    r=<span class="hljs-number">16</span>,
    lora_alpha=<span class="hljs-number">16</span>,
    lora_dropout=<span class="hljs-number">0.05</span>,
    target_modules=[
        <span class="hljs-string">&quot;q_proj&quot;</span>,
        <span class="hljs-string">&quot;gate_proj&quot;</span>,
        <span class="hljs-string">&quot;v_proj&quot;</span>,
        <span class="hljs-string">&quot;o_proj&quot;</span>,
        <span class="hljs-string">&quot;k_proj&quot;</span>,
        <span class="hljs-string">&quot;up_proj&quot;</span>,
        <span class="hljs-string">&quot;down_proj&quot;</span>
    ],
    bias=<span class="hljs-string">&quot;none&quot;</span>,
    task_type=<span class="hljs-string">&quot;CAUSAL_LM&quot;</span>,
)

<span class="hljs-comment"># training_args is an instance of NeuronTrainingArguments</span>
args = training_args.to_dict()
sft_config = NeuronSFTConfig(
    max_seq_length=<span class="hljs-number">1024</span>,
    packing=<span class="hljs-literal">False</span>,
    **args,
)

trainer = NeuronSFTTrainer(
    args=sft_config,
    model=model,
    peft_config=config,
    tokenizer=tokenizer,
    train_dataset=dataset,
    formatting_func=format_dolly,
)

<span class="hljs-comment"># Start training</span>
trainer.train()

trainer.save_model()  <span class="hljs-comment"># Saves the tokenizer too for easy upload</span>`,wrap:!1}}),oe=new f({props:{title:"4. Launch Training",local:"4-launch-training",headingTag:"h2"}}),Me=new f({props:{title:"Precompilation",local:"precompilation",headingTag:"h3"}}),_=new El({props:{$$slots:{default:[Gn]},$$scope:{ctx:J}}}),de=new y({props:{code:"JTIzISUyRmJpbiUyRmJhc2glMEFzZXQlMjAtZXglMEElMEFleHBvcnQlMjBORVVST05fRlVTRV9TT0ZUTUFYJTNEMSUwQWV4cG9ydCUyME5FVVJPTl9SVF9BU1lOQ19FWEVDX01BWF9JTkZMSUdIVF9SRVFVRVNUUyUzRDMlMEFleHBvcnQlMjBNQUxMT0NfQVJFTkFfTUFYJTNENjQlMjAlMjMlMjBsaW1pdCUyMHRoZSUyMENQVSUyMGFsbG9jYXRpb24lMjB0byUyMGF2b2lkJTIwcG90ZW50aWFsJTIwY3Jhc2hlcyUwQWV4cG9ydCUyME5FVVJPTl9DQ19GTEFHUyUzRCUyMi0tbW9kZWwtdHlwZSUzRHRyYW5zZm9ybWVyJTIwLS1kaXN0cmlidXRpb24tc3RyYXRlZ3klM0RsbG0tdHJhaW5pbmclMjAtLWVuYWJsZS1zYXR1cmF0ZS1pbmZpbml0eSUyMC0tY2FjaGVfZGlyJTNEJTJGaG9tZSUyRnVidW50dSUyRmNhY2hlX2Rpcl9uZXVyb24lMkYlMjIlMEElMEFQUk9DRVNTRVNfUEVSX05PREUlM0Q4JTBBJTBBVFBfREVHUkVFJTNEMiUwQVBQX0RFR1JFRSUzRDElMEFCUyUzRDElMEFHUkFESUVOVF9BQ0NVTVVMQVRJT05fU1RFUFMlM0Q4JTBBTE9HR0lOR19TVEVQUyUzRDElMEFNT0RFTF9OQU1FJTNEJTIybWV0YS1sbGFtYSUyRk1ldGEtTGxhbWEtMy04QiUyMiUwQU9VVFBVVF9ESVIlM0Rkb2xseV9sbGFtYV9vdXRwdXQlMEElMEFpZiUyMCU1QiUyMCUyMiUyNE5FVVJPTl9FWFRSQUNUX0dSQVBIU19PTkxZJTIyJTIwJTNEJTIwJTIyMSUyMiUyMCU1RCUzQiUyMHRoZW4lMEElMjAlMjAlMjAlMjBNQVhfU1RFUFMlM0QxMCUwQSUyMCUyMCUyMCUyME5VTV9FUE9DSFMlM0QxJTBBZWxzZSUwQSUyMCUyMCUyMCUyME1BWF9TVEVQUyUzRC0xJTBBJTIwJTIwJTIwJTIwTlVNX0VQT0NIUyUzRDMlMEFmaSUwQSUwQSUwQXRvcmNocnVuJTIwLS1ucHJvY19wZXJfbm9kZSUyMCUyNFBST0NFU1NFU19QRVJfTk9ERSUyMGRvY3MlMkZzb3VyY2UlMkZ0cmFpbmluZ190dXRvcmlhbHMlMkZzZnRfbG9yYV9maW5ldHVuZV9sbG0ucHklMjAlNUMlMEElMjAlMjAtLW1vZGVsX2lkJTIwJTI0TU9ERUxfTkFNRSUyMCU1QyUwQSUyMCUyMC0tbnVtX3RyYWluX2Vwb2NocyUyMCUyNE5VTV9FUE9DSFMlMjAlNUMlMEElMjAlMjAtLWRvX3RyYWluJTIwJTVDJTBBJTIwJTIwLS1sZWFybmluZ19yYXRlJTIwNWUtNSUyMCU1QyUwQSUyMCUyMC0td2FybXVwX3JhdGlvJTIwMC4wMyUyMCU1QyUwQSUyMCUyMC0tbWF4X3N0ZXBzJTIwJTI0TUFYX1NURVBTJTIwJTVDJTBBJTIwJTIwLS1wZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlMjAlMjRCUyUyMCU1QyUwQSUyMCUyMC0tcGVyX2RldmljZV9ldmFsX2JhdGNoX3NpemUlMjAlMjRCUyUyMCU1QyUwQSUyMCUyMC0tZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTIwJTI0R1JBRElFTlRfQUNDVU1VTEFUSU9OX1NURVBTJTIwJTVDJTBBJTIwJTIwLS1ncmFkaWVudF9jaGVja3BvaW50aW5nJTIwdHJ1ZSUyMCU1QyUwQSUyMCUyMC0tYmYxNiUyMCU1QyUwQSUyMCUyMC0tdGVuc29yX3BhcmFsbGVsX3NpemUlMjAlMjRUUF9ERUdSRUUlMjAlNUMlMEElMjAlMjAtLXBpcGVsaW5lX3BhcmFsbGVsX3NpemUlMjAlMjRQUF9ERUdSRUUlMjAlNUMlMEElMjAlMjAtLWxvZ2dpbmdfc3RlcHMlMjAlMjRMT0dHSU5HX1NURVBTJTIwJTVDJTBBJTIwJTIwLS1zYXZlX3RvdGFsX2xpbWl0JTIwMSUyMCU1QyUwQSUyMCUyMC0tb3V0cHV0X2RpciUyMCUyNE9VVFBVVF9ESVIlMjAlNUMlMEElMjAlMjAtLWxyX3NjaGVkdWxlcl90eXBlJTIwJTIyY29uc3RhbnQlMjIlMjAlNUMlMEElMjAlMjAtLW92ZXJ3cml0ZV9vdXRwdXRfZGly",highlighted:`<span class="hljs-meta">#!/bin/bash</span>
<span class="hljs-built_in">set</span> -ex

<span class="hljs-built_in">export</span> NEURON_FUSE_SOFTMAX=1
<span class="hljs-built_in">export</span> NEURON_RT_ASYNC_EXEC_MAX_INFLIGHT_REQUESTS=3
<span class="hljs-built_in">export</span> MALLOC_ARENA_MAX=64 <span class="hljs-comment"># limit the CPU allocation to avoid potential crashes</span>
<span class="hljs-built_in">export</span> NEURON_CC_FLAGS=<span class="hljs-string">&quot;--model-type=transformer --distribution-strategy=llm-training --enable-saturate-infinity --cache_dir=/home/ubuntu/cache_dir_neuron/&quot;</span>

PROCESSES_PER_NODE=8

TP_DEGREE=2
PP_DEGREE=1
BS=1
GRADIENT_ACCUMULATION_STEPS=8
LOGGING_STEPS=1
MODEL_NAME=<span class="hljs-string">&quot;meta-llama/Meta-Llama-3-8B&quot;</span>
OUTPUT_DIR=dolly_llama_output

<span class="hljs-keyword">if</span> [ <span class="hljs-string">&quot;<span class="hljs-variable">$NEURON_EXTRACT_GRAPHS_ONLY</span>&quot;</span> = <span class="hljs-string">&quot;1&quot;</span> ]; <span class="hljs-keyword">then</span>
    MAX_STEPS=10
    NUM_EPOCHS=1
<span class="hljs-keyword">else</span>
    MAX_STEPS=-1
    NUM_EPOCHS=3
<span class="hljs-keyword">fi</span>


torchrun --nproc_per_node <span class="hljs-variable">$PROCESSES_PER_NODE</span> docs/source/training_tutorials/sft_lora_finetune_llm.py \\
  --model_id <span class="hljs-variable">$MODEL_NAME</span> \\
  --num_train_epochs <span class="hljs-variable">$NUM_EPOCHS</span> \\
  --do_train \\
  --learning_rate 5e-5 \\
  --warmup_ratio 0.03 \\
  --max_steps <span class="hljs-variable">$MAX_STEPS</span> \\
  --per_device_train_batch_size <span class="hljs-variable">$BS</span> \\
  --per_device_eval_batch_size <span class="hljs-variable">$BS</span> \\
  --gradient_accumulation_steps <span class="hljs-variable">$GRADIENT_ACCUMULATION_STEPS</span> \\
  --gradient_checkpointing <span class="hljs-literal">true</span> \\
  --bf16 \\
  --tensor_parallel_size <span class="hljs-variable">$TP_DEGREE</span> \\
  --pipeline_parallel_size <span class="hljs-variable">$PP_DEGREE</span> \\
  --logging_steps <span class="hljs-variable">$LOGGING_STEPS</span> \\
  --save_total_limit 1 \\
  --output_dir <span class="hljs-variable">$OUTPUT_DIR</span> \\
  --lr_scheduler_type <span class="hljs-string">&quot;constant&quot;</span> \\
  --overwrite_output_dir`,wrap:!1}}),he=new y({props:{code:"bmV1cm9uX3BhcmFsbGVsX2NvbXBpbGUlMjBiYXNoJTIwZG9jcyUyRnNvdXJjZSUyRnRyYWluaW5nX3R1dG9yaWFscyUyRnNmdF9sb3JhX2ZpbmV0dW5lX2xsbS5zaA==",highlighted:"neuron_parallel_compile bash docs/source/training_tutorials/sft_lora_finetune_llm.sh",wrap:!1}}),Ue=new y({props:{code:"JTIzJTIwcmVtb3ZlJTIwZHVtbXklMjBhcnRpZmFjdHMlMjB3aGljaCUyMGFyZSUyMGNyZWF0ZWQlMjBieSUyMHRoZSUyMHByZWNvbXBpbGF0aW9uJTIwY29tbWFuZCUwQXJtJTIwLXJmJTIwZG9sbHlfbGxhbWFfb3V0cHV0",highlighted:`<span class="hljs-comment"># remove dummy artifacts which are created by the precompilation command</span>
<span class="hljs-built_in">rm</span> -rf dolly_llama_output`,wrap:!1}}),Je=new f({props:{title:"Actual Training",local:"actual-training",headingTag:"h3"}}),Ce=new y({props:{code:"YmFzaCUyMGRvY3MlMkZzb3VyY2UlMkZ0cmFpbmluZ190dXRvcmlhbHMlMkZzZnRfbG9yYV9maW5ldHVuZV9sbG0uc2glMEE=",highlighted:`bash docs/source/training_tutorials/sft_lora_finetune_llm.sh
`,wrap:!1}}),Ee=new f({props:{title:"Consolidate the checkpoint and merge model",local:"consolidate-the-checkpoint-and-merge-model",headingTag:"h3"}}),Be=new y({props:{code:"b3B0aW11bS1jbGklMjBuZXVyb24lMjBjb25zb2xpZGF0ZSUyMGRvbGx5X2xsYW1hX291dHB1dCUyMGRvbGx5X2xsYW1hX291dHB1dA==",highlighted:"optimum-cli neuron consolidate dolly_llama_output dolly_llama_output",wrap:!1}}),Re=new y({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQWZyb20lMjBwZWZ0JTIwaW1wb3J0JTIwUGVmdE1vZGVsJTJDJTIwUGVmdENvbmZpZyUwQSUwQU1PREVMX05BTUUlMjAlM0QlMjAnbWV0YS1sbGFtYSUyRk1ldGEtTGxhbWEtMy04QiclMEFBREFQVEVSX1BBVEglMjAlM0QlMjAnZG9sbHlfbGxhbWFfb3V0cHV0JyUwQU1FUkdFRF9NT0RFTF9QQVRIJTIwJTNEJTIwJ2RvbGx5X2xsYW1hJyUwQSUwQSUyMyUyMExvYWQlMjBiYXNlJTIwb2RlbCUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKE1PREVMX05BTUUpJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoTU9ERUxfTkFNRSklMEElMEElMjMlMjBMb2FkJTIwYWRhcHRlciUyMGNvbmZpZ3VyYXRpb24lMjBhbmQlMjBtb2RlbCUwQWFkYXB0ZXJfY29uZmlnJTIwJTNEJTIwUGVmdENvbmZpZy5mcm9tX3ByZXRyYWluZWQoQURBUFRFUl9QQVRIKSUwQWZpbmV0dW5lZF9tb2RlbCUyMCUzRCUyMFBlZnRNb2RlbC5mcm9tX3ByZXRyYWluZWQobW9kZWwlMkMlMjBBREFQVEVSX1BBVEglMkMlMjBjb25maWclM0RhZGFwdGVyX2NvbmZpZyklMEElMEFwcmludCglMjJTYXZpbmclMjB0b2tlbml6ZXIlMjIpJTBBdG9rZW5pemVyLnNhdmVfcHJldHJhaW5lZChNRVJHRURfTU9ERUxfUEFUSCklMEFwcmludCglMjJTYXZpbmclMjBtb2RlbCUyMiklMEFmaW5ldHVuZWRfbW9kZWwlMjAlM0QlMjBmaW5ldHVuZWRfbW9kZWwubWVyZ2VfYW5kX3VubG9hZCgpJTBBZmluZXR1bmVkX21vZGVsLnNhdmVfcHJldHJhaW5lZChNRVJHRURfTU9ERUxfUEFUSCklMEE=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> PeftModel, PeftConfig

MODEL_NAME = <span class="hljs-string">&#x27;meta-llama/Meta-Llama-3-8B&#x27;</span>
ADAPTER_PATH = <span class="hljs-string">&#x27;dolly_llama_output&#x27;</span>
MERGED_MODEL_PATH = <span class="hljs-string">&#x27;dolly_llama&#x27;</span>

<span class="hljs-comment"># Load base odel</span>
model = AutoModelForCausalLM.from_pretrained(MODEL_NAME)
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)

<span class="hljs-comment"># Load adapter configuration and model</span>
adapter_config = PeftConfig.from_pretrained(ADAPTER_PATH)
finetuned_model = PeftModel.from_pretrained(model, ADAPTER_PATH, config=adapter_config)

<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Saving tokenizer&quot;</span>)
tokenizer.save_pretrained(MERGED_MODEL_PATH)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Saving model&quot;</span>)
finetuned_model = finetuned_model.merge_and_unload()
finetuned_model.save_pretrained(MERGED_MODEL_PATH)
`,wrap:!1}}),Ae=new f({props:{title:"5. Evaluate and test fine-tuned Llama model",local:"5-evaluate-and-test-fine-tuned-llama-model",headingTag:"h2"}}),ke=new y({props:{code:"ZnJvbSUyMG9wdGltdW0ubmV1cm9uJTIwaW1wb3J0JTIwTmV1cm9uTW9kZWxGb3JDYXVzYWxMTSUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBdXRvVG9rZW5pemVyJTBBJTBBY29tcGlsZXJfYXJncyUyMCUzRCUyMCU3QiUyMm51bV9jb3JlcyUyMiUzQSUyMDIlMkMlMjAlMjJhdXRvX2Nhc3RfdHlwZSUyMiUzQSUyMCdmcDE2JyU3RCUwQWlucHV0X3NoYXBlcyUyMCUzRCUyMCU3QiUyMmJhdGNoX3NpemUlMjIlM0ElMjAxJTJDJTIwJTIyc2VxdWVuY2VfbGVuZ3RoJTIyJTNBJTIwMjA0OCU3RCUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmRvbGx5X2xsYW1hJTIyKSUwQW1vZGVsJTIwJTNEJTIwTmV1cm9uTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyZG9sbHlfbGxhbWElMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBleHBvcnQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKipjb21waWxlcl9hcmdzJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKippbnB1dF9zaGFwZXMp",highlighted:`<span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronModelForCausalLM
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

compiler_args = {<span class="hljs-string">&quot;num_cores&quot;</span>: <span class="hljs-number">2</span>, <span class="hljs-string">&quot;auto_cast_type&quot;</span>: <span class="hljs-string">&#x27;fp16&#x27;</span>}
input_shapes = {<span class="hljs-string">&quot;batch_size&quot;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&quot;sequence_length&quot;</span>: <span class="hljs-number">2048</span>}

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;dolly_llama&quot;</span>)
model = NeuronModelForCausalLM.from_pretrained(
        <span class="hljs-string">&quot;dolly_llama&quot;</span>,
        export=<span class="hljs-literal">True</span>,
        **compiler_args,
        **input_shapes)`,wrap:!1}}),Xe=new y({props:{code:"JTIzJTIwQ09NTUVOVCUyMElOJTIwaWYlMjB5b3UlMjB3YW50JTIwdG8lMjBzYXZlJTIwdGhlJTIwY29tcGlsZWQlMjBtb2RlbCUwQSUyMyUyMG1vZGVsLnNhdmVfcHJldHJhaW5lZCglMjJjb21waWxlZF9kb2xseV9sbGFtYV9vdXRwdXQlMjIp",highlighted:`<span class="hljs-comment"># COMMENT IN if you want to save the compiled model</span>
<span class="hljs-comment"># model.save_pretrained(&quot;compiled_dolly_llama_output&quot;)</span>`,wrap:!1}}),Fe=new y({props:{code:"ZGVmJTIwZm9ybWF0X2RvbGx5X2luZmVyZW5jZShzYW1wbGUpJTNBJTBBJTIwJTIwJTIwJTIwaW5zdHJ1Y3Rpb24lMjAlM0QlMjBmJTIyJTIzJTIzJTIzJTIwSW5zdHJ1Y3Rpb24lNUNuJTdCc2FtcGxlJTVCJ2luc3RydWN0aW9uJyU1RCU3RCUyMiUwQSUyMCUyMCUyMCUyMGNvbnRleHQlMjAlM0QlMjBmJTIyJTIzJTIzJTIzJTIwQ29udGV4dCU1Q24lN0JzYW1wbGUlNUInY29udGV4dCclNUQlN0QlMjIlMjBpZiUyMCUyMmNvbnRleHQlMjIlMjBpbiUyMHNhbXBsZSUyMGVsc2UlMjBOb25lJTBBJTIwJTIwJTIwJTIwcmVzcG9uc2UlMjAlM0QlMjBmJTIyJTIzJTIzJTIzJTIwQW5zd2VyJTVDbiUyMiUwQSUyMCUyMCUyMCUyMHByb21wdCUyMCUzRCUyMCUyMiU1Q24lNUNuJTIyLmpvaW4oJTVCaSUyMGZvciUyMGklMjBpbiUyMCU1Qmluc3RydWN0aW9uJTJDJTIwY29udGV4dCUyQyUyMHJlc3BvbnNlJTVEJTIwaWYlMjBpJTIwaXMlMjBub3QlMjBOb25lJTVEKSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHByb21wdCUwQSUwQSUwQWRlZiUyMGdlbmVyYXRlKHNhbXBsZSklM0ElMEElMjAlMjAlMjAlMjBwcm9tcHQlMjAlM0QlMjBmb3JtYXRfZG9sbHlfaW5mZXJlbmNlKHNhbXBsZSklMEElMjAlMjAlMjAlMjBpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIocHJvbXB0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEElMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKippbnB1dHMlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtYXhfbmV3X3Rva2VucyUzRDUxMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGRvX3NhbXBsZSUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0ZW1wZXJhdHVyZSUzRDAuOSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRvcF9rJTNENTAlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0b3BfcCUzRDAuOSUwQSUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjByZXR1cm4lMjB0b2tlbml6ZXIuZGVjb2RlKG91dHB1dHMlNUIwJTVEJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzREZhbHNlKSU1Qmxlbihwcm9tcHQpJTNBJTVE",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">format_dolly_inference</span>(<span class="hljs-params">sample</span>):
    instruction = <span class="hljs-string">f&quot;### Instruction\\n<span class="hljs-subst">{sample[<span class="hljs-string">&#x27;instruction&#x27;</span>]}</span>&quot;</span>
    context = <span class="hljs-string">f&quot;### Context\\n<span class="hljs-subst">{sample[<span class="hljs-string">&#x27;context&#x27;</span>]}</span>&quot;</span> <span class="hljs-keyword">if</span> <span class="hljs-string">&quot;context&quot;</span> <span class="hljs-keyword">in</span> sample <span class="hljs-keyword">else</span> <span class="hljs-literal">None</span>
    response = <span class="hljs-string">f&quot;### Answer\\n&quot;</span>
    prompt = <span class="hljs-string">&quot;\\n\\n&quot;</span>.join([i <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> [instruction, context, response] <span class="hljs-keyword">if</span> i <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>])
    <span class="hljs-keyword">return</span> prompt


<span class="hljs-keyword">def</span> <span class="hljs-title function_">generate</span>(<span class="hljs-params">sample</span>):
    prompt = format_dolly_inference(sample)
    inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
    outputs = model.generate(
        **inputs,
        max_new_tokens=<span class="hljs-number">512</span>,
        do_sample=<span class="hljs-literal">True</span>,
        temperature=<span class="hljs-number">0.9</span>,
        top_k=<span class="hljs-number">50</span>,
        top_p=<span class="hljs-number">0.9</span>
    )
    <span class="hljs-keyword">return</span> tokenizer.decode(outputs[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">False</span>)[<span class="hljs-built_in">len</span>(prompt):]`,wrap:!1}}),He=new y({props:{code:"cHJvbXB0JTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIyaW5zdHJ1Y3Rpb24lMjIlM0ElMjAlMjJDYW4lMjB5b3UlMjB0ZWxsJTIwbWUlMjBzb21ldGhpbmclMjBhYm91dCUyMEFXUyUzRiUyMiUwQSU3RCUwQXJlcyUyMCUzRCUyMGdlbmVyYXRlKHByb21wdCklMEElMEFwcmludChyZXMp",highlighted:`prompt = {
  <span class="hljs-string">&quot;instruction&quot;</span>: <span class="hljs-string">&quot;Can you tell me something about AWS?&quot;</span>
}
res = generate(prompt)

<span class="hljs-built_in">print</span>(res)`,wrap:!1}}),Ye=new y({props:{code:"cHJvbXB0JTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIyaW5zdHJ1Y3Rpb24lMjIlM0ElMjAlMjJIb3clMjBjYW4lMjBJJTIwdHJhaW4lMjBtb2RlbHMlMjBvbiUyMEFXUyUyMFRyYWluaXVtJTNGJTIyJTJDJTBBJTIwJTIwJTIyY29udGV4dCUyMiUzQSUyMCUyMiVGMCU5RiVBNCU5NyUyME9wdGltdW0lMjBOZXVyb24lMjBpcyUyMHRoZSUyMGludGVyZmFjZSUyMGJldHdlZW4lMjB0aGUlMjAlRjAlOUYlQTQlOTclMjBUcmFuc2Zvcm1lcnMlMjBsaWJyYXJ5JTIwYW5kJTIwQVdTJTIwQWNjZWxlcmF0b3JzJTIwaW5jbHVkaW5nJTIwJTVCQVdTJTIwVHJhaW5pdW0lNUQoaHR0cHMlM0ElMkYlMkZhd3MuYW1hem9uLmNvbSUyRm1hY2hpbmUtbGVhcm5pbmclMkZ0cmFpbml1bSUyRiUzRm5jMSUzRGhfbHMpJTIwYW5kJTIwJTVCQVdTJTIwSW5mZXJlbnRpYSU1RChodHRwcyUzQSUyRiUyRmF3cy5hbWF6b24uY29tJTJGbWFjaGluZS1sZWFybmluZyUyRmluZmVyZW50aWElMkYlM0ZuYzElM0RoX2xzKS4lMjBJdCUyMHByb3ZpZGVzJTIwYSUyMHNldCUyMG9mJTIwdG9vbHMlMjBlbmFibGluZyUyMGVhc3klMjBtb2RlbCUyMGxvYWRpbmclMkMlMjB0cmFpbmluZyUyMGFuZCUyMGluZmVyZW5jZSUyMG9uJTIwc2luZ2xlLSUyMGFuZCUyMG11bHRpLUFjY2VsZXJhdG9yJTIwc2V0dGluZ3MlMjBmb3IlMjBkaWZmZXJlbnQlMjBkb3duc3RyZWFtJTIwdGFza3MuJTIyJTBBJTdEJTBBcmVzJTIwJTNEJTIwZ2VuZXJhdGUocHJvbXB0KSUwQSUwQXByaW50KHJlcyk=",highlighted:`prompt = {
  <span class="hljs-string">&quot;instruction&quot;</span>: <span class="hljs-string">&quot;How can I train models on AWS Trainium?&quot;</span>,
  <span class="hljs-string">&quot;context&quot;</span>: <span class="hljs-string">&quot;🤗 Optimum Neuron is the interface between the 🤗 Transformers library and AWS Accelerators including [AWS Trainium](https://aws.amazon.com/machine-learning/trainium/?nc1=h_ls) and [AWS Inferentia](https://aws.amazon.com/machine-learning/inferentia/?nc1=h_ls). It provides a set of tools enabling easy model loading, training and inference on single- and multi-Accelerator settings for different downstream tasks.&quot;</span>
}
res = generate(prompt)

<span class="hljs-built_in">print</span>(res)`,wrap:!1}}),{c(){p=i("meta"),w=s(),h=i("p"),U=s(),M(v.$$.fragment),Ke=s(),E=i("p"),E.innerHTML=Bl,et=s(),Z=i("p"),Z.innerHTML=Wl,tt=s(),B=i("p"),B.textContent=Rl,lt=s(),W=i("ul"),W.innerHTML=Vl,nt=s(),M(j.$$.fragment),st=s(),M(R.$$.fragment),at=s(),V=i("p"),V.textContent=Al,it=s(),A=i("ol"),A.innerHTML=$l,ot=s(),M($.$$.fragment),rt=s(),b=i("ol"),b.innerHTML=Sl,pt=s(),S=i("ul"),S.innerHTML=kl,Mt=s(),g=i("ol"),g.innerHTML=Nl,mt=s(),M(k.$$.fragment),ut=s(),I=i("ol"),I.innerHTML=Xl,ct=s(),M(N.$$.fragment),dt=s(),M(X.$$.fragment),Tt=s(),G=i("p"),G.innerHTML=Gl,ht=s(),F=i("p"),F.textContent=Fl,yt=s(),M(x.$$.fragment),wt=s(),Q=i("p"),Q.innerHTML=xl,ft=s(),M(H.$$.fragment),Ut=s(),L=i("p"),L.textContent=Ql,Jt=s(),z=i("ol"),z.innerHTML=Hl,jt=s(),Y=i("p"),Y.innerHTML=Ll,bt=s(),M(q.$$.fragment),gt=s(),P=i("p"),P.innerHTML=zl,It=s(),D=i("p"),D.innerHTML=Yl,Ct=s(),M(O.$$.fragment),_t=s(),K=i("p"),K.innerHTML=ql,vt=s(),M(ee.$$.fragment),Et=s(),M(te.$$.fragment),Zt=s(),le=i("p"),le.innerHTML=Pl,Bt=s(),M(C.$$.fragment),Wt=s(),ne=i("p"),ne.textContent=Dl,Rt=s(),M(se.$$.fragment),Vt=s(),ae=i("p"),ae.textContent=Ol,At=s(),ie=i("ul"),ie.innerHTML=Kl,$t=s(),M(oe.$$.fragment),St=s(),re=i("p"),re.innerHTML=en,kt=s(),pe=i("p"),pe.innerHTML=tn,Nt=s(),M(Me.$$.fragment),Xt=s(),me=i("p"),me.textContent=ln,Gt=s(),ue=i("p"),ue.innerHTML=nn,Ft=s(),M(_.$$.fragment),xt=s(),ce=i("p"),ce.innerHTML=sn,Qt=s(),M(de.$$.fragment),Ht=s(),Te=i("p"),Te.innerHTML=an,Lt=s(),M(he.$$.fragment),zt=s(),ye=i("p"),ye.innerHTML=on,Yt=s(),we=i("p"),we.textContent=rn,qt=s(),fe=i("p"),fe.innerHTML=pn,Pt=s(),M(Ue.$$.fragment),Dt=s(),M(Je.$$.fragment),Ot=s(),je=i("p"),je.innerHTML=Mn,Kt=s(),be=i("p"),be.innerHTML=mn,el=s(),ge=i("p"),ge.innerHTML=un,tl=s(),Ie=i("p"),Ie.innerHTML=cn,ll=s(),M(Ce.$$.fragment),nl=s(),_e=i("p"),_e.textContent=dn,sl=s(),ve=i("p"),ve.textContent=Tn,al=s(),M(Ee.$$.fragment),il=s(),Ze=i("p"),Ze.innerHTML=hn,ol=s(),M(Be.$$.fragment),rl=s(),We=i("p"),We.innerHTML=yn,pl=s(),M(Re.$$.fragment),Ml=s(),Ve=i("p"),Ve.textContent=wn,ml=s(),M(Ae.$$.fragment),ul=s(),$e=i("p"),$e.innerHTML=fn,cl=s(),Se=i("p"),Se.innerHTML=Un,dl=s(),M(ke.$$.fragment),Tl=s(),Ne=i("p"),Ne.innerHTML=Jn,hl=s(),M(Xe.$$.fragment),yl=s(),Ge=i("p"),Ge.innerHTML=jn,wl=s(),M(Fe.$$.fragment),fl=s(),xe=i("p"),xe.textContent=bn,Ul=s(),Qe=i("p"),Qe.innerHTML=gn,Jl=s(),M(He.$$.fragment),jl=s(),Le=i("blockquote"),Le.innerHTML=In,bl=s(),ze=i("p"),ze.textContent=Cn,gl=s(),M(Ye.$$.fragment),Il=s(),qe=i("blockquote"),qe.innerHTML=_n,Cl=s(),Pe=i("p"),Pe.textContent=vn,_l=s(),De=i("p"),this.h()},l(e){const t=Sn("svelte-u9bgzb",document.head);p=o(t,"META",{name:!0,content:!0}),t.forEach(l),w=a(e),h=o(e,"P",{}),Wn(h).forEach(l),U=a(e),m(v.$$.fragment,e),Ke=a(e),E=o(e,"P",{"data-svelte-h":!0}),r(E)!=="svelte-uoutuw"&&(E.innerHTML=Bl),et=a(e),Z=o(e,"P",{"data-svelte-h":!0}),r(Z)!=="svelte-1w2bqvx"&&(Z.innerHTML=Wl),tt=a(e),B=o(e,"P",{"data-svelte-h":!0}),r(B)!=="svelte-1hahfn0"&&(B.textContent=Rl),lt=a(e),W=o(e,"UL",{"data-svelte-h":!0}),r(W)!=="svelte-1bkg6hx"&&(W.innerHTML=Vl),nt=a(e),m(j.$$.fragment,e),st=a(e),m(R.$$.fragment,e),at=a(e),V=o(e,"P",{"data-svelte-h":!0}),r(V)!=="svelte-w52dpm"&&(V.textContent=Al),it=a(e),A=o(e,"OL",{"data-svelte-h":!0}),r(A)!=="svelte-15ega2l"&&(A.innerHTML=$l),ot=a(e),m($.$$.fragment,e),rt=a(e),b=o(e,"OL",{start:!0,"data-svelte-h":!0}),r(b)!=="svelte-ksb30n"&&(b.innerHTML=Sl),pt=a(e),S=o(e,"UL",{"data-svelte-h":!0}),r(S)!=="svelte-1qiwkz0"&&(S.innerHTML=kl),Mt=a(e),g=o(e,"OL",{start:!0,"data-svelte-h":!0}),r(g)!=="svelte-qhor5j"&&(g.innerHTML=Nl),mt=a(e),m(k.$$.fragment,e),ut=a(e),I=o(e,"OL",{start:!0,"data-svelte-h":!0}),r(I)!=="svelte-dm3bsw"&&(I.innerHTML=Xl),ct=a(e),m(N.$$.fragment,e),dt=a(e),m(X.$$.fragment,e),Tt=a(e),G=o(e,"P",{"data-svelte-h":!0}),r(G)!=="svelte-s6kf7j"&&(G.innerHTML=Gl),ht=a(e),F=o(e,"P",{"data-svelte-h":!0}),r(F)!=="svelte-11lpom8"&&(F.textContent=Fl),yt=a(e),m(x.$$.fragment,e),wt=a(e),Q=o(e,"P",{"data-svelte-h":!0}),r(Q)!=="svelte-1snwaj0"&&(Q.innerHTML=xl),ft=a(e),m(H.$$.fragment,e),Ut=a(e),L=o(e,"P",{"data-svelte-h":!0}),r(L)!=="svelte-1993zti"&&(L.textContent=Ql),Jt=a(e),z=o(e,"OL",{"data-svelte-h":!0}),r(z)!=="svelte-5ew6jj"&&(z.innerHTML=Hl),jt=a(e),Y=o(e,"P",{"data-svelte-h":!0}),r(Y)!=="svelte-1u9ozqi"&&(Y.innerHTML=Ll),bt=a(e),m(q.$$.fragment,e),gt=a(e),P=o(e,"P",{"data-svelte-h":!0}),r(P)!=="svelte-uhn59v"&&(P.innerHTML=zl),It=a(e),D=o(e,"P",{"data-svelte-h":!0}),r(D)!=="svelte-1js0azg"&&(D.innerHTML=Yl),Ct=a(e),m(O.$$.fragment,e),_t=a(e),K=o(e,"P",{"data-svelte-h":!0}),r(K)!=="svelte-1w3fqpa"&&(K.innerHTML=ql),vt=a(e),m(ee.$$.fragment,e),Et=a(e),m(te.$$.fragment,e),Zt=a(e),le=o(e,"P",{"data-svelte-h":!0}),r(le)!=="svelte-sho0uq"&&(le.innerHTML=Pl),Bt=a(e),m(C.$$.fragment,e),Wt=a(e),ne=o(e,"P",{"data-svelte-h":!0}),r(ne)!=="svelte-1mymbn5"&&(ne.textContent=Dl),Rt=a(e),m(se.$$.fragment,e),Vt=a(e),ae=o(e,"P",{"data-svelte-h":!0}),r(ae)!=="svelte-t16ze3"&&(ae.textContent=Ol),At=a(e),ie=o(e,"UL",{"data-svelte-h":!0}),r(ie)!=="svelte-e6qxb9"&&(ie.innerHTML=Kl),$t=a(e),m(oe.$$.fragment,e),St=a(e),re=o(e,"P",{"data-svelte-h":!0}),r(re)!=="svelte-15svx9v"&&(re.innerHTML=en),kt=a(e),pe=o(e,"P",{"data-svelte-h":!0}),r(pe)!=="svelte-1vufr00"&&(pe.innerHTML=tn),Nt=a(e),m(Me.$$.fragment,e),Xt=a(e),me=o(e,"P",{"data-svelte-h":!0}),r(me)!=="svelte-1u0hmu2"&&(me.textContent=ln),Gt=a(e),ue=o(e,"P",{"data-svelte-h":!0}),r(ue)!=="svelte-fa7eag"&&(ue.innerHTML=nn),Ft=a(e),m(_.$$.fragment,e),xt=a(e),ce=o(e,"P",{"data-svelte-h":!0}),r(ce)!=="svelte-t9z3bd"&&(ce.innerHTML=sn),Qt=a(e),m(de.$$.fragment,e),Ht=a(e),Te=o(e,"P",{"data-svelte-h":!0}),r(Te)!=="svelte-15xsd3t"&&(Te.innerHTML=an),Lt=a(e),m(he.$$.fragment,e),zt=a(e),ye=o(e,"P",{"data-svelte-h":!0}),r(ye)!=="svelte-2ey5d2"&&(ye.innerHTML=on),Yt=a(e),we=o(e,"P",{"data-svelte-h":!0}),r(we)!=="svelte-tgf31c"&&(we.textContent=rn),qt=a(e),fe=o(e,"P",{"data-svelte-h":!0}),r(fe)!=="svelte-1q7xoex"&&(fe.innerHTML=pn),Pt=a(e),m(Ue.$$.fragment,e),Dt=a(e),m(Je.$$.fragment,e),Ot=a(e),je=o(e,"P",{"data-svelte-h":!0}),r(je)!=="svelte-1yiqpri"&&(je.innerHTML=Mn),Kt=a(e),be=o(e,"P",{"data-svelte-h":!0}),r(be)!=="svelte-1anw499"&&(be.innerHTML=mn),el=a(e),ge=o(e,"P",{"data-svelte-h":!0}),r(ge)!=="svelte-106984l"&&(ge.innerHTML=un),tl=a(e),Ie=o(e,"P",{"data-svelte-h":!0}),r(Ie)!=="svelte-11khgeu"&&(Ie.innerHTML=cn),ll=a(e),m(Ce.$$.fragment,e),nl=a(e),_e=o(e,"P",{"data-svelte-h":!0}),r(_e)!=="svelte-1v2czzn"&&(_e.textContent=dn),sl=a(e),ve=o(e,"P",{"data-svelte-h":!0}),r(ve)!=="svelte-1wuuxgn"&&(ve.textContent=Tn),al=a(e),m(Ee.$$.fragment,e),il=a(e),Ze=o(e,"P",{"data-svelte-h":!0}),r(Ze)!=="svelte-66io75"&&(Ze.innerHTML=hn),ol=a(e),m(Be.$$.fragment,e),rl=a(e),We=o(e,"P",{"data-svelte-h":!0}),r(We)!=="svelte-1cigzjn"&&(We.innerHTML=yn),pl=a(e),m(Re.$$.fragment,e),Ml=a(e),Ve=o(e,"P",{"data-svelte-h":!0}),r(Ve)!=="svelte-17mrerr"&&(Ve.textContent=wn),ml=a(e),m(Ae.$$.fragment,e),ul=a(e),$e=o(e,"P",{"data-svelte-h":!0}),r($e)!=="svelte-1s116uq"&&($e.innerHTML=fn),cl=a(e),Se=o(e,"P",{"data-svelte-h":!0}),r(Se)!=="svelte-1pgrxrm"&&(Se.innerHTML=Un),dl=a(e),m(ke.$$.fragment,e),Tl=a(e),Ne=o(e,"P",{"data-svelte-h":!0}),r(Ne)!=="svelte-px4ach"&&(Ne.innerHTML=Jn),hl=a(e),m(Xe.$$.fragment,e),yl=a(e),Ge=o(e,"P",{"data-svelte-h":!0}),r(Ge)!=="svelte-jx2yqv"&&(Ge.innerHTML=jn),wl=a(e),m(Fe.$$.fragment,e),fl=a(e),xe=o(e,"P",{"data-svelte-h":!0}),r(xe)!=="svelte-1ytr5g6"&&(xe.textContent=bn),Ul=a(e),Qe=o(e,"P",{"data-svelte-h":!0}),r(Qe)!=="svelte-1j8w2o0"&&(Qe.innerHTML=gn),Jl=a(e),m(He.$$.fragment,e),jl=a(e),Le=o(e,"BLOCKQUOTE",{"data-svelte-h":!0}),r(Le)!=="svelte-6l4k0q"&&(Le.innerHTML=In),bl=a(e),ze=o(e,"P",{"data-svelte-h":!0}),r(ze)!=="svelte-ovzcvi"&&(ze.textContent=Cn),gl=a(e),m(Ye.$$.fragment,e),Il=a(e),qe=o(e,"BLOCKQUOTE",{"data-svelte-h":!0}),r(qe)!=="svelte-15xrtpx"&&(qe.innerHTML=_n),Cl=a(e),Pe=o(e,"P",{"data-svelte-h":!0}),r(Pe)!=="svelte-q9f4rf"&&(Pe.textContent=vn),_l=a(e),De=o(e,"P",{}),Wn(De).forEach(l),this.h()},h(){Oe(p,"name","hf:doc:metadata"),Oe(p,"content",xn),Oe(b,"start","3"),Oe(g,"start","4"),Oe(I,"start","5")},m(e,t){kn(document.head,p),n(e,w,t),n(e,h,t),n(e,U,t),u(v,e,t),n(e,Ke,t),n(e,E,t),n(e,et,t),n(e,Z,t),n(e,tt,t),n(e,B,t),n(e,lt,t),n(e,W,t),n(e,nt,t),u(j,e,t),n(e,st,t),u(R,e,t),n(e,at,t),n(e,V,t),n(e,it,t),n(e,A,t),n(e,ot,t),u($,e,t),n(e,rt,t),n(e,b,t),n(e,pt,t),n(e,S,t),n(e,Mt,t),n(e,g,t),n(e,mt,t),u(k,e,t),n(e,ut,t),n(e,I,t),n(e,ct,t),u(N,e,t),n(e,dt,t),u(X,e,t),n(e,Tt,t),n(e,G,t),n(e,ht,t),n(e,F,t),n(e,yt,t),u(x,e,t),n(e,wt,t),n(e,Q,t),n(e,ft,t),u(H,e,t),n(e,Ut,t),n(e,L,t),n(e,Jt,t),n(e,z,t),n(e,jt,t),n(e,Y,t),n(e,bt,t),u(q,e,t),n(e,gt,t),n(e,P,t),n(e,It,t),n(e,D,t),n(e,Ct,t),u(O,e,t),n(e,_t,t),n(e,K,t),n(e,vt,t),u(ee,e,t),n(e,Et,t),u(te,e,t),n(e,Zt,t),n(e,le,t),n(e,Bt,t),u(C,e,t),n(e,Wt,t),n(e,ne,t),n(e,Rt,t),u(se,e,t),n(e,Vt,t),n(e,ae,t),n(e,At,t),n(e,ie,t),n(e,$t,t),u(oe,e,t),n(e,St,t),n(e,re,t),n(e,kt,t),n(e,pe,t),n(e,Nt,t),u(Me,e,t),n(e,Xt,t),n(e,me,t),n(e,Gt,t),n(e,ue,t),n(e,Ft,t),u(_,e,t),n(e,xt,t),n(e,ce,t),n(e,Qt,t),u(de,e,t),n(e,Ht,t),n(e,Te,t),n(e,Lt,t),u(he,e,t),n(e,zt,t),n(e,ye,t),n(e,Yt,t),n(e,we,t),n(e,qt,t),n(e,fe,t),n(e,Pt,t),u(Ue,e,t),n(e,Dt,t),u(Je,e,t),n(e,Ot,t),n(e,je,t),n(e,Kt,t),n(e,be,t),n(e,el,t),n(e,ge,t),n(e,tl,t),n(e,Ie,t),n(e,ll,t),u(Ce,e,t),n(e,nl,t),n(e,_e,t),n(e,sl,t),n(e,ve,t),n(e,al,t),u(Ee,e,t),n(e,il,t),n(e,Ze,t),n(e,ol,t),u(Be,e,t),n(e,rl,t),n(e,We,t),n(e,pl,t),u(Re,e,t),n(e,Ml,t),n(e,Ve,t),n(e,ml,t),u(Ae,e,t),n(e,ul,t),n(e,$e,t),n(e,cl,t),n(e,Se,t),n(e,dl,t),u(ke,e,t),n(e,Tl,t),n(e,Ne,t),n(e,hl,t),u(Xe,e,t),n(e,yl,t),n(e,Ge,t),n(e,wl,t),u(Fe,e,t),n(e,fl,t),n(e,xe,t),n(e,Ul,t),n(e,Qe,t),n(e,Jl,t),u(He,e,t),n(e,jl,t),n(e,Le,t),n(e,bl,t),n(e,ze,t),n(e,gl,t),u(Ye,e,t),n(e,Il,t),n(e,qe,t),n(e,Cl,t),n(e,Pe,t),n(e,_l,t),n(e,De,t),vl=!0},p(e,[t]){const En={};t&2&&(En.$$scope={dirty:t,ctx:e}),j.$set(En);const Zn={};t&2&&(Zn.$$scope={dirty:t,ctx:e}),C.$set(Zn);const Bn={};t&2&&(Bn.$$scope={dirty:t,ctx:e}),_.$set(Bn)},i(e){vl||(c(v.$$.fragment,e),c(j.$$.fragment,e),c(R.$$.fragment,e),c($.$$.fragment,e),c(k.$$.fragment,e),c(N.$$.fragment,e),c(X.$$.fragment,e),c(x.$$.fragment,e),c(H.$$.fragment,e),c(q.$$.fragment,e),c(O.$$.fragment,e),c(ee.$$.fragment,e),c(te.$$.fragment,e),c(C.$$.fragment,e),c(se.$$.fragment,e),c(oe.$$.fragment,e),c(Me.$$.fragment,e),c(_.$$.fragment,e),c(de.$$.fragment,e),c(he.$$.fragment,e),c(Ue.$$.fragment,e),c(Je.$$.fragment,e),c(Ce.$$.fragment,e),c(Ee.$$.fragment,e),c(Be.$$.fragment,e),c(Re.$$.fragment,e),c(Ae.$$.fragment,e),c(ke.$$.fragment,e),c(Xe.$$.fragment,e),c(Fe.$$.fragment,e),c(He.$$.fragment,e),c(Ye.$$.fragment,e),vl=!0)},o(e){d(v.$$.fragment,e),d(j.$$.fragment,e),d(R.$$.fragment,e),d($.$$.fragment,e),d(k.$$.fragment,e),d(N.$$.fragment,e),d(X.$$.fragment,e),d(x.$$.fragment,e),d(H.$$.fragment,e),d(q.$$.fragment,e),d(O.$$.fragment,e),d(ee.$$.fragment,e),d(te.$$.fragment,e),d(C.$$.fragment,e),d(se.$$.fragment,e),d(oe.$$.fragment,e),d(Me.$$.fragment,e),d(_.$$.fragment,e),d(de.$$.fragment,e),d(he.$$.fragment,e),d(Ue.$$.fragment,e),d(Je.$$.fragment,e),d(Ce.$$.fragment,e),d(Ee.$$.fragment,e),d(Be.$$.fragment,e),d(Re.$$.fragment,e),d(Ae.$$.fragment,e),d(ke.$$.fragment,e),d(Xe.$$.fragment,e),d(Fe.$$.fragment,e),d(He.$$.fragment,e),d(Ye.$$.fragment,e),vl=!1},d(e){e&&(l(w),l(h),l(U),l(Ke),l(E),l(et),l(Z),l(tt),l(B),l(lt),l(W),l(nt),l(st),l(at),l(V),l(it),l(A),l(ot),l(rt),l(b),l(pt),l(S),l(Mt),l(g),l(mt),l(ut),l(I),l(ct),l(dt),l(Tt),l(G),l(ht),l(F),l(yt),l(wt),l(Q),l(ft),l(Ut),l(L),l(Jt),l(z),l(jt),l(Y),l(bt),l(gt),l(P),l(It),l(D),l(Ct),l(_t),l(K),l(vt),l(Et),l(Zt),l(le),l(Bt),l(Wt),l(ne),l(Rt),l(Vt),l(ae),l(At),l(ie),l($t),l(St),l(re),l(kt),l(pe),l(Nt),l(Xt),l(me),l(Gt),l(ue),l(Ft),l(xt),l(ce),l(Qt),l(Ht),l(Te),l(Lt),l(zt),l(ye),l(Yt),l(we),l(qt),l(fe),l(Pt),l(Dt),l(Ot),l(je),l(Kt),l(be),l(el),l(ge),l(tl),l(Ie),l(ll),l(nl),l(_e),l(sl),l(ve),l(al),l(il),l(Ze),l(ol),l(rl),l(We),l(pl),l(Ml),l(Ve),l(ml),l(ul),l($e),l(cl),l(Se),l(dl),l(Tl),l(Ne),l(hl),l(yl),l(Ge),l(wl),l(fl),l(xe),l(Ul),l(Qe),l(Jl),l(jl),l(Le),l(bl),l(ze),l(gl),l(Il),l(qe),l(Cl),l(Pe),l(_l),l(De)),l(p),T(v,e),T(j,e),T(R,e),T($,e),T(k,e),T(N,e),T(X,e),T(x,e),T(H,e),T(q,e),T(O,e),T(ee,e),T(te,e),T(C,e),T(se,e),T(oe,e),T(Me,e),T(_,e),T(de,e),T(he,e),T(Ue,e),T(Je,e),T(Ce,e),T(Ee,e),T(Be,e),T(Re,e),T(Ae,e),T(ke,e),T(Xe,e),T(Fe,e),T(He,e),T(Ye,e)}}}const xn='{"title":"Supervised Fine-Tuning of Llama 3 8B on one AWS Trainium instance","local":"supervised-fine-tuning-of-llama-3-8b-on-one-aws-trainium-instance","sections":[{"title":"1. Setup AWS Environment","local":"1-setup-aws-environment","sections":[],"depth":2},{"title":"2. Load and prepare the dataset","local":"2-load-and-prepare-the-dataset","sections":[],"depth":2},{"title":"3. Supervised Fine-Tuning of Llama on AWS Trainium with the NeuronSFTTrainer","local":"3-supervised-fine-tuning-of-llama-on-aws-trainium-with-the-neuronsfttrainer","sections":[{"title":"Formatting our dataset","local":"formatting-our-dataset","sections":[],"depth":3},{"title":"Preparing the model","local":"preparing-the-model","sections":[],"depth":3}],"depth":2},{"title":"4. Launch Training","local":"4-launch-training","sections":[{"title":"Precompilation","local":"precompilation","sections":[],"depth":3},{"title":"Actual Training","local":"actual-training","sections":[],"depth":3},{"title":"Consolidate the checkpoint and merge model","local":"consolidate-the-checkpoint-and-merge-model","sections":[],"depth":3}],"depth":2},{"title":"5. Evaluate and test fine-tuned Llama model","local":"5-evaluate-and-test-fine-tuned-llama-model","sections":[],"depth":2}],"depth":1}';function Qn(J){return Vn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Pn extends An{constructor(p){super(),$n(this,p,Qn,Fn,Rn,{})}}export{Pn as component};