course/pr_1149/en/_app/immutable/nodes/38.b5bc2dcc.js

import{s as ln,o as nn}from"../chunks/scheduler.893fe8c9.js";import{S as an,i as tn,e as u,s as a,c as p,h as pn,a as h,d as l,b as t,f as sn,g as M,j as i,k as en,l as Mn,m as n,n as r,t as m,o as y,p as j}from"../chunks/index.b1df2166.js";import{C as rn,H as c,E as mn}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.bd52ba30.js";import{Y as yn}from"../chunks/Youtube.ec5d7916.js";import{C as J}from"../chunks/CodeBlock.eaf0b265.js";import{C as jn}from"../chunks/CourseFloatingBanner.c1c08878.js";import{F as un}from"../chunks/FrameworkSwitchCourse.4480e339.js";function hn(cl){let T,zs,b,Rs,w,Gs,o,qs,U,Ls,d,Fs,I,Ys,f,bl="In this section, we’ll take a closer look at creating and using models. We’ll use the <code>AutoModel</code> class, which is handy when you want to instantiate any model from a checkpoint.",Ps,A,Ks,k,wl="Let’s begin by examining what happens when we instantiate an <code>AutoModel</code>:",Os,$,se,D,ol='Similar to the tokenizer, the <code>from_pretrained()</code> method will download and cache the model data from the Hugging Face Hub. As mentioned previously, the checkpoint name corresponds to a specific model architecture and weights, in this case a BERT model with a basic architecture (12 layers, 768 hidden size, 12 attention heads) and cased inputs (meaning that the uppercase/lowercase distinction is important). There are many checkpoints available on the Hub — you can explore them <a href="https://huggingface.co/models" rel="nofollow">here</a>.',ee,C,Ul="The <code>AutoModel</code> class and its associates are actually simple wrappers designed to fetch the appropriate model architecture for a given checkpoint. It’s an “auto” class meaning it will guess the appropriate model architecture for you and instantiate the correct model class. However, if you know the type of model you want to use, you can use the class that defines its architecture directly:",le,g,ne,x,ae,E,dl="Saving a model is as simple as saving a tokenizer. In fact, the models actually have the same <code>save_pretrained()</code> method, which saves the model’s weights and architecture configuration:",te,Q,pe,v,Il="This will save two files to your disk:",Me,N,re,Z,fl="If you look inside the <em>config.json</em> file, you’ll see all the necessary attributes needed to build the model architecture. This file also contains some metadata, such as where the checkpoint originated and what 🤗 Transformers version you were using when you last saved the checkpoint.",me,S,Al="The <em>pytorch_model.safetensors</em> file is known as the state dictionary; it contains all your model’s weights. The two files work together: the configuration file is needed to know about the model architecture, while the model weights are the parameters of the model.",ye,V,kl="To reuse a saved model, use the <code>from_pretrained()</code> method again:",je,_,ue,B,$l='A wonderful feature of the 🤗 Transformers library is the ability to easily share models and tokenizers with the community. To do this, make sure you have an account on <a href="https://huggingface.co" rel="nofollow">Hugging Face</a>. If you’re using a notebook, you can easily log in with this:',he,H,ie,W,Dl="Otherwise, at your terminal run:",Je,z,Te,R,Cl="Then you can push the model to the Hub with the <code>push_to_hub()</code> method:",ce,X,be,G,gl="This will upload the model files to the Hub, in a repository under your namespace named <em>my-awesome-model</em>. Then, anyone can load your model with the <code>from_pretrained()</code> method!",we,q,oe,L,xl="You can do a lot more with the Hub API:",Ue,F,El="<li>Push a model from a local repository</li> <li>Update specific files without re-uploading everything</li> <li>Add model cards to document the model’s abilities, limitations, known biases, etc.</li>",de,Y,Ql='See <a href="https://huggingface.co/docs/huggingface_hub/how-to-upstream" rel="nofollow">the documentation</a> for a complete tutorial on this, or check out the advanced <a href="/course/chapter4">Chapter 4</a>.',Ie,P,fe,K,vl='Transformer models handle text by turning the inputs into numbers. Here we will look at exactly what happens when your text is processed by the tokenizer. We’ve already seen in <a href="/course/chapter1">Chapter 1</a> that tokenizers split the text into tokens and then convert these tokens into numbers. We can see this conversion through a simple tokenizer:',Ae,O,ke,ss,$e,es,Nl="We get a dictionary with the following fields:",De,ls,Zl="<li>input_ids: numerical representations of your tokens</li> <li>token_type_ids: these tell the model which part of the input is sentence A and which is sentence B (discussed more in the next section)</li> <li>attention_mask: this indicates which tokens should be attended to and which should not (discussed more in a bit)</li>",Ce,ns,Sl="We can decode the input IDs to get back the original text:",ge,as,xe,ts,Ee,ps,Vl="You’ll notice that the tokenizer has added special tokens — <code>[CLS]</code> and <code>[SEP]</code> — required by the model. Not all models need special tokens; they’re utilized when a model was pretrained with them, in which case the tokenizer needs to add them as that model expects these tokens.",Qe,Ms,_l="You can encode multiple sentences at once, either by batching them together (we’ll discuss this soon) or by passing a list:",ve,rs,Ne,ms,Ze,ys,Bl="Note that when passing multiple sentences, the tokenizer returns a list for each sentence for each dictionary value. We can also ask the tokenizer to return tensors directly from PyTorch:",Se,js,Ve,us,_e,hs,Hl="But there’s a problem: the two lists don’t have the same length! Arrays and tensors need to be rectangular, so we can’t simply convert these lists to a PyTorch tensor (or NumPy array). The tokenizer provides an option for that: padding.",Be,is,He,Js,Wl="If we ask the tokenizer to pad the inputs, it will make all sentences the same length by adding a special padding token to the sentences that are shorter than the longest one:",We,Ts,ze,cs,Re,bs,zl="Now we have rectangular tensors! Note that the padding tokens have been encoded into input IDs with ID 0, and they have an attention mask value of 0 as well. This is because those padding tokens shouldn’t be analyzed by the model: they’re not part of the actual sentence.",Xe,ws,Ge,os,Rl="The tensors might get too big to be processed by the model. For instance, BERT was only pretrained with sequences up to 512 tokens, so it cannot process longer sequences. If you have sequences longer than the model can handle, you’ll need to truncate them with the <code>truncation</code> parameter:",qe,Us,Le,ds,Fe,Is,Xl="By combining the padding and truncation arguments, you can make sure your tensors have the exact size you need:",Ye,fs,Pe,As,Ke,ks,Oe,$s,Gl="Special tokens (or at least the concept of them) is particularly important to BERT and derived models. These tokens are added to better represent the sentence boundaries, such as the beginning of a sentence (<code>[CLS]</code>) or separator between sentences (<code>[SEP]</code>). Let’s look at a simple example:",sl,Ds,el,Cs,ll,gs,ql="These special tokens are automatically added by the tokenizer. Not all models need special tokens; they are primarily used when a model was pretrained with them, in which case the tokenizer will add them since the model expects them.",nl,xs,al,Es,Ll="Here’s a concrete example. Consider these encoded sequences:",tl,Qs,pl,vs,Fl="Once tokenized, we have:",Ml,Ns,rl,Zs,Yl="This is a list of encoded sequences: a list of lists. Tensors only accept rectangular shapes (think matrices). This “array” is already of rectangular shape, so converting it to a tensor is easy:",ml,Ss,yl,Vs,jl,_s,Pl="Making use of the tensors with the model is extremely simple — we just call the model with the inputs:",ul,Bs,hl,Hs,Kl=`While the model accepts a lot of different arguments, only the input IDs are necessary. We’ll explain what the other arguments do and when they are required later,
but first we need to take a closer look at the tokenizers that build the inputs that a Transformer model can understand.`,il,Ws,Jl,Xs,Tl;return w=new un({props:{fw:cl[0]}}),o=new rn({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),U=new c({props:{title:"Models",local:"the-models",headingTag:"h1"}}),d=new jn({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter2/section3_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter2/section3_pt.ipynb"}]}}),I=new yn({props:{id:"AhChOFRegn4"}}),A=new c({props:{title:"Creating a Transformer",local:"creating-a-transformer",headingTag:"h2"}}),$=new J({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMjJiZXJ0LWJhc2UtY2FzZWQlMjIp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModel

model = AutoModel.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`,wrap:!1}}),g=new J({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJlcnRNb2RlbCUwQSUwQW1vZGVsJTIwJTNEJTIwQmVydE1vZGVsLmZyb21fcHJldHJhaW5lZCglMjJiZXJ0LWJhc2UtY2FzZWQlMjIp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BertModel

model = BertModel.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`,wrap:!1}}),x=new c({props:{title:"Loading and saving",local:"loading-and-saving",headingTag:"h2"}}),Q=new J({props:{code:"bW9kZWwuc2F2ZV9wcmV0cmFpbmVkKCUyMmRpcmVjdG9yeV9vbl9teV9jb21wdXRlciUyMik=",highlighted:'model.save_pretrained(<span class="hljs-string">&quot;directory_on_my_computer&quot;</span>)',wrap:!1}}),N=new J({props:{code:"bHMlMjBkaXJlY3Rvcnlfb25fbXlfY29tcHV0ZXIlMEElMEFjb25maWcuanNvbiUyMG1vZGVsLnNhZmV0ZW5zb3Jz",highlighted:`ls <span class="hljs-keyword">directory_on_my_computer
</span>
<span class="hljs-built_in">config</span>.<span class="hljs-keyword">json </span>model.safetensors`,wrap:!1}}),_=new J({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMjJkaXJlY3Rvcnlfb25fbXlfY29tcHV0ZXIlMjIp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModel

model = AutoModel.from_pretrained(<span class="hljs-string">&quot;directory_on_my_computer&quot;</span>)`,wrap:!1}}),H=new J({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`,wrap:!1}}),z=new J({props:{code:"aHVnZ2luZ2ZhY2UtY2xpJTIwbG9naW4=",highlighted:"huggingface-cli login",wrap:!1}}),X=new J({props:{code:"bW9kZWwucHVzaF90b19odWIoJTIybXktYXdlc29tZS1tb2RlbCUyMik=",highlighted:'model.push_to_hub(<span class="hljs-string">&quot;my-awesome-model&quot;</span>)',wrap:!1}}),q=new J({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMjJ5b3VyLXVzZXJuYW1lJTJGbXktYXdlc29tZS1tb2RlbCUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModel

model = AutoModel.from_pretrained(<span class="hljs-string">&quot;your-username/my-awesome-model&quot;</span>)`,wrap:!1}}),P=new c({props:{title:"Encoding text",local:"encoding-text",headingTag:"h2"}}),O=new J({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJiZXJ0LWJhc2UtY2FzZWQlMjIpJTBBJTBBZW5jb2RlZF9pbnB1dCUyMCUzRCUyMHRva2VuaXplciglMjJIZWxsbyUyQyUyMEknbSUyMGElMjBzaW5nbGUlMjBzZW50ZW5jZSElMjIpJTBBcHJpbnQoZW5jb2RlZF9pbnB1dCk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)

encoded_input = tokenizer(<span class="hljs-string">&quot;Hello, I&#x27;m a single sentence!&quot;</span>)
<span class="hljs-built_in">print</span>(encoded_input)`,wrap:!1}}),ss=new J({props:{code:"JTdCJ2lucHV0X2lkcyclM0ElMjAlNUIxMDElMkMlMjA4NjY3JTJDJTIwMTE3JTJDJTIwMTAwMCUyQyUyMDEwNDUlMkMlMjAxMDA1JTJDJTIwMTA0OSUyQyUyMDIyMzUlMkMlMjAxNzY2MiUyQyUyMDEyMTcyJTJDJTIwMTAxMiUyQyUyMDEwMiU1RCUyQyUyMCUwQSUyMCd0b2tlbl90eXBlX2lkcyclM0ElMjAlNUIwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlNUQlMkMlMjAlMEElMjAnYXR0ZW50aW9uX21hc2snJTNBJTIwJTVCMSUyQyUyMDElMkMlMjAxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTVEJTdE",highlighted:`{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">101</span>, <span class="hljs-number">8667</span>, <span class="hljs-number">117</span>, <span class="hljs-number">1000</span>, <span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">1049</span>, <span class="hljs-number">2235</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]}`,wrap:!1}}),as=new J({props:{code:"dG9rZW5pemVyLmRlY29kZShlbmNvZGVkX2lucHV0JTVCJTIyaW5wdXRfaWRzJTIyJTVEKQ==",highlighted:'tokenizer.decode(encoded_input[<span class="hljs-string">&quot;input_ids&quot;</span>])',wrap:!1}}),ts=new J({props:{code:"JTIyJTVCQ0xTJTVEJTIwSGVsbG8lMkMlMjBJJ20lMjBhJTIwc2luZ2xlJTIwc2VudGVuY2UhJTIwJTVCU0VQJTVEJTIy",highlighted:'<span class="hljs-string">&quot;[CLS] Hello, I&#x27;m a single sentence! [SEP]&quot;</span>',wrap:!1}}),rs=new J({props:{code:"ZW5jb2RlZF9pbnB1dCUyMCUzRCUyMHRva2VuaXplciglMjJIb3clMjBhcmUlMjB5b3UlM0YlMjIlMkMlMjAlMjJJJ20lMjBmaW5lJTJDJTIwdGhhbmslMjB5b3UhJTIyKSUwQXByaW50KGVuY29kZWRfaW5wdXQp",highlighted:`encoded_input = tokenizer(<span class="hljs-string">&quot;How are you?&quot;</span>, <span class="hljs-string">&quot;I&#x27;m fine, thank you!&quot;</span>)
<span class="hljs-built_in">print</span>(encoded_input)`,wrap:!1}}),ms=new J({props:{code:"JTdCJ2lucHV0X2lkcyclM0ElMjAlNUIlNUIxMDElMkMlMjAxNzMxJTJDJTIwMTEzMiUyQyUyMDExMjglMkMlMjAxMzYlMkMlMjAxMDIlNUQlMkMlMjAlNUIxMDElMkMlMjAxMDQ1JTJDJTIwMTAwNSUyQyUyMDEwNDklMkMlMjAyNTAzJTJDJTIwMTE3JTJDJTIwNTc2MyUyQyUyMDExMjglMkMlMjAxMzYlMkMlMjAxMDIlNUQlNUQlMkMlMjAlMEElMjAndG9rZW5fdHlwZV9pZHMnJTNBJTIwJTVCJTVCMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTVEJTJDJTIwJTVCMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCU1RCU1RCUyQyUyMCUwQSUyMCdhdHRlbnRpb25fbWFzayclM0ElMjAlNUIlNUIxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTJDJTIwMSUyQyUyMDElNUQlMkMlMjAlNUIxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTVEJTVEJTdE",highlighted:`{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1731</span>, <span class="hljs-number">1132</span>, <span class="hljs-number">1128</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>], [<span class="hljs-number">101</span>, <span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">1049</span>, <span class="hljs-number">2503</span>, <span class="hljs-number">117</span>, <span class="hljs-number">5763</span>, <span class="hljs-number">1128</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]]}`,wrap:!1}}),js=new J({props:{code:"ZW5jb2RlZF9pbnB1dCUyMCUzRCUyMHRva2VuaXplciglMjJIb3clMjBhcmUlMjB5b3UlM0YlMjIlMkMlMjAlMjJJJ20lMjBmaW5lJTJDJTIwdGhhbmslMjB5b3UhJTIyJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEFwcmludChlbmNvZGVkX2lucHV0KQ==",highlighted:`encoded_input = tokenizer(<span class="hljs-string">&quot;How are you?&quot;</span>, <span class="hljs-string">&quot;I&#x27;m fine, thank you!&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-built_in">print</span>(encoded_input)`,wrap:!1}}),us=new J({props:{code:"JTdCJ2lucHV0X2lkcyclM0ElMjB0ZW5zb3IoJTVCJTVCJTIwJTIwMTAxJTJDJTIwJTIwMTczMSUyQyUyMCUyMDExMzIlMkMlMjAlMjAxMTI4JTJDJTIwJTIwJTIwMTM2JTJDJTIwJTIwJTIwMTAyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVCJTIwJTIwMTAxJTJDJTIwJTIwMTA0NSUyQyUyMCUyMDEwMDUlMkMlMjAlMjAxMDQ5JTJDJTIwJTIwMjUwMyUyQyUyMCUyMCUyMDExNyUyQyUyMCUyMDU3NjMlMkMlMjAlMjAxMTI4JTJDJTIwJTIwJTIwMTM2JTJDJTIwJTIwJTIwMTAyJTVEJTVEKSUyQyUyMCUwQSUyMCd0b2tlbl90eXBlX2lkcyclM0ElMjB0ZW5zb3IoJTVCJTVCMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVCMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCU1RCU1RCklMkMlMjAlMEElMjAnYXR0ZW50aW9uX21hc2snJTNBJTIwdGVuc29yKCU1QiU1QjElMkMlMjAxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTJDJTIwMSU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU1QjElMkMlMjAxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTJDJTIwMSUyQyUyMDElNUQlNUQpJTdE",highlighted:`{<span class="hljs-string">&#x27;input_ids&#x27;</span>: tensor([[  <span class="hljs-number">101</span>,  <span class="hljs-number">1731</span>,  <span class="hljs-number">1132</span>,  <span class="hljs-number">1128</span>,   <span class="hljs-number">136</span>,   <span class="hljs-number">102</span>],
         [  <span class="hljs-number">101</span>,  <span class="hljs-number">1045</span>,  <span class="hljs-number">1005</span>,  <span class="hljs-number">1049</span>,  <span class="hljs-number">2503</span>,   <span class="hljs-number">117</span>,  <span class="hljs-number">5763</span>,  <span class="hljs-number">1128</span>,   <span class="hljs-number">136</span>,   <span class="hljs-number">102</span>]]), 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: tensor([[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
         [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]), 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: tensor([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
         [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]])}`,wrap:!1}}),is=new c({props:{title:"Padding inputs",local:"padding-inputs",headingTag:"h3"}}),Ts=new J({props:{code:"ZW5jb2RlZF9pbnB1dCUyMCUzRCUyMHRva2VuaXplciglMEElMjAlMjAlMjAlMjAlNUIlMjJIb3clMjBhcmUlMjB5b3UlM0YlMjIlMkMlMjAlMjJJJ20lMjBmaW5lJTJDJTIwdGhhbmslMjB5b3UhJTIyJTVEJTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyJTBBKSUwQXByaW50KGVuY29kZWRfaW5wdXQp",highlighted:`encoded_input = tokenizer(
    [<span class="hljs-string">&quot;How are you?&quot;</span>, <span class="hljs-string">&quot;I&#x27;m fine, thank you!&quot;</span>], padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>
)
<span class="hljs-built_in">print</span>(encoded_input)`,wrap:!1}}),cs=new J({props:{code:"JTdCJ2lucHV0X2lkcyclM0ElMjB0ZW5zb3IoJTVCJTVCJTIwJTIwMTAxJTJDJTIwJTIwMTczMSUyQyUyMCUyMDExMzIlMkMlMjAlMjAxMTI4JTJDJTIwJTIwJTIwMTM2JTJDJTIwJTIwJTIwMTAyJTJDJTIwJTIwJTIwJTIwJTIwMCUyQyUyMCUyMCUyMCUyMCUyMDAlMkMlMjAlMjAlMjAlMjAlMjAwJTJDJTIwJTIwJTIwJTIwJTIwMCU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU1QiUyMCUyMDEwMSUyQyUyMCUyMDEwNDUlMkMlMjAlMjAxMDA1JTJDJTIwJTIwMTA0OSUyQyUyMCUyMDI1MDMlMkMlMjAlMjAlMjAxMTclMkMlMjAlMjA1NzYzJTJDJTIwJTIwMTEyOCUyQyUyMCUyMCUyMDEzNiUyQyUyMCUyMCUyMDEwMiU1RCU1RCklMkMlMjAlMEElMjAndG9rZW5fdHlwZV9pZHMnJTNBJTIwdGVuc29yKCU1QiU1QjAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUIwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTVEJTVEKSUyQyUyMCUwQSUyMCdhdHRlbnRpb25fbWFzayclM0ElMjB0ZW5zb3IoJTVCJTVCMSUyQyUyMDElMkMlMjAxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU1QjElMkMlMjAxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTJDJTIwMSUyQyUyMDElNUQlNUQpJTdE",highlighted:`{<span class="hljs-string">&#x27;input_ids&#x27;</span>: tensor([[  <span class="hljs-number">101</span>,  <span class="hljs-number">1731</span>,  <span class="hljs-number">1132</span>,  <span class="hljs-number">1128</span>,   <span class="hljs-number">136</span>,   <span class="hljs-number">102</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>],
         [  <span class="hljs-number">101</span>,  <span class="hljs-number">1045</span>,  <span class="hljs-number">1005</span>,  <span class="hljs-number">1049</span>,  <span class="hljs-number">2503</span>,   <span class="hljs-number">117</span>,  <span class="hljs-number">5763</span>,  <span class="hljs-number">1128</span>,   <span class="hljs-number">136</span>,   <span class="hljs-number">102</span>]]), 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: tensor([[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
         [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]), 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: tensor([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
         [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]])}`,wrap:!1}}),ws=new c({props:{title:"Truncating inputs",local:"truncating-inputs",headingTag:"h3"}}),Us=new J({props:{code:"ZW5jb2RlZF9pbnB1dCUyMCUzRCUyMHRva2VuaXplciglMEElMjAlMjAlMjAlMjAlMjJUaGlzJTIwaXMlMjBhJTIwdmVyeSUyMHZlcnklMjB2ZXJ5JTIwdmVyeSUyMHZlcnklMjB2ZXJ5JTIwdmVyeSUyMHZlcnklMjB2ZXJ5JTIwdmVyeSUyMHZlcnklMjB2ZXJ5JTIwdmVyeSUyMHZlcnklMjB2ZXJ5JTIwdmVyeSUyMHZlcnklMjB2ZXJ5JTIwdmVyeSUyMHZlcnklMjB2ZXJ5JTIwdmVyeSUyMHZlcnklMjB2ZXJ5JTIwdmVyeSUyMHZlcnklMjB2ZXJ5JTIwdmVyeSUyMHZlcnklMjB2ZXJ5JTIwdmVyeSUyMHZlcnklMjB2ZXJ5JTIwdmVyeSUyMHZlcnklMjB2ZXJ5JTIwdmVyeSUyMHZlcnklMjB2ZXJ5JTIwdmVyeSUyMHZlcnklMjB2ZXJ5JTIwdmVyeSUyMHZlcnklMjB2ZXJ5JTIwdmVyeSUyMHZlcnklMjB2ZXJ5JTIwdmVyeSUyMGxvbmclMjBzZW50ZW5jZS4lMjIlMkMlMEElMjAlMjAlMjAlMjB0cnVuY2F0aW9uJTNEVHJ1ZSUyQyUwQSklMEFwcmludChlbmNvZGVkX2lucHV0JTVCJTIyaW5wdXRfaWRzJTIyJTVEKQ==",highlighted:`encoded_input = tokenizer(
    <span class="hljs-string">&quot;This is a very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very long sentence.&quot;</span>,
    truncation=<span class="hljs-literal">True</span>,
)
<span class="hljs-built_in">print</span>(encoded_input[<span class="hljs-string">&quot;input_ids&quot;</span>])`,wrap:!1}}),ds=new J({props:{code:"JTVCMTAxJTJDJTIwMTE4OCUyQyUyMDExMTAlMkMlMjAxNzAlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTUwNSUyQyUyMDE1MDUlMkMlMjAxNTA1JTJDJTIwMTE3OSUyQyUyMDU2NTAlMkMlMjAxMTklMkMlMjAxMDIlNUQ=",highlighted:'[<span class="hljs-number">101</span>, <span class="hljs-number">1188</span>, <span class="hljs-number">1110</span>, <span class="hljs-number">170</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1505</span>, <span class="hljs-number">1179</span>, <span class="hljs-number">5650</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>]',wrap:!1}}),fs=new J({props:{code:"ZW5jb2RlZF9pbnB1dCUyMCUzRCUyMHRva2VuaXplciglMEElMjAlMjAlMjAlMjAlNUIlMjJIb3clMjBhcmUlMjB5b3UlM0YlMjIlMkMlMjAlMjJJJ20lMjBmaW5lJTJDJTIwdGhhbmslMjB5b3UhJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwcGFkZGluZyUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjB0cnVuY2F0aW9uJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMG1heF9sZW5ndGglM0Q1JTJDJTBBJTIwJTIwJTIwJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiUyQyUwQSklMEFwcmludChlbmNvZGVkX2lucHV0KQ==",highlighted:`encoded_input = tokenizer(
    [<span class="hljs-string">&quot;How are you?&quot;</span>, <span class="hljs-string">&quot;I&#x27;m fine, thank you!&quot;</span>],
    padding=<span class="hljs-literal">True</span>,
    truncation=<span class="hljs-literal">True</span>,
    max_length=<span class="hljs-number">5</span>,
    return_tensors=<span class="hljs-string">&quot;pt&quot;</span>,
)
<span class="hljs-built_in">print</span>(encoded_input)`,wrap:!1}}),As=new J({props:{code:"JTdCJ2lucHV0X2lkcyclM0ElMjB0ZW5zb3IoJTVCJTVCJTIwJTIwMTAxJTJDJTIwJTIwMTczMSUyQyUyMCUyMDExMzIlMkMlMjAlMjAxMTI4JTJDJTIwJTIwJTIwMTAyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVCJTIwJTIwMTAxJTJDJTIwJTIwMTA0NSUyQyUyMCUyMDEwMDUlMkMlMjAlMjAxMDQ5JTJDJTIwJTIwJTIwMTAyJTVEJTVEKSUyQyUyMCUwQSUyMCd0b2tlbl90eXBlX2lkcyclM0ElMjB0ZW5zb3IoJTVCJTVCMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUIwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCU1RCU1RCklMkMlMjAlMEElMjAnYXR0ZW50aW9uX21hc2snJTNBJTIwdGVuc29yKCU1QiU1QjElMkMlMjAxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVCMSUyQyUyMDElMkMlMjAxJTJDJTIwMSUyQyUyMDElNUQlNUQpJTdE",highlighted:`{<span class="hljs-string">&#x27;input_ids&#x27;</span>: tensor([[  <span class="hljs-number">101</span>,  <span class="hljs-number">1731</span>,  <span class="hljs-number">1132</span>,  <span class="hljs-number">1128</span>,   <span class="hljs-number">102</span>],
         [  <span class="hljs-number">101</span>,  <span class="hljs-number">1045</span>,  <span class="hljs-number">1005</span>,  <span class="hljs-number">1049</span>,   <span class="hljs-number">102</span>]]), 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: tensor([[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
         [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]), 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: tensor([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
         [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]])}`,wrap:!1}}),ks=new c({props:{title:"Adding special tokens",local:"adding-special-tokens",headingTag:"h3"}}),Ds=new J({props:{code:"ZW5jb2RlZF9pbnB1dCUyMCUzRCUyMHRva2VuaXplciglMjJIb3clMjBhcmUlMjB5b3UlM0YlMjIpJTBBcHJpbnQoZW5jb2RlZF9pbnB1dCU1QiUyMmlucHV0X2lkcyUyMiU1RCklMEF0b2tlbml6ZXIuZGVjb2RlKGVuY29kZWRfaW5wdXQlNUIlMjJpbnB1dF9pZHMlMjIlNUQp",highlighted:`encoded_input = tokenizer(<span class="hljs-string">&quot;How are you?&quot;</span>)
<span class="hljs-built_in">print</span>(encoded_input[<span class="hljs-string">&quot;input_ids&quot;</span>])
tokenizer.decode(encoded_input[<span class="hljs-string">&quot;input_ids&quot;</span>])`,wrap:!1}}),Cs=new J({props:{code:"JTVCMTAxJTJDJTIwMTczMSUyQyUyMDExMzIlMkMlMjAxMTI4JTJDJTIwMTM2JTJDJTIwMTAyJTVEJTBBJyU1QkNMUyU1RCUyMEhvdyUyMGFyZSUyMHlvdSUzRiUyMCU1QlNFUCU1RCc=",highlighted:`[<span class="hljs-number">101</span>, <span class="hljs-number">1731</span>, <span class="hljs-number">1132</span>, <span class="hljs-number">1128</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>]
<span class="hljs-string">&#x27;[CLS] How are you? [SEP]&#x27;</span>`,wrap:!1}}),xs=new c({props:{title:"Why is all of this necessary?",local:"why-is-all-of-this-necessary",headingTag:"h3"}}),Qs=new J({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIySSUyMGhhdGUlMjB0aGlzJTIwc28lMjBtdWNoISUyMiUyQyUwQSU1RA==",highlighted:`sequences = [
    <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>,
    <span class="hljs-string">&quot;I hate this so much!&quot;</span>,
]`,wrap:!1}}),Ns=new J({props:{code:"ZW5jb2RlZF9zZXF1ZW5jZXMlMjAlM0QlMjAlNUIlMEElMjAlMjAlMjAlMjAlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAxMDElMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAxMDQ1JTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwMTAwNSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMDIzMTAlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAyMDQyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwMzQwMyUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMDIwMDUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAxMDM3JTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwMTc2NjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAxMjE3MiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMDI2MDclMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAyMDI2JTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwMjg3OCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMDIxNjYlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAxMDEyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwMTAyJTJDJTBBJTIwJTIwJTIwJTIwJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTVCMTAxJTJDJTIwMTA0NSUyQyUyMDUyMjMlMkMlMjAyMDIzJTJDJTIwMjA2MSUyQyUyMDIxNzIlMkMlMjA5OTklMkMlMjAxMDIlNUQlMkMlMEElNUQ=",highlighted:`encoded_sequences = [
    [
        <span class="hljs-number">101</span>,
        <span class="hljs-number">1045</span>,
        <span class="hljs-number">1005</span>,
        <span class="hljs-number">2310</span>,
        <span class="hljs-number">2042</span>,
        <span class="hljs-number">3403</span>,
        <span class="hljs-number">2005</span>,
        <span class="hljs-number">1037</span>,
        <span class="hljs-number">17662</span>,
        <span class="hljs-number">12172</span>,
        <span class="hljs-number">2607</span>,
        <span class="hljs-number">2026</span>,
        <span class="hljs-number">2878</span>,
        <span class="hljs-number">2166</span>,
        <span class="hljs-number">1012</span>,
        <span class="hljs-number">102</span>,
    ],
    [<span class="hljs-number">101</span>, <span class="hljs-number">1045</span>, <span class="hljs-number">5223</span>, <span class="hljs-number">2023</span>, <span class="hljs-number">2061</span>, <span class="hljs-number">2172</span>, <span class="hljs-number">999</span>, <span class="hljs-number">102</span>],
]`,wrap:!1}}),Ss=new J({props:{code:"aW1wb3J0JTIwdG9yY2glMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b3JjaC50ZW5zb3IoZW5jb2RlZF9zZXF1ZW5jZXMp",highlighted:`<span class="hljs-keyword">import</span> torch

model_inputs = torch.tensor(encoded_sequences)`,wrap:!1}}),Vs=new c({props:{title:"Using the tensors as inputs to the model",local:"using-the-tensors-as-inputs-to-the-model",headingTag:"h3"}}),Bs=new J({props:{code:"b3V0cHV0JTIwJTNEJTIwbW9kZWwobW9kZWxfaW5wdXRzKQ==",highlighted:"output = model(model_inputs)",wrap:!1}}),Ws=new mn({props:{source:"https://github.com/huggingface/course/blob/main/chapters/en/chapter2/3.mdx"}}),{c(){T=u("meta"),zs=a(),b=u("p"),Rs=a(),p(w.$$.fragment),Gs=a(),p(o.$$.fragment),qs=a(),p(U.$$.fragment),Ls=a(),p(d.$$.fragment),Fs=a(),p(I.$$.fragment),Ys=a(),f=u("p"),f.innerHTML=bl,Ps=a(),p(A.$$.fragment),Ks=a(),k=u("p"),k.innerHTML=wl,Os=a(),p($.$$.fragment),se=a(),D=u("p"),D.innerHTML=ol,ee=a(),C=u("p"),C.innerHTML=Ul,le=a(),p(g.$$.fragment),ne=a(),p(x.$$.fragment),ae=a(),E=u("p"),E.innerHTML=dl,te=a(),p(Q.$$.fragment),pe=a(),v=u("p"),v.textContent=Il,Me=a(),p(N.$$.fragment),re=a(),Z=u("p"),Z.innerHTML=fl,me=a(),S=u("p"),S.innerHTML=Al,ye=a(),V=u("p"),V.innerHTML=kl,je=a(),p(_.$$.fragment),ue=a(),B=u("p"),B.innerHTML=$l,he=a(),p(H.$$.fragment),ie=a(),W=u("p"),W.textContent=Dl,Je=a(),p(z.$$.fragment),Te=a(),R=u("p"),R.innerHTML=Cl,ce=a(),p(X.$$.fragment),be=a(),G=u("p"),G.innerHTML=gl,we=a(),p(q.$$.fragment),oe=a(),L=u("p"),L.textContent=xl,Ue=a(),F=u("ul"),F.innerHTML=El,de=a(),Y=u("p"),Y.innerHTML=Ql,Ie=a(),p(P.$$.fragment),fe=a(),K=u("p"),K.innerHTML=vl,Ae=a(),p(O.$$.fragment),ke=a(),p(ss.$$.fragment),$e=a(),es=u("p"),es.textContent=Nl,De=a(),ls=u("ul"),ls.innerHTML=Zl,Ce=a(),ns=u("p"),ns.textContent=Sl,ge=a(),p(as.$$.fragment),xe=a(),p(ts.$$.fragment),Ee=a(),ps=u("p"),ps.innerHTML=Vl,Qe=a(),Ms=u("p"),Ms.textContent=_l,ve=a(),p(rs.$$.fragment),Ne=a(),p(ms.$$.fragment),Ze=a(),ys=u("p"),ys.textContent=Bl,Se=a(),p(js.$$.fragment),Ve=a(),p(us.$$.fragment),_e=a(),hs=u("p"),hs.textContent=Hl,Be=a(),p(is.$$.fragment),He=a(),Js=u("p"),Js.textContent=Wl,We=a(),p(Ts.$$.fragment),ze=a(),p(cs.$$.fragment),Re=a(),bs=u("p"),bs.textContent=zl,Xe=a(),p(ws.$$.fragment),Ge=a(),os=u("p"),os.innerHTML=Rl,qe=a(),p(Us.$$.fragment),Le=a(),p(ds.$$.fragment),Fe=a(),Is=u("p"),Is.textContent=Xl,Ye=a(),p(fs.$$.fragment),Pe=a(),p(As.$$.fragment),Ke=a(),p(ks.$$.fragment),Oe=a(),$s=u("p"),$s.innerHTML=Gl,sl=a(),p(Ds.$$.fragment),el=a(),p(Cs.$$.fragment),ll=a(),gs=u("p"),gs.textContent=ql,nl=a(),p(xs.$$.fragment),al=a(),Es=u("p"),Es.textContent=Ll,tl=a(),p(Qs.$$.fragment),pl=a(),vs=u("p"),vs.textContent=Fl,Ml=a(),p(Ns.$$.fragment),rl=a(),Zs=u("p"),Zs.textContent=Yl,ml=a(),p(Ss.$$.fragment),yl=a(),p(Vs.$$.fragment),jl=a(),_s=u("p"),_s.textContent=Pl,ul=a(),p(Bs.$$.fragment),hl=a(),Hs=u("p"),Hs.textContent=Kl,il=a(),p(Ws.$$.fragment),Jl=a(),Xs=u("p"),this.h()},l(s){const e=pn("svelte-u9bgzb",document.head);T=h(e,"META",{name:!0,content:!0}),e.forEach(l),zs=t(s),b=h(s,"P",{}),sn(b).forEach(l),Rs=t(s),M(w.$$.fragment,s),Gs=t(s),M(o.$$.fragment,s),qs=t(s),M(U.$$.fragment,s),Ls=t(s),M(d.$$.fragment,s),Fs=t(s),M(I.$$.fragment,s),Ys=t(s),f=h(s,"P",{"data-svelte-h":!0}),i(f)!=="svelte-1nloweu"&&(f.innerHTML=bl),Ps=t(s),M(A.$$.fragment,s),Ks=t(s),k=h(s,"P",{"data-svelte-h":!0}),i(k)!=="svelte-17aypug"&&(k.innerHTML=wl),Os=t(s),M($.$$.fragment,s),se=t(s),D=h(s,"P",{"data-svelte-h":!0}),i(D)!=="svelte-1rqsf14"&&(D.innerHTML=ol),ee=t(s),C=h(s,"P",{"data-svelte-h":!0}),i(C)!=="svelte-gay9y4"&&(C.innerHTML=Ul),le=t(s),M(g.$$.fragment,s),ne=t(s),M(x.$$.fragment,s),ae=t(s),E=h(s,"P",{"data-svelte-h":!0}),i(E)!=="svelte-hdwas9"&&(E.innerHTML=dl),te=t(s),M(Q.$$.fragment,s),pe=t(s),v=h(s,"P",{"data-svelte-h":!0}),i(v)!=="svelte-1mevt8f"&&(v.textContent=Il),Me=t(s),M(N.$$.fragment,s),re=t(s),Z=h(s,"P",{"data-svelte-h":!0}),i(Z)!=="svelte-1dlg86"&&(Z.innerHTML=fl),me=t(s),S=h(s,"P",{"data-svelte-h":!0}),i(S)!=="svelte-1lituvv"&&(S.innerHTML=Al),ye=t(s),V=h(s,"P",{"data-svelte-h":!0}),i(V)!=="svelte-1h4c2vc"&&(V.innerHTML=kl),je=t(s),M(_.$$.fragment,s),ue=t(s),B=h(s,"P",{"data-svelte-h":!0}),i(B)!=="svelte-ddgh4s"&&(B.innerHTML=$l),he=t(s),M(H.$$.fragment,s),ie=t(s),W=h(s,"P",{"data-svelte-h":!0}),i(W)!=="svelte-l3oiw9"&&(W.textContent=Dl),Je=t(s),M(z.$$.fragment,s),Te=t(s),R=h(s,"P",{"data-svelte-h":!0}),i(R)!=="svelte-1pzfpy5"&&(R.innerHTML=Cl),ce=t(s),M(X.$$.fragment,s),be=t(s),G=h(s,"P",{"data-svelte-h":!0}),i(G)!=="svelte-1nxdobe"&&(G.innerHTML=gl),we=t(s),M(q.$$.fragment,s),oe=t(s),L=h(s,"P",{"data-svelte-h":!0}),i(L)!=="svelte-1wh37ph"&&(L.textContent=xl),Ue=t(s),F=h(s,"UL",{"data-svelte-h":!0}),i(F)!=="svelte-103qsj1"&&(F.innerHTML=El),de=t(s),Y=h(s,"P",{"data-svelte-h":!0}),i(Y)!=="svelte-1ml54wi"&&(Y.innerHTML=Ql),Ie=t(s),M(P.$$.fragment,s),fe=t(s),K=h(s,"P",{"data-svelte-h":!0}),i(K)!=="svelte-1i2mfbj"&&(K.innerHTML=vl),Ae=t(s),M(O.$$.fragment,s),ke=t(s),M(ss.$$.fragment,s),$e=t(s),es=h(s,"P",{"data-svelte-h":!0}),i(es)!=="svelte-mgm01i"&&(es.textContent=Nl),De=t(s),ls=h(s,"UL",{"data-svelte-h":!0}),i(ls)!=="svelte-1n80xkj"&&(ls.innerHTML=Zl),Ce=t(s),ns=h(s,"P",{"data-svelte-h":!0}),i(ns)!=="svelte-954hsi"&&(ns.textContent=Sl),ge=t(s),M(as.$$.fragment,s),xe=t(s),M(ts.$$.fragment,s),Ee=t(s),ps=h(s,"P",{"data-svelte-h":!0}),i(ps)!=="svelte-w3oug8"&&(ps.innerHTML=Vl),Qe=t(s),Ms=h(s,"P",{"data-svelte-h":!0}),i(Ms)!=="svelte-kdmifw"&&(Ms.textContent=_l),ve=t(s),M(rs.$$.fragment,s),Ne=t(s),M(ms.$$.fragment,s),Ze=t(s),ys=h(s,"P",{"data-svelte-h":!0}),i(ys)!=="svelte-zcnpfw"&&(ys.textContent=Bl),Se=t(s),M(js.$$.fragment,s),Ve=t(s),M(us.$$.fragment,s),_e=t(s),hs=h(s,"P",{"data-svelte-h":!0}),i(hs)!=="svelte-1jvf30l"&&(hs.textContent=Hl),Be=t(s),M(is.$$.fragment,s),He=t(s),Js=h(s,"P",{"data-svelte-h":!0}),i(Js)!=="svelte-1frpdsp"&&(Js.textContent=Wl),We=t(s),M(Ts.$$.fragment,s),ze=t(s),M(cs.$$.fragment,s),Re=t(s),bs=h(s,"P",{"data-svelte-h":!0}),i(bs)!=="svelte-1dmy5z2"&&(bs.textContent=zl),Xe=t(s),M(ws.$$.fragment,s),Ge=t(s),os=h(s,"P",{"data-svelte-h":!0}),i(os)!=="svelte-1jolo2s"&&(os.innerHTML=Rl),qe=t(s),M(Us.$$.fragment,s),Le=t(s),M(ds.$$.fragment,s),Fe=t(s),Is=h(s,"P",{"data-svelte-h":!0}),i(Is)!=="svelte-48hc6q"&&(Is.textContent=Xl),Ye=t(s),M(fs.$$.fragment,s),Pe=t(s),M(As.$$.fragment,s),Ke=t(s),M(ks.$$.fragment,s),Oe=t(s),$s=h(s,"P",{"data-svelte-h":!0}),i($s)!=="svelte-1mklzwf"&&($s.innerHTML=Gl),sl=t(s),M(Ds.$$.fragment,s),el=t(s),M(Cs.$$.fragment,s),ll=t(s),gs=h(s,"P",{"data-svelte-h":!0}),i(gs)!=="svelte-2judgi"&&(gs.textContent=ql),nl=t(s),M(xs.$$.fragment,s),al=t(s),Es=h(s,"P",{"data-svelte-h":!0}),i(Es)!=="svelte-zezeea"&&(Es.textContent=Ll),tl=t(s),M(Qs.$$.fragment,s),pl=t(s),vs=h(s,"P",{"data-svelte-h":!0}),i(vs)!=="svelte-k5jj6q"&&(vs.textContent=Fl),Ml=t(s),M(Ns.$$.fragment,s),rl=t(s),Zs=h(s,"P",{"data-svelte-h":!0}),i(Zs)!=="svelte-y3q2fn"&&(Zs.textContent=Yl),ml=t(s),M(Ss.$$.fragment,s),yl=t(s),M(Vs.$$.fragment,s),jl=t(s),_s=h(s,"P",{"data-svelte-h":!0}),i(_s)!=="svelte-1et2den"&&(_s.textContent=Pl),ul=t(s),M(Bs.$$.fragment,s),hl=t(s),Hs=h(s,"P",{"data-svelte-h":!0}),i(Hs)!=="svelte-1suqc52"&&(Hs.textContent=Kl),il=t(s),M(Ws.$$.fragment,s),Jl=t(s),Xs=h(s,"P",{}),sn(Xs).forEach(l),this.h()},h(){en(T,"name","hf:doc:metadata"),en(T,"content",Jn)},m(s,e){Mn(document.head,T),n(s,zs,e),n(s,b,e),n(s,Rs,e),r(w,s,e),n(s,Gs,e),r(o,s,e),n(s,qs,e),r(U,s,e),n(s,Ls,e),r(d,s,e),n(s,Fs,e),r(I,s,e),n(s,Ys,e),n(s,f,e),n(s,Ps,e),r(A,s,e),n(s,Ks,e),n(s,k,e),n(s,Os,e),r($,s,e),n(s,se,e),n(s,D,e),n(s,ee,e),n(s,C,e),n(s,le,e),r(g,s,e),n(s,ne,e),r(x,s,e),n(s,ae,e),n(s,E,e),n(s,te,e),r(Q,s,e),n(s,pe,e),n(s,v,e),n(s,Me,e),r(N,s,e),n(s,re,e),n(s,Z,e),n(s,me,e),n(s,S,e),n(s,ye,e),n(s,V,e),n(s,je,e),r(_,s,e),n(s,ue,e),n(s,B,e),n(s,he,e),r(H,s,e),n(s,ie,e),n(s,W,e),n(s,Je,e),r(z,s,e),n(s,Te,e),n(s,R,e),n(s,ce,e),r(X,s,e),n(s,be,e),n(s,G,e),n(s,we,e),r(q,s,e),n(s,oe,e),n(s,L,e),n(s,Ue,e),n(s,F,e),n(s,de,e),n(s,Y,e),n(s,Ie,e),r(P,s,e),n(s,fe,e),n(s,K,e),n(s,Ae,e),r(O,s,e),n(s,ke,e),r(ss,s,e),n(s,$e,e),n(s,es,e),n(s,De,e),n(s,ls,e),n(s,Ce,e),n(s,ns,e),n(s,ge,e),r(as,s,e),n(s,xe,e),r(ts,s,e),n(s,Ee,e),n(s,ps,e),n(s,Qe,e),n(s,Ms,e),n(s,ve,e),r(rs,s,e),n(s,Ne,e),r(ms,s,e),n(s,Ze,e),n(s,ys,e),n(s,Se,e),r(js,s,e),n(s,Ve,e),r(us,s,e),n(s,_e,e),n(s,hs,e),n(s,Be,e),r(is,s,e),n(s,He,e),n(s,Js,e),n(s,We,e),r(Ts,s,e),n(s,ze,e),r(cs,s,e),n(s,Re,e),n(s,bs,e),n(s,Xe,e),r(ws,s,e),n(s,Ge,e),n(s,os,e),n(s,qe,e),r(Us,s,e),n(s,Le,e),r(ds,s,e),n(s,Fe,e),n(s,Is,e),n(s,Ye,e),r(fs,s,e),n(s,Pe,e),r(As,s,e),n(s,Ke,e),r(ks,s,e),n(s,Oe,e),n(s,$s,e),n(s,sl,e),r(Ds,s,e),n(s,el,e),r(Cs,s,e),n(s,ll,e),n(s,gs,e),n(s,nl,e),r(xs,s,e),n(s,al,e),n(s,Es,e),n(s,tl,e),r(Qs,s,e),n(s,pl,e),n(s,vs,e),n(s,Ml,e),r(Ns,s,e),n(s,rl,e),n(s,Zs,e),n(s,ml,e),r(Ss,s,e),n(s,yl,e),r(Vs,s,e),n(s,jl,e),n(s,_s,e),n(s,ul,e),r(Bs,s,e),n(s,hl,e),n(s,Hs,e),n(s,il,e),r(Ws,s,e),n(s,Jl,e),n(s,Xs,e),Tl=!0},p(s,[e]){const Ol={};e&1&&(Ol.fw=s[0]),w.$set(Ol)},i(s){Tl||(m(w.$$.fragment,s),m(o.$$.fragment,s),m(U.$$.fragment,s),m(d.$$.fragment,s),m(I.$$.fragment,s),m(A.$$.fragment,s),m($.$$.fragment,s),m(g.$$.fragment,s),m(x.$$.fragment,s),m(Q.$$.fragment,s),m(N.$$.fragment,s),m(_.$$.fragment,s),m(H.$$.fragment,s),m(z.$$.fragment,s),m(X.$$.fragment,s),m(q.$$.fragment,s),m(P.$$.fragment,s),m(O.$$.fragment,s),m(ss.$$.fragment,s),m(as.$$.fragment,s),m(ts.$$.fragment,s),m(rs.$$.fragment,s),m(ms.$$.fragment,s),m(js.$$.fragment,s),m(us.$$.fragment,s),m(is.$$.fragment,s),m(Ts.$$.fragment,s),m(cs.$$.fragment,s),m(ws.$$.fragment,s),m(Us.$$.fragment,s),m(ds.$$.fragment,s),m(fs.$$.fragment,s),m(As.$$.fragment,s),m(ks.$$.fragment,s),m(Ds.$$.fragment,s),m(Cs.$$.fragment,s),m(xs.$$.fragment,s),m(Qs.$$.fragment,s),m(Ns.$$.fragment,s),m(Ss.$$.fragment,s),m(Vs.$$.fragment,s),m(Bs.$$.fragment,s),m(Ws.$$.fragment,s),Tl=!0)},o(s){y(w.$$.fragment,s),y(o.$$.fragment,s),y(U.$$.fragment,s),y(d.$$.fragment,s),y(I.$$.fragment,s),y(A.$$.fragment,s),y($.$$.fragment,s),y(g.$$.fragment,s),y(x.$$.fragment,s),y(Q.$$.fragment,s),y(N.$$.fragment,s),y(_.$$.fragment,s),y(H.$$.fragment,s),y(z.$$.fragment,s),y(X.$$.fragment,s),y(q.$$.fragment,s),y(P.$$.fragment,s),y(O.$$.fragment,s),y(ss.$$.fragment,s),y(as.$$.fragment,s),y(ts.$$.fragment,s),y(rs.$$.fragment,s),y(ms.$$.fragment,s),y(js.$$.fragment,s),y(us.$$.fragment,s),y(is.$$.fragment,s),y(Ts.$$.fragment,s),y(cs.$$.fragment,s),y(ws.$$.fragment,s),y(Us.$$.fragment,s),y(ds.$$.fragment,s),y(fs.$$.fragment,s),y(As.$$.fragment,s),y(ks.$$.fragment,s),y(Ds.$$.fragment,s),y(Cs.$$.fragment,s),y(xs.$$.fragment,s),y(Qs.$$.fragment,s),y(Ns.$$.fragment,s),y(Ss.$$.fragment,s),y(Vs.$$.fragment,s),y(Bs.$$.fragment,s),y(Ws.$$.fragment,s),Tl=!1},d(s){s&&(l(zs),l(b),l(Rs),l(Gs),l(qs),l(Ls),l(Fs),l(Ys),l(f),l(Ps),l(Ks),l(k),l(Os),l(se),l(D),l(ee),l(C),l(le),l(ne),l(ae),l(E),l(te),l(pe),l(v),l(Me),l(re),l(Z),l(me),l(S),l(ye),l(V),l(je),l(ue),l(B),l(he),l(ie),l(W),l(Je),l(Te),l(R),l(ce),l(be),l(G),l(we),l(oe),l(L),l(Ue),l(F),l(de),l(Y),l(Ie),l(fe),l(K),l(Ae),l(ke),l($e),l(es),l(De),l(ls),l(Ce),l(ns),l(ge),l(xe),l(Ee),l(ps),l(Qe),l(Ms),l(ve),l(Ne),l(Ze),l(ys),l(Se),l(Ve),l(_e),l(hs),l(Be),l(He),l(Js),l(We),l(ze),l(Re),l(bs),l(Xe),l(Ge),l(os),l(qe),l(Le),l(Fe),l(Is),l(Ye),l(Pe),l(Ke),l(Oe),l($s),l(sl),l(el),l(ll),l(gs),l(nl),l(al),l(Es),l(tl),l(pl),l(vs),l(Ml),l(rl),l(Zs),l(ml),l(yl),l(jl),l(_s),l(ul),l(hl),l(Hs),l(il),l(Jl),l(Xs)),l(T),j(w,s),j(o,s),j(U,s),j(d,s),j(I,s),j(A,s),j($,s),j(g,s),j(x,s),j(Q,s),j(N,s),j(_,s),j(H,s),j(z,s),j(X,s),j(q,s),j(P,s),j(O,s),j(ss,s),j(as,s),j(ts,s),j(rs,s),j(ms,s),j(js,s),j(us,s),j(is,s),j(Ts,s),j(cs,s),j(ws,s),j(Us,s),j(ds,s),j(fs,s),j(As,s),j(ks,s),j(Ds,s),j(Cs,s),j(xs,s),j(Qs,s),j(Ns,s),j(Ss,s),j(Vs,s),j(Bs,s),j(Ws,s)}}}const Jn='{"title":"Models","local":"the-models","sections":[{"title":"Creating a Transformer","local":"creating-a-transformer","sections":[],"depth":2},{"title":"Loading and saving","local":"loading-and-saving","sections":[],"depth":2},{"title":"Encoding text","local":"encoding-text","sections":[{"title":"Padding inputs","local":"padding-inputs","sections":[],"depth":3},{"title":"Truncating inputs","local":"truncating-inputs","sections":[],"depth":3},{"title":"Adding special tokens","local":"adding-special-tokens","sections":[],"depth":3},{"title":"Why is all of this necessary?","local":"why-is-all-of-this-necessary","sections":[],"depth":3},{"title":"Using the tensors as inputs to the model","local":"using-the-tensors-as-inputs-to-the-model","sections":[],"depth":3}],"depth":2}],"depth":1}';function Tn(cl,T,zs){let b="pt";return nn(()=>{const Rs=new URLSearchParams(window.location.search);zs(0,b=Rs.get("fw")||"pt")}),[b]}class fn extends an{constructor(T){super(),tn(this,T,Tn,hn,ln,{})}}export{fn as component};