course/pr_1213/pt/_app/immutable/nodes/18.dfa771ce.js

import{s as as,o as ns}from"../chunks/scheduler.505acc25.js";import{S as os,i as is,e as b,s as o,c as m,h as ps,a as J,d as l,b as i,f as Oe,g as u,j as T,k as es,l as rs,m as a,n as M,o as r,q as ss,t as c,p as d,r as ls}from"../chunks/index.1238bded.js";import{C as cs,H as Ge,E as ms}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.b263ef37.js";import{C as j}from"../chunks/CodeBlock.806cccc4.js";import{C as ts}from"../chunks/CourseFloatingBanner.2e302d0f.js";import{F as us}from"../chunks/FrameworkSwitchCourse.23952889.js";function Ms(w){let n,p;return n=new ts({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/pt/chapter2/section6_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/pt/chapter2/section6_tf.ipynb"}]}}),{c(){m(n.$$.fragment)},l(t){u(n.$$.fragment,t)},m(t,y){M(n,t,y),p=!0},i(t){p||(c(n.$$.fragment,t),p=!0)},o(t){r(n.$$.fragment,t),p=!1},d(t){d(n,t)}}}function ds(w){let n,p;return n=new ts({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/pt/chapter2/section6_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/pt/chapter2/section6_pt.ipynb"}]}}),{c(){m(n.$$.fragment)},l(t){u(n.$$.fragment,t)},m(t,y){M(n,t,y),p=!0},i(t){p||(c(n.$$.fragment,t),p=!0)},o(t){r(n.$$.fragment,t),p=!1},d(t){d(n,t)}}}function bs(w){let n,p;return n=new j({props:{code:"aW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMFRGQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbiUwQSUwQWNoZWNrcG9pbnQlMjAlM0QlMjAlMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZC1maW5ldHVuZWQtc3N0LTItZW5nbGlzaCUyMiUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBbW9kZWwlMjAlM0QlMjBURkF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBc2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEF0b2tlbnMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjB0cnVuY2F0aW9uJTNEVHJ1ZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIydGYlMjIpJTBBb3V0cHV0JTIwJTNEJTIwbW9kZWwoKip0b2tlbnMp",highlighted:`<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = TFAutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
output = model(**tokens)`,wrap:!1}}),{c(){m(n.$$.fragment)},l(t){u(n.$$.fragment,t)},m(t,y){M(n,t,y),p=!0},i(t){p||(c(n.$$.fragment,t),p=!0)},o(t){r(n.$$.fragment,t),p=!1},d(t){d(n,t)}}}function Js(w){let n,p;return n=new j({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24lMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQtZmluZXR1bmVkLXNzdC0yLWVuZ2xpc2glMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbi5mcm9tX3ByZXRyYWluZWQoY2hlY2twb2ludCklMEFzZXF1ZW5jZXMlMjAlM0QlMjAlNUIlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMkMlMjAlMjJTbyUyMGhhdmUlMjBJISUyMiU1RCUwQSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHRydW5jYXRpb24lM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEFvdXRwdXQlMjAlM0QlMjBtb2RlbCgqKnRva2Vucyk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
output = model(**tokens)`,wrap:!1}}),{c(){m(n.$$.fragment)},l(t){u(n.$$.fragment,t)},m(t,y){M(n,t,y),p=!0},i(t){p||(c(n.$$.fragment,t),p=!0)},o(t){r(n.$$.fragment,t),p=!1},d(t){d(n,t)}}}function ys(w){let n,p,t,y,Z,ae,V,ne,g,oe,f,h,se,k,Xe="Nas últimas seções, temos feito o nosso melhor para fazer a maior parte do trabalho à mão. Exploramos como funcionam os tokenizers e analisamos a tokenização, conversão para IDs de entrada, padding, truncagem e máscaras de atenção.",ie,$,Ee="Entretanto, como vimos na seção 2, a API dos 🤗 Transformers pode tratar de tudo isso para nós com uma função de alto nível, na qual mergulharemos aqui. Quando você chama seu <code>tokenizer</code> diretamente na frase, você recebe de volta entradas que estão prontas para passar pelo seu modelo:",pe,W,re,B,xe="Aqui, a variável <code>model_inputs</code> contém tudo o que é necessário para que um modelo funcione bem. Para DistilBERT, isso inclui os IDs de entrada, bem como a máscara de atenção. Outros modelos que aceitam entradas adicionais também terão essas saídas pelo objeto <code>tokenizer</code>.",ce,z,Ne="Como veremos em alguns exemplos abaixo, este método é muito poderoso. Primeiro, ele pode simbolizar uma única sequência:",me,v,ue,q,Se="Também lida com várias sequências de cada vez, sem nenhuma mudança na API:",Me,G,de,X,_e="Ela pode ser aplicada de acordo com vários objetivos:",be,E,Je,x,Qe="Também pode truncar sequências:",ye,N,Te,S,Re="O objeto <code>tokenizer</code> pode lidar com a conversão para tensores de estrutura específicos, que podem então ser enviados diretamente para o modelo. Por exemplo, na seguinte amostra de código, estamos solicitando que o tokenizer retorne tensores de diferentes estruturas - <code>&quot;pt&quot;</code> retorna tensores PyTorch, <code>&quot;tf&quot;</code> retorna tensores TensorFlow, e <code>&quot;np&quot;</code> retorna arrays NumPy:",je,_,we,Q,fe,R,Ye="Se dermos uma olhada nos IDs de entrada devolvidos pelo tokenizer, veremos que eles são um pouco diferentes do que tínhamos anteriormente:",he,Y,Ie,C,Ue,F,Ce="Um token ID foi adicionada no início e uma no final. Vamos decodificar as duas sequências de IDs acima para ver do que se trata:",Ze,H,Ve,D,ge,A,Fe="O tokenizer acrescentou a palavra especial <code>[CLS]</code> no início e a palavra especial <code>[SEP]</code> no final. Isto porque o modelo foi pré-treinado com esses, então para obter os mesmos resultados para inferência, precisamos adicioná-los também. Note que alguns modelos não acrescentam palavras especiais, ou acrescentam palavras diferentes; os modelos também podem acrescentar estas palavras especiais apenas no início, ou apenas no final. Em qualquer caso, o tokenizer sabe quais são as palavras que são esperadas e tratará disso para você.",ke,P,$e,L,He="Agora que já vimos todos os passos individuais que o objeto <code>tokenizer</code> utiliza quando aplicado em textos, vamos ver uma última vez como ele pode lidar com múltiplas sequências (padding!), sequências muito longas (truncagem!), e múltiplos tipos de tensores com seu API principal:",We,I,U,le,K,Be,te,ze;Z=new us({props:{fw:w[0]}}),V=new cs({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),g=new Ge({props:{title:"Colocando tudo junto",local:"colocando-tudo-junto",headingTag:"h1"}});const De=[ds,Ms],O=[];function Ae(e,s){return e[0]==="pt"?0:1}f=Ae(w),h=O[f]=De[f](w),W=new j({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQtZmluZXR1bmVkLXNzdC0yLWVuZ2xpc2glMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQSUwQXNlcXVlbmNlJTIwJTNEJTIwJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTBBJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`,wrap:!1}}),v=new j({props:{code:"c2VxdWVuY2UlMjAlM0QlMjAlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2Up",highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`,wrap:!1}}),G=new j({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzKQ==",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

model_inputs = tokenizer(sequences)`,wrap:!1}}),E=new j({props:{code:"JTIzJTIwSXIlQzMlQTElMjBwcmVlbmNoZXIlMjBhcyUyMHNlcXUlQzMlQUFuY2lhcyUyMGF0JUMzJUE5JTIwbyUyMGNvbXByaW1lbnRvJTIwbSVDMyVBMXhpbW8lMjBkYSUyMHNlcXUlQzMlQUFuY2lhJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMHBhZGRpbmclM0QlMjJsb25nZXN0JTIyKSUwQSUwQSUyMyUyMElyJUMzJUExJTIwcHJlZW5jaGVyJTIwYXMlMjBzZXF1JUMzJUFBbmNpYXMlMjBhdCVDMyVBOSUyMG8lMjBjb21wcmltZW50byUyMG0lQzMlQTF4aW1vJTIwZG8lMjBtb2RlbG8lMEElMjMlMjAoNTEyJTIwcGFyYSUyMG8lMjBtb2RlbG8lMjBCRVJUJTIwb3UlMjBEaXN0aWxCRVJUKSUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEJTIybWF4X2xlbmd0aCUyMiklMEElMEElMjMlMjBJciVDMyVBMSUyMHByZWVuY2hlciUyMGFzJTIwc2VxdSVDMyVBQW5jaWFzJTIwYXQlQzMlQTklMjBvJTIwY29tcHJpbWVudG8lMjBtJUMzJUExeGltbyUyMGVzcGVjaWZpY2FkbyUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEJTIybWF4X2xlbmd0aCUyMiUyQyUyMG1heF9sZW5ndGglM0Q4KQ==",highlighted:`<span class="hljs-comment"># Irá preencher as sequências até o comprimento máximo da sequência</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;longest&quot;</span>)

<span class="hljs-comment"># Irá preencher as sequências até o comprimento máximo do modelo</span>
<span class="hljs-comment"># (512 para o modelo BERT ou DistilBERT)</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>)

<span class="hljs-comment"># Irá preencher as sequências até o comprimento máximo especificado</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>, max_length=<span class="hljs-number">8</span>)`,wrap:!1}}),N=new j({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEElMjMlMjBJciVDMyVBMSUyMHByZWVuY2hlciUyMGFzJTIwc2VxdSVDMyVBQW5jaWFzJTIwYXQlQzMlQTklMjBvJTIwY29tcHJpbWVudG8lMjBtJUMzJUExeGltbyUyMGRvJTIwbW9kZWxvJTBBJTIzJTIwKDUxMiUyMHBhcmElMjBvJTIwbW9kZWxvJTIwQkVSVCUyMG91JTIwRGlzdGlsQkVSVCklMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwdHJ1bmNhdGlvbiUzRFRydWUpJTBBJTBBJTIzJTIwVHJ1bmNhciVDMyVBMSUyMGFzJTIwc2VxdSVDMyVBQW5jaWFzJTIwcXVlJTIwcyVDMyVBM28lMjBtYWlzJTIwbG9uZ2FzJTIwZG8lMjBxdWUlMjBvJTIwY29tcHJpbWVudG8lMjBtJUMzJUExeGltbyUyMGVzcGVjaWZpY2FkbyUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBtYXhfbGVuZ3RoJTNEOCUyQyUyMHRydW5jYXRpb24lM0RUcnVlKQ==",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># Irá preencher as sequências até o comprimento máximo do modelo</span>
<span class="hljs-comment"># (512 para o modelo BERT ou DistilBERT)</span>
model_inputs = tokenizer(sequences, truncation=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># Truncará as sequências que são mais longas do que o comprimento máximo especificado</span>
model_inputs = tokenizer(sequences, max_length=<span class="hljs-number">8</span>, truncation=<span class="hljs-literal">True</span>)`,wrap:!1}}),_=new j({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEElMjMlMjBSZXRvcm5hJTIwdGVuc29yZXMlMjBQeVRvcmNoJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMHBhZGRpbmclM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEElMEElMjMlMjBSZXRvcm5hJTIwdGVuc29yZXMlMjBUZW5zb3JGbG93JTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMHBhZGRpbmclM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJ0ZiUyMiklMEElMEElMjMlMjBSZXRvcm5hJTIwTnVtUHklMjBhcnJheXMlMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMm5wJTIyKQ==",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># Retorna tensores PyTorch</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-comment"># Retorna tensores TensorFlow</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)

<span class="hljs-comment"># Retorna NumPy arrays</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;np&quot;</span>)`,wrap:!1}}),Q=new Ge({props:{title:"Tokens especiais",local:"tokens-especiais",headingTag:"h2"}}),Y=new j({props:{code:"c2VxdWVuY2UlMjAlM0QlMjAlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2UpJTBBcHJpbnQobW9kZWxfaW5wdXRzJTVCJTIyaW5wdXRfaWRzJTIyJTVEKSUwQSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplci50b2tlbml6ZShzZXF1ZW5jZSklMEFpZHMlMjAlM0QlMjB0b2tlbml6ZXIuY29udmVydF90b2tlbnNfdG9faWRzKHRva2VucyklMEFwcmludChpZHMp",highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)
<span class="hljs-built_in">print</span>(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>])

tokens = tokenizer.tokenize(sequence)
ids = tokenizer.convert_tokens_to_ids(tokens)
<span class="hljs-built_in">print</span>(ids)`,wrap:!1}}),C=new j({props:{code:"JTVCMTAxJTJDJTIwMTA0NSUyQyUyMDEwMDUlMkMlMjAyMzEwJTJDJTIwMjA0MiUyQyUyMDM0MDMlMkMlMjAyMDA1JTJDJTIwMTAzNyUyQyUyMDE3NjYyJTJDJTIwMTIxNzIlMkMlMjAyNjA3JTJDJTIwMjAyNiUyQyUyMDI4NzglMkMlMjAyMTY2JTJDJTIwMTAxMiUyQyUyMDEwMiU1RCUwQSU1QjEwNDUlMkMlMjAxMDA1JTJDJTIwMjMxMCUyQyUyMDIwNDIlMkMlMjAzNDAzJTJDJTIwMjAwNSUyQyUyMDEwMzclMkMlMjAxNzY2MiUyQyUyMDEyMTcyJTJDJTIwMjYwNyUyQyUyMDIwMjYlMkMlMjAyODc4JTJDJTIwMjE2NiUyQyUyMDEwMTIlNUQ=",highlighted:`[<span class="hljs-number">101</span>, <span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>]
[<span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>]`,wrap:!1}}),H=new j({props:{code:"cHJpbnQodG9rZW5pemVyLmRlY29kZShtb2RlbF9pbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQpKSUwQXByaW50KHRva2VuaXplci5kZWNvZGUoaWRzKSk=",highlighted:`<span class="hljs-built_in">print</span>(tokenizer.decode(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>]))
<span class="hljs-built_in">print</span>(tokenizer.decode(ids))`,wrap:!1}}),D=new j({props:{code:"JTIyJTVCQ0xTJTVEJTIwaSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMGh1Z2dpbmdmYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIwJTVCU0VQJTVEJTIyJTBBJTIyaSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMGh1Z2dpbmdmYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIy",highlighted:`<span class="hljs-string">&quot;[CLS] i&#x27;ve been waiting for a huggingface course my whole life. [SEP]&quot;</span>
<span class="hljs-string">&quot;i&#x27;ve been waiting for a huggingface course my whole life.&quot;</span>`,wrap:!1}}),P=new Ge({props:{title:"Do tokenizer ao modelo",local:"do-tokenizer-ao-modelo",headingTag:"h2"}});const Pe=[Js,bs],ee=[];function Le(e,s){return e[0]==="pt"?0:1}return I=Le(w),U=ee[I]=Pe[I](w),K=new ms({props:{source:"https://github.com/huggingface/course/blob/main/chapters/pt/chapter2/6.mdx"}}),{c(){n=b("meta"),p=o(),t=b("p"),y=o(),m(Z.$$.fragment),ae=o(),m(V.$$.fragment),ne=o(),m(g.$$.fragment),oe=o(),h.c(),se=o(),k=b("p"),k.textContent=Xe,ie=o(),$=b("p"),$.innerHTML=Ee,pe=o(),m(W.$$.fragment),re=o(),B=b("p"),B.innerHTML=xe,ce=o(),z=b("p"),z.textContent=Ne,me=o(),m(v.$$.fragment),ue=o(),q=b("p"),q.textContent=Se,Me=o(),m(G.$$.fragment),de=o(),X=b("p"),X.textContent=_e,be=o(),m(E.$$.fragment),Je=o(),x=b("p"),x.textContent=Qe,ye=o(),m(N.$$.fragment),Te=o(),S=b("p"),S.innerHTML=Re,je=o(),m(_.$$.fragment),we=o(),m(Q.$$.fragment),fe=o(),R=b("p"),R.textContent=Ye,he=o(),m(Y.$$.fragment),Ie=o(),m(C.$$.fragment),Ue=o(),F=b("p"),F.textContent=Ce,Ze=o(),m(H.$$.fragment),Ve=o(),m(D.$$.fragment),ge=o(),A=b("p"),A.innerHTML=Fe,ke=o(),m(P.$$.fragment),$e=o(),L=b("p"),L.innerHTML=He,We=o(),U.c(),le=o(),m(K.$$.fragment),Be=o(),te=b("p"),this.h()},l(e){const s=ps("svelte-u9bgzb",document.head);n=J(s,"META",{name:!0,content:!0}),s.forEach(l),p=i(e),t=J(e,"P",{}),Oe(t).forEach(l),y=i(e),u(Z.$$.fragment,e),ae=i(e),u(V.$$.fragment,e),ne=i(e),u(g.$$.fragment,e),oe=i(e),h.l(e),se=i(e),k=J(e,"P",{"data-svelte-h":!0}),T(k)!=="svelte-xmqek0"&&(k.textContent=Xe),ie=i(e),$=J(e,"P",{"data-svelte-h":!0}),T($)!=="svelte-562dly"&&($.innerHTML=Ee),pe=i(e),u(W.$$.fragment,e),re=i(e),B=J(e,"P",{"data-svelte-h":!0}),T(B)!=="svelte-2razkz"&&(B.innerHTML=xe),ce=i(e),z=J(e,"P",{"data-svelte-h":!0}),T(z)!=="svelte-cyywnn"&&(z.textContent=Ne),me=i(e),u(v.$$.fragment,e),ue=i(e),q=J(e,"P",{"data-svelte-h":!0}),T(q)!=="svelte-19p2zjb"&&(q.textContent=Se),Me=i(e),u(G.$$.fragment,e),de=i(e),X=J(e,"P",{"data-svelte-h":!0}),T(X)!=="svelte-11z2iie"&&(X.textContent=_e),be=i(e),u(E.$$.fragment,e),Je=i(e),x=J(e,"P",{"data-svelte-h":!0}),T(x)!=="svelte-1ipipqt"&&(x.textContent=Qe),ye=i(e),u(N.$$.fragment,e),Te=i(e),S=J(e,"P",{"data-svelte-h":!0}),T(S)!=="svelte-1ycw8i5"&&(S.innerHTML=Re),je=i(e),u(_.$$.fragment,e),we=i(e),u(Q.$$.fragment,e),fe=i(e),R=J(e,"P",{"data-svelte-h":!0}),T(R)!=="svelte-vd6zjh"&&(R.textContent=Ye),he=i(e),u(Y.$$.fragment,e),Ie=i(e),u(C.$$.fragment,e),Ue=i(e),F=J(e,"P",{"data-svelte-h":!0}),T(F)!=="svelte-bf4s2h"&&(F.textContent=Ce),Ze=i(e),u(H.$$.fragment,e),Ve=i(e),u(D.$$.fragment,e),ge=i(e),A=J(e,"P",{"data-svelte-h":!0}),T(A)!=="svelte-n5lvwy"&&(A.innerHTML=Fe),ke=i(e),u(P.$$.fragment,e),$e=i(e),L=J(e,"P",{"data-svelte-h":!0}),T(L)!=="svelte-1yrr9no"&&(L.innerHTML=He),We=i(e),U.l(e),le=i(e),u(K.$$.fragment,e),Be=i(e),te=J(e,"P",{}),Oe(te).forEach(l),this.h()},h(){es(n,"name","hf:doc:metadata"),es(n,"content",Ts)},m(e,s){rs(document.head,n),a(e,p,s),a(e,t,s),a(e,y,s),M(Z,e,s),a(e,ae,s),M(V,e,s),a(e,ne,s),M(g,e,s),a(e,oe,s),O[f].m(e,s),a(e,se,s),a(e,k,s),a(e,ie,s),a(e,$,s),a(e,pe,s),M(W,e,s),a(e,re,s),a(e,B,s),a(e,ce,s),a(e,z,s),a(e,me,s),M(v,e,s),a(e,ue,s),a(e,q,s),a(e,Me,s),M(G,e,s),a(e,de,s),a(e,X,s),a(e,be,s),M(E,e,s),a(e,Je,s),a(e,x,s),a(e,ye,s),M(N,e,s),a(e,Te,s),a(e,S,s),a(e,je,s),M(_,e,s),a(e,we,s),M(Q,e,s),a(e,fe,s),a(e,R,s),a(e,he,s),M(Y,e,s),a(e,Ie,s),M(C,e,s),a(e,Ue,s),a(e,F,s),a(e,Ze,s),M(H,e,s),a(e,Ve,s),M(D,e,s),a(e,ge,s),a(e,A,s),a(e,ke,s),M(P,e,s),a(e,$e,s),a(e,L,s),a(e,We,s),ee[I].m(e,s),a(e,le,s),M(K,e,s),a(e,Be,s),a(e,te,s),ze=!0},p(e,[s]){const Ke={};s&1&&(Ke.fw=e[0]),Z.$set(Ke);let ve=f;f=Ae(e),f!==ve&&(ls(),r(O[ve],1,1,()=>{O[ve]=null}),ss(),h=O[f],h||(h=O[f]=De[f](e),h.c()),c(h,1),h.m(se.parentNode,se));let qe=I;I=Le(e),I!==qe&&(ls(),r(ee[qe],1,1,()=>{ee[qe]=null}),ss(),U=ee[I],U||(U=ee[I]=Pe[I](e),U.c()),c(U,1),U.m(le.parentNode,le))},i(e){ze||(c(Z.$$.fragment,e),c(V.$$.fragment,e),c(g.$$.fragment,e),c(h),c(W.$$.fragment,e),c(v.$$.fragment,e),c(G.$$.fragment,e),c(E.$$.fragment,e),c(N.$$.fragment,e),c(_.$$.fragment,e),c(Q.$$.fragment,e),c(Y.$$.fragment,e),c(C.$$.fragment,e),c(H.$$.fragment,e),c(D.$$.fragment,e),c(P.$$.fragment,e),c(U),c(K.$$.fragment,e),ze=!0)},o(e){r(Z.$$.fragment,e),r(V.$$.fragment,e),r(g.$$.fragment,e),r(h),r(W.$$.fragment,e),r(v.$$.fragment,e),r(G.$$.fragment,e),r(E.$$.fragment,e),r(N.$$.fragment,e),r(_.$$.fragment,e),r(Q.$$.fragment,e),r(Y.$$.fragment,e),r(C.$$.fragment,e),r(H.$$.fragment,e),r(D.$$.fragment,e),r(P.$$.fragment,e),r(U),r(K.$$.fragment,e),ze=!1},d(e){e&&(l(p),l(t),l(y),l(ae),l(ne),l(oe),l(se),l(k),l(ie),l($),l(pe),l(re),l(B),l(ce),l(z),l(me),l(ue),l(q),l(Me),l(de),l(X),l(be),l(Je),l(x),l(ye),l(Te),l(S),l(je),l(we),l(fe),l(R),l(he),l(Ie),l(Ue),l(F),l(Ze),l(Ve),l(ge),l(A),l(ke),l($e),l(L),l(We),l(le),l(Be),l(te)),l(n),d(Z,e),d(V,e),d(g,e),O[f].d(e),d(W,e),d(v,e),d(G,e),d(E,e),d(N,e),d(_,e),d(Q,e),d(Y,e),d(C,e),d(H,e),d(D,e),d(P,e),ee[I].d(e),d(K,e)}}}const Ts='{"title":"Colocando tudo junto","local":"colocando-tudo-junto","sections":[{"title":"Tokens especiais","local":"tokens-especiais","sections":[],"depth":2},{"title":"Do tokenizer ao modelo","local":"do-tokenizer-ao-modelo","sections":[],"depth":2}],"depth":1}';function js(w,n,p){let t="pt";return ns(()=>{const y=new URLSearchParams(window.location.search);p(0,t=y.get("fw")||"pt")}),[t]}class Vs extends os{constructor(n){super(),is(this,n,js,ys,as,{})}}export{Vs as component};