course/pr_1213/it/_app/immutable/nodes/18.dfc68efe.js

import{s as nl,o as al}from"../chunks/scheduler.505acc25.js";import{S as il,i as ol,e as d,s as i,c,h as pl,a as y,d as t,b as o,f as Oe,g as m,j as T,k as el,l as rl,m as n,n as M,o as r,q as ll,t as u,p as b,r as tl}from"../chunks/index.1238bded.js";import{C as ul,H as Se,E as cl}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.d247c7a1.js";import{C as f}from"../chunks/CodeBlock.81d6557a.js";import{C as sl}from"../chunks/CourseFloatingBanner.0b6e065b.js";import{F as ml}from"../chunks/FrameworkSwitchCourse.23952889.js";function Ml(w){let a,p;return a=new sl({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_tf.ipynb"}]}}),{c(){c(a.$$.fragment)},l(s){m(a.$$.fragment,s)},m(s,J){M(a,s,J),p=!0},i(s){p||(u(a.$$.fragment,s),p=!0)},o(s){r(a.$$.fragment,s),p=!1},d(s){b(a,s)}}}function bl(w){let a,p;return a=new sl({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_pt.ipynb"}]}}),{c(){c(a.$$.fragment)},l(s){m(a.$$.fragment,s)},m(s,J){M(a,s,J),p=!0},i(s){p||(u(a.$$.fragment,s),p=!0)},o(s){r(a.$$.fragment,s),p=!1},d(s){b(a,s)}}}function dl(w){let a,p;return a=new f({props:{code:"aW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMFRGQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbiUwQSUwQWNoZWNrcG9pbnQlMjAlM0QlMjAlMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZC1maW5ldHVuZWQtc3N0LTItZW5nbGlzaCUyMiUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBbW9kZWwlMjAlM0QlMjBURkF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBc2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEF0b2tlbnMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjB0cnVuY2F0aW9uJTNEVHJ1ZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIydGYlMjIpJTBBb3V0cHV0JTIwJTNEJTIwbW9kZWwoKip0b2tlbnMp",highlighted:`<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = TFAutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
output = model(**tokens)`,wrap:!1}}),{c(){c(a.$$.fragment)},l(s){m(a.$$.fragment,s)},m(s,J){M(a,s,J),p=!0},i(s){p||(u(a.$$.fragment,s),p=!0)},o(s){r(a.$$.fragment,s),p=!1},d(s){b(a,s)}}}function yl(w){let a,p;return a=new f({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24lMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQtZmluZXR1bmVkLXNzdC0yLWVuZ2xpc2glMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbi5mcm9tX3ByZXRyYWluZWQoY2hlY2twb2ludCklMEFzZXF1ZW5jZXMlMjAlM0QlMjAlNUIlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMkMlMjAlMjJTbyUyMGhhdmUlMjBJISUyMiU1RCUwQSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHRydW5jYXRpb24lM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEFvdXRwdXQlMjAlM0QlMjBtb2RlbCgqKnRva2Vucyk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
output = model(**tokens)`,wrap:!1}}),{c(){c(a.$$.fragment)},l(s){m(a.$$.fragment,s)},m(s,J){M(a,s,J),p=!0},i(s){p||(u(a.$$.fragment,s),p=!0)},o(s){r(a.$$.fragment,s),p=!1},d(s){b(a,s)}}}function Jl(w){let a,p,s,J,U,ne,Z,ae,k,ie,j,h,le,z,qe="Nelle ultime sezioni abbiamo fatto del nostro meglio per fare la maggior parte del lavoro a mano. Abbiamo esplorato il funzionamento dei tokenizer e abbiamo esaminato la tokenizzazione, la conversione in ID di input, il padding, il troncamento e le maschere di attenzione.",oe,$,Xe="Tuttavia, come abbiamo visto nella sezione 2, l’API 🤗 Transformers può gestire tutto questo con una funzione di alto livello che approfondiremo qui. Quando si chiama il <code>tokenizer</code> direttamente sulla frase, si ottengono input pronti per passare attraverso il modello:",pe,V,re,W,xe="Qui, la variabile <code>model_inputs</code> contiene tutto ciò che è necessario per il buon funzionamento del modello. Per DistilBERT, questo include gli ID degli ingressi e la maschera di attenzione. Altri modelli che accettano input aggiuntivi avranno anche questi output dall’oggetto <code>tokenizer</code>.",ue,B,Ee="Come vedremo in alcuni esempi, questo metodo è molto potente. Innanzitutto, può tokenizzare una singola sequenza:",ce,G,me,v,Re="Gestisce anche più sequenze alla volta, senza alcuna modifica dell’API:",Me,S,be,q,_e="Possiamo implementare il padding in diversi modi",de,X,ye,x,Ne="Può anche troncare le sequenze:",Je,E,Te,R,Ye="L’oggetto <code>tokenizer</code> può gestire la conversione in tensori di framework specifici, che possono successivamente essere inviati direttamente al modello. Per esempio, nel seguente esempio di codice si chiede al tokenizer di restituire i tensori dei diversi framework: <code>&quot;pt&quot;</code> restituisce i tensori di PyTorch, <code>&quot;tf&quot;</code> restituisce i tensori di TensorFlow e <code>&quot;np&quot;</code> restituisce gli array di NumPy:",fe,_,we,N,je,Y,Qe="Se diamo un’occhiata agli ID di input restituiti dal tokenizer, noteremo che sono leggermente diversi da quelli che avevamo prima:",he,Q,ge,C,Ie,F,Ce="Un ID token è stato aggiunto all’inizio e uno alla fine. Decodifichiamo le due sequenze di ID qui sopra per capire di cosa si tratta:",Ue,H,Ze,A,ke,D,Fe="Il tokenizer ha aggiunto la parola speciale <code>[CLS]</code> all’inizio e la parola speciale <code>[SEP]</code> alla fine. Questo perché il modello è stato preaddestrato con queste parole, quindi per ottenere gli stessi risultati per l’inferenza dobbiamo aggiungerle anche noi. Si noti che alcuni modelli non aggiungono parole speciali, o ne aggiungono di diverse; i modelli possono anche aggiungere queste parole speciali solo all’inizio o solo alla fine. In ogni caso, il tokenizer sa quali sono previste e se ne occuperà per voi.",ze,P,$e,L,He="Ora che abbiamo visto tutti i singoli passaggi che l’oggetto <code>tokenizer</code> utilizza quando viene applicato ai testi, vediamo un’ultima volta come può gestire sequenze multiple (padding!), sequenze molto lunghe (troncamento!) e diversi tipi di tensori con la sua API principale:",Ve,g,I,te,K,We,se,Be;U=new ml({props:{fw:w[0]}}),Z=new ul({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),k=new Se({props:{title:"Mettiamo insieme i pezzi",local:"mettiamo-insieme-i-pezzi",headingTag:"h1"}});const Ae=[bl,Ml],O=[];function De(e,l){return e[0]==="pt"?0:1}j=De(w),h=O[j]=Ae[j](w),V=new f({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQtZmluZXR1bmVkLXNzdC0yLWVuZ2xpc2glMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQSUwQXNlcXVlbmNlJTIwJTNEJTIwJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTBBJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`,wrap:!1}}),G=new f({props:{code:"c2VxdWVuY2UlMjAlM0QlMjAlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2Up",highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`,wrap:!1}}),S=new f({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzKQ==",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

model_inputs = tokenizer(sequences)`,wrap:!1}}),X=new f({props:{code:"JTIzJTIwRWZmZXR0dWElMjBpbCUyMHBhZGRpbmclMjBkZWxsYSUyMHNlcXVlbnphJTIwZmlubyUyMGFsbGxhJTIwbWFzc2ltYSUyMGx1bmdoZXp6YSUyMGRlbGxhJTIwc2VxdWVuemElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRCUyMmxvbmdlc3QlMjIpJTBBJTBBJTIzJTIwRWZmZXR0dWElMjBpbCUyMHBhZGRpbmclMjBmaW5vJTIwYWxsYSUyMGx1bmdoZXp6YSUyMG1hc3NpbWElMjBkZWwlMjBtb2RlbGxvJTBBJTIzJTIwKDUxMiUyMHBlciUyMEJFUlQlMjBvJTIwRGlzdGlsQkVSVCklMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRCUyMm1heF9sZW5ndGglMjIpJTBBJTBBJTIzJTIwRWZmZXR0dWElMjBpbCUyMHBhZGRpbmclMjBmaW5vJTIwYWxsYSUyMGx1bmdoZXp6YSUyMG1hc3NpbWElMjBzcGVjaWZpY2F0YSUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEJTIybWF4X2xlbmd0aCUyMiUyQyUyMG1heF9sZW5ndGglM0Q4KQ==",highlighted:`<span class="hljs-comment"># Effettua il padding della sequenza fino allla massima lunghezza della sequenza</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;longest&quot;</span>)

<span class="hljs-comment"># Effettua il padding fino alla lunghezza massima del modello</span>
<span class="hljs-comment"># (512 per BERT o DistilBERT)</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>)

<span class="hljs-comment"># Effettua il padding fino alla lunghezza massima specificata</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>, max_length=<span class="hljs-number">8</span>)`,wrap:!1}}),E=new f({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEElMjMlMjBUcm9uY2ElMjBsZSUyMHNlcXVlbnplJTIwcGklQzMlQjklMjBsdW5naGUlMjBkZWxsYSUyMGx1bmdoZXp6YSUyMG1hc3NpbWElMjBkZWwlMjBtb2RlbGxvLiUwQSUyMyUyMCg1MTIlMjBwZXIlMjBCRVJUJTIwbyUyMERpc3RpbEJFUlQpJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMHRydW5jYXRpb24lM0RUcnVlKSUwQSUwQSUyMyUyMFRyb25jYSUyMGxlJTIwc2VxdWVuemUlMjBwaSVDMyVCOSUyMGx1bmdoZSUyMGRlbGxhJTIwbHVuZ2hlenphJTIwbWFzc2ltYSUyMHNwZWNpZmljYXRhLiUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBtYXhfbGVuZ3RoJTNEOCUyQyUyMHRydW5jYXRpb24lM0RUcnVlKQ==",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># Tronca le sequenze più lunghe della lunghezza massima del modello.</span>
<span class="hljs-comment"># (512 per BERT o DistilBERT)</span>
model_inputs = tokenizer(sequences, truncation=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># Tronca le sequenze più lunghe della lunghezza massima specificata.</span>
model_inputs = tokenizer(sequences, max_length=<span class="hljs-number">8</span>, truncation=<span class="hljs-literal">True</span>)`,wrap:!1}}),_=new f({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEElMjMlMjBSaXRvcm5hJTIwdGVuc29yaSUyMFB5VG9yY2glMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQSUwQSUyMyUyMFJpdG9ybmElMjB0ZW5zb3JpJTIwVGVuc29yRmxvdyUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIydGYlMjIpJTBBJTBBJTIzJTIwUml0b3JuYSUyME51bVB5JTIwYXJyYXlzJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMHBhZGRpbmclM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJucCUyMik=",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># Ritorna tensori PyTorch</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-comment"># Ritorna tensori TensorFlow</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)

<span class="hljs-comment"># Ritorna NumPy arrays</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;np&quot;</span>)`,wrap:!1}}),N=new Se({props:{title:"Token speciali",local:"token-speciali",headingTag:"h2"}}),Q=new f({props:{code:"c2VxdWVuY2UlMjAlM0QlMjAlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2UpJTBBcHJpbnQobW9kZWxfaW5wdXRzJTVCJTIyaW5wdXRfaWRzJTIyJTVEKSUwQSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplci50b2tlbml6ZShzZXF1ZW5jZSklMEFpZHMlMjAlM0QlMjB0b2tlbml6ZXIuY29udmVydF90b2tlbnNfdG9faWRzKHRva2VucyklMEFwcmludChpZHMp",highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)
<span class="hljs-built_in">print</span>(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>])

tokens = tokenizer.tokenize(sequence)
ids = tokenizer.convert_tokens_to_ids(tokens)
<span class="hljs-built_in">print</span>(ids)`,wrap:!1}}),C=new f({props:{code:"JTVCMTAxJTJDJTIwMTA0NSUyQyUyMDEwMDUlMkMlMjAyMzEwJTJDJTIwMjA0MiUyQyUyMDM0MDMlMkMlMjAyMDA1JTJDJTIwMTAzNyUyQyUyMDE3NjYyJTJDJTIwMTIxNzIlMkMlMjAyNjA3JTJDJTIwMjAyNiUyQyUyMDI4NzglMkMlMjAyMTY2JTJDJTIwMTAxMiUyQyUyMDEwMiU1RCUwQSU1QjEwNDUlMkMlMjAxMDA1JTJDJTIwMjMxMCUyQyUyMDIwNDIlMkMlMjAzNDAzJTJDJTIwMjAwNSUyQyUyMDEwMzclMkMlMjAxNzY2MiUyQyUyMDEyMTcyJTJDJTIwMjYwNyUyQyUyMDIwMjYlMkMlMjAyODc4JTJDJTIwMjE2NiUyQyUyMDEwMTIlNUQ=",highlighted:`[<span class="hljs-number">101</span>, <span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>]
[<span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>]`,wrap:!1}}),H=new f({props:{code:"cHJpbnQodG9rZW5pemVyLmRlY29kZShtb2RlbF9pbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQpKSUwQXByaW50KHRva2VuaXplci5kZWNvZGUoaWRzKSk=",highlighted:`<span class="hljs-built_in">print</span>(tokenizer.decode(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>]))
<span class="hljs-built_in">print</span>(tokenizer.decode(ids))`,wrap:!1}}),A=new f({props:{code:"JTIyJTVCQ0xTJTVEJTIwaSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMGh1Z2dpbmdmYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIwJTVCU0VQJTVEJTIyJTBBJTIyaSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMGh1Z2dpbmdmYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIy",highlighted:`<span class="hljs-string">&quot;[CLS] i&#x27;ve been waiting for a huggingface course my whole life. [SEP]&quot;</span>
<span class="hljs-string">&quot;i&#x27;ve been waiting for a huggingface course my whole life.&quot;</span>`,wrap:!1}}),P=new Se({props:{title:"Conclusione: Dal tokenizer al modello",local:"conclusione-dal-tokenizer-al-modello",headingTag:"h2"}});const Pe=[yl,dl],ee=[];function Le(e,l){return e[0]==="pt"?0:1}return g=Le(w),I=ee[g]=Pe[g](w),K=new cl({props:{source:"https://github.com/huggingface/course/blob/main/chapters/it/chapter2/6.mdx"}}),{c(){a=d("meta"),p=i(),s=d("p"),J=i(),c(U.$$.fragment),ne=i(),c(Z.$$.fragment),ae=i(),c(k.$$.fragment),ie=i(),h.c(),le=i(),z=d("p"),z.textContent=qe,oe=i(),$=d("p"),$.innerHTML=Xe,pe=i(),c(V.$$.fragment),re=i(),W=d("p"),W.innerHTML=xe,ue=i(),B=d("p"),B.textContent=Ee,ce=i(),c(G.$$.fragment),me=i(),v=d("p"),v.textContent=Re,Me=i(),c(S.$$.fragment),be=i(),q=d("p"),q.textContent=_e,de=i(),c(X.$$.fragment),ye=i(),x=d("p"),x.textContent=Ne,Je=i(),c(E.$$.fragment),Te=i(),R=d("p"),R.innerHTML=Ye,fe=i(),c(_.$$.fragment),we=i(),c(N.$$.fragment),je=i(),Y=d("p"),Y.textContent=Qe,he=i(),c(Q.$$.fragment),ge=i(),c(C.$$.fragment),Ie=i(),F=d("p"),F.textContent=Ce,Ue=i(),c(H.$$.fragment),Ze=i(),c(A.$$.fragment),ke=i(),D=d("p"),D.innerHTML=Fe,ze=i(),c(P.$$.fragment),$e=i(),L=d("p"),L.innerHTML=He,Ve=i(),I.c(),te=i(),c(K.$$.fragment),We=i(),se=d("p"),this.h()},l(e){const l=pl("svelte-u9bgzb",document.head);a=y(l,"META",{name:!0,content:!0}),l.forEach(t),p=o(e),s=y(e,"P",{}),Oe(s).forEach(t),J=o(e),m(U.$$.fragment,e),ne=o(e),m(Z.$$.fragment,e),ae=o(e),m(k.$$.fragment,e),ie=o(e),h.l(e),le=o(e),z=y(e,"P",{"data-svelte-h":!0}),T(z)!=="svelte-1gq1u3k"&&(z.textContent=qe),oe=o(e),$=y(e,"P",{"data-svelte-h":!0}),T($)!=="svelte-1xosnnr"&&($.innerHTML=Xe),pe=o(e),m(V.$$.fragment,e),re=o(e),W=y(e,"P",{"data-svelte-h":!0}),T(W)!=="svelte-1aee5q6"&&(W.innerHTML=xe),ue=o(e),B=y(e,"P",{"data-svelte-h":!0}),T(B)!=="svelte-1518yby"&&(B.textContent=Ee),ce=o(e),m(G.$$.fragment,e),me=o(e),v=y(e,"P",{"data-svelte-h":!0}),T(v)!=="svelte-t4crif"&&(v.textContent=Re),Me=o(e),m(S.$$.fragment,e),be=o(e),q=y(e,"P",{"data-svelte-h":!0}),T(q)!=="svelte-110r6wo"&&(q.textContent=_e),de=o(e),m(X.$$.fragment,e),ye=o(e),x=y(e,"P",{"data-svelte-h":!0}),T(x)!=="svelte-c5od7r"&&(x.textContent=Ne),Je=o(e),m(E.$$.fragment,e),Te=o(e),R=y(e,"P",{"data-svelte-h":!0}),T(R)!=="svelte-avke6z"&&(R.innerHTML=Ye),fe=o(e),m(_.$$.fragment,e),we=o(e),m(N.$$.fragment,e),je=o(e),Y=y(e,"P",{"data-svelte-h":!0}),T(Y)!=="svelte-1r8bndt"&&(Y.textContent=Qe),he=o(e),m(Q.$$.fragment,e),ge=o(e),m(C.$$.fragment,e),Ie=o(e),F=y(e,"P",{"data-svelte-h":!0}),T(F)!=="svelte-di8hcs"&&(F.textContent=Ce),Ue=o(e),m(H.$$.fragment,e),Ze=o(e),m(A.$$.fragment,e),ke=o(e),D=y(e,"P",{"data-svelte-h":!0}),T(D)!=="svelte-1m0dtz8"&&(D.innerHTML=Fe),ze=o(e),m(P.$$.fragment,e),$e=o(e),L=y(e,"P",{"data-svelte-h":!0}),T(L)!=="svelte-8r23w7"&&(L.innerHTML=He),Ve=o(e),I.l(e),te=o(e),m(K.$$.fragment,e),We=o(e),se=y(e,"P",{}),Oe(se).forEach(t),this.h()},h(){el(a,"name","hf:doc:metadata"),el(a,"content",Tl)},m(e,l){rl(document.head,a),n(e,p,l),n(e,s,l),n(e,J,l),M(U,e,l),n(e,ne,l),M(Z,e,l),n(e,ae,l),M(k,e,l),n(e,ie,l),O[j].m(e,l),n(e,le,l),n(e,z,l),n(e,oe,l),n(e,$,l),n(e,pe,l),M(V,e,l),n(e,re,l),n(e,W,l),n(e,ue,l),n(e,B,l),n(e,ce,l),M(G,e,l),n(e,me,l),n(e,v,l),n(e,Me,l),M(S,e,l),n(e,be,l),n(e,q,l),n(e,de,l),M(X,e,l),n(e,ye,l),n(e,x,l),n(e,Je,l),M(E,e,l),n(e,Te,l),n(e,R,l),n(e,fe,l),M(_,e,l),n(e,we,l),M(N,e,l),n(e,je,l),n(e,Y,l),n(e,he,l),M(Q,e,l),n(e,ge,l),M(C,e,l),n(e,Ie,l),n(e,F,l),n(e,Ue,l),M(H,e,l),n(e,Ze,l),M(A,e,l),n(e,ke,l),n(e,D,l),n(e,ze,l),M(P,e,l),n(e,$e,l),n(e,L,l),n(e,Ve,l),ee[g].m(e,l),n(e,te,l),M(K,e,l),n(e,We,l),n(e,se,l),Be=!0},p(e,[l]){const Ke={};l&1&&(Ke.fw=e[0]),U.$set(Ke);let Ge=j;j=De(e),j!==Ge&&(tl(),r(O[Ge],1,1,()=>{O[Ge]=null}),ll(),h=O[j],h||(h=O[j]=Ae[j](e),h.c()),u(h,1),h.m(le.parentNode,le));let ve=g;g=Le(e),g!==ve&&(tl(),r(ee[ve],1,1,()=>{ee[ve]=null}),ll(),I=ee[g],I||(I=ee[g]=Pe[g](e),I.c()),u(I,1),I.m(te.parentNode,te))},i(e){Be||(u(U.$$.fragment,e),u(Z.$$.fragment,e),u(k.$$.fragment,e),u(h),u(V.$$.fragment,e),u(G.$$.fragment,e),u(S.$$.fragment,e),u(X.$$.fragment,e),u(E.$$.fragment,e),u(_.$$.fragment,e),u(N.$$.fragment,e),u(Q.$$.fragment,e),u(C.$$.fragment,e),u(H.$$.fragment,e),u(A.$$.fragment,e),u(P.$$.fragment,e),u(I),u(K.$$.fragment,e),Be=!0)},o(e){r(U.$$.fragment,e),r(Z.$$.fragment,e),r(k.$$.fragment,e),r(h),r(V.$$.fragment,e),r(G.$$.fragment,e),r(S.$$.fragment,e),r(X.$$.fragment,e),r(E.$$.fragment,e),r(_.$$.fragment,e),r(N.$$.fragment,e),r(Q.$$.fragment,e),r(C.$$.fragment,e),r(H.$$.fragment,e),r(A.$$.fragment,e),r(P.$$.fragment,e),r(I),r(K.$$.fragment,e),Be=!1},d(e){e&&(t(p),t(s),t(J),t(ne),t(ae),t(ie),t(le),t(z),t(oe),t($),t(pe),t(re),t(W),t(ue),t(B),t(ce),t(me),t(v),t(Me),t(be),t(q),t(de),t(ye),t(x),t(Je),t(Te),t(R),t(fe),t(we),t(je),t(Y),t(he),t(ge),t(Ie),t(F),t(Ue),t(Ze),t(ke),t(D),t(ze),t($e),t(L),t(Ve),t(te),t(We),t(se)),t(a),b(U,e),b(Z,e),b(k,e),O[j].d(e),b(V,e),b(G,e),b(S,e),b(X,e),b(E,e),b(_,e),b(N,e),b(Q,e),b(C,e),b(H,e),b(A,e),b(P,e),ee[g].d(e),b(K,e)}}}const Tl='{"title":"Mettiamo insieme i pezzi","local":"mettiamo-insieme-i-pezzi","sections":[{"title":"Token speciali","local":"token-speciali","sections":[],"depth":2},{"title":"Conclusione: Dal tokenizer al modello","local":"conclusione-dal-tokenizer-al-modello","sections":[],"depth":2}],"depth":1}';function fl(w,a,p){let s="pt";return al(()=>{const J=new URLSearchParams(window.location.search);p(0,s=J.get("fw")||"pt")}),[s]}class Zl extends il{constructor(a){super(),ol(this,a,fl,Jl,nl,{})}}export{Zl as component};