course/pr_1132/en/_app/immutable/nodes/44.ec00643e.js

import{s as We,o as ve}from"../chunks/scheduler.893fe8c9.js";import{S as Ie,i as Ae,e as ye,s as a,c as s,h as _e,a as ke,d as n,b as i,f as Te,g as r,k as Me,l as je,m as o,n as h,t as l,o as m,p}from"../chunks/index.b1df2166.js";import{C as Ze,H as f,E as Je}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.0a609dc6.js";import{C as qe}from"../chunks/CodeBlock.845f8dd4.js";import{C as Ce}from"../chunks/CourseFloatingBanner.c1c08878.js";import{Q as u}from"../chunks/Question.ea6d4cb0.js";import{F as Ee}from"../chunks/FrameworkSwitchCourse.4480e339.js";function Be(xe){let d,V,c,S,$,D,g,G,w,Q,b,Y,x,L,y,N,k,K,z,O,T,ee,M,te,q,ne,W,oe,v,ae,I,ie,A,se,_,re,j,he,Z,le,J,me,C,pe,E,fe,B,ue,U,de,H,ce,X,$e,F,ge,R,we,P,be;return $=new Ee({props:{fw:xe[0]}}),g=new Ze({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),w=new f({props:{title:"End-of-chapter quiz",local:"end-of-chapter-quiz",headingTag:"h1"}}),b=new Ce({props:{chapter:2,classNames:"absolute z-10 right-0 top-0"}}),x=new f({props:{title:"1. What is the order of the language modeling pipeline?",local:"1-what-is-the-order-of-the-language-modeling-pipeline",headingTag:"h3"}}),y=new u({props:{choices:[{text:"First, the model, which handles text and returns raw predictions. The tokenizer then makes sense of these predictions and converts them back to text when needed.",explain:"The model cannot understand text! The tokenizer must first tokenize the text and convert it to IDs so that it is understandable by the model."},{text:"First, the tokenizer, which handles text and returns IDs. The model handles these IDs and outputs a prediction, which can be some text.",explain:"The model's prediction cannot be text straight away. The tokenizer has to be used in order to convert the prediction back to text!"},{text:"The tokenizer handles text and returns IDs. The model handles these IDs and outputs a prediction. The tokenizer can then be used once again to convert these predictions back to some text.",explain:"The tokenizer can be used for both tokenizing and de-tokenizing.",correct:!0}]}}),k=new f({props:{title:"2. How many dimensions does the tensor output by the base Transformer model have, and what are they?",local:"2-how-many-dimensions-does-the-tensor-output-by-the-base-transformer-model-have-and-what-are-they",headingTag:"h3"}}),z=new u({props:{choices:[{text:"2: The sequence length and the batch size",explain:"False! The tensor output by the model has a third dimension: hidden size."},{text:"2: The sequence length and the hidden size",explain:"False! All Transformer models handle batches, even with a single sequence; that would be a batch size of 1!"},{text:"3: The sequence length, the batch size, and the hidden size",explain:"Nicely done!",correct:!0}]}}),T=new f({props:{title:"3. Which of the following is an example of subword tokenization?",local:"3-which-of-the-following-is-an-example-of-subword-tokenization",headingTag:"h3"}}),M=new u({props:{choices:[{text:"WordPiece",explain:"Yes, that's one example of subword tokenization!",correct:!0},{text:"Character-based tokenization",explain:"Character-based tokenization is not a type of subword tokenization."},{text:"Splitting on whitespace and punctuation",explain:"That's a word-based tokenization scheme!"},{text:"BPE",explain:"Yes, that's one example of subword tokenization!",correct:!0},{text:"Unigram",explain:"Yes, that's one example of subword tokenization!",correct:!0},{text:"None of the above",explain:"Wrong!"}]}}),q=new f({props:{title:"4. What is a model head?",local:"4-what-is-a-model-head",headingTag:"h3"}}),W=new u({props:{choices:[{text:"A component of the base Transformer network that redirects tensors to their correct layers",explain:"There's no such component."},{text:"Also known as the self-attention mechanism, it adapts the representation of a token according to the other tokens of the sequence",explain:'The self-attention layer does contain attention "heads," but these are not adaptation heads.'},{text:"An additional component, usually made up of one or a few layers, to convert the transformer predictions to a task-specific output",explain:"That's right. Adaptation heads, also known simply as heads, come up in different forms: language modeling heads, question answering heads, sequence classification heads... ",correct:!0}]}}),v=new f({props:{title:"5. What is an AutoModel?",local:"5-what-is-an-automodel",headingTag:"h3"}}),I=new u({props:{choices:[{text:"A model that automatically trains on your data",explain:"Are you mistaking this with our <a href='https://huggingface.co/autotrain'>AutoTrain</a> product?"},{text:"An object that returns the correct architecture based on the checkpoint",explain:"Exactly: the <code>AutoModel</code> only needs to know the checkpoint from which to initialize to return the correct architecture.",correct:!0},{text:"A model that automatically detects the language used for its inputs to load the correct weights",explain:"While some checkpoints and models are capable of handling multiple languages, there are no built-in tools for automatic checkpoint selection according to language. You should head over to the <a href='https://huggingface.co/models'>Model Hub</a> to find the best checkpoint for your task!"}]}}),A=new f({props:{title:"6. What are the techniques to be aware of when batching sequences of different lengths together?",local:"6-what-are-the-techniques-to-be-aware-of-when-batching-sequences-of-different-lengths-together",headingTag:"h3"}}),_=new u({props:{choices:[{text:"Truncating",explain:"Yes, truncation is a correct way of evening out sequences so that they fit in a rectangular shape. Is it the only one, though?",correct:!0},{text:"Returning tensors",explain:"While the other techniques allow you to return rectangular tensors, returning tensors isn't helpful when batching sequences together."},{text:"Padding",explain:"Yes, padding is a correct way of evening out sequences so that they fit in a rectangular shape. Is it the only one, though?",correct:!0},{text:"Attention masking",explain:"Absolutely! Attention masks are of prime importance when handling sequences of different lengths. That's not the only technique to be aware of, however.",correct:!0}]}}),j=new f({props:{title:"7. What is the point of applying a SoftMax function to the logits output by a sequence classification model?",local:"7-what-is-the-point-of-applying-a-softmax-function-to-the-logits-output-by-a-sequence-classification-model",headingTag:"h3"}}),Z=new u({props:{choices:[{text:"It softens the logits so that they're more reliable.",explain:"No, the SoftMax function does not affect the reliability of results."},{text:"It applies a lower and upper bound so that they're understandable.",explain:"The resulting values are bound between 0 and 1. That's not the only reason we use a SoftMax function, though.",correct:!0},{text:"The total sum of the output is then 1, resulting in a possible probabilistic interpretation.",explain:"Correct! That's not the only reason we use a SoftMax function, though.",correct:!0}]}}),J=new f({props:{title:"8. What method is most of the tokenizer API centered around?",local:"8-what-method-is-most-of-the-tokenizer-api-centered-around",headingTag:"h3"}}),C=new u({props:{choices:[{text:"<code>encode</code>, as it can encode text into IDs and IDs into predictions",explain:"Wrong! While the <code>encode</code> method does exist on tokenizers, it does not exist on models."},{text:"Calling the tokenizer object directly.",explain:"Exactly! The <code>__call__</code> method of the tokenizer is a very powerful method which can handle pretty much anything. It is also the method used to retrieve predictions from a model.",correct:!0},{text:"<code>pad</code>",explain:"Wrong! Padding is very useful, but it's just one part of the tokenizer API."},{text:"<code>tokenize</code>",explain:"The <code>tokenize</code> method is arguably one of the most useful methods, but it isn't the core of the tokenizer API."}]}}),E=new f({props:{title:"9. What does the result variable contain in this code sample?",local:"9-what-does-the-result-variable-contain-in-this-code-sample",headingTag:"h3"}}),B=new qe({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJiZXJ0LWJhc2UtY2FzZWQlMjIpJTBBcmVzdWx0JTIwJTNEJTIwdG9rZW5pemVyLnRva2VuaXplKCUyMkhlbGxvISUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)
result = tokenizer.tokenize(<span class="hljs-string">&quot;Hello!&quot;</span>)`,wrap:!1}}),U=new u({props:{choices:[{text:"A list of strings, each string being a token",explain:"Absolutely! Convert this to IDs, and send them to a model!",correct:!0},{text:"A list of IDs",explain:"Incorrect; that's what the <code>__call__</code> or <code>convert_tokens_to_ids</code> method is for!"},{text:"A string containing all of the tokens",explain:"This would be suboptimal, as the goal is to split the string into multiple tokens."}]}}),H=new f({props:{title:"10. Is there something wrong with the following code?",local:"10-is-there-something-wrong-with-the-following-code",headingTag:"h3"}}),X=new qe({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWwlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJiZXJ0LWJhc2UtY2FzZWQlMjIpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUyMmdwdDIlMjIpJTBBJTBBZW5jb2RlZCUyMCUzRCUyMHRva2VuaXplciglMjJIZXkhJTIyJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEFyZXN1bHQlMjAlM0QlMjBtb2RlbCgqKmVuY29kZWQp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModel

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)
model = AutoModel.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)

encoded = tokenizer(<span class="hljs-string">&quot;Hey!&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
result = model(**encoded)`,wrap:!1}}),F=new u({props:{choices:[{text:"No, it seems correct.",explain:"Unfortunately, coupling a model with a tokenizer that was trained with a different checkpoint is rarely a good idea. The model was not trained to make sense out of this tokenizer's output, so the model output (if it can even run!) will not make any sense."},{text:"The tokenizer and model should always be from the same checkpoint.",explain:"Right!",correct:!0},{text:"It's good practice to pad and truncate with the tokenizer as every input is a batch.",explain:"It's true that every model input needs to be a batch. However, truncating or padding this sequence wouldn't necessarily make sense as there is only one of it, and those are techniques to batch together a list of sentences."}]}}),R=new Je({props:{source:"https://github.com/huggingface/course/blob/main/chapters/en/chapter2/9.mdx"}}),{c(){d=ye("meta"),V=a(),c=ye("p"),S=a(),s($.$$.fragment),D=a(),s(g.$$.fragment),G=a(),s(w.$$.fragment),Q=a(),s(b.$$.fragment),Y=a(),s(x.$$.fragment),L=a(),s(y.$$.fragment),N=a(),s(k.$$.fragment),K=a(),s(z.$$.fragment),O=a(),s(T.$$.fragment),ee=a(),s(M.$$.fragment),te=a(),s(q.$$.fragment),ne=a(),s(W.$$.fragment),oe=a(),s(v.$$.fragment),ae=a(),s(I.$$.fragment),ie=a(),s(A.$$.fragment),se=a(),s(_.$$.fragment),re=a(),s(j.$$.fragment),he=a(),s(Z.$$.fragment),le=a(),s(J.$$.fragment),me=a(),s(C.$$.fragment),pe=a(),s(E.$$.fragment),fe=a(),s(B.$$.fragment),ue=a(),s(U.$$.fragment),de=a(),s(H.$$.fragment),ce=a(),s(X.$$.fragment),$e=a(),s(F.$$.fragment),ge=a(),s(R.$$.fragment),we=a(),P=ye("p"),this.h()},l(e){const t=_e("svelte-u9bgzb",document.head);d=ke(t,"META",{name:!0,content:!0}),t.forEach(n),V=i(e),c=ke(e,"P",{}),Te(c).forEach(n),S=i(e),r($.$$.fragment,e),D=i(e),r(g.$$.fragment,e),G=i(e),r(w.$$.fragment,e),Q=i(e),r(b.$$.fragment,e),Y=i(e),r(x.$$.fragment,e),L=i(e),r(y.$$.fragment,e),N=i(e),r(k.$$.fragment,e),K=i(e),r(z.$$.fragment,e),O=i(e),r(T.$$.fragment,e),ee=i(e),r(M.$$.fragment,e),te=i(e),r(q.$$.fragment,e),ne=i(e),r(W.$$.fragment,e),oe=i(e),r(v.$$.fragment,e),ae=i(e),r(I.$$.fragment,e),ie=i(e),r(A.$$.fragment,e),se=i(e),r(_.$$.fragment,e),re=i(e),r(j.$$.fragment,e),he=i(e),r(Z.$$.fragment,e),le=i(e),r(J.$$.fragment,e),me=i(e),r(C.$$.fragment,e),pe=i(e),r(E.$$.fragment,e),fe=i(e),r(B.$$.fragment,e),ue=i(e),r(U.$$.fragment,e),de=i(e),r(H.$$.fragment,e),ce=i(e),r(X.$$.fragment,e),$e=i(e),r(F.$$.fragment,e),ge=i(e),r(R.$$.fragment,e),we=i(e),P=ke(e,"P",{}),Te(P).forEach(n),this.h()},h(){Me(d,"name","hf:doc:metadata"),Me(d,"content",Ue)},m(e,t){je(document.head,d),o(e,V,t),o(e,c,t),o(e,S,t),h($,e,t),o(e,D,t),h(g,e,t),o(e,G,t),h(w,e,t),o(e,Q,t),h(b,e,t),o(e,Y,t),h(x,e,t),o(e,L,t),h(y,e,t),o(e,N,t),h(k,e,t),o(e,K,t),h(z,e,t),o(e,O,t),h(T,e,t),o(e,ee,t),h(M,e,t),o(e,te,t),h(q,e,t),o(e,ne,t),h(W,e,t),o(e,oe,t),h(v,e,t),o(e,ae,t),h(I,e,t),o(e,ie,t),h(A,e,t),o(e,se,t),h(_,e,t),o(e,re,t),h(j,e,t),o(e,he,t),h(Z,e,t),o(e,le,t),h(J,e,t),o(e,me,t),h(C,e,t),o(e,pe,t),h(E,e,t),o(e,fe,t),h(B,e,t),o(e,ue,t),h(U,e,t),o(e,de,t),h(H,e,t),o(e,ce,t),h(X,e,t),o(e,$e,t),h(F,e,t),o(e,ge,t),h(R,e,t),o(e,we,t),o(e,P,t),be=!0},p(e,[t]){const ze={};t&1&&(ze.fw=e[0]),$.$set(ze)},i(e){be||(l($.$$.fragment,e),l(g.$$.fragment,e),l(w.$$.fragment,e),l(b.$$.fragment,e),l(x.$$.fragment,e),l(y.$$.fragment,e),l(k.$$.fragment,e),l(z.$$.fragment,e),l(T.$$.fragment,e),l(M.$$.fragment,e),l(q.$$.fragment,e),l(W.$$.fragment,e),l(v.$$.fragment,e),l(I.$$.fragment,e),l(A.$$.fragment,e),l(_.$$.fragment,e),l(j.$$.fragment,e),l(Z.$$.fragment,e),l(J.$$.fragment,e),l(C.$$.fragment,e),l(E.$$.fragment,e),l(B.$$.fragment,e),l(U.$$.fragment,e),l(H.$$.fragment,e),l(X.$$.fragment,e),l(F.$$.fragment,e),l(R.$$.fragment,e),be=!0)},o(e){m($.$$.fragment,e),m(g.$$.fragment,e),m(w.$$.fragment,e),m(b.$$.fragment,e),m(x.$$.fragment,e),m(y.$$.fragment,e),m(k.$$.fragment,e),m(z.$$.fragment,e),m(T.$$.fragment,e),m(M.$$.fragment,e),m(q.$$.fragment,e),m(W.$$.fragment,e),m(v.$$.fragment,e),m(I.$$.fragment,e),m(A.$$.fragment,e),m(_.$$.fragment,e),m(j.$$.fragment,e),m(Z.$$.fragment,e),m(J.$$.fragment,e),m(C.$$.fragment,e),m(E.$$.fragment,e),m(B.$$.fragment,e),m(U.$$.fragment,e),m(H.$$.fragment,e),m(X.$$.fragment,e),m(F.$$.fragment,e),m(R.$$.fragment,e),be=!1},d(e){e&&(n(V),n(c),n(S),n(D),n(G),n(Q),n(Y),n(L),n(N),n(K),n(O),n(ee),n(te),n(ne),n(oe),n(ae),n(ie),n(se),n(re),n(he),n(le),n(me),n(pe),n(fe),n(ue),n(de),n(ce),n($e),n(ge),n(we),n(P)),n(d),p($,e),p(g,e),p(w,e),p(b,e),p(x,e),p(y,e),p(k,e),p(z,e),p(T,e),p(M,e),p(q,e),p(W,e),p(v,e),p(I,e),p(A,e),p(_,e),p(j,e),p(Z,e),p(J,e),p(C,e),p(E,e),p(B,e),p(U,e),p(H,e),p(X,e),p(F,e),p(R,e)}}}const Ue='{"title":"End-of-chapter quiz","local":"end-of-chapter-quiz","sections":[{"title":"1. What is the order of the language modeling pipeline?","local":"1-what-is-the-order-of-the-language-modeling-pipeline","sections":[],"depth":3},{"title":"2. How many dimensions does the tensor output by the base Transformer model have, and what are they?","local":"2-how-many-dimensions-does-the-tensor-output-by-the-base-transformer-model-have-and-what-are-they","sections":[],"depth":3},{"title":"3. Which of the following is an example of subword tokenization?","local":"3-which-of-the-following-is-an-example-of-subword-tokenization","sections":[],"depth":3},{"title":"4. What is a model head?","local":"4-what-is-a-model-head","sections":[],"depth":3},{"title":"5. What is an AutoModel?","local":"5-what-is-an-automodel","sections":[],"depth":3},{"title":"6. What are the techniques to be aware of when batching sequences of different lengths together?","local":"6-what-are-the-techniques-to-be-aware-of-when-batching-sequences-of-different-lengths-together","sections":[],"depth":3},{"title":"7. What is the point of applying a SoftMax function to the logits output by a sequence classification model?","local":"7-what-is-the-point-of-applying-a-softmax-function-to-the-logits-output-by-a-sequence-classification-model","sections":[],"depth":3},{"title":"8. What method is most of the tokenizer API centered around?","local":"8-what-method-is-most-of-the-tokenizer-api-centered-around","sections":[],"depth":3},{"title":"9. What does the result variable contain in this code sample?","local":"9-what-does-the-result-variable-contain-in-this-code-sample","sections":[],"depth":3},{"title":"10. Is there something wrong with the following code?","local":"10-is-there-something-wrong-with-the-following-code","sections":[],"depth":3}],"depth":1}';function He(xe,d,V){let c="pt";return ve(()=>{const S=new URLSearchParams(window.location.search);V(0,c=S.get("fw")||"pt")}),[c]}class Ge extends Ie{constructor(d){super(),Ae(this,d,He,Be,We,{})}}export{Ge as component};