course/pr_1149/te/_app/immutable/nodes/19.d75f798a.js

import{s as Le,o as Ke}from"../chunks/scheduler.cc52f4b9.js";import{S as Oe,i as el,e as r,s as n,c as p,h as ll,a as c,d as t,b as a,f as De,g as i,j as y,l as Pe,m as tl,n as s,o as M,p as m,q as u,r as o}from"../chunks/index.bd400c31.js";import{C as sl,H as Xe,E as nl}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.25fe1ea6.js";import{C as b}from"../chunks/CodeBlock.1794e33c.js";import{C as al}from"../chunks/CourseFloatingBanner.6f780cc4.js";import{F as pl}from"../chunks/FrameworkSwitchCourse.e8e44fa0.js";function il(xe){let J,P,T,L,j,O,w,ee,d,le,h,te,U,ze=`గత కొద్ది విభాగాల్లో, ఎక్కువ భాగం పనిని మన చేతులతో చేయడానికి ప్రయత్నించాం.<br/>
టోకెనైజర్లు ఎలా పని చేస్తాయో చూశాం, tokenization, input IDs మార్పు, padding, truncation, attention masks వంటి దశలను పరిశీలించాము.`,se,f,Ge=`అయితే, సెక్షన్ 2 లో చూసినట్లుగా, 🤗 Transformers APIలోని ఒక high-level ఫంక్షన్ ఈ అన్నింటినీ మన కోసం ఆటోమేటిక్‌గా చేయగలదు.<br/>
మీరు <code>tokenizer</code> ను నేరుగా వాక్యం మీద కాల్ చేస్తే, మోడల్‌కు పంపడానికి సిద్ధమైన ఇన్‌పుట్‌లు మీకు లభిస్తాయి:`,ne,I,ae,Z,Se=`ఇక్కడ <code>model_inputs</code> వేరియబుల్‌లో మోడల్ సరిగ్గా పనిచేయడానికి అవసరమైన ప్రతి అంశం ఉంటుంది. DistilBERT కోసం, ఇందులో input IDs అలాగే attention mask ఉంటాయి.<br/>
ఇతర మోడళ్లకు అదనపు ఇన్‌పుట్‌లు అవసరమైతే, tokenizer వాటినీ కూడా తయారు చేస్తుంది.`,pe,$,Ce="క్రింది ఉదాహరణల్లో చూస్తున్నట్లుగా, ఈ విధానం చాలా శక్తివంతమైనది. మొదటగా, ఇది ఒకే సీక్వెన్స్‌ను tokenize చేయగలదు:",ie,g,Me,V,Ee="అదే విధంగా, APIలో ఎలాంటి మార్పు లేకుండా బహుళ సీక్వెన్స్‌లను కూడా tokenize చేయగలదు:",me,k,ue,W,ve="ఇది padding‌ను కూడా వివిధ లక్ష్యాల ప్రకారం నిర్వహించగలదు:",oe,B,re,x,Re="ఇది సీక్వెన్స్‌లను truncate కూడా చేయగలదు:",ce,X,ye,z,He=`<code>tokenizer</code> ఆబ్జెక్ట్ ప్రత్యేక framework టెన్సర్లు (TensorFlow, PyTorch, NumPy) గా మార్పు చేయడం కూడా నిర్వహిస్తుంది.<br/>
ఉదాహరణకు, క్రింది కోడ్‌లో <code>&quot;pt&quot;</code> అంటే PyTorch టెన్సర్లు, <code>&quot;np&quot;</code> అంటే NumPy arrays:`,be,G,Je,S,Te,C,Ne="టోకెనైజర్ ఇచ్చిన input IDs ను పరిశీలిస్తే, అవి ముందుగా చూసిన IDs కంటే కొద్దిగా భిన్నంగా ఉంటాయి:",je,E,we,v,de,R,qe="ఆ input IDs ను decode చేసి చూస్తే విషయం స్పష్టమవుతుంది:",he,H,Ue,N,fe,q,Ye=`టోకెనైజర్ ప్రారంభంలో <code>[CLS]</code>, చివరలో <code>[SEP]</code> అనే ప్రత్యేక పదాలను జోడించింది.<br/>
దానికి కారణం — ఆ మోడల్ ప్రీట్రైనింగ్ సమయంలో ఈ tokens ను ఉపయోగించింది, కాబట్టి inference సమయంలో కూడా అవి అవసరం అవుతాయి.`,Ie,Y,Qe=`గమనించండి:<br/>
అన్ని మోడళ్లు ఇలాంటి ప్రత్యేక tokens ను జోడించవు.<br/>
కొన్ని మోడళ్లు వేర్వేరు special tokens ఉపయోగిస్తాయి.<br/>
కొన్నిసార్లు ప్రారంభంలో మాత్రమే, కొన్నిసార్లు చివరలో మాత్రమే జోడిస్తాయి.`,Ze,Q,Fe="ఏ మోడల్ ఏ tokens ను కోరుకుంటుందో tokenizer కు పూర్తిగా తెలుసు — దాన్ని అది మీ కోసం నిర్వహిస్తుంది.",$e,F,ge,_,_e=`ఇప్పుడు <code>tokenizer</code> టెక్స్ట్‌పై పనిచేసేటప్పుడు follow అయ్యే ప్రతి దశ గురించి తెలుసుకున్నాం.<br/>
చివరిసారి, ఇది padding (బహుళ సీక్వెన్స్‌లు!), truncation (పొడవైన సీక్వెన్స్‌లు!), మరియు framework టెన్సర్లను ఎలా నిర్వహిస్తుందో చూద్దాం:`,Ve,A,ke,D,We,K,Be;return j=new pl({props:{fw:xe[0]}}),w=new sl({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),d=new Xe({props:{title:"అన్నింటినీ కలిపి చూడడం",local:"putting-it-all-together",headingTag:"h1"}}),h=new al({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_pt.ipynb"}]}}),I=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQtZmluZXR1bmVkLXNzdC0yLWVuZ2xpc2glMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQSUwQXNlcXVlbmNlJTIwJTNEJTIwJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTBBJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`,wrap:!1}}),g=new b({props:{code:"c2VxdWVuY2UlMjAlM0QlMjAlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2Up",highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`,wrap:!1}}),k=new b({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzKQ==",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

model_inputs = tokenizer(sequences)`,wrap:!1}}),B=new b({props:{code:"JTIzJTIwV2lsbCUyMHBhZCUyMHRoZSUyMHNlcXVlbmNlcyUyMHVwJTIwdG8lMjB0aGUlMjBtYXhpbXVtJTIwc2VxdWVuY2UlMjBsZW5ndGglMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRCUyMmxvbmdlc3QlMjIpJTBBJTBBJTIzJTIwV2lsbCUyMHBhZCUyMHRoZSUyMHNlcXVlbmNlcyUyMHVwJTIwdG8lMjB0aGUlMjBtb2RlbCUyMG1heCUyMGxlbmd0aCUwQSUyMyUyMCg1MTIlMjBmb3IlMjBCRVJUJTIwb3IlMjBEaXN0aWxCRVJUKSUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEJTIybWF4X2xlbmd0aCUyMiklMEElMEElMjMlMjBXaWxsJTIwcGFkJTIwdGhlJTIwc2VxdWVuY2VzJTIwdXAlMjB0byUyMHRoZSUyMHNwZWNpZmllZCUyMG1heCUyMGxlbmd0aCUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEJTIybWF4X2xlbmd0aCUyMiUyQyUyMG1heF9sZW5ndGglM0Q4KQ==",highlighted:`<span class="hljs-comment"># Will pad the sequences up to the maximum sequence length</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;longest&quot;</span>)

<span class="hljs-comment"># Will pad the sequences up to the model max length</span>
<span class="hljs-comment"># (512 for BERT or DistilBERT)</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>)

<span class="hljs-comment"># Will pad the sequences up to the specified max length</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>, max_length=<span class="hljs-number">8</span>)`,wrap:!1}}),X=new b({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEElMjMlMjBXaWxsJTIwdHJ1bmNhdGUlMjB0aGUlMjBzZXF1ZW5jZXMlMjB0aGF0JTIwYXJlJTIwbG9uZ2VyJTIwdGhhbiUyMHRoZSUyMG1vZGVsJTIwbWF4JTIwbGVuZ3RoJTBBJTIzJTIwKDUxMiUyMGZvciUyMEJFUlQlMjBvciUyMERpc3RpbEJFUlQpJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMHRydW5jYXRpb24lM0RUcnVlKSUwQSUwQSUyMyUyMFdpbGwlMjB0cnVuY2F0ZSUyMHRoZSUyMHNlcXVlbmNlcyUyMHRoYXQlMjBhcmUlMjBsb25nZXIlMjB0aGFuJTIwdGhlJTIwc3BlY2lmaWVkJTIwbWF4JTIwbGVuZ3RoJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMG1heF9sZW5ndGglM0Q4JTJDJTIwdHJ1bmNhdGlvbiUzRFRydWUp",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># Will truncate the sequences that are longer than the model max length</span>
<span class="hljs-comment"># (512 for BERT or DistilBERT)</span>
model_inputs = tokenizer(sequences, truncation=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># Will truncate the sequences that are longer than the specified max length</span>
model_inputs = tokenizer(sequences, max_length=<span class="hljs-number">8</span>, truncation=<span class="hljs-literal">True</span>)`,wrap:!1}}),G=new b({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEElMjMlMjBSZXR1cm5zJTIwUHlUb3JjaCUyMHRlbnNvcnMlMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQSUwQSUyMyUyMFJldHVybnMlMjBOdW1QeSUyMGFycmF5cyUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIybnAlMjIp",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># Returns PyTorch tensors</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-comment"># Returns NumPy arrays</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;np&quot;</span>)`,wrap:!1}}),S=new Xe({props:{title:"ప్రత్యేక tokens",local:"special-tokens",headingTag:"h2"}}),E=new b({props:{code:"c2VxdWVuY2UlMjAlM0QlMjAlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2UpJTBBcHJpbnQobW9kZWxfaW5wdXRzJTVCJTIyaW5wdXRfaWRzJTIyJTVEKSUwQSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplci50b2tlbml6ZShzZXF1ZW5jZSklMEFpZHMlMjAlM0QlMjB0b2tlbml6ZXIuY29udmVydF90b2tlbnNfdG9faWRzKHRva2VucyklMEFwcmludChpZHMp",highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)
<span class="hljs-built_in">print</span>(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>])

tokens = tokenizer.tokenize(sequence)
ids = tokenizer.convert_tokens_to_ids(tokens)
<span class="hljs-built_in">print</span>(ids)`,wrap:!1}}),v=new b({props:{code:"JTVCMTAxJTJDJTIwMTA0NSUyQyUyMDEwMDUlMkMlMjAyMzEwJTJDJTIwMjA0MiUyQyUyMDM0MDMlMkMlMjAyMDA1JTJDJTIwMTAzNyUyQyUyMDE3NjYyJTJDJTIwMTIxNzIlMkMlMjAyNjA3JTJDJTIwMjAyNiUyQyUyMDI4NzglMkMlMjAyMTY2JTJDJTIwMTAxMiUyQyUyMDEwMiU1RCUwQSU1QjEwNDUlMkMlMjAxMDA1JTJDJTIwMjMxMCUyQyUyMDIwNDIlMkMlMjAzNDAzJTJDJTIwMjAwNSUyQyUyMDEwMzclMkMlMjAxNzY2MiUyQyUyMDEyMTcyJTJDJTIwMjYwNyUyQyUyMDIwMjYlMkMlMjAyODc4JTJDJTIwMjE2NiUyQyUyMDEwMTIlNUQ=",highlighted:`[<span class="hljs-number">101</span>, <span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>]
[<span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>]`,wrap:!1}}),H=new b({props:{code:"cHJpbnQodG9rZW5pemVyLmRlY29kZShtb2RlbF9pbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQpKSUwQXByaW50KHRva2VuaXplci5kZWNvZGUoaWRzKSk=",highlighted:`<span class="hljs-built_in">print</span>(tokenizer.decode(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>]))
<span class="hljs-built_in">print</span>(tokenizer.decode(ids))`,wrap:!1}}),N=new b({props:{code:"JTIyJTVCQ0xTJTVEJTIwaSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMGh1Z2dpbmdmYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIwJTVCU0VQJTVEJTIyJTBBJTIyaSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMGh1Z2dpbmdmYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIy",highlighted:`<span class="hljs-string">&quot;[CLS] i&#x27;ve been waiting for a huggingface course my whole life. [SEP]&quot;</span>
<span class="hljs-string">&quot;i&#x27;ve been waiting for a huggingface course my whole life.&quot;</span>`,wrap:!1}}),F=new Xe({props:{title:"ముగింపు: Tokenizer నుండి Model వరకు",local:"wrapping-up-from-tokenizer-to-model",headingTag:"h2"}}),A=new b({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24lMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQtZmluZXR1bmVkLXNzdC0yLWVuZ2xpc2glMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbi5mcm9tX3ByZXRyYWluZWQoY2hlY2twb2ludCklMEFzZXF1ZW5jZXMlMjAlM0QlMjAlNUIlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMkMlMjAlMjJTbyUyMGhhdmUlMjBJISUyMiU1RCUwQSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHRydW5jYXRpb24lM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEFvdXRwdXQlMjAlM0QlMjBtb2RlbCgqKnRva2Vucyk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
output = model(**tokens)`,wrap:!1}}),D=new nl({props:{source:"https://github.com/huggingface/course/blob/main/chapters/te/chapter2/6.mdx"}}),{c(){J=r("meta"),P=n(),T=r("p"),L=n(),p(j.$$.fragment),O=n(),p(w.$$.fragment),ee=n(),p(d.$$.fragment),le=n(),p(h.$$.fragment),te=n(),U=r("p"),U.innerHTML=ze,se=n(),f=r("p"),f.innerHTML=Ge,ne=n(),p(I.$$.fragment),ae=n(),Z=r("p"),Z.innerHTML=Se,pe=n(),$=r("p"),$.textContent=Ce,ie=n(),p(g.$$.fragment),Me=n(),V=r("p"),V.textContent=Ee,me=n(),p(k.$$.fragment),ue=n(),W=r("p"),W.textContent=ve,oe=n(),p(B.$$.fragment),re=n(),x=r("p"),x.textContent=Re,ce=n(),p(X.$$.fragment),ye=n(),z=r("p"),z.innerHTML=He,be=n(),p(G.$$.fragment),Je=n(),p(S.$$.fragment),Te=n(),C=r("p"),C.textContent=Ne,je=n(),p(E.$$.fragment),we=n(),p(v.$$.fragment),de=n(),R=r("p"),R.textContent=qe,he=n(),p(H.$$.fragment),Ue=n(),p(N.$$.fragment),fe=n(),q=r("p"),q.innerHTML=Ye,Ie=n(),Y=r("p"),Y.innerHTML=Qe,Ze=n(),Q=r("p"),Q.textContent=Fe,$e=n(),p(F.$$.fragment),ge=n(),_=r("p"),_.innerHTML=_e,Ve=n(),p(A.$$.fragment),ke=n(),p(D.$$.fragment),We=n(),K=r("p"),this.h()},l(e){const l=ll("svelte-u9bgzb",document.head);J=c(l,"META",{name:!0,content:!0}),l.forEach(t),P=a(e),T=c(e,"P",{}),De(T).forEach(t),L=a(e),i(j.$$.fragment,e),O=a(e),i(w.$$.fragment,e),ee=a(e),i(d.$$.fragment,e),le=a(e),i(h.$$.fragment,e),te=a(e),U=c(e,"P",{"data-svelte-h":!0}),y(U)!=="svelte-626tqx"&&(U.innerHTML=ze),se=a(e),f=c(e,"P",{"data-svelte-h":!0}),y(f)!=="svelte-8gv2do"&&(f.innerHTML=Ge),ne=a(e),i(I.$$.fragment,e),ae=a(e),Z=c(e,"P",{"data-svelte-h":!0}),y(Z)!=="svelte-1i4p3yj"&&(Z.innerHTML=Se),pe=a(e),$=c(e,"P",{"data-svelte-h":!0}),y($)!=="svelte-10ms3pt"&&($.textContent=Ce),ie=a(e),i(g.$$.fragment,e),Me=a(e),V=c(e,"P",{"data-svelte-h":!0}),y(V)!=="svelte-vqc9s3"&&(V.textContent=Ee),me=a(e),i(k.$$.fragment,e),ue=a(e),W=c(e,"P",{"data-svelte-h":!0}),y(W)!=="svelte-v15bp3"&&(W.textContent=ve),oe=a(e),i(B.$$.fragment,e),re=a(e),x=c(e,"P",{"data-svelte-h":!0}),y(x)!=="svelte-s3830m"&&(x.textContent=Re),ce=a(e),i(X.$$.fragment,e),ye=a(e),z=c(e,"P",{"data-svelte-h":!0}),y(z)!=="svelte-13nknzu"&&(z.innerHTML=He),be=a(e),i(G.$$.fragment,e),Je=a(e),i(S.$$.fragment,e),Te=a(e),C=c(e,"P",{"data-svelte-h":!0}),y(C)!=="svelte-snznlo"&&(C.textContent=Ne),je=a(e),i(E.$$.fragment,e),we=a(e),i(v.$$.fragment,e),de=a(e),R=c(e,"P",{"data-svelte-h":!0}),y(R)!=="svelte-qv1aux"&&(R.textContent=qe),he=a(e),i(H.$$.fragment,e),Ue=a(e),i(N.$$.fragment,e),fe=a(e),q=c(e,"P",{"data-svelte-h":!0}),y(q)!=="svelte-stb40v"&&(q.innerHTML=Ye),Ie=a(e),Y=c(e,"P",{"data-svelte-h":!0}),y(Y)!=="svelte-adb0if"&&(Y.innerHTML=Qe),Ze=a(e),Q=c(e,"P",{"data-svelte-h":!0}),y(Q)!=="svelte-otg5t6"&&(Q.textContent=Fe),$e=a(e),i(F.$$.fragment,e),ge=a(e),_=c(e,"P",{"data-svelte-h":!0}),y(_)!=="svelte-drm5bj"&&(_.innerHTML=_e),Ve=a(e),i(A.$$.fragment,e),ke=a(e),i(D.$$.fragment,e),We=a(e),K=c(e,"P",{}),De(K).forEach(t),this.h()},h(){Pe(J,"name","hf:doc:metadata"),Pe(J,"content",Ml)},m(e,l){tl(document.head,J),s(e,P,l),s(e,T,l),s(e,L,l),M(j,e,l),s(e,O,l),M(w,e,l),s(e,ee,l),M(d,e,l),s(e,le,l),M(h,e,l),s(e,te,l),s(e,U,l),s(e,se,l),s(e,f,l),s(e,ne,l),M(I,e,l),s(e,ae,l),s(e,Z,l),s(e,pe,l),s(e,$,l),s(e,ie,l),M(g,e,l),s(e,Me,l),s(e,V,l),s(e,me,l),M(k,e,l),s(e,ue,l),s(e,W,l),s(e,oe,l),M(B,e,l),s(e,re,l),s(e,x,l),s(e,ce,l),M(X,e,l),s(e,ye,l),s(e,z,l),s(e,be,l),M(G,e,l),s(e,Je,l),M(S,e,l),s(e,Te,l),s(e,C,l),s(e,je,l),M(E,e,l),s(e,we,l),M(v,e,l),s(e,de,l),s(e,R,l),s(e,he,l),M(H,e,l),s(e,Ue,l),M(N,e,l),s(e,fe,l),s(e,q,l),s(e,Ie,l),s(e,Y,l),s(e,Ze,l),s(e,Q,l),s(e,$e,l),M(F,e,l),s(e,ge,l),s(e,_,l),s(e,Ve,l),M(A,e,l),s(e,ke,l),M(D,e,l),s(e,We,l),s(e,K,l),Be=!0},p(e,[l]){const Ae={};l&1&&(Ae.fw=e[0]),j.$set(Ae)},i(e){Be||(m(j.$$.fragment,e),m(w.$$.fragment,e),m(d.$$.fragment,e),m(h.$$.fragment,e),m(I.$$.fragment,e),m(g.$$.fragment,e),m(k.$$.fragment,e),m(B.$$.fragment,e),m(X.$$.fragment,e),m(G.$$.fragment,e),m(S.$$.fragment,e),m(E.$$.fragment,e),m(v.$$.fragment,e),m(H.$$.fragment,e),m(N.$$.fragment,e),m(F.$$.fragment,e),m(A.$$.fragment,e),m(D.$$.fragment,e),Be=!0)},o(e){u(j.$$.fragment,e),u(w.$$.fragment,e),u(d.$$.fragment,e),u(h.$$.fragment,e),u(I.$$.fragment,e),u(g.$$.fragment,e),u(k.$$.fragment,e),u(B.$$.fragment,e),u(X.$$.fragment,e),u(G.$$.fragment,e),u(S.$$.fragment,e),u(E.$$.fragment,e),u(v.$$.fragment,e),u(H.$$.fragment,e),u(N.$$.fragment,e),u(F.$$.fragment,e),u(A.$$.fragment,e),u(D.$$.fragment,e),Be=!1},d(e){e&&(t(P),t(T),t(L),t(O),t(ee),t(le),t(te),t(U),t(se),t(f),t(ne),t(ae),t(Z),t(pe),t($),t(ie),t(Me),t(V),t(me),t(ue),t(W),t(oe),t(re),t(x),t(ce),t(ye),t(z),t(be),t(Je),t(Te),t(C),t(je),t(we),t(de),t(R),t(he),t(Ue),t(fe),t(q),t(Ie),t(Y),t(Ze),t(Q),t($e),t(ge),t(_),t(Ve),t(ke),t(We),t(K)),t(J),o(j,e),o(w,e),o(d,e),o(h,e),o(I,e),o(g,e),o(k,e),o(B,e),o(X,e),o(G,e),o(S,e),o(E,e),o(v,e),o(H,e),o(N,e),o(F,e),o(A,e),o(D,e)}}}const Ml='{"title":"అన్నింటినీ కలిపి చూడడం","local":"putting-it-all-together","sections":[{"title":"ప్రత్యేక tokens","local":"special-tokens","sections":[],"depth":2},{"title":"ముగింపు: Tokenizer నుండి Model వరకు","local":"wrapping-up-from-tokenizer-to-model","sections":[],"depth":2}],"depth":1}';function ml(xe,J,P){let T="pt";return Ke(()=>{const L=new URLSearchParams(window.location.search);P(0,T=L.get("fw")||"pt")}),[T]}class Jl extends Oe{constructor(J){super(),el(this,J,ml,il,Le,{})}}export{Jl as component};