course/pr_1107/my/chapter2/9.html

<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;အခန်းပြီးဆုံးခြင်း စစ်ဆေးမှု&quot;,&quot;local&quot;:&quot;end-of-chapter-quiz&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;1. Language modeling pipeline ၏ အစီအစဉ်က ဘာလဲ။&quot;,&quot;local&quot;:&quot;1-language-modeling-pipeline--အစအစဉက-ဘလ&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;2. Base Transformer model က ထုတ်ပေးတဲ့ tensor မှာ dimension ဘယ်နှစ်ခုရှိပြီး၊ ဘာတွေလဲ။&quot;,&quot;local&quot;:&quot;2-base-transformer-model-က-ထတပတ-tensor-မ-dimension-ဘယနစခရပ-ဘတလ&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;3. အောက်ပါတို့ထဲမှ မည်သည့်အရာက subword tokenization ဥပမာတစ်ခုလဲ။&quot;,&quot;local&quot;:&quot;3-အကပတထမ-မညသညအရက-subword-tokenization-ဥပမတစခလ&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;4. Model head ဆိုတာ ဘာလဲ။&quot;,&quot;local&quot;:&quot;4-model-head-ဆတ-ဘလ&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;5. AutoModel ဆိုတာ ဘာလဲ။&quot;,&quot;local&quot;:&quot;5-automodel-ဆတ-ဘလ&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;6. အရှည်မတူညီသော sequences များကို အတူတကွ batch လုပ်သည့်အခါ မည်သည့်နည်းလမ်းများကို သိရှိထားသင့်သလဲ။&quot;,&quot;local&quot;:&quot;6-အရညမတညသ-sequences-မက-အတတက-batch-လပသညအခ-မညသညနညလမမက-သရထသငသလ&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;7. sequence classification model က ထုတ်ပေးတဲ့ logits တွေပေါ်မှာ SoftMax function ကို အသုံးပြုရခြင်းရဲ့ ရည်ရွယ်ချက်က ဘာလဲ။&quot;,&quot;local&quot;:&quot;7-sequence-classification-model-က-ထတပတ-logits-တပမ-softmax-function-က-အသပရခငရ-ရညရယခကက-ဘလ&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;8. tokenizer API ရဲ့ အများစုက ဘယ် method ပေါ်မှာ အခြေခံထားလဲ။&quot;,&quot;local&quot;:&quot;8-tokenizer-api-ရ-အမစက-ဘယ-method-ပမ-အခခထလ&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;9. ဒီ code sample မှာ result variable က ဘာတွေ ပါဝင်သလဲ။&quot;,&quot;local&quot;:&quot;9-ဒ-code-sample-မ-result-variable-က-ဘတ-ပဝငသလ&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;10. အောက်ပါ code မှာ တစ်ခုခု မှားနေတာ ရှိပါသလား။&quot;,&quot;local&quot;:&quot;10-အကပ-code-မ-တစခခ-မနတ-ရပသလ&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:1}">
		<link href="/docs/course/pr_1107/my/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
		<link rel="modulepreload" href="/docs/course/pr_1107/my/_app/immutable/entry/start.5c6233a8.js">
		<link rel="modulepreload" href="/docs/course/pr_1107/my/_app/immutable/chunks/scheduler.0835143d.js">
		<link rel="modulepreload" href="/docs/course/pr_1107/my/_app/immutable/chunks/singletons.c8b11329.js">
		<link rel="modulepreload" href="/docs/course/pr_1107/my/_app/immutable/chunks/index.1bab75e2.js">
		<link rel="modulepreload" href="/docs/course/pr_1107/my/_app/immutable/chunks/paths.e4a366ea.js">
		<link rel="modulepreload" href="/docs/course/pr_1107/my/_app/immutable/entry/app.55586789.js">
		<link rel="modulepreload" href="/docs/course/pr_1107/my/_app/immutable/chunks/preload-helper.5f7c8393.js">
		<link rel="modulepreload" href="/docs/course/pr_1107/my/_app/immutable/chunks/index.3d7efe79.js">
		<link rel="modulepreload" href="/docs/course/pr_1107/my/_app/immutable/nodes/0.0cec3d6c.js">
		<link rel="modulepreload" href="/docs/course/pr_1107/my/_app/immutable/chunks/each.e59479a4.js">
		<link rel="modulepreload" href="/docs/course/pr_1107/my/_app/immutable/nodes/22.9a2de22b.js">
		<link rel="modulepreload" href="/docs/course/pr_1107/my/_app/immutable/chunks/CodeBlock.116ed840.js">
		<link rel="modulepreload" href="/docs/course/pr_1107/my/_app/immutable/chunks/CourseFloatingBanner.860ea6e4.js">
		<link rel="modulepreload" href="/docs/course/pr_1107/my/_app/immutable/chunks/Question.a2f852b0.js">
		<link rel="modulepreload" href="/docs/course/pr_1107/my/_app/immutable/chunks/stores.6af6d5ae.js">
		<link rel="modulepreload" href="/docs/course/pr_1107/my/_app/immutable/chunks/FrameworkSwitchCourse.ff2bd9ab.js">
		<link rel="modulepreload" href="/docs/course/pr_1107/my/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.0b02b772.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;အခန်းပြီးဆုံးခြင်း စစ်ဆေးမှု&quot;,&quot;local&quot;:&quot;end-of-chapter-quiz&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;1. Language modeling pipeline ၏ အစီအစဉ်က ဘာလဲ။&quot;,&quot;local&quot;:&quot;1-language-modeling-pipeline--အစအစဉက-ဘလ&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;2. Base Transformer model က ထုတ်ပေးတဲ့ tensor မှာ dimension ဘယ်နှစ်ခုရှိပြီး၊ ဘာတွေလဲ။&quot;,&quot;local&quot;:&quot;2-base-transformer-model-က-ထတပတ-tensor-မ-dimension-ဘယနစခရပ-ဘတလ&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;3. အောက်ပါတို့ထဲမှ မည်သည့်အရာက subword tokenization ဥပမာတစ်ခုလဲ။&quot;,&quot;local&quot;:&quot;3-အကပတထမ-မညသညအရက-subword-tokenization-ဥပမတစခလ&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;4. Model head ဆိုတာ ဘာလဲ။&quot;,&quot;local&quot;:&quot;4-model-head-ဆတ-ဘလ&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;5. AutoModel ဆိုတာ ဘာလဲ။&quot;,&quot;local&quot;:&quot;5-automodel-ဆတ-ဘလ&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;6. အရှည်မတူညီသော sequences များကို အတူတကွ batch လုပ်သည့်အခါ မည်သည့်နည်းလမ်းများကို သိရှိထားသင့်သလဲ။&quot;,&quot;local&quot;:&quot;6-အရညမတညသ-sequences-မက-အတတက-batch-လပသညအခ-မညသညနညလမမက-သရထသငသလ&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;7. sequence classification model က ထုတ်ပေးတဲ့ logits တွေပေါ်မှာ SoftMax function ကို အသုံးပြုရခြင်းရဲ့ ရည်ရွယ်ချက်က ဘာလဲ။&quot;,&quot;local&quot;:&quot;7-sequence-classification-model-က-ထတပတ-logits-တပမ-softmax-function-က-အသပရခငရ-ရညရယခကက-ဘလ&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;8. tokenizer API ရဲ့ အများစုက ဘယ် method ပေါ်မှာ အခြေခံထားလဲ။&quot;,&quot;local&quot;:&quot;8-tokenizer-api-ရ-အမစက-ဘယ-method-ပမ-အခခထလ&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;9. ဒီ code sample မှာ result variable က ဘာတွေ ပါဝင်သလဲ။&quot;,&quot;local&quot;:&quot;9-ဒ-code-sample-မ-result-variable-က-ဘတ-ပဝငသလ&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;10. အောက်ပါ code မှာ တစ်ခုခု မှားနေတာ ရှိပါသလား။&quot;,&quot;local&quot;:&quot;10-အကပ-code-မ-တစခခ-မနတ-ရပသလ&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:1}"><!-- HEAD_svelte-u9bgzb_END -->      <p></p> <div class="bg-white leading-none border border-gray-100 rounded-lg flex p-0.5 w-56 text-sm mb-4"><a class="flex justify-center flex-1 py-1.5 px-2.5 focus:outline-none !no-underline rounded-l bg-red-50 dark:bg-transparent text-red-600" href="?fw=pt"><svg class="mr-1.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><defs><clipPath id="a"><rect x="3.05" y="0.5" width="25.73" height="31" fill="none"></rect></clipPath></defs><g clip-path="url(#a)"><path d="M24.94,9.51a12.81,12.81,0,0,1,0,18.16,12.68,12.68,0,0,1-18,0,12.81,12.81,0,0,1,0-18.16l9-9V5l-.84.83-6,6a9.58,9.58,0,1,0,13.55,0ZM20.44,9a1.68,1.68,0,1,1,1.67-1.67A1.68,1.68,0,0,1,20.44,9Z" fill="#ee4c2c"></path></g></svg> Pytorch </a><a class="flex justify-center flex-1 py-1.5 px-2.5 focus:outline-none !no-underline rounded-r text-gray-500 filter grayscale" href="?fw=tf"><svg class="mr-1.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="0.94em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 274"><path d="M145.726 42.065v42.07l72.861 42.07v-42.07l-72.86-42.07zM0 84.135v42.07l36.43 21.03V105.17L0 84.135zm109.291 21.035l-36.43 21.034v126.2l36.43 21.035v-84.135l36.435 21.035v-42.07l-36.435-21.034V105.17z" fill="#E55B2D"></path><path d="M145.726 42.065L36.43 105.17v42.065l72.861-42.065v42.065l36.435-21.03v-84.14zM255.022 63.1l-36.435 21.035v42.07l36.435-21.035V63.1zm-72.865 84.135l-36.43 21.035v42.07l36.43-21.036v-42.07zm-36.43 63.104l-36.436-21.035v84.135l36.435-21.035V210.34z" fill="#ED8E24"></path><path d="M145.726 0L0 84.135l36.43 21.035l109.296-63.105l72.861 42.07L255.022 63.1L145.726 0zm0 126.204l-36.435 21.03l36.435 21.036l36.43-21.035l-36.43-21.03z" fill="#F8BF3C"></path></svg> TensorFlow </a></div>   <h1 class="relative group"><a id="end-of-chapter-quiz" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#end-of-chapter-quiz"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>အခန်းပြီးဆုံးခြင်း စစ်ဆေးမှု</span></h1>  <div class="flex space-x-1 absolute z-10 right-0 top-0"><a href="https://discuss.huggingface.co/t/chapter-2-questions" target="_blank"><img alt="Ask a Question" class="!m-0" src="https://img.shields.io/badge/Ask%20a%20question-ffcb4c.svg?logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgLTEgMTA0IDEwNiI+PGRlZnM+PHN0eWxlPi5jbHMtMXtmaWxsOiMyMzFmMjA7fS5jbHMtMntmaWxsOiNmZmY5YWU7fS5jbHMtM3tmaWxsOiMwMGFlZWY7fS5jbHMtNHtmaWxsOiMwMGE5NGY7fS5jbHMtNXtmaWxsOiNmMTVkMjI7fS5jbHMtNntmaWxsOiNlMzFiMjM7fTwvc3R5bGU+PC9kZWZzPjx0aXRsZT5EaXNjb3Vyc2VfbG9nbzwvdGl0bGU+PGcgaWQ9IkxheWVyXzIiPjxnIGlkPSJMYXllcl8zIj48cGF0aCBjbGFzcz0iY2xzLTEiIGQ9Ik01MS44NywwQzIzLjcxLDAsMCwyMi44MywwLDUxYzAsLjkxLDAsNTIuODEsMCw1Mi44MWw1MS44Ni0uMDVjMjguMTYsMCw1MS0yMy43MSw1MS01MS44N1M4MCwwLDUxLjg3LDBaIi8+PHBhdGggY2xhc3M9ImNscy0yIiBkPSJNNTIuMzcsMTkuNzRBMzEuNjIsMzEuNjIsMCwwLDAsMjQuNTgsNjYuNDFsLTUuNzIsMTguNEwzOS40LDgwLjE3YTMxLjYxLDMxLjYxLDAsMSwwLDEzLTYwLjQzWiIvPjxwYXRoIGNsYXNzPSJjbHMtMyIgZD0iTTc3LjQ1LDMyLjEyYTMxLjYsMzEuNiwwLDAsMS0zOC4wNSw0OEwxOC44Niw4NC44MmwyMC45MS0yLjQ3QTMxLjYsMzEuNiwwLDAsMCw3Ny40NSwzMi4xMloiLz48cGF0aCBjbGFzcz0iY2xzLTQiIGQ9Ik03MS42MywyNi4yOUEzMS42LDMxLjYsMCwwLDEsMzguOCw3OEwxOC44Niw4NC44MiwzOS40LDgwLjE3QTMxLjYsMzEuNiwwLDAsMCw3MS42MywyNi4yOVoiLz48cGF0aCBjbGFzcz0iY2xzLTUiIGQ9Ik0yNi40Nyw2Ny4xMWEzMS42MSwzMS42MSwwLDAsMSw1MS0zNUEzMS42MSwzMS42MSwwLDAsMCwyNC41OCw2Ni40MWwtNS43MiwxOC40WiIvPjxwYXRoIGNsYXNzPSJjbHMtNiIgZD0iTTI0LjU4LDY2LjQxQTMxLjYxLDMxLjYxLDAsMCwxLDcxLjYzLDI2LjI5YTMxLjYxLDMxLjYxLDAsMCwwLTQ5LDM5LjYzbC0zLjc2LDE4LjlaIi8+PC9nPjwvZz48L3N2Zz4="></a>  </div>  <h3 class="relative group"><a id="1-language-modeling-pipeline--အစအစဉက-ဘလ" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#1-language-modeling-pipeline--အစအစဉက-ဘလ"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>1. Language modeling pipeline ၏ အစီအစဉ်က ဘာလဲ။</span></h3> <div><form><label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="0"> <!-- HTML_TAG_START -->ပထမဆုံး၊ text ကို ကိုင်တွယ်ပြီး raw predictions တွေကို ပြန်ပေးတဲ့ model ဖြစ်ပါတယ်။ ထို့နောက် tokenizer က ဒီ predictions တွေကို နားလည်ပြီး လိုအပ်တဲ့အခါ text အဖြစ် ပြန်ပြောင်းပေးပါတယ်။<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="1"> <!-- HTML_TAG_START -->ပထမဆုံး၊ text ကို ကိုင်တွယ်ပြီး IDs တွေကို ပြန်ပေးတဲ့ tokenizer ဖြစ်ပါတယ်။ Model က ဒီ IDs တွေကို ကိုင်တွယ်ပြီး text ဖြစ်နိုင်တဲ့ prediction တစ်ခုကို ထုတ်ပေးပါတယ်။<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="2"> <!-- HTML_TAG_START -->Tokenizer က text ကို ကိုင်တွယ်ပြီး IDs တွေကို ပြန်ပေးပါတယ်။ Model က ဒီ IDs တွေကို ကိုင်တွယ်ပြီး prediction တစ်ခုကို ထုတ်ပေးပါတယ်။ ထို့နောက် tokenizer ကို ဒီ predictions တွေကို text အဖြစ် ပြန်ပြောင်းဖို့အတွက် တစ်ဖန် ထပ်မံအသုံးပြုနိုင်ပါတယ်။<!-- HTML_TAG_END --></label>  <div class="flex flex-row items-center mt-3"><button class="btn px-4 mr-4" type="submit" disabled>Submit</button> </div></form></div>  <h3 class="relative group"><a id="2-base-transformer-model-က-ထတပတ-tensor-မ-dimension-ဘယနစခရပ-ဘတလ" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#2-base-transformer-model-က-ထတပတ-tensor-မ-dimension-ဘယနစခရပ-ဘတလ"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>2. Base Transformer model က ထုတ်ပေးတဲ့ tensor မှာ dimension ဘယ်နှစ်ခုရှိပြီး၊ ဘာတွေလဲ။</span></h3> <div><form><label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="0"> <!-- HTML_TAG_START -->2 ခု: Sequence length နဲ့ batch size<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="1"> <!-- HTML_TAG_START -->2 ခု: Sequence length နဲ့ hidden size<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="2"> <!-- HTML_TAG_START -->3 ခု: Sequence length, batch size နဲ့ hidden size<!-- HTML_TAG_END --></label>  <div class="flex flex-row items-center mt-3"><button class="btn px-4 mr-4" type="submit" disabled>Submit</button> </div></form></div>  <h3 class="relative group"><a id="3-အကပတထမ-မညသညအရက-subword-tokenization-ဥပမတစခလ" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#3-အကပတထမ-မညသညအရက-subword-tokenization-ဥပမတစခလ"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>3. အောက်ပါတို့ထဲမှ မည်သည့်အရာက subword tokenization ဥပမာတစ်ခုလဲ။</span></h3> <div><form><label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="0"> <!-- HTML_TAG_START -->WordPiece<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="1"> <!-- HTML_TAG_START -->Character-based tokenization<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="2"> <!-- HTML_TAG_START -->Whitespace နဲ့ punctuation တွေနဲ့ ပိုင်းခြားခြင်း<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="3"> <!-- HTML_TAG_START -->BPE<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="4"> <!-- HTML_TAG_START -->Unigram<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="5"> <!-- HTML_TAG_START -->အထက်ပါအဖြေများမှ တစ်ခုမှ မဟုတ်ပါ။<!-- HTML_TAG_END --></label>  <div class="flex flex-row items-center mt-3"><button class="btn px-4 mr-4" type="submit" disabled>Submit</button> </div></form></div>  <h3 class="relative group"><a id="4-model-head-ဆတ-ဘလ" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#4-model-head-ဆတ-ဘလ"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>4. Model head ဆိုတာ ဘာလဲ။</span></h3> <div><form><label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="0"> <!-- HTML_TAG_START -->Base Transformer network ရဲ့ အစိတ်အပိုင်းတစ်ခုဖြစ်ပြီး tensors တွေကို ၎င်းတို့ရဲ့ မှန်ကန်တဲ့ layers တွေဆီ ပြန်လည်လမ်းကြောင်းပြောင်းပေးပါတယ်။<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="1"> <!-- HTML_TAG_START -->Self-attention mechanism လို့လည်း လူသိများပြီး၊ ၎င်းသည် sequence ၏ အခြား tokens များနှင့်အညီ token တစ်ခု၏ ကိုယ်စားပြုမှုကို လိုက်လျောညီထွေဖြစ်အောင် ပြုလုပ်ပေးပါတယ်။<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="2"> <!-- HTML_TAG_START -->Transformer predictions တွေကို task-specific output တစ်ခုအဖြစ် ပြောင်းလဲဖို့အတွက် ပုံမှန်အားဖြင့် layers တစ်ခု သို့မဟုတ် အနည်းငယ်နဲ့ ဖွဲ့စည်းထားတဲ့ အပိုအစိတ်အပိုင်းတစ်ခု။<!-- HTML_TAG_END --></label>  <div class="flex flex-row items-center mt-3"><button class="btn px-4 mr-4" type="submit" disabled>Submit</button> </div></form></div>  <h3 class="relative group"><a id="5-automodel-ဆတ-ဘလ" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#5-automodel-ဆတ-ဘလ"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>5. AutoModel ဆိုတာ ဘာလဲ။</span></h3> <div><form><label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="0"> <!-- HTML_TAG_START -->သင်၏ data ပေါ်တွင် အလိုအလျောက် လေ့ကျင့်ပေးသော model တစ်ခု။<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="1"> <!-- HTML_TAG_START -->Checkpoint ကို အခြေခံပြီး မှန်ကန်တဲ့ architecture ကို ပြန်ပေးတဲ့ object တစ်ခု။<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="2"> <!-- HTML_TAG_START -->၎င်း၏ inputs များအတွက် အသုံးပြုသော ဘာသာစကားကို အလိုအလျောက် ထောက်လှမ်းပြီး မှန်ကန်သော weights များကို load လုပ်ပေးသော model တစ်ခု။<!-- HTML_TAG_END --></label>  <div class="flex flex-row items-center mt-3"><button class="btn px-4 mr-4" type="submit" disabled>Submit</button> </div></form></div>  <h3 class="relative group"><a id="6-အရညမတညသ-sequences-မက-အတတက-batch-လပသညအခ-မညသညနညလမမက-သရထသငသလ" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#6-အရညမတညသ-sequences-မက-အတတက-batch-လပသညအခ-မညသညနညလမမက-သရထသငသလ"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>6. အရှည်မတူညီသော sequences များကို အတူတကွ batch လုပ်သည့်အခါ မည်သည့်နည်းလမ်းများကို သိရှိထားသင့်သလဲ။</span></h3> <div><form><label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="0"> <!-- HTML_TAG_START -->Truncating<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="1"> <!-- HTML_TAG_START -->Returning tensors<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="2"> <!-- HTML_TAG_START -->Padding<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="3"> <!-- HTML_TAG_START -->Attention masking<!-- HTML_TAG_END --></label>  <div class="flex flex-row items-center mt-3"><button class="btn px-4 mr-4" type="submit" disabled>Submit</button> </div></form></div>  <h3 class="relative group"><a id="7-sequence-classification-model-က-ထတပတ-logits-တပမ-softmax-function-က-အသပရခငရ-ရညရယခကက-ဘလ" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#7-sequence-classification-model-က-ထတပတ-logits-တပမ-softmax-function-က-အသပရခငရ-ရညရယခကက-ဘလ"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>7. sequence classification model က ထုတ်ပေးတဲ့ logits တွေပေါ်မှာ SoftMax function ကို အသုံးပြုရခြင်းရဲ့ ရည်ရွယ်ချက်က ဘာလဲ။</span></h3> <div><form><label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="0"> <!-- HTML_TAG_START -->Logits တွေကို ပိုမိုယုံကြည်စိတ်ချရအောင် ပြုလုပ်ပေးပါတယ်။<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="1"> <!-- HTML_TAG_START -->၎င်းတို့ နားလည်နိုင်အောင် အနိမ့်ဆုံးနှင့် အမြင့်ဆုံးကန့်သတ်ချက်ကို သတ်မှတ်ပေးပါတယ်။<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="2"> <!-- HTML_TAG_START -->output ရဲ့ စုစုပေါင်းတန်ဖိုးက 1 ဖြစ်လာပြီး ဖြစ်နိုင်ခြေဆိုင်ရာ အဓိပ္ပာယ်ဖွင့်ဆိုနိုင်ခြေ ရှိလာပါတယ်။<!-- HTML_TAG_END --></label>  <div class="flex flex-row items-center mt-3"><button class="btn px-4 mr-4" type="submit" disabled>Submit</button> </div></form></div>  <h3 class="relative group"><a id="8-tokenizer-api-ရ-အမစက-ဘယ-method-ပမ-အခခထလ" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#8-tokenizer-api-ရ-အမစက-ဘယ-method-ပမ-အခခထလ"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>8. tokenizer API ရဲ့ အများစုက ဘယ် method ပေါ်မှာ အခြေခံထားလဲ။</span></h3> <div><form><label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="0"> <!-- HTML_TAG_START --><code>encode</code>၊ text ကို IDs အဖြစ် encode လုပ်နိုင်ပြီး IDs တွေကို predictions အဖြစ် encode လုပ်နိုင်လို့ပါ။<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="1"> <!-- HTML_TAG_START -->tokenizer object ကို တိုက်ရိုက်ခေါ်ခြင်း။<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="2"> <!-- HTML_TAG_START --><code>pad</code><!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="3"> <!-- HTML_TAG_START --><code>tokenize</code><!-- HTML_TAG_END --></label>  <div class="flex flex-row items-center mt-3"><button class="btn px-4 mr-4" type="submit" disabled>Submit</button> </div></form></div>  <h3 class="relative group"><a id="9-ဒ-code-sample-မ-result-variable-က-ဘတ-ပဝငသလ" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#9-ဒ-code-sample-မ-result-variable-က-ဘတ-ပဝငသလ"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>9. ဒီ code sample မှာ result variable က ဘာတွေ ပါဝင်သလဲ။</span></h3> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)
result = tokenizer.tokenize(<span class="hljs-string">&quot;Hello!&quot;</span>)<!-- HTML_TAG_END --></pre></div> <div><form><label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="0"> <!-- HTML_TAG_START -->Strings များ၏ list တစ်ခု၊ string တစ်ခုစီသည် token တစ်ခုဖြစ်သည်။<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="1"> <!-- HTML_TAG_START -->IDs များ၏ list တစ်ခု။<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="2"> <!-- HTML_TAG_START -->Tokens များအားလုံး ပါဝင်သော string တစ်ခု။<!-- HTML_TAG_END --></label>  <div class="flex flex-row items-center mt-3"><button class="btn px-4 mr-4" type="submit" disabled>Submit</button> </div></form></div>  <h3 class="relative group"><a id="10-အကပ-code-မ-တစခခ-မနတ-ရပသလ" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#10-အကပ-code-မ-တစခခ-မနတ-ရပသလ"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>10. အောက်ပါ code မှာ တစ်ခုခု မှားနေတာ ရှိပါသလား။</span></h3> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModel

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)
model = AutoModel.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)

encoded = tokenizer(<span class="hljs-string">&quot;Hey!&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
result = model(**encoded)<!-- HTML_TAG_END --></pre></div> <div><form><label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="0"> <!-- HTML_TAG_START -->မရှိပါဘူး၊ မှန်ကန်ပုံရပါတယ်။<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="1"> <!-- HTML_TAG_START -->Tokenizer နဲ့ model ဟာ အမြဲတမ်း checkpoint တူတူကနေ ဖြစ်သင့်ပါတယ်။<!-- HTML_TAG_END --></label> <label class="block"><input autocomplete="off" class="form-input -mt-1.5 mr-2" name="choice" type="checkbox" value="2"> <!-- HTML_TAG_START -->Input တိုင်းဟာ batch ဖြစ်တာကြောင့် tokenizer နဲ့ pad လုပ်ခြင်းနဲ့ truncate လုပ်ခြင်းက ကောင်းတဲ့ အလေ့အကျင့်ပါ။<!-- HTML_TAG_END --></label>  <div class="flex flex-row items-center mt-3"><button class="btn px-4 mr-4" type="submit" disabled>Submit</button> </div></form></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/course/blob/main/chapters/my/chapter2/9.mdx" target="_blank"><span data-svelte-h="svelte-1kd6by1">&lt;</span> <span data-svelte-h="svelte-x0xyl0">&gt;</span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a>  <p></p> 
			
			<script>
				{
					__sveltekit_dep9rk = {
						assets: "/docs/course/pr_1107/my",
						base: "/docs/course/pr_1107/my",
						env: {}
					};

					const element = document.currentScript.parentElement;

					const data = [null,null];

					Promise.all([
						import("/docs/course/pr_1107/my/_app/immutable/entry/start.5c6233a8.js"),
						import("/docs/course/pr_1107/my/_app/immutable/entry/app.55586789.js")
					]).then(([kit, app]) => {
						kit.start(app, element, {
							node_ids: [0, 22],
							data,
							form: null,
							error: null
						});
					});
				}
			</script>