Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Kekuatan Khusus Tokenizer Cepat","local":"fast-tokenizers-special-powers","sections":[{"title":"Enkode dalam Batch","local":"batch-encoding","sections":[],"depth":2},{"title":"Di Balik Pipeline token-classification","local":"inside-the-token-classification-pipeline","sections":[{"title":"Mendapatkan Hasil Dasar dari Pipeline","local":"getting-the-base-results-with-the-pipeline","sections":[],"depth":3},{"title":"Dari Input ke Prediksi","local":"from-inputs-to-predictions","sections":[],"depth":3},{"title":"Mengelompokkan Entitas","local":"grouping-entities","sections":[],"depth":3}],"depth":2}],"depth":1}"> | |
| <link href="/docs/course/pr_1054/id/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload"> | |
| <link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/entry/start.4f92af03.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/scheduler.36a0863c.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/singletons.7dc7b9a4.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/index.733708bb.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/paths.cf097d06.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/entry/app.19cef1b6.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/index.156fee99.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/nodes/0.1203e4a0.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/each.e59479a4.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/nodes/47.23e09f42.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/Tip.8a648467.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/Youtube.a5d6d567.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/CodeBlock.4cf998e6.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/CourseFloatingBanner.16bb8bff.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/FrameworkSwitchCourse.d8719b67.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/getInferenceSnippets.472bc46d.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Kekuatan Khusus Tokenizer Cepat","local":"fast-tokenizers-special-powers","sections":[{"title":"Enkode dalam Batch","local":"batch-encoding","sections":[],"depth":2},{"title":"Di Balik Pipeline token-classification","local":"inside-the-token-classification-pipeline","sections":[{"title":"Mendapatkan Hasil Dasar dari Pipeline","local":"getting-the-base-results-with-the-pipeline","sections":[],"depth":3},{"title":"Dari Input ke Prediksi","local":"from-inputs-to-predictions","sections":[],"depth":3},{"title":"Mengelompokkan Entitas","local":"grouping-entities","sections":[],"depth":3}],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="bg-white leading-none border border-gray-100 rounded-lg flex p-0.5 w-56 text-sm mb-4"><a class="flex justify-center flex-1 py-1.5 px-2.5 focus:outline-none !no-underline rounded-l bg-red-50 dark:bg-transparent text-red-600" href="?fw=pt"><svg class="mr-1.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><defs><clipPath id="a"><rect x="3.05" y="0.5" width="25.73" height="31" fill="none"></rect></clipPath></defs><g clip-path="url(#a)"><path d="M24.94,9.51a12.81,12.81,0,0,1,0,18.16,12.68,12.68,0,0,1-18,0,12.81,12.81,0,0,1,0-18.16l9-9V5l-.84.83-6,6a9.58,9.58,0,1,0,13.55,0ZM20.44,9a1.68,1.68,0,1,1,1.67-1.67A1.68,1.68,0,0,1,20.44,9Z" fill="#ee4c2c"></path></g></svg> Pytorch </a><a class="flex justify-center flex-1 py-1.5 px-2.5 focus:outline-none !no-underline rounded-r text-gray-500 filter grayscale" href="?fw=tf"><svg class="mr-1.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="0.94em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 274"><path d="M145.726 42.065v42.07l72.861 42.07v-42.07l-72.86-42.07zM0 84.135v42.07l36.43 21.03V105.17L0 84.135zm109.291 21.035l-36.43 21.034v126.2l36.43 21.035v-84.135l36.435 21.035v-42.07l-36.435-21.034V105.17z" fill="#E55B2D"></path><path d="M145.726 42.065L36.43 105.17v42.065l72.861-42.065v42.065l36.435-21.03v-84.14zM255.022 63.1l-36.435 21.035v42.07l36.435-21.035V63.1zm-72.865 84.135l-36.43 21.035v42.07l36.43-21.036v-42.07zm-36.43 63.104l-36.436-21.035v84.135l36.435-21.035V210.34z" fill="#ED8E24"></path><path d="M145.726 0L0 84.135l36.43 21.035l109.296-63.105l72.861 42.07L255.022 63.1L145.726 0zm0 126.204l-36.435 21.03l36.435 21.036l36.43-21.035l-36.43-21.03z" fill="#F8BF3C"></path></svg> TensorFlow </a></div> <h1 class="relative group"><a id="fast-tokenizers-special-powers" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#fast-tokenizers-special-powers"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Kekuatan Khusus Tokenizer Cepat</span></h1> <div class="flex space-x-1 absolute z-10 right-0 top-0"><a href="https://discuss.huggingface.co/t/chapter-6-questions" target="_blank"><img alt="Ask a Question" class="!m-0" src="https://img.shields.io/badge/Ask%20a%20question-ffcb4c.svg?logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgLTEgMTA0IDEwNiI+PGRlZnM+PHN0eWxlPi5jbHMtMXtmaWxsOiMyMzFmMjA7fS5jbHMtMntmaWxsOiNmZmY5YWU7fS5jbHMtM3tmaWxsOiMwMGFlZWY7fS5jbHMtNHtmaWxsOiMwMGE5NGY7fS5jbHMtNXtmaWxsOiNmMTVkMjI7fS5jbHMtNntmaWxsOiNlMzFiMjM7fTwvc3R5bGU+PC9kZWZzPjx0aXRsZT5EaXNjb3Vyc2VfbG9nbzwvdGl0bGU+PGcgaWQ9IkxheWVyXzIiPjxnIGlkPSJMYXllcl8zIj48cGF0aCBjbGFzcz0iY2xzLTEiIGQ9Ik01MS44NywwQzIzLjcxLDAsMCwyMi44MywwLDUxYzAsLjkxLDAsNTIuODEsMCw1Mi44MWw1MS44Ni0uMDVjMjguMTYsMCw1MS0yMy43MSw1MS01MS44N1M4MCwwLDUxLjg3LDBaIi8+PHBhdGggY2xhc3M9ImNscy0yIiBkPSJNNTIuMzcsMTkuNzRBMzEuNjIsMzEuNjIsMCwwLDAsMjQuNTgsNjYuNDFsLTUuNzIsMTguNEwzOS40LDgwLjE3YTMxLjYxLDMxLjYxLDAsMSwwLDEzLTYwLjQzWiIvPjxwYXRoIGNsYXNzPSJjbHMtMyIgZD0iTTc3LjQ1LDMyLjEyYTMxLjYsMzEuNiwwLDAsMS0zOC4wNSw0OEwxOC44Niw4NC44MmwyMC45MS0yLjQ3QTMxLjYsMzEuNiwwLDAsMCw3Ny40NSwzMi4xMloiLz48cGF0aCBjbGFzcz0iY2xzLTQiIGQ9Ik03MS42MywyNi4yOUEzMS42LDMxLjYsMCwwLDEsMzguOCw3OEwxOC44Niw4NC44MiwzOS40LDgwLjE3QTMxLjYsMzEuNiwwLDAsMCw3MS42MywyNi4yOVoiLz48cGF0aCBjbGFzcz0iY2xzLTUiIGQ9Ik0yNi40Nyw2Ny4xMWEzMS42MSwzMS42MSwwLDAsMSw1MS0zNUEzMS42MSwzMS42MSwwLDAsMCwyNC41OCw2Ni40MWwtNS43MiwxOC40WiIvPjxwYXRoIGNsYXNzPSJjbHMtNiIgZD0iTTI0LjU4LDY2LjQxQTMxLjYxLDMxLjYxLDAsMCwxLDcxLjYzLDI2LjI5YTMxLjYxLDMxLjYxLDAsMCwwLTQ5LDM5LjYzbC0zLjc2LDE4LjlaIi8+PC9nPjwvZz48L3N2Zz4="></a> <a href="https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter6/section3_pt.ipynb" target="_blank"><img alt="Open In Colab" class="!m-0" src="https://colab.research.google.com/assets/colab-badge.svg"></a> <a href="https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter6/section3_pt.ipynb" target="_blank"><img alt="Open In Studio Lab" class="!m-0" src="https://studiolab.sagemaker.aws/studiolab.svg"></a></div> <p data-svelte-h="svelte-1hhwwqy">Di bagian ini, kita akan melihat lebih dekat kemampuan tokenizer di 🤗 Transformers. Sampai sekarang kita hanya menggunakannya untuk men-tokenisasi input atau mendekode ID kembali menjadi teks, tetapi tokenizer — terutama yang didukung oleh pustaka 🤗 Tokenizers — bisa melakukan jauh lebih banyak. Untuk mengilustrasikan fitur tambahan ini, kita akan menjelajahi cara mereplikasi hasil dari pipeline <code>token-classification</code> (yang kita sebut <code>ner</code>) dan <code>question-answering</code> yang pertama kali kita temui di <a href="/course/chapter1">Bab 1</a>.</p> <iframe class="w-full xl:w-4/6 h-80" src="https://www.youtube-nocookie.com/embed/g8quOxoqhHQ" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> <p data-svelte-h="svelte-arhrvd">Dalam pembahasan berikut, kita akan sering membedakan antara tokenizer “lambat” dan “cepat”. Tokenizer lambat adalah tokenizer yang ditulis dalam Python di dalam pustaka 🤗 Transformers, sementara versi cepat disediakan oleh 🤗 Tokenizers, yang ditulis dalam bahasa Rust. Jika Anda masih ingat tabel dari <a href="/course/chapter5/3">Bab 5</a> yang menunjukkan berapa lama waktu yang dibutuhkan oleh tokenizer cepat dan lambat untuk melakukan tokenisasi pada Drug Review Dataset, Anda pasti sudah punya gambaran mengapa kita menyebutnya cepat dan lambat:</p> <table data-svelte-h="svelte-52p298"><thead><tr><th align="center"></th> <th align="center">Tokenizer Cepat</th> <th align="center">Tokenizer Lambat</th></tr></thead> <tbody><tr><td align="center"><code>batched=True</code></td> <td align="center">10.8s</td> <td align="center">4m41s</td></tr> <tr><td align="center"><code>batched=False</code></td> <td align="center">59.2s</td> <td align="center">5m3s</td></tr></tbody></table> <div class="course-tip course-tip-orange bg-gradient-to-br dark:bg-gradient-to-r before:border-orange-500 dark:before:border-orange-800 from-orange-50 dark:from-gray-900 to-white dark:to-gray-950 border border-orange-50 text-orange-700 dark:text-gray-400"><p data-svelte-h="svelte-1jjtjm5">⚠️ Saat men-tokenisasi satu kalimat, Anda mungkin tidak melihat perbedaan kecepatan antara tokenizer lambat dan cepat. Bahkan, tokenizer cepat bisa lebih lambat! Perbedaannya baru terlihat jelas saat Anda men-tokenisasi banyak teks sekaligus.</p></div> <h2 class="relative group"><a id="batch-encoding" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#batch-encoding"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Enkode dalam Batch</span></h2> <iframe class="w-full xl:w-4/6 h-80" src="https://www.youtube-nocookie.com/embed/3umI3tm27Vw" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> <p data-svelte-h="svelte-qltl4q">Output dari tokenizer bukan hanya dictionary Python biasa; yang kita dapatkan sebenarnya adalah objek khusus <code>BatchEncoding</code>. Ini adalah turunan dari dictionary (itulah mengapa kita bisa mengaksesnya seperti dictionary), tetapi memiliki metode tambahan yang umumnya digunakan oleh tokenizer cepat.</p> <p data-svelte-h="svelte-2oan4y">Selain kemampuannya untuk melakukan paralelisasi, fungsi utama dari tokenizer cepat adalah mereka <strong>selalu melacak rentang asli teks</strong> dari mana token akhir berasal — fitur ini disebut <em>offset mapping</em>. Ini memungkinkan kita melakukan hal-hal seperti memetakan setiap kata ke token yang dihasilkannya, atau memetakan setiap karakter dalam teks asli ke token tempat karakter itu berada, dan sebaliknya.</p> <p data-svelte-h="svelte-1d1gbfd">Mari kita lihat contoh:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer | |
| tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">"bert-base-cased"</span>) | |
| example = <span class="hljs-string">"My name is Sylvain and I work at Hugging Face in Brooklyn."</span> | |
| encoding = tokenizer(example) | |
| <span class="hljs-built_in">print</span>(<span class="hljs-built_in">type</span>(encoding))<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-x0br4t">Seperti disebutkan sebelumnya, kita mendapatkan objek <code>BatchEncoding</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><<span class="hljs-keyword">class</span> <span class="hljs-string">'transformers.tokenization_utils_base.BatchEncoding'</span>><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-4o2aab">Karena <code>AutoTokenizer</code> secara default memilih tokenizer cepat, kita bisa menggunakan metode tambahan dari objek ini. Ada dua cara untuk mengecek apakah tokenizer kita adalah versi cepat atau lambat: kita bisa mengecek atribut <code>is_fast</code> dari <code>tokenizer</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->tokenizer.is_fast<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-literal">True</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1ja6cma">atau dari objek <code>encoding</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->encoding.is_fast<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-literal">True</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1j7oy88">Mari lihat apa yang memungkinkan dilakukan oleh tokenizer cepat. Pertama, kita bisa mengakses token tanpa perlu mengonversi ID kembali ke token:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->encoding.tokens()<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->[<span class="hljs-string">'[CLS]'</span>, <span class="hljs-string">'My'</span>, <span class="hljs-string">'name'</span>, <span class="hljs-string">'is'</span>, <span class="hljs-string">'S'</span>, <span class="hljs-string">'##yl'</span>, <span class="hljs-string">'##va'</span>, <span class="hljs-string">'##in'</span>, <span class="hljs-string">'and'</span>, <span class="hljs-string">'I'</span>, <span class="hljs-string">'work'</span>, <span class="hljs-string">'at'</span>, <span class="hljs-string">'Hu'</span>, <span class="hljs-string">'##gging'</span>, <span class="hljs-string">'Face'</span>, <span class="hljs-string">'in'</span>, | |
| <span class="hljs-string">'Brooklyn'</span>, <span class="hljs-string">'.'</span>, <span class="hljs-string">'[SEP]'</span>]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-8tqoq2">Token dengan indeks ke-5 adalah <code>##yl</code>, bagian dari kata “Sylvain” dalam kalimat asli. Kita juga bisa menggunakan metode <code>word_ids()</code> untuk mendapatkan indeks kata asal dari setiap token:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->encoding.word_ids()<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->[<span class="hljs-literal">None</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">3</span>, <span class="hljs-number">3</span>, <span class="hljs-number">3</span>, <span class="hljs-number">4</span>, <span class="hljs-number">5</span>, <span class="hljs-number">6</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">8</span>, <span class="hljs-number">9</span>, <span class="hljs-number">10</span>, <span class="hljs-number">11</span>, <span class="hljs-number">12</span>, <span class="hljs-literal">None</span>]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-haslrs">Kita bisa melihat bahwa token khusus milik tokenizer seperti <code>[CLS]</code> dan <code>[SEP]</code> dipetakan ke <code>None</code>, dan kemudian setiap token dipetakan ke kata asalnya. Ini sangat berguna untuk menentukan apakah suatu token berada di awal sebuah kata atau apakah dua token berasal dari kata yang sama. Kita memang bisa mengandalkan awalan <code>##</code> untuk itu, tetapi cara ini hanya berfungsi pada tokenizer seperti BERT; metode ini bisa digunakan pada jenis tokenizer apa pun selama merupakan tokenizer cepat. Pada bab selanjutnya, kita akan melihat bagaimana kita bisa menggunakan kemampuan ini untuk menerapkan label yang kita miliki untuk setiap kata secara tepat ke token-token dalam tugas seperti named entity recognition (NER) dan part-of-speech (POS) tagging. Kita juga bisa menggunakannya untuk memask semua token yang berasal dari kata yang sama dalam masked language modeling (teknik yang disebut <em>whole word masking</em>).</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-1ftyolh">Konsep “kata” bisa membingungkan. Misalnya, apakah “I’ll” (gabungan dari “I will”) dihitung sebagai satu atau dua kata? Ini tergantung tokenizer dan proses <em>pre-tokenization</em> yang digunakan. Beberapa tokenizer hanya memisahkan berdasarkan spasi, sehingga akan menganggap ini sebagai satu kata. Lainnya menggunakan tanda baca juga, jadi akan memisahkannya menjadi dua kata.</p> <p data-svelte-h="svelte-gsfp1">✏️ <strong>Coba sendiri!</strong> Buat tokenizer dari checkpoint <code>bert-base-cased</code> dan <code>roberta-base</code>, lalu tokenisasi kata ”81s”. Apa yang Anda amati? Bagaimana hasil dari <code>word_ids()</code>?</p></div> <p data-svelte-h="svelte-1ljdwuu">Ada juga metode <code>sentence_ids()</code> untuk memetakan token ke kalimat asalnya (meskipun atribut <code>token_type_ids</code> juga menyediakan informasi yang serupa).</p> <p data-svelte-h="svelte-i6pys8">Akhirnya, kita dapat memetakan token atau kata ke karakter dalam teks asli dan sebaliknya menggunakan metode <code>word_to_chars()</code>, <code>token_to_chars()</code>, <code>char_to_word()</code>, atau <code>char_to_token()</code>. Misalnya, <code>word_ids()</code> menunjukkan bahwa <code>##yl</code> adalah bagian dari kata ke-3. Tapi kata apa itu di kalimat?</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->start, end = encoding.word_to_chars(<span class="hljs-number">3</span>) | |
| example[start:end]<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->Sylvain<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1byj7zn">Seperti disebutkan, semua ini dimungkinkan karena tokenizer cepat menyimpan informasi rentang teks asal setiap token dalam daftar <em>offset</em>. Selanjutnya, kita akan tunjukkan bagaimana mereplikasi hasil pipeline <code>token-classification</code> secara manual.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-kutff1">✏️ <strong>Coba sendiri!</strong> Buat teks contoh Anda sendiri, dan coba pahami token apa yang dikaitkan dengan ID kata tertentu. Juga, coba ekstrak rentang karakter untuk sebuah kata. Bonus: coba gunakan dua kalimat dan lihat apakah ID kalimatnya masuk akal.</p></div> <h2 class="relative group"><a id="inside-the-token-classification-pipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#inside-the-token-classification-pipeline"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Di Balik Pipeline token-classification</span></h2> <p data-svelte-h="svelte-9yd9fy">Dalam <a href="/course/chapter1">Bab 1</a>, kita telah mencoba menerapkan NER untuk pertama kalinya — yaitu tugas untuk mengidentifikasi bagian-bagian teks yang merupakan entitas seperti orang, lokasi, atau organisasi — menggunakan fungsi <code>pipeline()</code> dari 🤗 Transformers. Kemudian, di <a href="/course/chapter2">Bab 2</a>, kita melihat bahwa sebuah pipeline menggabungkan tiga tahap yang diperlukan untuk mendapatkan prediksi dari teks mentah: tokenisasi, menjalankan input melalui model, dan pasca-pemrosesan. Dua langkah pertama dalam pipeline <code>token-classification</code> sama seperti pipeline lainnya, tetapi proses pasca-pemrosesannya sedikit lebih kompleks — mari kita lihat bagaimana caranya!</p> <iframe class="w-full xl:w-4/6 h-80" src="https://www.youtube-nocookie.com/embed/0E7ltQB7fM8" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> <h3 class="relative group"><a id="getting-the-base-results-with-the-pipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#getting-the-base-results-with-the-pipeline"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Mendapatkan Hasil Dasar dari Pipeline</span></h3> <p data-svelte-h="svelte-b59y6f">Pertama, mari ambil pipeline <code>token-classification</code> agar kita bisa mendapatkan hasil untuk dibandingkan secara manual. Model default yang digunakan adalah <a href="https://huggingface.co/dbmdz/bert-large-cased-finetuned-conll03-english" rel="nofollow"><code>dbmdz/bert-large-cased-finetuned-conll03-english</code></a>, yang melakukan NER pada kalimat:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline | |
| token_classifier = pipeline(<span class="hljs-string">"token-classification"</span>) | |
| token_classifier(<span class="hljs-string">"My name is Sylvain and I work at Hugging Face in Brooklyn."</span>)<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->[{<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-PER'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.9993828</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">4</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'S'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">11</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">12</span>}, | |
| {<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-PER'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.99815476</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">5</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'##yl'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">12</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">14</span>}, | |
| {<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-PER'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.99590725</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">6</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'##va'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">14</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">16</span>}, | |
| {<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-PER'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.9992327</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">7</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'##in'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">16</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">18</span>}, | |
| {<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-ORG'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.97389334</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">12</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'Hu'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">33</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">35</span>}, | |
| {<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-ORG'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.976115</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">13</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'##gging'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">35</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">40</span>}, | |
| {<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-ORG'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.98879766</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">14</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'Face'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">41</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">45</span>}, | |
| {<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-LOC'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">16</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'Brooklyn'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">49</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">57</span>}]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1x60ra8">Model berhasil mengidentifikasi setiap token dari kata “Sylvain” sebagai entitas orang, “Hugging Face” sebagai organisasi, dan “Brooklyn” sebagai lokasi. Kita juga bisa meminta pipeline untuk mengelompokkan token-token yang berasal dari entitas yang sama:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline | |
| token_classifier = pipeline(<span class="hljs-string">"token-classification"</span>, aggregation_strategy=<span class="hljs-string">"simple"</span>) | |
| token_classifier(<span class="hljs-string">"My name is Sylvain and I work at Hugging Face in Brooklyn."</span>)<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->[{<span class="hljs-string">'entity_group'</span>: <span class="hljs-string">'PER'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.9981694</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'Sylvain'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">11</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">18</span>}, | |
| {<span class="hljs-string">'entity_group'</span>: <span class="hljs-string">'ORG'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.97960204</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'Hugging Face'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">33</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">45</span>}, | |
| {<span class="hljs-string">'entity_group'</span>: <span class="hljs-string">'LOC'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'Brooklyn'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">49</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">57</span>}]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-o9i7py">Strategi <code>aggregation_strategy</code> yang dipilih akan memengaruhi skor yang dihitung untuk setiap entitas yang dikelompokkan. Dengan <code>"simple"</code>, skornya hanyalah rata-rata dari skor setiap token dalam entitas tersebut: misalnya, skor untuk “Sylvain” adalah rata-rata dari skor yang kita lihat pada contoh sebelumnya untuk token <code>S</code>, <code>##yl</code>, <code>##va</code>, dan <code>##in</code>. Strategi lain yang tersedia adalah:</p> <ul data-svelte-h="svelte-n0codw"><li><code>"first"</code>: skor entitas diambil dari skor token pertama (untuk “Sylvain” = 0.993828)</li> <li><code>"max"</code>: skor entitas adalah skor token tertinggi (untuk “Hugging Face” = 0.98879766 dari “Face”)</li> <li><code>"average"</code>: rata-rata skor dari kata yang membentuk entitas (untuk “Hugging Face”, rata-rata skor dari “Hugging” dan “Face”)</li></ul> <p data-svelte-h="svelte-50h0cv">Sekarang mari kita lihat bagaimana cara mendapatkan hasil ini <strong>tanpa menggunakan fungsi <code>pipeline()</code></strong>!</p> <h3 class="relative group"><a id="from-inputs-to-predictions" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#from-inputs-to-predictions"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Dari Input ke Prediksi</span></h3> <p data-svelte-h="svelte-14mixun">Pertama, kita perlu men-tokenisasi input kita dan melewatkannya ke dalam model. Ini dilakukan persis seperti di <a href="/course/chapter2">Bab 2</a>; kita menginstansiasi tokenizer dan model menggunakan kelas <code>AutoXxx</code>, lalu menggunakannya pada contoh:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForTokenClassification | |
| model_checkpoint = <span class="hljs-string">"dbmdz/bert-large-cased-finetuned-conll03-english"</span> | |
| tokenizer = AutoTokenizer.from_pretrained(model_checkpoint) | |
| model = AutoModelForTokenClassification.from_pretrained(model_checkpoint) | |
| example = <span class="hljs-string">"My name is Sylvain and I work at Hugging Face in Brooklyn."</span> | |
| inputs = tokenizer(example, return_tensors=<span class="hljs-string">"pt"</span>) | |
| outputs = model(**inputs)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-y4ur3d">Karena kita menggunakan <code>AutoModelForTokenClassification</code>, kita mendapatkan satu set <em>logits</em> untuk setiap token dalam urutan input:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-built_in">print</span>(inputs[<span class="hljs-string">"input_ids"</span>].shape) | |
| <span class="hljs-built_in">print</span>(outputs.logits.shape)<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">19</span>]) | |
| torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">19</span>, <span class="hljs-number">9</span>])<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-4yhtr7">Kita memiliki satu <em>batch</em> yang terdiri dari 1 urutan berisi 19 token, dan model memiliki 9 label berbeda, sehingga output model memiliki bentuk (shape) 1 × 19 × 9. Seperti pada pipeline klasifikasi teks, kita menggunakan fungsi <em>softmax</em> untuk mengubah logits menjadi probabilitas, lalu menggunakan <em>argmax</em> untuk mendapatkan prediksi (karena urutan tidak berubah oleh <em>softmax</em>, kita bisa langsung mengambil argmax dari logits).</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> torch | |
| probabilities = torch.nn.functional.softmax(outputs.logits, dim=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>].tolist() | |
| predictions = outputs.logits.argmax(dim=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>].tolist() | |
| <span class="hljs-built_in">print</span>(predictions)<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">4</span>, <span class="hljs-number">4</span>, <span class="hljs-number">4</span>, <span class="hljs-number">4</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">6</span>, <span class="hljs-number">6</span>, <span class="hljs-number">6</span>, <span class="hljs-number">0</span>, <span class="hljs-number">8</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-z4vpxd">Atribut <code>model.config.id2label</code> berisi pemetaan dari indeks ke label, yang bisa kita gunakan untuk memahami hasil prediksi:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->model.config.id2label<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->{<span class="hljs-number">0</span>: <span class="hljs-string">'O'</span>, | |
| <span class="hljs-number">1</span>: <span class="hljs-string">'B-MISC'</span>, | |
| <span class="hljs-number">2</span>: <span class="hljs-string">'I-MISC'</span>, | |
| <span class="hljs-number">3</span>: <span class="hljs-string">'B-PER'</span>, | |
| <span class="hljs-number">4</span>: <span class="hljs-string">'I-PER'</span>, | |
| <span class="hljs-number">5</span>: <span class="hljs-string">'B-ORG'</span>, | |
| <span class="hljs-number">6</span>: <span class="hljs-string">'I-ORG'</span>, | |
| <span class="hljs-number">7</span>: <span class="hljs-string">'B-LOC'</span>, | |
| <span class="hljs-number">8</span>: <span class="hljs-string">'I-LOC'</span>}<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-lbssed">Seperti yang telah kita lihat sebelumnya, terdapat 9 label: <code>O</code> adalah label untuk token-token yang tidak termasuk dalam entitas apa pun (singkatan dari “outside”), dan kemudian kita memiliki dua label untuk setiap jenis entitas (miscellaneous, person, organization, dan location). Label <code>B-XXX</code> menunjukkan bahwa token berada di awal entitas <code>XXX</code>, sedangkan label <code>I-XXX</code> menunjukkan bahwa token berada di dalam entitas <code>XXX</code>. Sebagai contoh, dalam contoh saat ini kita mengharapkan model mengklasifikasikan token <code>S</code> sebagai <code>B-PER</code> (awal dari entitas person) dan token <code>##yl</code>, <code>##va</code>, serta <code>##in</code> sebagai <code>I-PER</code> (di dalam entitas person).</p> <p data-svelte-h="svelte-tmoa5">Mungkin terlihat bahwa model salah karena memberikan <code>I-PER</code> ke semua empat token ini, tapi itu sebenarnya tidak sepenuhnya salah. Ada dua format untuk label <code>B-</code> dan <code>I-</code>: yaitu <em>IOB1</em> dan <em>IOB2</em>. Format IOB2 (ditunjukkan dengan warna merah muda di bawah) adalah yang kita perkenalkan, sedangkan format IOB1 (biru) menggunakan label <code>B-</code> hanya untuk memisahkan dua entitas yang berdampingan dari tipe yang sama. Model yang kita gunakan telah disetel ulang menggunakan dataset dengan format IOB1, itulah sebabnya token <code>S</code> diberi label <code>I-PER</code>.</p> <div class="flex justify-center" data-svelte-h="svelte-3i2zqr"><img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/IOB_versions.svg" alt="Format IOB1 vs IOB2"> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/IOB_versions-dark.svg" alt="Format IOB1 vs IOB2"></div> <p data-svelte-h="svelte-g0s27w">Dengan peta ini, kita siap untuk mereplikasi (hampir seluruhnya) hasil dari pipeline pertama — kita hanya perlu mengambil skor dan label dari setiap token yang tidak diklasifikasikan sebagai <code>O</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->results = [] | |
| tokens = inputs.tokens() | |
| <span class="hljs-keyword">for</span> idx, pred <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(predictions): | |
| label = model.config.id2label[pred] | |
| <span class="hljs-keyword">if</span> label != <span class="hljs-string">"O"</span>: | |
| results.append( | |
| {<span class="hljs-string">"entity"</span>: label, <span class="hljs-string">"score"</span>: probabilities[idx][pred], <span class="hljs-string">"word"</span>: tokens[idx]} | |
| ) | |
| <span class="hljs-built_in">print</span>(results)<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->[{<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-PER'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.9993828</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">4</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'S'</span>}, | |
| {<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-PER'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.99815476</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">5</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'##yl'</span>}, | |
| {<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-PER'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.99590725</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">6</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'##va'</span>}, | |
| {<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-PER'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.9992327</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">7</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'##in'</span>}, | |
| {<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-ORG'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.97389334</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">12</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'Hu'</span>}, | |
| {<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-ORG'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.976115</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">13</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'##gging'</span>}, | |
| {<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-ORG'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.98879766</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">14</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'Face'</span>}, | |
| {<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-LOC'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">16</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'Brooklyn'</span>}]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-ladciu">Ini sangat mirip dengan hasil yang kita dapatkan sebelumnya, dengan satu pengecualian: pipeline juga memberikan informasi tentang posisi <code>start</code> dan <code>end</code> dari setiap entitas dalam kalimat asli. Di sinilah offset mapping akan digunakan. Untuk mendapatkan offset, kita cukup mengatur <code>return_offsets_mapping=True</code> saat menggunakan tokenizer:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->inputs_with_offsets = tokenizer(example, return_offsets_mapping=<span class="hljs-literal">True</span>) | |
| inputs_with_offsets[<span class="hljs-string">"offset_mapping"</span>]<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->[(<span class="hljs-number">0</span>, <span class="hljs-number">0</span>), (<span class="hljs-number">0</span>, <span class="hljs-number">2</span>), (<span class="hljs-number">3</span>, <span class="hljs-number">7</span>), (<span class="hljs-number">8</span>, <span class="hljs-number">10</span>), (<span class="hljs-number">11</span>, <span class="hljs-number">12</span>), (<span class="hljs-number">12</span>, <span class="hljs-number">14</span>), (<span class="hljs-number">14</span>, <span class="hljs-number">16</span>), (<span class="hljs-number">16</span>, <span class="hljs-number">18</span>), (<span class="hljs-number">19</span>, <span class="hljs-number">22</span>), (<span class="hljs-number">23</span>, <span class="hljs-number">24</span>), (<span class="hljs-number">25</span>, <span class="hljs-number">29</span>), (<span class="hljs-number">30</span>, <span class="hljs-number">32</span>), | |
| (<span class="hljs-number">33</span>, <span class="hljs-number">35</span>), (<span class="hljs-number">35</span>, <span class="hljs-number">40</span>), (<span class="hljs-number">41</span>, <span class="hljs-number">45</span>), (<span class="hljs-number">46</span>, <span class="hljs-number">48</span>), (<span class="hljs-number">49</span>, <span class="hljs-number">57</span>), (<span class="hljs-number">57</span>, <span class="hljs-number">58</span>), (<span class="hljs-number">0</span>, <span class="hljs-number">0</span>)]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-9ybuvm">Setiap pasangan tuple menunjukkan rentang teks yang sesuai untuk setiap token, di mana <code>(0, 0)</code> dicadangkan untuk token khusus. Kita telah melihat bahwa token pada indeks ke-5 adalah <code>##yl</code>, dan offset-nya adalah <code>(12, 14)</code>. Jika kita ambil potongan teks dari contoh:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->example[<span class="hljs-number">12</span>:<span class="hljs-number">14</span>]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-n4omb5">kita mendapatkan rentang teks yang tepat tanpa <code>##</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->yl<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1h3i7qa">Dengan ini, kita bisa menyempurnakan hasil sebelumnya:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->results = [] | |
| inputs_with_offsets = tokenizer(example, return_offsets_mapping=<span class="hljs-literal">True</span>) | |
| tokens = inputs_with_offsets.tokens() | |
| offsets = inputs_with_offsets[<span class="hljs-string">"offset_mapping"</span>] | |
| <span class="hljs-keyword">for</span> idx, pred <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(predictions): | |
| label = model.config.id2label[pred] | |
| <span class="hljs-keyword">if</span> label != <span class="hljs-string">"O"</span>: | |
| start, end = offsets[idx] | |
| results.append( | |
| { | |
| <span class="hljs-string">"entity"</span>: label, | |
| <span class="hljs-string">"score"</span>: probabilities[idx][pred], | |
| <span class="hljs-string">"word"</span>: tokens[idx], | |
| <span class="hljs-string">"start"</span>: start, | |
| <span class="hljs-string">"end"</span>: end, | |
| } | |
| ) | |
| <span class="hljs-built_in">print</span>(results)<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->[{<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-PER'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.9993828</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">4</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'S'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">11</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">12</span>}, | |
| {<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-PER'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.99815476</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">5</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'##yl'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">12</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">14</span>}, | |
| {<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-PER'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.99590725</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">6</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'##va'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">14</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">16</span>}, | |
| {<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-PER'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.9992327</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">7</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'##in'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">16</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">18</span>}, | |
| {<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-ORG'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.97389334</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">12</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'Hu'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">33</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">35</span>}, | |
| {<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-ORG'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.976115</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">13</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'##gging'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">35</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">40</span>}, | |
| {<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-ORG'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.98879766</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">14</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'Face'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">41</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">45</span>}, | |
| {<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-LOC'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">'index'</span>: <span class="hljs-number">16</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'Brooklyn'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">49</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">57</span>}]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-3ooswk">Ini adalah hasil yang <strong>sama persis</strong> dengan pipeline pertama!</p> <h3 class="relative group"><a id="grouping-entities" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#grouping-entities"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Mengelompokkan Entitas</span></h3> <p data-svelte-h="svelte-1u43ki3">Menggunakan offset untuk menentukan posisi awal dan akhir setiap entitas memang berguna, tetapi informasi tersebut sebenarnya tidak sepenuhnya diperlukan. Namun, saat kita ingin mengelompokkan entitas, offset akan sangat membantu dan menghindarkan kita dari kode yang rumit. Misalnya, jika kita ingin menggabungkan token <code>Hu</code>, <code>##gging</code>, dan <code>Face</code>, kita bisa membuat aturan khusus yang menyatakan bahwa dua token pertama harus digabung dengan menghapus <code>##</code>, dan <code>Face</code> ditambahkan dengan spasi karena tidak diawali dengan <code>##</code> — tetapi aturan ini hanya akan bekerja untuk jenis tokenizer tertentu. Kita harus menulis seperangkat aturan lain untuk tokenizer seperti SentencePiece atau Byte-Pair Encoding (yang akan dibahas lebih lanjut di bab ini).</p> <p data-svelte-h="svelte-1gj3qro">Dengan menggunakan offset, semua kode khusus tersebut tidak lagi diperlukan: kita cukup mengambil rentang dalam teks asli yang dimulai dari token pertama dan berakhir pada token terakhir. Jadi, dalam kasus token <code>Hu</code>, <code>##gging</code>, dan <code>Face</code>, kita harus mulai dari karakter ke-33 (awal dari <code>Hu</code>) dan berakhir sebelum karakter ke-45 (akhir dari <code>Face</code>):</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->example[<span class="hljs-number">33</span>:<span class="hljs-number">45</span>]<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->Hugging Face<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-7e9s3v">Untuk menulis kode yang melakukan pasca-pemrosesan terhadap prediksi sambil mengelompokkan entitas, kita akan mengelompokkan entitas yang berurutan dan dilabeli dengan <code>I-XXX</code>, kecuali entitas pertama yang bisa saja dilabeli sebagai <code>B-XXX</code> atau <code>I-XXX</code> (jadi, kita berhenti mengelompokkan suatu entitas ketika kita mendapatkan label <code>O</code>, jenis entitas yang baru, atau <code>B-XXX</code> yang menunjukkan bahwa entitas baru dari tipe yang sama telah dimulai):</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np | |
| results = [] | |
| inputs_with_offsets = tokenizer(example, return_offsets_mapping=<span class="hljs-literal">True</span>) | |
| tokens = inputs_with_offsets.tokens() | |
| offsets = inputs_with_offsets[<span class="hljs-string">"offset_mapping"</span>] | |
| idx = <span class="hljs-number">0</span> | |
| <span class="hljs-keyword">while</span> idx < <span class="hljs-built_in">len</span>(predictions): | |
| pred = predictions[idx] | |
| label = model.config.id2label[pred] | |
| <span class="hljs-keyword">if</span> label != <span class="hljs-string">"O"</span>: | |
| <span class="hljs-comment"># Hapus awalan B- atau I-</span> | |
| label = label[<span class="hljs-number">2</span>:] | |
| start, _ = offsets[idx] | |
| <span class="hljs-comment"># Ambil semua token yang diberi label I</span> | |
| all_scores = [] | |
| <span class="hljs-keyword">while</span> ( | |
| idx < <span class="hljs-built_in">len</span>(predictions) | |
| <span class="hljs-keyword">and</span> model.config.id2label[predictions[idx]] == <span class="hljs-string">f"I-<span class="hljs-subst">{label}</span>"</span> | |
| ): | |
| all_scores.append(probabilities[idx][pred]) | |
| _, end = offsets[idx] | |
| idx += <span class="hljs-number">1</span> | |
| <span class="hljs-comment"># Skor adalah rata-rata skor dari semua token dalam entitas tersebut</span> | |
| score = np.mean(all_scores).item() | |
| word = example[start:end] | |
| results.append( | |
| { | |
| <span class="hljs-string">"entity_group"</span>: label, | |
| <span class="hljs-string">"score"</span>: score, | |
| <span class="hljs-string">"word"</span>: word, | |
| <span class="hljs-string">"start"</span>: start, | |
| <span class="hljs-string">"end"</span>: end, | |
| } | |
| ) | |
| idx += <span class="hljs-number">1</span> | |
| <span class="hljs-built_in">print</span>(results)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-wrxr7l">Dan hasilnya akan sama seperti pipeline kedua:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->[{<span class="hljs-string">'entity_group'</span>: <span class="hljs-string">'PER'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.9981694</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'Sylvain'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">11</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">18</span>}, | |
| {<span class="hljs-string">'entity_group'</span>: <span class="hljs-string">'ORG'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.97960204</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'Hugging Face'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">33</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">45</span>}, | |
| {<span class="hljs-string">'entity_group'</span>: <span class="hljs-string">'LOC'</span>, <span class="hljs-string">'score'</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">'word'</span>: <span class="hljs-string">'Brooklyn'</span>, <span class="hljs-string">'start'</span>: <span class="hljs-number">49</span>, <span class="hljs-string">'end'</span>: <span class="hljs-number">57</span>}]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-f2jwt5">Contoh lain dari tugas yang sangat terbantu dengan offset adalah <strong>question answering</strong>. Pembahasan pipeline tersebut (yang akan kita lakukan di bagian selanjutnya) juga akan membawa kita pada satu fitur terakhir dari tokenizer di 🤗 Transformers: menangani <strong>token yang meluap (overflow)</strong> saat input terlalu panjang dan perlu dipotong (truncate).</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/course/blob/main/chapters/id/chapter6/3.mdx" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p> | |
| <script> | |
| { | |
| __sveltekit_ojy514 = { | |
| assets: "/docs/course/pr_1054/id", | |
| base: "/docs/course/pr_1054/id", | |
| env: {} | |
| }; | |
| const element = document.currentScript.parentElement; | |
| const data = [null,null]; | |
| Promise.all([ | |
| import("/docs/course/pr_1054/id/_app/immutable/entry/start.4f92af03.js"), | |
| import("/docs/course/pr_1054/id/_app/immutable/entry/app.19cef1b6.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 47], | |
| data, | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 103 kB
- Xet hash:
- 9ee9cfbf54322c28286e4b076ac8f7cc5e78c621428e35c8029a802d0f1ec226
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.