Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"การสร้าง tokenizer ทีละขั้นตอน","local":"การสราง-tokenizer-ทละขนตอน","sections":[{"title":"การโหลด corpus","local":"การโหลด-corpus","sections":[],"depth":2},{"title":"สร้าง WordPiece tokenizer ตั้งแต่เริ่มต้น","local":"สราง-wordpiece-tokenizer-ตงแตเรมตน","sections":[],"depth":2},{"title":"การสร้าง BPE tokenizer ตั้งแต่เริ่มต้น","local":"การสราง-bpe-tokenizer-ตงแตเรมตน","sections":[],"depth":2},{"title":"การสร้าง Unigram tokenizer ตั้งแต่เริ่มต้น","local":"การสราง-unigram-tokenizer-ตงแตเรมตน","sections":[],"depth":2}],"depth":1}"/> | |
| <link href="/docs/course/pr_1339/th/_app/immutable/entry/start.B-F4PHHy.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1339/th/_app/immutable/chunks/9kyLaLrF.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1339/th/_app/immutable/chunks/DClKtzkC.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1339/th/_app/immutable/entry/app.EwjAFzMw.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1339/th/_app/immutable/chunks/qDsaV5xM.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1339/th/_app/immutable/chunks/DAcRH6O-.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1339/th/_app/immutable/chunks/D_4dMjXp.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1339/th/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1339/th/_app/immutable/nodes/0.oElJE2wH.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1339/th/_app/immutable/chunks/DO68AkFq.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1339/th/_app/immutable/nodes/43.D0Da3Ney.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1339/th/_app/immutable/chunks/CY2X-9Af.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1339/th/_app/immutable/chunks/D-AR32sm.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1339/th/_app/immutable/chunks/DAT0v8Wh.js" rel="modulepreload"> | |
| <!--meji3z--><meta name="hf:doc:metadata" content="{"title":"การสร้าง tokenizer ทีละขั้นตอน","local":"การสราง-tokenizer-ทละขนตอน","sections":[{"title":"การโหลด corpus","local":"การโหลด-corpus","sections":[],"depth":2},{"title":"สร้าง WordPiece tokenizer ตั้งแต่เริ่มต้น","local":"สราง-wordpiece-tokenizer-ตงแตเรมตน","sections":[],"depth":2},{"title":"การสร้าง BPE tokenizer ตั้งแต่เริ่มต้น","local":"การสราง-bpe-tokenizer-ตงแตเรมตน","sections":[],"depth":2},{"title":"การสร้าง Unigram tokenizer ตั้งแต่เริ่มต้น","local":"การสราง-unigram-tokenizer-ตงแตเรมตน","sections":[],"depth":2}],"depth":1}"/><!----> | |
| <link href="/docs/course/pr_1339/th/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="การสราง-tokenizer-ทละขนตอน" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#การสราง-tokenizer-ทละขนตอน"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>การสร้าง tokenizer ทีละขั้นตอน</span></h1><!--]--><!----> <div class="flex space-x-1 absolute z-10 right-0 top-0"><a href="https://discuss.huggingface.co/t/chapter-6-questions" target="_blank"><img alt="Ask a Question" class="!m-0" src="https://img.shields.io/badge/Ask%20a%20question-ffcb4c.svg?logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgLTEgMTA0IDEwNiI+PGRlZnM+PHN0eWxlPi5jbHMtMXtmaWxsOiMyMzFmMjA7fS5jbHMtMntmaWxsOiNmZmY5YWU7fS5jbHMtM3tmaWxsOiMwMGFlZWY7fS5jbHMtNHtmaWxsOiMwMGE5NGY7fS5jbHMtNXtmaWxsOiNmMTVkMjI7fS5jbHMtNntmaWxsOiNlMzFiMjM7fTwvc3R5bGU+PC9kZWZzPjx0aXRsZT5EaXNjb3Vyc2VfbG9nbzwvdGl0bGU+PGcgaWQ9IkxheWVyXzIiPjxnIGlkPSJMYXllcl8zIj48cGF0aCBjbGFzcz0iY2xzLTEiIGQ9Ik01MS44NywwQzIzLjcxLDAsMCwyMi44MywwLDUxYzAsLjkxLDAsNTIuODEsMCw1Mi44MWw1MS44Ni0uMDVjMjguMTYsMCw1MS0yMy43MSw1MS01MS44N1M4MCwwLDUxLjg3LDBaIi8+PHBhdGggY2xhc3M9ImNscy0yIiBkPSJNNTIuMzcsMTkuNzRBMzEuNjIsMzEuNjIsMCwwLDAsMjQuNTgsNjYuNDFsLTUuNzIsMTguNEwzOS40LDgwLjE3YTMxLjYxLDMxLjYxLDAsMSwwLDEzLTYwLjQzWiIvPjxwYXRoIGNsYXNzPSJjbHMtMyIgZD0iTTc3LjQ1LDMyLjEyYTMxLjYsMzEuNiwwLDAsMS0zOC4wNSw0OEwxOC44Niw4NC44MmwyMC45MS0yLjQ3QTMxLjYsMzEuNiwwLDAsMCw3Ny40NSwzMi4xMloiLz48cGF0aCBjbGFzcz0iY2xzLTQiIGQ9Ik03MS42MywyNi4yOUEzMS42LDMxLjYsMCwwLDEsMzguOCw3OEwxOC44Niw4NC44MiwzOS40LDgwLjE3QTMxLjYsMzEuNiwwLDAsMCw3MS42MywyNi4yOVoiLz48cGF0aCBjbGFzcz0iY2xzLTUiIGQ9Ik0yNi40Nyw2Ny4xMWEzMS42MSwzMS42MSwwLDAsMSw1MS0zNUEzMS42MSwzMS42MSwwLDAsMCwyNC41OCw2Ni40MWwtNS43MiwxOC40WiIvPjxwYXRoIGNsYXNzPSJjbHMtNiIgZD0iTTI0LjU4LDY2LjQxQTMxLjYxLDMxLjYxLDAsMCwxLDcxLjYzLDI2LjI5YTMxLjYxLDMxLjYxLDAsMCwwLTQ5LDM5LjYzbC0zLjc2LDE4LjlaIi8+PC9nPjwvZz48L3N2Zz4="/></a><!----> <!--[0--><a href="https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/th/chapter6/section8.ipynb" target="_blank"><img alt="Open In Colab" class="!m-0" src="https://colab.research.google.com/assets/colab-badge.svg"/></a><!--]--> <!--[0--><a href="https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/th/chapter6/section8.ipynb" target="_blank"><img alt="Open In Studio Lab" class="!m-0" src="https://studiolab.sagemaker.aws/studiolab.svg"/></a><!--]--></div><!----> <p>จากบทก่อนๆ คุณจะเห็นว่า การตัดคำ ประกอบไปด้วยหลายขั้นตอน :</p> <ul><li>Normalization (หรือ การปรับข้อความให้เป็นมาตรฐาน หมายถึงการทำความสะอาดข้อความ เช่น ลบช่องว่างหรือเครื่องหมายเน้นเสียง รวมถึงการทำ Unicode normalization และอื่นๆ)</li> <li>Pre-tokenization (ขั้นตอนก่อนตัดคำ หมายถึง การแยกข้อความออกเป็นคำๆ)</li> <li>ส่ง input เข้าไปในโมเดล (แยกคำที่ได้จากขั้นตอน pre-tokenization ออกเป็นคำย่อยหลายๆคำ)</li> <li>Post-processing (ขั้นตอนปรับแต่งผลลัพธ์ เช่น การใส่ token พิเศษของ tokenizer เข้าไปในผลลัพธ์, การสร้าง attention mask และ token type IDs)</li></ul> <p>เพื่อเป็นการเตือนความจำ มาดูกระบวนการโดยรวมอีกครั้ง :</p> <div class="flex justify-center"><img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/tokenization_pipeline.svg" alt="The tokenization pipeline."/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/tokenization_pipeline-dark.svg" alt="The tokenization pipeline."/></div> <p>🤗 Tokenizers library เป็นเครื่องมือสำหรับช่วยดำเนินการขั้นตอนพวกนี้ โดยคุณสามารถผสมผสานเครื่องมือพวกนี้ตามความต้องการได้ | |
| ในบทนี้เราจะมาเรียนวิธีสร้าง tokenizer ขึ้นมาตั้งแต่ต้น แทนที่จะเทรนจากตัวที่ถูก implement แล้วด้วยข้อมูลใหม่เท่านั้น อย่างที่เราได้ลองทำกันใน<a href="/course/chapter6/2">บทที่ 2</a> เมื่อจบบทนี้ คุณจะสามารถสร้าง tokenizer แบบใดก็ได้ตามที่คุณต้องการ</p> <iframe class="w-full xl:w-4/6 h-80" src="https://www.youtube-nocookie.com/embed/MR8tZm5ViWU" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen=""></iframe><!----> <p>library นี้ ประกอบด้วยส่วนหลักคือ <code>Tokenizer</code> class ที่มีส่วนประกอบย่อยสำคัญอื่นๆ แบ่งเป็นหลาย submodules</p> <ul><li><code>normalizers</code> ประกอบไปด้วย <code>Normalizer</code> หลายประเภทที่คุณสามารถใช้ได้ (รายชื่อทั้งหมดดูได้<a href="https://huggingface.co/docs/tokenizers/api/normalizers" rel="nofollow">ที่นี่</a>).</li> <li><code>pre_tokenizers</code> ประกอบไปด้วย <code>PreTokenizer</code> หลายประเภทที่คุณสามารถใช้ได้ (รายชื่อทั้งหมดดูได้<a href="https://huggingface.co/docs/tokenizers/api/pre-tokenizers" rel="nofollow">ที่นี่</a>).</li> <li><code>models</code> ประกอบไปด้วย <code>Model</code> หลายประเภทที่คุณสามารถใช้ได้ เช่น <code>BPE</code>, <code>WordPiece</code>, และ <code>Unigram</code> (รายชื่อทั้งหมดดูได้<a href="https://huggingface.co/docs/tokenizers/api/models" rel="nofollow">ที่นี่</a>).</li> <li><code>trainers</code> ประกอบไปด้วย <code>Trainer</code> หลายประเภทที่คุณสามารถใช้เพื่อเทรนโมเดลด้วย corpus ที่มีได้ (แต่ละโมเดลจะมีเทรนเนอร์อย่างละตัว; รายชื่อทั้งหมดดูได้<a href="https://huggingface.co/docs/tokenizers/api/trainers" rel="nofollow">ที่นี่</a>).</li> <li><code>post_processors</code> ประกอบไปด้วย <code>PostProcessor</code> หลายประเภทที่คุณสามารถใช้ได้ (รายชื่อทั้งหมดดูได้<a href="https://huggingface.co/docs/tokenizers/api/post-processors" rel="nofollow">ที่นี่</a>)</li> <li><code>decoders</code> ประกอบไปด้วย <code>Decoder</code> หลายประเภทที่ใช้เพื่อ decode ผลลัพธ์จากการ tokenization (รายชื่อทั้งหมดดูได้<a href="https://huggingface.co/docs/tokenizers/components#decoders" rel="nofollow">ที่นี่</a>)</li></ul> <p>คุณสามารถดูรายชื่อของส่วนประกอบสำคัญๆต่างๆได้<a href="https://huggingface.co/docs/tokenizers/components" rel="nofollow">ที่นี่</a></p> <!--[1--><h2 class="relative group"><a id="การโหลด-corpus" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#การโหลด-corpus"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>การโหลด corpus</span></h2><!--]--><!----> <p>ในการเทรน tokenizer ตัวใหม่ เราจะใช้ corpus เล็กๆ เพื่อที่การคำนวณจะได้รวดเร็ว | |
| การเรียกใช้งาน corpus จะทำคล้ายๆกับวิธีที่เราใช้ใน<a href="/course/chapter6/2">ตอนต้นของบทนี้</a> แต่ครั้งนี้เราจะใช้ชุดข้อมูลชื่อ <a href="https://huggingface.co/datasets/wikitext" rel="nofollow">WikiText-2</a></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| dataset = load_dataset(<span class="hljs-string">"wikitext"</span>, name=<span class="hljs-string">"wikitext-2-raw-v1"</span>, split=<span class="hljs-string">"train"</span>) | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">get_training_corpus</span>(): | |
| <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, <span class="hljs-built_in">len</span>(dataset), <span class="hljs-number">1000</span>): | |
| <span class="hljs-keyword">yield</span> dataset[i : i + <span class="hljs-number">1000</span>][<span class="hljs-string">"text"</span>]<!----></pre></div><!----> <p><code>get_training_corpus()</code> เป็น generator ที่จะ yield ข้อมูลในรูปแบบ batch โดยแต่ละ batch ประกอบไปด้วย 1,000 ข้อความ | |
| 🤗 Tokenizers สามารถเทรนได้จากไฟล์ข้อความโดยตรง ดังนั้นเราจะสร้างไฟล์ข้อความ ที่ประกอบไปด้วย ข้อความทั้งหมดจาก WikiText-2</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(<span class="hljs-string">"wikitext-2.txt"</span>, <span class="hljs-string">"w"</span>, encoding=<span class="hljs-string">"utf-8"</span>) <span class="hljs-keyword">as</span> f: | |
| <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(dataset)): | |
| f.write(dataset[i][<span class="hljs-string">"text"</span>] + <span class="hljs-string">"\n"</span>)<!----></pre></div><!----> <p>ต่อไปเราจะพาคุณสร้าง tokenizer แบบ BERT, GPT-2, and XLNet ของคุณเอง ทีละขั้นตอน | |
| ซึ่งคุณก็จะได้ลองใช้งาน tokenization algorithm ต่างๆที่ได้เรียนมาแล้ว เช่น WordPiece, BPE, and Unigram มาเริ่มจาก BERT กัน</p> <!--[1--><h2 class="relative group"><a id="สราง-wordpiece-tokenizer-ตงแตเรมตน" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#สราง-wordpiece-tokenizer-ตงแตเรมตน"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>สร้าง WordPiece tokenizer ตั้งแต่เริ่มต้น</span></h2><!--]--><!----> <p>ในการสร้าง tokenizer โดยใช้ 🤗 Tokenizers library เราจะเริ่มจากการสร้าง <code>Tokenizer</code> object จาก <code>model</code> และตั้งค่า attribute ต่างๆ ได้แก่ <code>normalizer</code>, <code>pre_tokenizer</code>, <code>post_processor</code>, และ <code>decoder</code> ด้วยค่าที่เราต้องการ</p> <p>ในตัวอย่างนี้ เราจะสร้าง <code>Tokenizer</code> ด้วย WordPiece :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> ( | |
| decoders, | |
| models, | |
| normalizers, | |
| pre_tokenizers, | |
| processors, | |
| trainers, | |
| Tokenizer, | |
| ) | |
| tokenizer = Tokenizer(models.WordPiece(unk_token=<span class="hljs-string">"[UNK]"</span>))<!----></pre></div><!----> <p>จากนั้นเราจะตั้งค่า <code>unk_token</code> เพื่อบอกโมเดลว่าให้มัน return ค่าอะไรหากมันเจอตัวอักษรที่มันไม่รู้จัก | |
| ส่วน argument อื่นๆที่เราสามารถตั้งค่าได้ก็คือ <code>vocab</code> (แต่เนื่องจากเราจะเทรนโมเดล ทำให้ตอนนี้เรายังไม่ต้องตั้งค่านี้) และ <code>max_input_chars_per_word</code> ซึ่งหมายถึง ความยาวสูงสุดของแต่ละคำ (คำที่ยาวกว่าค่านี้จะถูกแบ่งเป็นหลายๆส่วน)</p> <p>ขั้นตอนแรกคือ normalization เนื่องจาก BERT เป็นโมเดลที่ได้รับความนิยมมาก เราจึงมี <code>BertNormalizer</code> เฉพาะ ซึ่งมี option ต่างๆดังนี้: <code>lowercase</code>, <code>strip_accents</code>, <code>clean_text</code> (ลบ control characters และแทนที่ช่องว่างที่อยู่ต่อกันหลายๆอันด้วยช่องว่างเดียว), และ <code>handle_chinese_chars</code> (เพิ่มช่องว่างในตัวอักษรจีน)</p> <p>เราจะมาเลียนแบบ <code>bert-base-uncased</code> tokenizer โดยตั้งค่า normalizer ดังนี้ :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.normalizer = normalizers.BertNormalizer(lowercase=<span class="hljs-literal">True</span>)<!----></pre></div><!----> <p>แต่ปกติแล้ว สำหรับ tokenizer ใหม่ คุณอาจจะไม่สามารถใช้ normalizer ที่จาก 🤗 Tokenizers library ได้ ดังนั้นเราจะมาเรียนวิธีการสร้าง BERT normalizer เองกัน</p> <p>library นี้ มี normalizer เพื่อ <code>Lowercase</code> และเพื่อ <code>StripAccents</code> ซึ่งคุณสามารถรวมทั้งสองตัวได้โดยใช้ <code>Sequence</code> :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.normalizer = normalizers.<span class="hljs-type">Sequence</span>( | |
| [normalizers.NFD(), normalizers.Lowercase(), normalizers.StripAccents()] | |
| )<!----></pre></div><!----> <p>เราจะใช้ <code>NFD</code> Unicode normalizer ด้วย เพื่อที่จะให้ <code>StripAccents</code> สามารถหาสัญลักษณ์ accents ได้ และจะได้ลบพวกมันออก</p> <p>เพื่อเช็คผลลัพธ์ของ normalizer เราจะใช้ <code>normalize_str()</code> method จาก <code>normalizer</code> :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-built_in">print</span>(tokenizer.normalizer.normalize_str(<span class="hljs-string">"Héllò hôw are ü?"</span>))<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->hello how are u?<!----></pre></div><!----> <blockquote class="tip"><p><strong>รายละเอียดเพิ่มเติม</strong> ถ้าคุณทดลองใช้งาน normalizer ทั้งสองเวอร์ชันกับข้อความที่มีตัวอักษร unicode <code>u"\u0085"</code> คุณจะได้ผลลัพธ์ที่แตกต่างกัน | |
| อย่างไรก็ตาม เราไม่อยากทำให้เวอร์ชันที่สร้างจาก <code>normalizers.Sequence</code> ของเรานั้นซับซ้อนเกินไป เราจึงไม่ใช้ Regex ที่ <code>BertNormalizer</code> ใช้เวลาที่ <code>clean_text</code> ถูกตั้งค่าเป็น <code>True</code> ซึ่งเป็นค่าตั้งต้น | |
| แต่คุณไม่ต้องกังวลไป เพราะมันยังมีวิธีที่จะทำให้ผลลัพธ์ออกมาเป็นเหมือนกันโดยที่ไม่ต้องใช้ <code>BertNormalizer</code> นั่นคือโดยการเพิ่ม <code>normalizers.Replace</code> สองครั้ง เข้าไปใน <code>normalizers.Sequence</code></p></blockquote> <p>ขั้นตอนต่อไปคือ การ pre-tokenization เราจะใช้ <code>BertPreTokenizer</code> ที่ถูกสร้างมาแล้ว :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer()<!----></pre></div><!----> <p>หรือจะใช้ตัวที่เราสร้างขึ้นมาเองก็ได้ :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()<!----></pre></div><!----> <p>โปรดทราบว่า <code>Whitespace</code> pre-tokenizer จะตัดข้อความตรงที่มีช่องว่าง และ รวมถึงตัวสัญลักษณ์ที่ไม่ใช้ตัวอักษร ตัวเลข หรือ underscore หรือพูดอีกแบบก็คือ มันจะแบ่งข้อความตรงที่มีช่องว่างและเครื่องหมายวรรคตอน นั่นเอง</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.pre_tokenizer.pre_tokenize_str(<span class="hljs-string">"Let's test my pre-tokenizer."</span>)<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->[(<span class="hljs-string">'Let'</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">3</span>)), (<span class="hljs-string">"'"</span>, (<span class="hljs-number">3</span>, <span class="hljs-number">4</span>)), (<span class="hljs-string">'s'</span>, (<span class="hljs-number">4</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">'test'</span>, (<span class="hljs-number">6</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">'my'</span>, (<span class="hljs-number">11</span>, <span class="hljs-number">13</span>)), (<span class="hljs-string">'pre'</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">17</span>)), | |
| (<span class="hljs-string">'-'</span>, (<span class="hljs-number">17</span>, <span class="hljs-number">18</span>)), (<span class="hljs-string">'tokenizer'</span>, (<span class="hljs-number">18</span>, <span class="hljs-number">27</span>)), (<span class="hljs-string">'.'</span>, (<span class="hljs-number">27</span>, <span class="hljs-number">28</span>))]<!----></pre></div><!----> <p>แต่ถ้าคุณต้องการจะแบ่งข้อความตามช่องว่างเท่านั้น ให้ใช้ <code>WhitespaceSplit</code> pre-tokenizer :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->pre_tokenizer = pre_tokenizers.WhitespaceSplit() | |
| pre_tokenizer.pre_tokenize_str(<span class="hljs-string">"Let's test my pre-tokenizer."</span>)<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->[(<span class="hljs-string">"Let's"</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">'test'</span>, (<span class="hljs-number">6</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">'my'</span>, (<span class="hljs-number">11</span>, <span class="hljs-number">13</span>)), (<span class="hljs-string">'pre-tokenizer.'</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">28</span>))]<!----></pre></div><!----> <p>เหมือนกับใน normalizer เราสามารถใช้ <code>Sequence</code> เพื่อรวมหลายๆ pre-tokenizers เข้าด้วยกัน :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->pre_tokenizer = pre_tokenizers.<span class="hljs-type">Sequence</span>( | |
| [pre_tokenizers.WhitespaceSplit(), pre_tokenizers.Punctuation()] | |
| ) | |
| pre_tokenizer.pre_tokenize_str(<span class="hljs-string">"Let's test my pre-tokenizer."</span>)<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->[(<span class="hljs-string">'Let'</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">3</span>)), (<span class="hljs-string">"'"</span>, (<span class="hljs-number">3</span>, <span class="hljs-number">4</span>)), (<span class="hljs-string">'s'</span>, (<span class="hljs-number">4</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">'test'</span>, (<span class="hljs-number">6</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">'my'</span>, (<span class="hljs-number">11</span>, <span class="hljs-number">13</span>)), (<span class="hljs-string">'pre'</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">17</span>)), | |
| (<span class="hljs-string">'-'</span>, (<span class="hljs-number">17</span>, <span class="hljs-number">18</span>)), (<span class="hljs-string">'tokenizer'</span>, (<span class="hljs-number">18</span>, <span class="hljs-number">27</span>)), (<span class="hljs-string">'.'</span>, (<span class="hljs-number">27</span>, <span class="hljs-number">28</span>))]<!----></pre></div><!----> <p>ขั้นตอนต่อไปใน tokenization pipeline คือการใส่ input เข้าไปในโมเดล | |
| เราได้สร้างโมเดลขึ้นมาแล้ว แต่เรายังคงต้องเทรนมัน ซึ่งเราจะใช้ <code>WordPieceTrainer</code></p> <p>สิ่งสำคัญที่คุณต้องจำคือ เวลาสร้าง (instantiate) trainer ใน 🤗 Tokenizers คุณจะต้องเพิ่ม token พิเศษต่างๆ เข้าไปในเทรนเนอร์เอง ไม่เช่นนั้น มันจะไม่เพิ่ม token พวกนี้เข้าไปใน vocabulary เพราะว่า token พิเศษไม่ได้อยู่ใน training corpus :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->special_tokens = [<span class="hljs-string">"[UNK]"</span>, <span class="hljs-string">"[PAD]"</span>, <span class="hljs-string">"[CLS]"</span>, <span class="hljs-string">"[SEP]"</span>, <span class="hljs-string">"[MASK]"</span>] | |
| trainer = trainers.WordPieceTrainer(vocab_size=<span class="hljs-number">25000</span>, special_tokens=special_tokens)<!----></pre></div><!----> <p>นอกจากเราจะสามารถตั้งค่า <code>vocab_size</code> และ <code>special_tokens</code> แล้ว เรายังสามารถตั้งค่า <code>min_frequency</code> (ความถี่ขั้นต่ำของคำที่ต้องมี เพื่อที่จะได้ถูกเพิ่มลงไปใน vocabulary) หรือ <code>continuing_subword_prefix</code> (ถ้าหากเราต้องการใช้สัญลักษณ์อื่น ที่ไม่ใช่<code>##</code>) ได้อีกด้วย</p> <p>เราจะเทรนโมเดลโดยรันคำสั่งต่อไปนี้ :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)<!----></pre></div><!----> <p>คุณสามารถใช้ไฟล์ข้อความเพื่อเทรนได้ด้วย (แต่ก่อนอื่น คุณจะต้องสร้างโมเดลขึ้นมาใหม่โดยใช้ <code>WordPiece</code> ที่ยังว่างเปล่าอยู่) :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.model = models.WordPiece(unk_token=<span class="hljs-string">"[UNK]"</span>) | |
| tokenizer.train([<span class="hljs-string">"wikitext-2.txt"</span>], trainer=trainer)<!----></pre></div><!----> <p>สำหรับทั้งสองกรณี คุณสามารถทดลองใช้ tokenizer กับข้อความได้ โดยเรียกใช้ <code>encode()</code> method :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->encoding = tokenizer.encode(<span class="hljs-string">"Let's test this tokenizer."</span>) | |
| <span class="hljs-built_in">print</span>(encoding.tokens)<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->[<span class="hljs-string">'let'</span>, <span class="hljs-string">"'"</span>, <span class="hljs-string">'s'</span>, <span class="hljs-string">'test'</span>, <span class="hljs-string">'this'</span>, <span class="hljs-string">'tok'</span>, <span class="hljs-string">'##eni'</span>, <span class="hljs-string">'##zer'</span>, <span class="hljs-string">'.'</span>]<!----></pre></div><!----> <p><code>encoding</code> ที่เราได้คือ <code>Encoding</code> ที่ประกอบไปด้วยข้อมูลจำเป็นต่างๆสำหรับ tokenizer ได้แก่ <code>ids</code>, <code>type_ids</code>, <code>tokens</code>, <code>offsets</code>, <code>attention_mask</code>, <code>special_tokens_mask</code>, และ <code>overflowing</code></p> <p>ขั้นตอนสุดท้ายของ pipeline คือ post-processing | |
| เราจะเพิ่ม <code>[CLS]</code> ไว้ที่ตอนต้นของผลลัพธ์จากการ tokenize และ <code>[SEP]</code> ไว้ที่ตอนท้าย หรือหลังสิ้นสุดประโยค ถ้า input ของเรามีสองประโยค</p> <p>เราจะใช้ <code>TemplateProcessor</code> เพื่อช่วยเราทำ แต่ก่อนอื่นเราต้องหา ID ของ <code>[CLS]</code> และ <code>[SEP]</code> ก่อน :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->cls_token_id = tokenizer.token_to_id(<span class="hljs-string">"[CLS]"</span>) | |
| sep_token_id = tokenizer.token_to_id(<span class="hljs-string">"[SEP]"</span>) | |
| <span class="hljs-built_in">print</span>(cls_token_id, sep_token_id)<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->(<span class="hljs-number">2</span>, <span class="hljs-number">3</span>)<!----></pre></div><!----> <p>เราจะมาสร้าง template ให้กับ <code>TemplateProcessor</code> โดยจะต้องกำหนดว่าเราต้องการให้มันประมวลผล input ที่เป็น ประโยคเดี่ยว และ input ที่มีสองประโยคอย่างไร | |
| สำหรับทั้งสองกรณี เราจะต้องกำหนด token พิเศษ เพื่อแทนประโยคขึ้นมา สำหรับประโยคเดี่ยวหรือประโยคแรก เราจะใช้ <code>$A</code> ส่วนสำหรับประโยคที่สอง เราจะใช้ <code>$B</code> สำหรับ token พิเศษพวกนี้และประโยค เราจะสร้าง token type ID ขึ้นมา โดยจะเขียน ID นี้หลังเครื่องหมาย colon</p> <p>template ของ BERT มีรายละเอียดดังนี้ :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.post_processor = processors.TemplateProcessing( | |
| single=<span class="hljs-string">f"[CLS]:0 $A:0 [SEP]:0"</span>, | |
| pair=<span class="hljs-string">f"[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"</span>, | |
| special_tokens=[(<span class="hljs-string">"[CLS]"</span>, cls_token_id), (<span class="hljs-string">"[SEP]"</span>, sep_token_id)], | |
| )<!----></pre></div><!----> <p>อย่าลืมว่า เราจะต้องให้ข้อมูลโมเดลเกี่ยวกับ ID ของ token พิเศษด้วย เพื่อที่โมเดลจะได้แปลงมันได้อย่างถูกต้อง</p> <p>เราจะกลับมาดูตัวอย่างก่อนหน้ากัน :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->encoding = tokenizer.encode(<span class="hljs-string">"Let's test this tokenizer."</span>) | |
| <span class="hljs-built_in">print</span>(encoding.tokens)<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->[<span class="hljs-string">'[CLS]'</span>, <span class="hljs-string">'let'</span>, <span class="hljs-string">"'"</span>, <span class="hljs-string">'s'</span>, <span class="hljs-string">'test'</span>, <span class="hljs-string">'this'</span>, <span class="hljs-string">'tok'</span>, <span class="hljs-string">'##eni'</span>, <span class="hljs-string">'##zer'</span>, <span class="hljs-string">'.'</span>, <span class="hljs-string">'[SEP]'</span>]<!----></pre></div><!----> <p>ถ้าเราใส่ input ที่เป็นสองประโยค เราจะได้ :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->encoding = tokenizer.encode(<span class="hljs-string">"Let's test this tokenizer..."</span>, <span class="hljs-string">"on a pair of sentences."</span>) | |
| <span class="hljs-built_in">print</span>(encoding.tokens) | |
| <span class="hljs-built_in">print</span>(encoding.type_ids)<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->[<span class="hljs-string">'[CLS]'</span>, <span class="hljs-string">'let'</span>, <span class="hljs-string">"'"</span>, <span class="hljs-string">'s'</span>, <span class="hljs-string">'test'</span>, <span class="hljs-string">'this'</span>, <span class="hljs-string">'tok'</span>, <span class="hljs-string">'##eni'</span>, <span class="hljs-string">'##zer'</span>, <span class="hljs-string">'...'</span>, <span class="hljs-string">'[SEP]'</span>, <span class="hljs-string">'on'</span>, <span class="hljs-string">'a'</span>, <span class="hljs-string">'pair'</span>, <span class="hljs-string">'of'</span>, <span class="hljs-string">'sentences'</span>, <span class="hljs-string">'.'</span>, <span class="hljs-string">'[SEP]'</span>] | |
| [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]<!----></pre></div><!----> <p>ขั้นตอนสุดท้าย เราจะเพิ่ม decoder เข้าไปใน pipeline :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.decoder = decoders.WordPiece(prefix=<span class="hljs-string">"##"</span>)<!----></pre></div><!----> <p>มาดูผลลัพธ์ของ <code>encoding</code> กัน :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.decode(encoding.ids)<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-string">"let's test this tokenizer... on a pair of sentences."</span><!----></pre></div><!----> <p>เยี่ยมมาก! ตอนนี้เราสามารถบันทึก tokenizer นี้เป็นไฟล์ JSON ได้แล้ว ดังนี้ :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.save(<span class="hljs-string">"tokenizer.json"</span>)<!----></pre></div><!----> <p>คุณสามารถโหลดไฟล์นี้ให้เป็น <code>Tokenizer</code> object ได้โดยใช้ <code>from_file()</code> method :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->new_tokenizer = Tokenizer.from_file(<span class="hljs-string">"tokenizer.json"</span>)<!----></pre></div><!----> <p>การจะนำ tokenizer นี้มาใช้ใน 🤗 Transformers เราจะต้อง wrap มันให้เป็น <code>PreTrainedTokenizerFast</code> ก่อน | |
| โดยเราใช้ class ปกติ (ถ้า tokenizer ของเรามีโครงสร้างสอดคล้องกันโมเดลหลักเราที่จะใช้งาน) หรือ class ที่ถูกสร้างขึ้นมาแล้ว เช่น <code>BertTokenizerFast</code> ในกรณีที่คุณสร้าง tokenizer ขึ้นมาเองอย่างที่เราได้สอนไว้ข้างต้น คุณจะต้องใช้ตัวเลือกแรก</p> <p>การจะ wrap tokenizer ของเราให้เป็น <code>PreTrainedTokenizerFast</code> เราจะต้องส่งผ่าน tokenizer ของเราเข้าไปเป็น <code>tokenizer_object</code> หรือ ส่งผ่านไฟล์ของ tokenizer เป็น <code>tokenizer_file</code> สิ่งสำคัญอีกอย่างคือ คุณจะต้องส่งผ่านข้อมูลเกี่ยวกับ token พิเศษ ต่างๆให้โมเดลเอง เพราะว่า class นี้จะไม่สามารถ infer จาก <code>tokenizer</code> object ได้เอง ว่า token ตัวไหนเป็น mask token เช่น <code>[CLS]</code> :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast | |
| wrapped_tokenizer = PreTrainedTokenizerFast( | |
| tokenizer_object=tokenizer, | |
| <span class="hljs-comment"># tokenizer_file="tokenizer.json", # You can load from the tokenizer file, alternatively</span> | |
| unk_token=<span class="hljs-string">"[UNK]"</span>, | |
| pad_token=<span class="hljs-string">"[PAD]"</span>, | |
| cls_token=<span class="hljs-string">"[CLS]"</span>, | |
| sep_token=<span class="hljs-string">"[SEP]"</span>, | |
| mask_token=<span class="hljs-string">"[MASK]"</span>, | |
| )<!----></pre></div><!----> <p>ถ้าคุณใช้ class ที่เฉพาะเจาะจง เช่น <code>BertTokenizerFast</code> คุณเพียงแค่ต้องเพิ่มข้อมูลเกี่ยวกับ token พิเศษที่ต่างจากตัวที่โมเดลใช้อยู่แล้ว ในตัวอย่างของเรา เราไม่ได้ใช้ token พิเศษอื่น จึงไม่ต้องเพิ่มอะไร :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BertTokenizerFast | |
| wrapped_tokenizer = BertTokenizerFast(tokenizer_object=tokenizer)<!----></pre></div><!----> <p>ตอนนี้เราก็สามารถใช้ tokenizer ที่เราได้สร้างขึ้นมาเอง เหมือนกับ tokenizer ตัวอื่นๆจาก 🤗 Transformers ได้แล้ว นอกจากนั้นคุณยังสามารถเซฟมันได้ โดยใช้ <code>save_pretrained()</code> หรืออัพโหลดมันไปที่ Hub โดยใช้ <code>push_to_hub()</code></p> <p>หลังจากที่เราได้ดูกันแล้วว่าจะสร้าง WordPiece tokenizer อย่างไร เราจะมาดูกันว่า จะทำแบบเดียวกันกับ BPE tokenizer ได้อย่างไร เราจะอธิบายคร่าวๆเท่านั้น เพราะคุณได้เรียนรายละเอียดมาแล้ว และจะพูดถึงแค่ข้อแตกต่างเท่านั้น</p> <!--[1--><h2 class="relative group"><a id="การสราง-bpe-tokenizer-ตงแตเรมตน" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#การสราง-bpe-tokenizer-ตงแตเรมตน"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>การสร้าง BPE tokenizer ตั้งแต่เริ่มต้น</span></h2><!--]--><!----> <p>เราจะมาสร้าง GPT-2 tokenizer กัน เช่นเดียวกับตัวอย่างของ BERT tokenizer เราจะเริ่มด้วยกัน initialize <code>Tokenizer</code> ด้วยโมเดล BPE :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer = Tokenizer(models.BPE())<!----></pre></div><!----> <p>เราสามารถสร้างโมเดลนี้ด้วย vocabulary ที่มีอยู่ได้ โดยส่งผ่าน <code>vocab</code> และ <code>merges</code> แต่เราจะเทรนตั้งแต่ต้น แปลว่าเราไม่จำเป็นต้องทำขั้นตอนนี้ | |
| เราไม่จำเป็นต้องกำหนด <code>unk_token</code> เพราะ GPT-2 ใช้ byte-level BPE</p> <p>นอกจากนั้น GPT-2 ยังไม่ใช้ normalizer อีกด้วย เราจึงจะข้ามขั้นตอนนี้ไปทำ pre-tokenization เลย :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=<span class="hljs-literal">False</span>)<!----></pre></div><!----> <p><code>add_prefix_space=False</code> หมายถึงเราจะไม่เพิ่มช่องว่างตรงต้นประโยค (ค่า default จะมีการเพิ่มช่องว่างนี้)</p> <p>มาดูผลลัพธ์ตัวอย่างของการ pre-tokenization กัน :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.pre_tokenizer.pre_tokenize_str(<span class="hljs-string">"Let's test pre-tokenization!"</span>)<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->[(<span class="hljs-string">'Let'</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">3</span>)), (<span class="hljs-string">"'s"</span>, (<span class="hljs-number">3</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">'Ġtest'</span>, (<span class="hljs-number">5</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">'Ġpre'</span>, (<span class="hljs-number">10</span>, <span class="hljs-number">14</span>)), (<span class="hljs-string">'-'</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">15</span>)), | |
| (<span class="hljs-string">'tokenization'</span>, (<span class="hljs-number">15</span>, <span class="hljs-number">27</span>)), (<span class="hljs-string">'!'</span>, (<span class="hljs-number">27</span>, <span class="hljs-number">28</span>))]<!----></pre></div><!----> <p>ขั้นตอนต่อไปคือการเทรนโมเดล สำหรับ GPT-2 มันจะมี token พิเศษตัวเดียวคือ end-of-text token (อยู่ตรงท้ายของข้อความ) :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->trainer = trainers.BpeTrainer(vocab_size=<span class="hljs-number">25000</span>, special_tokens=[<span class="hljs-string">"<|endoftext|>"</span>]) | |
| tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)<!----></pre></div><!----> <p>เช่นเดียวกันกับ <code>WordPieceTrainer</code> คุณสามารถกำหนด <code>vocab_size</code>, <code>special_tokens</code>, <code>min_frequency</code> ได้ ถ้าหากคุณใช้ end-of-word suffix (เช่น <code></w></code>) คุณก็สามารถตั้งค่ามันได้ด้วย <code>end_of_word_suffix</code></p> <p>คุณสามารถเทรนด้วยไฟล์ข้อความได้ด้วย :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.model = models.BPE() | |
| tokenizer.train([<span class="hljs-string">"wikitext-2.txt"</span>], trainer=trainer)<!----></pre></div><!----> <p>มาดูผลลัพธ์ของการ tokenize ข้อความตัวอย่างกัน :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->encoding = tokenizer.encode(<span class="hljs-string">"Let's test this tokenizer."</span>) | |
| <span class="hljs-built_in">print</span>(encoding.tokens)<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->[<span class="hljs-string">'L'</span>, <span class="hljs-string">'et'</span>, <span class="hljs-string">"'"</span>, <span class="hljs-string">'s'</span>, <span class="hljs-string">'Ġtest'</span>, <span class="hljs-string">'Ġthis'</span>, <span class="hljs-string">'Ġto'</span>, <span class="hljs-string">'ken'</span>, <span class="hljs-string">'izer'</span>, <span class="hljs-string">'.'</span>]<!----></pre></div><!----> <p>เราจะทำการ post-processing แบบ byte-level ให้กับ GPT-2 tokenizer ดังนี้ :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.post_processor = processors.ByteLevel(trim_offsets=<span class="hljs-literal">False</span>)<!----></pre></div><!----> <p><code>trim_offsets = False</code> แปลว่าเราจะไม่เปลี่ยนแปลงค่า offset ของ token ที่ขึ้นต้นด้วย <code>Ġ</code> ซึ่งแปลว่าตำแหน่งเริ่มต้นของ offset จะหมายถึง ช่องว่าง และไม่ใช่ตัวอักษรแรกของ token นั้น</p> <p>มาดูผลลัพธ์ของข้อความที่เราเพิ่งจะ encode กันไป ในตัวอย่างนี้ token ในตำแหน่งที่ 4 คือ <code>'Ġtest'</code> :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->sentence = <span class="hljs-string">"Let's test this tokenizer."</span> | |
| encoding = tokenizer.encode(sentence) | |
| start, end = encoding.offsets[<span class="hljs-number">4</span>] | |
| sentence[start:end]<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-string">' test'</span><!----></pre></div><!----> <p>สุดท้ายเราจะเพิ่มส่วนที่เป็น byte-level decoder :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.decoder = decoders.ByteLevel()<!----></pre></div><!----> <p>เช็คดูอีกทีว่าผลลัพธ์ถูกต้องหรือไม่ :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.decode(encoding.ids)<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-string">"Let's test this tokenizer."</span><!----></pre></div><!----> <p>เยี่ยมมาก! ตอนนี้คุณสามารถเซฟโมเดลได้ ส่วนขั้นต่อไปคือ เราจะ wrap tokenizer ของเราให้เป็น <code>PreTrainedTokenizerFast</code> หรือ <code>GPT2TokenizerFast</code> เพื่อจะได้เรียกใช้มันได้ใน 🤗 Transformers</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast | |
| wrapped_tokenizer = PreTrainedTokenizerFast( | |
| tokenizer_object=tokenizer, | |
| bos_token=<span class="hljs-string">"<|endoftext|>"</span>, | |
| eos_token=<span class="hljs-string">"<|endoftext|>"</span>, | |
| )<!----></pre></div><!----> <p>หรือ :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPT2TokenizerFast | |
| wrapped_tokenizer = GPT2TokenizerFast(tokenizer_object=tokenizer)<!----></pre></div><!----> <p>ต่อไปเราจะสอนวิธีสร้าง Unigram tokenizer บ้าง</p> <!--[1--><h2 class="relative group"><a id="การสราง-unigram-tokenizer-ตงแตเรมตน" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#การสราง-unigram-tokenizer-ตงแตเรมตน"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>การสร้าง Unigram tokenizer ตั้งแต่เริ่มต้น</span></h2><!--]--><!----> <p>เราจะสร้าง XLNet tokenizer โดยเริ่มจาก initialize <code>Tokenizer</code> ด้วย Unigram model !</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer = Tokenizer(models.Unigram())<!----></pre></div><!----> <p>คุณสามารถ initialize มันโดยส่งผ่าน vocabulary ที่มีอยู่เข้าไปด้วย</p> <p>ในขั้นตอน normalization โมเดล XLNet จะมีการแทนที่ตัวอักษรต่างๆ :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Regex | |
| tokenizer.normalizer = normalizers.<span class="hljs-type">Sequence</span>( | |
| [ | |
| normalizers.Replace(<span class="hljs-string">"``"</span>, <span class="hljs-string">'"'</span>), | |
| normalizers.Replace(<span class="hljs-string">"''"</span>, <span class="hljs-string">'"'</span>), | |
| normalizers.NFKD(), | |
| normalizers.StripAccents(), | |
| normalizers.Replace(Regex(<span class="hljs-string">" {2,}"</span>), <span class="hljs-string">" "</span>), | |
| ] | |
| )<!----></pre></div><!----> <p>โค้ดข้างบนนี้จะแทนที่ <code>“</code> และ <code>”</code> ด้วย <code>”</code> และถ้ามีช่องว่างที่อยู่ต่อๆกัน มันจะถูกแปลงให้เป็นช่องว่างเดียว และสุดท้ายมันจะลบสัญลักษณ์ accent ออกด้วย</p> <p>pre-tokenizer ที่ใช้ใน SentencePiece tokenizer คือ <code>Metaspace</code> :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.pre_tokenizer = pre_tokenizers.Metaspace()<!----></pre></div><!----> <p>มาดูผลลัพธ์ของการ pre-tokenization กับข้อความตัวอย่างกัน :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.pre_tokenizer.pre_tokenize_str(<span class="hljs-string">"Let's test the pre-tokenizer!"</span>)<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->[(<span class="hljs-string">"▁Let's"</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">'▁test'</span>, (<span class="hljs-number">5</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">'▁the'</span>, (<span class="hljs-number">10</span>, <span class="hljs-number">14</span>)), (<span class="hljs-string">'▁pre-tokenizer!'</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">29</span>))]<!----></pre></div><!----> <p>ขั้นต่อไปคือการเทรน XLNet มีการใช้ token พิเศษอยู่จำนวนหนึ่ง :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->special_tokens = [<span class="hljs-string">"<cls>"</span>, <span class="hljs-string">"<sep>"</span>, <span class="hljs-string">"<unk>"</span>, <span class="hljs-string">"<pad>"</span>, <span class="hljs-string">"<mask>"</span>, <span class="hljs-string">"<s>"</span>, <span class="hljs-string">"</s>"</span>] | |
| trainer = trainers.UnigramTrainer( | |
| vocab_size=<span class="hljs-number">25000</span>, special_tokens=special_tokens, unk_token=<span class="hljs-string">"<unk>"</span> | |
| ) | |
| tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)<!----></pre></div><!----> <p>สิ่งสำคัญเวลาใช้ <code>UnigramTrainer</code> คือ อย่าลืมกำหนด argument <code>unk_token</code> นอกจากนั้นคุณยังสามารถตั้งค่า argument อื่นๆที่ต้องใช้กับ Unigram algorithm ได้ด้วย เช่น <code>shrinking_factor</code> (ค่าเริ่มต้นคือ 0.75) หรือ <code>max_piece_length</code> (ค่าเริ่มต้นคือ 16)</p> <p>เราสามารถเทรนจากไฟล์ข้อความได้ด้วย :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.model = models.Unigram() | |
| tokenizer.train([<span class="hljs-string">"wikitext-2.txt"</span>], trainer=trainer)<!----></pre></div><!----> <p>ลอง tokenize ตัวอย่างง่ายๆดู :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->encoding = tokenizer.encode(<span class="hljs-string">"Let's test this tokenizer."</span>) | |
| <span class="hljs-built_in">print</span>(encoding.tokens)<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->[<span class="hljs-string">'▁Let'</span>, <span class="hljs-string">"'"</span>, <span class="hljs-string">'s'</span>, <span class="hljs-string">'▁test'</span>, <span class="hljs-string">'▁this'</span>, <span class="hljs-string">'▁to'</span>, <span class="hljs-string">'ken'</span>, <span class="hljs-string">'izer'</span>, <span class="hljs-string">'.'</span>]<!----></pre></div><!----> <p>XLNet จะเพิ่ม token พิเศษ <code><cls></code> ใส่ในตอนท้ายของประโยค และตั้งค่า type ID มันเป็น 2 เพื่อให้มันแตกต่างจาก token อื่น | |
| การทำแบบนี้ถือว่าเป็นการ padding ทางด้ายซ้าย</p> <p>เพื่อจัดการกับ token พิเศษ เราจะต้องสร้าง template ขึ้นมา ก่อนอื่นเราต้องดูว่า ID ของ <code><cls></code> และ <code><sep></code> คืออะไร :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->cls_token_id = tokenizer.token_to_id(<span class="hljs-string">"<cls>"</span>) | |
| sep_token_id = tokenizer.token_to_id(<span class="hljs-string">"<sep>"</span>) | |
| <span class="hljs-built_in">print</span>(cls_token_id, sep_token_id)<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-number">0</span> <span class="hljs-number">1</span><!----></pre></div><!----> <p>นี่คือตัวอย่างการสร้าง template :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.post_processor = processors.TemplateProcessing( | |
| single=<span class="hljs-string">"$A:0 <sep>:0 <cls>:2"</span>, | |
| pair=<span class="hljs-string">"$A:0 <sep>:0 $B:1 <sep>:1 <cls>:2"</span>, | |
| special_tokens=[(<span class="hljs-string">"<sep>"</span>, sep_token_id), (<span class="hljs-string">"<cls>"</span>, cls_token_id)], | |
| )<!----></pre></div><!----> <p>เราจะทดลองใช้งานมันโดยการ encode ประโยค input สองประโยค :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->encoding = tokenizer.encode(<span class="hljs-string">"Let's test this tokenizer..."</span>, <span class="hljs-string">"on a pair of sentences!"</span>) | |
| <span class="hljs-built_in">print</span>(encoding.tokens) | |
| <span class="hljs-built_in">print</span>(encoding.type_ids)<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->[<span class="hljs-string">'▁Let'</span>, <span class="hljs-string">"'"</span>, <span class="hljs-string">'s'</span>, <span class="hljs-string">'▁test'</span>, <span class="hljs-string">'▁this'</span>, <span class="hljs-string">'▁to'</span>, <span class="hljs-string">'ken'</span>, <span class="hljs-string">'izer'</span>, <span class="hljs-string">'.'</span>, <span class="hljs-string">'.'</span>, <span class="hljs-string">'.'</span>, <span class="hljs-string">'<sep>'</span>, <span class="hljs-string">'▁'</span>, <span class="hljs-string">'on'</span>, <span class="hljs-string">'▁'</span>, <span class="hljs-string">'a'</span>, <span class="hljs-string">'▁pair'</span>, | |
| <span class="hljs-string">'▁of'</span>, <span class="hljs-string">'▁sentence'</span>, <span class="hljs-string">'s'</span>, <span class="hljs-string">'!'</span>, <span class="hljs-string">'<sep>'</span>, <span class="hljs-string">'<cls>'</span>] | |
| [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>]<!----></pre></div><!----> <p>จากนั้นตั้งค่า decoder เป็น <code>Metaspace</code> :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->tokenizer.decoder = decoders.Metaspace()<!----></pre></div><!----> <p>ตอนนี้เราก็เสร็จแล้ว คุณสามารถเซฟ tokenizer และ wrap มันให้เป็น <code>PreTrainedTokenizerFast</code> หรือ <code>XLNetTokenizerFast</code> ก็ได้ ถ้าคุณต้องการใช้มันใน 🤗 Transformers</p> <p>สิ่งสำคัญอีกอย่างเวลาใช้ <code>PreTrainedTokenizerFast</code> ก็คือเราจะต้องบอก 🤗 Transformers library ว่าเราได้ทำการ padding ทางซ้าย :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast | |
| wrapped_tokenizer = PreTrainedTokenizerFast( | |
| tokenizer_object=tokenizer, | |
| bos_token=<span class="hljs-string">"<s>"</span>, | |
| eos_token=<span class="hljs-string">"</s>"</span>, | |
| unk_token=<span class="hljs-string">"<unk>"</span>, | |
| pad_token=<span class="hljs-string">"<pad>"</span>, | |
| cls_token=<span class="hljs-string">"<cls>"</span>, | |
| sep_token=<span class="hljs-string">"<sep>"</span>, | |
| mask_token=<span class="hljs-string">"<mask>"</span>, | |
| padding_side=<span class="hljs-string">"left"</span>, | |
| )<!----></pre></div><!----> <p>อีกวิธีก็คือ :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> XLNetTokenizerFast | |
| wrapped_tokenizer = XLNetTokenizerFast(tokenizer_object=tokenizer)<!----></pre></div><!----> <p>ตอนนี้ คุณก็ได้เรียนรู้ขั้นตอนในการสร้าง tokenizer ขึ้นมาเองแล้ว โดยการใช้เครื่องมือจาก 🤗 Tokenizers library และได้เรียนวิธีการนำ tokenizer ของคุณไปใช้ใน 🤗 Transformers อีกด้วย</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/course/blob/main/chapters/th/chapter6/8.mdx" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]--> | |
| <script> | |
| { | |
| __sveltekit_13u241c = { | |
| base: "/docs/course/pr_1339/th", | |
| assets: "/docs/course/pr_1339/th" | |
| }; | |
| const element = document.currentScript.parentElement; | |
| Promise.all([ | |
| import("/docs/course/pr_1339/th/_app/immutable/entry/start.B-F4PHHy.js"), | |
| import("/docs/course/pr_1339/th/_app/immutable/entry/app.EwjAFzMw.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 43], | |
| data: [null,null], | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 151 kB
- Xet hash:
- 4b51ffe07e6f30200a26dd5e59ea2d95ffcaa218b5b8e975686e5528fcfa49b5
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.