Hugging Face's logo

Buckets:
hf-doc-build
/
doc-dev

Files
xet
hf-doc-build/doc-dev / course /pr_1052 /id /chapter1 /6.html
rtrm's picture
rtrm
download
raw
50.1 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Arsitektur Transformer&quot;,&quot;local&quot;:&quot;transformer-architectures&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Model Encoder&quot;,&quot;local&quot;:&quot;encoder-models&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Model Decoder&quot;,&quot;local&quot;:&quot;decoder-models&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Large Language Models (LLMs) Modern&quot;,&quot;local&quot;:&quot;large-language-models-llms-modern&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Dua fase pelatihan:&quot;,&quot;local&quot;:&quot;dua-fase-pelatihan&quot;,&quot;sections&quot;:[],&quot;depth&quot;:4},{&quot;title&quot;:&quot;Kemampuan umum LLM saat ini:&quot;,&quot;local&quot;:&quot;kemampuan-umum-llm-saat-ini&quot;,&quot;sections&quot;:[],&quot;depth&quot;:4}],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Model Sequence-to-Sequence&quot;,&quot;local&quot;:&quot;sequence-to-sequence-models&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Aplikasi praktis:&quot;,&quot;local&quot;:&quot;aplikasi-praktis&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Memilih Arsitektur yang Tepat&quot;,&quot;local&quot;:&quot;choosing-the-right-architecture&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Mekanisme Attention&quot;,&quot;local&quot;:&quot;attention-mechanisms&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Attention LSH&quot;,&quot;local&quot;:&quot;attention-lsh&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Attention Lokal&quot;,&quot;local&quot;:&quot;attention-lokal&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Positional Encoding Aksial&quot;,&quot;local&quot;:&quot;positional-encoding-aksial&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Kesimpulan&quot;,&quot;local&quot;:&quot;conclusion&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}">
 <link href="/docs/course/pr_1052/id/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
 <link rel="modulepreload" href="/docs/course/pr_1052/id/_app/immutable/entry/start.5943204e.js">
 <link rel="modulepreload" href="/docs/course/pr_1052/id/_app/immutable/chunks/scheduler.1d51f4c0.js">
 <link rel="modulepreload" href="/docs/course/pr_1052/id/_app/immutable/chunks/singletons.8dbecaac.js">
 <link rel="modulepreload" href="/docs/course/pr_1052/id/_app/immutable/chunks/index.fa8592cf.js">
 <link rel="modulepreload" href="/docs/course/pr_1052/id/_app/immutable/chunks/paths.547a3d4b.js">
 <link rel="modulepreload" href="/docs/course/pr_1052/id/_app/immutable/entry/app.b3380ec8.js">
 <link rel="modulepreload" href="/docs/course/pr_1052/id/_app/immutable/chunks/index.86f0ceea.js">
 <link rel="modulepreload" href="/docs/course/pr_1052/id/_app/immutable/nodes/0.b0267e25.js">
 <link rel="modulepreload" href="/docs/course/pr_1052/id/_app/immutable/chunks/each.e59479a4.js">
 <link rel="modulepreload" href="/docs/course/pr_1052/id/_app/immutable/nodes/10.15e74578.js">
 <link rel="modulepreload" href="/docs/course/pr_1052/id/_app/immutable/chunks/Youtube.d8ae3a4d.js">
 <link rel="modulepreload" href="/docs/course/pr_1052/id/_app/immutable/chunks/CourseFloatingBanner.9ea31445.js">
 <link rel="modulepreload" href="/docs/course/pr_1052/id/_app/immutable/chunks/getInferenceSnippets.462a0ab5.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Arsitektur Transformer&quot;,&quot;local&quot;:&quot;transformer-architectures&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Model Encoder&quot;,&quot;local&quot;:&quot;encoder-models&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Model Decoder&quot;,&quot;local&quot;:&quot;decoder-models&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Large Language Models (LLMs) Modern&quot;,&quot;local&quot;:&quot;large-language-models-llms-modern&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Dua fase pelatihan:&quot;,&quot;local&quot;:&quot;dua-fase-pelatihan&quot;,&quot;sections&quot;:[],&quot;depth&quot;:4},{&quot;title&quot;:&quot;Kemampuan umum LLM saat ini:&quot;,&quot;local&quot;:&quot;kemampuan-umum-llm-saat-ini&quot;,&quot;sections&quot;:[],&quot;depth&quot;:4}],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Model Sequence-to-Sequence&quot;,&quot;local&quot;:&quot;sequence-to-sequence-models&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Aplikasi praktis:&quot;,&quot;local&quot;:&quot;aplikasi-praktis&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Memilih Arsitektur yang Tepat&quot;,&quot;local&quot;:&quot;choosing-the-right-architecture&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Mekanisme Attention&quot;,&quot;local&quot;:&quot;attention-mechanisms&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Attention LSH&quot;,&quot;local&quot;:&quot;attention-lsh&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Attention Lokal&quot;,&quot;local&quot;:&quot;attention-lokal&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Positional Encoding Aksial&quot;,&quot;local&quot;:&quot;positional-encoding-aksial&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Kesimpulan&quot;,&quot;local&quot;:&quot;conclusion&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="flex space-x-1 absolute z-10 right-0 top-0"><a href="https://discuss.huggingface.co/t/chapter-1-questions" target="_blank"><img alt="Ask a Question" class="!m-0" src="https://img.shields.io/badge/Ask%20a%20question-ffcb4c.svg?logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgLTEgMTA0IDEwNiI+PGRlZnM+PHN0eWxlPi5jbHMtMXtmaWxsOiMyMzFmMjA7fS5jbHMtMntmaWxsOiNmZmY5YWU7fS5jbHMtM3tmaWxsOiMwMGFlZWY7fS5jbHMtNHtmaWxsOiMwMGE5NGY7fS5jbHMtNXtmaWxsOiNmMTVkMjI7fS5jbHMtNntmaWxsOiNlMzFiMjM7fTwvc3R5bGU+PC9kZWZzPjx0aXRsZT5EaXNjb3Vyc2VfbG9nbzwvdGl0bGU+PGcgaWQ9IkxheWVyXzIiPjxnIGlkPSJMYXllcl8zIj48cGF0aCBjbGFzcz0iY2xzLTEiIGQ9Ik01MS44NywwQzIzLjcxLDAsMCwyMi44MywwLDUxYzAsLjkxLDAsNTIuODEsMCw1Mi44MWw1MS44Ni0uMDVjMjguMTYsMCw1MS0yMy43MSw1MS01MS44N1M4MCwwLDUxLjg3LDBaIi8+PHBhdGggY2xhc3M9ImNscy0yIiBkPSJNNTIuMzcsMTkuNzRBMzEuNjIsMzEuNjIsMCwwLDAsMjQuNTgsNjYuNDFsLTUuNzIsMTguNEwzOS40LDgwLjE3YTMxLjYxLDMxLjYxLDAsMSwwLDEzLTYwLjQzWiIvPjxwYXRoIGNsYXNzPSJjbHMtMyIgZD0iTTc3LjQ1LDMyLjEyYTMxLjYsMzEuNiwwLDAsMS0zOC4wNSw0OEwxOC44Niw4NC44MmwyMC45MS0yLjQ3QTMxLjYsMzEuNiwwLDAsMCw3Ny40NSwzMi4xMloiLz48cGF0aCBjbGFzcz0iY2xzLTQiIGQ9Ik03MS42MywyNi4yOUEzMS42LDMxLjYsMCwwLDEsMzguOCw3OEwxOC44Niw4NC44MiwzOS40LDgwLjE3QTMxLjYsMzEuNiwwLDAsMCw3MS42MywyNi4yOVoiLz48cGF0aCBjbGFzcz0iY2xzLTUiIGQ9Ik0yNi40Nyw2Ny4xMWEzMS42MSwzMS42MSwwLDAsMSw1MS0zNUEzMS42MSwzMS42MSwwLDAsMCwyNC41OCw2Ni40MWwtNS43MiwxOC40WiIvPjxwYXRoIGNsYXNzPSJjbHMtNiIgZD0iTTI0LjU4LDY2LjQxQTMxLjYxLDMxLjYxLDAsMCwxLDcxLjYzLDI2LjI5YTMxLjYxLDMxLjYxLDAsMCwwLTQ5LDM5LjYzbC0zLjc2LDE4LjlaIi8+PC9nPjwvZz48L3N2Zz4="></a> </div> <h1 class="relative group"><a id="transformer-architectures" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformer-architectures"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Arsitektur Transformer</span></h1> <p data-svelte-h="svelte-1l9v1ww">Di bagian sebelumnya, kita telah memperkenalkan arsitektur umum Transformer dan menjelajahi bagaimana model-model ini menyelesaikan berbagai tugas. Sekarang, mari kita lihat lebih dekat tiga varian arsitektur utama dari model Transformer dan pahami kapan sebaiknya menggunakan masing-masing arsitektur.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">Ingat bahwa sebagian besar model Transformer menggunakan salah satu dari tiga arsitektur: encoder-only, decoder-only, atau encoder-decoder (sequence-to-sequence). Memahami perbedaan ini akan membantu kamu memilih model yang tepat untuk tugas tertentu.</div> <h2 class="relative group"><a id="encoder-models" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#encoder-models"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Model Encoder</span></h2> <iframe class="w-full xl:w-4/6 h-80" src="https://www.youtube-nocookie.com/embed/MUqNwgPjJvQ" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> <p data-svelte-h="svelte-1ufajjb">Model encoder hanya menggunakan bagian encoder dari arsitektur Transformer. Pada setiap tahap, lapisan atensi dapat mengakses seluruh kata dalam kalimat awal. Model ini sering disebut memiliki atensi “bi-directional”, dan dikenal sebagai <em>auto-encoding models</em>.</p> <p data-svelte-h="svelte-pr6tph">Pretraining biasanya dilakukan dengan merusak kalimat (misalnya dengan memask kata secara acak), lalu meminta model untuk memulihkan kalimat aslinya.</p> <p data-svelte-h="svelte-o9suvm">Model encoder sangat cocok untuk tugas-tugas yang membutuhkan pemahaman penuh terhadap kalimat, seperti klasifikasi kalimat, named entity recognition, dan question answering ekstraktif.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">Seperti yang kita lihat di [Bagaimana 🤗 Transformers menyelesaikan tugas](/chapter1/5), model encoder seperti BERT sangat baik dalam memahami teks karena dapat melihat konteks dari dua arah. Ini membuatnya ideal untuk tugas-tugas yang membutuhkan pemahaman menyeluruh terhadap input.</div> <p data-svelte-h="svelte-yukqbk">Contoh model:</p> <ul data-svelte-h="svelte-vv3q6"><li><a href="https://huggingface.co/docs/transformers/model_doc/bert" rel="nofollow">BERT</a></li> <li><a href="https://huggingface.co/docs/transformers/model_doc/distilbert" rel="nofollow">DistilBERT</a></li> <li><a href="https://huggingface.co/docs/transformers/en/model_doc/modernbert" rel="nofollow">ModernBERT</a></li></ul> <h2 class="relative group"><a id="decoder-models" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#decoder-models"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Model Decoder</span></h2> <iframe class="w-full xl:w-4/6 h-80" src="https://www.youtube-nocookie.com/embed/d_ixlCubqQw" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> <p data-svelte-h="svelte-xqd3ic">Model decoder hanya menggunakan bagian decoder dari Transformer. Setiap kata hanya dapat melihat kata-kata sebelumnya dalam kalimat. Model ini dikenal sebagai <em>auto-regressive models</em>.</p> <p data-svelte-h="svelte-1q03t8h">Pretraining biasanya melibatkan prediksi kata berikutnya dalam kalimat.</p> <p data-svelte-h="svelte-1419m7k">Model ini sangat cocok untuk tugas-tugas yang melibatkan generasi teks.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">Model decoder seperti GPT dirancang untuk menghasilkan teks dengan memprediksi satu token pada satu waktu. Seperti yang telah kita bahas di [Bagaimana 🤗 Transformers menyelesaikan tugas](/chapter1/5), model ini hanya dapat melihat token-token sebelumnya, yang membuatnya sangat baik untuk generasi teks kreatif, namun kurang ideal untuk tugas-tugas yang membutuhkan pemahaman dua arah.</div> <p data-svelte-h="svelte-yukqbk">Contoh model:</p> <ul data-svelte-h="svelte-b0ymuu"><li><a href="https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct" rel="nofollow">SmolLM Series</a></li> <li><a href="https://huggingface.co/docs/transformers/en/model_doc/llama4" rel="nofollow">LLaMA</a></li> <li><a href="https://huggingface.co/docs/transformers/main/en/model_doc/gemma3" rel="nofollow">Gemma</a></li> <li><a href="https://huggingface.co/deepseek-ai/DeepSeek-V3" rel="nofollow">DeepSeek V3</a></li></ul> <h3 class="relative group"><a id="large-language-models-llms-modern" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#large-language-models-llms-modern"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Large Language Models (LLMs) Modern</span></h3> <p data-svelte-h="svelte-ahcvnj">Sebagian besar LLM saat ini menggunakan arsitektur decoder-only. Model ini mengalami peningkatan pesat baik dari segi ukuran maupun kemampuan.</p> <h4 class="relative group"><a id="dua-fase-pelatihan" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#dua-fase-pelatihan"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Dua fase pelatihan:</span></h4> <ol data-svelte-h="svelte-7h9f3r"><li><strong>Pretraining</strong> – belajar memprediksi token berikutnya</li> <li><strong>Instruction tuning</strong> – dilatih untuk mengikuti instruksi manusia</li></ol> <h4 class="relative group"><a id="kemampuan-umum-llm-saat-ini" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#kemampuan-umum-llm-saat-ini"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Kemampuan umum LLM saat ini:</span></h4> <table data-svelte-h="svelte-1jyegsw"><thead><tr><th>Kemampuan</th> <th>Deskripsi</th> <th>Contoh</th></tr></thead> <tbody><tr><td>Generasi teks</td> <td>Menulis teks relevan &amp; koheren</td> <td>Menulis cerita, esai</td></tr> <tr><td>Ringkasan</td> <td>Meringkas dokumen panjang</td> <td>Ringkasan eksekutif</td></tr> <tr><td>Terjemahan</td> <td>Bahasa-ke-bahasa</td> <td>Inggris ke Spanyol</td></tr> <tr><td>QA</td> <td>Jawaban fakta</td> <td>“Apa ibu kota Prancis?”</td></tr> <tr><td>Generasi kode</td> <td>Membuat fungsi dari deskripsi</td> <td>Kode Python</td></tr> <tr><td>Penalaran</td> <td>Menyelesaikan langkah demi langkah</td> <td>Masalah logika</td></tr> <tr><td>Few-shot learning</td> <td>Belajar dari 2–3 contoh</td> <td>Klasifikasi teks</td></tr></tbody></table> <p data-svelte-h="svelte-xx5bxr">Anda dapat bereksperimen dengan LLM berbasis decoder langsung di peramban Anda melalui halaman repositori model di Hub. Berikut contoh dengan <a href="https://huggingface.co/openai-community/gpt2" rel="nofollow">GPT-2</a>, model open source klasik dari OpenAI:</p> <iframe src="https://huggingface.co/openai-community/gpt2" frameborder="0" width="100%" height="450"></iframe> <h2 class="relative group"><a id="sequence-to-sequence-models" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#sequence-to-sequence-models"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Model Sequence-to-Sequence</span></h2> <iframe class="w-full xl:w-4/6 h-80" src="https://www.youtube-nocookie.com/embed/0_4KEb08xrE" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> <p data-svelte-h="svelte-kg557u">Model encoder-decoder (atau <em>sequence-to-sequence</em>) menggunakan kedua bagian arsitektur Transformer. Encoder melihat seluruh input, sementara decoder hanya melihat token sebelumnya.</p> <p data-svelte-h="svelte-15g1hay">Contoh pretraining: T5 mengganti span teks dengan token [MASK], dan model diminta memulihkan teks tersebut.</p> <p data-svelte-h="svelte-1xjynbw">Model ini sangat cocok untuk tugas yang melibatkan transformasi teks ke teks, seperti ringkasan, terjemahan, atau QA generatif.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">Model seperti BART dan T5 menggabungkan kekuatan dua arsitektur: pemahaman input (encoder) dan generasi output (decoder). Ini membuat mereka unggul dalam tugas seperti ringkasan dan terjemahan.</div> <h3 class="relative group"><a id="aplikasi-praktis" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#aplikasi-praktis"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Aplikasi praktis:</span></h3> <table data-svelte-h="svelte-t7nmlg"><thead><tr><th>Aplikasi</th> <th>Deskripsi</th> <th>Contoh model</th></tr></thead> <tbody><tr><td>Terjemahan</td> <td>Bahasa-ke-bahasa</td> <td>Marian, T5</td></tr> <tr><td>Ringkasan</td> <td>Meringkas teks panjang</td> <td>BART, T5</td></tr> <tr><td>Data-to-text</td> <td>Data terstruktur → teks</td> <td>T5</td></tr> <tr><td>Koreksi tata bahasa</td> <td>Memperbaiki kalimat</td> <td>T5</td></tr> <tr><td>QA generatif</td> <td>Jawaban berdasarkan konteks</td> <td>BART, T5</td></tr></tbody></table> <iframe src="https://course-demos-speech-to-speech-translation.hf.space" frameborder="0" width="850" height="450"></iframe> <p data-svelte-h="svelte-yukqbk">Contoh model:</p> <ul data-svelte-h="svelte-sn73k4"><li><a href="https://huggingface.co/docs/transformers/model_doc/bart" rel="nofollow">BART</a></li> <li><a href="https://huggingface.co/docs/transformers/model_doc/mbart" rel="nofollow">mBART</a></li> <li><a href="https://huggingface.co/docs/transformers/model_doc/marian" rel="nofollow">Marian</a></li> <li><a href="https://huggingface.co/docs/transformers/model_doc/t5" rel="nofollow">T5</a></li></ul> <h2 class="relative group"><a id="choosing-the-right-architecture" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#choosing-the-right-architecture"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Memilih Arsitektur yang Tepat</span></h2> <table data-svelte-h="svelte-2em03l"><thead><tr><th>Tugas</th> <th>Arsitektur</th> <th>Contoh Model</th></tr></thead> <tbody><tr><td>Klasifikasi teks</td> <td>Encoder</td> <td>BERT, RoBERTa</td></tr> <tr><td>Generasi teks</td> <td>Decoder</td> <td>GPT, LLaMA</td></tr> <tr><td>Terjemahan</td> <td>Encoder-Decoder</td> <td>T5, BART</td></tr> <tr><td>Ringkasan</td> <td>Encoder-Decoder</td> <td>BART, T5</td></tr> <tr><td>Named Entity Recognition</td> <td>Encoder</td> <td>BERT, RoBERTa</td></tr> <tr><td>QA ekstraktif</td> <td>Encoder</td> <td>BERT</td></tr> <tr><td>QA generatif</td> <td>Decoder atau Seq2Seq</td> <td>GPT, T5</td></tr> <tr><td>Chatbot</td> <td>Decoder</td> <td>GPT, LLaMA</td></tr></tbody></table> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">Saat memilih model:
1. Apakah tugasmu membutuhkan pemahaman dua arah?
2. Apakah kamu menganalisis teks atau membuat teks baru?
3. Apakah kamu mentransformasi satu bentuk teks ke bentuk lain?
<p data-svelte-h="svelte-1ig8b93">Jawaban-jawaban ini akan membantumu memilih arsitektur yang tepat.</p></div> <h2 class="relative group"><a id="attention-mechanisms" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#attention-mechanisms"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Mekanisme Attention</span></h2> <p data-svelte-h="svelte-6lw55l">Sebagian besar model Transformer menggunakan full attention, dalam arti bahwa matriks attention berbentuk persegi. Ini bisa menjadi hambatan komputasi besar saat menangani teks panjang. Longformer dan Reformer adalah model yang mencoba menjadi lebih efisien dengan menggunakan versi sparsity (jarang) dari matriks attention untuk mempercepat pelatihan.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-3u9nd">Mekanisme attention standar memiliki kompleksitas komputasi sebesar O(n²), di mana n adalah panjang urutan. Ini menjadi masalah untuk urutan yang sangat panjang. Mekanisme attention khusus di bawah ini membantu mengatasi keterbatasan tersebut.</p></div> <h3 class="relative group"><a id="attention-lsh" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#attention-lsh"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Attention LSH</span></h3> <p data-svelte-h="svelte-10nubzf"><a href="https://huggingface.co/docs/transformers/model_doc/reformer" rel="nofollow">Reformer</a> menggunakan attention LSH (Locality-Sensitive Hashing). Dalam perhitungan softmax(QKᵗ), hanya elemen terbesar (pada dimensi softmax) dari matriks QKᵗ yang akan memberikan kontribusi berarti. Jadi, untuk setiap query <code>q</code> dalam Q, kita hanya mempertimbangkan key <code>k</code> dalam K yang dekat dengan <code>q</code>. Fungsi hash digunakan untuk menentukan apakah <code>q</code> dan <code>k</code> cukup dekat. Attention mask dimodifikasi untuk memblokir token saat ini (kecuali pada posisi pertama), karena query dan key akan sama (dan jadi sangat mirip). Karena hasil hash bisa bersifat acak, beberapa fungsi hash digunakan dalam praktik (ditentukan oleh parameter <code>n_rounds</code>) dan hasilnya dirata-rata.</p> <h3 class="relative group"><a id="attention-lokal" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#attention-lokal"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Attention Lokal</span></h3> <p data-svelte-h="svelte-10e141f"><a href="https://huggingface.co/docs/transformers/model_doc/longformer" rel="nofollow">Longformer</a> menggunakan attention lokal: sering kali, konteks lokal (misalnya, dua token di kiri dan kanan) sudah cukup untuk menentukan aksi dari sebuah token. Selain itu, dengan menumpuk lapisan attention yang memiliki jendela kecil (small window), lapisan terakhir akan memiliki bidang pengamatan (receptive field) yang lebih luas dari hanya token dalam jendela, sehingga memungkinkan model membangun representasi dari seluruh kalimat.</p> <p data-svelte-h="svelte-1hu1kbi">Beberapa token input yang telah dipilih juga diberi global attention: untuk token-token ini, matriks attention bisa mengakses semua token, dan proses ini bersifat simetris — semua token lain juga dapat mengakses token khusus tersebut (di samping token dalam jendela lokal mereka). Ini digambarkan dalam Gambar 2d pada makalah aslinya, dan contoh attention mask-nya dapat dilihat di bawah:</p> <div class="flex justify-center" data-svelte-h="svelte-d4kpls"><img scale="50 %" align="center" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/local_attention_mask.png"></div> <p data-svelte-h="svelte-eq2kba">Dengan menggunakan matriks attention yang memiliki parameter lebih sedikit, model dapat menerima input dengan panjang urutan yang lebih besar.</p> <h3 class="relative group"><a id="positional-encoding-aksial" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#positional-encoding-aksial"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Positional Encoding Aksial</span></h3> <p><a href="https://huggingface.co/docs/transformers/model_doc/reformer" rel="nofollow" data-svelte-h="svelte-1bpx0mm">Reformer</a> menggunakan <em data-svelte-h="svelte-e01b01">axial positional encodings</em>: dalam model Transformer tradisional, positional encoding <code data-svelte-h="svelte-vy3mw7">E</code> adalah matriks berukuran<!-- HTML_TAG_START --><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>l</mi></mrow><annotation encoding="application/x-tex">l</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal" style="margin-right:0.01968em;">l</span></span></span></span><!-- HTML_TAG_END --> ×<!-- HTML_TAG_START --><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>d</mi></mrow><annotation encoding="application/x-tex">d</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal">d</span></span></span></span><!-- HTML_TAG_END -->, dengan<!-- HTML_TAG_START --><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>l</mi></mrow><annotation encoding="application/x-tex">l</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal" style="margin-right:0.01968em;">l</span></span></span></span><!-- HTML_TAG_END --> sebagai panjang urutan dan<!-- HTML_TAG_START --><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>d</mi></mrow><annotation encoding="application/x-tex">d</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal">d</span></span></span></span><!-- HTML_TAG_END --> sebagai dimensi <em data-svelte-h="svelte-18fwkaj">hidden state</em>. Untuk teks yang sangat panjang, matriks ini bisa menjadi sangat besar dan memakan banyak ruang di GPU.</p> <p>Untuk mengatasinya, axial positional encodings memfaktorkan matriks besar <code data-svelte-h="svelte-vy3mw7">E</code> menjadi dua matriks yang lebih kecil, <code data-svelte-h="svelte-a9nvfq">E1</code> dan <code data-svelte-h="svelte-1mjm48j">E2</code>, dengan ukuran masing-masing<!-- HTML_TAG_START --><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>l</mi><mn>1</mn></msub><mo>×</mo><msub><mi>d</mi><mn>1</mn></msub></mrow><annotation encoding="application/x-tex">l_{1} \times d_{1}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0197em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">1</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">×</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">d</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">1</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span><!-- HTML_TAG_END --> dan<!-- HTML_TAG_START --><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>l</mi><mn>2</mn></msub><mo>×</mo><msub><mi>d</mi><mn>2</mn></msub></mrow><annotation encoding="application/x-tex">l_{2} \times d_{2}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0197em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">2</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">×</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">d</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">2</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span><!-- HTML_TAG_END -->, sedemikian hingga<!-- HTML_TAG_START --><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>l</mi><mn>1</mn></msub><mo>×</mo><msub><mi>l</mi><mn>2</mn></msub><mo>=</mo><mi>l</mi></mrow><annotation encoding="application/x-tex">l_{1} \times l_{2} = l</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0197em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">1</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">×</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0197em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">2</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal" style="margin-right:0.01968em;">l</span></span></span></span><!-- HTML_TAG_END --> dan<!-- HTML_TAG_START --><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>d</mi><mn>1</mn></msub><mo>+</mo><msub><mi>d</mi><mn>2</mn></msub><mo>=</mo><mi>d</mi></mrow><annotation encoding="application/x-tex">d_{1} + d_{2} = d</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">d</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">1</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">d</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">2</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal">d</span></span></span></span><!-- HTML_TAG_END -->. (Dengan hasil perkalian panjang urutan, ini jauh lebih kecil ukurannya.) Embedding untuk langkah waktu<!-- HTML_TAG_START --><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>j</mi></mrow><annotation encoding="application/x-tex">j</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.854em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.05724em;">j</span></span></span></span><!-- HTML_TAG_END --> dalam <code data-svelte-h="svelte-vy3mw7">E</code> diperoleh dengan menggabungkan (concatenate) embedding dari<!-- HTML_TAG_START --><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>j</mi><mi mathvariant="normal">%</mi><mi>l</mi><mn>1</mn></mrow><annotation encoding="application/x-tex">j \% l1</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.9444em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.05724em;">j</span><span class="mord">%</span><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="mord">1</span></span></span></span><!-- HTML_TAG_END --> di <code data-svelte-h="svelte-a9nvfq">E1</code> dan<!-- HTML_TAG_START --><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>j</mi><mi mathvariant="normal">/</mi><mi mathvariant="normal">/</mi><mi>l</mi><mn>1</mn></mrow><annotation encoding="application/x-tex">j // l1</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.05724em;">j</span><span class="mord">//</span><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="mord">1</span></span></span></span><!-- HTML_TAG_END --> di <code data-svelte-h="svelte-1mjm48j">E2</code>.</p> <h2 class="relative group"><a id="conclusion" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#conclusion"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Kesimpulan</span></h2> <p data-svelte-h="svelte-17fy409">Di bagian ini, kita telah mempelajari tiga arsitektur utama Transformer serta beberapa mekanisme attention khusus. Memahami perbedaan arsitektur ini sangat penting untuk memilih model yang tepat untuk tugas NLP tertentu.</p> <p data-svelte-h="svelte-i0clr5">Saat Anda melanjutkan kursus ini, Anda akan mendapatkan pengalaman langsung dengan berbagai arsitektur tersebut dan mempelajari cara <em>fine-tuning</em> untuk kebutuhan spesifik Anda. Di bagian selanjutnya, kita akan melihat beberapa keterbatasan dan bias yang ada dalam model-model ini — hal-hal penting yang perlu Anda pahami saat akan menggunakannya dalam dunia nyata.</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/course/blob/main/chapters/id/chapter1/6.mdx" target="_blank"><span data-svelte-h="svelte-1kd6by1">&lt;</span> <span data-svelte-h="svelte-x0xyl0">&gt;</span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p>
<script>
 {
 __sveltekit_wnfanp = {
 assets: "/docs/course/pr_1052/id",
 base: "/docs/course/pr_1052/id",
 env: {}
 };
 const element = document.currentScript.parentElement;
 const data = [null,null];
 Promise.all([
 import("/docs/course/pr_1052/id/_app/immutable/entry/start.5943204e.js"),
 import("/docs/course/pr_1052/id/_app/immutable/entry/app.b3380ec8.js")
 ]).then(([kit, app]) => {
 kit.start(app, element, {
 node_ids: [0, 10],
 data,
 form: null,
 error: null
 });
 });
 }
 </script>

Xet Storage Details

Size:
50.1 kB
·
Xet hash:
53aefa592892654ea08a92eaad3564928b71bdc176fd8d9ddcd31218a9ecd9d2

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.