Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / course /pr_1054 /id /chapter5 /5.html

rtrm

about 1 month ago

download

raw

83.2 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Membuat Dataset Sendiri","local":"creating-your-own-dataset","sections":[{"title":"Mendapatkan Data","local":"getting-the-data","sections":[],"depth":2},{"title":"Membersihkan Data","local":"cleaning-up-the-data","sections":[],"depth":2},{"title":"Menambahkan Data Komentar","local":"augmenting-the-dataset","sections":[],"depth":2},{"title":"Mengunggah dataset ke Hugging Face Hub","local":"uploading-the-dataset-to-the-hugging-face-hub","sections":[],"depth":2},{"title":"Membuat dataset card","local":"creating-a-dataset-card","sections":[],"depth":2}],"depth":1}">
	<link href="/docs/course/pr_1054/id/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/entry/start.4f92af03.js">
	<link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/scheduler.36a0863c.js">
	<link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/singletons.7dc7b9a4.js">
	<link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/index.733708bb.js">
	<link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/paths.cf097d06.js">
	<link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/entry/app.19cef1b6.js">
	<link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/index.156fee99.js">
	<link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/nodes/0.1203e4a0.js">
	<link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/nodes/40.03873a71.js">
	<link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/Tip.8a648467.js">
	<link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/Youtube.a5d6d567.js">
	<link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/CodeBlock.4cf998e6.js">
	<link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/CourseFloatingBanner.16bb8bff.js">
	<link rel="modulepreload" href="/docs/course/pr_1054/id/_app/immutable/chunks/getInferenceSnippets.472bc46d.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Membuat Dataset Sendiri","local":"creating-your-own-dataset","sections":[{"title":"Mendapatkan Data","local":"getting-the-data","sections":[],"depth":2},{"title":"Membersihkan Data","local":"cleaning-up-the-data","sections":[],"depth":2},{"title":"Menambahkan Data Komentar","local":"augmenting-the-dataset","sections":[],"depth":2},{"title":"Mengunggah dataset ke Hugging Face Hub","local":"uploading-the-dataset-to-the-hugging-face-hub","sections":[],"depth":2},{"title":"Membuat dataset card","local":"creating-a-dataset-card","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="creating-your-own-dataset" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#creating-your-own-dataset"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Membuat Dataset Sendiri</span></h1> <div class="flex space-x-1 absolute z-10 right-0 top-0"><a href="https://discuss.huggingface.co/t/chapter-5-questions" target="_blank"><img alt="Ask a Question" class="!m-0" src="https://img.shields.io/badge/Ask%20a%20question-ffcb4c.svg?logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgLTEgMTA0IDEwNiI+PGRlZnM+PHN0eWxlPi5jbHMtMXtmaWxsOiMyMzFmMjA7fS5jbHMtMntmaWxsOiNmZmY5YWU7fS5jbHMtM3tmaWxsOiMwMGFlZWY7fS5jbHMtNHtmaWxsOiMwMGE5NGY7fS5jbHMtNXtmaWxsOiNmMTVkMjI7fS5jbHMtNntmaWxsOiNlMzFiMjM7fTwvc3R5bGU+PC9kZWZzPjx0aXRsZT5EaXNjb3Vyc2VfbG9nbzwvdGl0bGU+PGcgaWQ9IkxheWVyXzIiPjxnIGlkPSJMYXllcl8zIj48cGF0aCBjbGFzcz0iY2xzLTEiIGQ9Ik01MS44NywwQzIzLjcxLDAsMCwyMi44MywwLDUxYzAsLjkxLDAsNTIuODEsMCw1Mi44MWw1MS44Ni0uMDVjMjguMTYsMCw1MS0yMy43MSw1MS01MS44N1M4MCwwLDUxLjg3LDBaIi8+PHBhdGggY2xhc3M9ImNscy0yIiBkPSJNNTIuMzcsMTkuNzRBMzEuNjIsMzEuNjIsMCwwLDAsMjQuNTgsNjYuNDFsLTUuNzIsMTguNEwzOS40LDgwLjE3YTMxLjYxLDMxLjYxLDAsMSwwLDEzLTYwLjQzWiIvPjxwYXRoIGNsYXNzPSJjbHMtMyIgZD0iTTc3LjQ1LDMyLjEyYTMxLjYsMzEuNiwwLDAsMS0zOC4wNSw0OEwxOC44Niw4NC44MmwyMC45MS0yLjQ3QTMxLjYsMzEuNiwwLDAsMCw3Ny40NSwzMi4xMloiLz48cGF0aCBjbGFzcz0iY2xzLTQiIGQ9Ik03MS42MywyNi4yOUEzMS42LDMxLjYsMCwwLDEsMzguOCw3OEwxOC44Niw4NC44MiwzOS40LDgwLjE3QTMxLjYsMzEuNiwwLDAsMCw3MS42MywyNi4yOVoiLz48cGF0aCBjbGFzcz0iY2xzLTUiIGQ9Ik0yNi40Nyw2Ny4xMWEzMS42MSwzMS42MSwwLDAsMSw1MS0zNUEzMS42MSwzMS42MSwwLDAsMCwyNC41OCw2Ni40MWwtNS43MiwxOC40WiIvPjxwYXRoIGNsYXNzPSJjbHMtNiIgZD0iTTI0LjU4LDY2LjQxQTMxLjYxLDMxLjYxLDAsMCwxLDcxLjYzLDI2LjI5YTMxLjYxLDMxLjYxLDAsMCwwLTQ5LDM5LjYzbC0zLjc2LDE4LjlaIi8+PC9nPjwvZz48L3N2Zz4="></a> <a href="https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter5/section5.ipynb" target="_blank"><img alt="Open In Colab" class="!m-0" src="https://colab.research.google.com/assets/colab-badge.svg"></a> <a href="https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter5/section5.ipynb" target="_blank"><img alt="Open In Studio Lab" class="!m-0" src="https://studiolab.sagemaker.aws/studiolab.svg"></a></div> <p data-svelte-h="svelte-g9wag9">Terkadang, dataset yang kamu butuhkan untuk membangun aplikasi NLP belum tersedia — jadi kamu perlu membuatnya sendiri. Di bagian ini, kita akan menunjukkan cara membuat korpus dari <a href="https://github.com/features/issues" rel="nofollow">GitHub issues</a>, yang sering digunakan untuk melacak bug atau permintaan fitur pada repositori GitHub. Dataset ini bisa digunakan untuk berbagai tujuan, seperti:</p> <ul data-svelte-h="svelte-1et9a7x"><li>Menganalisis berapa lama waktu yang dibutuhkan untuk menyelesaikan issues atau pull request</li> <li>Melatih <em>multilabel classifier</em> untuk menandai issue dengan metadata berdasarkan deskripsinya (misalnya: “bug”, “enhancement”, atau “question”)</li> <li>Membuat mesin pencari semantik untuk menemukan issue yang cocok dengan kueri pengguna</li></ul> <p data-svelte-h="svelte-128jp6r">Di sini kita akan fokus pada <strong>pembuatan korpus</strong>, dan di bagian selanjutnya kita akan membahas bagaimana membangun aplikasi pencarian semantik. Untuk membuatnya lebih menarik, kita akan menggunakan issue dari proyek open source yang populer: <strong>🤗 Datasets!</strong> Mari kita mulai dengan mendapatkan data dan mengeksplorasi informasi di dalamnya.</p> <h2 class="relative group"><a id="getting-the-data" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#getting-the-data"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Mendapatkan Data</span></h2> <p data-svelte-h="svelte-1017ywi">Kamu bisa menemukan semua issue di 🤗 Datasets dengan membuka tab <a href="https://github.com/huggingface/datasets/issues" rel="nofollow">Issues</a> pada repositorinya. Pada saat penulisan, terdapat <strong>331</strong> issue terbuka dan <strong>668</strong> yang sudah ditutup.</p> <div class="flex justify-center" data-svelte-h="svelte-jsxj4"><img src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/datasets-issues.png" alt="Issue GitHub dari repositori 🤗 Datasets." width="80%"></div> <p data-svelte-h="svelte-aiutzp">Jika kamu klik salah satu issue, kamu akan melihat informasi seperti judul, deskripsi, dan label yang menjelaskan isi issue tersebut. Contohnya terlihat pada gambar berikut:</p> <div class="flex justify-center" data-svelte-h="svelte-7oemdk"><img src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/datasets-issues-single.png" alt="Contoh issue di repositori 🤗 Datasets." width="80%"></div> <p data-svelte-h="svelte-3psj0z">Untuk mengunduh semua issue dari repositori, kita akan menggunakan <a href="https://docs.github.com/en/rest" rel="nofollow">GitHub REST API</a> dan mengakses endpoint <a href="https://docs.github.com/en/rest/reference/issues#list-repository-issues" rel="nofollow"><code>Issues</code></a>. Endpoint ini akan mengembalikan daftar objek JSON, masing-masing berisi berbagai informasi seperti judul, deskripsi, status, dan lainnya.</p> <p data-svelte-h="svelte-fl2yau">Cara paling mudah untuk mengambil issue adalah dengan library <code>requests</code>, yaitu pustaka standar Python untuk membuat HTTP request. Instal dulu dengan:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->!pip install requests<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-mta968">Setelah pustaka diinstal, Anda dapat melakukan permintaan GET ke endpoint <code>Issues</code> dengan memanggil fungsi <code>requests.get()</code>. Sebagai contoh, Anda bisa menjalankan perintah berikut untuk mengambil isu pertama pada halaman pertama:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> requests

	url = <span class="hljs-string">"https://api.github.com/repos/huggingface/datasets/issues?page=1&per_page=1"</span>
	response = requests.get(url)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1kg56eg">Objek <code>response</code> berisi banyak informasi termasuk status kode HTTP:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->response.status_code<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-number">200</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-spxljx">Kode status <code>200</code> berarti permintaan berhasil (Anda dapat menemukan daftar kode status HTTP yang mungkin <a href="https://en.wikipedia.org/wiki/List_of_HTTP_status_codes" rel="nofollow">di sini</a>). Namun, yang benar-benar ingin kita lihat adalah <em>payload</em>-nya, yang dapat diakses dalam berbagai format seperti byte, string, atau JSON. Karena kita tahu bahwa isu-isu kita berada dalam format JSON, mari kita periksa payload-nya sebagai berikut:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->response.json()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-128r092">Contoh output:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->[{<span class="hljs-string">'url'</span>: <span class="hljs-string">'https://api.github.com/repos/huggingface/datasets/issues/2792'</span>,
	<span class="hljs-string">'repository_url'</span>: <span class="hljs-string">'https://api.github.com/repos/huggingface/datasets'</span>,
	<span class="hljs-string">'labels_url'</span>: <span class="hljs-string">'https://api.github.com/repos/huggingface/datasets/issues/2792/labels{/name}'</span>,
	<span class="hljs-string">'comments_url'</span>: <span class="hljs-string">'https://api.github.com/repos/huggingface/datasets/issues/2792/comments'</span>,
	<span class="hljs-string">'events_url'</span>: <span class="hljs-string">'https://api.github.com/repos/huggingface/datasets/issues/2792/events'</span>,
	<span class="hljs-string">'html_url'</span>: <span class="hljs-string">'https://github.com/huggingface/datasets/pull/2792'</span>,
	<span class="hljs-string">'id'</span>: <span class="hljs-number">968650274</span>,
	<span class="hljs-string">'node_id'</span>: <span class="hljs-string">'MDExOlB1bGxSZXF1ZXN0NzEwNzUyMjc0'</span>,
	<span class="hljs-string">'number'</span>: <span class="hljs-number">2792</span>,
	<span class="hljs-string">'title'</span>: <span class="hljs-string">'Update GooAQ'</span>,
	<span class="hljs-string">'user'</span>: {<span class="hljs-string">'login'</span>: <span class="hljs-string">'bhavitvyamalik'</span>,
	<span class="hljs-string">'id'</span>: <span class="hljs-number">19718818</span>,
	<span class="hljs-string">'node_id'</span>: <span class="hljs-string">'MDQ6VXNlcjE5NzE4ODE4'</span>,
	<span class="hljs-string">'avatar_url'</span>: <span class="hljs-string">'https://avatars.githubusercontent.com/u/19718818?v=4'</span>,
	<span class="hljs-string">'gravatar_id'</span>: <span class="hljs-string">''</span>,
	<span class="hljs-string">'url'</span>: <span class="hljs-string">'https://api.github.com/users/bhavitvyamalik'</span>,
	<span class="hljs-string">'html_url'</span>: <span class="hljs-string">'https://github.com/bhavitvyamalik'</span>,
	<span class="hljs-string">'followers_url'</span>: <span class="hljs-string">'https://api.github.com/users/bhavitvyamalik/followers'</span>,
	<span class="hljs-string">'following_url'</span>: <span class="hljs-string">'https://api.github.com/users/bhavitvyamalik/following{/other_user}'</span>,
	<span class="hljs-string">'gists_url'</span>: <span class="hljs-string">'https://api.github.com/users/bhavitvyamalik/gists{/gist_id}'</span>,
	<span class="hljs-string">'starred_url'</span>: <span class="hljs-string">'https://api.github.com/users/bhavitvyamalik/starred{/owner}{/repo}'</span>,
	<span class="hljs-string">'subscriptions_url'</span>: <span class="hljs-string">'https://api.github.com/users/bhavitvyamalik/subscriptions'</span>,
	<span class="hljs-string">'organizations_url'</span>: <span class="hljs-string">'https://api.github.com/users/bhavitvyamalik/orgs'</span>,
	<span class="hljs-string">'repos_url'</span>: <span class="hljs-string">'https://api.github.com/users/bhavitvyamalik/repos'</span>,
	<span class="hljs-string">'events_url'</span>: <span class="hljs-string">'https://api.github.com/users/bhavitvyamalik/events{/privacy}'</span>,
	<span class="hljs-string">'received_events_url'</span>: <span class="hljs-string">'https://api.github.com/users/bhavitvyamalik/received_events'</span>,
	<span class="hljs-string">'type'</span>: <span class="hljs-string">'User'</span>,
	<span class="hljs-string">'site_admin'</span>: <span class="hljs-literal">False</span>},
	<span class="hljs-string">'labels'</span>: [],
	<span class="hljs-string">'state'</span>: <span class="hljs-string">'open'</span>,
	<span class="hljs-string">'locked'</span>: <span class="hljs-literal">False</span>,
	<span class="hljs-string">'assignee'</span>: <span class="hljs-literal">None</span>,
	<span class="hljs-string">'assignees'</span>: [],
	<span class="hljs-string">'milestone'</span>: <span class="hljs-literal">None</span>,
	<span class="hljs-string">'comments'</span>: <span class="hljs-number">1</span>,
	<span class="hljs-string">'created_at'</span>: <span class="hljs-string">'2021-08-12T11:40:18Z'</span>,
	<span class="hljs-string">'updated_at'</span>: <span class="hljs-string">'2021-08-12T12:31:17Z'</span>,
	<span class="hljs-string">'closed_at'</span>: <span class="hljs-literal">None</span>,
	<span class="hljs-string">'author_association'</span>: <span class="hljs-string">'CONTRIBUTOR'</span>,
	<span class="hljs-string">'active_lock_reason'</span>: <span class="hljs-literal">None</span>,
	<span class="hljs-string">'pull_request'</span>: {<span class="hljs-string">'url'</span>: <span class="hljs-string">'https://api.github.com/repos/huggingface/datasets/pulls/2792'</span>,
	<span class="hljs-string">'html_url'</span>: <span class="hljs-string">'https://github.com/huggingface/datasets/pull/2792'</span>,
	<span class="hljs-string">'diff_url'</span>: <span class="hljs-string">'https://github.com/huggingface/datasets/pull/2792.diff'</span>,
	<span class="hljs-string">'patch_url'</span>: <span class="hljs-string">'https://github.com/huggingface/datasets/pull/2792.patch'</span>},
	<span class="hljs-string">'body'</span>: <span class="hljs-string">'[GooAQ](https://github.com/allenai/gooaq) dataset was recently updated after splits were added for the same. This PR contains new updated GooAQ with train/val/test splits and updated README as well.'</span>,
	<span class="hljs-string">'performed_via_github_app'</span>: <span class="hljs-literal">None</span>}]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-zs0imu">Seperti yang bisa dilihat, kita mendapat banyak informasi, termasuk <code>title</code>, <code>body</code>, dan <code>number</code> dari issue, serta detail user yang membukanya.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-s1073s">✏️ <strong>Coba sendiri!</strong> Klik beberapa URL di payload JSON untuk melihat informasi apa saja yang terhubung dengan setiap issue GitHub.</p></div> <p data-svelte-h="svelte-1wr337y">Seperti dijelaskan dalam <a href="https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting" rel="nofollow">dokumentasi GitHub</a>, permintaan tanpa autentikasi dibatasi hingga 60 permintaan per jam. Meskipun Anda dapat meningkatkan parameter kueri <code>per_page</code> untuk mengurangi jumlah permintaan yang dikirim, Anda tetap akan mencapai batas tersebut pada repositori yang memiliki lebih dari beberapa ribu isu. Sebagai gantinya, Anda sebaiknya mengikuti <a href="https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token" rel="nofollow">instruksi GitHub</a> untuk membuat <em>personal access token</em> agar dapat meningkatkan batas permintaan menjadi 5.000 per jam. Setelah Anda memiliki token, Anda bisa menyertakannya sebagai bagian dari header permintaan:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->GITHUB_TOKEN = <span class="hljs-string">"xxx"</span> <span class="hljs-comment"># Ganti dengan token milikmu</span>
	headers = {<span class="hljs-string">"Authorization"</span>: <span class="hljs-string">f"token <span class="hljs-subst">{GITHUB_TOKEN}</span>"</span>}<!-- HTML_TAG_END --></pre></div> <div class="course-tip course-tip-orange bg-gradient-to-br dark:bg-gradient-to-r before:border-orange-500 dark:before:border-orange-800 from-orange-50 dark:from-gray-900 to-white dark:to-gray-950 border border-orange-50 text-orange-700 dark:text-gray-400"><p data-svelte-h="svelte-1uz5x2m">⚠️ Jangan pernah membagikan notebook yang berisi <code>GITHUB_TOKEN</code> Anda secara langsung. Kami menyarankan Anda menghapus sel terakhir setelah mengeksekusinya untuk menghindari kebocoran informasi secara tidak sengaja. Lebih baik lagi, simpan token tersebut di dalam file <em>.env</em> dan gunakan pustaka <a href="https://github.com/theskumar/python-dotenv" rel="nofollow"><code>python-dotenv</code></a> untuk memuatnya secara otomatis sebagai variabel lingkungan (environment variable).</p></div> <p data-svelte-h="svelte-1cj22fx">Berikut fungsi Python untuk mengunduh semua issue dari repositori GitHub:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> time
	<span class="hljs-keyword">import</span> math
	<span class="hljs-keyword">from</span> pathlib <span class="hljs-keyword">import</span> Path
	<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd
	<span class="hljs-keyword">from</span> tqdm.notebook <span class="hljs-keyword">import</span> tqdm


	<span class="hljs-keyword">def</span> <span class="hljs-title function_">fetch_issues</span>(<span class="hljs-params">
	owner=<span class="hljs-string">"huggingface"</span>,
	repo=<span class="hljs-string">"datasets"</span>,
	num_issues=<span class="hljs-number">10_000</span>,
	rate_limit=<span class="hljs-number">5_000</span>,
	issues_path=Path(<span class="hljs-params"><span class="hljs-string">"."</span></span>),
	</span>):
	<span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> issues_path.is_dir():
	issues_path.mkdir(exist_ok=<span class="hljs-literal">True</span>)

	batch = []
	all_issues = []
	per_page = <span class="hljs-number">100</span> <span class="hljs-comment"># Jumlah issue per halaman</span>
	num_pages = math.ceil(num_issues / per_page)
	base_url = <span class="hljs-string">"https://api.github.com/repos"</span>

	<span class="hljs-keyword">for</span> page <span class="hljs-keyword">in</span> tqdm(<span class="hljs-built_in">range</span>(num_pages)):
	query = <span class="hljs-string">f"issues?page=<span class="hljs-subst">{page}</span>&per_page=<span class="hljs-subst">{per_page}</span>&state=all"</span>
	issues = requests.get(<span class="hljs-string">f"<span class="hljs-subst">{base_url}</span>/<span class="hljs-subst">{owner}</span>/<span class="hljs-subst">{repo}</span>/<span class="hljs-subst">{query}</span>"</span>, headers=headers)
	batch.extend(issues.json())

	<span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(batch) > rate_limit <span class="hljs-keyword">and</span> <span class="hljs-built_in">len</span>(all_issues) < num_issues:
	all_issues.extend(batch)
	batch = [] <span class="hljs-comment"># Kosongkan batch</span>
	<span class="hljs-built_in">print</span>(<span class="hljs-string">"Batas permintaan GitHub tercapai. Menunggu 1 jam ..."</span>)
	time.sleep(<span class="hljs-number">60</span> * <span class="hljs-number">60</span> + <span class="hljs-number">1</span>)

	all_issues.extend(batch)
	df = pd.DataFrame.from_records(all_issues)
	df.to_json(<span class="hljs-string">f"<span class="hljs-subst">{issues_path}</span>/<span class="hljs-subst">{repo}</span>-issues.jsonl"</span>, orient=<span class="hljs-string">"records"</span>, lines=<span class="hljs-literal">True</span>)
	<span class="hljs-built_in">print</span>(
	<span class="hljs-string">f"Berhasil mengunduh semua issue untuk <span class="hljs-subst">{repo}</span>! Dataset disimpan di <span class="hljs-subst">{issues_path}</span>/<span class="hljs-subst">{repo}</span>-issues.jsonl"</span>
	)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-k4043s">Sekarang ketika kita memanggil <code>fetch_issues()</code>, fungsi ini akan mengunduh semua issue secara bertahap (batch) untuk menghindari melampaui batas permintaan per jam dari GitHub. Hasilnya akan disimpan dalam file <em>repository_name-issues.jsonl</em>, di mana setiap baris adalah objek JSON yang merepresentasikan satu issue. Mari kita gunakan fungsi ini untuk mengambil semua issue dari proyek 🤗 Datasets:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Bergantung pada koneksi internetmu, proses ini bisa memakan waktu beberapa menit...</span>
	fetch_issues()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-ivno77">Setelah issue berhasil diunduh, kita bisa memuatnya secara lokal menggunakan teknik yang telah kita pelajari di <a href="/course/chapter5/2">bagian 2</a>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->issues_dataset = load_dataset(<span class="hljs-string">"json"</span>, data_files=<span class="hljs-string">"datasets-issues.jsonl"</span>, split=<span class="hljs-string">"train"</span>)
	issues_dataset<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->Dataset({
	features: [<span class="hljs-string">'url'</span>, <span class="hljs-string">'repository_url'</span>, <span class="hljs-string">'labels_url'</span>, <span class="hljs-string">'comments_url'</span>, <span class="hljs-string">'events_url'</span>, <span class="hljs-string">'html_url'</span>, <span class="hljs-string">'id'</span>, <span class="hljs-string">'node_id'</span>, <span class="hljs-string">'number'</span>, <span class="hljs-string">'title'</span>, <span class="hljs-string">'user'</span>, <span class="hljs-string">'labels'</span>, <span class="hljs-string">'state'</span>, <span class="hljs-string">'locked'</span>, <span class="hljs-string">'assignee'</span>, <span class="hljs-string">'assignees'</span>, <span class="hljs-string">'milestone'</span>, <span class="hljs-string">'comments'</span>, <span class="hljs-string">'created_at'</span>, <span class="hljs-string">'updated_at'</span>, <span class="hljs-string">'closed_at'</span>, <span class="hljs-string">'author_association'</span>, <span class="hljs-string">'active_lock_reason'</span>, <span class="hljs-string">'pull_request'</span>, <span class="hljs-string">'body'</span>, <span class="hljs-string">'timeline_url'</span>, <span class="hljs-string">'performed_via_github_app'</span>],
	num_rows: <span class="hljs-number">3019</span>
	})<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-eb9xbh">Bagus! Kita telah berhasil membuat dataset pertama kita dari nol! Tapi, kenapa jumlahnya ribuan issue padahal di <a href="https://github.com/huggingface/datasets/issues" rel="nofollow">tab Issues</a> repositori 🤗 Datasets hanya terlihat sekitar 1.000 saja 🤔? Seperti yang dijelaskan dalam <a href="https://docs.github.com/en/rest/reference/issues#list-issues-assigned-to-the-authenticated-user" rel="nofollow">dokumentasi GitHub</a>:</p> <blockquote data-svelte-h="svelte-196gbda"><p>API REST GitHub v3 menganggap setiap <em>pull request</em> sebagai sebuah issue, tetapi tidak semua issue adalah <em>pull request</em>. Oleh karena itu, endpoint “Issues” dapat mengembalikan baik issue maupun pull request. Kamu dapat membedakan pull request dengan adanya kunci <code>pull_request</code>. Perlu diketahui bahwa <code>id</code> dari pull request yang dikembalikan dari endpoint “Issues” adalah ID issue.</p></blockquote> <p data-svelte-h="svelte-ewf5fi">Karena konten antara issue dan pull request berbeda, mari kita lakukan sedikit pembersihan agar bisa membedakannya.</p> <h2 class="relative group"><a id="cleaning-up-the-data" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#cleaning-up-the-data"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Membersihkan Data</span></h2> <p data-svelte-h="svelte-1g5n8m9">Berdasarkan dokumentasi di atas, kolom <code>pull_request</code> dapat digunakan untuk membedakan antara issue dan pull request. Mari kita lihat beberapa contoh acak untuk membandingkan. Sama seperti di <a href="/course/chapter5/3">bagian 3</a>, kita akan menggunakan <code>Dataset.shuffle()</code> dan <code>Dataset.select()</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->sample = issues_dataset.shuffle(seed=<span class="hljs-number">666</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">3</span>))

	<span class="hljs-comment"># Cetak URL dan entri pull request-nya</span>
	<span class="hljs-keyword">for</span> url, pr <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(sample[<span class="hljs-string">"html_url"</span>], sample[<span class="hljs-string">"pull_request"</span>]):
	<span class="hljs-built_in">print</span>(<span class="hljs-string">f">> URL: <span class="hljs-subst">{url}</span>"</span>)
	<span class="hljs-built_in">print</span>(<span class="hljs-string">f">> Pull request: <span class="hljs-subst">{pr}</span>\n"</span>)<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->>> URL: https://github.com/huggingface/datasets/pull/<span class="hljs-number">850</span>
	>> Pull request: {...}

	>> URL: https://github.com/huggingface/datasets/issues/<span class="hljs-number">2773</span>
	>> Pull request: <span class="hljs-literal">None</span>

	>> URL: https://github.com/huggingface/datasets/pull/<span class="hljs-number">783</span>
	>> Pull request: {...}<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-e4upaq">Dari sini kita bisa lihat bahwa setiap pull request memiliki berbagai URL terkait, sementara issue biasa memiliki nilai <code>None</code>. Berdasarkan hal ini, kita bisa menambahkan kolom baru <code>is_pull_request</code> untuk membedakan keduanya:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->issues_dataset = issues_dataset.<span class="hljs-built_in">map</span>(
	<span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">"is_pull_request"</span>: <span class="hljs-literal">False</span> <span class="hljs-keyword">if</span> x[<span class="hljs-string">"pull_request"</span>] <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">else</span> <span class="hljs-literal">True</span>}
	)<!-- HTML_TAG_END --></pre></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-18t2m75">✏️ <strong>Coba sendiri!</strong> Hitung rata-rata waktu yang dibutuhkan untuk menutup issue pada proyek 🤗 Datasets. Kamu bisa menggunakan <code>Dataset.filter()</code> untuk menyaring pull request dan issue yang masih terbuka, lalu gunakan <code>Dataset.set_format()</code> untuk mengubah dataset menjadi <code>DataFrame</code> agar lebih mudah memproses kolom <code>created_at</code> dan <code>closed_at</code>. Untuk tantangan tambahan, hitung juga rata-rata waktu penutupan pull request.</p></div> <p data-svelte-h="svelte-rkgleh">Meskipun kita bisa lanjut membersihkan dataset dengan menghapus atau mengganti nama kolom, umumnya lebih baik menyimpan dataset dalam bentuk mentah agar bisa digunakan dalam berbagai aplikasi.</p> <p data-svelte-h="svelte-1pjgkrk">Sebelum kita unggah dataset ini ke Hugging Face Hub, ada satu hal lagi yang perlu kita tambahkan: <strong>komentar</strong> pada setiap issue atau pull request. Mari kita tambahkan menggunakan — seperti yang bisa ditebak — GitHub REST API!</p> <h2 class="relative group"><a id="augmenting-the-dataset" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#augmenting-the-dataset"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Menambahkan Data Komentar</span></h2> <p data-svelte-h="svelte-1rygm07">Seperti terlihat pada gambar berikut, komentar pada issue atau pull request memberikan informasi yang sangat kaya, terutama jika kita ingin membuat mesin pencari untuk menjawab pertanyaan pengguna tentang pustaka ini.</p> <div class="flex justify-center" data-svelte-h="svelte-1socmpm"><img src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/datasets-issues-comment.png" alt="Komentar pada salah satu issue 🤗 Datasets." width="80%"></div> <p data-svelte-h="svelte-8a35wx">GitHub REST API menyediakan endpoint <a href="https://docs.github.com/en/rest/reference/issues#list-issue-comments" rel="nofollow"><code>Comments</code></a> yang mengembalikan semua komentar terkait dengan nomor issue tertentu. Mari kita uji endpoint ini:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->issue_number = <span class="hljs-number">2792</span>
	url = <span class="hljs-string">f"https://api.github.com/repos/huggingface/datasets/issues/<span class="hljs-subst">{issue_number}</span>/comments"</span>
	response = requests.get(url, headers=headers)
	response.json()<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->[{<span class="hljs-string">'url'</span>: <span class="hljs-string">'https://api.github.com/repos/huggingface/datasets/issues/comments/897594128'</span>,
	<span class="hljs-string">'html_url'</span>: <span class="hljs-string">'https://github.com/huggingface/datasets/pull/2792#issuecomment-897594128'</span>,
	<span class="hljs-string">'issue_url'</span>: <span class="hljs-string">'https://api.github.com/repos/huggingface/datasets/issues/2792'</span>,
	<span class="hljs-string">'id'</span>: <span class="hljs-number">897594128</span>,
	<span class="hljs-string">'node_id'</span>: <span class="hljs-string">'IC_kwDODunzps41gDMQ'</span>,
	<span class="hljs-string">'user'</span>: {<span class="hljs-string">'login'</span>: <span class="hljs-string">'bhavitvyamalik'</span>,
	<span class="hljs-string">'id'</span>: <span class="hljs-number">19718818</span>,
	<span class="hljs-string">'node_id'</span>: <span class="hljs-string">'MDQ6VXNlcjE5NzE4ODE4'</span>,
	<span class="hljs-string">'avatar_url'</span>: <span class="hljs-string">'https://avatars.githubusercontent.com/u/19718818?v=4'</span>,
	<span class="hljs-string">'gravatar_id'</span>: <span class="hljs-string">''</span>,
	<span class="hljs-string">'url'</span>: <span class="hljs-string">'https://api.github.com/users/bhavitvyamalik'</span>,
	<span class="hljs-string">'html_url'</span>: <span class="hljs-string">'https://github.com/bhavitvyamalik'</span>,
	<span class="hljs-string">'followers_url'</span>: <span class="hljs-string">'https://api.github.com/users/bhavitvyamalik/followers'</span>,
	<span class="hljs-string">'following_url'</span>: <span class="hljs-string">'https://api.github.com/users/bhavitvyamalik/following{/other_user}'</span>,
	<span class="hljs-string">'gists_url'</span>: <span class="hljs-string">'https://api.github.com/users/bhavitvyamalik/gists{/gist_id}'</span>,
	<span class="hljs-string">'starred_url'</span>: <span class="hljs-string">'https://api.github.com/users/bhavitvyamalik/starred{/owner}{/repo}'</span>,
	<span class="hljs-string">'subscriptions_url'</span>: <span class="hljs-string">'https://api.github.com/users/bhavitvyamalik/subscriptions'</span>,
	<span class="hljs-string">'organizations_url'</span>: <span class="hljs-string">'https://api.github.com/users/bhavitvyamalik/orgs'</span>,
	<span class="hljs-string">'repos_url'</span>: <span class="hljs-string">'https://api.github.com/users/bhavitvyamalik/repos'</span>,
	<span class="hljs-string">'events_url'</span>: <span class="hljs-string">'https://api.github.com/users/bhavitvyamalik/events{/privacy}'</span>,
	<span class="hljs-string">'received_events_url'</span>: <span class="hljs-string">'https://api.github.com/users/bhavitvyamalik/received_events'</span>,
	<span class="hljs-string">'type'</span>: <span class="hljs-string">'User'</span>,
	<span class="hljs-string">'site_admin'</span>: <span class="hljs-literal">False</span>},
	<span class="hljs-string">'created_at'</span>: <span class="hljs-string">'2021-08-12T12:21:52Z'</span>,
	<span class="hljs-string">'updated_at'</span>: <span class="hljs-string">'2021-08-12T12:31:17Z'</span>,
	<span class="hljs-string">'author_association'</span>: <span class="hljs-string">'CONTRIBUTOR'</span>,
	<span class="hljs-string">'body'</span>: <span class="hljs-string">"@albertvillanova my tests are failing here:\r\n```\r\ndataset_name = 'gooaq'\r\n\r\n def test_load_dataset(self, dataset_name):\r\n configs = self.dataset_tester.load_all_configs(dataset_name, is_local=True)[:1]\r\n> self.dataset_tester.check_load_dataset(dataset_name, configs, is_local=True, use_local_dummy_data=True)\r\n\r\ntests/test_dataset_common.py:234: \r\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \r\ntests/test_dataset_common.py:187: in check_load_dataset\r\n self.parent.assertTrue(len(dataset[split]) > 0)\r\nE AssertionError: False is not true\r\n```\r\nWhen I try loading dataset on local machine it works fine. Any suggestions on how can I avoid this error?"</span>,
	<span class="hljs-string">'performed_via_github_app'</span>: <span class="hljs-literal">None</span>}]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-wwcygy">Kita dapat melihat bahwa komentar disimpan dalam field <code>body</code>, jadi mari kita tulis fungsi sederhana yang mengembalikan semua komentar yang terkait dengan sebuah issue dengan mengambil isi <code>body</code> dari setiap elemen dalam <code>response.json()</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">def</span> <span class="hljs-title function_">get_comments</span>(<span class="hljs-params">issue_number</span>):
	url = <span class="hljs-string">f"https://api.github.com/repos/huggingface/datasets/issues/<span class="hljs-subst">{issue_number}</span>/comments"</span>
	response = requests.get(url, headers=headers)
	<span class="hljs-keyword">return</span> [r[<span class="hljs-string">"body"</span>] <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> response.json()]


	<span class="hljs-comment"># Uji apakah fungsi kita bekerja sebagaimana mestinya</span>
	get_comments(<span class="hljs-number">2792</span>)<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->[<span class="hljs-string">"@albertvillanova tes saya gagal di sini:\r\n```\r\ndataset_name = 'gooaq'\r\n\r\n def test_load_dataset(self, dataset_name):\r\n configs = self.dataset_tester.load_all_configs(dataset_name, is_local=True)[:1]\r\n> self.dataset_tester.check_load_dataset(dataset_name, configs, is_local=True, use_local_dummy_data=True)\r\n\r\ntests/test_dataset_common.py:234: \r\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \r\ntests/test_dataset_common.py:187: in check_load_dataset\r\n self.parent.assertTrue(len(dataset[split]) > 0)\r\nE AssertionError: False is not true\r\n```\r\nSaat saya mencoba memuat dataset di mesin lokal, semuanya berjalan baik. Ada saran bagaimana cara menghindari error ini?"</span>]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-wdg0k4">Ini terlihat bagus, jadi mari kita gunakan <code>Dataset.map()</code> untuk menambahkan kolom <code>comments</code> baru ke setiap issue dalam dataset kita:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Tergantung pada koneksi internet Anda, ini bisa memakan waktu beberapa menit...</span>
	issues_with_comments_dataset = issues_dataset.<span class="hljs-built_in">map</span>(
	<span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">"comments"</span>: get_comments(x[<span class="hljs-string">"number"</span>])}
	)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-10y8j8w">Langkah terakhir adalah mengunggah dataset kita ke Hub. Mari kita lihat bagaimana caranya.</p> <h2 class="relative group"><a id="uploading-the-dataset-to-the-hugging-face-hub" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#uploading-the-dataset-to-the-hugging-face-hub"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Mengunggah dataset ke Hugging Face Hub</span></h2> <iframe class="w-full xl:w-4/6 h-80" src="https://www.youtube-nocookie.com/embed/HaN6qCr_Afc" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> <p data-svelte-h="svelte-uy6ydo">Sekarang kita memiliki dataset yang sudah ditambah dengan komentar, saatnya untuk mengunggahnya ke Hub agar bisa dibagikan kepada komunitas! Mengunggah dataset sangat mudah: sama seperti model dan tokenizer dari 🤗 Transformers, kita bisa menggunakan metode <code>push_to_hub()</code> untuk mengunggah dataset. Untuk melakukannya, kita memerlukan token autentikasi, yang bisa didapatkan dengan login ke Hugging Face Hub menggunakan fungsi <code>notebook_login()</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

	notebook_login()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1jndr71">Ini akan menampilkan widget tempat Anda bisa memasukkan nama pengguna dan kata sandi, dan token API akan disimpan di <em>~/.huggingface/token</em>. Jika Anda menjalankan kode di terminal, Anda bisa login lewat CLI:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->huggingface-cli login<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1hejsrr">Setelah itu, kita bisa mengunggah dataset kita dengan menjalankan:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->issues_with_comments_dataset.push_to_hub(<span class="hljs-string">"github-issues"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1rxnds9">Mulai dari sini, siapa pun bisa mengunduh dataset ini cukup dengan memberikan ID repositori sebagai argumen <code>path</code> ke <code>load_dataset()</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->remote_dataset = load_dataset(<span class="hljs-string">"lewtun/github-issues"</span>, split=<span class="hljs-string">"train"</span>)
	remote_dataset<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->Dataset({
	features: [<span class="hljs-string">'url'</span>, <span class="hljs-string">'repository_url'</span>, <span class="hljs-string">'labels_url'</span>, <span class="hljs-string">'comments_url'</span>, <span class="hljs-string">'events_url'</span>, <span class="hljs-string">'html_url'</span>, <span class="hljs-string">'id'</span>, <span class="hljs-string">'node_id'</span>, <span class="hljs-string">'number'</span>, <span class="hljs-string">'title'</span>, <span class="hljs-string">'user'</span>, <span class="hljs-string">'labels'</span>, <span class="hljs-string">'state'</span>, <span class="hljs-string">'locked'</span>, <span class="hljs-string">'assignee'</span>, <span class="hljs-string">'assignees'</span>, <span class="hljs-string">'milestone'</span>, <span class="hljs-string">'comments'</span>, <span class="hljs-string">'created_at'</span>, <span class="hljs-string">'updated_at'</span>, <span class="hljs-string">'closed_at'</span>, <span class="hljs-string">'author_association'</span>, <span class="hljs-string">'active_lock_reason'</span>, <span class="hljs-string">'pull_request'</span>, <span class="hljs-string">'body'</span>, <span class="hljs-string">'performed_via_github_app'</span>, <span class="hljs-string">'is_pull_request'</span>],
	num_rows: <span class="hljs-number">2855</span>
	})<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-s6j1co">Keren, kita sudah mengunggah dataset ke Hub dan sekarang bisa digunakan oleh orang lain! Masih ada satu hal penting yang perlu dilakukan: menambahkan <em>dataset card</em> yang menjelaskan bagaimana korpus dibuat dan menyediakan informasi berguna lainnya untuk komunitas.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-x5bji1">💡 Anda juga bisa mengunggah dataset ke Hugging Face Hub langsung dari terminal dengan menggunakan <code>huggingface-cli</code> dan sedikit perintah Git. Lihat <a href="https://huggingface.co/docs/datasets/share#share-a-dataset-using-the-cli" rel="nofollow">panduan 🤗 Datasets</a> untuk detailnya.</p></div> <h2 class="relative group"><a id="creating-a-dataset-card" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#creating-a-dataset-card"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Membuat dataset card</span></h2> <p data-svelte-h="svelte-1nu1723">Dataset yang terdokumentasi dengan baik lebih mungkin bermanfaat bagi orang lain (termasuk diri Anda di masa depan!), karena mereka menyediakan konteks yang memungkinkan pengguna memutuskan apakah dataset tersebut relevan untuk tugas mereka dan mengevaluasi potensi bias atau risiko dalam penggunaannya.</p> <p data-svelte-h="svelte-rdszv7">Di Hugging Face Hub, informasi ini disimpan dalam file <em>README.md</em> di setiap repositori dataset. Ada dua langkah utama yang perlu Anda lakukan sebelum membuat file ini:</p> <ol data-svelte-h="svelte-1lbja9p"><li>Gunakan <a href="https://huggingface.co/datasets/tagging/" rel="nofollow">aplikasi <code>datasets-tagging</code></a> untuk membuat tag metadata dalam format YAML. Tag ini digunakan untuk berbagai fitur pencarian di Hugging Face Hub dan memastikan dataset Anda dapat dengan mudah ditemukan oleh komunitas. Karena kita membuat dataset kustom di sini, Anda perlu mengkloning repositori <code>datasets-tagging</code> dan menjalankan aplikasi secara lokal. Berikut tampilan antarmukanya:</li></ol> <div class="flex justify-center" data-svelte-h="svelte-sd2n2n"><img src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/datasets-tagger.png" alt="Antarmuka 'datasets-tagging'." width="80%"></div> <ol start="2" data-svelte-h="svelte-rp0o8"><li>Baca <a href="https://github.com/huggingface/datasets/blob/master/templates/README_guide.md" rel="nofollow">panduan 🤗 Datasets</a> tentang cara membuat dataset card yang informatif dan gunakan sebagai template.</li></ol> <p data-svelte-h="svelte-16krxpl">Anda bisa membuat file <em>README.md</em> langsung di Hub, dan Anda bisa menemukan contoh template dataset card di repositori dataset <code>lewtun/github-issues</code>. Berikut adalah tangkapan layar dari dataset card yang telah diisi:</p> <div class="flex justify-center" data-svelte-h="svelte-1l5z6p3"><img src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/dataset-card.png" alt="Sebuah dataset card." width="80%"></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-l7y93x">✏️ <strong>Coba sendiri!</strong> Gunakan aplikasi <code>dataset-tagging</code> dan <a href="https://github.com/huggingface/datasets/blob/master/templates/README_guide.md" rel="nofollow">panduan 🤗 Datasets</a> untuk menyelesaikan file <em>README.md</em> untuk dataset GitHub issues Anda.</p></div> <p data-svelte-h="svelte-wuebg">Itu saja! Kita telah melihat dalam bagian ini bahwa membuat dataset yang bagus bisa jadi cukup rumit, tapi untungnya proses mengunggah dan membagikannya ke komunitas sangatlah mudah. Di bagian selanjutnya kita akan menggunakan dataset baru ini untuk membuat mesin pencari semantik dengan 🤗 Datasets yang dapat mencocokkan pertanyaan ke issue dan komentar yang paling relevan.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-118coqs">✏️ <strong>Coba sendiri!</strong> Ikuti langkah-langkah di bagian ini untuk membuat dataset GitHub issues dari pustaka open source favorit Anda (pilih yang selain 🤗 Datasets, tentu saja!). Untuk poin tambahan, latih model klasifikasi multilabel untuk memprediksi tag yang ada di field <code>labels</code>.</p></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/course/blob/main/chapters/id/chapter5/5.mdx" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_ojy514 = {
	assets: "/docs/course/pr_1054/id",
	base: "/docs/course/pr_1054/id",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/course/pr_1054/id/_app/immutable/entry/start.4f92af03.js"),
	import("/docs/course/pr_1054/id/_app/immutable/entry/app.19cef1b6.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 40],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 83.2 kB
Xet hash:: 0ff4c95bfc3020c76a297b4915ef17095009b6c5727e969ba088fce53952a1e0

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.