Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Entrenar un modelo de lenguaje causal desde cero","local":"entrenar-un-modelo-de-lenguaje-causal-desde-cero","sections":[{"title":"Recopilar los datos","local":"recopilar-los-datos","sections":[],"depth":2},{"title":"Preparar el conjunto de datos","local":"preparar-el-conjunto-de-datos","sections":[],"depth":2},{"title":"Inicializar un nuevo modelo","local":"inicializar-un-nuevo-modelo","sections":[],"depth":2},{"title":"Generación de código con un pipeline","local":"generacion-de-codigo-con-un-pipeline","sections":[],"depth":2},{"title":"Entrenamiento con 🤗 Accelerate","local":"entrenamiento-con-accelerate","sections":[],"depth":2}],"depth":1}"/> | |
| <link href="/docs/course/pr_1352/es/_app/immutable/entry/start.DV9gcfqJ.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1352/es/_app/immutable/chunks/DfLptDcl.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1352/es/_app/immutable/chunks/BA_dZbsI.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1352/es/_app/immutable/entry/app.C68Ze5cV.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1352/es/_app/immutable/chunks/C0jRxzmK.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1352/es/_app/immutable/chunks/DvbUaM1Q.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1352/es/_app/immutable/chunks/BtJu3x8T.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1352/es/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1352/es/_app/immutable/nodes/0.BzDzkAzS.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1352/es/_app/immutable/chunks/C3nIf8Go.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1352/es/_app/immutable/nodes/45.CiLK4I2U.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1352/es/_app/immutable/chunks/DX9wbpOW.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1352/es/_app/immutable/chunks/b7zOzquf.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1352/es/_app/immutable/chunks/r2R_RyXK.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1352/es/_app/immutable/chunks/C-UOVayx.js" rel="modulepreload"> | |
| <link href="/docs/course/pr_1352/es/_app/immutable/chunks/Khighb8g.js" rel="modulepreload"> | |
| <!--gfnkp8--><meta name="hf:doc:metadata" content="{"title":"Entrenar un modelo de lenguaje causal desde cero","local":"entrenar-un-modelo-de-lenguaje-causal-desde-cero","sections":[{"title":"Recopilar los datos","local":"recopilar-los-datos","sections":[],"depth":2},{"title":"Preparar el conjunto de datos","local":"preparar-el-conjunto-de-datos","sections":[],"depth":2},{"title":"Inicializar un nuevo modelo","local":"inicializar-un-nuevo-modelo","sections":[],"depth":2},{"title":"Generación de código con un pipeline","local":"generacion-de-codigo-con-un-pipeline","sections":[],"depth":2},{"title":"Entrenamiento con 🤗 Accelerate","local":"entrenamiento-con-accelerate","sections":[],"depth":2}],"depth":1}"/><!----> | |
| <link href="/docs/course/pr_1352/es/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="bg-white leading-none border border-gray-100 rounded-lg flex p-0.5 w-56 text-sm mb-4"><!--[--><a class="flex justify-center flex-1 py-1.5 px-2.5 focus:outline-none !no-underline rounded-l bg-red-50 dark:bg-transparent text-red-600" href="?fw=pt"><!--[--><svg class="mr-1.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><defs><clipPath id="a"><rect x="3.05" y="0.5" width="25.73" height="31" fill="none"></rect></clipPath></defs><g clip-path="url(#a)"><path d="M24.94,9.51a12.81,12.81,0,0,1,0,18.16,12.68,12.68,0,0,1-18,0,12.81,12.81,0,0,1,0-18.16l9-9V5l-.84.83-6,6a9.58,9.58,0,1,0,13.55,0ZM20.44,9a1.68,1.68,0,1,1,1.67-1.67A1.68,1.68,0,0,1,20.44,9Z" fill="#ee4c2c"></path></g></svg><!--]--> Pytorch</a><a class="flex justify-center flex-1 py-1.5 px-2.5 focus:outline-none !no-underline rounded-r text-gray-500 filter grayscale" href="?fw=tf"><!--[--><svg class="mr-1.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="0.94em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 274"><path d="M145.726 42.065v42.07l72.861 42.07v-42.07l-72.86-42.07zM0 84.135v42.07l36.43 21.03V105.17L0 84.135zm109.291 21.035l-36.43 21.034v126.2l36.43 21.035v-84.135l36.435 21.035v-42.07l-36.435-21.034V105.17z" fill="#E55B2D"></path><path d="M145.726 42.065L36.43 105.17v42.065l72.861-42.065v42.065l36.435-21.03v-84.14zM255.022 63.1l-36.435 21.035v42.07l36.435-21.035V63.1zm-72.865 84.135l-36.43 21.035v42.07l36.43-21.036v-42.07zm-36.43 63.104l-36.436-21.035v84.135l36.435-21.035V210.34z" fill="#ED8E24"></path><path d="M145.726 0L0 84.135l36.43 21.035l109.296-63.105l72.861 42.07L255.022 63.1L145.726 0zm0 126.204l-36.435 21.03l36.435 21.036l36.43-21.035l-36.43-21.03z" fill="#F8BF3C"></path></svg><!--]--> TensorFlow</a><!--]--></div><!----> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="entrenar-un-modelo-de-lenguaje-causal-desde-cero" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#entrenar-un-modelo-de-lenguaje-causal-desde-cero"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Entrenar un modelo de lenguaje causal desde cero</span></h1><!--]--><!----> <!--[0--><div class="flex space-x-1 absolute z-10 right-0 top-0"><a href="https://discuss.huggingface.co/t/chapter-7-questions" target="_blank"><img alt="Ask a Question" class="!m-0" src="https://img.shields.io/badge/Ask%20a%20question-ffcb4c.svg?logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgLTEgMTA0IDEwNiI+PGRlZnM+PHN0eWxlPi5jbHMtMXtmaWxsOiMyMzFmMjA7fS5jbHMtMntmaWxsOiNmZmY5YWU7fS5jbHMtM3tmaWxsOiMwMGFlZWY7fS5jbHMtNHtmaWxsOiMwMGE5NGY7fS5jbHMtNXtmaWxsOiNmMTVkMjI7fS5jbHMtNntmaWxsOiNlMzFiMjM7fTwvc3R5bGU+PC9kZWZzPjx0aXRsZT5EaXNjb3Vyc2VfbG9nbzwvdGl0bGU+PGcgaWQ9IkxheWVyXzIiPjxnIGlkPSJMYXllcl8zIj48cGF0aCBjbGFzcz0iY2xzLTEiIGQ9Ik01MS44NywwQzIzLjcxLDAsMCwyMi44MywwLDUxYzAsLjkxLDAsNTIuODEsMCw1Mi44MWw1MS44Ni0uMDVjMjguMTYsMCw1MS0yMy43MSw1MS01MS44N1M4MCwwLDUxLjg3LDBaIi8+PHBhdGggY2xhc3M9ImNscy0yIiBkPSJNNTIuMzcsMTkuNzRBMzEuNjIsMzEuNjIsMCwwLDAsMjQuNTgsNjYuNDFsLTUuNzIsMTguNEwzOS40LDgwLjE3YTMxLjYxLDMxLjYxLDAsMSwwLDEzLTYwLjQzWiIvPjxwYXRoIGNsYXNzPSJjbHMtMyIgZD0iTTc3LjQ1LDMyLjEyYTMxLjYsMzEuNiwwLDAsMS0zOC4wNSw0OEwxOC44Niw4NC44MmwyMC45MS0yLjQ3QTMxLjYsMzEuNiwwLDAsMCw3Ny40NSwzMi4xMloiLz48cGF0aCBjbGFzcz0iY2xzLTQiIGQ9Ik03MS42MywyNi4yOUEzMS42LDMxLjYsMCwwLDEsMzguOCw3OEwxOC44Niw4NC44MiwzOS40LDgwLjE3QTMxLjYsMzEuNiwwLDAsMCw3MS42MywyNi4yOVoiLz48cGF0aCBjbGFzcz0iY2xzLTUiIGQ9Ik0yNi40Nyw2Ny4xMWEzMS42MSwzMS42MSwwLDAsMSw1MS0zNUEzMS42MSwzMS42MSwwLDAsMCwyNC41OCw2Ni40MWwtNS43MiwxOC40WiIvPjxwYXRoIGNsYXNzPSJjbHMtNiIgZD0iTTI0LjU4LDY2LjQxQTMxLjYxLDMxLjYxLDAsMCwxLDcxLjYzLDI2LjI5YTMxLjYxLDMxLjYxLDAsMCwwLTQ5LDM5LjYzbC0zLjc2LDE4LjlaIi8+PC9nPjwvZz48L3N2Zz4="/></a><!----> <!--[0--><a href="https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter7/section6_pt.ipynb" target="_blank"><img alt="Open In Colab" class="!m-0" src="https://colab.research.google.com/assets/colab-badge.svg"/></a><!--]--> <!--[0--><a href="https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter7/section6_pt.ipynb" target="_blank"><img alt="Open In Studio Lab" class="!m-0" src="https://studiolab.sagemaker.aws/studiolab.svg"/></a><!--]--></div><!--]--> <p>Hasta ahora, hemos utilizado principalmente modelos preentrenados y los hemos ajustado finamente para nuevos casos de uso reutilizando los pesos del preentrenamiento. Como vimos en el <a href="/course/chapter1">Capítulo 1</a>, esto se conoce comúnmente como <em>aprendizaje por transferencia</em> (<em>transfer learning</em>), y es una estrategia muy exitosa para aplicar modelos Transformer a la mayoría de los casos de uso del mundo real donde los datos etiquetados son escasos. En este capítulo adoptaremos un enfoque diferente y entrenaremos un modelo completamente nuevo desde cero. Este es un buen enfoque si tienes muchos datos y estos son muy diferentes de los datos de preentrenamiento utilizados para los modelos disponibles. Sin embargo, preentrenar un modelo de lenguaje requiere considerablemente más recursos computacionales que simplemente ajustar uno existente. Algunos ejemplos en los que puede tener sentido entrenar un nuevo modelo incluyen conjuntos de datos consistentes en notas musicales, secuencias moleculares como el ADN o lenguajes de programación. Estos últimos han ganado gran popularidad recientemente gracias a herramientas como TabNine y Copilot de GitHub, impulsadas por el modelo Codex de OpenAI, que pueden generar secuencias largas de código. Esta tarea de generación de texto se aborda mejor con modelos de lenguaje autorregresivos o causales como GPT-2.</p> <p>En esta sección construiremos una versión a escala reducida de un modelo de generación de código: nos centraremos en completar una sola línea en lugar de funciones o clases completas, utilizando un subconjunto de código Python. Al trabajar con datos en Python estás en contacto frecuente con el stack de ciencia de datos de Python, compuesto por las librerías <code>matplotlib</code>, <code>seaborn</code>, <code>pandas</code> y <code>scikit-learn</code>. Al usar estos frameworks, es común necesitar buscar comandos específicos, por lo que sería genial si pudiéramos usar un modelo para completar estas llamadas por nosotros.</p> <iframe class="w-full xl:w-4/6 h-80" src="https://www.youtube-nocookie.com/embed/Vpjb1lu0MDk" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen=""></iframe><!----> <p>En el <a href="/course/chapter6">Capítulo 6</a> creamos un tokenizador eficiente para procesar código fuente en Python, pero lo que todavía necesitamos es un conjunto de datos a gran escala sobre el cual preentrenar un modelo. Aquí aplicaremos nuestro tokenizador a un corpus de código Python derivado de repositorios de GitHub. Luego utilizaremos la API <code>Trainer</code> y 🤗 Accelerate para entrenar el modelo. ¡Manos a la obra!</p> <iframe src="https://course-demos-codeparrot-ds.hf.space" frameborder="0" height="300" title="Gradio app" class="block dark:hidden container p-0 flex-grow space-iframe" allow="accelerometer; ambient-light-sensor; autoplay; battery; camera; document-domain; encrypted-media; fullscreen; geolocation; gyroscope; layout-animations; legacy-image-formats; magnetometer; microphone; midi; oversized-images; payment; picture-in-picture; publickey-credentials-get; sync-xhr; usb; vr ; wake-lock; xr-spatial-tracking" sandbox="allow-forms allow-modals allow-popups allow-popups-to-escape-sandbox allow-same-origin allow-scripts allow-downloads"></iframe> <p>Esto en realidad muestra el modelo que fue entrenado y subido al Hub usando el código que se muestra en esta sección. Puedes encontrarlo <a href="https://huggingface.co/huggingface-course/codeparrot-ds?text=plt.imshow%28" rel="nofollow">aquí</a>. Ten en cuenta que dado que ocurre cierta aleatorización en la generación de texto, es probable que obtengas un resultado ligeramente diferente.</p> <!--[1--><h2 class="relative group"><a id="recopilar-los-datos" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#recopilar-los-datos"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Recopilar los datos</span></h2><!--]--><!----> <p>El código Python abunda en repositorios de código como GitHub, que podemos utilizar para crear un conjunto de datos extrayendo información de cada repositorio de Python. Este fue el enfoque adoptado en el <a href="https://learning.oreilly.com/library/view/natural-language-processing/9781098136789/" rel="nofollow">libro de texto de Transformers</a> para preentrenar un modelo GPT-2 grande. A partir de un volcado de GitHub de unos 180 GB que contenía aproximadamente 20 millones de archivos Python llamado <code>codeparrot</code>, los autores construyeron un conjunto de datos que luego compartieron en el <a href="https://huggingface.co/datasets/transformersbook/codeparrot" rel="nofollow">Hugging Face Hub</a>.</p> <p>Sin embargo, entrenar sobre el corpus completo consume mucho tiempo y recursos de cómputo, y solo necesitamos el subconjunto del conjunto de datos relacionado con el stack de ciencia de datos de Python. Por lo tanto, comencemos filtrando el conjunto de datos <code>codeparrot</code> para obtener todos los archivos que incluyan alguna de las librerías de este stack. Debido al tamaño del conjunto de datos, queremos evitar descargarlo; en su lugar, utilizaremos la función de transmisión (<em>streaming</em>) para filtrarlo sobre la marcha. Para ayudarnos a filtrar los ejemplos de código que utilizan las librerías mencionadas anteriormente, usaremos la siguiente función:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">def</span> <span class="hljs-title function_">any_keyword_in_string</span>(<span class="hljs-params">string, keywords</span>): | |
| <span class="hljs-keyword">for</span> keyword <span class="hljs-keyword">in</span> keywords: | |
| <span class="hljs-keyword">if</span> keyword <span class="hljs-keyword">in</span> string: | |
| <span class="hljs-keyword">return</span> <span class="hljs-literal">True</span> | |
| <span class="hljs-keyword">return</span> <span class="hljs-literal">False</span><!----></pre></div><!----> <p>Probémosla en dos ejemplos:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->filters = [<span class="hljs-string">"pandas"</span>, <span class="hljs-string">"sklearn"</span>, <span class="hljs-string">"matplotlib"</span>, <span class="hljs-string">"seaborn"</span>] | |
| example_1 = <span class="hljs-string">"import numpy as np"</span> | |
| example_2 = <span class="hljs-string">"import pandas as pd"</span> | |
| <span class="hljs-built_in">print</span>( | |
| any_keyword_in_string(example_1, filters), any_keyword_in_string(example_2, filters) | |
| )<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-literal">False</span> <span class="hljs-literal">True</span><!----></pre></div><!----> <p>Podemos usar esto para crear una función que transmitirá el conjunto de datos mediante streaming y filtrará los elementos que queremos:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">from</span> collections <span class="hljs-keyword">import</span> defaultdict | |
| <span class="hljs-keyword">from</span> tqdm <span class="hljs-keyword">import</span> tqdm | |
| <span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">filter_streaming_dataset</span>(<span class="hljs-params">dataset, filters</span>): | |
| filtered_dict = defaultdict(<span class="hljs-built_in">list</span>) | |
| total = <span class="hljs-number">0</span> | |
| <span class="hljs-keyword">for</span> sample <span class="hljs-keyword">in</span> tqdm(<span class="hljs-built_in">iter</span>(dataset)): | |
| total += <span class="hljs-number">1</span> | |
| <span class="hljs-keyword">if</span> any_keyword_in_string(sample[<span class="hljs-string">"content"</span>], filters): | |
| <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> sample.items(): | |
| filtered_dict[k].append(v) | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"<span class="hljs-subst">{<span class="hljs-built_in">len</span>(filtered_dict[<span class="hljs-string">'content'</span>])/total:<span class="hljs-number">.2</span>%}</span> of data after filtering."</span>) | |
| <span class="hljs-keyword">return</span> Dataset.from_dict(filtered_dict)<!----></pre></div><!----> <p>Luego podemos simplemente aplicar esta función al conjunto de datos en streaming:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-comment"># ¡Esta celda tardará mucho tiempo en ejecutarse, así que deberías saltártela y pasar a</span> | |
| <span class="hljs-comment"># la siguiente!</span> | |
| <span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| split = <span class="hljs-string">"train"</span> <span class="hljs-comment"># "valid"</span> | |
| filters = [<span class="hljs-string">"pandas"</span>, <span class="hljs-string">"sklearn"</span>, <span class="hljs-string">"matplotlib"</span>, <span class="hljs-string">"seaborn"</span>] | |
| data = load_dataset(<span class="hljs-string">f"transformersbook/codeparrot-<span class="hljs-subst">{split}</span>"</span>, split=split, streaming=<span class="hljs-literal">True</span>) | |
| filtered_data = filter_streaming_dataset(data, filters)<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-number">3.26</span>% of data after filtering.<!----></pre></div><!----> <p>Esto nos deja con aproximadamente el 3% del conjunto de datos original, lo cual sigue siendo bastante considerable: ¡el conjunto de datos resultante tiene 6 GB y consta de 600.000 scripts de Python!</p> <p>Filtrar el conjunto de datos completo puede tardar de 2 a 3 horas dependiendo de tu máquina y ancho de banda. Si no quieres pasar por este largo proceso tú mismo, ponemos a tu disposición el conjunto de datos filtrado en el Hub para que lo descargues:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, DatasetDict | |
| ds_train = load_dataset(<span class="hljs-string">"huggingface-course/codeparrot-ds-train"</span>, split=<span class="hljs-string">"train"</span>) | |
| ds_valid = load_dataset(<span class="hljs-string">"huggingface-course/codeparrot-ds-valid"</span>, split=<span class="hljs-string">"validation"</span>) | |
| raw_datasets = DatasetDict( | |
| { | |
| <span class="hljs-string">"train"</span>: ds_train, <span class="hljs-comment"># .shuffle().select(range(50000)),</span> | |
| <span class="hljs-string">"valid"</span>: ds_valid, <span class="hljs-comment"># .shuffle().select(range(500))</span> | |
| } | |
| ) | |
| raw_datasets<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->DatasetDict({ | |
| train: Dataset({ | |
| features: [<span class="hljs-string">'repo_name'</span>, <span class="hljs-string">'path'</span>, <span class="hljs-string">'copies'</span>, <span class="hljs-string">'size'</span>, <span class="hljs-string">'content'</span>, <span class="hljs-string">'license'</span>], | |
| num_rows: <span class="hljs-number">606720</span> | |
| }) | |
| valid: Dataset({ | |
| features: [<span class="hljs-string">'repo_name'</span>, <span class="hljs-string">'path'</span>, <span class="hljs-string">'copies'</span>, <span class="hljs-string">'size'</span>, <span class="hljs-string">'content'</span>, <span class="hljs-string">'license'</span>], | |
| num_rows: <span class="hljs-number">3322</span> | |
| }) | |
| })<!----></pre></div><!----> <blockquote class="tip"><p>El preentrenamiento del modelo de lenguaje llevará un tiempo. Te sugerimos que primero ejecutes el bucle de entrenamiento en una muestra de los datos descomentando las dos líneas parciales anteriores, y te asegures de que el entrenamiento se complete exitosamente y los modelos se guarden. ¡No hay nada más frustrante que una ejecución de entrenamiento falle en el último paso porque olvidaste crear una carpeta o porque hay un error tipográfico al final del bucle de entrenamiento!</p></blockquote> <p>Veamos un ejemplo del conjunto de datos. Solo mostraremos los primeros 200 caracteres de cada campo:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">for</span> key <span class="hljs-keyword">in</span> raw_datasets[<span class="hljs-string">"train"</span>][<span class="hljs-number">0</span>]: | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"<span class="hljs-subst">{key.upper()}</span>: <span class="hljs-subst">{raw_datasets[<span class="hljs-string">'train'</span>][<span class="hljs-number">0</span>][key][:<span class="hljs-number">200</span>]}</span>"</span>)<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-string">'REPO_NAME: kmike/scikit-learn'</span> | |
| <span class="hljs-string">'PATH: sklearn/utils/__init__.py'</span> | |
| <span class="hljs-string">'COPIES: 3'</span> | |
| <span class="hljs-string">'SIZE: 10094'</span> | |
| <span class="hljs-string">'''CONTENT: """ | |
| The :mod:`sklearn.utils` module includes various utilites. | |
| """ | |
| from collections import Sequence | |
| import numpy as np | |
| from scipy.sparse import issparse | |
| import warnings | |
| from .murmurhash import murm | |
| LICENSE: bsd-3-clause'''</span><!----></pre></div><!----> <p>Podemos ver que el campo <code>content</code> contiene el código sobre el cual queremos que nuestro modelo se entrene. Ahora que tenemos un conjunto de datos, necesitamos preparar los textos para que estén en un formato adecuado para el preentrenamiento.</p> <!--[1--><h2 class="relative group"><a id="preparar-el-conjunto-de-datos" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#preparar-el-conjunto-de-datos"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Preparar el conjunto de datos</span></h2><!--]--><!----> <iframe class="w-full xl:w-4/6 h-80" src="https://www.youtube-nocookie.com/embed/ma1TrR7gE7I" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen=""></iframe><!----> <p>El primer paso será tokenizar los datos, para que podamos usarlos en el entrenamiento. Dado que nuestro objetivo principal es autocompletar llamadas a funciones cortas, podemos mantener el tamaño de contexto relativamente pequeño. Esto tiene la ventaja de que podemos entrenar el modelo mucho más rápido y requiere significativamente menos memoria. Si para tu aplicación es importante tener más contexto (por ejemplo, si deseas que el modelo escriba pruebas unitarias basadas en un archivo con la definición de la función), asegúrate de aumentar ese número, pero ten en cuenta también que esto conlleva un mayor consumo de memoria en la GPU. Por ahora, fijemos el tamaño de contexto en 128 tokens, en contraposición a los 1.024 o 2.048 utilizados en GPT-2 o GPT-3, respectivamente.</p> <p>La mayoría de los documentos contienen muchos más de 128 tokens, por lo que simplemente truncar las entradas a la longitud máxima eliminaría una gran fracción de nuestro conjunto de datos. En su lugar, usaremos la opción <code>return_overflowing_tokens</code> para tokenizar toda la entrada y dividirla en varios fragmentos (<em>chunks</em>), tal como hicimos en el <a href="/course/chapter6/4">Capítulo 6</a>. También utilizaremos la opción <code>return_length</code> para obtener la longitud de cada fragmento creado automáticamente. Con frecuencia, el último fragmento será menor que el tamaño de contexto, y nos desharemos de estas partes para evitar problemas de relleno (<em>padding</em>); realmente no las necesitamos ya que de todos modos disponemos de abundantes datos.</p> <div class="flex justify-center"><img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/chunking_texts.svg" alt="Chunking a large texts in several pieces."/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/chunking_texts-dark.svg" alt="Chunking a large texts in several pieces."/></div> <p>Veamos exactamente cómo funciona esto observando los dos primeros ejemplos:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer | |
| context_length = <span class="hljs-number">128</span> | |
| tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">"huggingface-course/code-search-net-tokenizer"</span>) | |
| outputs = tokenizer( | |
| raw_datasets[<span class="hljs-string">"train"</span>][:<span class="hljs-number">2</span>][<span class="hljs-string">"content"</span>], | |
| truncation=<span class="hljs-literal">True</span>, | |
| max_length=context_length, | |
| return_overflowing_tokens=<span class="hljs-literal">True</span>, | |
| return_length=<span class="hljs-literal">True</span>, | |
| ) | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"Input IDs length: <span class="hljs-subst">{<span class="hljs-built_in">len</span>(outputs[<span class="hljs-string">'input_ids'</span>])}</span>"</span>) | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"Input chunk lengths: <span class="hljs-subst">{(outputs[<span class="hljs-string">'length'</span>])}</span>"</span>) | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"Chunk mapping: <span class="hljs-subst">{outputs[<span class="hljs-string">'overflow_to_sample_mapping'</span>]}</span>"</span>)<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->Input IDs length: <span class="hljs-number">34</span> | |
| Input chunk lengths: [<span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">117</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">41</span>] | |
| Chunk mapping: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]<!----></pre></div><!----> <p>Podemos ver que obtenemos 34 segmentos en total a partir de esos dos ejemplos. Observando las longitudes de los fragmentos, podemos ver que los fragmentos al final de ambos documentos tienen menos de 128 tokens (117 y 41, respectivamente). Estos representan solo una pequeña fracción del total de fragmentos que tenemos, por lo que podemos descartarlos de forma segura. Con el campo <code>overflow_to_sample_mapping</code>, también podemos reconstruir qué fragmentos pertenecían a qué muestras de entrada.</p> <p>Con esta operación estamos utilizando una práctica característica de la función <code>Dataset.map()</code> en 🤗 Datasets, que consiste en que no requiere mapeos uno a uno; como vimos en la <a href="/course/chapter7/3">sección 3</a>, podemos crear lotes con más o menos elementos que el lote de entrada. Esto es útil al realizar operaciones como el aumento de datos o el filtrado de datos que cambian la cantidad de elementos. En nuestro caso, al tokenizar cada elemento en fragmentos del tamaño de contexto especificado, creamos muchas muestras a partir de cada documento. Solo debemos asegurarnos de eliminar las columnas existentes, ya que tienen un tamaño en conflicto. Si quisiéramos conservarlas, podríamos repetirlas adecuadamente y devolverlas dentro de la llamada a <code>Dataset.map()</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize</span>(<span class="hljs-params">element</span>): | |
| outputs = tokenizer( | |
| element[<span class="hljs-string">"content"</span>], | |
| truncation=<span class="hljs-literal">True</span>, | |
| max_length=context_length, | |
| return_overflowing_tokens=<span class="hljs-literal">True</span>, | |
| return_length=<span class="hljs-literal">True</span>, | |
| ) | |
| input_batch = [] | |
| <span class="hljs-keyword">for</span> length, input_ids <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(outputs[<span class="hljs-string">"length"</span>], outputs[<span class="hljs-string">"input_ids"</span>]): | |
| <span class="hljs-keyword">if</span> length == context_length: | |
| input_batch.append(input_ids) | |
| <span class="hljs-keyword">return</span> {<span class="hljs-string">"input_ids"</span>: input_batch} | |
| tokenized_datasets = raw_datasets.<span class="hljs-built_in">map</span>( | |
| tokenize, batched=<span class="hljs-literal">True</span>, remove_columns=raw_datasets[<span class="hljs-string">"train"</span>].column_names | |
| ) | |
| tokenized_datasets<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->DatasetDict({ | |
| train: Dataset({ | |
| features: [<span class="hljs-string">'input_ids'</span>], | |
| num_rows: <span class="hljs-number">16702061</span> | |
| }) | |
| valid: Dataset({ | |
| features: [<span class="hljs-string">'input_ids'</span>], | |
| num_rows: <span class="hljs-number">93164</span> | |
| }) | |
| })<!----></pre></div><!----> <p>Ahora tenemos 16,7 millones de ejemplos con 128 tokens cada uno, lo que corresponde a aproximadamente 2.100 millones de tokens en total. Como referencia, los modelos GPT-3 y Codex de OpenAI se entrenan en 300 y 100 mil millones de tokens, respectivamente, donde los modelos Codex se inicializan a partir de los puntos de control de GPT-3. Nuestro objetivo en esta sección no es competir con estos modelos, que pueden generar textos largos y coherentes, sino crear una versión a escala reducida que proporcione una función de autocompletado rápido para científicos de datos.</p> <p>Ahora que tenemos el conjunto de datos listo, ¡configuremos el modelo!</p> <blockquote class="tip"><p>✏️ <strong>¡Pruébalo!</strong> Deshacerse de todos los fragmentos que son más pequeños que el tamaño de contexto no fue un gran problema aquí porque estamos usando ventanas de contexto pequeñas. A medida que aumentes el tamaño de contexto (o si tienes un corpus de documentos cortos), la fracción de fragmentos que se descartan también crecerá. Una forma más eficiente de preparar los datos es unir todas las muestras tokenizadas en un lote con un token <code>eos_token_id</code> en medio, y luego realizar la fragmentación en las secuencias concatenadas. Como ejercicio, modifica la función <code>tokenize()</code> para hacer uso de ese enfoque. Ten en cuenta que querrás establecer <code>truncation=False</code> y eliminar los demás argumentos del tokenizador para obtener la secuencia completa de ID de tokens.</p></blockquote> <!--[1--><h2 class="relative group"><a id="inicializar-un-nuevo-modelo" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#inicializar-un-nuevo-modelo"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Inicializar un nuevo modelo</span></h2><!--]--><!----> <p>Nuestro primer paso es inicializar desde cero un modelo GPT-2. Usaremos la misma configuración para nuestro modelo que para el modelo GPT-2 pequeño, por lo que cargamos la configuración preentrenada, nos aseguramos de que el tamaño del tokenizador coincida con el tamaño del vocabulario del modelo y pasamos los ID de los tokens <code>bos</code> y <code>eos</code> (inicio y final de secuencia):</p> <!--[0--><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, GPT2LMHeadModel, AutoConfig | |
| config = AutoConfig.from_pretrained( | |
| <span class="hljs-string">"gpt2"</span>, | |
| vocab_size=<span class="hljs-built_in">len</span>(tokenizer), | |
| n_ctx=context_length, | |
| bos_token_id=tokenizer.bos_token_id, | |
| eos_token_id=tokenizer.eos_token_id, | |
| )<!----></pre></div><!----> <p>Con esa configuración, podemos cargar un nuevo modelo. Ten en cuenta que esta es la primera vez que no usamos la función <code>from_pretrained()</code>, ya que en realidad estamos inicializando un modelo nosotros mismos:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->model = GPT2LMHeadModel(config) | |
| model_size = <span class="hljs-built_in">sum</span>(t.numel() <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> model.parameters()) | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"GPT-2 size: <span class="hljs-subst">{model_size/<span class="hljs-number">1000</span>**<span class="hljs-number">2</span>:<span class="hljs-number">.1</span>f}</span>M parameters"</span>)<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->GPT-<span class="hljs-number">2</span> size: <span class="hljs-number">124.2</span>M parameters<!----></pre></div><!----><!--]--> <p>Nuestro modelo tiene 124M de parámetros que tendremos que ajustar. Antes de poder comenzar el entrenamiento, necesitamos configurar un ensamblador de datos (<em>data collator</em>) que se encargará de crear los lotes. Podemos usar el agrupador <code>DataCollatorForLanguageModeling</code>, que está diseñado específicamente para el modelado de lenguaje (como su nombre sugiere sutilmente). Además de apilar y rellenar los lotes, también se encarga de crear las etiquetas del modelo de lenguaje: en el modelado de lenguaje causal las entradas también sirven como etiquetas (simplemente desplazadas un elemento), y este agrupador de datos las crea sobre la marcha durante el entrenamiento para que no necesitemos duplicar los <code>input_ids</code>.</p> <p>Ten en cuenta que <code>DataCollatorForLanguageModeling</code> admite tanto el modelado de lenguaje enmascarado (MLM) como el modelado de lenguaje causal (CLM). Por defecto prepara datos para MLM, pero podemos cambiar a CLM configurando el argumento <code>mlm=False</code>:</p> <!--[0--><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling | |
| tokenizer.pad_token = tokenizer.eos_token | |
| data_collator = DataCollatorForLanguageModeling(tokenizer, mlm=<span class="hljs-literal">False</span>)<!----></pre></div><!--]--> <p>Echemos un vistazo a un ejemplo:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->out = data_collator([tokenized_datasets[<span class="hljs-string">"train"</span>][i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">5</span>)]) | |
| <span class="hljs-keyword">for</span> key <span class="hljs-keyword">in</span> out: | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"<span class="hljs-subst">{key}</span> shape: <span class="hljs-subst">{out[key].shape}</span>"</span>)<!----></pre></div><!----> <!--[0--><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->input_ids shape: torch.Size([<span class="hljs-number">5</span>, <span class="hljs-number">128</span>]) | |
| attention_mask shape: torch.Size([<span class="hljs-number">5</span>, <span class="hljs-number">128</span>]) | |
| labels shape: torch.Size([<span class="hljs-number">5</span>, <span class="hljs-number">128</span>])<!----></pre></div><!--]--> <p>Podemos ver que los ejemplos se han apilado y todos los tensores tienen la misma forma.</p> <!--[-1--><!--]--> <blockquote class="warning"><p>⚠️ El desplazamiento de las entradas y las etiquetas para alinearlas ocurre dentro del modelo, por lo que el ensamblador de datos simplemente copia las entradas para crear las etiquetas.</p></blockquote> <p>Ahora tenemos todo listo para entrenar nuestro modelo; ¡después de todo, no fue tanto trabajo! Antes de comenzar a entrenar debemos iniciar sesión en Hugging Face. Si estás trabajando en un cuaderno, puedes hacerlo con la siguiente función de utilidad:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login | |
| notebook_login()<!----></pre></div><!----> <p>Esto mostrará un widget donde podrás ingresar tus credenciales de inicio de sesión de Hugging Face.</p> <p>Si no estás trabajando en un cuaderno, simplemente escribe la siguiente línea en tu terminal:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!---->huggingface-cli login<!----></pre></div><!----> <!--[0--><p>Todo lo que queda por hacer es configurar los argumentos de entrenamiento y poner en marcha el <code>Trainer</code>. Utilizaremos un esquema de tasa de aprendizaje de coseno con algo de calentamiento (<em>warmup</em>) y un tamaño de lote efectivo de 256 (<code>per_device_train_batch_size</code> * <code>gradient_accumulation_steps</code>). La acumulación de gradiente se utiliza cuando un solo lote no cabe en la memoria, y acumula incrementalmente el gradiente a través de varios pasos hacia adelante y hacia atrás. Veremos esto en acción cuando creemos el bucle de entrenamiento con 🤗 Accelerate.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer, TrainingArguments | |
| args = TrainingArguments( | |
| output_dir=<span class="hljs-string">"codeparrot-ds"</span>, | |
| per_device_train_batch_size=<span class="hljs-number">32</span>, | |
| per_device_eval_batch_size=<span class="hljs-number">32</span>, | |
| evaluation_strategy=<span class="hljs-string">"steps"</span>, | |
| eval_steps=<span class="hljs-number">5_000</span>, | |
| logging_steps=<span class="hljs-number">5_000</span>, | |
| gradient_accumulation_steps=<span class="hljs-number">8</span>, | |
| num_train_epochs=<span class="hljs-number">1</span>, | |
| weight_decay=<span class="hljs-number">0.1</span>, | |
| warmup_steps=<span class="hljs-number">1_000</span>, | |
| lr_scheduler_type=<span class="hljs-string">"cosine"</span>, | |
| learning_rate=<span class="hljs-number">5e-4</span>, | |
| save_steps=<span class="hljs-number">5_000</span>, | |
| fp16=<span class="hljs-literal">True</span>, | |
| push_to_hub=<span class="hljs-literal">True</span>, | |
| ) | |
| trainer = Trainer( | |
| model=model, | |
| tokenizer=tokenizer, | |
| args=args, | |
| data_collator=data_collator, | |
| train_dataset=tokenized_datasets[<span class="hljs-string">"train"</span>], | |
| eval_dataset=tokenized_datasets[<span class="hljs-string">"valid"</span>], | |
| )<!----></pre></div><!----> <p>Ahora simplemente podemos iniciar el <code>Trainer</code> y esperar a que termine el entrenamiento. Dependiendo de si lo ejecutas en el conjunto de entrenamiento completo o en un subconjunto, esto llevará 20 o 2 horas, respectivamente, ¡así que tómate unos cafés y lee un buen libro!</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->trainer.train()<!----></pre></div><!----> <p>Una vez completado el entrenamiento, podemos subir el modelo y el tokenizador al Hub:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->trainer.push_to_hub()<!----></pre></div><!----><!--]--> <blockquote class="tip"><p>✏️ <strong>¡Pruébalo!</strong> Solo nos tomó alrededor de 30 líneas de código además de los <code>TrainingArguments</code> pasar de los textos sin procesar al entrenamiento de GPT-2. ¡Pruébalo con tu propio conjunto de datos y mira si puedes obtener buenos resultados!</p></blockquote> <blockquote class="tip"></blockquote> <!--[0--><blockquote><p>💡 Si tienes acceso a una máquina con varias GPUs, intenta ejecutar el código allí. El <code>Trainer</code> gestiona automáticamente múltiples máquinas, y esto puede acelerar el entrenamiento enormemente.</p></blockquote><!--]--> <!--[1--><h2 class="relative group"><a id="generacion-de-codigo-con-un-pipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#generacion-de-codigo-con-un-pipeline"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Generación de código con un pipeline</span></h2><!--]--><!----> <p>Ahora es el momento de la verdad: ¡veamos qué tan bien funciona realmente el modelo entrenado! Podemos ver en los registros que la pérdida disminuyó de manera constante, pero para poner a prueba el modelo, veamos cómo funciona en algunas instrucciones (<em>prompts</em>). Para hacer eso, envolveremos el modelo en un <code>pipeline</code> de generación de texto y lo colocaremos en la GPU para generaciones rápidas si hay una disponible:</p> <!--[0--><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline | |
| device = torch.device(<span class="hljs-string">"cuda"</span>) <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> torch.device(<span class="hljs-string">"cpu"</span>) | |
| pipe = pipeline( | |
| <span class="hljs-string">"text-generation"</span>, model=<span class="hljs-string">"huggingface-course/codeparrot-ds"</span>, device=device | |
| )<!----></pre></div><!--]--> <p>Comencemos con la sencilla tarea de crear un gráfico de dispersión:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->txt = <span class="hljs-string">"""\ | |
| # create some data | |
| x = np.random.randn(100) | |
| y = np.random.randn(100) | |
| # create scatter plot with x, y | |
| """</span> | |
| <span class="hljs-built_in">print</span>(pipe(txt, num_return_sequences=<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>][<span class="hljs-string">"generated_text"</span>])<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-comment"># create some data</span> | |
| x = np.random.randn(<span class="hljs-number">100</span>) | |
| y = np.random.randn(<span class="hljs-number">100</span>) | |
| <span class="hljs-comment"># create scatter plot with x, y</span> | |
| plt.scatter(x, y) | |
| <span class="hljs-comment"># create scatter</span><!----></pre></div><!----> <p>El resultado parece correcto. ¿Funciona también para una operación de <code>pandas</code>? Veamos si podemos crear un <code>DataFrame</code> a partir de dos matrices:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->txt = <span class="hljs-string">"""\ | |
| # create some data | |
| x = np.random.randn(100) | |
| y = np.random.randn(100) | |
| # create dataframe from x and y | |
| """</span> | |
| <span class="hljs-built_in">print</span>(pipe(txt, num_return_sequences=<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>][<span class="hljs-string">"generated_text"</span>])<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-comment"># create some data</span> | |
| x = np.random.randn(<span class="hljs-number">100</span>) | |
| y = np.random.randn(<span class="hljs-number">100</span>) | |
| <span class="hljs-comment"># create dataframe from x and y</span> | |
| df = pd.DataFrame({<span class="hljs-string">'x'</span>: x, <span class="hljs-string">'y'</span>: y}) | |
| df.insert(<span class="hljs-number">0</span>,<span class="hljs-string">'x'</span>, x) | |
| <span class="hljs-keyword">for</span><!----></pre></div><!----> <p>Genial, esa es la respuesta correcta, aunque luego vuelve a insertar la columna <code>x</code>. Dado que la cantidad de tokens generados es limitada, el bucle <code>for</code> siguiente queda cortado. Veamos si podemos hacer algo un poco más complejo y hacer que el modelo nos ayude a usar la operación <code>groupby</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->txt = <span class="hljs-string">"""\ | |
| # dataframe with profession, income and name | |
| df = pd.DataFrame({'profession': x, 'income':y, 'name': z}) | |
| # calculate the mean income per profession | |
| """</span> | |
| <span class="hljs-built_in">print</span>(pipe(txt, num_return_sequences=<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>][<span class="hljs-string">"generated_text"</span>])<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-comment"># dataframe with profession, income and name</span> | |
| df = pd.DataFrame({<span class="hljs-string">'profession'</span>: x, <span class="hljs-string">'income'</span>:y, <span class="hljs-string">'name'</span>: z}) | |
| <span class="hljs-comment"># calculate the mean income per profession</span> | |
| profession = df.groupby([<span class="hljs-string">'profession'</span>]).mean() | |
| <span class="hljs-comment"># compute the</span><!----></pre></div><!----> <p>Nada mal; esa es la forma correcta de hacerlo. Por último, veamos si también podemos usarlo para <code>scikit-learn</code> y configurar un modelo de Random Forest:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->txt = <span class="hljs-string">""" | |
| # import random forest regressor from scikit-learn | |
| from sklearn.ensemble import RandomForestRegressor | |
| # fit random forest model with 300 estimators on X, y: | |
| """</span> | |
| <span class="hljs-built_in">print</span>(pipe(txt, num_return_sequences=<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>][<span class="hljs-string">"generated_text"</span>])<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-comment"># import random forest regressor from scikit-learn</span> | |
| <span class="hljs-keyword">from</span> sklearn.ensemble <span class="hljs-keyword">import</span> RandomForestRegressor | |
| <span class="hljs-comment"># fit random forest model with 300 estimators on X, y:</span> | |
| rf = RandomForestRegressor(n_estimators=<span class="hljs-number">300</span>, random_state=random_state, max_depth=<span class="hljs-number">3</span>) | |
| rf.fit(X, y) | |
| rf<!----></pre></div><!----> <!--[-1--><p>Mirando estos pocos ejemplos, parece que el modelo ha aprendido parte de la sintaxis del stack de ciencia de datos de Python (por supuesto, necesitaríamos evaluarlo más a fondo antes de implementar el modelo en el mundo real). Sin embargo, a veces se requiere una mayor personalización del entrenamiento del modelo para lograr el rendimiento necesario para un caso de uso determinado. Por ejemplo, ¿qué pasaría si quisiéramos actualizar dinámicamente el tamaño del lote o tener un bucle de entrenamiento condicional que omita los ejemplos defectuosos sobre la marcha? Una opción sería crear una subclase de <code>Trainer</code> y agregar los cambios necesarios, pero a veces es más sencillo escribir el bucle de entrenamiento desde cero. Ahí es donde entra 🤗 Accelerate.</p><!--]--> <!--[0--><!--[1--><h2 class="relative group"><a id="entrenamiento-con-accelerate" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#entrenamiento-con-accelerate"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Entrenamiento con 🤗 Accelerate</span></h2><!--]--><!----> <p>Hemos visto cómo entrenar un modelo con el <code>Trainer</code>, lo que permite cierta personalización. Sin embargo, a veces queremos un control total sobre el bucle de entrenamiento o queremos realizar algunos cambios exóticos. En este caso, 🤗 Accelerate es una excelente opción, y en esta sección repasaremos los pasos para usarlo en el entrenamiento de nuestro modelo. Para hacer las cosas más interesantes, también agregaremos un toque especial al bucle de entrenamiento.</p> <iframe class="w-full xl:w-4/6 h-80" src="https://www.youtube-nocookie.com/embed/Hm8_PgVTFuc" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen=""></iframe><!----> <p>Dado que nos interesa principalmente la autocompletación sensata para las librerías de ciencia de datos, tiene sentido dar más peso a las muestras de entrenamiento que hacen un mayor uso de estas librerías. Podemos identificar fácilmente estos ejemplos mediante el uso de palabras clave como <code>plt</code>, <code>pd</code>, <code>sk</code>, <code>fit</code> y <code>predict</code>, que son los nombres de importación más frecuentes para <code>matplotlib.pyplot</code>, <code>pandas</code> y <code>sklearn</code>, así como el patrón de ajuste/predicción de este último. Si cada uno de estos está representado como un solo token, podemos comprobar fácilmente si ocurren en la secuencia de entrada. Los tokens pueden tener un prefijo de espacio en blanco, por lo que también verificaremos esas versiones en el vocabulario del tokenizador. Para comprobar que funciona, agregaremos un token de prueba que debería dividirse en múltiples tokens:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->keytoken_ids = [] | |
| <span class="hljs-keyword">for</span> keyword <span class="hljs-keyword">in</span> [ | |
| <span class="hljs-string">"plt"</span>, | |
| <span class="hljs-string">"pd"</span>, | |
| <span class="hljs-string">"sk"</span>, | |
| <span class="hljs-string">"fit"</span>, | |
| <span class="hljs-string">"predict"</span>, | |
| <span class="hljs-string">" plt"</span>, | |
| <span class="hljs-string">" pd"</span>, | |
| <span class="hljs-string">" sk"</span>, | |
| <span class="hljs-string">" fit"</span>, | |
| <span class="hljs-string">" predict"</span>, | |
| <span class="hljs-string">"testtest"</span>, | |
| ]: | |
| ids = tokenizer([keyword]).input_ids[<span class="hljs-number">0</span>] | |
| <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(ids) == <span class="hljs-number">1</span>: | |
| keytoken_ids.append(ids[<span class="hljs-number">0</span>]) | |
| <span class="hljs-keyword">else</span>: | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"Keyword has not single token: <span class="hljs-subst">{keyword}</span>"</span>)<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-string">'Keyword has not single token: testtest'</span><!----></pre></div><!----> <p>Genial, ¡parece que funciona muy bien! Ahora podemos escribir una función de pérdida personalizada que tome como entradas la secuencia de entrada, los logits y los tokens clave que acabamos de seleccionar. Primero necesitamos alinear los logits y las entradas: la secuencia de entrada desplazada uno hacia la derecha forma las etiquetas, ya que el siguiente token es la etiqueta para el token actual. Podemos lograr esto comenzando las etiquetas a partir del segundo token de la secuencia de entrada, ya que el modelo no hace una predicción para el primer token de todos modos. Luego cortamos el último logit, ya que no tenemos una etiqueta para el token que sigue a la secuencia de entrada completa. Con eso podemos calcular la pérdida por muestra y contar las apariciones de todas las palabras clave en cada muestra. Finalmente, calculamos el promedio ponderado sobre todas las muestras utilizando las apariciones como pesos. Dado que no queremos descartar todas las muestras que no tienen palabras clave, sumamos 1 a los pesos:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">from</span> torch.nn <span class="hljs-keyword">import</span> CrossEntropyLoss | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">keytoken_weighted_loss</span>(<span class="hljs-params">inputs, logits, keytoken_ids, alpha=<span class="hljs-number">1.0</span></span>): | |
| <span class="hljs-comment"># Shift so that tokens < n predict n</span> | |
| shift_labels = inputs[..., <span class="hljs-number">1</span>:].contiguous() | |
| shift_logits = logits[..., :-<span class="hljs-number">1</span>, :].contiguous() | |
| <span class="hljs-comment"># Calculate per-token loss</span> | |
| loss_fct = CrossEntropyLoss(reduce=<span class="hljs-literal">False</span>) | |
| loss = loss_fct(shift_logits.view(-<span class="hljs-number">1</span>, shift_logits.size(-<span class="hljs-number">1</span>)), shift_labels.view(-<span class="hljs-number">1</span>)) | |
| <span class="hljs-comment"># Resize and average loss per sample</span> | |
| loss_per_sample = loss.view(shift_logits.size(<span class="hljs-number">0</span>), shift_logits.size(<span class="hljs-number">1</span>)).mean(axis=<span class="hljs-number">1</span>) | |
| <span class="hljs-comment"># Calculate and scale weighting</span> | |
| weights = torch.stack([(inputs == kt).<span class="hljs-built_in">float</span>() <span class="hljs-keyword">for</span> kt <span class="hljs-keyword">in</span> keytoken_ids]).<span class="hljs-built_in">sum</span>( | |
| axis=[<span class="hljs-number">0</span>, <span class="hljs-number">2</span>] | |
| ) | |
| weights = alpha * (<span class="hljs-number">1.0</span> + weights) | |
| <span class="hljs-comment"># Calculate weighted average</span> | |
| weighted_loss = (loss_per_sample * weights).mean() | |
| <span class="hljs-keyword">return</span> weighted_loss<!----></pre></div><!----> <p>Antes de poder comenzar a entrenar con esta fantástica nueva función de pérdida, necesitamos preparar algunas cosas:</p> <ul><li>Necesitamos cargadores de datos (<em>dataloaders</em>) para cargar los datos en lotes.</li> <li>Necesitamos configurar los parámetros de decaimiento de peso (<em>weight decay</em>).</li> <li>De vez en cuando queremos evaluar, por lo que tiene sentido envolver el código de evaluación en una función.</li></ul> <p>Comencemos con los dataloaders. Solo necesitamos establecer el formato del conjunto de datos en <code>"torch"</code>, y luego podemos pasarlo a un <code>DataLoader</code> de PyTorch con el tamaño de lote apropiado:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">from</span> torch.utils.data.dataloader <span class="hljs-keyword">import</span> DataLoader | |
| tokenized_datasets.set_format(<span class="hljs-string">"torch"</span>) | |
| train_dataloader = DataLoader(tokenized_datasets[<span class="hljs-string">"train"</span>], batch_size=<span class="hljs-number">32</span>, shuffle=<span class="hljs-literal">True</span>) | |
| eval_dataloader = DataLoader(tokenized_datasets[<span class="hljs-string">"valid"</span>], batch_size=<span class="hljs-number">32</span>)<!----></pre></div><!----> <p>A continuación, agrupamos los parámetros para que el optimizador sepa cuáles recibirán un decaimiento de peso adicional. Por lo general, todos los términos de sesgo (<em>bias</em>) y pesos de LayerNorm están exentos de esto; así es como podemos hacerlo:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->weight_decay = <span class="hljs-number">0.1</span> | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">get_grouped_params</span>(<span class="hljs-params">model, no_decay=[<span class="hljs-string">"bias"</span>, <span class="hljs-string">"LayerNorm.weight"</span>]</span>): | |
| params_with_wd, params_without_wd = [], [] | |
| <span class="hljs-keyword">for</span> n, p <span class="hljs-keyword">in</span> model.named_parameters(): | |
| <span class="hljs-keyword">if</span> <span class="hljs-built_in">any</span>(nd <span class="hljs-keyword">in</span> n <span class="hljs-keyword">for</span> nd <span class="hljs-keyword">in</span> no_decay): | |
| params_without_wd.append(p) | |
| <span class="hljs-keyword">else</span>: | |
| params_with_wd.append(p) | |
| <span class="hljs-keyword">return</span> [ | |
| {<span class="hljs-string">"params"</span>: params_with_wd, <span class="hljs-string">"weight_decay"</span>: weight_decay}, | |
| {<span class="hljs-string">"params"</span>: params_without_wd, <span class="hljs-string">"weight_decay"</span>: <span class="hljs-number">0.0</span>}, | |
| ]<!----></pre></div><!----> <p>Dado que queremos evaluar el modelo con regularidad en el conjunto de validación durante el entrenamiento, escribamos también una función para eso. Simplemente recorre el dataloader de evaluación y recopila todas las pérdidas a través de los procesos:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">def</span> <span class="hljs-title function_">evaluate</span>(): | |
| model.<span class="hljs-built_in">eval</span>() | |
| losses = [] | |
| <span class="hljs-keyword">for</span> step, batch <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(eval_dataloader): | |
| <span class="hljs-keyword">with</span> torch.no_grad(): | |
| outputs = model(batch[<span class="hljs-string">"input_ids"</span>], labels=batch[<span class="hljs-string">"input_ids"</span>]) | |
| losses.append(accelerator.gather(outputs.loss)) | |
| loss = torch.mean(torch.cat(losses)) | |
| <span class="hljs-keyword">try</span>: | |
| perplexity = torch.exp(loss) | |
| <span class="hljs-keyword">except</span> OverflowError: | |
| perplexity = <span class="hljs-built_in">float</span>(<span class="hljs-string">"inf"</span>) | |
| <span class="hljs-keyword">return</span> loss.item(), perplexity.item()<!----></pre></div><!----> <p>Con la función <code>evaluate()</code> podemos reportar la pérdida y la <a href="/course/chapter7/3">perplejidad</a> a intervalos regulares. A continuación, redefinimos nuestro modelo para asegurarnos de volver a entrenar desde cero:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->model = GPT2LMHeadModel(config)<!----></pre></div><!----> <p>Luego podemos definir nuestro optimizador, utilizando la función anterior para dividir los parámetros para el decaimiento de peso:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">from</span> torch.optim <span class="hljs-keyword">import</span> AdamW | |
| optimizer = AdamW(get_grouped_params(model), lr=<span class="hljs-number">5e-4</span>)<!----></pre></div><!----> <p>Ahora preparemos el modelo, el optimizador y los dataloaders para poder comenzar a entrenar:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator | |
| accelerator = Accelerator(fp16=<span class="hljs-literal">True</span>) | |
| model, optimizer, train_dataloader, eval_dataloader = accelerator.prepare( | |
| model, optimizer, train_dataloader, eval_dataloader | |
| )<!----></pre></div><!----> <blockquote class="tip"><p>🚨 Si estás entrenando en una TPU, deberás mover todo el código a partir de la celda anterior a una función de entrenamiento dedicada. Consulta el <a href="/course/chapter3">Capítulo 3</a> para obtener más detalles.</p></blockquote> <p>Ahora que hemos enviado nuestro <code>train_dataloader</code> a <code>accelerator.prepare()</code>, podemos usar su longitud para calcular el número de pasos de entrenamiento. Recuerda que siempre debemos hacer esto después de preparar el dataloader, ya que ese método cambiará su longitud. Usamos un esquema lineal clásico desde la tasa de aprendizaje hasta 0:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> get_scheduler | |
| num_train_epochs = <span class="hljs-number">1</span> | |
| num_update_steps_per_epoch = <span class="hljs-built_in">len</span>(train_dataloader) | |
| num_training_steps = num_train_epochs * num_update_steps_per_epoch | |
| lr_scheduler = get_scheduler( | |
| name=<span class="hljs-string">"linear"</span>, | |
| optimizer=optimizer, | |
| num_warmup_steps=<span class="hljs-number">1_000</span>, | |
| num_training_steps=num_training_steps, | |
| )<!----></pre></div><!----> <p>Por último, para subir nuestro modelo al Hub, necesitaremos crear un objeto <code>Repository</code> en una carpeta de trabajo. Primero inicia sesión en el Hugging Face Hub, si aún no lo has hecho. Determinaremos el nombre del repositorio a partir del ID de modelo que queremos darle a nuestro modelo (siéntete libre de reemplazar <code>repo_name</code> con tu propia elección; solo necesita contener tu nombre de usuario, que es lo que hace la función <code>get_full_repo_name()</code>):</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> Repository, get_full_repo_name | |
| model_name = <span class="hljs-string">"codeparrot-ds-accelerate"</span> | |
| repo_name = get_full_repo_name(model_name) | |
| repo_name<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-string">'sgugger/codeparrot-ds-accelerate'</span><!----></pre></div><!----> <p>Luego podemos clonar ese repositorio en una carpeta local. Si ya existe, esta carpeta local debe ser un clon existente del repositorio con el que estamos trabajando:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->output_dir = <span class="hljs-string">"codeparrot-ds-accelerate"</span> | |
| repo = Repository(output_dir, clone_from=repo_name)<!----></pre></div><!----> <p>Ahora podemos subir cualquier cosa que guardemos en <code>output_dir</code> llamando al método <code>repo.push_to_hub()</code>. Esto nos ayudará a subir los modelos intermedios al final de cada época.</p> <p>Antes de entrenar, ejecutemos una prueba rápida para ver si la función de evaluación funciona correctamente:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->evaluate()<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->(<span class="hljs-number">10.934126853942871</span>, <span class="hljs-number">56057.14453125</span>)<!----></pre></div><!----> <p>Esos son valores muy altos para la pérdida y la perplejidad, pero no es de extrañar ya que aún no hemos entrenado el modelo. Con esto, tenemos todo preparado para escribir la parte central del script de entrenamiento: el bucle de entrenamiento. En el bucle de entrenamiento iteramos sobre el dataloader y pasamos los lotes al modelo. Con los logits, podemos entonces evaluar nuestra función de pérdida personalizada. Escalamos la pérdida por la cantidad de pasos de acumulación de gradiente para no generar pérdidas mayores al agregar más pasos. Antes de optimizar, también recortamos los gradientes (<em>gradient clipping</em>) para una mejor convergencia. Finalmente, cada pocos pasos evaluamos el modelo en el conjunto de evaluación con nuestra nueva función <code>evaluate()</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">from</span> tqdm.notebook <span class="hljs-keyword">import</span> tqdm | |
| gradient_accumulation_steps = <span class="hljs-number">8</span> | |
| eval_steps = <span class="hljs-number">5_000</span> | |
| model.train() | |
| completed_steps = <span class="hljs-number">0</span> | |
| <span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(num_train_epochs): | |
| <span class="hljs-keyword">for</span> step, batch <span class="hljs-keyword">in</span> tqdm( | |
| <span class="hljs-built_in">enumerate</span>(train_dataloader, start=<span class="hljs-number">1</span>), total=num_training_steps | |
| ): | |
| logits = model(batch[<span class="hljs-string">"input_ids"</span>]).logits | |
| loss = keytoken_weighted_loss(batch[<span class="hljs-string">"input_ids"</span>], logits, keytoken_ids) | |
| <span class="hljs-keyword">if</span> step % <span class="hljs-number">100</span> == <span class="hljs-number">0</span>: | |
| accelerator.<span class="hljs-built_in">print</span>( | |
| { | |
| <span class="hljs-string">"samples"</span>: step * samples_per_step, | |
| <span class="hljs-string">"steps"</span>: completed_steps, | |
| <span class="hljs-string">"loss/train"</span>: loss.item() * gradient_accumulation_steps, | |
| } | |
| ) | |
| loss = loss / gradient_accumulation_steps | |
| accelerator.backward(loss) | |
| <span class="hljs-keyword">if</span> step % gradient_accumulation_steps == <span class="hljs-number">0</span>: | |
| accelerator.clip_grad_norm_(model.parameters(), <span class="hljs-number">1.0</span>) | |
| optimizer.step() | |
| lr_scheduler.step() | |
| optimizer.zero_grad() | |
| completed_steps += <span class="hljs-number">1</span> | |
| <span class="hljs-keyword">if</span> (step % (eval_steps * gradient_accumulation_steps)) == <span class="hljs-number">0</span>: | |
| eval_loss, perplexity = evaluate() | |
| accelerator.<span class="hljs-built_in">print</span>({<span class="hljs-string">"loss/eval"</span>: eval_loss, <span class="hljs-string">"perplexity"</span>: perplexity}) | |
| model.train() | |
| accelerator.wait_for_everyone() | |
| unwrapped_model = accelerator.unwrap_model(model) | |
| unwrapped_model.save_pretrained(output_dir, save_function=accelerator.save) | |
| <span class="hljs-keyword">if</span> accelerator.is_main_process: | |
| tokenizer.save_pretrained(output_dir) | |
| repo.push_to_hub( | |
| commit_message=<span class="hljs-string">f"Training in progress step <span class="hljs-subst">{step}</span>"</span>, blocking=<span class="hljs-literal">False</span> | |
| )<!----></pre></div><!----> <p>Y eso es todo: ahora tienes tu propio bucle de entrenamiento personalizado para modelos de lenguaje causal como GPT-2 que puedes personalizar aún más según tus necesidades.</p> <blockquote class="tip"><p>✏️ <strong>¡Pruébalo!</strong> Crea tu propia función de pérdida personalizada adaptada a tu caso de uso o agrega otro paso personalizado en el bucle de entrenamiento.</p></blockquote> <blockquote class="tip"><p>✏️ <strong>¡Pruébalo!</strong> Al ejecutar experimentos de entrenamiento prolongados, es una buena idea registrar métricas importantes utilizando herramientas como TensorBoard o Weights & Biases. Agrega un registro adecuado al bucle de entrenamiento para que siempre puedas verificar cómo va el entrenamiento.</p></blockquote><!--]--> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/course/blob/main/chapters/es/chapter7/6.mdx" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]--> | |
| <script> | |
| { | |
| __sveltekit_agf4mu = { | |
| base: "/docs/course/pr_1352/es", | |
| assets: "/docs/course/pr_1352/es" | |
| }; | |
| const element = document.currentScript.parentElement; | |
| Promise.all([ | |
| import("/docs/course/pr_1352/es/_app/immutable/entry/start.DV9gcfqJ.js"), | |
| import("/docs/course/pr_1352/es/_app/immutable/entry/app.C68Ze5cV.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 45], | |
| data: [null,null], | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 136 kB
- Xet hash:
- 34bcac7f4d85e301ab4d428db7aaf7984a451e15677ae298a95093b9bb871a1c
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.