course/pr_1213/es/_app/immutable/nodes/33.b6dd7dbd.js

import{s as il,n as rl,o as ol}from"../chunks/scheduler.505acc25.js";import{S as ml,i as pl,e as i,s as n,c as o,h as Ml,a as r,d as l,b as s,f as sl,g as m,j as d,k as Bt,l as cl,m as a,n as p,t as M,o as c,p as u}from"../chunks/index.e22abd30.js";import{C as ul,H as y,E as dl}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.a144e953.js";import{C as J}from"../chunks/CodeBlock.f6688f67.js";import{C as yl}from"../chunks/CourseFloatingBanner.f0a2dc21.js";function Jl(Zt){let T,ge,Ue,je,w,$e,f,Ie,U,Ce,h,Wt="Ahora que has visto la teoría, pongámosla en práctica. En este ejercicio, ajustarás un modelo con GRPO.",Ge,b,Rt='<p>Este ejercicio fue escrito por el experto en ajuste fino de LLM <a href="https://huggingface.co/mlabonne" rel="nofollow">@mlabonne</a>.</p>',Be,g,Ze,j,vt="Primero, instalemos las dependencias para este ejercicio.",We,$,Re,I,xt="Ahora importaremos las librerías necesarias.",ve,C,xe,G,Xe,B,Xt="Weights & Biases es una herramienta para registrar y monitorizar experimentos. La usaremos para registrar nuestro proceso de ajuste fino.",Fe,Z,ke,W,Ft="Puedes hacer este ejercicio sin iniciar sesión en Weights & Biases, pero es recomendable hacerlo para seguir tus experimentos e interpretar los resultados.",_e,R,Qe,v,kt='Usaremos el dataset <a href="https://huggingface.co/datasets/mlabonne/smoltldr" rel="nofollow"><code>mlabonne/smoltldr</code></a>, que contiene una lista de historias cortas.',Ve,x,Ee,X,ze,F,_t='Para este ejercicio usaremos <a href="https://huggingface.co/HuggingFaceTB/SmolLM2-135M" rel="nofollow"><code>SmolLM2-135M</code></a>.',Ye,k,Qt='Es un modelo pequeño de 135M parámetros que funciona en hardware limitado. Eso lo hace ideal para aprender, aunque no sea el modelo más potente disponible. Si tienes acceso a mejor hardware, puedes probar con <a href="https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B" rel="nofollow"><code>SmolLM2-1.7B</code></a>.',He,_,Ne,Q,qe,V,Vt="Ahora cargaremos la configuración de LoRA para reducir el número de parámetros entrenables y, con ello, la memoria necesaria para ajustar el modelo.",Se,E,Et='Si no conoces LoRA, puedes leer más en el <a href="https://huggingface.co/learn/course/en/chapter11/3" rel="nofollow">Capítulo 11</a>.',Ae,z,Le,Y,Pe,H,De,N,zt="Como mencionamos en la sección anterior, GRPO puede usar cualquier función de recompensa para mejorar el modelo. En este caso usaremos una función sencilla que anima al modelo a generar texto de 50 tokens.",Oe,q,Ke,S,et,A,Yt="Usaremos la clase <code>GRPOConfig</code> para definir los argumentos al estilo de <code>transformers</code>.",tt,L,Ht='Si es la primera vez que defines argumentos de entrenamiento, puedes consultar <a href="https://huggingface.co/docs/transformers/en/main_classes/trainer#trainingarguments" rel="nofollow">TrainingArguments</a> o el <a href="https://huggingface.co/learn/course/en/chapter2/1" rel="nofollow">Capítulo 2</a>.',lt,P,at,D,Nt="Ahora podemos inicializar el entrenador con el modelo, el dataset y los argumentos de entrenamiento, y empezar el ajuste.",nt,O,st,K,qt="El entrenamiento tarda alrededor de una hora en una sola GPU A10G, disponible en Google Colab o mediante Hugging Face Spaces.",it,ee,rt,te,St="Si configuramos <code>push_to_hub=True</code> y <code>model_id</code> con un nombre de modelo válido, el modelo se subirá a Hugging Face Hub mientras entrenamos.",ot,le,mt,ae,At="<code>GRPOTrainer</code> registra la recompensa de tu función de recompensa, la pérdida y otras métricas.",pt,ne,Lt="Nos centraremos en la recompensa y la pérdida.",Mt,se,Pt="Como puedes ver, la recompensa se acerca a 0 a medida que el modelo aprende. Es una buena señal de que está aprendiendo a generar texto con la longitud correcta.",ct,ie,Dt='<img src="https://huggingface.co/reasoning-course/images/resolve/main/grpo/13.png" alt="Reward from reward function"/>',ut,re,Ot="Puede que observes que la pérdida empieza en cero y luego aumenta durante el entrenamiento. Aunque parezca contraintuitivo, ese comportamiento es esperable en GRPO. La pérdida es proporcional a la divergencia KL respecto a la política original. A medida que el modelo aprende a optimizar mejor la función de recompensa, se aleja más de su política inicial y eso se refleja en la pérdida.",dt,oe,Kt='<img src="https://huggingface.co/reasoning-course/images/resolve/main/grpo/14.png" alt="Loss"/>',yt,me,Jt,pe,el="Compartamos el modelo con la comunidad.",Tt,Me,bt,ce,wt,ue,tl="Ya has ajustado un modelo con GRPO. Ahora generemos texto con él.",ft,de,ll="Primero definiremos un documento largo.",Ut,ye,ht,Je,al="Ahora podemos generar texto con el modelo.",gt,Te,jt,be,$t,we,nl="En este capítulo hemos visto cómo ajustar un modelo con GRPO, interpretar los resultados del entrenamiento y generar texto con el modelo resultante.",It,fe,Ct,he,Gt;return w=new yl({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/course/en/chapter12/grpo_finetune.ipynb"}]}}),f=new ul({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),U=new y({props:{title:"Ejercicio práctico: ajusta un modelo con GRPO",local:"ejercicio-práctico-ajusta-un-modelo-con-grpo",headingTag:"h1"}}),g=new y({props:{title:"Instalar dependencias",local:"instalar-dependencias",headingTag:"h2"}}),$=new J({props:{code:"IXBpcCUyMGluc3RhbGwlMjAtcXFxJTIwZGF0YXNldHMlM0QlM0QzLjIuMCUyMHRyYW5zZm9ybWVycyUzRCUzRDQuNDcuMSUyMHRybCUzRCUzRDAuMTQuMCUyMHBlZnQlM0QlM0QwLjE0LjAlMjBhY2NlbGVyYXRlJTNEJTNEMS4yLjElMjBiaXRzYW5kYnl0ZXMlM0QlM0QwLjQ1LjIlMjB3YW5kYiUzRCUzRDAuMTkuNyUyMC0tcHJvZ3Jlc3MtYmFyJTIwb2ZmJTBBIXBpcCUyMGluc3RhbGwlMjAtcXFxJTIwZmxhc2gtYXR0biUyMC0tbm8tYnVpbGQtaXNvbGF0aW9uJTIwLS1wcm9ncmVzcy1iYXIlMjBvZmY=",highlighted:`!pip install -qqq datasets==3.2.0 transformers==4.47.1 trl==0.14.0 peft==0.14.0 accelerate==1.2.1 bitsandbytes==0.45.2 wandb==0.19.7 --progress-bar off
!pip install -qqq flash-attn --no-build-isolation --progress-bar off`,wrap:!1}}),C=new J({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGF0YXNldHMlMjBpbXBvcnQlMjBsb2FkX2RhdGFzZXQlMEFmcm9tJTIwcGVmdCUyMGltcG9ydCUyMExvcmFDb25maWclMkMlMjBnZXRfcGVmdF9tb2RlbCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUyQyUyMEF1dG9Ub2tlbml6ZXIlMEFmcm9tJTIwdHJsJTIwaW1wb3J0JTIwR1JQT0NvbmZpZyUyQyUyMEdSUE9UcmFpbmVy",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> LoraConfig, get_peft_model
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOConfig, GRPOTrainer`,wrap:!1}}),G=new y({props:{title:"Importar e iniciar sesión en Weights & Biases",local:"importar-e-iniciar-sesión-en-weights--biases",headingTag:"h2"}}),Z=new J({props:{code:"aW1wb3J0JTIwd2FuZGIlMEElMEF3YW5kYi5sb2dpbigp",highlighted:`<span class="hljs-keyword">import</span> wandb

wandb.login()`,wrap:!1}}),R=new y({props:{title:"Cargar el dataset",local:"cargar-el-dataset",headingTag:"h2"}}),x=new J({props:{code:"ZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJtbGFib25uZSUyRnNtb2x0bGRyJTIyKSUwQXByaW50KGRhdGFzZXQp",highlighted:`dataset = load_dataset(<span class="hljs-string">&quot;mlabonne/smoltldr&quot;</span>)
<span class="hljs-built_in">print</span>(dataset)`,wrap:!1}}),X=new y({props:{title:"Cargar el modelo",local:"cargar-el-modelo",headingTag:"h2"}}),_=new J({props:{code:"bW9kZWxfaWQlMjAlM0QlMjAlMjJIdWdnaW5nRmFjZVRCJTJGU21vbExNLTEzNU0tSW5zdHJ1Y3QlMjIlMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjBtb2RlbF9pZCUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwYXR0bl9pbXBsZW1lbnRhdGlvbiUzRCUyMmZsYXNoX2F0dGVudGlvbl8yJTIyJTJDJTBBKSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKQ==",highlighted:`model_id = <span class="hljs-string">&quot;HuggingFaceTB/SmolLM-135M-Instruct&quot;</span>
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
    attn_implementation=<span class="hljs-string">&quot;flash_attention_2&quot;</span>,
)
tokenizer = AutoTokenizer.from_pretrained(model_id)`,wrap:!1}}),Q=new y({props:{title:"Cargar LoRA",local:"cargar-lora",headingTag:"h2"}}),z=new J({props:{code:"JTIzJTIwTG9hZCUyMExvUkElMEFsb3JhX2NvbmZpZyUyMCUzRCUyMExvcmFDb25maWcoJTBBJTIwJTIwJTIwJTIwdGFza190eXBlJTNEJTIyQ0FVU0FMX0xNJTIyJTJDJTBBJTIwJTIwJTIwJTIwciUzRDE2JTJDJTBBJTIwJTIwJTIwJTIwbG9yYV9hbHBoYSUzRDMyJTJDJTBBJTIwJTIwJTIwJTIwdGFyZ2V0X21vZHVsZXMlM0QlMjJhbGwtbGluZWFyJTIyJTJDJTBBKSUwQW1vZGVsJTIwJTNEJTIwZ2V0X3BlZnRfbW9kZWwobW9kZWwlMkMlMjBsb3JhX2NvbmZpZyklMEFwcmludChtb2RlbC5wcmludF90cmFpbmFibGVfcGFyYW1ldGVycygpKQ==",highlighted:`<span class="hljs-comment"># Load LoRA</span>
lora_config = LoraConfig(
    task_type=<span class="hljs-string">&quot;CAUSAL_LM&quot;</span>,
    r=<span class="hljs-number">16</span>,
    lora_alpha=<span class="hljs-number">32</span>,
    target_modules=<span class="hljs-string">&quot;all-linear&quot;</span>,
)
model = get_peft_model(model, lora_config)
<span class="hljs-built_in">print</span>(model.print_trainable_parameters())`,wrap:!1}}),Y=new J({props:{code:"VG90YWwlMjB0cmFpbmFibGUlMjBwYXJhbWV0ZXJzJTNBJTIwMTM1TQ==",highlighted:"Total trainable parameters: 135M",wrap:!1}}),H=new y({props:{title:"Definir la función de recompensa",local:"definir-la-función-de-recompensa",headingTag:"h2"}}),q=new J({props:{code:"JTIzJTIwUmV3YXJkJTIwZnVuY3Rpb24lMEFpZGVhbF9sZW5ndGglMjAlM0QlMjA1MCUwQSUwQSUwQWRlZiUyMHJld2FyZF9sZW4oY29tcGxldGlvbnMlMkMlMjAqKmt3YXJncyklM0ElMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlNUItYWJzKGlkZWFsX2xlbmd0aCUyMC0lMjBsZW4oY29tcGxldGlvbikpJTIwZm9yJTIwY29tcGxldGlvbiUyMGluJTIwY29tcGxldGlvbnMlNUQ=",highlighted:`<span class="hljs-comment"># Reward function</span>
ideal_length = <span class="hljs-number">50</span>


<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_len</span>(<span class="hljs-params">completions, **kwargs</span>):
    <span class="hljs-keyword">return</span> [-<span class="hljs-built_in">abs</span>(ideal_length - <span class="hljs-built_in">len</span>(completion)) <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]`,wrap:!1}}),S=new y({props:{title:"Definir los argumentos de entrenamiento",local:"definir-los-argumentos-de-entrenamiento",headingTag:"h2"}}),P=new J({props:{code:"JTIzJTIwVHJhaW5pbmclMjBhcmd1bWVudHMlMEF0cmFpbmluZ19hcmdzJTIwJTNEJTIwR1JQT0NvbmZpZyglMEElMjAlMjAlMjAlMjBvdXRwdXRfZGlyJTNEJTIyR1JQTyUyMiUyQyUwQSUyMCUyMCUyMCUyMGxlYXJuaW5nX3JhdGUlM0QyZS01JTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTNEOCUyQyUwQSUyMCUyMCUyMCUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUzRDIlMkMlMEElMjAlMjAlMjAlMjBtYXhfcHJvbXB0X2xlbmd0aCUzRDUxMiUyQyUwQSUyMCUyMCUyMCUyMG1heF9jb21wbGV0aW9uX2xlbmd0aCUzRDk2JTJDJTBBJTIwJTIwJTIwJTIwbnVtX2dlbmVyYXRpb25zJTNEOCUyQyUwQSUyMCUyMCUyMCUyMG9wdGltJTNEJTIyYWRhbXdfOGJpdCUyMiUyQyUwQSUyMCUyMCUyMCUyMG51bV90cmFpbl9lcG9jaHMlM0QxJTJDJTBBJTIwJTIwJTIwJTIwYmYxNiUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjByZXBvcnRfdG8lM0QlNUIlMjJ3YW5kYiUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMHJlbW92ZV91bnVzZWRfY29sdW1ucyUzREZhbHNlJTJDJTBBJTIwJTIwJTIwJTIwbG9nZ2luZ19zdGVwcyUzRDElMkMlMEEp",highlighted:`<span class="hljs-comment"># Training arguments</span>
training_args = GRPOConfig(
    output_dir=<span class="hljs-string">&quot;GRPO&quot;</span>,
    learning_rate=<span class="hljs-number">2e-5</span>,
    per_device_train_batch_size=<span class="hljs-number">8</span>,
    gradient_accumulation_steps=<span class="hljs-number">2</span>,
    max_prompt_length=<span class="hljs-number">512</span>,
    max_completion_length=<span class="hljs-number">96</span>,
    num_generations=<span class="hljs-number">8</span>,
    optim=<span class="hljs-string">&quot;adamw_8bit&quot;</span>,
    num_train_epochs=<span class="hljs-number">1</span>,
    bf16=<span class="hljs-literal">True</span>,
    report_to=[<span class="hljs-string">&quot;wandb&quot;</span>],
    remove_unused_columns=<span class="hljs-literal">False</span>,
    logging_steps=<span class="hljs-number">1</span>,
)`,wrap:!1}}),O=new J({props:{code:"JTIzJTIwVHJhaW5lciUwQXRyYWluZXIlMjAlM0QlMjBHUlBPVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwcmV3YXJkX2Z1bmNzJTNEJTVCcmV3YXJkX2xlbiU1RCUyQyUwQSUyMCUyMCUyMCUyMGFyZ3MlM0R0cmFpbmluZ19hcmdzJTJDJTBBJTIwJTIwJTIwJTIwdHJhaW5fZGF0YXNldCUzRGRhdGFzZXQlNUIlMjJ0cmFpbiUyMiU1RCUyQyUwQSklMEElMEElMjMlMjBUcmFpbiUyMG1vZGVsJTBBd2FuZGIuaW5pdChwcm9qZWN0JTNEJTIyR1JQTyUyMiklMEF0cmFpbmVyLnRyYWluKCk=",highlighted:`<span class="hljs-comment"># Trainer</span>
trainer = GRPOTrainer(
    model=model,
    reward_funcs=[reward_len],
    args=training_args,
    train_dataset=dataset[<span class="hljs-string">&quot;train&quot;</span>],
)

<span class="hljs-comment"># Train model</span>
wandb.init(project=<span class="hljs-string">&quot;GRPO&quot;</span>)
trainer.train()`,wrap:!1}}),ee=new y({props:{title:"Subir el modelo al Hub durante el entrenamiento",local:"subir-el-modelo-al-hub-durante-el-entrenamiento",headingTag:"h2"}}),le=new y({props:{title:"Interpretar los resultados del entrenamiento",local:"interpretar-los-resultados-del-entrenamiento",headingTag:"h2"}}),me=new y({props:{title:"Guardar y publicar el modelo",local:"guardar-y-publicar-el-modelo",headingTag:"h2"}}),Me=new J({props:{code:"bWVyZ2VkX21vZGVsJTIwJTNEJTIwdHJhaW5lci5tb2RlbC5tZXJnZV9hbmRfdW5sb2FkKCklMEFtZXJnZWRfbW9kZWwucHVzaF90b19odWIoJTBBJTIwJTIwJTIwJTIwJTIyU21vbEdSUE8tMTM1TSUyMiUyQyUyMHByaXZhdGUlM0RGYWxzZSUyQyUyMHRhZ3MlM0QlNUIlMjJHUlBPJTIyJTJDJTIwJTIyUmVhc29uaW5nLUNvdXJzZSUyMiU1RCUwQSk=",highlighted:`merged_model = trainer.model.merge_and_unload()
merged_model.push_to_hub(
    <span class="hljs-string">&quot;SmolGRPO-135M&quot;</span>, private=<span class="hljs-literal">False</span>, tags=[<span class="hljs-string">&quot;GRPO&quot;</span>, <span class="hljs-string">&quot;Reasoning-Course&quot;</span>]
)`,wrap:!1}}),ce=new y({props:{title:"Generar texto",local:"generar-texto",headingTag:"h2"}}),ye=new J({props:{code:"cHJvbXB0JTIwJTNEJTIwJTIyJTIyJTIyJTBBJTIzJTIwQSUyMGxvbmclMjBkb2N1bWVudCUyMGFib3V0JTIwdGhlJTIwQ2F0JTBBJTBBVGhlJTIwY2F0JTIwKEZlbGlzJTIwY2F0dXMpJTJDJTIwYWxzbyUyMHJlZmVycmVkJTIwdG8lMjBhcyUyMHRoZSUyMGRvbWVzdGljJTIwY2F0JTIwb3IlMjBob3VzZSUyMGNhdCUyQyUyMGlzJTIwYSUyMHNtYWxsJTIwJTBBZG9tZXN0aWNhdGVkJTIwY2Fybml2b3JvdXMlMjBtYW1tYWwuJTIwSXQlMjBpcyUyMHRoZSUyMG9ubHklMjBkb21lc3RpY2F0ZWQlMjBzcGVjaWVzJTIwb2YlMjB0aGUlMjBmYW1pbHklMjBGZWxpZGFlLiUwQUFkdmFuY2VzJTIwaW4lMjBhcmNoYWVvbG9neSUyMGFuZCUyMGdlbmV0aWNzJTIwaGF2ZSUyMHNob3duJTIwdGhhdCUyMHRoZSUyMGRvbWVzdGljYXRpb24lMjBvZiUyMHRoZSUyMGNhdCUyMG9jY3VycmVkJTBBaW4lMjB0aGUlMjBOZWFyJTIwRWFzdCUyMGFyb3VuZCUyMDc1MDAlMjBCQy4lMjBJdCUyMGlzJTIwY29tbW9ubHklMjBrZXB0JTIwYXMlMjBhJTIwcGV0JTIwYW5kJTIwZmFybSUyMGNhdCUyQyUyMGJ1dCUyMGFsc28lMjByYW5nZXMlMEFmcmVlbHklMjBhcyUyMGElMjBmZXJhbCUyMGNhdCUyMGF2b2lkaW5nJTIwaHVtYW4lMjBjb250YWN0LiUyMEl0JTIwaXMlMjB2YWx1ZWQlMjBieSUyMGh1bWFucyUyMGZvciUyMGNvbXBhbmlvbnNoaXAlMjBhbmQlMEFpdHMlMjBhYmlsaXR5JTIwdG8lMjBraWxsJTIwdmVybWluLiUyMEl0cyUyMHJldHJhY3RhYmxlJTIwY2xhd3MlMjBhcmUlMjBhZGFwdGVkJTIwdG8lMjBraWxsaW5nJTIwc21hbGwlMjBwcmV5JTIwc3BlY2llcyUwQXN1Y2glMjBhcyUyMG1pY2UlMjBhbmQlMjByYXRzLiUyMEl0JTIwaGFzJTIwYSUyMHN0cm9uZyUyQyUyMGZsZXhpYmxlJTIwYm9keSUyQyUyMHF1aWNrJTIwcmVmbGV4ZXMlMkMlMjBhbmQlMjBzaGFycCUyMHRlZXRoJTJDJTBBYW5kJTIwaXRzJTIwbmlnaHQlMjB2aXNpb24lMjBhbmQlMjBzZW5zZSUyMG9mJTIwc21lbGwlMjBhcmUlMjB3ZWxsJTIwZGV2ZWxvcGVkLiUyMEl0JTIwaXMlMjBhJTIwc29jaWFsJTIwc3BlY2llcyUyQyUwQWJ1dCUyMGElMjBzb2xpdGFyeSUyMGh1bnRlciUyMGFuZCUyMGElMjBjcmVwdXNjdWxhciUyMHByZWRhdG9yLiUyMENhdCUyMGNvbW11bmljYXRpb24lMjBpbmNsdWRlcyUwQXZvY2FsaXphdGlvbnMlRTIlODAlOTRpbmNsdWRpbmclMjBtZW93aW5nJTJDJTIwcHVycmluZyUyQyUyMHRyaWxsaW5nJTJDJTIwaGlzc2luZyUyQyUyMGdyb3dsaW5nJTJDJTIwYW5kJTIwZ3J1bnRpbmclRTIlODAlOTRhcyUwQXdlbGwlMjBhcyUyMGJvZHklMjBsYW5ndWFnZS4lMjBJdCUyMGNhbiUyMGhlYXIlMjBzb3VuZHMlMjB0b28lMjBmYWludCUyMG9yJTIwdG9vJTIwaGlnaCUyMGluJTIwZnJlcXVlbmN5JTIwZm9yJTIwaHVtYW4lMjBlYXJzJTJDJTBBc3VjaCUyMGFzJTIwdGhvc2UlMjBtYWRlJTIwYnklMjBzbWFsbCUyMG1hbW1hbHMuJTIwSXQlMjBzZWNyZXRlcyUyMGFuZCUyMHBlcmNlaXZlcyUyMHBoZXJvbW9uZXMuJTBBJTIyJTIyJTIyJTBBJTBBbWVzc2FnZXMlMjAlM0QlMjAlNUIlMEElMjAlMjAlMjAlMjAlN0IlMjJyb2xlJTIyJTNBJTIwJTIydXNlciUyMiUyQyUyMCUyMmNvbnRlbnQlMjIlM0ElMjBwcm9tcHQlN0QlMkMlMEElNUQ=",highlighted:`prompt = <span class="hljs-string">&quot;&quot;&quot;
# A long document about the Cat

The cat (Felis catus), also referred to as the domestic cat or house cat, is a small 
domesticated carnivorous mammal. It is the only domesticated species of the family Felidae.
Advances in archaeology and genetics have shown that the domestication of the cat occurred
in the Near East around 7500 BC. It is commonly kept as a pet and farm cat, but also ranges
freely as a feral cat avoiding human contact. It is valued by humans for companionship and
its ability to kill vermin. Its retractable claws are adapted to killing small prey species
such as mice and rats. It has a strong, flexible body, quick reflexes, and sharp teeth,
and its night vision and sense of smell are well developed. It is a social species,
but a solitary hunter and a crepuscular predator. Cat communication includes
vocalizations—including meowing, purring, trilling, hissing, growling, and grunting—as
well as body language. It can hear sounds too faint or too high in frequency for human ears,
such as those made by small mammals. It secretes and perceives pheromones.
&quot;&quot;&quot;</span>

messages = [
    {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: prompt},
]`,wrap:!1}}),Te=new J({props:{code:"JTIzJTIwR2VuZXJhdGUlMjB0ZXh0JTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBZ2VuZXJhdG9yJTIwJTNEJTIwcGlwZWxpbmUoJTIydGV4dC1nZW5lcmF0aW9uJTIyJTJDJTIwbW9kZWwlM0QlMjJTbW9sR1JQTy0xMzVNJTIyKSUwQSUwQSUyMyUyMyUyME9yJTIwdXNlJTIwdGhlJTIwbW9kZWwlMjBhbmQlMjB0b2tlbml6ZXIlMjB3ZSUyMGRlZmluZWQlMjBlYXJsaWVyJTBBJTIzJTIwZ2VuZXJhdG9yJTIwJTNEJTIwcGlwZWxpbmUoJTIydGV4dC1nZW5lcmF0aW9uJTIyJTJDJTIwbW9kZWwlM0Rtb2RlbCUyQyUyMHRva2VuaXplciUzRHRva2VuaXplciklMEElMEFnZW5lcmF0ZV9rd2FyZ3MlMjAlM0QlMjAlN0IlMEElMjAlMjAlMjAlMjAlMjJtYXhfbmV3X3Rva2VucyUyMiUzQSUyMDI1NiUyQyUwQSUyMCUyMCUyMCUyMCUyMmRvX3NhbXBsZSUyMiUzQSUyMFRydWUlMkMlMEElMjAlMjAlMjAlMjAlMjJ0ZW1wZXJhdHVyZSUyMiUzQSUyMDAuNSUyQyUwQSUyMCUyMCUyMCUyMCUyMm1pbl9wJTIyJTNBJTIwMC4xJTJDJTBBJTdEJTBBJTBBZ2VuZXJhdGVkX3RleHQlMjAlM0QlMjBnZW5lcmF0b3IobWVzc2FnZXMlMkMlMjBnZW5lcmF0ZV9rd2FyZ3MlM0RnZW5lcmF0ZV9rd2FyZ3MpJTBBJTBBcHJpbnQoZ2VuZXJhdGVkX3RleHQp",highlighted:`<span class="hljs-comment"># Generate text</span>
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

generator = pipeline(<span class="hljs-string">&quot;text-generation&quot;</span>, model=<span class="hljs-string">&quot;SmolGRPO-135M&quot;</span>)

<span class="hljs-comment">## Or use the model and tokenizer we defined earlier</span>
<span class="hljs-comment"># generator = pipeline(&quot;text-generation&quot;, model=model, tokenizer=tokenizer)</span>

generate_kwargs = {
    <span class="hljs-string">&quot;max_new_tokens&quot;</span>: <span class="hljs-number">256</span>,
    <span class="hljs-string">&quot;do_sample&quot;</span>: <span class="hljs-literal">True</span>,
    <span class="hljs-string">&quot;temperature&quot;</span>: <span class="hljs-number">0.5</span>,
    <span class="hljs-string">&quot;min_p&quot;</span>: <span class="hljs-number">0.1</span>,
}

generated_text = generator(messages, generate_kwargs=generate_kwargs)

<span class="hljs-built_in">print</span>(generated_text)`,wrap:!1}}),be=new y({props:{title:"Conclusión",local:"conclusión",headingTag:"h1"}}),fe=new dl({props:{source:"https://github.com/huggingface/course/blob/main/chapters/es/chapter12/5.mdx"}}),{c(){T=i("meta"),ge=n(),Ue=i("p"),je=n(),o(w.$$.fragment),$e=n(),o(f.$$.fragment),Ie=n(),o(U.$$.fragment),Ce=n(),h=i("p"),h.textContent=Wt,Ge=n(),b=i("blockquote"),b.innerHTML=Rt,Be=n(),o(g.$$.fragment),Ze=n(),j=i("p"),j.textContent=vt,We=n(),o($.$$.fragment),Re=n(),I=i("p"),I.textContent=xt,ve=n(),o(C.$$.fragment),xe=n(),o(G.$$.fragment),Xe=n(),B=i("p"),B.textContent=Xt,Fe=n(),o(Z.$$.fragment),ke=n(),W=i("p"),W.textContent=Ft,_e=n(),o(R.$$.fragment),Qe=n(),v=i("p"),v.innerHTML=kt,Ve=n(),o(x.$$.fragment),Ee=n(),o(X.$$.fragment),ze=n(),F=i("p"),F.innerHTML=_t,Ye=n(),k=i("p"),k.innerHTML=Qt,He=n(),o(_.$$.fragment),Ne=n(),o(Q.$$.fragment),qe=n(),V=i("p"),V.textContent=Vt,Se=n(),E=i("p"),E.innerHTML=Et,Ae=n(),o(z.$$.fragment),Le=n(),o(Y.$$.fragment),Pe=n(),o(H.$$.fragment),De=n(),N=i("p"),N.textContent=zt,Oe=n(),o(q.$$.fragment),Ke=n(),o(S.$$.fragment),et=n(),A=i("p"),A.innerHTML=Yt,tt=n(),L=i("p"),L.innerHTML=Ht,lt=n(),o(P.$$.fragment),at=n(),D=i("p"),D.textContent=Nt,nt=n(),o(O.$$.fragment),st=n(),K=i("p"),K.textContent=qt,it=n(),o(ee.$$.fragment),rt=n(),te=i("p"),te.innerHTML=St,ot=n(),o(le.$$.fragment),mt=n(),ae=i("p"),ae.innerHTML=At,pt=n(),ne=i("p"),ne.textContent=Lt,Mt=n(),se=i("p"),se.textContent=Pt,ct=n(),ie=i("p"),ie.innerHTML=Dt,ut=n(),re=i("p"),re.textContent=Ot,dt=n(),oe=i("p"),oe.innerHTML=Kt,yt=n(),o(me.$$.fragment),Jt=n(),pe=i("p"),pe.textContent=el,Tt=n(),o(Me.$$.fragment),bt=n(),o(ce.$$.fragment),wt=n(),ue=i("p"),ue.textContent=tl,ft=n(),de=i("p"),de.textContent=ll,Ut=n(),o(ye.$$.fragment),ht=n(),Je=i("p"),Je.textContent=al,gt=n(),o(Te.$$.fragment),jt=n(),o(be.$$.fragment),$t=n(),we=i("p"),we.textContent=nl,It=n(),o(fe.$$.fragment),Ct=n(),he=i("p"),this.h()},l(e){const t=Ml("svelte-u9bgzb",document.head);T=r(t,"META",{name:!0,content:!0}),t.forEach(l),ge=s(e),Ue=r(e,"P",{}),sl(Ue).forEach(l),je=s(e),m(w.$$.fragment,e),$e=s(e),m(f.$$.fragment,e),Ie=s(e),m(U.$$.fragment,e),Ce=s(e),h=r(e,"P",{"data-svelte-h":!0}),d(h)!=="svelte-131nt95"&&(h.textContent=Wt),Ge=s(e),b=r(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),d(b)!=="svelte-1c3007"&&(b.innerHTML=Rt),Be=s(e),m(g.$$.fragment,e),Ze=s(e),j=r(e,"P",{"data-svelte-h":!0}),d(j)!=="svelte-14mro9g"&&(j.textContent=vt),We=s(e),m($.$$.fragment,e),Re=s(e),I=r(e,"P",{"data-svelte-h":!0}),d(I)!=="svelte-hdo7zi"&&(I.textContent=xt),ve=s(e),m(C.$$.fragment,e),xe=s(e),m(G.$$.fragment,e),Xe=s(e),B=r(e,"P",{"data-svelte-h":!0}),d(B)!=="svelte-6h6joa"&&(B.textContent=Xt),Fe=s(e),m(Z.$$.fragment,e),ke=s(e),W=r(e,"P",{"data-svelte-h":!0}),d(W)!=="svelte-18k0at8"&&(W.textContent=Ft),_e=s(e),m(R.$$.fragment,e),Qe=s(e),v=r(e,"P",{"data-svelte-h":!0}),d(v)!=="svelte-1qmirj4"&&(v.innerHTML=kt),Ve=s(e),m(x.$$.fragment,e),Ee=s(e),m(X.$$.fragment,e),ze=s(e),F=r(e,"P",{"data-svelte-h":!0}),d(F)!=="svelte-13gco5a"&&(F.innerHTML=_t),Ye=s(e),k=r(e,"P",{"data-svelte-h":!0}),d(k)!=="svelte-41qzy1"&&(k.innerHTML=Qt),He=s(e),m(_.$$.fragment,e),Ne=s(e),m(Q.$$.fragment,e),qe=s(e),V=r(e,"P",{"data-svelte-h":!0}),d(V)!=="svelte-1yic9hm"&&(V.textContent=Vt),Se=s(e),E=r(e,"P",{"data-svelte-h":!0}),d(E)!=="svelte-ivspty"&&(E.innerHTML=Et),Ae=s(e),m(z.$$.fragment,e),Le=s(e),m(Y.$$.fragment,e),Pe=s(e),m(H.$$.fragment,e),De=s(e),N=r(e,"P",{"data-svelte-h":!0}),d(N)!=="svelte-cc8088"&&(N.textContent=zt),Oe=s(e),m(q.$$.fragment,e),Ke=s(e),m(S.$$.fragment,e),et=s(e),A=r(e,"P",{"data-svelte-h":!0}),d(A)!=="svelte-1kczmfu"&&(A.innerHTML=Yt),tt=s(e),L=r(e,"P",{"data-svelte-h":!0}),d(L)!=="svelte-1mw5e34"&&(L.innerHTML=Ht),lt=s(e),m(P.$$.fragment,e),at=s(e),D=r(e,"P",{"data-svelte-h":!0}),d(D)!=="svelte-1b3kokq"&&(D.textContent=Nt),nt=s(e),m(O.$$.fragment,e),st=s(e),K=r(e,"P",{"data-svelte-h":!0}),d(K)!=="svelte-10o0sxx"&&(K.textContent=qt),it=s(e),m(ee.$$.fragment,e),rt=s(e),te=r(e,"P",{"data-svelte-h":!0}),d(te)!=="svelte-1fm8eaw"&&(te.innerHTML=St),ot=s(e),m(le.$$.fragment,e),mt=s(e),ae=r(e,"P",{"data-svelte-h":!0}),d(ae)!=="svelte-1kve4uf"&&(ae.innerHTML=At),pt=s(e),ne=r(e,"P",{"data-svelte-h":!0}),d(ne)!=="svelte-19qtkhx"&&(ne.textContent=Lt),Mt=s(e),se=r(e,"P",{"data-svelte-h":!0}),d(se)!=="svelte-hlx6o1"&&(se.textContent=Pt),ct=s(e),ie=r(e,"P",{"data-svelte-h":!0}),d(ie)!=="svelte-nb9yq5"&&(ie.innerHTML=Dt),ut=s(e),re=r(e,"P",{"data-svelte-h":!0}),d(re)!=="svelte-1k438xn"&&(re.textContent=Ot),dt=s(e),oe=r(e,"P",{"data-svelte-h":!0}),d(oe)!=="svelte-1bbe9id"&&(oe.innerHTML=Kt),yt=s(e),m(me.$$.fragment,e),Jt=s(e),pe=r(e,"P",{"data-svelte-h":!0}),d(pe)!=="svelte-6ag5r0"&&(pe.textContent=el),Tt=s(e),m(Me.$$.fragment,e),bt=s(e),m(ce.$$.fragment,e),wt=s(e),ue=r(e,"P",{"data-svelte-h":!0}),d(ue)!=="svelte-yz8jx9"&&(ue.textContent=tl),ft=s(e),de=r(e,"P",{"data-svelte-h":!0}),d(de)!=="svelte-1m8gect"&&(de.textContent=ll),Ut=s(e),m(ye.$$.fragment,e),ht=s(e),Je=r(e,"P",{"data-svelte-h":!0}),d(Je)!=="svelte-12zozzh"&&(Je.textContent=al),gt=s(e),m(Te.$$.fragment,e),jt=s(e),m(be.$$.fragment,e),$t=s(e),we=r(e,"P",{"data-svelte-h":!0}),d(we)!=="svelte-1ei1xp2"&&(we.textContent=nl),It=s(e),m(fe.$$.fragment,e),Ct=s(e),he=r(e,"P",{}),sl(he).forEach(l),this.h()},h(){Bt(T,"name","hf:doc:metadata"),Bt(T,"content",Tl),Bt(b,"class","tip")},m(e,t){cl(document.head,T),a(e,ge,t),a(e,Ue,t),a(e,je,t),p(w,e,t),a(e,$e,t),p(f,e,t),a(e,Ie,t),p(U,e,t),a(e,Ce,t),a(e,h,t),a(e,Ge,t),a(e,b,t),a(e,Be,t),p(g,e,t),a(e,Ze,t),a(e,j,t),a(e,We,t),p($,e,t),a(e,Re,t),a(e,I,t),a(e,ve,t),p(C,e,t),a(e,xe,t),p(G,e,t),a(e,Xe,t),a(e,B,t),a(e,Fe,t),p(Z,e,t),a(e,ke,t),a(e,W,t),a(e,_e,t),p(R,e,t),a(e,Qe,t),a(e,v,t),a(e,Ve,t),p(x,e,t),a(e,Ee,t),p(X,e,t),a(e,ze,t),a(e,F,t),a(e,Ye,t),a(e,k,t),a(e,He,t),p(_,e,t),a(e,Ne,t),p(Q,e,t),a(e,qe,t),a(e,V,t),a(e,Se,t),a(e,E,t),a(e,Ae,t),p(z,e,t),a(e,Le,t),p(Y,e,t),a(e,Pe,t),p(H,e,t),a(e,De,t),a(e,N,t),a(e,Oe,t),p(q,e,t),a(e,Ke,t),p(S,e,t),a(e,et,t),a(e,A,t),a(e,tt,t),a(e,L,t),a(e,lt,t),p(P,e,t),a(e,at,t),a(e,D,t),a(e,nt,t),p(O,e,t),a(e,st,t),a(e,K,t),a(e,it,t),p(ee,e,t),a(e,rt,t),a(e,te,t),a(e,ot,t),p(le,e,t),a(e,mt,t),a(e,ae,t),a(e,pt,t),a(e,ne,t),a(e,Mt,t),a(e,se,t),a(e,ct,t),a(e,ie,t),a(e,ut,t),a(e,re,t),a(e,dt,t),a(e,oe,t),a(e,yt,t),p(me,e,t),a(e,Jt,t),a(e,pe,t),a(e,Tt,t),p(Me,e,t),a(e,bt,t),p(ce,e,t),a(e,wt,t),a(e,ue,t),a(e,ft,t),a(e,de,t),a(e,Ut,t),p(ye,e,t),a(e,ht,t),a(e,Je,t),a(e,gt,t),p(Te,e,t),a(e,jt,t),p(be,e,t),a(e,$t,t),a(e,we,t),a(e,It,t),p(fe,e,t),a(e,Ct,t),a(e,he,t),Gt=!0},p:rl,i(e){Gt||(M(w.$$.fragment,e),M(f.$$.fragment,e),M(U.$$.fragment,e),M(g.$$.fragment,e),M($.$$.fragment,e),M(C.$$.fragment,e),M(G.$$.fragment,e),M(Z.$$.fragment,e),M(R.$$.fragment,e),M(x.$$.fragment,e),M(X.$$.fragment,e),M(_.$$.fragment,e),M(Q.$$.fragment,e),M(z.$$.fragment,e),M(Y.$$.fragment,e),M(H.$$.fragment,e),M(q.$$.fragment,e),M(S.$$.fragment,e),M(P.$$.fragment,e),M(O.$$.fragment,e),M(ee.$$.fragment,e),M(le.$$.fragment,e),M(me.$$.fragment,e),M(Me.$$.fragment,e),M(ce.$$.fragment,e),M(ye.$$.fragment,e),M(Te.$$.fragment,e),M(be.$$.fragment,e),M(fe.$$.fragment,e),Gt=!0)},o(e){c(w.$$.fragment,e),c(f.$$.fragment,e),c(U.$$.fragment,e),c(g.$$.fragment,e),c($.$$.fragment,e),c(C.$$.fragment,e),c(G.$$.fragment,e),c(Z.$$.fragment,e),c(R.$$.fragment,e),c(x.$$.fragment,e),c(X.$$.fragment,e),c(_.$$.fragment,e),c(Q.$$.fragment,e),c(z.$$.fragment,e),c(Y.$$.fragment,e),c(H.$$.fragment,e),c(q.$$.fragment,e),c(S.$$.fragment,e),c(P.$$.fragment,e),c(O.$$.fragment,e),c(ee.$$.fragment,e),c(le.$$.fragment,e),c(me.$$.fragment,e),c(Me.$$.fragment,e),c(ce.$$.fragment,e),c(ye.$$.fragment,e),c(Te.$$.fragment,e),c(be.$$.fragment,e),c(fe.$$.fragment,e),Gt=!1},d(e){e&&(l(ge),l(Ue),l(je),l($e),l(Ie),l(Ce),l(h),l(Ge),l(b),l(Be),l(Ze),l(j),l(We),l(Re),l(I),l(ve),l(xe),l(Xe),l(B),l(Fe),l(ke),l(W),l(_e),l(Qe),l(v),l(Ve),l(Ee),l(ze),l(F),l(Ye),l(k),l(He),l(Ne),l(qe),l(V),l(Se),l(E),l(Ae),l(Le),l(Pe),l(De),l(N),l(Oe),l(Ke),l(et),l(A),l(tt),l(L),l(lt),l(at),l(D),l(nt),l(st),l(K),l(it),l(rt),l(te),l(ot),l(mt),l(ae),l(pt),l(ne),l(Mt),l(se),l(ct),l(ie),l(ut),l(re),l(dt),l(oe),l(yt),l(Jt),l(pe),l(Tt),l(bt),l(wt),l(ue),l(ft),l(de),l(Ut),l(ht),l(Je),l(gt),l(jt),l($t),l(we),l(It),l(Ct),l(he)),l(T),u(w,e),u(f,e),u(U,e),u(g,e),u($,e),u(C,e),u(G,e),u(Z,e),u(R,e),u(x,e),u(X,e),u(_,e),u(Q,e),u(z,e),u(Y,e),u(H,e),u(q,e),u(S,e),u(P,e),u(O,e),u(ee,e),u(le,e),u(me,e),u(Me,e),u(ce,e),u(ye,e),u(Te,e),u(be,e),u(fe,e)}}}const Tl='{"title":"Ejercicio práctico: ajusta un modelo con GRPO","local":"ejercicio-práctico-ajusta-un-modelo-con-grpo","sections":[{"title":"Instalar dependencias","local":"instalar-dependencias","sections":[],"depth":2},{"title":"Importar e iniciar sesión en Weights & Biases","local":"importar-e-iniciar-sesión-en-weights--biases","sections":[],"depth":2},{"title":"Cargar el dataset","local":"cargar-el-dataset","sections":[],"depth":2},{"title":"Cargar el modelo","local":"cargar-el-modelo","sections":[],"depth":2},{"title":"Cargar LoRA","local":"cargar-lora","sections":[],"depth":2},{"title":"Definir la función de recompensa","local":"definir-la-función-de-recompensa","sections":[],"depth":2},{"title":"Definir los argumentos de entrenamiento","local":"definir-los-argumentos-de-entrenamiento","sections":[],"depth":2},{"title":"Subir el modelo al Hub durante el entrenamiento","local":"subir-el-modelo-al-hub-durante-el-entrenamiento","sections":[],"depth":2},{"title":"Interpretar los resultados del entrenamiento","local":"interpretar-los-resultados-del-entrenamiento","sections":[],"depth":2},{"title":"Guardar y publicar el modelo","local":"guardar-y-publicar-el-modelo","sections":[],"depth":2},{"title":"Generar texto","local":"generar-texto","sections":[],"depth":2}],"depth":1}';function bl(Zt){return ol(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class jl extends ml{constructor(T){super(),pl(this,T,bl,Jl,il,{})}}export{jl as component};