Hugging Face's logo

Buckets:
hf-doc-build
/
doc-dev

Files
xet
hf-doc-build/doc-dev / course /pr_1356 /es /chapter12 /4.html
HuggingFaceDocBuilder's picture
HuggingFaceDocBuilder
download
raw
41.7 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Implementación de GRPO en TRL&quot;,&quot;local&quot;:&quot;implementacion-de-grpo-en-trl&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Componentes clave&quot;,&quot;local&quot;:&quot;componentes-clave&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;1. Formato del conjunto de datos&quot;,&quot;local&quot;:&quot;1-formato-del-conjunto-de-datos&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;2. Función de recompensa&quot;,&quot;local&quot;:&quot;2-función-de-recompensa&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;3. Configuración del entrenamiento&quot;,&quot;local&quot;:&quot;3-configuración-del-entrenamiento&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Consejos para el éxito&quot;,&quot;local&quot;:&quot;consejos-para-el-éxito&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Diseño de funciones de recompensa&quot;,&quot;local&quot;:&quot;diseño-de-funciones-de-recompensa&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;1. Recompensas basadas en la longitud&quot;,&quot;local&quot;:&quot;1-recompensas-basadas-en-la-longitud&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;2. Recompensas basadas en reglas para tareas verificables&quot;,&quot;local&quot;:&quot;2-recompensas-basadas-en-reglas-para-tareas-verificables&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;3. Recompensas basadas en formato&quot;,&quot;local&quot;:&quot;3-recompensas-basadas-en-formato&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;¡Eso es todo!&quot;,&quot;local&quot;:&quot;eso-es-todo&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
<link href="/docs/course/pr_1356/es/_app/immutable/entry/start.DeQFEJ0_.js" rel="modulepreload">
<link href="/docs/course/pr_1356/es/_app/immutable/chunks/MQVuurK6.js" rel="modulepreload">
<link href="/docs/course/pr_1356/es/_app/immutable/chunks/BA_dZbsI.js" rel="modulepreload">
<link href="/docs/course/pr_1356/es/_app/immutable/entry/app.fni0K-10.js" rel="modulepreload">
<link href="/docs/course/pr_1356/es/_app/immutable/chunks/C0jRxzmK.js" rel="modulepreload">
<link href="/docs/course/pr_1356/es/_app/immutable/chunks/DvbUaM1Q.js" rel="modulepreload">
<link href="/docs/course/pr_1356/es/_app/immutable/chunks/DPIcjav0.js" rel="modulepreload">
<link href="/docs/course/pr_1356/es/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
<link href="/docs/course/pr_1356/es/_app/immutable/nodes/0.dUMKTqid.js" rel="modulepreload">
<link href="/docs/course/pr_1356/es/_app/immutable/chunks/C3nIf8Go.js" rel="modulepreload">
<link href="/docs/course/pr_1356/es/_app/immutable/nodes/13.yJLfADa2.js" rel="modulepreload">
<link href="/docs/course/pr_1356/es/_app/immutable/chunks/DX9wbpOW.js" rel="modulepreload">
<link href="/docs/course/pr_1356/es/_app/immutable/chunks/Khighb8g.js" rel="modulepreload">
<!--v383be--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Implementación de GRPO en TRL&quot;,&quot;local&quot;:&quot;implementacion-de-grpo-en-trl&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Componentes clave&quot;,&quot;local&quot;:&quot;componentes-clave&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;1. Formato del conjunto de datos&quot;,&quot;local&quot;:&quot;1-formato-del-conjunto-de-datos&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;2. Función de recompensa&quot;,&quot;local&quot;:&quot;2-función-de-recompensa&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;3. Configuración del entrenamiento&quot;,&quot;local&quot;:&quot;3-configuración-del-entrenamiento&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Consejos para el éxito&quot;,&quot;local&quot;:&quot;consejos-para-el-éxito&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Diseño de funciones de recompensa&quot;,&quot;local&quot;:&quot;diseño-de-funciones-de-recompensa&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;1. Recompensas basadas en la longitud&quot;,&quot;local&quot;:&quot;1-recompensas-basadas-en-la-longitud&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;2. Recompensas basadas en reglas para tareas verificables&quot;,&quot;local&quot;:&quot;2-recompensas-basadas-en-reglas-para-tareas-verificables&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;3. Recompensas basadas en formato&quot;,&quot;local&quot;:&quot;3-recompensas-basadas-en-formato&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;¡Eso es todo!&quot;,&quot;local&quot;:&quot;eso-es-todo&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
<link href="/docs/course/pr_1356/es/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="implementacion-de-grpo-en-trl" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#implementacion-de-grpo-en-trl"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Implementación de GRPO en TRL</span></h1><!--]--><!----> <p>En esta página aprenderemos a implementar Group Relative Policy Optimization (GRPO) utilizando la librería Transformer Reinforcement Learning (TRL). Nos centraremos en la implementación práctica con una cantidad mínima de código.</p> <p>Exploraremos los conceptos centrales de GRPO tal como se incorporan en el <code>GRPOTrainer</code> de TRL, utilizando fragmentos de la documentación oficial de TRL como guía.</p> <blockquote class="tip"><p>Este capítulo está dirigido a principiantes en TRL. Si ya estás familiarizado con TRL, quizás también quieras consultar la <a href="https://github.com/huggingface/open-r1/blob/main/src/open_r1/grpo.py" rel="nofollow">implementación de Open R1</a> de GRPO.</p></blockquote> <p>En primer lugar, recordemos algunos de los conceptos importantes del algoritmo GRPO:</p> <ul><li>Formación de grupos: El modelo genera múltiples terminaciones para cada prompt.</li> <li>Aprendizaje de preferencias: El modelo aprende de una función de recompensa que compara grupos de terminaciones.</li> <li>Configuración del entrenamiento: El modelo utiliza una configuración para controlar el proceso de entrenamiento.</li></ul> <p>¿Qué necesitamos hacer para implementar GRPO?</p> <ul><li>Definir un conjunto de datos de prompts.</li> <li>Definir una función de recompensa que tome una lista de terminaciones y devuelva una lista de recompensas.</li> <li>Configurar el proceso de entrenamiento con un <code>GRPOConfig</code>.</li> <li>Entrenar el modelo utilizando el <code>GRPOTrainer</code>.</li></ul> <p>Aquí tienes un ejemplo mínimo para comenzar con el entrenamiento de GRPO:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOTrainer, GRPOConfig
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-comment"># 1. Cargar tu conjunto de datos</span>
dataset = load_dataset(<span class="hljs-string">&quot;your_dataset&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
<span class="hljs-comment"># 2. Definir una función de recompensa simple</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_func</span>(<span class="hljs-params">completions, **kwargs</span>):
<span class="hljs-string">&quot;&quot;&quot;Ejemplo: Recompensar terminaciones más largas&quot;&quot;&quot;</span>
<span class="hljs-keyword">return</span> [<span class="hljs-built_in">float</span>(<span class="hljs-built_in">len</span>(completion)) <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]
<span class="hljs-comment"># 3. Configurar el entrenamiento</span>
training_args = GRPOConfig(
output_dir=<span class="hljs-string">&quot;output&quot;</span>,
num_train_epochs=<span class="hljs-number">3</span>,
per_device_train_batch_size=<span class="hljs-number">4</span>,
gradient_accumulation_steps=<span class="hljs-number">2</span>,
logging_steps=<span class="hljs-number">10</span>,
)
<span class="hljs-comment"># 4. Inicializar y entrenar</span>
trainer = GRPOTrainer(
model=<span class="hljs-string">&quot;your_model&quot;</span>, <span class="hljs-comment"># ej. &quot;Qwen/Qwen2-0.5B-Instruct&quot;</span>
args=training_args,
train_dataset=dataset,
reward_funcs=reward_func,
)
trainer.train()<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="componentes-clave" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#componentes-clave"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Componentes clave</span></h2><!--]--><!----> <!--[2--><h3 class="relative group"><a id="1-formato-del-conjunto-de-datos" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#1-formato-del-conjunto-de-datos"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>1. Formato del conjunto de datos</span></h3><!--]--><!----> <p>Tu conjunto de datos debe contener prompts a los que responderá el modelo. El entrenador de GRPO generará múltiples terminaciones para cada prompt y utilizará la función de recompensa para compararlas.</p> <!--[2--><h3 class="relative group"><a id="2-función-de-recompensa" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#2-función-de-recompensa"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>2. Función de recompensa</span></h3><!--]--><!----> <p>La función de recompensa es crucial: determina cómo aprende el modelo. Aquí hay dos ejemplos prácticos:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-comment"># Ejemplo 1: Recompensa basada en la longitud de la terminación</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_length</span>(<span class="hljs-params">completions, **kwargs</span>):
<span class="hljs-keyword">return</span> [<span class="hljs-built_in">float</span>(<span class="hljs-built_in">len</span>(completion)) <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]
<span class="hljs-comment"># Ejemplo 2: Recompensa basada en coincidir con un patrón</span>
<span class="hljs-keyword">import</span> re
<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_format</span>(<span class="hljs-params">completions, **kwargs</span>):
pattern = <span class="hljs-string">r&quot;^&lt;think&gt;.*?&lt;/think&gt;&lt;answer&gt;.*?&lt;/answer&gt;$&quot;</span>
<span class="hljs-keyword">return</span> [<span class="hljs-number">1.0</span> <span class="hljs-keyword">if</span> re.<span class="hljs-keyword">match</span>(pattern, c) <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span> <span class="hljs-keyword">for</span> c <span class="hljs-keyword">in</span> completions]<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="3-configuración-del-entrenamiento" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#3-configuración-del-entrenamiento"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>3. Configuración del entrenamiento</span></h3><!--]--><!----> <p>Parámetros clave a considerar en <code>GRPOConfig</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->training_args = GRPOConfig(
<span class="hljs-comment"># Parámetros esenciales</span>
output_dir=<span class="hljs-string">&quot;output&quot;</span>,
num_train_epochs=<span class="hljs-number">3</span>,
num_generations=<span class="hljs-number">4</span>, <span class="hljs-comment"># Número de terminaciones a generar para cada prompt</span>
per_device_train_batch_size=<span class="hljs-number">4</span>, <span class="hljs-comment"># Queremos obtener todas las generaciones en un lote por dispositivo</span>
<span class="hljs-comment"># Opcionales pero útiles</span>
gradient_accumulation_steps=<span class="hljs-number">2</span>,
learning_rate=<span class="hljs-number">1e-5</span>,
logging_steps=<span class="hljs-number">10</span>,
<span class="hljs-comment"># Específicos de GRPO (opcionales)</span>
use_vllm=<span class="hljs-literal">True</span>, <span class="hljs-comment"># Acelerar la generación</span>
)<!----></pre></div><!----> <p>El parámetro <code>num_generations</code> es particularmente importante para GRPO ya que define el tamaño del grupo: cuántas terminaciones diferentes generará el modelo para cada prompt. Este es un diferenciador clave de otros métodos de RL:</p> <ul><li>Demasiado pequeño (ej., 2-3): Puede no proporcionar suficiente diversidad para comparaciones significativas</li> <li>Recomendado (4-16): Proporciona un buen equilibrio entre diversidad y eficiencia computacional</li> <li>Valores mayores: Pueden mejorar el aprendizaje pero aumentan significativamente el costo computacional</li></ul> <p>El tamaño del grupo debe elegirse según tus recursos computacionales y la complejidad de tu tarea. Para tareas simples, grupos más pequeños (4-8) pueden ser suficientes, mientras que tareas de razonamiento más complejas podrían beneficiarse de grupos más grandes (8-16).</p> <!--[1--><h2 class="relative group"><a id="consejos-para-el-éxito" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#consejos-para-el-éxito"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Consejos para el éxito</span></h2><!--]--><!----> <ol><li><strong>Gestión de memoria</strong>: Ajusta <code>per_device_train_batch_size</code> y <code>gradient_accumulation_steps</code> en función de la memoria de tu GPU.</li> <li><strong>Velocidad</strong>: Habilita <code>use_vllm=True</code> para una generación más rápida si tu modelo es compatible.</li> <li><strong>Monitoreo</strong>: Observa las métricas registradas durante el entrenamiento: <ul><li><code>reward</code>: Recompensa promedio en todas las terminaciones</li> <li><code>reward_std</code>: Desviación estándar dentro de los grupos de recompensa</li> <li><code>kl</code>: Divergencia KL del modelo de referencia</li></ul></li></ol> <!--[1--><h2 class="relative group"><a id="diseño-de-funciones-de-recompensa" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diseño-de-funciones-de-recompensa"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Diseño de funciones de recompensa</span></h2><!--]--><!----> <p>El artículo de DeepSeek R1 demuestra varios enfoques eficaces para el diseño de funciones de recompensa que puedes adaptar para tu propia implementación de GRPO:</p> <!--[2--><h3 class="relative group"><a id="1-recompensas-basadas-en-la-longitud" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#1-recompensas-basadas-en-la-longitud"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>1. Recompensas basadas en la longitud</span></h3><!--]--><!----> <p>Una de las recompensas más fáciles de implementar es una recompensa basada en la longitud. Puedes penalizar terminaciones demasiado cortas o demasiado largas:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_len</span>(<span class="hljs-params">completions, **kwargs</span>):
ideal_length = <span class="hljs-number">20</span>
<span class="hljs-keyword">return</span> [-<span class="hljs-built_in">abs</span>(ideal_length - <span class="hljs-built_in">len</span>(completion)) <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]<!----></pre></div><!----> <p>Esta función de recompensa penaliza las terminaciones que son demasiado cortas o demasiado largas, animando al modelo a generar respuestas cercanas a la longitud ideal de 20 tokens.</p> <iframe src="https://marimo.app/gh/huggingface/notebooks/main/e?entrypoint=course%2Fen%2Fchapter13%2Fgrpo_length.py&amp;embed=true&amp;show-chrome=false" title="Marimo Notebook" width="100%" height="800px" frameborder="0" allow="clipboard-write"></iframe> <!--[1--><h2 class="relative group"><a id="2-recompensas-basadas-en-reglas-para-tareas-verificables" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#2-recompensas-basadas-en-reglas-para-tareas-verificables"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>2. Recompensas basadas en reglas para tareas verificables</span></h2><!--]--><!----> <p>Para tareas con respuestas objetivamente correctas (como matemáticas o programación), puedes implementar funciones de recompensa basadas en reglas:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">def</span> <span class="hljs-title function_">problem_reward</span>(<span class="hljs-params">completions, answers, **kwargs</span>):
<span class="hljs-string">&quot;&quot;&quot;Función de recompensa para problemas matemáticos con respuestas verificables
completions: lista de terminaciones a evaluar
answers: lista de respuestas a los problemas del conjunto de datos
&quot;&quot;&quot;</span>
rewards = []
<span class="hljs-keyword">for</span> completion, correct_answer <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(completions, answers):
<span class="hljs-comment"># Extraer la respuesta de la terminación</span>
<span class="hljs-keyword">try</span>:
<span class="hljs-comment"># Este es un ejemplo simplificado; necesitarías un análisis adecuado</span>
answer = extract_final_answer(completion)
<span class="hljs-comment"># Recompensa binaria: 1 para correcta, 0 para incorrecta</span>
reward = <span class="hljs-number">1.0</span> <span class="hljs-keyword">if</span> answer == correct_answer <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span>
rewards.append(reward)
<span class="hljs-keyword">except</span>:
<span class="hljs-comment"># Si no podemos analizar una respuesta, otorgamos una recompensa baja</span>
rewards.append(<span class="hljs-number">0.0</span>)
<span class="hljs-keyword">return</span> rewards<!----></pre></div><!----> <iframe src="https://marimo.app/gh/huggingface/notebooks/main/e?entrypoint=course%2Fen%2Fchapter13%2Fgrpo_math.py&amp;embed=true&amp;show-chrome=false" title="Marimo Notebook" width="100%" height="800px" frameborder="0" allow="clipboard-write"></iframe> <!--[1--><h2 class="relative group"><a id="3-recompensas-basadas-en-formato" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#3-recompensas-basadas-en-formato"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>3. Recompensas basadas en formato</span></h2><!--]--><!----> <p>También puedes recompensar el formato adecuado, lo cual fue importante en el entrenamiento de DeepSeek R1:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">def</span> <span class="hljs-title function_">format_reward</span>(<span class="hljs-params">completions, **kwargs</span>):
<span class="hljs-string">&quot;&quot;&quot;Recompensar las terminaciones que siguen el formato deseado&quot;&quot;&quot;</span>
<span class="hljs-comment"># Ejemplo: Comprobar si la terminación sigue un formato pensar-luego-responder</span>
pattern = <span class="hljs-string">r&quot;&lt;think&gt;(.*?)&lt;/think&gt;\s*&lt;answer&gt;(.*?)&lt;/answer&gt;&quot;</span>
rewards = []
<span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions:
<span class="hljs-keyword">match</span> = re.search(pattern, completion, re.DOTALL)
<span class="hljs-keyword">if</span> <span class="hljs-keyword">match</span>:
<span class="hljs-comment"># Comprobar si hay contenido sustancial en ambas secciones</span>
think_content = <span class="hljs-keyword">match</span>.group(<span class="hljs-number">1</span>).strip()
answer_content = <span class="hljs-keyword">match</span>.group(<span class="hljs-number">2</span>).strip()
<span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(think_content) &gt; <span class="hljs-number">20</span> <span class="hljs-keyword">and</span> <span class="hljs-built_in">len</span>(answer_content) &gt; <span class="hljs-number">0</span>:
rewards.append(<span class="hljs-number">1.0</span>)
<span class="hljs-keyword">else</span>:
rewards.append(
<span class="hljs-number">0.5</span>
) <span class="hljs-comment"># Recompensa parcial por formato correcto pero contenido limitado</span>
<span class="hljs-keyword">else</span>:
rewards.append(<span class="hljs-number">0.0</span>) <span class="hljs-comment"># Sin recompensa por formato incorrecto</span>
<span class="hljs-keyword">return</span> rewards<!----></pre></div><!----> <iframe src="https://marimo.app/gh/huggingface/notebooks/main/e?entrypoint=course%2Fen%2Fchapter13%2Fgrpo_format.py&amp;embed=true&amp;show-chrome=false" title="Marimo Notebook" width="100%" height="800px" frameborder="0" allow="clipboard-write"></iframe> <p>Estos ejemplos demuestran cómo puedes implementar funciones de recompensa inspiradas en el proceso de entrenamiento de DeepSeek R1, centrándote en la corrección, el formato y las señales combinadas.</p> <!--[1--><h2 class="relative group"><a id="eso-es-todo" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#eso-es-todo"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>¡Eso es todo!</span></h2><!--]--><!----> <p>En la siguiente sección, realizarás un ejercicio para implementar GRPO en TRL.</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/course/blob/main/chapters/es/chapter12/4.mdx" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
<script>
{
__sveltekit_1tvmzrn = {
base: "/docs/course/pr_1356/es",
assets: "/docs/course/pr_1356/es"
};
const element = document.currentScript.parentElement;
Promise.all([
import("/docs/course/pr_1356/es/_app/immutable/entry/start.DeQFEJ0_.js"),
import("/docs/course/pr_1356/es/_app/immutable/entry/app.fni0K-10.js")
]).then(([kit, app]) => {
kit.start(app, element, {
node_ids: [0, 13],
data: [null,null],
form: null,
error: null
});
});
}
</script>

Xet Storage Details

Size:
41.7 kB
·
Xet hash:
9979ea30cd6fc763adf629dd2847c76c9eff5191b6aeee110c4bf17f81331fed

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.