Hugging Face's logo

Buckets:
hf-doc-build
/
doc-dev

Files
xet
hf-doc-build/doc-dev / course /pr_1069 /rum /chapter12 /6.html
rtrm's picture
rtrm
download
raw
59.7 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Exercițiu Practic: GRPO cu Unsloth&quot;,&quot;local&quot;:&quot;exercițiu-practic-grpo-cu-unsloth&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Instalează dependențele&quot;,&quot;local&quot;:&quot;instalează-dependențele&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Configurarea Unsloth&quot;,&quot;local&quot;:&quot;configurarea-unsloth&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Pregătirea Datelor&quot;,&quot;local&quot;:&quot;pregătirea-datelor&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Definirea Funcțiilor de Recompensă&quot;,&quot;local&quot;:&quot;definirea-funcțiilor-de-recompensă&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Antrenarea cu GRPO&quot;,&quot;local&quot;:&quot;antrenarea-cu-grpo&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Testarea Modelului&quot;,&quot;local&quot;:&quot;testarea-modelului&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Salvarea Modelului&quot;,&quot;local&quot;:&quot;salvarea-modelului&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Încărcarea pe Hugging Face Hub&quot;,&quot;local&quot;:&quot;încărcarea-pe-hugging-face-hub&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Concluzie&quot;,&quot;local&quot;:&quot;concluzie&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}">
 <link href="/docs/course/pr_1069/rum/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
 <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/entry/start.1de7c3d2.js">
 <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/chunks/scheduler.37c15a92.js">
 <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/chunks/singletons.e13b7dfd.js">
 <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/chunks/index.18351ede.js">
 <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/chunks/paths.e130b7b0.js">
 <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/entry/app.1f82014c.js">
 <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/chunks/index.2bf4358c.js">
 <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/nodes/0.3c83e1ab.js">
 <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/chunks/each.e59479a4.js">
 <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/nodes/33.0b46b416.js">
 <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/chunks/Tip.363c041f.js">
 <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/chunks/CodeBlock.4e987730.js">
 <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/chunks/CourseFloatingBanner.6add7356.js">
 <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/chunks/getInferenceSnippets.24b50994.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Exercițiu Practic: GRPO cu Unsloth&quot;,&quot;local&quot;:&quot;exercițiu-practic-grpo-cu-unsloth&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Instalează dependențele&quot;,&quot;local&quot;:&quot;instalează-dependențele&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Configurarea Unsloth&quot;,&quot;local&quot;:&quot;configurarea-unsloth&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Pregătirea Datelor&quot;,&quot;local&quot;:&quot;pregătirea-datelor&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Definirea Funcțiilor de Recompensă&quot;,&quot;local&quot;:&quot;definirea-funcțiilor-de-recompensă&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Antrenarea cu GRPO&quot;,&quot;local&quot;:&quot;antrenarea-cu-grpo&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Testarea Modelului&quot;,&quot;local&quot;:&quot;testarea-modelului&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Salvarea Modelului&quot;,&quot;local&quot;:&quot;salvarea-modelului&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Încărcarea pe Hugging Face Hub&quot;,&quot;local&quot;:&quot;încărcarea-pe-hugging-face-hub&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Concluzie&quot;,&quot;local&quot;:&quot;concluzie&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="flex space-x-1 absolute z-10 right-0 top-0"><a href="https://discuss.huggingface.co/t/chapter-2-questions" target="_blank"><img alt="Ask a Question" class="!m-0" src="https://img.shields.io/badge/Ask%20a%20question-ffcb4c.svg?logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgLTEgMTA0IDEwNiI+PGRlZnM+PHN0eWxlPi5jbHMtMXtmaWxsOiMyMzFmMjA7fS5jbHMtMntmaWxsOiNmZmY5YWU7fS5jbHMtM3tmaWxsOiMwMGFlZWY7fS5jbHMtNHtmaWxsOiMwMGE5NGY7fS5jbHMtNXtmaWxsOiNmMTVkMjI7fS5jbHMtNntmaWxsOiNlMzFiMjM7fTwvc3R5bGU+PC9kZWZzPjx0aXRsZT5EaXNjb3Vyc2VfbG9nbzwvdGl0bGU+PGcgaWQ9IkxheWVyXzIiPjxnIGlkPSJMYXllcl8zIj48cGF0aCBjbGFzcz0iY2xzLTEiIGQ9Ik01MS44NywwQzIzLjcxLDAsMCwyMi44MywwLDUxYzAsLjkxLDAsNTIuODEsMCw1Mi44MWw1MS44Ni0uMDVjMjguMTYsMCw1MS0yMy43MSw1MS01MS44N1M4MCwwLDUxLjg3LDBaIi8+PHBhdGggY2xhc3M9ImNscy0yIiBkPSJNNTIuMzcsMTkuNzRBMzEuNjIsMzEuNjIsMCwwLDAsMjQuNTgsNjYuNDFsLTUuNzIsMTguNEwzOS40LDgwLjE3YTMxLjYxLDMxLjYxLDAsMSwwLDEzLTYwLjQzWiIvPjxwYXRoIGNsYXNzPSJjbHMtMyIgZD0iTTc3LjQ1LDMyLjEyYTMxLjYsMzEuNiwwLDAsMS0zOC4wNSw0OEwxOC44Niw4NC44MmwyMC45MS0yLjQ3QTMxLjYsMzEuNiwwLDAsMCw3Ny40NSwzMi4xMloiLz48cGF0aCBjbGFzcz0iY2xzLTQiIGQ9Ik03MS42MywyNi4yOUEzMS42LDMxLjYsMCwwLDEsMzguOCw3OEwxOC44Niw4NC44MiwzOS40LDgwLjE3QTMxLjYsMzEuNiwwLDAsMCw3MS42MywyNi4yOVoiLz48cGF0aCBjbGFzcz0iY2xzLTUiIGQ9Ik0yNi40Nyw2Ny4xMWEzMS42MSwzMS42MSwwLDAsMSw1MS0zNUEzMS42MSwzMS42MSwwLDAsMCwyNC41OCw2Ni40MWwtNS43MiwxOC40WiIvPjxwYXRoIGNsYXNzPSJjbHMtNiIgZD0iTTI0LjU4LDY2LjQxQTMxLjYxLDMxLjYxLDAsMCwxLDcxLjYzLDI2LjI5YTMxLjYxLDMxLjYxLDAsMCwwLTQ5LDM5LjYzbC0zLjc2LDE4LjlaIi8+PC9nPjwvZz48L3N2Zz4="></a> <a href="https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/HuggingFace%20Course-Gemma3_(1B)-GRPO.ipynb" target="_blank"><img alt="Open In Colab" class="!m-0" src="https://colab.research.google.com/assets/colab-badge.svg"></a> </div> <h1 class="relative group"><a id="exercițiu-practic-grpo-cu-unsloth" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#exercițiu-practic-grpo-cu-unsloth"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Exercițiu Practic: GRPO cu Unsloth</span></h1> <p data-svelte-h="svelte-1czfiii">În acest exercițiu, vei ajusta fin un model cu GRPO (Optimizarea Relativă a Politicii de Grup) folosind Unsloth, pentru a îmbunătăți capacitățile de raționament ale unui model. Am acoperit GRPO în <a href="/course/chapter3/3">Capitolul 3</a>.</p> <p data-svelte-h="svelte-gujhi0">Unsloth este o bibliotecă care accelerează ajustarea fină a LLM-urilor, făcând posibilă antrenarea modelelor mai repede și cu mai puține resurse computaționale. Unsloth se conectează la TRL, deci vom construi pe ceea ce am învățat în secțiunile anterioare, și o vom adapta pentru specificațiile Unsloth.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-1u7ab9d">Acest exercițiu poate fi rulat pe un GPU T4 Google Colab gratuit. Pentru cea mai bună experiență, urmărește notebook-ul legat mai sus și încearcă-l singur.</p></div> <h2 class="relative group"><a id="instalează-dependențele" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#instalează-dependențele"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Instalează dependențele</span></h2> <p data-svelte-h="svelte-2per9o">În primul rând, să instalăm bibliotecile necesare. Vom avea nevoie de Unsloth pentru ajustarea fină accelerată și vLLM pentru inferența rapidă.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->pip install unsloth vllm
pip install --upgrade pillow<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="configurarea-unsloth" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#configurarea-unsloth"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Configurarea Unsloth</span></h2> <p data-svelte-h="svelte-1uxs7kh">Unsloth oferă o clasă (<code>FastLanguageModel</code>) care integrează transformers cu optimizările Unsloth. Să o importăm:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> unsloth <span class="hljs-keyword">import</span> FastLanguageModel<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1w7fftr">Acum, să încărcăm modelul Google Gemma 3 1B Instruct și să-l configurăm pentru ajustarea fină:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> unsloth <span class="hljs-keyword">import</span> FastLanguageModel
<span class="hljs-keyword">import</span> torch
max_seq_length = <span class="hljs-number">1024</span> <span class="hljs-comment"># Poate crește pentru urme de raționament mai lungi</span>
lora_rank = <span class="hljs-number">32</span> <span class="hljs-comment"># Rang mai mare = mai inteligent, dar mai lent</span>
model, tokenizer = FastLanguageModel.from_pretrained(
 model_name=<span class="hljs-string">&quot;google/gemma-3-1b-it&quot;</span>,
 max_seq_length=max_seq_length,
 load_in_4bit=<span class="hljs-literal">True</span>, <span class="hljs-comment"># False pentru LoRA 16bit</span>
 fast_inference=<span class="hljs-literal">True</span>, <span class="hljs-comment"># Activează inferența rapidă vLLM</span>
 max_lora_rank=lora_rank,
 gpu_memory_utilization=<span class="hljs-number">0.6</span>, <span class="hljs-comment"># Reduce dacă rămâi fără memorie</span>
)
model = FastLanguageModel.get_peft_model(
 model,
 r=lora_rank, <span class="hljs-comment"># Alege orice număr &gt; 0 ! Sugerat 8, 16, 32, 64, 128</span>
 target_modules=[
 <span class="hljs-string">&quot;q_proj&quot;</span>,
 <span class="hljs-string">&quot;k_proj&quot;</span>,
 <span class="hljs-string">&quot;v_proj&quot;</span>,
 <span class="hljs-string">&quot;o_proj&quot;</span>,
 <span class="hljs-string">&quot;gate_proj&quot;</span>,
 <span class="hljs-string">&quot;up_proj&quot;</span>,
 <span class="hljs-string">&quot;down_proj&quot;</span>,
 ], <span class="hljs-comment"># Elimină QKVO dacă rămâi fără memorie</span>
 lora_alpha=lora_rank,
 use_gradient_checkpointing=<span class="hljs-string">&quot;unsloth&quot;</span>, <span class="hljs-comment"># Activează ajustarea fină pentru context lung</span>
 random_state=<span class="hljs-number">3407</span>,
)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-105vqkb">Acest cod încarcă modelul în cuantizare 4-bit pentru a economisi memoria și aplică LoRA (Adaptarea de Rang Mic) pentru ajustarea fină eficientă. Parametrul <code>target_modules</code> specifică care straturi ale modelului să fie ajustate fin, și <code>use_gradient_checkpointing</code> permite antrenarea cu contexte mai lungi.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-nce7ys">Nu vom acoperi detaliile LoRA în acest capitol, dar poți învăța mai multe în <a href="/course/chapter11/3">Capitolul 11</a>.</p></div> <h2 class="relative group"><a id="pregătirea-datelor" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#pregătirea-datelor"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Pregătirea Datelor</span></h2> <p data-svelte-h="svelte-1936rd4">Pentru acest exercițiu, vom folosi setul de date GSM8K, care conține probleme de matematică de gimnaziu. Vom formata datele pentru a încuraja modelul să-și arate raționamentul înainte de a oferi un răspuns.</p> <p data-svelte-h="svelte-savllt">În primul rând, vom defini formatul prompt-urilor și răspunsurilor:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Definește prompt-ul de sistem care instruiește modelul să folosească un format specific</span>
SYSTEM_PROMPT = <span class="hljs-string">&quot;&quot;&quot;
Răspunde în următorul format:
&lt;reasoning&gt;
...
&lt;/reasoning&gt;
&lt;answer&gt;
...
&lt;/answer&gt;
&quot;&quot;&quot;</span>
XML_COT_FORMAT = <span class="hljs-string">&quot;&quot;&quot;\
&lt;reasoning&gt;
{reasoning}
&lt;/reasoning&gt;
&lt;answer&gt;
{answer}
&lt;/answer&gt;
&quot;&quot;&quot;</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1tiu5id">Acum, să pregătim setul de date:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> re
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, Dataset
<span class="hljs-comment"># Funcții helper pentru a extrage răspunsuri din formate diferite</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">extract_xml_answer</span>(<span class="hljs-params">text: <span class="hljs-built_in">str</span></span>) -&gt; <span class="hljs-built_in">str</span>:
 answer = text.split(<span class="hljs-string">&quot;&lt;answer&gt;&quot;</span>)[-<span class="hljs-number">1</span>]
 answer = answer.split(<span class="hljs-string">&quot;&lt;/answer&gt;&quot;</span>)[<span class="hljs-number">0</span>]
 <span class="hljs-keyword">return</span> answer.strip()
<span class="hljs-keyword">def</span> <span class="hljs-title function_">extract_hash_answer</span>(<span class="hljs-params">text: <span class="hljs-built_in">str</span></span>) -&gt; <span class="hljs-built_in">str</span> | <span class="hljs-literal">None</span>:
 <span class="hljs-keyword">if</span> <span class="hljs-string">&quot;####&quot;</span> <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> text:
 <span class="hljs-keyword">return</span> <span class="hljs-literal">None</span>
 <span class="hljs-keyword">return</span> text.split(<span class="hljs-string">&quot;####&quot;</span>)[<span class="hljs-number">1</span>].strip()
<span class="hljs-comment"># Funcție pentru a pregăti setul de date GSM8K</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_gsm8k_questions</span>(<span class="hljs-params">split=<span class="hljs-string">&quot;train&quot;</span></span>) -&gt; Dataset:
 data = load_dataset(<span class="hljs-string">&quot;openai/gsm8k&quot;</span>, <span class="hljs-string">&quot;main&quot;</span>)[split]
 data = data.<span class="hljs-built_in">map</span>(
 <span class="hljs-keyword">lambda</span> x: {
 <span class="hljs-string">&quot;prompt&quot;</span>: [
 {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: SYSTEM_PROMPT},
 {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: x[<span class="hljs-string">&quot;question&quot;</span>]},
 ],
 <span class="hljs-string">&quot;answer&quot;</span>: extract_hash_answer(x[<span class="hljs-string">&quot;answer&quot;</span>]),
 }
 )
 <span class="hljs-keyword">return</span> data
dataset = get_gsm8k_questions()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-qv1780">Setul de date este pregătit prin extragerea răspunsului din setul de date și formatarea acestuia ca șir de caractere.</p> <h2 class="relative group"><a id="definirea-funcțiilor-de-recompensă" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#definirea-funcțiilor-de-recompensă"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Definirea Funcțiilor de Recompensă</span></h2> <p data-svelte-h="svelte-1858ain">După cum am discutat într-o <a href="/course/chapter13/4">pagină anterioară</a>, GRPO poate folosi funcții de recompensă pentru a ghida învățarea modelului bazată pe criterii verificabile precum lungimea și formatarea.</p> <p data-svelte-h="svelte-b0lkty">În acest exercițiu, vom defini mai multe funcții de recompensă care încurajează diferite aspecte ale unui raționament bun. De exemplu, vom recompensa modelul pentru oferirea unui răspuns întreg, și pentru urmarea formatului strict.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Funcția de recompensă care verifică dacă răspunsul este corect</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">correctness_reward_func</span>(<span class="hljs-params">prompts, completions, answer, **kwargs</span>) -&gt; <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span>]:
 responses = [completion[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;content&quot;</span>] <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]
 q = prompts[<span class="hljs-number">0</span>][-<span class="hljs-number">1</span>][<span class="hljs-string">&quot;content&quot;</span>]
 extracted_responses = [extract_xml_answer(r) <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> responses]
 <span class="hljs-built_in">print</span>(
 <span class="hljs-string">&quot;-&quot;</span> * <span class="hljs-number">20</span>,
 <span class="hljs-string">f&quot;Întrebare:\n<span class="hljs-subst">{q}</span>&quot;</span>,
 <span class="hljs-string">f&quot;\nRăspuns:\n<span class="hljs-subst">{answer[<span class="hljs-number">0</span>]}</span>&quot;</span>,
 <span class="hljs-string">f&quot;\nRăspuns model:\n<span class="hljs-subst">{responses[<span class="hljs-number">0</span>]}</span>&quot;</span>,
 <span class="hljs-string">f&quot;\nExtras:\n<span class="hljs-subst">{extracted_responses[<span class="hljs-number">0</span>]}</span>&quot;</span>,
 )
 <span class="hljs-keyword">return</span> [<span class="hljs-number">2.0</span> <span class="hljs-keyword">if</span> r == a <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span> <span class="hljs-keyword">for</span> r, a <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(extracted_responses, answer)]
<span class="hljs-comment"># Funcția de recompensă care verifică dacă răspunsul este un întreg</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">int_reward_func</span>(<span class="hljs-params">completions, **kwargs</span>) -&gt; <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span>]:
 responses = [completion[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;content&quot;</span>] <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]
 extracted_responses = [extract_xml_answer(r) <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> responses]
 <span class="hljs-keyword">return</span> [<span class="hljs-number">0.5</span> <span class="hljs-keyword">if</span> r.isdigit() <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span> <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> extracted_responses]
<span class="hljs-comment"># Funcția de recompensă care verifică dacă completarea urmează formatul strict</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">strict_format_reward_func</span>(<span class="hljs-params">completions, **kwargs</span>) -&gt; <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span>]:
 pattern = <span class="hljs-string">r&quot;^&lt;reasoning&gt;\n.*?\n&lt;/reasoning&gt;\n&lt;answer&gt;\n.*?\n&lt;/answer&gt;\n$&quot;</span>
 responses = [completion[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;content&quot;</span>] <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]
 matches = [re.<span class="hljs-keyword">match</span>(pattern, r) <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> responses]
 <span class="hljs-keyword">return</span> [<span class="hljs-number">0.5</span> <span class="hljs-keyword">if</span> <span class="hljs-keyword">match</span> <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span> <span class="hljs-keyword">for</span> <span class="hljs-keyword">match</span> <span class="hljs-keyword">in</span> matches]
<span class="hljs-comment"># Funcția de recompensă care verifică dacă completarea urmează un format mai relaxat</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">soft_format_reward_func</span>(<span class="hljs-params">completions, **kwargs</span>) -&gt; <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span>]:
 pattern = <span class="hljs-string">r&quot;&lt;reasoning&gt;.*?&lt;/reasoning&gt;\s*&lt;answer&gt;.*?&lt;/answer&gt;&quot;</span>
 responses = [completion[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;content&quot;</span>] <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]
 matches = [re.<span class="hljs-keyword">match</span>(pattern, r) <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> responses]
 <span class="hljs-keyword">return</span> [<span class="hljs-number">0.5</span> <span class="hljs-keyword">if</span> <span class="hljs-keyword">match</span> <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span> <span class="hljs-keyword">for</span> <span class="hljs-keyword">match</span> <span class="hljs-keyword">in</span> matches]
<span class="hljs-comment"># Funcția de recompensă care numără tag-urile XML și penalizează conținutul extra</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">count_xml</span>(<span class="hljs-params">text</span>) -&gt; <span class="hljs-built_in">float</span>:
 count = <span class="hljs-number">0.0</span>
 <span class="hljs-keyword">if</span> text.count(<span class="hljs-string">&quot;&lt;reasoning&gt;\n&quot;</span>) == <span class="hljs-number">1</span>:
 count += <span class="hljs-number">0.125</span>
 <span class="hljs-keyword">if</span> text.count(<span class="hljs-string">&quot;\n&lt;/reasoning&gt;\n&quot;</span>) == <span class="hljs-number">1</span>:
 count += <span class="hljs-number">0.125</span>
 <span class="hljs-keyword">if</span> text.count(<span class="hljs-string">&quot;\n&lt;answer&gt;\n&quot;</span>) == <span class="hljs-number">1</span>:
 count += <span class="hljs-number">0.125</span>
 count -= <span class="hljs-built_in">len</span>(text.split(<span class="hljs-string">&quot;\n&lt;/answer&gt;\n&quot;</span>)[-<span class="hljs-number">1</span>]) * <span class="hljs-number">0.001</span>
 <span class="hljs-keyword">if</span> text.count(<span class="hljs-string">&quot;\n&lt;/answer&gt;&quot;</span>) == <span class="hljs-number">1</span>:
 count += <span class="hljs-number">0.125</span>
 count -= (<span class="hljs-built_in">len</span>(text.split(<span class="hljs-string">&quot;\n&lt;/answer&gt;&quot;</span>)[-<span class="hljs-number">1</span>]) - <span class="hljs-number">1</span>) * <span class="hljs-number">0.001</span>
 <span class="hljs-keyword">return</span> count
<span class="hljs-keyword">def</span> <span class="hljs-title function_">xmlcount_reward_func</span>(<span class="hljs-params">completions, **kwargs</span>) -&gt; <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span>]:
 contents = [completion[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;content&quot;</span>] <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]
 <span class="hljs-keyword">return</span> [count_xml(c) <span class="hljs-keyword">for</span> c <span class="hljs-keyword">in</span> contents]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1szvi14">Aceste funcții de recompensă servesc diferite scopuri:</p> <table data-svelte-h="svelte-18ti4cp"><thead><tr><th>Funcția de Recompensă</th> <th>Scopul</th></tr></thead> <tbody><tr><td><code>correctness_reward_func</code></td> <td>Recompensează modelul când răspunsul său se potrivește cu răspunsul corect</td></tr> <tr><td><code>int_reward_func</code></td> <td>Recompensează modelul pentru oferirea unui răspuns numeric</td></tr> <tr><td><code>strict_format_reward_func</code> și <code>soft_format_reward_func</code></td> <td>Recompensează modelul pentru urmarea formatului specificat</td></tr> <tr><td><code>xmlcount_reward_func</code></td> <td>Recompensează utilizarea corectă a tag-urilor XML și penalizează conținutul extra după tag-urile de închidere</td></tr></tbody></table> <h2 class="relative group"><a id="antrenarea-cu-grpo" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#antrenarea-cu-grpo"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Antrenarea cu GRPO</span></h2> <p data-svelte-h="svelte-wxqqvx">Acum vom configura antrenorul GRPO cu modelul nostru, tokenizer-ul și funcțiile de recompensă. Această parte urmează aceeași abordare ca <a href="/course/chapter12/5">exercițiul anterior</a>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOConfig, GRPOTrainer
max_prompt_length = <span class="hljs-number">256</span>
training_args = GRPOConfig(
 learning_rate=<span class="hljs-number">5e-6</span>,
 adam_beta1=<span class="hljs-number">0.9</span>,
 adam_beta2=<span class="hljs-number">0.99</span>,
 weight_decay=<span class="hljs-number">0.1</span>,
 warmup_ratio=<span class="hljs-number">0.1</span>,
 lr_scheduler_type=<span class="hljs-string">&quot;cosine&quot;</span>,
 optim=<span class="hljs-string">&quot;paged_adamw_8bit&quot;</span>,
 logging_steps=<span class="hljs-number">1</span>,
 per_device_train_batch_size=<span class="hljs-number">1</span>,
 gradient_accumulation_steps=<span class="hljs-number">1</span>, <span class="hljs-comment"># Crește la 4 pentru antrenare mai lină</span>
 num_generations=<span class="hljs-number">6</span>, <span class="hljs-comment"># Scade dacă rămâi fără memorie</span>
 max_prompt_length=max_prompt_length,
 max_completion_length=max_seq_length - max_prompt_length,
 <span class="hljs-comment"># num_train_epochs = 1, # Setează la 1 pentru o rulare completă de antrenare</span>
 max_steps=<span class="hljs-number">250</span>,
 save_steps=<span class="hljs-number">250</span>,
 max_grad_norm=<span class="hljs-number">0.1</span>,
 report_to=<span class="hljs-string">&quot;none&quot;</span>, <span class="hljs-comment"># Poate folosi Weights &amp; Biases</span>
 output_dir=<span class="hljs-string">&quot;outputs&quot;</span>,
)
trainer = GRPOTrainer(
 model=model,
 processing_class=tokenizer,
 reward_funcs=[
 xmlcount_reward_func,
 soft_format_reward_func,
 strict_format_reward_func,
 int_reward_func,
 correctness_reward_func,
 ],
 args=training_args,
 train_dataset=dataset,
)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-yoo396"><code>GRPOConfig</code> setează diferiți hiperparametri pentru antrenare:</p> <ul data-svelte-h="svelte-1xmdbk7"><li><code>use_vllm</code>: Activează inferența rapidă cu vLLM</li> <li><code>learning_rate</code>: Controlează cât de repede învață modelul</li> <li><code>num_generations</code>: Numărul de completări de generat pentru fiecare prompt</li> <li><code>max_steps</code>: Numărul total de pași de antrenare de efectuat</li></ul> <p data-svelte-h="svelte-32t6p">Acum să începem antrenarea:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->trainer.train()<!-- HTML_TAG_END --></pre></div> <div class="course-tip course-tip-orange bg-gradient-to-br dark:bg-gradient-to-r before:border-orange-500 dark:before:border-orange-800 from-orange-50 dark:from-gray-900 to-white dark:to-gray-950 border border-orange-50 text-orange-700 dark:text-gray-400"><p data-svelte-h="svelte-f1p6xp">Antrenarea poate dura ceva timp. S-ar putea să nu vezi recompensele crescând imediat - poate dura 150-200 de pași înainte să începi să vezi îmbunătățiri. Fii răbdător!</p></div> <h2 class="relative group"><a id="testarea-modelului" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#testarea-modelului"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Testarea Modelului</span></h2> <p data-svelte-h="svelte-wwrmb7">După antrenare, să testăm modelul nostru pentru a vedea cum performează. În primul rând, vom salva greutățile LoRA:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->model.save_lora(<span class="hljs-string">&quot;grpo_saved_lora&quot;</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-fmezoh">Acum, să testăm modelul cu o întrebare nouă:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> vllm <span class="hljs-keyword">import</span> SamplingParams
text = tokenizer.apply_chat_template(
 [
 {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: SYSTEM_PROMPT},
 {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;Calculează pi.&quot;</span>},
 ],
 tokenize=<span class="hljs-literal">False</span>,
 add_generation_prompt=<span class="hljs-literal">True</span>,
)
sampling_params = SamplingParams(
 temperature=<span class="hljs-number">0.8</span>,
 top_p=<span class="hljs-number">0.95</span>,
 max_tokens=<span class="hljs-number">1024</span>,
)
output = (
 model.fast_generate(
 text,
 sampling_params=sampling_params,
 lora_request=model.load_lora(<span class="hljs-string">&quot;grpo_saved_lora&quot;</span>),
 )[<span class="hljs-number">0</span>]
 .outputs[<span class="hljs-number">0</span>]
 .text
)
<span class="hljs-built_in">print</span>(output)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1k72oxj">Ar trebui să vezi că modelul urmează acum formatul specificat, arătându-și raționamentul înainte de a oferi un răspuns.</p> <h2 class="relative group"><a id="salvarea-modelului" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#salvarea-modelului"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Salvarea Modelului</span></h2> <p data-svelte-h="svelte-1gfjffu">Unsloth oferă mai multe opțiuni pentru salvarea modelului tău ajustat fin, dar ne vom concentra pe cea mai comună.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Salvează cu precizie de 16 biți</span>
model.save_pretrained_merged(<span class="hljs-string">&quot;model&quot;</span>, tokenizer, save_method=<span class="hljs-string">&quot;merged_16bit&quot;</span>)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="încărcarea-pe-hugging-face-hub" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#încărcarea-pe-hugging-face-hub"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Încărcarea pe Hugging Face Hub</span></h2> <p data-svelte-h="svelte-3fz42z">Vom încărca modelul pe Hugging Face Hub folosind metoda <code>push_to_hub_merged</code>. Această metodă ne permite să încărcăm modelul în multiple formate de cuantizare.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Încarcă pe Hugging Face Hub (necesită un token)</span>
model.push_to_hub_merged(
 <span class="hljs-string">&quot;numele-tau/numele-modelului&quot;</span>,
 tokenizer,
 save_method=<span class="hljs-string">&quot;merged_16bit&quot;</span>,
 token=<span class="hljs-string">&quot;token-ul-tau&quot;</span>,
)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-rhxj0n">Unsloth suportă de asemenea salvarea în format GGUF pentru utilizare cu llama.cpp:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->model.push_to_hub_gguf(
 <span class="hljs-string">&quot;numele-tau/numele-modelului&quot;</span>,
 tokenizer,
 quantization_method=[<span class="hljs-string">&quot;q4_k_m&quot;</span>, <span class="hljs-string">&quot;q8_0&quot;</span>, <span class="hljs-string">&quot;q5_k_m&quot;</span>],
 token=<span class="hljs-string">&quot;token-ul-tau&quot;</span>,
)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-ru1qrx">Fișierele GGUF pot fi folosite cu llama.cpp sau sisteme bazate pe UI precum Jan sau Open WebUI.</p> <h2 class="relative group"><a id="concluzie" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#concluzie"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Concluzie</span></h2> <p data-svelte-h="svelte-1hcn9j3">În acest exercițiu, ai învățat cum să:</p> <ol data-svelte-h="svelte-yuxkjo"><li>Configurezi Unsloth pentru ajustarea fină accelerată</li> <li>Pregătești datele pentru antrenarea GRPO</li> <li>Definești funcții de recompensă personalizate pentru a ghida învățarea modelului</li> <li>Antrenezi un model folosind GRPO</li> <li>Testezi modelul ajustat fin</li> <li>Salvezi modelul în diverse formate</li></ol> <p data-svelte-h="svelte-1ic5kr1">GRPO este o tehnică puternică pentru alinierea modelelor de limbaj cu comportamente specifice, iar Unsloth o face accesibilă chiar și pe hardware limitat. Prin combinarea mai multor funcții de recompensă, poți ghida modelul să urmeze un format specific în timp ce îi îmbunătățești și capacitățile de raționament.</p> <p data-svelte-h="svelte-cg9ehp">Pentru mai multe informații și resurse, verifică:</p> <ul data-svelte-h="svelte-1c0k3wb"><li><a href="https://docs.unsloth.ai/" rel="nofollow">Documentația Unsloth</a></li> <li><a href="https://discord.gg/unsloth" rel="nofollow">Discord Unsloth</a></li> <li><a href="https://github.com/unslothai/unsloth" rel="nofollow">GitHub Unsloth</a></li></ul> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/course/blob/main/chapters/rum/chapter12/6.mdx" target="_blank"><span data-svelte-h="svelte-1kd6by1">&lt;</span> <span data-svelte-h="svelte-x0xyl0">&gt;</span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p>
<script>
 {
 __sveltekit_1ftlxhy = {
 assets: "/docs/course/pr_1069/rum",
 base: "/docs/course/pr_1069/rum",
 env: {}
 };
 const element = document.currentScript.parentElement;
 const data = [null,null];
 Promise.all([
 import("/docs/course/pr_1069/rum/_app/immutable/entry/start.1de7c3d2.js"),
 import("/docs/course/pr_1069/rum/_app/immutable/entry/app.1f82014c.js")
 ]).then(([kit, app]) => {
 kit.start(app, element, {
 node_ids: [0, 33],
 data,
 form: null,
 error: null
 });
 });
 }
 </script>

Xet Storage Details

Size:
59.7 kB
·
Xet hash:
24ed22f12e9e7021a9e5a4372e2fb262965149bc887c0de6529667016600ff66

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.