course/pr_1139/ro/_app/immutable/nodes/31.cdf35aaa.js

import{s as Zl,a as sl,n as $l,o as Vl}from"../chunks/scheduler.893fe8c9.js";import{S as Ql,i as kl,e as i,s as n,c as m,h as vl,a as p,d as t,b as s,f as ce,g as c,j as r,k as M,l as Wl,m as a,n as u,t as o,o as y,p as w}from"../chunks/index.6ee278c6.js";import{C as Xl,H as d,E as xl}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.1d264bd0.js";import{C as re}from"../chunks/CodeBlock.fba51239.js";function _l(il){let T,ue,Me,oe,b,ye,f,we,I,pl="În această pagină, vom învăța cum să implementăm Optimizarea Relativă a Politicii de Grup (GRPO) folosind biblioteca Transformer Reinforcement Learning (TRL). Ne vom concentra pe implementarea practică cu cod minimal.",Je,h,rl="Vom explora conceptele centrale ale GRPO așa cum sunt întruchipate în GRPOTrainer din TRL, folosind fragmente din documentația oficială TRL pentru a ne ghida.",Ue,C,Ml='<p>Acest capitol este destinat începătorilor TRL. Dacă ești deja familiar cu TRL, ai putea de asemenea să consulți <a href="https://github.com/huggingface/open-r1/blob/main/src/open_r1/grpo.py" rel="nofollow">implementarea Open R1</a> a GRPO.</p>',je,B,ml="În primul rând, să ne reamintim unele dintre conceptele importante ale algoritmului GRPO:",de,g,cl="<li>Formarea Grupului: Modelul generează multiple completări pentru fiecare prompt.</li> <li>Învățarea Preferințelor: Modelul învață dintr-o funcție de recompensă care compară grupuri de completări.</li> <li>Configurația Antrenamentului: Modelul folosește o configurație pentru a controla procesul de antrenare.</li>",Te,G,ul="Ce trebuie să facem pentru a implementa GRPO?",Ce,z,ol="<li>Să definim un set de date de prompt-uri.</li> <li>Să definim o funcție de recompensă care ia o listă de completări și returnează o listă de recompense.</li> <li>Să configurăm procesul de antrenare cu un GRPOConfig.</li> <li>Să antrenăm modelul folosind GRPOTrainer.</li>",be,A,yl="Iată un exemplu minimal pentru a începe antrenamentul GRPO:",fe,R,Ie,Z,he,$,Be,V,wl="Setul tău de date ar trebui să conțină prompt-uri la care modelul va răspunde. Antrenorul GRPO va genera multiple completări pentru fiecare prompt și va folosi funcția de recompensă pentru a le compara.",ge,Q,Ge,k,Jl="Funcția de recompensă este crucială - determină cum învață modelul. Iată două exemple practice:",ze,v,Ae,W,Re,X,Ul="Parametrii cheie de considerat în <code>GRPOConfig</code>:",Ze,x,$e,_,jl="Parametrul <code>num_generation</code> este deosebit de important pentru GRPO deoarece definește dimensiunea grupului - câte completări diferite va genera modelul pentru fiecare prompt. Acesta este un diferențiator cheie de alte metode RL:",Ve,S,dl="<li>Prea mic (de exemplu, 2-3): S-ar putea să nu ofere suficientă diversitate pentru comparații semnificative</li> <li>Recomandat (4-16): Oferă un echilibru bun între diversitate și eficiența computațională</li> <li>Valori mai mari: Pot îmbunătăți învățarea dar cresc semnificativ costul computațional</li>",Qe,N,Tl="Dimensiunea grupului ar trebui aleasă în funcție de resursele tale computaționale și complexitatea sarcinii tale. Pentru sarcini simple, grupuri mai mici (4-8) pot fi suficiente, în timp ce sarcinile de raționament mai complexe ar putea beneficia de grupuri mai mari (8-16).",ke,E,ve,F,Cl=`<li><strong>Gestionarea Memoriei</strong>: Ajustează <code>per_device_train_batch_size</code> și <code>gradient_accumulation_steps</code> în funcție de memoria GPU-ului tău.</li> <li><strong>Viteza</strong>: Activează <code>use_vllm=True</code> pentru generare mai rapidă dacă modelul tău este suportat.</li> <li><strong>Monitorizarea</strong>: Urmărește metricile înregistrate în timpul antrenamentului:
<ul><li><code>reward</code>: Recompensa medie pe completări</li> <li><code>reward_std</code>: Deviația standard în cadrul grupurilor de recompense</li> <li><code>kl</code>: Divergența KL de la modelul de referință</li></ul></li>`,We,Y,Xe,H,bl="Lucrarea DeepSeek R1 demonstrează mai multe abordări eficiente pentru designul funcției de recompensă pe care le poți adapta pentru propria ta implementare GRPO:",xe,D,_e,O,fl="Una dintre cele mai ușoare recompense de implementat este o recompensă bazată pe lungime. Poți recompensa completări mai lungi:",Se,L,Ne,P,Il="Această funcție de recompensă penalizează completările care sunt prea scurte sau prea lungi, încurajând modelul să genereze completări care sunt aproape de lungimea ideală de 20 de token-uri.",Ee,J,hl,Fe,q,Ye,K,Bl="Pentru sarcini cu răspunsuri obiectiv corecte (cum ar fi matematica sau codarea), poți implementa funcții de recompensă bazate pe reguli:",He,ee,De,U,gl,Oe,le,Le,te,Gl="Poți de asemenea să recompensezi formatarea corespunzătoare, care a fost importantă în antrenamentul DeepSeek R1:",Pe,ae,qe,j,zl,Ke,ne,Al="Aceste exemple demonstrează cum poți implementa funcții de recompensă inspirate din procesul de antrenare DeepSeek R1, concentrându-se pe corectitudine, formatare și semnale combinate.",el,se,ll,ie,Rl="În următoarea secțiune, vei urma un exercițiu pentru a implementa GRPO în TRL.",tl,pe,al,me,nl;return b=new Xl({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),f=new d({props:{title:"Implementarea GRPO în TRL",local:"implementarea-grpo-în-trl",headingTag:"h1"}}),R=new re({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMEdSUE9UcmFpbmVyJTJDJTIwR1JQT0NvbmZpZyUwQWZyb20lMjBkYXRhc2V0cyUyMGltcG9ydCUyMGxvYWRfZGF0YXNldCUwQSUwQSUyMyUyMDEuJTIwJUMzJThFbmNhcmMlQzQlODMlMjBzZXR1bCUyMHQlQzQlODN1JTIwZGUlMjBkYXRlJTBBZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJzZXR1bF90YXVfZGVfZGF0ZSUyMiUyQyUyMHNwbGl0JTNEJTIydHJhaW4lMjIpJTBBJTBBJTBBJTIzJTIwMi4lMjBEZWZpbmUlQzglOTl0ZSUyMG8lMjBmdW5jJUM4JTlCaWUlMjBkZSUyMHJlY29tcGVucyVDNCU4MyUyMHNpbXBsJUM0JTgzJTBBZGVmJTIwcmV3YXJkX2Z1bmMoY29tcGxldGlvbnMlMkMlMjAqKmt3YXJncyklM0ElMEElMjAlMjAlMjAlMjAlMjIlMjIlMjJFeGVtcGx1JTNBJTIwUmVjb21wZW5zZWF6JUM0JTgzJTIwY29tcGxldCVDNCU4M3JpbGUlMjBtYWklMjBsdW5naSUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMHJldHVybiUyMCU1QmZsb2F0KGxlbihjb21wbGV0aW9uKSklMjBmb3IlMjBjb21wbGV0aW9uJTIwaW4lMjBjb21wbGV0aW9ucyU1RCUwQSUwQSUwQSUyMyUyMDMuJTIwQ29uZmlndXJlYXolQzQlODMlMjBhbnRyZW5hbWVudHVsJTBBdHJhaW5pbmdfYXJncyUyMCUzRCUyMEdSUE9Db25maWcoJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMm91dHB1dCUyMiUyQyUwQSUyMCUyMCUyMCUyMG51bV90cmFpbl9lcG9jaHMlM0QzJTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTNENCUyQyUwQSUyMCUyMCUyMCUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUzRDIlMkMlMEElMjAlMjAlMjAlMjBsb2dnaW5nX3N0ZXBzJTNEMTAlMkMlMEEpJTBBJTBBJTIzJTIwNC4lMjBJbmklQzglOUJpYWxpemVheiVDNCU4MyUyMCVDOCU5OWklMjBhbnRyZW5lYXolQzQlODMlMEF0cmFpbmVyJTIwJTNEJTIwR1JQT1RyYWluZXIoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0QlMjJtb2RlbHVsX3RhdSUyMiUyQyUyMCUyMCUyMyUyMGRlJTIwZXhlbXBsdSUyMCUyMlF3ZW4lMkZRd2VuMi0wLjVCLUluc3RydWN0JTIyJTBBJTIwJTIwJTIwJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMHJld2FyZF9mdW5jcyUzRHJld2FyZF9mdW5jJTJDJTBBKSUwQXRyYWluZXIudHJhaW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOTrainer, GRPOConfig
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-comment"># 1. Încarcă setul tău de date</span>
dataset = load_dataset(<span class="hljs-string">&quot;setul_tau_de_date&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)


<span class="hljs-comment"># 2. Definește o funcție de recompensă simplă</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_func</span>(<span class="hljs-params">completions, **kwargs</span>):
    <span class="hljs-string">&quot;&quot;&quot;Exemplu: Recompensează completările mai lungi&quot;&quot;&quot;</span>
    <span class="hljs-keyword">return</span> [<span class="hljs-built_in">float</span>(<span class="hljs-built_in">len</span>(completion)) <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]


<span class="hljs-comment"># 3. Configurează antrenamentul</span>
training_args = GRPOConfig(
    output_dir=<span class="hljs-string">&quot;output&quot;</span>,
    num_train_epochs=<span class="hljs-number">3</span>,
    per_device_train_batch_size=<span class="hljs-number">4</span>,
    gradient_accumulation_steps=<span class="hljs-number">2</span>,
    logging_steps=<span class="hljs-number">10</span>,
)

<span class="hljs-comment"># 4. Inițializează și antrenează</span>
trainer = GRPOTrainer(
    model=<span class="hljs-string">&quot;modelul_tau&quot;</span>,  <span class="hljs-comment"># de exemplu &quot;Qwen/Qwen2-0.5B-Instruct&quot;</span>
    args=training_args,
    train_dataset=dataset,
    reward_funcs=reward_func,
)
trainer.train()`,wrap:!1}}),Z=new d({props:{title:"Componentele Cheie",local:"componentele-cheie",headingTag:"h2"}}),$=new d({props:{title:"1. Formatul Setului de Date",local:"1-formatul-setului-de-date",headingTag:"h3"}}),Q=new d({props:{title:"2. Funcția de Recompensă",local:"2-funcția-de-recompensă",headingTag:"h3"}}),v=new re({props:{code:"JTIzJTIwRXhlbXBsdWwlMjAxJTNBJTIwUmVjb21wZW5zJUM0JTgzJTIwYmF6YXQlQzQlODMlMjBwZSUyMGx1bmdpbWVhJTIwY29tcGxldCVDNCU4M3JpaSUwQWRlZiUyMHJld2FyZF9sZW5ndGgoY29tcGxldGlvbnMlMkMlMjAqKmt3YXJncyklM0ElMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlNUJmbG9hdChsZW4oY29tcGxldGlvbikpJTIwZm9yJTIwY29tcGxldGlvbiUyMGluJTIwY29tcGxldGlvbnMlNUQlMEElMEElMEElMjMlMjBFeGVtcGx1bCUyMDIlM0ElMjBSZWNvbXBlbnMlQzQlODMlMjBiYXphdCVDNCU4MyUyMHBlJTIwcG90cml2aXJlYSUyMHVudWklMjBtb2RlbCUwQWltcG9ydCUyMHJlJTBBJTBBJTBBZGVmJTIwcmV3YXJkX2Zvcm1hdChjb21wbGV0aW9ucyUyQyUyMCoqa3dhcmdzKSUzQSUwQSUyMCUyMCUyMCUyMHBhdHRlcm4lMjAlM0QlMjByJTIyJTVFJTNDdGhpbmslM0UuKiUzRiUzQyUyRnRoaW5rJTNFJTNDYW5zd2VyJTNFLiolM0YlM0MlMkZhbnN3ZXIlM0UlMjQlMjIlMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlNUIxLjAlMjBpZiUyMHJlLm1hdGNoKHBhdHRlcm4lMkMlMjBjKSUyMGVsc2UlMjAwLjAlMjBmb3IlMjBjJTIwaW4lMjBjb21wbGV0aW9ucyU1RA==",highlighted:`<span class="hljs-comment"># Exemplul 1: Recompensă bazată pe lungimea completării</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_length</span>(<span class="hljs-params">completions, **kwargs</span>):
    <span class="hljs-keyword">return</span> [<span class="hljs-built_in">float</span>(<span class="hljs-built_in">len</span>(completion)) <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]


<span class="hljs-comment"># Exemplul 2: Recompensă bazată pe potrivirea unui model</span>
<span class="hljs-keyword">import</span> re


<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_format</span>(<span class="hljs-params">completions, **kwargs</span>):
    pattern = <span class="hljs-string">r&quot;^&lt;think&gt;.*?&lt;/think&gt;&lt;answer&gt;.*?&lt;/answer&gt;$&quot;</span>
    <span class="hljs-keyword">return</span> [<span class="hljs-number">1.0</span> <span class="hljs-keyword">if</span> re.<span class="hljs-keyword">match</span>(pattern, c) <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span> <span class="hljs-keyword">for</span> c <span class="hljs-keyword">in</span> completions]`,wrap:!1}}),W=new d({props:{title:"3. Configurația Antrenamentului",local:"3-configurația-antrenamentului",headingTag:"h3"}}),x=new re({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMEdSUE9Db25maWcoJTBBJTIwJTIwJTIwJTIwJTIzJTIwUGFyYW1ldHJpaSUyMGVzZW4lQzglOUJpYWxpJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMm91dHB1dCUyMiUyQyUwQSUyMCUyMCUyMCUyMG51bV90cmFpbl9lcG9jaHMlM0QzJTJDJTBBJTIwJTIwJTIwJTIwbnVtX2dlbmVyYXRpb24lM0Q0JTJDJTIwJTIwJTIzJTIwTnVtJUM0JTgzcnVsJTIwZGUlMjBjb21wbGV0JUM0JTgzcmklMjBkZSUyMGdlbmVyYXQlMjBwZW50cnUlMjBmaWVjYXJlJTIwcHJvbXB0JTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTNENCUyQyUyMCUyMCUyMyUyMFZyZW0lMjBzJUM0JTgzJTIwb2IlQzglOUJpbmVtJTIwdG9hdGUlMjBnZW5lciVDNCU4M3JpbGUlMjAlQzMlQUVudHItdW4lMjBsb3QlMjBkZSUyMGRpc3Bveml0aXYlMEElMjAlMjAlMjAlMjAlMjMlMjBPcCVDOCU5QmlvbmFsJTIwZGFyJTIwdXRpbCUwQSUyMCUyMCUyMCUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUzRDIlMkMlMEElMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNEMWUtNSUyQyUwQSUyMCUyMCUyMCUyMGxvZ2dpbmdfc3RlcHMlM0QxMCUyQyUwQSUyMCUyMCUyMCUyMCUyMyUyMFNwZWNpZmljJTIwR1JQTyUyMChvcCVDOCU5QmlvbmFsKSUwQSUyMCUyMCUyMCUyMHVzZV92bGxtJTNEVHJ1ZSUyQyUyMCUyMCUyMyUyMEFjY2VsZXJlYXolQzQlODMlMjBnZW5lcmFyZWElMEEp",highlighted:`training_args = GRPOConfig(
    <span class="hljs-comment"># Parametrii esențiali</span>
    output_dir=<span class="hljs-string">&quot;output&quot;</span>,
    num_train_epochs=<span class="hljs-number">3</span>,
    num_generation=<span class="hljs-number">4</span>,  <span class="hljs-comment"># Numărul de completări de generat pentru fiecare prompt</span>
    per_device_train_batch_size=<span class="hljs-number">4</span>,  <span class="hljs-comment"># Vrem să obținem toate generările într-un lot de dispozitiv</span>
    <span class="hljs-comment"># Opțional dar util</span>
    gradient_accumulation_steps=<span class="hljs-number">2</span>,
    learning_rate=<span class="hljs-number">1e-5</span>,
    logging_steps=<span class="hljs-number">10</span>,
    <span class="hljs-comment"># Specific GRPO (opțional)</span>
    use_vllm=<span class="hljs-literal">True</span>,  <span class="hljs-comment"># Accelerează generarea</span>
)`,wrap:!1}}),E=new d({props:{title:"Sfaturi pentru Succes",local:"sfaturi-pentru-succes",headingTag:"h2"}}),Y=new d({props:{title:"Designul Funcției de Recompensă",local:"designul-funcției-de-recompensă",headingTag:"h2"}}),D=new d({props:{title:"1. Recompense Bazate pe Lungime",local:"1-recompense-bazate-pe-lungime",headingTag:"h3"}}),L=new re({props:{code:"ZGVmJTIwcmV3YXJkX2xlbihjb21wbGV0aW9ucyUyQyUyMCoqa3dhcmdzKSUzQSUwQSUyMCUyMCUyMCUyMGlkZWFsX2xlbmd0aCUyMCUzRCUyMDIwJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwJTVCLWFicyhpZGVhbF9sZW5ndGglMjAtJTIwbGVuKGNvbXBsZXRpb24pKSUyMGZvciUyMGNvbXBsZXRpb24lMjBpbiUyMGNvbXBsZXRpb25zJTVE",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">reward_len</span>(<span class="hljs-params">completions, **kwargs</span>):
    ideal_length = <span class="hljs-number">20</span>
    <span class="hljs-keyword">return</span> [-<span class="hljs-built_in">abs</span>(ideal_length - <span class="hljs-built_in">len</span>(completion)) <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]`,wrap:!1}}),q=new d({props:{title:"2. Recompense Bazate pe Reguli pentru Sarcini Verificabile",local:"2-recompense-bazate-pe-reguli-pentru-sarcini-verificabile",headingTag:"h2"}}),ee=new re({props:{code:"ZGVmJTIwcHJvYmxlbV9yZXdhcmQoY29tcGxldGlvbnMlMkMlMjBhbnN3ZXJzJTJDJTIwKiprd2FyZ3MpJTNBJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyRnVuYyVDOCU5QmllJTIwZGUlMjByZWNvbXBlbnMlQzQlODMlMjBwZW50cnUlMjBwcm9ibGVtZSUyMGRlJTIwbWF0ZW1hdGljJUM0JTgzJTIwY3UlMjByJUM0JTgzc3B1bnN1cmklMjB2ZXJpZmljYWJpbGUlMEElMjAlMjAlMjAlMjBjb21wbGV0aW9ucyUzQSUyMGxpc3RhJTIwZGUlMjBjb21wbGV0JUM0JTgzcmklMjBkZSUyMGV2YWx1YXQlMEElMjAlMjAlMjAlMjBhbnN3ZXJzJTNBJTIwbGlzdGElMjBkZSUyMHIlQzQlODNzcHVuc3VyaSUyMGxhJTIwcHJvYmxlbWVsZSUyMGRpbiUyMHNldHVsJTIwZGUlMjBkYXRlJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTBBJTIwJTIwJTIwJTIwcmV3YXJkcyUyMCUzRCUyMCU1QiU1RCUwQSUyMCUyMCUyMCUyMGZvciUyMGNvbXBsZXRpb24lMkMlMjBjb3JyZWN0X2Fuc3dlciUyMGluJTIwemlwKGNvbXBsZXRpb25zJTJDJTIwYW5zd2VycyklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBFeHRyYWdlJTIwciVDNCU4M3NwdW5zdWwlMjBkaW4lMjBjb21wbGV0YXJlJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdHJ5JTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwQWNlc3RhJTIwZXN0ZSUyMHVuJTIwZXhlbXBsdSUyMHNpbXBsaWZpY2F0JTIwLSUyMGFpJTIwYXZlYSUyMG5ldm9pZSUyMGRlJTIwcGFyc2luZyUyMGNvcmVzcHVueiVDNCU4M3RvciUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFuc3dlciUyMCUzRCUyMGV4dHJhY3RfZmluYWxfYW5zd2VyKGNvbXBsZXRpb24pJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwUmVjb21wZW5zJUM0JTgzJTIwYmluYXIlQzQlODMlM0ElMjAxJTIwcGVudHJ1JTIwY29yZWN0JTJDJTIwMCUyMHBlbnRydSUyMGluY29yZWN0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV3YXJkJTIwJTNEJTIwMS4wJTIwaWYlMjBhbnN3ZXIlMjAlM0QlM0QlMjBjb3JyZWN0X2Fuc3dlciUyMGVsc2UlMjAwLjAlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXdhcmRzLmFwcGVuZChyZXdhcmQpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZXhjZXB0JTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwRGFjJUM0JTgzJTIwbnUlMjBwdXRlbSUyMHBhcnNhJTIwdW4lMjByJUM0JTgzc3B1bnMlMkMlMjBkJUM0JTgzbSUyMG8lMjByZWNvbXBlbnMlQzQlODMlMjBtaWMlQzQlODMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXdhcmRzLmFwcGVuZCgwLjApJTBBJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwcmV3YXJkcw==",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">problem_reward</span>(<span class="hljs-params">completions, answers, **kwargs</span>):
    <span class="hljs-string">&quot;&quot;&quot;Funcție de recompensă pentru probleme de matematică cu răspunsuri verificabile
    completions: lista de completări de evaluat
    answers: lista de răspunsuri la problemele din setul de date
    &quot;&quot;&quot;</span>

    rewards = []
    <span class="hljs-keyword">for</span> completion, correct_answer <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(completions, answers):
        <span class="hljs-comment"># Extrage răspunsul din completare</span>
        <span class="hljs-keyword">try</span>:
            <span class="hljs-comment"># Acesta este un exemplu simplificat - ai avea nevoie de parsing corespunzător</span>
            answer = extract_final_answer(completion)
            <span class="hljs-comment"># Recompensă binară: 1 pentru corect, 0 pentru incorect</span>
            reward = <span class="hljs-number">1.0</span> <span class="hljs-keyword">if</span> answer == correct_answer <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span>
            rewards.append(reward)
        <span class="hljs-keyword">except</span>:
            <span class="hljs-comment"># Dacă nu putem parsa un răspuns, dăm o recompensă mică</span>
            rewards.append(<span class="hljs-number">0.0</span>)

    <span class="hljs-keyword">return</span> rewards`,wrap:!1}}),le=new d({props:{title:"3. Recompense Bazate pe Format",local:"3-recompense-bazate-pe-format",headingTag:"h2"}}),ae=new re({props:{code:"ZGVmJTIwZm9ybWF0X3Jld2FyZChjb21wbGV0aW9ucyUyQyUyMCoqa3dhcmdzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMlJlY29tcGVuc2VheiVDNCU4MyUyMGNvbXBsZXQlQzQlODNyaWxlJTIwY2FyZSUyMHVybWVheiVDNCU4MyUyMGZvcm1hdHVsJTIwZG9yaXQlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjAlMjMlMjBFeGVtcGx1JTNBJTIwVmVyaWZpYyVDNCU4MyUyMGRhYyVDNCU4MyUyMGNvbXBsZXRhcmVhJTIwdXJtZWF6JUM0JTgzJTIwdW4lMjBmb3JtYXQlMjBnJUMzJUEybmRlJUM4JTk5dGUtYXBvaS1yJUM0JTgzc3B1bmRlJTBBJTIwJTIwJTIwJTIwcGF0dGVybiUyMCUzRCUyMHIlMjIlM0N0aGluayUzRSguKiUzRiklM0MlMkZ0aGluayUzRSU1Q3MqJTNDYW5zd2VyJTNFKC4qJTNGKSUzQyUyRmFuc3dlciUzRSUyMiUwQSUwQSUyMCUyMCUyMCUyMHJld2FyZHMlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjBmb3IlMjBjb21wbGV0aW9uJTIwaW4lMjBjb21wbGV0aW9ucyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1hdGNoJTIwJTNEJTIwcmUuc2VhcmNoKHBhdHRlcm4lMkMlMjBjb21wbGV0aW9uJTJDJTIwcmUuRE9UQUxMKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwbWF0Y2glM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBWZXJpZmljJUM0JTgzJTIwZGFjJUM0JTgzJTIwZXhpc3QlQzQlODMlMjBjb24lQzglOUJpbnV0JTIwc3Vic3RhbiVDOCU5QmlhbCUyMCVDMyVBRW4lMjBhbWJlbGUlMjBzZWMlQzglOUJpdW5pJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdGhpbmtfY29udGVudCUyMCUzRCUyMG1hdGNoLmdyb3VwKDEpLnN0cmlwKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhbnN3ZXJfY29udGVudCUyMCUzRCUyMG1hdGNoLmdyb3VwKDIpLnN0cmlwKCklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMGxlbih0aGlua19jb250ZW50KSUyMCUzRSUyMDIwJTIwYW5kJTIwbGVuKGFuc3dlcl9jb250ZW50KSUyMCUzRSUyMDAlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXdhcmRzLmFwcGVuZCgxLjApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxzZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJld2FyZHMuYXBwZW5kKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMDAuNSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMjAlMjAlMjMlMjBSZWNvbXBlbnMlQzQlODMlMjBwYXIlQzglOUJpYWwlQzQlODMlMjBwZW50cnUlMjBmb3JtYXQlMjBjb3JlY3QlMjBkYXIlMjBjb24lQzglOUJpbnV0JTIwbGltaXRhdCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVsc2UlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXdhcmRzLmFwcGVuZCgwLjApJTIwJTIwJTIzJTIwTmljaW8lMjByZWNvbXBlbnMlQzQlODMlMjBwZW50cnUlMjBmb3JtYXQlMjBpbmNvcmVjdCUwQSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHJld2FyZHM=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">format_reward</span>(<span class="hljs-params">completions, **kwargs</span>):
    <span class="hljs-string">&quot;&quot;&quot;Recompensează completările care urmează formatul dorit&quot;&quot;&quot;</span>
    <span class="hljs-comment"># Exemplu: Verifică dacă completarea urmează un format gândește-apoi-răspunde</span>
    pattern = <span class="hljs-string">r&quot;&lt;think&gt;(.*?)&lt;/think&gt;\\s*&lt;answer&gt;(.*?)&lt;/answer&gt;&quot;</span>

    rewards = []
    <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions:
        <span class="hljs-keyword">match</span> = re.search(pattern, completion, re.DOTALL)
        <span class="hljs-keyword">if</span> <span class="hljs-keyword">match</span>:
            <span class="hljs-comment"># Verifică dacă există conținut substanțial în ambele secțiuni</span>
            think_content = <span class="hljs-keyword">match</span>.group(<span class="hljs-number">1</span>).strip()
            answer_content = <span class="hljs-keyword">match</span>.group(<span class="hljs-number">2</span>).strip()

            <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(think_content) &gt; <span class="hljs-number">20</span> <span class="hljs-keyword">and</span> <span class="hljs-built_in">len</span>(answer_content) &gt; <span class="hljs-number">0</span>:
                rewards.append(<span class="hljs-number">1.0</span>)
            <span class="hljs-keyword">else</span>:
                rewards.append(
                    <span class="hljs-number">0.5</span>
                )  <span class="hljs-comment"># Recompensă parțială pentru format corect dar conținut limitat</span>
        <span class="hljs-keyword">else</span>:
            rewards.append(<span class="hljs-number">0.0</span>)  <span class="hljs-comment"># Nicio recompensă pentru format incorect</span>

    <span class="hljs-keyword">return</span> rewards`,wrap:!1}}),se=new d({props:{title:"Asta e tot!",local:"asta-e-tot",headingTag:"h2"}}),pe=new xl({props:{source:"https://github.com/huggingface/course/blob/main/chapters/ro/chapter12/4.mdx"}}),{c(){T=i("meta"),ue=n(),Me=i("p"),oe=n(),m(b.$$.fragment),ye=n(),m(f.$$.fragment),we=n(),I=i("p"),I.textContent=pl,Je=n(),h=i("p"),h.textContent=rl,Ue=n(),C=i("blockquote"),C.innerHTML=Ml,je=n(),B=i("p"),B.textContent=ml,de=n(),g=i("ul"),g.innerHTML=cl,Te=n(),G=i("p"),G.textContent=ul,Ce=n(),z=i("ul"),z.innerHTML=ol,be=n(),A=i("p"),A.textContent=yl,fe=n(),m(R.$$.fragment),Ie=n(),m(Z.$$.fragment),he=n(),m($.$$.fragment),Be=n(),V=i("p"),V.textContent=wl,ge=n(),m(Q.$$.fragment),Ge=n(),k=i("p"),k.textContent=Jl,ze=n(),m(v.$$.fragment),Ae=n(),m(W.$$.fragment),Re=n(),X=i("p"),X.innerHTML=Ul,Ze=n(),m(x.$$.fragment),$e=n(),_=i("p"),_.innerHTML=jl,Ve=n(),S=i("ul"),S.innerHTML=dl,Qe=n(),N=i("p"),N.textContent=Tl,ke=n(),m(E.$$.fragment),ve=n(),F=i("ol"),F.innerHTML=Cl,We=n(),m(Y.$$.fragment),Xe=n(),H=i("p"),H.textContent=bl,xe=n(),m(D.$$.fragment),_e=n(),O=i("p"),O.textContent=fl,Se=n(),m(L.$$.fragment),Ne=n(),P=i("p"),P.textContent=Il,Ee=n(),J=i("iframe"),Fe=n(),m(q.$$.fragment),Ye=n(),K=i("p"),K.textContent=Bl,He=n(),m(ee.$$.fragment),De=n(),U=i("iframe"),Oe=n(),m(le.$$.fragment),Le=n(),te=i("p"),te.textContent=Gl,Pe=n(),m(ae.$$.fragment),qe=n(),j=i("iframe"),Ke=n(),ne=i("p"),ne.textContent=Al,el=n(),m(se.$$.fragment),ll=n(),ie=i("p"),ie.textContent=Rl,tl=n(),m(pe.$$.fragment),al=n(),me=i("p"),this.h()},l(e){const l=vl("svelte-u9bgzb",document.head);T=p(l,"META",{name:!0,content:!0}),l.forEach(t),ue=s(e),Me=p(e,"P",{}),ce(Me).forEach(t),oe=s(e),c(b.$$.fragment,e),ye=s(e),c(f.$$.fragment,e),we=s(e),I=p(e,"P",{"data-svelte-h":!0}),r(I)!=="svelte-1293vxv"&&(I.textContent=pl),Je=s(e),h=p(e,"P",{"data-svelte-h":!0}),r(h)!=="svelte-na6ewk"&&(h.textContent=rl),Ue=s(e),C=p(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),r(C)!=="svelte-5c4xko"&&(C.innerHTML=Ml),je=s(e),B=p(e,"P",{"data-svelte-h":!0}),r(B)!=="svelte-1kpbqds"&&(B.textContent=ml),de=s(e),g=p(e,"UL",{"data-svelte-h":!0}),r(g)!=="svelte-ez6ot5"&&(g.innerHTML=cl),Te=s(e),G=p(e,"P",{"data-svelte-h":!0}),r(G)!=="svelte-1a3xk8g"&&(G.textContent=ul),Ce=s(e),z=p(e,"UL",{"data-svelte-h":!0}),r(z)!=="svelte-xpn84y"&&(z.innerHTML=ol),be=s(e),A=p(e,"P",{"data-svelte-h":!0}),r(A)!=="svelte-f204iz"&&(A.textContent=yl),fe=s(e),c(R.$$.fragment,e),Ie=s(e),c(Z.$$.fragment,e),he=s(e),c($.$$.fragment,e),Be=s(e),V=p(e,"P",{"data-svelte-h":!0}),r(V)!=="svelte-j8bpdm"&&(V.textContent=wl),ge=s(e),c(Q.$$.fragment,e),Ge=s(e),k=p(e,"P",{"data-svelte-h":!0}),r(k)!=="svelte-iw81bf"&&(k.textContent=Jl),ze=s(e),c(v.$$.fragment,e),Ae=s(e),c(W.$$.fragment,e),Re=s(e),X=p(e,"P",{"data-svelte-h":!0}),r(X)!=="svelte-1cxbitg"&&(X.innerHTML=Ul),Ze=s(e),c(x.$$.fragment,e),$e=s(e),_=p(e,"P",{"data-svelte-h":!0}),r(_)!=="svelte-zu7ra"&&(_.innerHTML=jl),Ve=s(e),S=p(e,"UL",{"data-svelte-h":!0}),r(S)!=="svelte-1al5l6o"&&(S.innerHTML=dl),Qe=s(e),N=p(e,"P",{"data-svelte-h":!0}),r(N)!=="svelte-1rh1v9n"&&(N.textContent=Tl),ke=s(e),c(E.$$.fragment,e),ve=s(e),F=p(e,"OL",{"data-svelte-h":!0}),r(F)!=="svelte-59ut5t"&&(F.innerHTML=Cl),We=s(e),c(Y.$$.fragment,e),Xe=s(e),H=p(e,"P",{"data-svelte-h":!0}),r(H)!=="svelte-6gbmmo"&&(H.textContent=bl),xe=s(e),c(D.$$.fragment,e),_e=s(e),O=p(e,"P",{"data-svelte-h":!0}),r(O)!=="svelte-18bht15"&&(O.textContent=fl),Se=s(e),c(L.$$.fragment,e),Ne=s(e),P=p(e,"P",{"data-svelte-h":!0}),r(P)!=="svelte-1xyrfe1"&&(P.textContent=Il),Ee=s(e),J=p(e,"IFRAME",{src:!0,title:!0,width:!0,height:!0,frameborder:!0,allow:!0}),ce(J).forEach(t),Fe=s(e),c(q.$$.fragment,e),Ye=s(e),K=p(e,"P",{"data-svelte-h":!0}),r(K)!=="svelte-11f5ljd"&&(K.textContent=Bl),He=s(e),c(ee.$$.fragment,e),De=s(e),U=p(e,"IFRAME",{src:!0,title:!0,width:!0,height:!0,frameborder:!0,allow:!0}),ce(U).forEach(t),Oe=s(e),c(le.$$.fragment,e),Le=s(e),te=p(e,"P",{"data-svelte-h":!0}),r(te)!=="svelte-15alen6"&&(te.textContent=Gl),Pe=s(e),c(ae.$$.fragment,e),qe=s(e),j=p(e,"IFRAME",{src:!0,title:!0,width:!0,height:!0,frameborder:!0,allow:!0}),ce(j).forEach(t),Ke=s(e),ne=p(e,"P",{"data-svelte-h":!0}),r(ne)!=="svelte-5dp67z"&&(ne.textContent=Al),el=s(e),c(se.$$.fragment,e),ll=s(e),ie=p(e,"P",{"data-svelte-h":!0}),r(ie)!=="svelte-11kcv9w"&&(ie.textContent=Rl),tl=s(e),c(pe.$$.fragment,e),al=s(e),me=p(e,"P",{}),ce(me).forEach(t),this.h()},h(){M(T,"name","hf:doc:metadata"),M(T,"content",Sl),M(C,"class","tip"),sl(J.src,hl="https://marimo.app/gh/huggingface/notebooks/main/e?entrypoint=course%2Fen%2Fchapter13%2Fgrpo_length.py&embed=true&show-chrome=false")||M(J,"src",hl),M(J,"title","Marimo Notebook"),M(J,"width","100%"),M(J,"height","800px"),M(J,"frameborder","0"),M(J,"allow","clipboard-write"),sl(U.src,gl="https://marimo.app/gh/huggingface/notebooks/main/e?entrypoint=course%2Fen%2Fchapter13%2Fgrpo_math.py&embed=true&show-chrome=false")||M(U,"src",gl),M(U,"title","Marimo Notebook"),M(U,"width","100%"),M(U,"height","800px"),M(U,"frameborder","0"),M(U,"allow","clipboard-write"),sl(j.src,zl="https://marimo.app/gh/huggingface/notebooks/main/e?entrypoint=course%2Fen%2Fchapter13%2Fgrpo_format.py&embed=true&show-chrome=false")||M(j,"src",zl),M(j,"title","Marimo Notebook"),M(j,"width","100%"),M(j,"height","800px"),M(j,"frameborder","0"),M(j,"allow","clipboard-write")},m(e,l){Wl(document.head,T),a(e,ue,l),a(e,Me,l),a(e,oe,l),u(b,e,l),a(e,ye,l),u(f,e,l),a(e,we,l),a(e,I,l),a(e,Je,l),a(e,h,l),a(e,Ue,l),a(e,C,l),a(e,je,l),a(e,B,l),a(e,de,l),a(e,g,l),a(e,Te,l),a(e,G,l),a(e,Ce,l),a(e,z,l),a(e,be,l),a(e,A,l),a(e,fe,l),u(R,e,l),a(e,Ie,l),u(Z,e,l),a(e,he,l),u($,e,l),a(e,Be,l),a(e,V,l),a(e,ge,l),u(Q,e,l),a(e,Ge,l),a(e,k,l),a(e,ze,l),u(v,e,l),a(e,Ae,l),u(W,e,l),a(e,Re,l),a(e,X,l),a(e,Ze,l),u(x,e,l),a(e,$e,l),a(e,_,l),a(e,Ve,l),a(e,S,l),a(e,Qe,l),a(e,N,l),a(e,ke,l),u(E,e,l),a(e,ve,l),a(e,F,l),a(e,We,l),u(Y,e,l),a(e,Xe,l),a(e,H,l),a(e,xe,l),u(D,e,l),a(e,_e,l),a(e,O,l),a(e,Se,l),u(L,e,l),a(e,Ne,l),a(e,P,l),a(e,Ee,l),a(e,J,l),a(e,Fe,l),u(q,e,l),a(e,Ye,l),a(e,K,l),a(e,He,l),u(ee,e,l),a(e,De,l),a(e,U,l),a(e,Oe,l),u(le,e,l),a(e,Le,l),a(e,te,l),a(e,Pe,l),u(ae,e,l),a(e,qe,l),a(e,j,l),a(e,Ke,l),a(e,ne,l),a(e,el,l),u(se,e,l),a(e,ll,l),a(e,ie,l),a(e,tl,l),u(pe,e,l),a(e,al,l),a(e,me,l),nl=!0},p:$l,i(e){nl||(o(b.$$.fragment,e),o(f.$$.fragment,e),o(R.$$.fragment,e),o(Z.$$.fragment,e),o($.$$.fragment,e),o(Q.$$.fragment,e),o(v.$$.fragment,e),o(W.$$.fragment,e),o(x.$$.fragment,e),o(E.$$.fragment,e),o(Y.$$.fragment,e),o(D.$$.fragment,e),o(L.$$.fragment,e),o(q.$$.fragment,e),o(ee.$$.fragment,e),o(le.$$.fragment,e),o(ae.$$.fragment,e),o(se.$$.fragment,e),o(pe.$$.fragment,e),nl=!0)},o(e){y(b.$$.fragment,e),y(f.$$.fragment,e),y(R.$$.fragment,e),y(Z.$$.fragment,e),y($.$$.fragment,e),y(Q.$$.fragment,e),y(v.$$.fragment,e),y(W.$$.fragment,e),y(x.$$.fragment,e),y(E.$$.fragment,e),y(Y.$$.fragment,e),y(D.$$.fragment,e),y(L.$$.fragment,e),y(q.$$.fragment,e),y(ee.$$.fragment,e),y(le.$$.fragment,e),y(ae.$$.fragment,e),y(se.$$.fragment,e),y(pe.$$.fragment,e),nl=!1},d(e){e&&(t(ue),t(Me),t(oe),t(ye),t(we),t(I),t(Je),t(h),t(Ue),t(C),t(je),t(B),t(de),t(g),t(Te),t(G),t(Ce),t(z),t(be),t(A),t(fe),t(Ie),t(he),t(Be),t(V),t(ge),t(Ge),t(k),t(ze),t(Ae),t(Re),t(X),t(Ze),t($e),t(_),t(Ve),t(S),t(Qe),t(N),t(ke),t(ve),t(F),t(We),t(Xe),t(H),t(xe),t(_e),t(O),t(Se),t(Ne),t(P),t(Ee),t(J),t(Fe),t(Ye),t(K),t(He),t(De),t(U),t(Oe),t(Le),t(te),t(Pe),t(qe),t(j),t(Ke),t(ne),t(el),t(ll),t(ie),t(tl),t(al),t(me)),t(T),w(b,e),w(f,e),w(R,e),w(Z,e),w($,e),w(Q,e),w(v,e),w(W,e),w(x,e),w(E,e),w(Y,e),w(D,e),w(L,e),w(q,e),w(ee,e),w(le,e),w(ae,e),w(se,e),w(pe,e)}}}const Sl='{"title":"Implementarea GRPO în TRL","local":"implementarea-grpo-în-trl","sections":[{"title":"Componentele Cheie","local":"componentele-cheie","sections":[{"title":"1. Formatul Setului de Date","local":"1-formatul-setului-de-date","sections":[],"depth":3},{"title":"2. Funcția de Recompensă","local":"2-funcția-de-recompensă","sections":[],"depth":3},{"title":"3. Configurația Antrenamentului","local":"3-configurația-antrenamentului","sections":[],"depth":3}],"depth":2},{"title":"Sfaturi pentru Succes","local":"sfaturi-pentru-succes","sections":[],"depth":2},{"title":"Designul Funcției de Recompensă","local":"designul-funcției-de-recompensă","sections":[{"title":"1. Recompense Bazate pe Lungime","local":"1-recompense-bazate-pe-lungime","sections":[],"depth":3}],"depth":2},{"title":"2. Recompense Bazate pe Reguli pentru Sarcini Verificabile","local":"2-recompense-bazate-pe-reguli-pentru-sarcini-verificabile","sections":[],"depth":2},{"title":"3. Recompense Bazate pe Format","local":"3-recompense-bazate-pe-format","sections":[],"depth":2},{"title":"Asta e tot!","local":"asta-e-tot","sections":[],"depth":2}],"depth":1}';function Nl(il){return Vl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Dl extends Ql{constructor(T){super(),kl(this,T,Nl,_l,Zl,{})}}export{Dl as component};