Buckets:
| import{s as w0,n as M0,o as f0}from"../chunks/scheduler.893fe8c9.js";import{S as x0,i as b0,e as m,s as e,c as v,q as h,H as d,h as T0,a as i,d as a,b as l,f as u,g as w,j as r,r as g,u as y,k as Te,l as t,m as p,n as M,t as f,o as x,p as b}from"../chunks/index.b1df2166.js";import{C as _0,H as T,E as C0}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.e6f5b9e1.js";import{C as j}from"../chunks/CodeBlock.61812ef0.js";function z0(To){let P,_e,ce,Ce,os,ze,hs,Je,F,_o='<p>ဒီအပိုင်းက GRPO ရဲ့ နည်းပညာနဲ့ သင်္ချာဆိုင်ရာ အသေးစိတ်အချက်အလက်တွေကို နက်နက်နဲနဲ လေ့လာထားပါတယ်။ <a href="https://huggingface.com/shirinyamani" target="_blank">Shirin Yamani</a> က ရေးသားခဲ့တာ ဖြစ်ပါတယ်။</p>',Ue,gs,Co="ကျွန်တော်တို့ရဲ့ model ရဲ့ training process ကို ပိုမိုကောင်းမွန်အောင် လုပ်ဆောင်နိုင်ဖို့ GRPO ကို ပိုနားလည်အောင် လုပ်ကြရအောင်။",je,ds,zo="GRPO က policy model ကို optimization လုပ်ဖို့ သီးခြား value model (Critic) ကို training လုပ်မယ့်အစား၊ model က ထုတ်ပေးထားတဲ့ responses တွေကို အုပ်စုလိုက် နှိုင်းယှဉ်ခြင်းဖြင့် တိုက်ရိုက်အကဲဖြတ်ပါတယ်။ ဒီနည်းလမ်းက computational cost ကို သိသိသာသာ လျှော့ချပေးပါတယ်။",ke,ys,Jo="GRPO ကို responses တွေရဲ့ မှန်ကန်မှုကို ဆုံးဖြတ်နိုင်တဲ့ မည်သည့် စစ်ဆေးနိုင်တဲ့ task မှာမဆို အသုံးပြုနိုင်ပါတယ်။ ဥပမာ၊ math reasoning မှာ response ရဲ့ မှန်ကန်မှုကို ground truth နဲ့ နှိုင်းယှဉ်ခြင်းဖြင့် အလွယ်တကူ စစ်ဆေးနိုင်ပါတယ်။",Le,us,Uo="နည်းပညာဆိုင်ရာ အသေးစိတ်အချက်အလက်တွေကို မလေ့လာခင်၊ GRPO က အပေါ်ယံအားဖြင့် ဘယ်လိုအလုပ်လုပ်လဲဆိုတာကို မြင်သာအောင် ကြည့်ရအောင်။",$e,vs,jo='<img src="https://huggingface.co/reasoning-course/images/resolve/main/grpo/16.png" alt="deep"/>',Ie,ws,ko="အခု ကျွန်တော်တို့ မြင်သာတဲ့ overview တစ်ခု ရရှိပြီဆိုတော့၊ GRPO က တစ်ဆင့်ချင်းစီ ဘယ်လိုအလုပ်လုပ်လဲဆိုတာကို ဖော်ပြပေးပါမယ်။",Ae,Ms,He,fs,Lo="GRPO ရဲ့ အဓိက ဆန်းသစ်တီထွင်မှုကတော့ responses များစွာကို တစ်ပြိုင်နက်တည်း အကဲဖြတ်ပြီး သင်ယူတဲ့ နည်းလမ်းပဲဖြစ်ပါတယ်။ သီးခြား reward model တစ်ခုပေါ် အားကိုးမယ့်အစား၊ အတူတူ group ထဲက outputs တွေကို နှိုင်းယှဉ်ပြီး ဘယ်ဟာတွေကို ပိုပြီး အားဖြည့်သင့်သလဲဆိုတာ ဆုံးဖြတ်ပါတယ်။",Ge,xs,$o="algorithm ရဲ့ အဆင့်တစ်ခုစီကို အသေးစိတ် ကြည့်ရအောင်။",Re,bs,Ee,Ts,Io="ပထမအဆင့်ကတော့ မေးခွန်းတစ်ခုစီအတွက် ဖြစ်နိုင်ခြေရှိတဲ့ အဖြေများစွာကို ထုတ်ပေးဖို့ပါပဲ။ ဒါက တစ်ခုနဲ့တစ်ခု နှိုင်းယှဉ်နိုင်တဲ့ ကွဲပြားခြားနားတဲ့ outputs တွေကို ဖန်တီးပေးပါတယ်။",Be,_,pr,Qe,dd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>q</mi></mrow><annotation encoding="application/x-tex"> q </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.625em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.03588em;">q</span></span></span></span>',De,Ze,yd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>G</mi></mrow><annotation encoding="application/x-tex"> G </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6833em;"></span><span class="mord mathnormal">G</span></span></span></span>',Pe,Se,ud='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mrow><msub><mi>o</mi><mn>1</mn></msub><mo separator="true">,</mo><msub><mi>o</mi><mn>2</mn></msub><mo separator="true">,</mo><msub><mi>o</mi><mn>3</mn></msub><mo separator="true">,</mo><mo>…</mo><mo separator="true">,</mo><msub><mi>o</mi><mi>G</mi></msub></mrow><msub><mi>π</mi><msub><mi>θ</mi><mtext>old</mtext></msub></msub></mrow><annotation encoding="application/x-tex"> {o_1, o_2, o_3, \\dots, o_G}\\pi_{\\theta_{\\text{old}}} </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6864em;vertical-align:-0.2559em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">1</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">2</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">3</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="minner">…</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3283em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">G</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3448em;"><span style="top:-2.3488em;margin-left:-0.0278em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mtight"><span class="mord text mtight"><span class="mord mtight">old</span></span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.1512em;"><span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.2559em;"><span></span></span></span></span></span></span></span></span></span>',Ne,Xe,vd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>G</mi><mo>=</mo><mn>8</mn></mrow><annotation encoding="application/x-tex"> G=8 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6833em;"></span><span class="mord mathnormal">G</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">8</span></span></span></span>',Ve,Fe,wd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>o</mi><mi>i</mi></msub></mrow><annotation encoding="application/x-tex"> o_i </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.5806em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',qe,We,_s,Oe,Cs,Ao="ဒါကို ပိုပြီး ထင်သာမြင်သာဖြစ်အောင်၊ ရိုးရှင်းတဲ့ ဂဏန်းတွက်ချက်မှု ပြဿနာတစ်ခုကို ကြည့်ရအောင်။",Ye,G,_t,Ho="မေးခွန်း",mr,Ke,Md='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>q</mi></mrow><annotation encoding="application/x-tex"> q </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.625em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.03588em;">q</span></span></span></span>',sl,al,fd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mtext>Calculate </mtext><mn>2</mn><mo>+</mo><mn>2</mn><mo>×</mo><mn>6</mn></mrow><annotation encoding="application/x-tex"> \\text{Calculate}\\space2 + 2 \\times 6 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.7778em;vertical-align:-0.0833em;"></span><span class="mord text"><span class="mord">Calculate</span></span><span class="mspace"> </span><span class="mord">2</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.7278em;vertical-align:-0.0833em;"></span><span class="mord">2</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">×</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">6</span></span></span></span>',tl,R,Ct,Go="Outputs",ir,nl,xd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">(</mo><mi>G</mi><mo>=</mo><mn>8</mn><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex"> (G = 8) </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mopen">(</span><span class="mord mathnormal">G</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord">8</span><span class="mclose">)</span></span></span></span>',el,ll,bd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">{</mo><msub><mi>o</mi><mn>1</mn></msub><mo>:</mo><mn>14</mn><mtext> (မှန်)</mtext><mo separator="true">,</mo><msub><mi>o</mi><mn>2</mn></msub><mo>:</mo><mn>16</mn><mtext> (မှား)</mtext><mo separator="true">,</mo><msub><mi>o</mi><mn>3</mn></msub><mo>:</mo><mn>10</mn><mtext> (မှား)</mtext><mo separator="true">,</mo><mo>…</mo><mo separator="true">,</mo><msub><mi>o</mi><mn>8</mn></msub><mo>:</mo><mn>14</mn><mtext> (မှန်)</mtext><mo stretchy="false">}</mo></mrow><annotation encoding="application/x-tex"> \\{o_1:14 \\text{ (မှန်)}, o_2:16 \\text{ (မှား)}, o_3:10 \\text{ (မှား)}, \\ldots, o_8:14 \\text{ (မှန်)}\\} </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mopen">{</span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">1</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">:</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord">14</span><span class="mord text"><span class="mord"> (</span><span class="mord brahmic_fallback">မှန်</span><span class="mord">)</span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">2</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">:</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord">16</span><span class="mord text"><span class="mord"> (</span><span class="mord brahmic_fallback">မှား</span><span class="mord">)</span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">3</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">:</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord">10</span><span class="mord text"><span class="mord"> (</span><span class="mord brahmic_fallback">မှား</span><span class="mord">)</span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="minner">…</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">8</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">:</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord">14</span><span class="mord text"><span class="mord"> (</span><span class="mord brahmic_fallback">မှန်</span><span class="mord">)</span></span><span class="mclose">}</span></span></span></span>',pl,zs,Ro="ထုတ်ပေးထားတဲ့ အဖြေအချို့က ဘယ်လိုမှန် (14) ပြီး တချို့က မှား (16 သို့မဟုတ် 10) ဆိုတာ သတိပြုပါ။ ဒီကွဲပြားမှုက နောက်အဆင့်အတွက် အရေးကြီးပါတယ်။",ml,Js,il,Us,Eo="ကျွန်တော်တို့ responses များစွာ ရရှိပြီးတာနဲ့၊ ဘယ်ဟာတွေက တခြားဟာတွေထက် ပိုကောင်းလဲဆိုတာ ဆုံးဖြတ်ဖို့ နည်းလမ်းတစ်ခု လိုအပ်ပါတယ်။ ဒီနေရာမှာ advantage calculation က ပါဝင်လာပါတယ်။",rl,js,cl,ks,Bo="ပထမဆုံး၊ ထုတ်ပေးထားတဲ့ response တစ်ခုစီကို reward score တစ်ခု သတ်မှတ်ပါတယ်။ ဒီဥပမာမှာ၊ reward model ကို အသုံးပြုပါမယ်၊ ဒါပေမယ့် ယခင်အပိုင်းမှာ ကျွန်တော်တို့ သင်ယူခဲ့တဲ့အတိုင်း၊ မည်သည့် reward ပြန်ပေးတဲ့ function ကိုမဆို အသုံးပြုနိုင်ပါတယ်။",ol,E,rr,hl,Td='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>r</mi><mi>i</mi></msub></mrow><annotation encoding="application/x-tex"> r_i </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.5806em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',gl,dl,_d='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>r</mi><mi>i</mi></msub></mrow><annotation encoding="application/x-tex"> r_i </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.5806em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',yl,ul,Ls,vl,$s,cr,wl,Cd='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi>A</mi><mi>i</mi></msub><mo>=</mo><mfrac><mrow><msub><mi>r</mi><mi>i</mi></msub><mo>−</mo><mtext>mean</mtext><mo stretchy="false">(</mo><mo stretchy="false">{</mo><msub><mi>r</mi><mn>1</mn></msub><mo separator="true">,</mo><msub><mi>r</mi><mn>2</mn></msub><mo separator="true">,</mo><mo>…</mo><mo separator="true">,</mo><msub><mi>r</mi><mi>G</mi></msub><mo stretchy="false">}</mo><mo stretchy="false">)</mo></mrow><mrow><mtext>std</mtext><mo stretchy="false">(</mo><mo stretchy="false">{</mo><msub><mi>r</mi><mn>1</mn></msub><mo separator="true">,</mo><msub><mi>r</mi><mn>2</mn></msub><mo separator="true">,</mo><mo>…</mo><mo separator="true">,</mo><msub><mi>r</mi><mi>G</mi></msub><mo stretchy="false">}</mo><mo stretchy="false">)</mo></mrow></mfrac></mrow><annotation encoding="application/x-tex">A_i = \\frac{r_i - \\text{mean}(\\{r_1, r_2, \\ldots, r_G\\})}{\\text{std}(\\{r_1, r_2, \\ldots, r_G\\})}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.363em;vertical-align:-0.936em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.427em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord text"><span class="mord">std</span></span><span class="mopen">({</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">1</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">2</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="minner">…</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3283em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">G</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mclose">})</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord text"><span class="mord">mean</span></span><span class="mopen">({</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">1</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">2</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="minner">…</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3283em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">G</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mclose">})</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.936em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span>',Ml,Is,fl,As,Qo="အထက်ပါ ဥပမာအတွက် ကျွန်တော်တို့ရဲ့ ဂဏန်းတွက်ချက်မှု ဥပမာနဲ့ ဆက်လက်လုပ်ဆောင်ပါက၊ ကျွန်တော်တို့မှာ 8 responses ရှိပြီး၊ 4 ခုက မှန်ကန်ပြီး ကျန်တာတွေက မှားယွင်းတယ်လို့ မြင်ယောင်ကြည့်ပါ၊ ဒါကြောင့်-",xl,q,zt,Do="<tr><th>Metric</th> <th>Value</th></tr>",or,I,Hs,Jt,Zo="Group Average",hr,oe,bl,zd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>m</mi><mi>e</mi><mi>a</mi><mi>n</mi><mo stretchy="false">(</mo><msub><mi>r</mi><mi>i</mi></msub><mo stretchy="false">)</mo><mo>=</mo><mn>0.5</mn></mrow><annotation encoding="application/x-tex"> mean(r_i) = 0.5 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal">m</span><span class="mord mathnormal">e</span><span class="mord mathnormal">an</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0.5</span></span></span></span>',gr,Gs,Ut,Po="Standard Deviation",dr,he,Tl,Jd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>s</mi><mi>t</mi><mi>d</mi><mo stretchy="false">(</mo><msub><mi>r</mi><mi>i</mi></msub><mo stretchy="false">)</mo><mo>=</mo><mn>0.53</mn></mrow><annotation encoding="application/x-tex"> std(r_i) = 0.53 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal">s</span><span class="mord mathnormal">t</span><span class="mord mathnormal">d</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0.53</span></span></span></span>',yr,Rs,jt,So="Advantage Value (မှန်ကန်သော response)",ur,ge,_l,Ud='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>A</mi><mi>i</mi></msub><mo>=</mo><mfrac><mrow><mn>1</mn><mo>−</mo><mn>0.5</mn></mrow><mn>0.53</mn></mfrac><mo>=</mo><mn>0.94</mn></mrow><annotation encoding="application/x-tex"> A_i = \\frac{1 - 0.5}{0.53}= 0.94 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1.1901em;vertical-align:-0.345em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.8451em;"><span style="top:-2.655em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">0.53</span></span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.394em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">1</span><span class="mbin mtight">−</span><span class="mord mtight">0.5</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.345em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0.94</span></span></span></span>',vr,Es,kt,No="Advantage Value (မှားယွင်းသော response)",wr,de,Cl,jd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>A</mi><mi>i</mi></msub><mo>=</mo><mfrac><mrow><mn>0</mn><mo>−</mo><mn>0.5</mn></mrow><mn>0.53</mn></mfrac><mo>=</mo><mo>−</mo><mn>0.94</mn></mrow><annotation encoding="application/x-tex"> A_i = \\frac{0 - 0.5}{0.53}= -0.94 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1.1901em;vertical-align:-0.345em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.8451em;"><span style="top:-2.655em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">0.53</span></span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.394em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">0</span><span class="mbin mtight">−</span><span class="mord mtight">0.5</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.345em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.7278em;vertical-align:-0.0833em;"></span><span class="mord">−</span><span class="mord">0.94</span></span></span></span>',zl,Bs,Jl,Qs,Xo="အခု ကျွန်တော်တို့ advantage values တွေကို တွက်ချက်ပြီးပြီဆိုတော့၊ ဒါတွေက ဘာကိုဆိုလိုသလဲ နားလည်အောင် ကြိုးစားကြည့်ရအောင်။",Ul,C,Mr,jl,kd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>A</mi><mi>i</mi></msub></mrow><annotation encoding="application/x-tex"> A_i </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',kl,Ll,Ld='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>A</mi><mi>i</mi></msub><mo>></mo><mn>0</mn></mrow><annotation encoding="application/x-tex"> A_i > 0 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">></span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0</span></span></span></span>',$l,Il,$d='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>o</mi><mi>i</mi></msub></mrow><annotation encoding="application/x-tex"> o_i </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.5806em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',Al,Hl,Id='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>A</mi><mi>i</mi></msub><mo><</mo><mn>0</mn></mrow><annotation encoding="application/x-tex"> A_i < 0 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel"><</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0</span></span></span></span>',Gl,Rl,Ad='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>o</mi><mi>i</mi></msub></mrow><annotation encoding="application/x-tex"> o_i </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.5806em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',El,Bl,W,fr,Ql,Hd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>A</mi><mi>i</mi></msub><mo>=</mo><mn>0.94</mn><mtext>(မှန်ကန်သော output)</mtext></mrow><annotation encoding="application/x-tex"> A_i = 0.94 \\text{(မှန်ကန်သော output)} </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord">0.94</span><span class="mord text"><span class="mord">(</span><span class="mord brahmic_fallback">မှန်ကန်သော</span><span class="mord"> output)</span></span></span></span></span>',Dl,Zl,Ds,Vo="ကျွန်တော်တို့ advantage values တွေကို တွက်ချက်ပြီးပြီဆိုတော့၊ policy ကို update လုပ်ဖို့ အဆင်သင့်ဖြစ်ပါပြီ။",Pl,Zs,Sl,Ps,Fo="နောက်ဆုံးအဆင့်ကတော့ အနာဂတ်မှာ ကောင်းမွန်တဲ့ responses တွေကို ထုတ်ပေးနိုင်ခြေ ပိုများလာအောင် ကျွန်တော်တို့ရဲ့ model ကို update လုပ်ဖို့ ဒီ advantage values တွေကို အသုံးပြုခြင်းပဲ ဖြစ်ပါတယ်။",Nl,Ss,xr,Xl,Gd='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi>J</mi><mrow><mi>G</mi><mi>R</mi><mi>P</mi><mi>O</mi></mrow></msub><mo stretchy="false">(</mo><mi>θ</mi><mo stretchy="false">)</mo><mo>=</mo><mrow><mo fence="true">[</mo><mfrac><mn>1</mn><mi>G</mi></mfrac><munderover><mo>∑</mo><mrow><mi>i</mi><mo>=</mo><mn>1</mn></mrow><mi>G</mi></munderover><mi>min</mi><mo></mo><mrow><mo fence="true">(</mo><mfrac><mrow><msub><mi>π</mi><mi>θ</mi></msub><mo stretchy="false">(</mo><msub><mi>o</mi><mi>i</mi></msub><mi mathvariant="normal">∣</mi><mi>q</mi><mo stretchy="false">)</mo></mrow><mrow><msub><mi>π</mi><msub><mi>θ</mi><mrow><mi>o</mi><mi>l</mi><mi>d</mi></mrow></msub></msub><mo stretchy="false">(</mo><msub><mi>o</mi><mi>i</mi></msub><mi mathvariant="normal">∣</mi><mi>q</mi><mo stretchy="false">)</mo></mrow></mfrac><msub><mi>A</mi><mi>i</mi></msub><mtext>clip</mtext><mrow><mo fence="true">(</mo><mfrac><mrow><msub><mi>π</mi><mi>θ</mi></msub><mo stretchy="false">(</mo><msub><mi>o</mi><mi>i</mi></msub><mi mathvariant="normal">∣</mi><mi>q</mi><mo stretchy="false">)</mo></mrow><mrow><msub><mi>π</mi><msub><mi>θ</mi><mrow><mi>o</mi><mi>l</mi><mi>d</mi></mrow></msub></msub><mo stretchy="false">(</mo><msub><mi>o</mi><mi>i</mi></msub><mi mathvariant="normal">∣</mi><mi>q</mi><mo stretchy="false">)</mo></mrow></mfrac><mo separator="true">,</mo><mn>1</mn><mo>−</mo><mi>ϵ</mi><mo separator="true">,</mo><mn>1</mn><mo>+</mo><mi>ϵ</mi><mo fence="true">)</mo></mrow><msub><mi>A</mi><mi>i</mi></msub><mo fence="true">)</mo></mrow><mo fence="true">]</mo></mrow><mo>−</mo><mi>β</mi><msub><mi>D</mi><mrow><mi>K</mi><mi>L</mi></mrow></msub><mo stretchy="false">(</mo><msub><mi>π</mi><mi>θ</mi></msub><mi mathvariant="normal">∥</mi><mi mathvariant="normal">∥</mi><msub><mi>π</mi><mrow><mi>r</mi><mi>e</mi><mi>f</mi></mrow></msub><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">J_{GRPO}(\\theta) = \\left[\\frac{1}{G} \\sum_{i=1}^{G} \\min \\left( \\frac{\\pi_{\\theta}(o_i|q)}{\\pi_{\\theta_{old}}(o_i|q)} A_i \\text{clip}\\left( \\frac{\\pi_{\\theta}(o_i|q)}{\\pi_{\\theta_{old}}(o_i|q)}, 1 - \\epsilon, 1 + \\epsilon \\right) A_i \\right)\\right]- \\beta D_{KL}(\\pi_{\\theta} \\|\\| \\pi_{ref})</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.09618em;">J</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3283em;"><span style="top:-2.55em;margin-left:-0.0962em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">GRPO</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.02778em;">θ</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:3.106em;vertical-align:-1.2777em;"></span><span class="minner"><span class="mopen delimcenter" style="top:0em;"><span class="delimsizing size4">[</span></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal">G</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">1</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mop op-limits"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.8283em;"><span style="top:-1.8723em;margin-left:0em;"><span class="pstrut" style="height:3.05em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">i</span><span class="mrel mtight">=</span><span class="mord mtight">1</span></span></span></span><span style="top:-3.05em;"><span class="pstrut" style="height:3.05em;"></span><span><span class="mop op-symbol large-op">∑</span></span></span><span style="top:-4.3em;margin-left:0em;"><span class="pstrut" style="height:3.05em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">G</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:1.2777em;"><span></span></span></span></span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mop">min</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="minner"><span class="mopen delimcenter" style="top:0em;"><span class="delimsizing size3">(</span></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.427em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3448em;"><span style="top:-2.3488em;margin-left:-0.0278em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">o</span><span class="mord mathnormal mtight" style="margin-right:0.01968em;">l</span><span class="mord mathnormal mtight">d</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.1512em;"><span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.2559em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mord">∣</span><span class="mord mathnormal" style="margin-right:0.03588em;">q</span><span class="mclose">)</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mord">∣</span><span class="mord mathnormal" style="margin-right:0.03588em;">q</span><span class="mclose">)</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.9419em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mord text"><span class="mord">clip</span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="minner"><span class="mopen delimcenter" style="top:0em;"><span class="delimsizing size3">(</span></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.427em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3448em;"><span style="top:-2.3488em;margin-left:-0.0278em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">o</span><span class="mord mathnormal mtight" style="margin-right:0.01968em;">l</span><span class="mord mathnormal mtight">d</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.1512em;"><span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.2559em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mord">∣</span><span class="mord mathnormal" style="margin-right:0.03588em;">q</span><span class="mclose">)</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mord">∣</span><span class="mord mathnormal" style="margin-right:0.03588em;">q</span><span class="mclose">)</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.9419em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">1</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord mathnormal">ϵ</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">1</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord mathnormal">ϵ</span><span class="mclose delimcenter" style="top:0em;"><span class="delimsizing size3">)</span></span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mclose delimcenter" style="top:0em;"><span class="delimsizing size3">)</span></span></span><span class="mclose delimcenter" style="top:0em;"><span class="delimsizing size4">]</span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1.0361em;vertical-align:-0.2861em;"></span><span class="mord mathnormal" style="margin-right:0.05278em;">β</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">D</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3283em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.07153em;">K</span><span class="mord mathnormal mtight">L</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mord">∥∥</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">re</span><span class="mord mathnormal mtight" style="margin-right:0.10764em;">f</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.2861em;"><span></span></span></span></span></span></span><span class="mclose">)</span></span></span></span></span>',Vl,Ns,qo="ဒီ formula က အစပိုင်းမှာ ကြောက်စရာကောင်းတယ်လို့ ထင်ရပေမယ့်၊ အရေးကြီးတဲ့ ရည်ရွယ်ချက်တစ်ခုစီကို ဆောင်ရွက်ပေးတဲ့ အစိတ်အပိုင်းများစွာနဲ့ တည်ဆောက်ထားတာပါ။ တစ်ခုချင်းစီကို ဖော်ပြပေးပါမယ်။",Fl,Xs,ql,Vs,Wo="GRPO update function က တည်ငြိမ်ပြီး ထိရောက်တဲ့ သင်ယူမှုကို သေချာစေဖို့ နည်းလမ်းများစွာကို ပေါင်းစပ်ထားပါတယ်။ အစိတ်အပိုင်းတစ်ခုစီကို ကြည့်ရအောင်။",Wl,Fs,Ol,qs,br,Yl,Rd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo fence="true">(</mo><mfrac><mrow><msub><mi>π</mi><mi>θ</mi></msub><mo stretchy="false">(</mo><msub><mi>o</mi><mi>i</mi></msub><mi mathvariant="normal">∣</mi><mi>q</mi><mo stretchy="false">)</mo></mrow><mrow><msub><mi>π</mi><msub><mi>θ</mi><mrow><mi>o</mi><mi>l</mi><mi>d</mi></mrow></msub></msub><mo stretchy="false">(</mo><msub><mi>o</mi><mi>i</mi></msub><mi mathvariant="normal">∣</mi><mi>q</mi><mo stretchy="false">)</mo></mrow></mfrac><mo fence="true">)</mo></mrow><annotation encoding="application/x-tex"> \\left(\\frac{\\pi_{\\theta}(o_i|q)}{\\pi_{\\theta_{old}}(o_i|q)}\\right) </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1.8em;vertical-align:-0.65em;"></span><span class="minner"><span class="mopen delimcenter" style="top:0em;"><span class="delimsizing size2">(</span></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.01em;"><span style="top:-2.655em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3448em;"><span style="top:-2.3488em;margin-left:-0.0359em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3448em;"><span style="top:-2.3448em;margin-left:-0.0278em;margin-right:0.1em;"><span class="pstrut" style="height:2.6944em;"></span><span class="mord mtight"><span class="mord mathnormal mtight">o</span><span class="mord mathnormal mtight" style="margin-right:0.01968em;">l</span><span class="mord mathnormal mtight">d</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.3496em;"><span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.401em;"><span></span></span></span></span></span></span><span class="mopen mtight">(</span><span class="mord mtight"><span class="mord mathnormal mtight">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3281em;"><span style="top:-2.357em;margin-left:0em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.143em;"><span></span></span></span></span></span></span><span class="mord mtight">∣</span><span class="mord mathnormal mtight" style="margin-right:0.03588em;">q</span><span class="mclose mtight">)</span></span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.485em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3448em;"><span style="top:-2.3488em;margin-left:-0.0359em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.1512em;"><span></span></span></span></span></span></span><span class="mopen mtight">(</span><span class="mord mtight"><span class="mord mathnormal mtight">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3281em;"><span style="top:-2.357em;margin-left:0em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.143em;"><span></span></span></span></span></span></span><span class="mord mtight">∣</span><span class="mord mathnormal mtight" style="margin-right:0.03588em;">q</span><span class="mclose mtight">)</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.6257em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mclose delimcenter" style="top:0em;"><span class="delimsizing size2">)</span></span></span></span></span></span>',Kl,Ws,Oo="concept အားဖြင့်၊ ဒီ formula က new model ရဲ့ response probability က old model ရဲ့ response probability နဲ့ ဘယ်လောက်ကွာခြားသလဲဆိုတာကို နှိုင်းယှဉ်ပြီး၊ မျှော်မှန်းထားတဲ့ ရလဒ်ကို ပိုမိုကောင်းမွန်စေတဲ့ responses တွေအတွက် preference ကို ပေါင်းစပ်ထားပါတယ်။",sp,Os,ap,O,S,Tr,tp,Ed='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mtext>ratio</mtext><mo>></mo><mn>1</mn></mrow><annotation encoding="application/x-tex"> \\text{ratio} > 1 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.707em;vertical-align:-0.0391em;"></span><span class="mord text"><span class="mord">ratio</span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">></span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">1</span></span></span></span>',np,ep,Bd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>o</mi><mi>i</mi></msub></mrow><annotation encoding="application/x-tex"> o_i </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.5806em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',lp,_r,N,Cr,pp,Qd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mtext>ratio</mtext><mo><</mo><mn>1</mn></mrow><annotation encoding="application/x-tex"> \\text{ratio} < 1 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.707em;vertical-align:-0.0391em;"></span><span class="mord text"><span class="mord">ratio</span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel"><</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">1</span></span></span></span>',mp,ip,Dd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>o</mi><mi>i</mi></msub></mrow><annotation encoding="application/x-tex"> o_i </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.5806em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',rp,cp,Ys,Yo="ဒီ ratio က model က step တစ်ခုစီမှာ ဘယ်လောက်ပြောင်းလဲနိုင်လဲဆိုတာကို ကျွန်တော်တို့ကို ထိန်းချုပ်နိုင်စေပြီး၊ ဒါက ကျွန်တော်တို့ကို နောက်တစ်ဆင့်သို့ ဦးတည်စေပါတယ်။",op,Ks,hp,sa,zr,gp,Zd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mtext>clip</mtext><mrow><mo fence="true">(</mo><mfrac><mrow><msub><mi>π</mi><mi>θ</mi></msub><mo stretchy="false">(</mo><msub><mi>o</mi><mi>i</mi></msub><mi mathvariant="normal">∣</mi><mi>q</mi><mo stretchy="false">)</mo></mrow><mrow><msub><mi>π</mi><msub><mi>θ</mi><mrow><mi>o</mi><mi>l</mi><mi>d</mi></mrow></msub></msub><mo stretchy="false">(</mo><msub><mi>o</mi><mi>i</mi></msub><mi mathvariant="normal">∣</mi><mi>q</mi><mo stretchy="false">)</mo></mrow></mfrac><mo separator="true">,</mo><mn>1</mn><mo>−</mo><mi>ϵ</mi><mo separator="true">,</mo><mn>1</mn><mo>+</mo><mi>ϵ</mi><mo fence="true">)</mo></mrow></mrow><annotation encoding="application/x-tex"> \\text{clip}\\left( \\frac{\\pi_{\\theta}(o_i|q)}{\\pi_{\\theta_{old}}(o_i|q)}, 1 - \\epsilon, 1 + \\epsilon\\right) </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1.8em;vertical-align:-0.65em;"></span><span class="mord text"><span class="mord">clip</span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="minner"><span class="mopen delimcenter" style="top:0em;"><span class="delimsizing size2">(</span></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.01em;"><span style="top:-2.655em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3448em;"><span style="top:-2.3488em;margin-left:-0.0359em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3448em;"><span style="top:-2.3448em;margin-left:-0.0278em;margin-right:0.1em;"><span class="pstrut" style="height:2.6944em;"></span><span class="mord mtight"><span class="mord mathnormal mtight">o</span><span class="mord mathnormal mtight" style="margin-right:0.01968em;">l</span><span class="mord mathnormal mtight">d</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.3496em;"><span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.401em;"><span></span></span></span></span></span></span><span class="mopen mtight">(</span><span class="mord mtight"><span class="mord mathnormal mtight">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3281em;"><span style="top:-2.357em;margin-left:0em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.143em;"><span></span></span></span></span></span></span><span class="mord mtight">∣</span><span class="mord mathnormal mtight" style="margin-right:0.03588em;">q</span><span class="mclose mtight">)</span></span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.485em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3448em;"><span style="top:-2.3488em;margin-left:-0.0359em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.1512em;"><span></span></span></span></span></span></span><span class="mopen mtight">(</span><span class="mord mtight"><span class="mord mathnormal mtight">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3281em;"><span style="top:-2.357em;margin-left:0em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.143em;"><span></span></span></span></span></span></span><span class="mord mtight">∣</span><span class="mord mathnormal mtight" style="margin-right:0.03588em;">q</span><span class="mclose mtight">)</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.6257em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">1</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord mathnormal">ϵ</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">1</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord mathnormal">ϵ</span><span class="mclose delimcenter" style="top:0em;"><span class="delimsizing size2">)</span></span></span></span></span></span>',dp,Y,Jr,yp,Pd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><mn>1</mn><mo>−</mo><mi>ϵ</mi><mo separator="true">,</mo><mn>1</mn><mo>+</mo><mi>ϵ</mi><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex"> [1 - \\epsilon, 1 + \\epsilon] </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mopen">[</span><span class="mord">1</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.8389em;vertical-align:-0.1944em;"></span><span class="mord mathnormal">ϵ</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">1</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal">ϵ</span><span class="mclose">]</span></span></span></span>',up,vp,aa,wp,ta,Ko="ဒီ clipping function ကို ပိုမိုနားလည်နိုင်ဖို့ မတူညီတဲ့ အခြေအနေနှစ်ခုကို ကြည့်ရအောင်။",Mp,K,ss,Lt,sh="Case 1",Ur,fp,Sd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mtext>Ratio</mtext><mo>:</mo><mfrac><mrow><msub><mi>π</mi><mi>θ</mi></msub><mo stretchy="false">(</mo><msub><mi>o</mi><mi>i</mi></msub><mi mathvariant="normal">∣</mi><mi>q</mi><mo stretchy="false">)</mo></mrow><mrow><msub><mi>π</mi><msub><mi>θ</mi><mrow><mi>o</mi><mi>l</mi><mi>d</mi></mrow></msub></msub><mo stretchy="false">(</mo><msub><mi>o</mi><mi>i</mi></msub><mi mathvariant="normal">∣</mi><mi>q</mi><mo stretchy="false">)</mo></mrow></mfrac><mo>=</mo><mfrac><mn>0.9</mn><mn>0.5</mn></mfrac><mo>=</mo><mn>1.8</mn><mo>→</mo><mtext>Clip </mtext><mn>1.2</mn></mrow><annotation encoding="application/x-tex"> \\text{Ratio}: \\frac{\\pi_{\\theta}(o_i|q)}{\\pi_{\\theta_{old}}(o_i|q)} = \\frac{0.9}{0.5} = 1.8 → \\text{Clip}\\space1.2 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6833em;"></span><span class="mord text"><span class="mord">Ratio</span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">:</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1.6357em;vertical-align:-0.6257em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.01em;"><span style="top:-2.655em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3448em;"><span style="top:-2.3488em;margin-left:-0.0359em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3448em;"><span style="top:-2.3448em;margin-left:-0.0278em;margin-right:0.1em;"><span class="pstrut" style="height:2.6944em;"></span><span class="mord mtight"><span class="mord mathnormal mtight">o</span><span class="mord mathnormal mtight" style="margin-right:0.01968em;">l</span><span class="mord mathnormal mtight">d</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.3496em;"><span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.401em;"><span></span></span></span></span></span></span><span class="mopen mtight">(</span><span class="mord mtight"><span class="mord mathnormal mtight">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3281em;"><span style="top:-2.357em;margin-left:0em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.143em;"><span></span></span></span></span></span></span><span class="mord mtight">∣</span><span class="mord mathnormal mtight" style="margin-right:0.03588em;">q</span><span class="mclose mtight">)</span></span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.485em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3448em;"><span style="top:-2.3488em;margin-left:-0.0359em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.1512em;"><span></span></span></span></span></span></span><span class="mopen mtight">(</span><span class="mord mtight"><span class="mord mathnormal mtight">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3281em;"><span style="top:-2.357em;margin-left:0em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.143em;"><span></span></span></span></span></span></span><span class="mord mtight">∣</span><span class="mord mathnormal mtight" style="margin-right:0.03588em;">q</span><span class="mclose mtight">)</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.6257em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1.1901em;vertical-align:-0.345em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.8451em;"><span style="top:-2.655em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">0.5</span></span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.394em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">0.9</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.345em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">1.8</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">→</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.8889em;vertical-align:-0.1944em;"></span><span class="mord text"><span class="mord">Clip</span></span><span class="mspace"> </span><span class="mord">1.2</span></span></span></span>',xp,jr,as,$t,ah="Case 2",kr,bp,Nd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mtext>Ratio</mtext><mo>:</mo><mfrac><mrow><msub><mi>π</mi><mi>θ</mi></msub><mo stretchy="false">(</mo><msub><mi>o</mi><mi>i</mi></msub><mi mathvariant="normal">∣</mi><mi>q</mi><mo stretchy="false">)</mo></mrow><mrow><msub><mi>π</mi><msub><mi>θ</mi><mrow><mi>o</mi><mi>l</mi><mi>d</mi></mrow></msub></msub><mo stretchy="false">(</mo><msub><mi>o</mi><mi>i</mi></msub><mi mathvariant="normal">∣</mi><mi>q</mi><mo stretchy="false">)</mo></mrow></mfrac><mo>=</mo><mfrac><mn>0.2</mn><mn>0.5</mn></mfrac><mo>=</mo><mn>0.4</mn><mo>→</mo><mtext>Clip </mtext><mn>0.8</mn></mrow><annotation encoding="application/x-tex"> \\text{Ratio}: \\frac{\\pi_{\\theta}(o_i|q)}{\\pi_{\\theta_{old}}(o_i|q)} = \\frac{0.2}{0.5} = 0.4 →\\text{Clip}\\space0.8 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6833em;"></span><span class="mord text"><span class="mord">Ratio</span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">:</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1.6357em;vertical-align:-0.6257em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.01em;"><span style="top:-2.655em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3448em;"><span style="top:-2.3488em;margin-left:-0.0359em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3448em;"><span style="top:-2.3448em;margin-left:-0.0278em;margin-right:0.1em;"><span class="pstrut" style="height:2.6944em;"></span><span class="mord mtight"><span class="mord mathnormal mtight">o</span><span class="mord mathnormal mtight" style="margin-right:0.01968em;">l</span><span class="mord mathnormal mtight">d</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.3496em;"><span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.401em;"><span></span></span></span></span></span></span><span class="mopen mtight">(</span><span class="mord mtight"><span class="mord mathnormal mtight">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3281em;"><span style="top:-2.357em;margin-left:0em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.143em;"><span></span></span></span></span></span></span><span class="mord mtight">∣</span><span class="mord mathnormal mtight" style="margin-right:0.03588em;">q</span><span class="mclose mtight">)</span></span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.485em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3448em;"><span style="top:-2.3488em;margin-left:-0.0359em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.1512em;"><span></span></span></span></span></span></span><span class="mopen mtight">(</span><span class="mord mtight"><span class="mord mathnormal mtight">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3281em;"><span style="top:-2.357em;margin-left:0em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.143em;"><span></span></span></span></span></span></span><span class="mord mtight">∣</span><span class="mord mathnormal mtight" style="margin-right:0.03588em;">q</span><span class="mclose mtight">)</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.6257em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1.1901em;vertical-align:-0.345em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.8451em;"><span style="top:-2.655em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">0.5</span></span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.394em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">0.2</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.345em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0.4</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">→</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.8889em;vertical-align:-0.1944em;"></span><span class="mord text"><span class="mord">Clip</span></span><span class="mspace"> </span><span class="mord">0.8</span></span></span></span>',Tp,_p,na,Cp,k,It,th="ဒီ formula က old model က အလေးမထားခဲ့တဲ့ responses တွေကို new model က ပိုနှစ်သက်အောင် တိုက်တွန်းပါတယ် <strong>အကယ်၍ ၎င်းတို့က ရလဒ်ကို ပိုမိုကောင်းမွန်စေတယ်ဆိုရင်ပေါ့</strong>။",Lr,ea,$r,X,Ir,zp,Xd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><mn>1</mn><mo>−</mo><mi>ϵ</mi><mo separator="true">,</mo><mn>1</mn><mo>+</mo><mi>ϵ</mi><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex"> [1 - \\epsilon, 1 + \\epsilon] </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mopen">[</span><span class="mord">1</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.8389em;vertical-align:-0.1944em;"></span><span class="mord mathnormal">ϵ</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">1</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal">ϵ</span><span class="mclose">]</span></span></span></span>',Jp,Up,Vd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>ϵ</mi><mo>=</mo><mn>0.2</mn><mo separator="true">,</mo><mtext> ဆိုတော့ </mtext><mo stretchy="false">[</mo><mn>0.8</mn><mo>−</mo><mn>1.2</mn><mo stretchy="false">]</mo><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex"> \\epsilon = 0.2, \\space \\text{ဆိုတော့} \\space [0.8-1.2]) </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.4306em;"></span><span class="mord mathnormal">ϵ</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord">0.2</span><span class="mpunct">,</span><span class="mspace"> </span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord text"><span class="mord brahmic_fallback">ဆိုတော့</span></span><span class="mspace"> </span><span class="mopen">[</span><span class="mord">0.8</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord">1.2</span><span class="mclose">])</span></span></span></span>',jp,Ar,Hr,At,nh="အကယ်၍ old model က စွမ်းဆောင်ရည် ညံ့ဖျင်းတဲ့ response တစ်ခုကို အလွန်အကျွံ ခန့်မှန်းခဲ့တယ်ဆိုရင်၊ new model က အဲဒီ မြင့်မားတဲ့ probability ကို ဆက်လက်ထိန်းသိမ်းဖို့ <strong>အားမပေးပါဘူး</strong>။",Gr,la,Rr,kp,Fd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>A</mi><mi>i</mi></msub></mrow><annotation encoding="application/x-tex"> A_i </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',Lp,$p,pa,eh="Clipping function က ကြီးမားတဲ့ ပြောင်းလဲမှုတွေကို ကာကွယ်ရာမှာ ကူညီပေးပေမယ့်၊ ကျွန်တော်တို့ရဲ့ model က မူရင်း behavior ကနေ အဝေးကြီး မသွေဖည်အောင် သေချာစေဖို့ နောက်ထပ် ကာကွယ်မှုတစ်ခု လိုအပ်ပါသေးတယ်။",Ip,ma,Ap,ia,Er,Hp,qd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>β</mi><msub><mi>D</mi><mrow><mi>K</mi><mi>L</mi></mrow></msub><mo stretchy="false">(</mo><msub><mi>π</mi><mi>θ</mi></msub><mi mathvariant="normal">∥</mi><mi mathvariant="normal">∥</mi><msub><mi>π</mi><mrow><mi>r</mi><mi>e</mi><mi>f</mi></mrow></msub><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex"> \\beta D_{KL}(\\pi_{\\theta} \\|\\| \\pi_{ref}) </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1.0361em;vertical-align:-0.2861em;"></span><span class="mord mathnormal" style="margin-right:0.05278em;">β</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">D</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3283em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.07153em;">K</span><span class="mord mathnormal mtight">L</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mord">∥∥</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">re</span><span class="mord mathnormal mtight" style="margin-right:0.10764em;">f</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.2861em;"><span></span></span></span></span></span></span><span class="mclose">)</span></span></span></span>',Gp,z,Br,Rp,Wd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>π</mi><mrow><mi>r</mi><mi>e</mi><mi>f</mi></mrow></msub></mrow><annotation encoding="application/x-tex"> \\pi_{ref} </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.7167em;vertical-align:-0.2861em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">re</span><span class="mord mathnormal mtight" style="margin-right:0.10764em;">f</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.2861em;"><span></span></span></span></span></span></span></span></span></span>',Ep,Ht,lh="per_token_logps",Qr,Bp,Od='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>π</mi><mi>θ</mi></msub></mrow><annotation encoding="application/x-tex"> \\pi_{\\theta} </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.5806em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',Qp,Gt,ph="new_per_token_logps",Dr,Dp,ra,Zp,ca,mh="<li>KL divergence penalty က model ရဲ့ outputs တွေကို ၎င်းရဲ့ မူရင်း distribution နဲ့ နီးကပ်အောင် ထိန်းထားပြီး၊ ကြီးမားတဲ့ ပြောင်းလဲမှုတွေကို ကာကွယ်ပေးပါတယ်။</li> <li>လုံးဝမဆီလျော်တဲ့ outputs တွေဆီ လွင့်မျောသွားမယ့်အစား၊ model က အချို့သော exploration ကို ခွင့်ပြုရင်း ၎င်းရဲ့ နားလည်မှုကို ပိုမိုကောင်းမွန်အောင် လုပ်ဆောင်ပါလိမ့်မယ်။</li>",Pp,oa,Sp,ha,ih="သင်္ချာဆိုင်ရာ အသေးစိတ်အချက်အလက်တွေကို စိတ်ဝင်စားသူများအတွက်၊ တရားဝင် အဓိပ္ပာယ်ဖွင့်ဆိုချက်ကို ကြည့်ကြရအောင်။",Np,ts,Zr,Xp,Yd='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi>D</mi><mrow><mi>K</mi><mi>L</mi></mrow></msub><mo stretchy="false">(</mo><mi>P</mi><mi mathvariant="normal">∥</mi><mi mathvariant="normal">∥</mi><mi>Q</mi><mo stretchy="false">)</mo><mo>=</mo><munder><mo>∑</mo><mrow><mi>x</mi><mo>∈</mo><mi>X</mi></mrow></munder><mi>P</mi><mo stretchy="false">(</mo><mi>x</mi><mo stretchy="false">)</mo><mi>log</mi><mo></mo><mfrac><mrow><mi>P</mi><mo stretchy="false">(</mo><mi>x</mi><mo stretchy="false">)</mo></mrow><mrow><mi>Q</mi><mo stretchy="false">(</mo><mi>x</mi><mo stretchy="false">)</mo></mrow></mfrac></mrow><annotation encoding="application/x-tex">D_{KL}(P \\|\\| Q) = \\sum_{x \\in X} P(x) \\log \\frac{P(x)}{Q(x)}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">D</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3283em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.07153em;">K</span><span class="mord mathnormal mtight">L</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mord">∥∥</span><span class="mord mathnormal">Q</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.7487em;vertical-align:-1.3217em;"></span><span class="mop op-limits"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.05em;"><span style="top:-1.8557em;margin-left:0em;"><span class="pstrut" style="height:3.05em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">x</span><span class="mrel mtight">∈</span><span class="mord mathnormal mtight" style="margin-right:0.07847em;">X</span></span></span></span><span style="top:-3.05em;"><span class="pstrut" style="height:3.05em;"></span><span><span class="mop op-symbol large-op">∑</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:1.3217em;"><span></span></span></span></span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord mathnormal">x</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mop">lo<span style="margin-right:0.01389em;">g</span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.427em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal">Q</span><span class="mopen">(</span><span class="mord mathnormal">x</span><span class="mclose">)</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord mathnormal">x</span><span class="mclose">)</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.936em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span>',Vp,Fp,ga,qp,ns,Pr,Wp,Kd='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>β</mi></mrow><annotation encoding="application/x-tex"> \\beta </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8889em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.05278em;">β</span></span></span></span>',Op,Yp,B,Rt,rh="<strong>β ပိုမြင့်ခြင်း (KL Penalty ပိုအားကောင်းခြင်း)</strong> <ul><li>Policy updates တွေပေါ်မှာ ကန့်သတ်ချက် ပိုများပါတယ်။ Model က ၎င်းရဲ့ reference distribution နဲ့ နီးကပ်စွာ တည်ရှိနေပါတယ်။</li> <li>Adaptation ကို နှေးကွေးစေနိုင်ပါတယ်- Model က ပိုကောင်းတဲ့ responses တွေကို ရှာဖွေရာမှာ ခက်ခဲနိုင်ပါတယ်။</li></ul>",Sr,Et,ch="<strong>β ပိုနိမ့်ခြင်း (KL Penalty ပိုအားနည်းခြင်း)</strong> <ul><li>Policy ကို update လုပ်ဖို့ လွတ်လပ်မှု ပိုများပါတယ်။ Model က reference ကနေ ပိုပြီး သွေဖည်နိုင်ပါတယ်။</li> <li>Adaptation ပိုမြန်ပေမယ့် မတည်ငြိမ်နိုင်ခြေ ရှိပါတယ်- Model က reward-hacking behaviors တွေကို သင်ယူနိုင်ပါတယ်။</li> <li>Over-optimization risk: အကယ်၍ reward model က ချို့ယွင်းနေတယ်ဆိုရင်၊ policy က အဓိပ္ပာယ်မရှိတဲ့ outputs တွေကို ထုတ်ပေးနိုင်ပါတယ်။</li></ul>",Nr,Q,Bt,oh="မူရင်း",Xr,es,hh="DeepSeekMath",Vr,Kp,s0='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>β</mi><mo>=</mo><mn>0.04</mn></mrow><annotation encoding="application/x-tex"> \\beta= 0.04 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8889em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.05278em;">β</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0.04</span></span></span></span>',sm,am,da,gh="အခု ကျွန်တော်တို့ GRPO ရဲ့ အစိတ်အပိုင်းတွေကို နားလည်ပြီဆိုတော့၊ ဒါတွေက ပြည့်စုံတဲ့ ဥပမာတစ်ခုမှာ ဘယ်လိုအတူတကွ အလုပ်လုပ်လဲဆိုတာ ကြည့်ရအောင်။",tm,ya,nm,ua,dh="GRPO အပေါ် ကျွန်တော်တို့ရဲ့ နားလည်မှုကို ခိုင်မာစေဖို့၊ အစကနေ အဆုံးထိ ပြည့်စုံတဲ့ ဥပမာတစ်ခုကို လုပ်ဆောင်ကြည့်ရအောင်။",em,va,lm,Qt,pm,a0='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mtext>Q: Calculate </mtext><mn>2</mn><mo>+</mo><mn>2</mn><mo>×</mo><mn>6</mn></mrow><annotation encoding="application/x-tex">\\text{Q: Calculate}\\space2 + 2 \\times 6</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8889em;vertical-align:-0.1944em;"></span><span class="mord text"><span class="mord">Q: Calculate</span></span><span class="mspace"> </span><span class="mord">2</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.7278em;vertical-align:-0.0833em;"></span><span class="mord">2</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">×</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">6</span></span></span></span></span>',mm,wa,im,Ma,yh="ပထမဆုံး၊ ကျွန်တော်တို့ model ကနေ responses များစွာကို ထုတ်ပေးပါတယ်။",rm,U,Fr,cm,t0='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">(</mo><mi>G</mi><mo>=</mo><mn>8</mn><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex"> (G = 8) </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mopen">(</span><span class="mord mathnormal">G</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord">8</span><span class="mclose">)</span></span></span></span>',om,hm,n0='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mn>4</mn></mrow><annotation encoding="application/x-tex"> 4 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">4</span></span></span></span>',gm,dm,e0='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mn>4</mn></mrow><annotation encoding="application/x-tex"> 4 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">4</span></span></span></span>',ym,um,l0='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi>o</mi><mn>1</mn></msub><mo>:</mo><mn>14</mn><mo stretchy="false">(</mo><mi>c</mi><mi>o</mi><mi>r</mi><mi>r</mi><mi>e</mi><mi>c</mi><mi>t</mi><mo stretchy="false">)</mo><mo separator="true">,</mo><msub><mi>o</mi><mn>2</mn></msub><mo>:</mo><mn>10</mn><mo stretchy="false">(</mo><mi>w</mi><mi>r</mi><mi>o</mi><mi>n</mi><mi>g</mi><mo stretchy="false">)</mo><mo separator="true">,</mo><msub><mi>o</mi><mn>3</mn></msub><mo>:</mo><mn>16</mn><mo stretchy="false">(</mo><mi>w</mi><mi>r</mi><mi>o</mi><mi>n</mi><mi>g</mi><mo stretchy="false">)</mo><mo separator="true">,</mo><mi mathvariant="normal">.</mi><mi mathvariant="normal">.</mi><mi mathvariant="normal">.</mi><msub><mi>o</mi><mi>G</mi></msub><mo>:</mo><mn>14</mn><mo stretchy="false">(</mo><mi>c</mi><mi>o</mi><mi>r</mi><mi>r</mi><mi>e</mi><mi>c</mi><mi>t</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">{o_1:14(correct), o_2:10 (wrong), o_3:16 (wrong), ... o_G:14(correct)}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">1</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">:</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mord">14</span><span class="mopen">(</span><span class="mord mathnormal">correc</span><span class="mord mathnormal">t</span><span class="mclose">)</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">2</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">:</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mord">10</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.02691em;">w</span><span class="mord mathnormal">ro</span><span class="mord mathnormal">n</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mclose">)</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">3</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">:</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mord">16</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.02691em;">w</span><span class="mord mathnormal">ro</span><span class="mord mathnormal">n</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mclose">)</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">...</span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3283em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">G</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">:</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mord">14</span><span class="mopen">(</span><span class="mord mathnormal">correc</span><span class="mord mathnormal">t</span><span class="mclose">)</span></span></span></span></span></span>',vm,fa,wm,xa,uh="နောက်တစ်ခုက၊ ဘယ် responses တွေက ပျမ်းမျှထက် ပိုကောင်းလဲဆိုတာ ဆုံးဖြတ်ဖို့ advantage values တွေကို တွက်ချက်ပါတယ်။",Mm,ls,Dt,vh="<tr><th>Statistic</th> <th>Value</th></tr>",qr,A,ba,Zt,wh="Group Average",Wr,ye,fm,p0='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>m</mi><mi>e</mi><mi>a</mi><mi>n</mi><mo stretchy="false">(</mo><msub><mi>r</mi><mi>i</mi></msub><mo stretchy="false">)</mo><mo>=</mo><mn>0.5</mn></mrow><annotation encoding="application/x-tex"> mean(r_i) = 0.5 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal">m</span><span class="mord mathnormal">e</span><span class="mord mathnormal">an</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0.5</span></span></span></span>',Or,Ta,Pt,Mh="Standard Deviation",Yr,ue,xm,m0='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>s</mi><mi>t</mi><mi>d</mi><mo stretchy="false">(</mo><msub><mi>r</mi><mi>i</mi></msub><mo stretchy="false">)</mo><mo>=</mo><mn>0.53</mn></mrow><annotation encoding="application/x-tex"> std(r_i) = 0.53 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal">s</span><span class="mord mathnormal">t</span><span class="mord mathnormal">d</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0.53</span></span></span></span>',Kr,_a,St,fh="Advantage Value (မှန်ကန်သော response)",sc,ve,bm,i0='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>A</mi><mi>i</mi></msub><mo>=</mo><mfrac><mrow><mn>1</mn><mo>−</mo><mn>0.5</mn></mrow><mn>0.53</mn></mfrac><mo>=</mo><mn>0.94</mn></mrow><annotation encoding="application/x-tex"> A_i = \\frac{1 - 0.5}{0.53}= 0.94 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1.1901em;vertical-align:-0.345em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.8451em;"><span style="top:-2.655em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">0.53</span></span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.394em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">1</span><span class="mbin mtight">−</span><span class="mord mtight">0.5</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.345em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0.94</span></span></span></span>',ac,Ca,Nt,xh="Advantage Value (မှားယွင်းသော response)",tc,we,Tm,r0='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>A</mi><mi>i</mi></msub><mo>=</mo><mfrac><mrow><mn>0</mn><mo>−</mo><mn>0.5</mn></mrow><mn>0.53</mn></mfrac><mo>=</mo><mo>−</mo><mn>0.94</mn></mrow><annotation encoding="application/x-tex"> A_i = \\frac{0 - 0.5}{0.53}= -0.94 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1.1901em;vertical-align:-0.345em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.8451em;"><span style="top:-2.655em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">0.53</span></span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.394em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">0</span><span class="mbin mtight">−</span><span class="mord mtight">0.5</span></span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.345em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.7278em;vertical-align:-0.0833em;"></span><span class="mord">−</span><span class="mord">0.94</span></span></span></span>',_m,za,Cm,Ja,bh="နောက်ဆုံးအနေနဲ့၊ မှန်ကန်တဲ့ responses တွေကို အားဖြည့်ဖို့ ကျွန်တော်တို့ရဲ့ model ကို update လုပ်ပါတယ်။",zm,ps,J,nc,Jm,c0='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>o</mi><mn>1</mn></msub></mrow><annotation encoding="application/x-tex"> o_1 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.5806em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">1</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',Um,Xt,Th="{\\theta",ec,jm,o0='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mn>0.5</mn></mrow><annotation encoding="application/x-tex"> 0.5 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0.5</span></span></span></span>',km,Lm,h0='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mn>0.7</mn></mrow><annotation encoding="application/x-tex"> 0.7 </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0.7</span></span></span></span>',$m,Im,g0='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mtext>Ratio</mtext><mo>:</mo><mfrac><mn>0.7</mn><mn>0.5</mn></mfrac><mo>=</mo><mn>1.4</mn><mo>→</mo><mtext>after Clip </mtext><mn>1.2</mn><mtext> </mtext><mo stretchy="false">(</mo><mi>ϵ</mi><mo>=</mo><mn>0.2</mn><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">\\text{Ratio}: \\frac{0.7}{0.5} = 1.4 →\\text{after Clip}\\space1.2 \\space (\\epsilon = 0.2)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6833em;"></span><span class="mord text"><span class="mord">Ratio</span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">:</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">0.5</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">0.7</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">1.4</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">→</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord text"><span class="mord">after Clip</span></span><span class="mspace"> </span><span class="mord">1.2</span><span class="mspace"> </span><span class="mopen">(</span><span class="mord mathnormal">ϵ</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord">0.2</span><span class="mclose">)</span></span></span></span></span>',lc,Ua,pc,Am,d0='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mtext>KL Divergence</mtext></mrow><annotation encoding="application/x-tex"> \\text{KL Divergence} </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8778em;vertical-align:-0.1944em;"></span><span class="mord text"><span class="mord">KL Divergence</span></span></span></span></span>',Hm,Gm,ja,_h="သီအိုရီဆိုင်ရာ နားလည်မှု ရှိပြီဆိုတော့၊ GRPO ကို code မှာ ဘယ်လို အကောင်အထည်ဖော်နိုင်လဲ ကြည့်ကြရအောင်။",Rm,ka,Em,La,Ch="လက်တွေ့ဥပမာတစ်ခုမှာ အရာအားလုံးကို ပေါင်းစပ်ကြည့်ရအောင်။ အောက်ပါ code က PyTorch မှာ GRPO ကို ဘယ်လို implement လုပ်ရမယ်ဆိုတာ ပြသထားပါတယ်။",Bm,$a,Qm,Ia,zh="ပထမဆုံး၊ ကျွန်တော်တို့ model တစ်ခုကို load လုပ်ပြီး ပေးထားတဲ့ မေးခွန်းတစ်ခုအတွက် responses များစွာကို ထုတ်ပေးဖို့ လိုအပ်ပါတယ်။",Dm,Aa,Zm,Ha,Jh="ဒီကနဦး Generation (မည်သည့်အဆင့်မှ မလုပ်ခင်) မှာ အောက်ပါအတိုင်း output ထုတ်ပေးပါလိမ့်မယ်။",Pm,Ga,Sm,Ra,Nm,Ea,Uh="အခု၊ ဘယ် responses တွေက မှန်ကန်ပြီး အဲဒါတွေအတိုင်း rewards တွေကို ဘယ်လိုသတ်မှတ်ရမလဲဆိုတာ ဆုံးဖြတ်ဖို့ လိုအပ်ပါတယ်။",Xm,Ba,jh="GRPO နဲ့အတူ၊ prompt ဥပမာတူတူအတွက်၊ ကျွန်တော်တို့ completions များစွာကို ထုတ်ပေးပါတယ်။ ဒါကြောင့် ဥပမာ၊ ကျွန်တော်တို့ရဲ့ prompts တွေဖြစ်တဲ့ <code>"Solve y = 2x + 1 for x = 2, y = "</code> နဲ့ <code>Solve y = 2x + 1 for x = 4, y = "</code> အတွက်၊ ပေးထားတဲ့ prompt အတွက် group of generated outputs နှစ်ခုရှိပါတယ်။ တစ်ခုကတော့",Vm,Qa,kh="<li><code>[5, 6, 7, 5]</code> ဖြစ်ပြီး နောက်တစ်ခုက</li> <li><code>[10, 2, 9, 9]</code> ဖြစ်ပါတယ်။ မှန်ကန်တဲ့အဖြေကတော့ 5 နဲ့ 9 ဖြစ်ပါတယ်။</li>",Fm,Da,Lh="လက်တွေ့မှာတော့ ဒီ reward scores တွေကို rule-based reward function တစ်ခုကနေ ရရှိပြီး response ရဲ့ မှန်ကန်မှုအပေါ် အခြေခံပြီး rewards တွေ သတ်မှတ်ပေးပါတယ်။ ဒါမှမဟုတ် response ရဲ့ မှန်ကန်မှုအပေါ် ဒါမှမဟုတ် နှစ်ခုပေါင်းစပ်ပြီး rewards တွေ သတ်မှတ်ဖို့ လေ့ကျင့်ထားတဲ့ ပိုရှုပ်ထွေးတဲ့ neural network-based model တစ်ခုကို အသုံးပြုနိုင်ပါတယ်။ ဒါပေမယ့် ရိုးရှင်းစေဖို့အတွက်၊ response က မှန်ကန်ရင် reward 1၊ မှားယွင်းရင် 0 လို့ ပြောကြပါစို့၊ ဒါကြောင့်",qm,Za,Wm,Pa,$h="နောက်တစ်ခုကတော့ rewards တွေရဲ့ group-wise mean နဲ့ std ကို ရယူပါမယ်။",Om,Sa,Ym,Na,Ih="ဒါက အောက်ပါအတိုင်း output ထုတ်ပေးပါလိမ့်မယ်။",Km,Xa,si,Va,Ah="အခု ကျွန်တော်တို့ response တစ်ခုစီအတွက် advantage values တွေကို တွက်ချက်နိုင်ပါပြီ။",ai,Fa,ti,qa,Hh="ဒါက အောက်ပါအတိုင်း output ထုတ်ပေးပါလိမ့်မယ်။",ni,Wa,ei,Oa,Gh="ဒါက အထက်ပါ Advantage formula ကနေ လာတာဖြစ်ပြီး-",li,Ya,pi,Ka,Rh="သို့သော်လည်း၊ ဒီမှာ shape က <code>(B*G,) = (8,)</code> ဖြစ်ပေမယ့်၊ လက်တွေ့မှာ logits shape နဲ့ ကိုက်ညီဖို့ <code>(B, G) = (2, 4)</code> shape လိုအပ်ပါတယ်၊ ဟုတ်တယ်မလား။ ဒါကြောင့်၊ logits shape နဲ့ ကိုက်ညီဖို့ advantages tensor ကို <code>(B*G, 1) = (8, 1)</code> shape ရအောင် unsqueeze လုပ်ဖို့ လိုအပ်ပါတယ်။",mi,st,ii,at,Eh="ဒါက အောက်ပါအတိုင်း output ထုတ်ပေးပါလိမ့်မယ်။",ri,tt,ci,nt,Bh="အခု ကျွန်တော်တို့ အဆင်သင့်ဖြစ်ပါပြီ၊ advantage values တွေအပေါ် အခြေခံပြီး policy model ကို update လုပ်မယ့် နောက်အဆင့်ကို သွားကြရအောင်။",oi,et,hi,lt,Qh="နောက်ဆုံးအနေနဲ့၊ advantage values တွေကို အသုံးပြုပြီး ကျွန်တော်တို့ model ကို update လုပ်ပါတယ်။",gi,pt,di,mt,Dh="<code>per_token_logps</code> ကို generated outputs တွေကို model ကို ပေးပို့ပြီး logits တွေ ရယူကာ softmax function ကို အသုံးပြုပြီး probabilities <code>F.softmax(logits, dim=-1)</code> ကို ရယူခြင်းဖြင့် ရရှိနိုင်ကြောင်း မှတ်သားပါ။",yi,it,ui,rt,Zh="<code>per_token_kl</code> ကိုလည်း အောက်ပါအတိုင်း တွက်ချက်နိုင်ပါတယ်။",vi,ct,wi,ot,Ph='ပြည့်စုံတဲ့ ဥပမာကို <a href="./basic_example.py">ဒီမှာ</a> ရှာတွေ့နိုင်ပါတယ်။ GRPO ကို အလွန်ကောင်းမွန်တဲ့ TRL team ကလည်း implement လုပ်ထားပါတယ်၊ အသေးစိတ်အချက်အလက်တွေအတွက် <a href="https://github.com/huggingface/trl/blob/main/trl/trainer/grpo_trainer.py" rel="nofollow">TRL/GRPO_trainer</a> ကို ကြည့်ရှုနိုင်ပါတယ်။',Mi,ht,fi,gt,Sh="ဂုဏ်ယူပါတယ်။ အခုဆိုရင် သင် Group Relative Policy Optimization (GRPO) အကြောင်းကို သင်ယူခဲ့ပါပြီ။ ကျွန်တော်တို့ ဖော်ပြခဲ့တာတွေကို ပြန်လည်အကျဉ်းချုပ်ရရင်…",xi,dt,Nh=`၁။ GRPO က သီးခြား value model မလိုအပ်ဘဲ group တစ်ခုအတွင်းရှိ outputs များစွာကို နှိုင်းယှဉ်ပြီး ဘယ်ဟာတွေက တခြားဟာတွေထက် ပိုကောင်းလဲဆိုတာ ဆုံးဖြတ်ပါတယ်။ | |
| ၂။ Advantage calculation က rewards တွေကို standardize လုပ်ပြီး ဘယ် responses တွေက ပျမ်းမျှထက် အပေါ် ဒါမှမဟုတ် အောက်လဲဆိုတာ ဖော်ထုတ်ပါတယ်။ | |
| ၃။ Policy update က KL divergence penalty ပါဝင်တဲ့ clipped objective function ကို အသုံးပြုပြီး တည်ငြိမ်တဲ့ သင်ယူမှုကို သေချာစေပါတယ်။`,bi,yt,Xh="ဒီနည်းလမ်းက သင်္ချာဆိုင်ရာ reasoning tasks တွေအတွက် အထူးသဖြင့် အစွမ်းထက်ပါတယ်။ အဲဒီ tasks တွေမှာ မှန်ကန်မှုကို ပုံသဏ္ဍာန်ကျကျ စစ်ဆေးနိုင်ပါတယ်။ GRPO နည်းလမ်းက သီးခြား critic model တစ်ခု လိုအပ်တဲ့ traditional RLHF နည်းလမ်းတွေနဲ့ နှိုင်းယှဉ်ရင် ပိုမိုထိရောက်တဲ့ training ကို ခွင့်ပြုပါတယ်။",Ti,ut,Vh="GRPO ကို ဆက်လက်လေ့လာရင်းနဲ့ မတူညီတဲ့ group sizes, reward functions, နဲ့ KL penalty coefficients တွေကို စမ်းသပ်ကြည့်ပြီး ဒါတွေက သင့် model ရဲ့ စွမ်းဆောင်ရည်ကို ဘယ်လိုသက်ရောက်မှုရှိလဲ ကြည့်ရှုဖို့ စဉ်းစားပါ။",_i,vt,Fh="ပျော်ရွှင်စွာ train ပါ! 🚀",Ci,wt,zi,Mt,qh='<li><a href="https://github.com/natolambert/rlhf-book" rel="nofollow">RLHF Book by Nathan Lambert</a></li> <li><a href="https://huggingface.co/papers/2412.19437" rel="nofollow">DeepSeek-V3 Technical Report</a></li> <li><a href="https://huggingface.co/papers/2402.03300" rel="nofollow">DeepSeekMath</a></li>',Ji,Ui,ji,ft,ki,c,Vt,Wh="<strong>Group Relative Policy Optimization (GRPO)</strong>: DeepSeekMath စာတမ်းတွင် မိတ်ဆက်ထားသော Reinforcement Learning (RL) algorithm တစ်ခုဖြစ်ပြီး၊ model-generated responses များကို group အတွင်း နှိုင်းယှဉ်ခြင်းဖြင့် policy model ကို အကဲဖြတ်ပြီး optimization လုပ်သည်။ သီးခြား value model (Critic) မလိုအပ်ပေ။",mc,Ft,Oh="<strong>DeepSeekMath</strong>: သင်္ချာဆိုင်ရာ reasoning အတွက် ဒီဇိုင်းထုတ်ထားသော Large Language Model (LLM) တစ်မျိုး။",ic,qt,Yh="<strong>Policy Model</strong>: Reinforcement Learning (RL) တွင် agent ၏ decision-making strategy ကို ကိုယ်စားပြုသော model။ ၎င်းသည် ပေးထားသော state တစ်ခုအတွက် မည်သည့် action (response) ကို လုပ်ဆောင်သင့်ကြောင်း သတ်မှတ်သည်။",rc,Wt,Kh="<strong>Computational Cost</strong>: algorithm တစ်ခုကို run ရန် လိုအပ်သော ကွန်ပျူတာ အရင်းအမြစ်များ (ဥပမာ- CPU/GPU time, memory)။",cc,Ot,sg="<strong>Verifiable Task</strong>: response ၏ မှန်ကန်မှုကို အ objective ကျကျ စစ်ဆေးနိုင်သော task။",oc,Yt,ag="<strong>Math Reasoning</strong>: သင်္ချာဆိုင်ရာ ပြဿနာများကို ဖြေရှင်းရန်အတွက် ဆင်ခြင်တုံတရားနှင့် ဆက်စပ်တွေးခေါ်မှုများ အသုံးပြုခြင်း။",hc,Kt,tg="<strong>Ground Truth</strong>: အမှန်တကယ် မှန်ကန်သော အဖြေ သို့မဟုတ် အချက်အလက်။",gc,sn,ng="<strong>Group Sampling</strong>: GRPO တွင် မေးခွန်းတစ်ခုစီအတွက် responses များစွာကို ထုတ်ပေးခြင်း။",dc,an,eg="<strong><code>G</code> (Group Size)</strong>: မေးခွန်းတစ်ခုစီအတွက် ထုတ်ပေးသော responses အရေအတွက်။",yc,tn,lg="<strong><code>o_i</code></strong>: ထုတ်ပေးသော response တစ်ခုစီ။",uc,nn,pg="<strong><code>\\pi_{\\theta_{old}}</code></strong>: Policy model ၏ ယခင် version (pre-update)။",vc,en,mg="<strong>Reward Score (<code>r_i</code>)</strong>: response တစ်ခု၏ အရည်အသွေး သို့မဟုတ် မှန်ကန်မှုအပေါ် အခြေခံ၍ သတ်မှတ်သော တန်ဖိုး။",wc,ln,ig="<strong>Reward Model (RM)</strong>: response များကို အကဲဖြတ်ပြီး reward score များပေးရန် လေ့ကျင့်ထားသော model။",Mc,pn,rg="<strong>Advantage Calculation</strong>: response တစ်ခု၏ reward ကို ၎င်း၏ group အတွင်းရှိ ပျမ်းမျှ reward နှင့် နှိုင်းယှဉ်ခြင်း။",fc,mn,cg="<strong>Standardization</strong>: ဒေတာအချက်အလက်များကို common scale တစ်ခုသို့ ပြောင်းလဲခြင်း (ဥပမာ- mean 0, standard deviation 1)။",xc,rn,og="<strong><code>mean(r_i)</code></strong>: rewards များ၏ ပျမ်းမျှတန်ဖိုး။",bc,cn,hg="<strong><code>std(r_i)</code></strong>: rewards များ၏ standard deviation (စံသွေဖည်မှု)။",Tc,on,gg="<strong>Policy Update</strong>: model ၏ parameters များကို ပြန်လည်ချိန်ညှိခြင်း (update လုပ်ခြင်း)။",_c,hn,dg="<strong>Target Function (<code>J_{GRPO}(\\theta)</code>)</strong>: Policy ကို optimization လုပ်ရာတွင် အသုံးပြုသော objective function။",Cc,D,gn,yg="<code>\\pi_{\\theta}(o_i|q)</code>",zc,Li,y0='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>o</mi><mi>i</mi></msub></mrow><annotation encoding="application/x-tex"> o_i </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.5806em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">o</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s"></span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',$i,Ii,u0='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>q</mi></mrow><annotation encoding="application/x-tex"> q </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.625em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.03588em;">q</span></span></span></span>',Ai,Jc,ms,dn,ug="<code>\\text{clip}(\\dots, 1 - \\epsilon, 1 + \\epsilon)</code>",Uc,Hi,v0='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><mn>1</mn><mo>−</mo><mi>ϵ</mi><mo separator="true">,</mo><mn>1</mn><mo>+</mo><mi>ϵ</mi><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex"> [1 - \\epsilon, 1 + \\epsilon] </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mopen">[</span><span class="mord">1</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.8389em;vertical-align:-0.1944em;"></span><span class="mord mathnormal">ϵ</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">1</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal">ϵ</span><span class="mclose">]</span></span></span></span>',Gi,jc,yn,vg="<strong><code>\\epsilon</code></strong>: Clipping function ၏ parameter ဖြစ်ပြီး ratio က မူရင်း policy မှ မည်မျှသွေဖည်နိုင်သည်ကို ထိန်းချုပ်သည်။",kc,un,wg="<strong><code>\\beta D_{KL}(\\pi_{\\theta} \\|\\| \\pi_{ref})</code></strong>: KL Divergence penalty term။ New policy ကို reference policy နှင့် နီးကပ်စွာ ထိန်းထားရန် အသုံးပြုသည်။",Lc,vn,Mg="<strong>Probability Ratio</strong>: New policy ၏ response probability ကို old policy ၏ response probability နှင့် နှိုင်းယှဉ်ခြင်း။",$c,wn,fg="<strong><code>D_{KL}(P \\|\\| Q)</code> (KL Divergence)</strong>: Distribution P က distribution Q မှ မည်မျှကွာခြားသည်ကို တိုင်းတာသော သင်္ချာဆိုင်ရာ တန်ဖိုး။",Ic,Mn,xg="<strong><code>P(x)</code></strong>: New model version ၏ output distribution။",Ac,fn,bg="<strong><code>Q(x)</code> / <code>\\pi_{ref}</code></strong>: Reference policy ၏ output distribution (pre-update model ၏ output)။",Hc,xn,Tg="<strong><code>per_token_logps</code></strong>: Token တစ်ခုစီ၏ log-probabilities များ။",Gc,bn,_g="<strong><code>new_per_token_logps</code></strong>: New model မှ ထုတ်ပေးသော token တစ်ခုစီ၏ log-probabilities များ။",Rc,Tn,Cg="<strong><code>\\beta</code> Parameter</strong>: KL divergence penalty ၏ အလေးချိန်ကို ထိန်းချုပ်သော coefficient။",Ec,_n,zg="<strong>Reward-Hacking Behaviors</strong>: Reward function တွင် ချို့ယွင်းချက်များကို အကျိုးယူပြီး မကောင်းမွန်သော (သို့မဟုတ် မရည်ရွယ်သော) responses များကို ထုတ်ပေးခြင်း။",Bc,Cn,Jg="<strong><code>torch</code></strong>: PyTorch library။",Qc,zn,Ug="<strong><code>torch.nn.functional as F</code></strong>: PyTorch ၏ functional API။",Dc,Jn,jg="<strong><code>AutoModelForCausalLM</code></strong>: 🤗 Transformers မှ Causal Language Model အမျိုးအစားကို အလိုအလျောက် load လုပ်ရန် class။",Zc,Un,kg="<strong><code>AutoTokenizer</code></strong>: 🤗 Transformers မှ Tokenizer အမျိုးအစားကို အလိုအလျောက် load လုပ်ရန် class။",Pc,jn,Lg="<strong><code>model.eval()</code></strong>: PyTorch model ကို evaluation mode သို့ ပြောင်းလဲရန်။",Sc,kn,$g="<strong><code>device</code></strong>: Model ကို run မည့် hardware (CPU သို့မဟုတ် GPU)။",Nc,Ln,Ig="<strong><code>tokenizer(prompt, return_tensors="pt", padding=True)</code></strong>: Input prompt ကို tokenize လုပ်ပြီး PyTorch tensors အဖြစ် ပြန်ပေးခြင်း။",Xc,$n,Ag="<strong><code>input_ids</code></strong>: Tokenized input ၏ ID များ။",Vc,In,Hg="<strong><code>attention_mask</code></strong>: Tokenized input ၏ attention mask။",Fc,An,Gg="<strong><code>model.generate()</code></strong>: Model မှ text များကို ထုတ်ပေးရန် method။",qc,Hn,Rg="<strong><code>max_new_tokens</code></strong>: ထုတ်ပေးမည့် tokens အရေအတွက် အများဆုံး။",Wc,Gn,Eg="<strong><code>num_return_sequences</code></strong>: ပြန်ပေးမည့် sequences အရေအတွက်။",Oc,Rn,Bg="<strong><code>do_sample</code></strong>: Sampling လုပ်ခြင်းကို ဖွင့်/ပိတ်။",Yc,En,Qg="<strong><code>top_k</code></strong>: Sampling လုပ်ရာတွင် ထိပ်ဆုံး k tokens ကိုသာ စဉ်းစားပါ။",Kc,Bn,Dg="<strong><code>temperature</code></strong>: Sampling လုပ်ရာတွင် randomness ကို ထိန်းချုပ်သော parameter။",so,Qn,Zg="<strong><code>pad_token_id</code></strong>: Padding token ၏ ID။",ao,Dn,Pg="<strong><code>tokenizer.eos_token_id</code></strong>: End-of-sequence token ၏ ID။",to,Zn,Sg="<strong><code>return_dict_in_generate</code></strong>: generate method မှ output များကို dictionary အဖြစ် ပြန်ပေးမလား။",no,Pn,Ng="<strong><code>output_scores</code></strong>: generate method မှ scores များကို ပြန်ပေးမလား။",eo,Sn,Xg="<strong><code>rewards_grouped.mean(dim=1)</code></strong>: Group အလိုက် rewards များ၏ ပျမ်းမျှကို တွက်ချက်ခြင်း။",lo,Nn,Vg="<strong><code>rewards_grouped.std(dim=1)</code></strong>: Group အလိုက် rewards များ၏ standard deviation ကို တွက်ချက်ခြင်း။",po,Xn,Fg="<strong><code>repeat_interleave()</code></strong>: Tensor ၏ element များကို သတ်မှတ်ထားသော အကြိမ်အရေအတွက်အတိုင်း ထပ်ခါတလဲလဲ ပြုလုပ်ခြင်း။",mo,Vn,qg="<strong><code>+ 1e-8</code></strong>: Division by zero ကို ကာကွယ်ရန်အတွက် denominator (အောက်ခြေဂဏန်း) တွင် ထည့်သွင်းထားသော သေးငယ်သည့် တန်ဖိုး။",io,Fn,Wg="<strong><code>advantages.unsqueeze(1)</code></strong>: Tensor ၏ dimension တစ်ခုကို ထပ်ထည့်ခြင်း (ဥပမာ- (8,) မှ (8, 1) သို့)။",ro,qn,Og="<strong><code>torch.exp()</code></strong>: Exponential function ကို တွက်ချက်ခြင်း။",co,Wn,Yg="<strong><code>torch.clamp()</code></strong>: Tensor ၏ တန်ဖိုးများကို သတ်မှတ်ထားသော အတိုင်းအတာအတွင်း ကန့်သတ်ခြင်း။",oo,On,Kg="<strong><code>pg_losses1</code>, <code>pg_losses2</code>, <code>pg_loss_max</code></strong>: Policy Gradient (PG) loss ၏ အမျိုးမျိုးသော ပုံစံများ။",ho,Yn,sd="<strong><code>F.kl_div()</code></strong>: KL Divergence ကို တွက်ချက်ခြင်း။",go,Kn,ad="<strong><code>F.log_softmax()</code></strong>: Log Softmax function ကို တွက်ချက်ခြင်း။",yo,se,td="<strong><code>reduction="none"</code></strong>: KL Divergence တွက်ချက်မှု၏ reduction type (reduction မလုပ်ခြင်း)။",uo,ae,nd="<strong><code>sum(dim=-1, keepdim=True)</code></strong>: သတ်မှတ်ထားသော dimension ပေါ်တွင် sum လုပ်ပြီး dimension ကို ဆက်လက်ထိန်းသိမ်းထားခြင်း။",vo,te,ed="<strong>TRL (Transformer Reinforcement Learning)</strong>: Hugging Face မှ Reinforcement Learning အတွက် library။",Ri,xt,Ei,Me,Bi;return os=new _0({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),hs=new T({props:{title:"DeepSeekMath တွင် Group Relative Policy Optimization (GRPO) ကို အဆင့်မြင့် နားလည်ခြင်း",local:"advanced-understanding-of-group-relative-policy-optimization-grpo-in-deepseekmath",headingTag:"h1"}}),Ms=new T({props:{title:"GRPO Algorithm",local:"grpo-algorithm",headingTag:"h2"}}),bs=new T({props:{title:"အဆင့် ၁: Group Sampling",local:"အဆင-၁-group-sampling",headingTag:"h3"}}),_s=new T({props:{title:"ဥပမာ",local:"ဥပမ",headingTag:"h4"}}),Js=new T({props:{title:"အဆင့် ၂: Advantage Calculation",local:"အဆင-၂-advantage-calculation",headingTag:"h3"}}),js=new T({props:{title:"Reward Distribution",local:"reward-distribution",headingTag:"h4"}}),Ls=new T({props:{title:"Advantage Value Formula",local:"advantage-value-formula",headingTag:"h4"}}),Is=new T({props:{title:"ဥပမာ",local:"ဥပမ",headingTag:"h4"}}),Bs=new T({props:{title:"အဓိပ္ပာယ်ဖွင့်ဆိုချက်",local:"အဓပပယဖငဆခက",headingTag:"h4"}}),Zs=new T({props:{title:"အဆင့် ၃: Policy Update",local:"အဆင-၃-policy-update",headingTag:"h3"}}),Xs=new T({props:{title:"Target Function ၏ အဓိက အစိတ်အပိုင်းများ",local:"target-function--အဓက-အစတအပငမ",headingTag:"h2"}}),Fs=new T({props:{title:"၁။ Probability Ratio",local:"၁-probability-ratio",headingTag:"h3"}}),Os=new T({props:{title:"အဓိပ္ပာယ်ဖွင့်ဆိုချက်",local:"အဓပပယဖငဆခက",headingTag:"h4"}}),Ks=new T({props:{title:"၂။ Clip Function",local:"၂-clip-function",headingTag:"h3"}}),aa=new T({props:{title:"ဥပမာ (ε = 0.2)",local:"ဥပမ-ε--02",headingTag:"h4"}}),na=new T({props:{title:"အဓိပ္ပာယ်ဖွင့်ဆိုချက်",local:"အဓပပယဖငဆခက",headingTag:"h4"}}),ma=new T({props:{title:"၃။ KL Divergence",local:"၃-kl-divergence",headingTag:"h3"}}),ra=new T({props:{title:"အဓိပ္ပာယ်ဖွင့်ဆိုချက်",local:"အဓပပယဖငဆခက",headingTag:"h4"}}),oa=new T({props:{title:"သင်္ချာဆိုင်ရာ အဓိပ္ပာယ်ဖွင့်ဆိုချက်",local:"သငခဆငရ-အဓပပယဖငဆခက",headingTag:"h4"}}),ga=new T({props:{title:"β Parameter ၏ အခန်းကဏ္ဍ",local:"β-parameter--အခနကဏဍ",headingTag:"h4"}}),ya=new T({props:{title:"GRPO ဖြင့် လုပ်ဆောင်ခဲ့သော ဥပမာ",local:"grpo-ဖင-လပဆငခသ-ဥပမ",headingTag:"h2"}}),va=new T({props:{title:"ဥပမာ ပြဿနာ",local:"ဥပမ-ပဿန",headingTag:"h3"}}),wa=new T({props:{title:"အဆင့် ၁: Group Sampling",local:"အဆင-၁-group-sampling",headingTag:"h3"}}),fa=new T({props:{title:"အဆင့် ၂: Advantage Calculation",local:"အဆင-၂-advantage-calculation",headingTag:"h3"}}),za=new T({props:{title:"အဆင့် ၃: Policy Update",local:"အဆင-၃-policy-update",headingTag:"h3"}}),ka=new T({props:{title:"Implementation Example",local:"implementation-example",headingTag:"h2"}}),$a=new T({props:{title:"၁။ Model ကို Loading လုပ်ခြင်းနှင့် Responses များ ထုတ်ပေးခြင်း",local:"၁-model-က-loading-လပခငနင-responses-မ-ထတပခင",headingTag:"h3"}}),Aa=new j({props:{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjB0b3JjaC5ubi5mdW5jdGlvbmFsJTIwYXMlMjBGJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQSUyMyUyMExvYWQlMjB0aGUlMjBtb2RlbCUyMGFuZCUyMHRva2VuaXplciUwQW1vZGVsX25hbWUlMjAlM0QlMjAlMjJRd2VuJTJGUXdlbjItTWF0aC0xLjVCJTIyJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfbmFtZSklMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9uYW1lKSUwQW1vZGVsLmV2YWwoKSUwQSUwQSUyMyUyME1vdmUlMjBtb2RlbCUyMHRvJTIwR1BVJTIwaWYlMjBhdmFpbGFibGUlMEFkZXZpY2UlMjAlM0QlMjB0b3JjaC5kZXZpY2UoJTIyY3VkYSUyMiUyMGlmJTIwdG9yY2guY3VkYS5pc19hdmFpbGFibGUoKSUyMGVsc2UlMjAlMjJjcHUlMjIpJTBBbW9kZWwudG8oZGV2aWNlKSUwQSUwQSUyMyUyMElucHV0JTIwcHJvbXB0JTBBcHJvbXB0JTIwJTNEJTIwJTIyU29sdmUlMjB5JTIwJTNEJTIwMnglMjAlMkIlMjAxJTIwZm9yJTIweCUyMCUzRCUyMDIlMkMlMjB5JTIwJTNEJTIwJTIyJTIwJTIwJTIzJTIwQ29ycmVjdCUyMGFuc3dlciUzQSUyMDUlMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIocHJvbXB0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiUyQyUyMHBhZGRpbmclM0RUcnVlKSUwQWlucHV0X2lkcyUyMCUzRCUyMGlucHV0cyU1QiUyMmlucHV0X2lkcyUyMiU1RC50byhkZXZpY2UpJTIwJTIwJTIzJTIwU2hhcGUlM0ElMjAoMSUyQyUyMHByb21wdF9sZW4pJTBBYXR0ZW50aW9uX21hc2slMjAlM0QlMjBpbnB1dHMlNUIlMjJhdHRlbnRpb25fbWFzayUyMiU1RC50byhkZXZpY2UpJTBBJTBBJTIzJTIwU3RlcCUyMDElM0ElMjBHZW5lcmF0ZSUyMDglMjByZXNwb25zZXMlMjAoQiUyMCUzRCUyMDIlMjBncm91cHMlMkMlMjBHJTIwJTNEJTIwNCUyMHJlc3BvbnNlcyUyMHBlciUyMGdyb3VwKSUwQWJhdGNoX3NpemUlMkMlMjBudW1fZ2VuZXJhdGlvbnMlMjAlM0QlMjAyJTJDJTIwNCUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSglMEElMjAlMjAlMjAlMjBpbnB1dF9pZHMlM0RpbnB1dF9pZHMlMkMlMjAlMjAlMjMlMjBTaGFwZSUzQSUyMCgxJTJDJTIwcHJvbXB0X2xlbiklMEElMjAlMjAlMjAlMjBhdHRlbnRpb25fbWFzayUzRGF0dGVudGlvbl9tYXNrJTJDJTBBJTIwJTIwJTIwJTIwbWF4X25ld190b2tlbnMlM0QxJTJDJTIwJTIwJTIzJTIwc2VxX2xlbiUyMCUzRCUyMDElMjAoc2luZ2xlJTIwdG9rZW4lMjBwZXIlMjByZXNwb25zZSklMEElMjAlMjAlMjAlMjBudW1fcmV0dXJuX3NlcXVlbmNlcyUzRGJhdGNoX3NpemUlMjAqJTIwbnVtX2dlbmVyYXRpb25zJTJDJTIwJTIwJTIzJTIwOCUyMHJlc3BvbnNlcyUyMHRvdGFsJTBBJTIwJTIwJTIwJTIwZG9fc2FtcGxlJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMHRvcF9rJTNEMTAlMkMlMEElMjAlMjAlMjAlMjB0ZW1wZXJhdHVyZSUzRDAuNyUyQyUwQSUyMCUyMCUyMCUyMHBhZF90b2tlbl9pZCUzRHRva2VuaXplci5lb3NfdG9rZW5faWQlMkMlMEElMjAlMjAlMjAlMjByZXR1cm5fZGljdF9pbl9nZW5lcmF0ZSUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBvdXRwdXRfc2NvcmVzJTNEVHJ1ZSUyQyUwQSk=",highlighted:`<span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">import</span> torch.nn.functional <span class="hljs-keyword">as</span> F | |
| <span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer | |
| <span class="hljs-comment"># Load the model and tokenizer</span> | |
| model_name = <span class="hljs-string">"Qwen/Qwen2-Math-1.5B"</span> | |
| model = AutoModelForCausalLM.from_pretrained(model_name) | |
| tokenizer = AutoTokenizer.from_pretrained(model_name) | |
| model.<span class="hljs-built_in">eval</span>() | |
| <span class="hljs-comment"># Move model to GPU if available</span> | |
| device = torch.device(<span class="hljs-string">"cuda"</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">"cpu"</span>) | |
| model.to(device) | |
| <span class="hljs-comment"># Input prompt</span> | |
| prompt = <span class="hljs-string">"Solve y = 2x + 1 for x = 2, y = "</span> <span class="hljs-comment"># Correct answer: 5</span> | |
| inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">"pt"</span>, padding=<span class="hljs-literal">True</span>) | |
| input_ids = inputs[<span class="hljs-string">"input_ids"</span>].to(device) <span class="hljs-comment"># Shape: (1, prompt_len)</span> | |
| attention_mask = inputs[<span class="hljs-string">"attention_mask"</span>].to(device) | |
| <span class="hljs-comment"># Step 1: Generate 8 responses (B = 2 groups, G = 4 responses per group)</span> | |
| batch_size, num_generations = <span class="hljs-number">2</span>, <span class="hljs-number">4</span> | |
| outputs = model.generate( | |
| input_ids=input_ids, <span class="hljs-comment"># Shape: (1, prompt_len)</span> | |
| attention_mask=attention_mask, | |
| max_new_tokens=<span class="hljs-number">1</span>, <span class="hljs-comment"># seq_len = 1 (single token per response)</span> | |
| num_return_sequences=batch_size * num_generations, <span class="hljs-comment"># 8 responses total</span> | |
| do_sample=<span class="hljs-literal">True</span>, | |
| top_k=<span class="hljs-number">10</span>, | |
| temperature=<span class="hljs-number">0.7</span>, | |
| pad_token_id=tokenizer.eos_token_id, | |
| return_dict_in_generate=<span class="hljs-literal">True</span>, | |
| output_scores=<span class="hljs-literal">True</span>, | |
| )`,wrap:!1}}),Ga=new j({props:{code:"T3V0cHV0JTIwMSUzQSUyMDUuMCUwQU91dHB1dCUyMDIlM0ElMjA2LjAlMEFPdXRwdXQlMjAzJTNBJTIwNy4wJTBBT3V0cHV0JTIwNCUzQSUyMDUuMCUwQU91dHB1dCUyMDUlM0ElMjAxMC4wJTBBT3V0cHV0JTIwNiUzQSUyMDIuMCUwQU91dHB1dCUyMDclM0ElMjA1LjAlMEFPdXRwdXQlMjA4JTNBJTIwNS4w",highlighted:`Output 1: 5.0 | |
| Output 2: 6.0 | |
| Output 3: 7.0 | |
| Output 4: 5.0 | |
| Output 5: 10.0 | |
| Output 6: 2.0 | |
| Output 7: 5.0 | |
| Output 8: 5.0`,wrap:!1}}),Ra=new T({props:{title:"၂။ Rewards များကို တွက်ချက်ခြင်း",local:"၂-rewards-မက-တကခကခင",headingTag:"h3"}}),Za=new j({props:{code:"cmV3YXJkXzElMjAlM0QlMjAlNUIxJTJDJTIwMCUyQyUyMDAlMkMlMjAxJTVEJTBBcmV3YXJkXzIlMjAlM0QlMjAlNUIwJTJDJTIwMCUyQyUyMDElMkMlMjAxJTVE",highlighted:`reward_1 = [<span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>] | |
| reward_2 = [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]`,wrap:!1}}),Sa=new j({props:{code:"JTIzJTIwU2hhcGUlM0ElMjAoQiUyMColMjBHJTJDKSUyMCUzRCUyMCg4JTJDKSUyMGJjJTIwd2UlMjBoYXZlJTIwMiUyMGdyb3VwcyUyMG9mJTIwNCUyMGdlbmVyYXRpb25zJTIwdGhhdCUyMHdlJTIwZmxhdHRlbiUwQXJld2FyZHMlMjAlM0QlMjB0b3JjaC50ZW5zb3IoJTVCMSUyQyUyMDAlMkMlMjAwJTJDJTIwMSUyQyUyMDAlMkMlMjAwJTJDJTIwMSUyQyUyMDElNUQlMkMlMjBkdHlwZSUzRHRvcmNoLmZsb2F0MzIpJTBBbnVtX2dlbmVyYXRpb25zJTIwJTNEJTIwNCUwQSUwQSUyMyUyMEdyb3VwJTIwcmV3YXJkcyUzQSUyMFNoYXBlJTIwKEIlMkMlMjBHKSUyMCUzRCUyMDIlMkMlMjA0KSUwQXJld2FyZHNfZ3JvdXBlZCUyMCUzRCUyMHJld2FyZHMudmlldygtMSUyQyUyMG51bV9nZW5lcmF0aW9ucyklMEElMEElMjMlMjBNZWFuJTIwcGVyJTIwZ3JvdXAlM0ElMjBTaGFwZSUyMChCJTJDKSUyMCUzRCUyMCgyJTJDKSUwQW1lYW5fZ3JvdXBlZF9yZXdhcmRzJTIwJTNEJTIwcmV3YXJkc19ncm91cGVkLm1lYW4oZGltJTNEMSklMEElMEElMjMlMjBTdGQlMjBwZXIlMjBncm91cCUzQSUyMFNoYXBlJTIwKEIlMkMpJTIwJTNEJTIwKDIlMkMpJTBBc3RkX2dyb3VwZWRfcmV3YXJkcyUyMCUzRCUyMHJld2FyZHNfZ3JvdXBlZC5zdGQoZGltJTNEMSklMEElMEElMjMlMjBCcm9hZGNhc3QlMjB0byUyMG1hdGNoJTIwcmV3YXJkcyUyMGFuZCUyMG5vcm1hbGl6ZSUzQSUyMFNoYXBlJTIwKEIlMjAqJTIwRyUyQyklMjAlM0QlMjAoOCUyQyklMEElMjMlMjB3aHklMjB3ZSUyMG5lZWQlMjB0byUyMGJyb2FkY2FzdCUzRiUyMGJlY2F1c2UlMjB3ZSUyMG5lZWQlMjB0byUyMGNhbGN1bGF0ZSUyMHRoZSUyMGFkdmFudGFnZSUyMHZhbHVlcyUyMGZvciUyMGVhY2glMjByZXNwb25zZSUyMHdpdGhpbiUyMHRoZSUyMGdyb3VwJTBBbWVhbl9ncm91cGVkX3Jld2FyZHMlMjAlM0QlMjBtZWFuX2dyb3VwZWRfcmV3YXJkcy5yZXBlYXRfaW50ZXJsZWF2ZShudW1fZ2VuZXJhdGlvbnMlMkMlMjBkaW0lM0QwKSUwQXN0ZF9ncm91cGVkX3Jld2FyZHMlMjAlM0QlMjBzdGRfZ3JvdXBlZF9yZXdhcmRzLnJlcGVhdF9pbnRlcmxlYXZlKG51bV9nZW5lcmF0aW9ucyUyQyUyMGRpbSUzRDAp",highlighted:`<span class="hljs-comment"># Shape: (B * G,) = (8,) bc we have 2 groups of 4 generations that we flatten</span> | |
| rewards = torch.tensor([<span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], dtype=torch.float32) | |
| num_generations = <span class="hljs-number">4</span> | |
| <span class="hljs-comment"># Group rewards: Shape (B, G) = 2, 4)</span> | |
| rewards_grouped = rewards.view(-<span class="hljs-number">1</span>, num_generations) | |
| <span class="hljs-comment"># Mean per group: Shape (B,) = (2,)</span> | |
| mean_grouped_rewards = rewards_grouped.mean(dim=<span class="hljs-number">1</span>) | |
| <span class="hljs-comment"># Std per group: Shape (B,) = (2,)</span> | |
| std_grouped_rewards = rewards_grouped.std(dim=<span class="hljs-number">1</span>) | |
| <span class="hljs-comment"># Broadcast to match rewards and normalize: Shape (B * G,) = (8,)</span> | |
| <span class="hljs-comment"># why we need to broadcast? because we need to calculate the advantage values for each response within the group</span> | |
| mean_grouped_rewards = mean_grouped_rewards.repeat_interleave(num_generations, dim=<span class="hljs-number">0</span>) | |
| std_grouped_rewards = std_grouped_rewards.repeat_interleave(num_generations, dim=<span class="hljs-number">0</span>)`,wrap:!1}}),Xa=new j({props:{code:"R3JvdXBlZCUyMFJld2FyZHMlM0ElMjB0ZW5zb3IoJTVCJTVCMS4lMkMlMjAwLiUyQyUyMDAuJTJDJTIwMS4lNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUIwLiUyQyUyMDAuJTJDJTIwMS4lMkMlMjAxLiU1RCU1RCklMEFNZWFuJTIwcGVyJTIwZ3JvdXAlM0ElMjB0ZW5zb3IoJTVCMC41MDAwJTJDJTIwMC41MDAwJTVEKSUwQVN0ZCUyMHBlciUyMGdyb3VwJTNBJTIwdGVuc29yKCU1QjAuNTc3NCUyQyUyMDAuNTc3NCU1RCklMEFCcm9hZGNhc3RlZCUyME1lYW4lM0ElMjB0ZW5zb3IoJTVCMC41MDAwJTJDJTIwMC41MDAwJTJDJTIwMC41MDAwJTJDJTIwMC41MDAwJTJDJTIwMC41MDAwJTJDJTIwMC41MDAwJTJDJTIwMC41MDAwJTJDJTIwMC41MDAwJTVEKSUwQUJyb2FkY2FzdGVkJTIwU3RkJTNBJTIwdGVuc29yKCU1QjAuNTc3NCUyQyUyMDAuNTc3NCUyQyUyMDAuNTc3NCUyQyUyMDAuNTc3NCUyQyUyMDAuNTc3NCUyQyUyMDAuNTc3NCUyQyUyMDAuNTc3NCUyQyUyMDAuNTc3NCU1RCk=",highlighted:`Grouped Rewards: tensor([[1., 0., 0., 1.], | |
| [0., 0., 1., 1.]]) | |
| Mean per group: tensor([0.5000, 0.5000]) | |
| Std per group: tensor([0.5774, 0.5774]) | |
| Broadcasted Mean: tensor([0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000]) | |
| Broadcasted Std: tensor([0.5774, 0.5774, 0.5774, 0.5774, 0.5774, 0.5774, 0.5774, 0.5774])`,wrap:!1}}),Fa=new j({props:{code:"JTIzJTIwQWR2YW50YWdlcyUzQSUyMFNoYXBlJTIwKEIlMjAqJTIwRyUyQyklMjAlM0QlMjAoOCUyQyklMEFhZHZhbnRhZ2VzJTIwJTNEJTIwKHJld2FyZHMlMjAtJTIwbWVhbl9ncm91cGVkX3Jld2FyZHMpJTIwJTJGJTIwKHN0ZF9ncm91cGVkX3Jld2FyZHMlMjAlMkIlMjAxZS04KQ==",highlighted:`<span class="hljs-comment"># Advantages: Shape (B * G,) = (8,)</span> | |
| advantages = (rewards - mean_grouped_rewards) / (std_grouped_rewards + <span class="hljs-number">1e-8</span>)`,wrap:!1}}),Wa=new j({props:{code:"QWR2YW50YWdlcyUzQSUyMHRlbnNvciglNUIlMjAwLjg2NTklMkMlMjAtMC44NjYwJTJDJTIwLTAuODY2MCUyQyUyMCUyMDAuODY1OSUyQyUyMC0wLjg2NjAlMkMlMjAtMC44NjYwJTJDJTIwJTIwMC44NjU5JTJDJTIwJTIwMC44NjU5JTVEKQ==",highlighted:"Advantages: tensor([ 0.8659, -0.8660, -0.8660, 0.8659, -0.8660, -0.8660, 0.8659, 0.8659])",wrap:!1}}),Ya=new j({props:{code:"cmV3YXJkXzElMjAlM0QlMjAlNUIxJTJDJTIwMCUyQyUyMDAlMkMlMjAxJTVEJTIwJUUxJTgwJUExJUUxJTgwJTkwJUUxJTgwJUJEJUUxJTgwJTgwJUUxJTgwJUJBJTNBJTBBMSUyMC0lMjAwLjUlMjAlMkYlMjAwLjU3NzQlMjAlRTIlODklODglMjAwLjg2NTklMEEwJTIwLSUyMDAuNSUyMCUyRiUyMDAuNTc3NCUyMCVFMiU4OSU4OCUyMC0wLjg2NjAlMEFyZXdhcmRfMiUyMCUzRCUyMCU1QjAlMkMlMjAwJTJDJTIwMSUyQyUyMDElNUQlMjAlRTElODAlQTElRTElODAlOTAlRTElODAlQkQlRTElODAlODAlRTElODAlQkElM0ElMjAlRTElODAlOTUlRTElODAlQUYlRTElODAlQjYlRTElODAlODUlRTElODAlQjYlRTElODAlOTAlRTElODAlQjAlRTElODElOEI=",highlighted:`reward_1 = [1, 0, 0, 1] အတွက်: | |
| 1 - 0.5 / 0.5774 ≈ 0.8659 | |
| 0 - 0.5 / 0.5774 ≈ -0.8660 | |
| reward_2 = [0, 0, 1, 1] အတွက်: ပုံစံတူ။`,wrap:!1}}),st=new j({props:{code:"JTIzJTIwU2hhcGUlMjAoQiUyMColMjBHJTJDJTIwMSklMjAlM0QlMjAoOCUyQyUyMDEpJTIwdG8lMjBtYXRjaCUyMHRoZSUyMGxvZ2l0cyUyMHNoYXBlJTBBYWR2YW50YWdlcyUyMCUzRCUyMGFkdmFudGFnZXMudW5zcXVlZXplKDEp",highlighted:`<span class="hljs-comment"># Shape (B * G, 1) = (8, 1) to match the logits shape</span> | |
| advantages = advantages.unsqueeze(<span class="hljs-number">1</span>)`,wrap:!1}}),tt=new j({props:{code:"QWR2YW50YWdlcyUzQSUyMHRlbnNvciglNUIlNUIlMjAwLjg2NTklNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUItMC44NjYwJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVCLTAuODY2MCU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU1QiUyMDAuODY1OSU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU1Qi0wLjg2NjAlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUItMC44NjYwJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVCJTIwMC44NjU5JTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVCJTIwMC44NjU5JTVEJTVEKQ==",highlighted:`Advantages: tensor([[ 0.8659], | |
| [-0.8660], | |
| [-0.8660], | |
| [ 0.8659], | |
| [-0.8660], | |
| [-0.8660], | |
| [ 0.8659], | |
| [ 0.8659]])`,wrap:!1}}),et=new T({props:{title:"၃။ Policy ကို Update လုပ်ခြင်း",local:"၃-policy-က-update-လပခင",headingTag:"h3"}}),pt=new j({props:{code:"JTIzJTIwQ29tcHV0ZSUyMHByb2JhYmlsaXR5JTIwcmF0aW8lMjBiZXR3ZWVuJTIwbmV3JTIwYW5kJTIwb2xkJTIwcG9saWNpZXMlMEFyYXRpbyUyMCUzRCUyMHRvcmNoLmV4cCglMEElMjAlMjAlMjAlMjBuZXdfcGVyX3Rva2VuX2xvZ3BzJTIwLSUyMHBlcl90b2tlbl9sb2dwcyUwQSklMjAlMjAlMjMlMjBTaGFwZSUzQSUyMChCKkclMkMlMjBzZXFfbGVuKSUyMHNlcV9sZW4lMjBpcyUyMHRoZSUyMGxlbmd0aCUyMG9mJTIwdGhlJTIwb3V0cHV0JTIwaS5lLiUyMHRoZSUyMG51bSUyMG9mJTIwZ2VuZXJhdGVkJTIwdG9rZW5zJTIwc28lMjBoZXJlJTIwZm9yJTIwc2ltcGxpY2l0eSUyMGxldCdzJTIwYXNzdW1lJTIwaXQlMjBpcyUyMDElMjAlMjMlMjAoOCUyQyUyMDEp",highlighted:`<span class="hljs-comment"># Compute probability ratio between new and old policies</span> | |
| ratio = torch.exp( | |
| new_per_token_logps - per_token_logps | |
| ) <span class="hljs-comment"># Shape: (B*G, seq_len) seq_len is the length of the output i.e. the num of generated tokens so here for simplicity let's assume it is 1 # (8, 1)</span>`,wrap:!1}}),it=new j({props:{code:"JTIzJTIwQ2xpcHBpbmclMjBGdW5jdGlvbiUwQWVwcyUyMCUzRCUyMHNlbGYuY2xpcHJhbmdlJTIwJTIwJTIzJTIwZS5nLiUyMDAuMiUwQXBnX2xvc3NlczElMjAlM0QlMjAtYWR2YW50YWdlcyUyMColMjByYXRpbyUyMCUyMCUyMyUyMFNoYXBlJTNBJTIwKEIqRyUyQyUyMHNlcV9sZW4pJTIwJTIwJTIzKDglMkMlMjAxKSUwQXBnX2xvc3NlczIlMjAlM0QlMjAtYWR2YW50YWdlcyUyMColMjB0b3JjaC5jbGFtcCglMEElMjAlMjAlMjAlMjByYXRpbyUyQyUyMDEuMCUyMC0lMjBlcHMlMkMlMjAxLjAlMjAlMkIlMjBlcHMlMEEpJTIwJTIwJTIzJTIwU2hhcGUlM0ElMjAoQipHJTJDJTIwc2VxX2xlbiklMjAlMjMoOCUyQyUyMDEpJTBBcGdfbG9zc19tYXglMjAlM0QlMjB0b3JjaC5tYXgocGdfbG9zc2VzMSUyQyUyMHBnX2xvc3NlczIpJTIwJTIwJTIzJTIwU2hhcGUlM0ElMjAoQipHJTJDJTIwc2VxX2xlbiklMjAlMjMoOCUyQyUyMDEpJTBBJTBBJTBBJTIzJTIwTm93JTIwQ29tYmluZSUyMHdpdGglMjBLTCUyMHBlbmFsdHklMjAlMjMlMjBTaGFwZSUzQSUyMChCKkclMkMlMjBzZXFfbGVuKSUyMCUyMyg4JTJDJTIwMSklMEFwZXJfdG9rZW5fbG9zcyUyMCUzRCUyMHBnX2xvc3NfbWF4JTIwJTJCJTIwc2VsZi5iZXRhJTIwKiUyMHBlcl90b2tlbl9rbA==",highlighted:`<span class="hljs-comment"># Clipping Function</span> | |
| eps = self.cliprange <span class="hljs-comment"># e.g. 0.2</span> | |
| pg_losses1 = -advantages * ratio <span class="hljs-comment"># Shape: (B*G, seq_len) #(8, 1)</span> | |
| pg_losses2 = -advantages * torch.clamp( | |
| ratio, <span class="hljs-number">1.0</span> - eps, <span class="hljs-number">1.0</span> + eps | |
| ) <span class="hljs-comment"># Shape: (B*G, seq_len) #(8, 1)</span> | |
| pg_loss_max = torch.<span class="hljs-built_in">max</span>(pg_losses1, pg_losses2) <span class="hljs-comment"># Shape: (B*G, seq_len) #(8, 1)</span> | |
| <span class="hljs-comment"># Now Combine with KL penalty # Shape: (B*G, seq_len) #(8, 1)</span> | |
| per_token_loss = pg_loss_max + self.beta * per_token_kl`,wrap:!1}}),ct=new j({props:{code:"JTIzJTIwU2hhcGUlM0ElMjAoQipHJTJDJTIwc2VxX2xlbiklMjAlMjMoOCUyQyUyMDEpJTBBcGVyX3Rva2VuX2tsJTIwJTNEJTIwRi5rbF9kaXYoJTBBJTIwJTIwJTIwJTIwRi5sb2dfc29mdG1heChuZXdfcGVyX3Rva2VuX2xvZ3BzJTJDJTIwZGltJTNELTEpJTJDJTBBJTIwJTIwJTIwJTIwRi5zb2Z0bWF4KHBlcl90b2tlbl9sb2dwcyUyQyUyMGRpbSUzRC0xKSUyQyUwQSUyMCUyMCUyMCUyMHJlZHVjdGlvbiUzRCUyMm5vbmUlMjIlMkMlMEEpLnN1bShkaW0lM0QtMSUyQyUyMGtlZXBkaW0lM0RUcnVlKQ==",highlighted:`<span class="hljs-comment"># Shape: (B*G, seq_len) #(8, 1)</span> | |
| per_token_kl = F.kl_div( | |
| F.log_softmax(new_per_token_logps, dim=-<span class="hljs-number">1</span>), | |
| F.softmax(per_token_logps, dim=-<span class="hljs-number">1</span>), | |
| reduction=<span class="hljs-string">"none"</span>, | |
| ).<span class="hljs-built_in">sum</span>(dim=-<span class="hljs-number">1</span>, keepdim=<span class="hljs-literal">True</span>)`,wrap:!1}}),ht=new T({props:{title:"အနှစ်ချုပ်နှင့် နောက်ထပ် အဆင့်များ",local:"အနစခပနင-နကထပ-အဆငမ",headingTag:"h2"}}),wt=new T({props:{title:"References",local:"references",headingTag:"h2"}}),ft=new T({props:{title:"ဝေါဟာရ ရှင်းလင်းချက် (Glossary)",local:"ဝဟရ-ရငလငခက-glossary",headingTag:"h2"}}),xt=new C0({props:{source:"https://github.com/huggingface/course/blob/main/chapters/my/chapter12/3a.mdx"}}),{c(){P=m("meta"),_e=e(),ce=m("p"),Ce=e(),v(os.$$.fragment),ze=e(),v(hs.$$.fragment),Je=e(),F=m("blockquote"),F.innerHTML=_o,Ue=e(),gs=m("p"),gs.textContent=Co,je=e(),ds=m("p"),ds.textContent=zo,ke=e(),ys=m("p"),ys.textContent=Jo,Le=e(),us=m("p"),us.textContent=Uo,$e=e(),vs=m("p"),vs.innerHTML=jo,Ie=e(),ws=m("p"),ws.textContent=ko,Ae=e(),v(Ms.$$.fragment),He=e(),fs=m("p"),fs.textContent=Lo,Ge=e(),xs=m("p"),xs.textContent=$o,Re=e(),v(bs.$$.fragment),Ee=e(),Ts=m("p"),Ts.textContent=Io,Be=e(),_=m("p"),pr=h("မေးခွန်း"),Qe=new d(!1),De=h(" တစ်ခုစီအတွက်၊ model က trained policy ကနေ outputs"),Ze=new d(!1),Pe=h(" ခု (group size) ကို ထုတ်ပေးပါလိမ့်မယ်- {"),Se=new d(!1),Ne=h(" }၊"),Xe=new d(!1),Ve=h(" ဖြစ်ပြီး"),Fe=new d(!1),qe=h(" တစ်ခုစီက model ကနေ ထုတ်ပေးတဲ့ completion တစ်ခုကို ကိုယ်စားပြုပါတယ်။"),We=e(),v(_s.$$.fragment),Oe=e(),Cs=m("p"),Cs.textContent=Ao,Ye=e(),G=m("p"),_t=m("strong"),_t.textContent=Ho,mr=e(),Ke=new d(!1),sl=h(" :"),al=new d(!1),tl=e(),R=m("p"),Ct=m("strong"),Ct.textContent=Go,ir=e(),nl=new d(!1),el=h(":"),ll=new d(!1),pl=e(),zs=m("p"),zs.textContent=Ro,ml=e(),v(Js.$$.fragment),il=e(),Us=m("p"),Us.textContent=Eo,rl=e(),v(js.$$.fragment),cl=e(),ks=m("p"),ks.textContent=Bo,ol=e(),E=m("p"),rr=h("မှန်ကန်မှု"),hl=new d(!1),gl=h(" အပေါ်အခြေခံပြီး ထုတ်ပေးထားတဲ့ response တစ်ခုစီကို RM score တစ်ခု သတ်မှတ်ပါ (ဥပမာ- မှန်ကန်တဲ့ response အတွက် 1၊ မှားယွင်းတဲ့ response အတွက် 0)၊ ပြီးရင်"),dl=new d(!1),yl=h(" တစ်ခုစီအတွက် အောက်ပါ Advantage value ကို တွက်ချက်ပါ။"),ul=e(),v(Ls.$$.fragment),vl=e(),$s=m("p"),cr=h(`GRPO ရဲ့ အဓိက အမြင်ကတော့ ကျွန်တော်တို့ဟာ အရည်အသွေးရဲ့ absolute measure တွေ မလိုအပ်ပါဘူး — အတူတူ group ထဲက outputs တွေကို နှိုင်းယှဉ်နိုင်ပါတယ်။ ဒါကို standardization ကို အသုံးပြုပြီး လုပ်ဆောင်ပါတယ်- | |
| `),wl=new d(!1),Ml=e(),v(Is.$$.fragment),fl=e(),As=m("p"),As.textContent=Qo,xl=e(),q=m("table"),zt=m("thead"),zt.innerHTML=Do,or=e(),I=m("tbody"),Hs=m("tr"),Jt=m("td"),Jt.textContent=Zo,hr=e(),oe=m("td"),bl=new d(!1),gr=e(),Gs=m("tr"),Ut=m("td"),Ut.textContent=Po,dr=e(),he=m("td"),Tl=new d(!1),yr=e(),Rs=m("tr"),jt=m("td"),jt.textContent=So,ur=e(),ge=m("td"),_l=new d(!1),vr=e(),Es=m("tr"),kt=m("td"),kt.textContent=No,wr=e(),de=m("td"),Cl=new d(!1),zl=e(),v(Bs.$$.fragment),Jl=e(),Qs=m("p"),Qs.textContent=Xo,Ul=e(),C=m("p"),Mr=h("ဒီ standardization (ဆိုလိုသည်မှာ"),jl=new d(!1),kl=h(" weighting) က model ကို response တစ်ခုစီရဲ့ ဆက်စပ်စွမ်းဆောင်ရည်ကို အကဲဖြတ်နိုင်စေပြီး၊ ပိုကောင်းတဲ့ (reward မြင့်မားတဲ့) responses တွေကို optimization process က ဦးတည်စေကာ၊ ပိုဆိုးတဲ့ဟာတွေကို ရှောင်ရှားစေပါတယ်။ ဥပမာအားဖြင့်၊"),Ll=new d(!1),$l=h(" ဖြစ်ရင်၊"),Il=new d(!1),Al=h(" ဟာ သူ့ group အတွင်းက ပျမ်းမျှအဆင့်ထက် ပိုကောင်းတဲ့ response ဖြစ်ပါတယ်၊ ပြီးတော့"),Hl=new d(!1),Gl=h(" ဖြစ်ရင်၊"),Rl=new d(!1),El=h(" ဟာ ပျမ်းမျှထက် နိမ့်တဲ့ အရည်အသွေး (ဆိုလိုသည်မှာ အရည်အသွေးညံ့/စွမ်းဆောင်ရည်ညံ့) ရှိပါတယ်။"),Bl=e(),W=m("p"),fr=h("အထက်ပါ ဥပမာအတွက်၊ အကယ်၍"),Ql=new d(!1),Dl=h(" ဖြစ်ပါက optimization steps များအတွင်း ၎င်း၏ generation probability ကို တိုးမြှင့်ပါလိမ့်မည်။"),Zl=e(),Ds=m("p"),Ds.textContent=Vo,Pl=e(),v(Zs.$$.fragment),Sl=e(),Ps=m("p"),Ps.textContent=Fo,Nl=e(),Ss=m("p"),xr=h(`policy update အတွက် target function က… | |
| `),Xl=new d(!1),Vl=e(),Ns=m("p"),Ns.textContent=qo,Fl=e(),v(Xs.$$.fragment),ql=e(),Vs=m("p"),Vs.textContent=Wo,Wl=e(),v(Fs.$$.fragment),Ol=e(),qs=m("p"),br=h(`Probability ratio ကို အောက်ပါအတိုင်း သတ်မှတ်ပါတယ်။ | |
| `),Yl=new d(!1),Kl=e(),Ws=m("p"),Ws.textContent=Oo,sp=e(),v(Os.$$.fragment),ap=e(),O=m("ul"),S=m("li"),Tr=h("အကယ်၍"),tp=new d(!1),np=h(" ဖြစ်ရင်၊ new model က response"),ep=new d(!1),lp=h(" ကို old model ထက် ပိုမြင့်တဲ့ probability သတ်မှတ်ပါတယ်။"),_r=e(),N=m("li"),Cr=h("အကယ်၍"),pp=new d(!1),mp=h(" ဖြစ်ရင်၊ new model က"),ip=new d(!1),rp=h(" ကို ပိုနိမ့်တဲ့ probability သတ်မှတ်ပါတယ်။"),cp=e(),Ys=m("p"),Ys.textContent=Yo,op=e(),v(Ks.$$.fragment),hp=e(),sa=m("p"),zr=h(`Clipping function ကို အောက်ပါအတိုင်း သတ်မှတ်ပါတယ်။ | |
| `),gp=new d(!1),dp=e(),Y=m("p"),Jr=h("အထက်မှာ ဆွေးနွေးခဲ့တဲ့ ratio ကို"),yp=new d(!1),up=h(" အတွင်းမှာ ကန့်သတ်ခြင်းဖြင့် ကြီးမားတဲ့ ပြောင်းလဲမှုတွေ ဒါမှမဟုတ် မမှန်ကန်တဲ့ updates တွေ ဖြစ်ပေါ်တာကို ရှောင်ရှားပြီး old policy ကနေ အဝေးကြီး မသွေဖည်အောင် ထိန်းချုပ်ပါတယ်။ တနည်းအားဖြင့်၊ ဒါက probability ratio ဘယ်လောက်အထိ တိုးမြှင့်နိုင်လဲဆိုတာကို ကန့်သတ်ပြီး new model ကို old model ကနေ အဝေးကြီး မတွန်းပို့မိစေဖို့ တည်ငြိမ်မှုကို ထိန်းသိမ်းရာမှာ ကူညီပေးပါတယ်။"),vp=e(),v(aa.$$.fragment),wp=e(),ta=m("p"),ta.textContent=Ko,Mp=e(),K=m("ul"),ss=m("li"),Lt=m("strong"),Lt.textContent=sh,Ur=h(`: အကယ်၍ new policy မှာ သီးခြား response တစ်ခုအတွက် probability 0.9 ရှိပြီး old policy မှာ probability 0.5 ရှိတယ်ဆိုရင်၊ ဒီ response က new policy ကနေ ပိုမြင့်တဲ့ probability ရရှိဖို့ အားဖြည့်ပေးခံရတာကို ဆိုလိုပါတယ်။ ဒါပေမယ့် clipping ဆိုတဲ့ ထိန်းချုပ်ထားတဲ့ ကန့်သတ်ချက် (upper bound limit 1.2) အတွင်းမှာပဲ ဖြစ်ပါတယ် | |
| -`),fp=new d(!1),xp=h(" (upper bound limit 1.2)"),jr=e(),as=m("li"),$t=m("strong"),$t.textContent=ah,kr=h(`: အကယ်၍ new policy က response တစ်ခုကို မထောက်ခံဘူးဆိုရင် (lower probability ဥပမာ- 0.2)၊ ဆိုလိုတာက response က အကျိုးမပြုဘူးဆိုရင် တိုးမြှင့်မှုက မမှန်ကန်နိုင်ပြီး model ကို ပြစ်ဒဏ်ပေးပါလိမ့်မယ်။ | |
| -`),bp=new d(!1),Tp=h(" (lower bound limit 0.8)"),_p=e(),v(na.$$.fragment),Cp=e(),k=m("ul"),It=m("li"),It.innerHTML=th,Lr=e(),ea=m("li"),$r=h("အကယ်၍ old model က မြင့်မားတဲ့ probability နဲ့ response တစ်ခုကို နှစ်သက်ပြီးသားဆိုရင်၊ new model က အဲဒါကို ဆက်လက်အားဖြည့်နိုင်ပါတယ် "),X=m("strong"),Ir=h("ဒါပေမယ့် ထိန်းချုပ်ထားတဲ့ ကန့်သတ်ချက်"),zp=new d(!1),Jp=h(" (ဥပမာ-"),Up=new d(!1),jp=h(" အတွင်းမှာသာ ဖြစ်ပါတယ်"),Ar=h("။"),Hr=e(),At=m("li"),At.innerHTML=nh,Gr=e(),la=m("li"),Rr=h("ဒါကြောင့်၊ concept အားဖြင့်၊ probability ratio ကို ပေါင်းစပ်ခြင်းဖြင့်၊ objective function က policy အပေါ် updates တွေကို advantage"),kp=new d(!1),Lp=h(" နဲ့ အချိုးကျဖြစ်စေပြီး ကြီးမားတဲ့ ပြောင်းလဲမှုတွေကို ကာကွယ်ပေးပါတယ်။"),$p=e(),pa=m("p"),pa.textContent=eh,Ip=e(),v(ma.$$.fragment),Ap=e(),ia=m("p"),Er=h(`KL divergence term က | |
| `),Hp=new d(!1),Gp=e(),z=m("p"),Br=h("KL divergence term မှာ၊"),Rp=new d(!1),Ep=h(" က အခြေခံအားဖြင့် pre-update model ရဲ့ output ဖြစ်တဲ့ "),Ht=m("code"),Ht.textContent=lh,Qr=h(" ဖြစ်ပြီး"),Bp=new d(!1),Qp=h(" က new model ရဲ့ output ဖြစ်တဲ့ "),Gt=m("code"),Gt.textContent=ph,Dr=h(" ဖြစ်ပါတယ်။ သီအိုရီအရ၊ KL divergence ကို optimization လုပ်နေစဉ် model က ၎င်းရဲ့ မူရင်း behavior ကနေ အဝေးကြီး မသွေဖည်အောင် ကာကွယ်ဖို့ minimize လုပ်ပါတယ်။ ဒါက reward signal ပေါ် အခြေခံပြီး စွမ်းဆောင်ရည်ကို မြှင့်တင်ခြင်းနဲ့ coherence ကို ထိန်းသိမ်းခြင်းကြား ဟန်ချက်ညီစေဖို့ ကူညီပေးပါတယ်။ ဒီအခြေအနေမှာ၊ KL divergence ကို minimize လုပ်ခြင်းက model က အဓိပ္ပာယ်မရှိတဲ့ စာသားတွေ ဒါမှမဟုတ်၊ သင်္ချာဆိုင်ရာ reasoning မှာဆိုရင်၊ လုံးဝမမှန်ကန်တဲ့ အဖြေတွေကို ထုတ်ပေးနိုင်ခြေကို လျှော့ချပေးပါတယ်။"),Dp=e(),v(ra.$$.fragment),Zp=e(),ca=m("ul"),ca.innerHTML=mh,Pp=e(),v(oa.$$.fragment),Sp=e(),ha=m("p"),ha.textContent=ih,Np=e(),ts=m("p"),Zr=h("KL distance ကို အောက်ပါအတိုင်း သတ်မှတ်ကြောင်း ပြန်လည်မှတ်မိပါ။"),Xp=new d(!1),Vp=h(` | |
| RLHF မှာ၊ စိတ်ဝင်စားတဲ့ distribution နှစ်ခုက မကြာခဏဆိုသလို new model version ရဲ့ distribution ဖြစ်တဲ့ P(x) နဲ့ reference policy ရဲ့ distribution ဖြစ်တဲ့ Q(x) တို့ ဖြစ်ပါတယ်။`),Fp=e(),v(ga.$$.fragment),qp=e(),ns=m("p"),Pr=h("coefficient"),Wp=new d(!1),Op=h(" က KL divergence ကန့်သတ်ချက်ကို ကျွန်တော်တို့ ဘယ်လောက်ပြင်းပြင်းထန်ထန် သတ်မှတ်မလဲဆိုတာ ထိန်းချုပ်ပါတယ်။"),Yp=e(),B=m("ul"),Rt=m("li"),Rt.innerHTML=rh,Sr=e(),Et=m("li"),Et.innerHTML=ch,Nr=e(),Q=m("li"),Bt=m("strong"),Bt.textContent=oh,Xr=e(),es=m("a"),es.textContent=hh,Vr=h(" paper က ဒီ"),Kp=new d(!1),sm=h(" လို့ သတ်မှတ်ထားပါတယ်။"),am=e(),da=m("p"),da.textContent=gh,tm=e(),v(ya.$$.fragment),nm=e(),ua=m("p"),ua.textContent=dh,em=e(),v(va.$$.fragment),lm=e(),Qt=m("p"),pm=new d(!1),mm=e(),v(wa.$$.fragment),im=e(),Ma=m("p"),Ma.textContent=yh,rm=e(),U=m("p"),Fr=h("responses"),cm=new d(!1),om=h(" ခုကို ထုတ်ပေးပါ၊"),hm=new d(!1),gm=h(" ခုက မှန်ကန်တဲ့အဖြေ (\\( 14, \\text{reward=} 1 \\)) ဖြစ်ပြီး"),dm=new d(!1),ym=h(` ခုက မမှန်ကန်ပါ (\\( \\text{reward= 0)} \\))၊ ဒါကြောင့်- | |
| `),um=new d(!1),vm=e(),v(fa.$$.fragment),wm=e(),xa=m("p"),xa.textContent=uh,Mm=e(),ls=m("table"),Dt=m("thead"),Dt.innerHTML=vh,qr=e(),A=m("tbody"),ba=m("tr"),Zt=m("td"),Zt.textContent=wh,Wr=e(),ye=m("td"),fm=new d(!1),Or=e(),Ta=m("tr"),Pt=m("td"),Pt.textContent=Mh,Yr=e(),ue=m("td"),xm=new d(!1),Kr=e(),_a=m("tr"),St=m("td"),St.textContent=fh,sc=e(),ve=m("td"),bm=new d(!1),ac=e(),Ca=m("tr"),Nt=m("td"),Nt.textContent=xh,tc=e(),we=m("td"),Tm=new d(!1),_m=e(),v(za.$$.fragment),Cm=e(),Ja=m("p"),Ja.textContent=bh,zm=e(),ps=m("ul"),J=m("li"),nc=h("correct output"),Jm=new d(!1),Um=h(" အတွက် old policy (\\( \\pi"),Xt=m("em"),Xt.textContent=Th,ec=h("{old}} \\)) ရဲ့ probability က"),jm=new d(!1),km=h(" ဖြစ်ပြီး new policy က"),Lm=new d(!1),$m=h(" အထိ တိုးမြှင့်လိုက်တယ်လို့ ယူဆပါက-"),Im=new d(!1),lc=e(),Ua=m("li"),pc=h("ထို့နောက် target function ကို ပြန်လည်ချိန်ညှိတဲ့အခါ၊ model က correct output ကို ထုတ်ပေးခြင်းကို အားဖြည့်ဖို့ ကြိုးစားပြီး"),Am=new d(!1),Hm=h(" က reference policy ကနေ သွေဖည်မှုကို ကန့်သတ်ပါတယ်။"),Gm=e(),ja=m("p"),ja.textContent=_h,Rm=e(),v(ka.$$.fragment),Em=e(),La=m("p"),La.textContent=Ch,Bm=e(),v($a.$$.fragment),Qm=e(),Ia=m("p"),Ia.textContent=zh,Dm=e(),v(Aa.$$.fragment),Zm=e(),Ha=m("p"),Ha.textContent=Jh,Pm=e(),v(Ga.$$.fragment),Sm=e(),v(Ra.$$.fragment),Nm=e(),Ea=m("p"),Ea.textContent=Uh,Xm=e(),Ba=m("p"),Ba.innerHTML=jh,Vm=e(),Qa=m("ul"),Qa.innerHTML=kh,Fm=e(),Da=m("p"),Da.textContent=Lh,qm=e(),v(Za.$$.fragment),Wm=e(),Pa=m("p"),Pa.textContent=$h,Om=e(),v(Sa.$$.fragment),Ym=e(),Na=m("p"),Na.textContent=Ih,Km=e(),v(Xa.$$.fragment),si=e(),Va=m("p"),Va.textContent=Ah,ai=e(),v(Fa.$$.fragment),ti=e(),qa=m("p"),qa.textContent=Hh,ni=e(),v(Wa.$$.fragment),ei=e(),Oa=m("p"),Oa.textContent=Gh,li=e(),v(Ya.$$.fragment),pi=e(),Ka=m("p"),Ka.innerHTML=Rh,mi=e(),v(st.$$.fragment),ii=e(),at=m("p"),at.textContent=Eh,ri=e(),v(tt.$$.fragment),ci=e(),nt=m("p"),nt.textContent=Bh,oi=e(),v(et.$$.fragment),hi=e(),lt=m("p"),lt.textContent=Qh,gi=e(),v(pt.$$.fragment),di=e(),mt=m("p"),mt.innerHTML=Dh,yi=e(),v(it.$$.fragment),ui=e(),rt=m("p"),rt.innerHTML=Zh,vi=e(),v(ct.$$.fragment),wi=e(),ot=m("p"),ot.innerHTML=Ph,Mi=e(),v(ht.$$.fragment),fi=e(),gt=m("p"),gt.textContent=Sh,xi=e(),dt=m("p"),dt.textContent=Nh,bi=e(),yt=m("p"),yt.textContent=Xh,Ti=e(),ut=m("p"),ut.textContent=Vh,_i=e(),vt=m("p"),vt.textContent=Fh,Ci=e(),v(wt.$$.fragment),zi=e(),Mt=m("ol"),Mt.innerHTML=qh,Ji=e(),Ui=m("hr"),ji=e(),v(ft.$$.fragment),ki=e(),c=m("ul"),Vt=m("li"),Vt.innerHTML=Wh,mc=e(),Ft=m("li"),Ft.innerHTML=Oh,ic=e(),qt=m("li"),qt.innerHTML=Yh,rc=e(),Wt=m("li"),Wt.innerHTML=Kh,cc=e(),Ot=m("li"),Ot.innerHTML=sg,oc=e(),Yt=m("li"),Yt.innerHTML=ag,hc=e(),Kt=m("li"),Kt.innerHTML=tg,gc=e(),sn=m("li"),sn.innerHTML=ng,dc=e(),an=m("li"),an.innerHTML=eg,yc=e(),tn=m("li"),tn.innerHTML=lg,uc=e(),nn=m("li"),nn.innerHTML=pg,vc=e(),en=m("li"),en.innerHTML=mg,wc=e(),ln=m("li"),ln.innerHTML=ig,Mc=e(),pn=m("li"),pn.innerHTML=rg,fc=e(),mn=m("li"),mn.innerHTML=cg,xc=e(),rn=m("li"),rn.innerHTML=og,bc=e(),cn=m("li"),cn.innerHTML=hg,Tc=e(),on=m("li"),on.innerHTML=gg,_c=e(),hn=m("li"),hn.innerHTML=dg,Cc=e(),D=m("li"),gn=m("strong"),gn.innerHTML=yg,zc=h(": New policy အောက်တွင် response"),Li=new d(!1),$i=h(" ကို query"),Ii=new d(!1),Ai=h(" အတွက် ထုတ်ပေးနိုင်ခြေ probability။"),Jc=e(),ms=m("li"),dn=m("strong"),dn.innerHTML=ug,Uc=h(": Probability ratio ကို သတ်မှတ်ထားသော အတိုင်းအတာ"),Hi=new d(!1),Gi=h(" အတွင်း ကန့်သတ်သော clipping function။"),jc=e(),yn=m("li"),yn.innerHTML=vg,kc=e(),un=m("li"),un.innerHTML=wg,Lc=e(),vn=m("li"),vn.innerHTML=Mg,$c=e(),wn=m("li"),wn.innerHTML=fg,Ic=e(),Mn=m("li"),Mn.innerHTML=xg,Ac=e(),fn=m("li"),fn.innerHTML=bg,Hc=e(),xn=m("li"),xn.innerHTML=Tg,Gc=e(),bn=m("li"),bn.innerHTML=_g,Rc=e(),Tn=m("li"),Tn.innerHTML=Cg,Ec=e(),_n=m("li"),_n.innerHTML=zg,Bc=e(),Cn=m("li"),Cn.innerHTML=Jg,Qc=e(),zn=m("li"),zn.innerHTML=Ug,Dc=e(),Jn=m("li"),Jn.innerHTML=jg,Zc=e(),Un=m("li"),Un.innerHTML=kg,Pc=e(),jn=m("li"),jn.innerHTML=Lg,Sc=e(),kn=m("li"),kn.innerHTML=$g,Nc=e(),Ln=m("li"),Ln.innerHTML=Ig,Xc=e(),$n=m("li"),$n.innerHTML=Ag,Vc=e(),In=m("li"),In.innerHTML=Hg,Fc=e(),An=m("li"),An.innerHTML=Gg,qc=e(),Hn=m("li"),Hn.innerHTML=Rg,Wc=e(),Gn=m("li"),Gn.innerHTML=Eg,Oc=e(),Rn=m("li"),Rn.innerHTML=Bg,Yc=e(),En=m("li"),En.innerHTML=Qg,Kc=e(),Bn=m("li"),Bn.innerHTML=Dg,so=e(),Qn=m("li"),Qn.innerHTML=Zg,ao=e(),Dn=m("li"),Dn.innerHTML=Pg,to=e(),Zn=m("li"),Zn.innerHTML=Sg,no=e(),Pn=m("li"),Pn.innerHTML=Ng,eo=e(),Sn=m("li"),Sn.innerHTML=Xg,lo=e(),Nn=m("li"),Nn.innerHTML=Vg,po=e(),Xn=m("li"),Xn.innerHTML=Fg,mo=e(),Vn=m("li"),Vn.innerHTML=qg,io=e(),Fn=m("li"),Fn.innerHTML=Wg,ro=e(),qn=m("li"),qn.innerHTML=Og,co=e(),Wn=m("li"),Wn.innerHTML=Yg,oo=e(),On=m("li"),On.innerHTML=Kg,ho=e(),Yn=m("li"),Yn.innerHTML=sd,go=e(),Kn=m("li"),Kn.innerHTML=ad,yo=e(),se=m("li"),se.innerHTML=td,uo=e(),ae=m("li"),ae.innerHTML=nd,vo=e(),te=m("li"),te.innerHTML=ed,Ri=e(),v(xt.$$.fragment),Ei=e(),Me=m("p"),this.h()},l(s){const n=T0("svelte-u9bgzb",document.head);P=i(n,"META",{name:!0,content:!0}),n.forEach(a),_e=l(s),ce=i(s,"P",{}),u(ce).forEach(a),Ce=l(s),w(os.$$.fragment,s),ze=l(s),w(hs.$$.fragment,s),Je=l(s),F=i(s,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),r(F)!=="svelte-1hooit0"&&(F.innerHTML=_o),Ue=l(s),gs=i(s,"P",{"data-svelte-h":!0}),r(gs)!=="svelte-j399di"&&(gs.textContent=Co),je=l(s),ds=i(s,"P",{"data-svelte-h":!0}),r(ds)!=="svelte-1rxazip"&&(ds.textContent=zo),ke=l(s),ys=i(s,"P",{"data-svelte-h":!0}),r(ys)!=="svelte-1a8eej3"&&(ys.textContent=Jo),Le=l(s),us=i(s,"P",{"data-svelte-h":!0}),r(us)!=="svelte-nm74y5"&&(us.textContent=Uo),$e=l(s),vs=i(s,"P",{"data-svelte-h":!0}),r(vs)!=="svelte-1fuvyr0"&&(vs.innerHTML=jo),Ie=l(s),ws=i(s,"P",{"data-svelte-h":!0}),r(ws)!=="svelte-ajoivp"&&(ws.textContent=ko),Ae=l(s),w(Ms.$$.fragment,s),He=l(s),fs=i(s,"P",{"data-svelte-h":!0}),r(fs)!=="svelte-wauzre"&&(fs.textContent=Lo),Ge=l(s),xs=i(s,"P",{"data-svelte-h":!0}),r(xs)!=="svelte-r7lh5w"&&(xs.textContent=$o),Re=l(s),w(bs.$$.fragment,s),Ee=l(s),Ts=i(s,"P",{"data-svelte-h":!0}),r(Ts)!=="svelte-13d37jh"&&(Ts.textContent=Io),Be=l(s),_=i(s,"P",{});var L=u(_);pr=g(L,"မေးခွန်း"),Qe=y(L,!1),De=g(L," တစ်ခုစီအတွက်၊ model က trained policy ကနေ outputs"),Ze=y(L,!1),Pe=g(L," ခု (group size) ကို ထုတ်ပေးပါလိမ့်မယ်- {"),Se=y(L,!1),Ne=g(L," }၊"),Xe=y(L,!1),Ve=g(L," ဖြစ်ပြီး"),Fe=y(L,!1),qe=g(L," တစ်ခုစီက model ကနေ ထုတ်ပေးတဲ့ completion တစ်ခုကို ကိုယ်စားပြုပါတယ်။"),L.forEach(a),We=l(s),w(_s.$$.fragment,s),Oe=l(s),Cs=i(s,"P",{"data-svelte-h":!0}),r(Cs)!=="svelte-197cph9"&&(Cs.textContent=Ao),Ye=l(s),G=i(s,"P",{});var ne=u(G);_t=i(ne,"STRONG",{"data-svelte-h":!0}),r(_t)!=="svelte-3hsjwm"&&(_t.textContent=Ho),mr=l(ne),Ke=y(ne,!1),sl=g(ne," :"),al=y(ne,!1),ne.forEach(a),tl=l(s),R=i(s,"P",{});var ee=u(R);Ct=i(ee,"STRONG",{"data-svelte-h":!0}),r(Ct)!=="svelte-svhppw"&&(Ct.textContent=Go),ir=l(ee),nl=y(ee,!1),el=g(ee,":"),ll=y(ee,!1),ee.forEach(a),pl=l(s),zs=i(s,"P",{"data-svelte-h":!0}),r(zs)!=="svelte-1aoszpa"&&(zs.textContent=Ro),ml=l(s),w(Js.$$.fragment,s),il=l(s),Us=i(s,"P",{"data-svelte-h":!0}),r(Us)!=="svelte-1c7sest"&&(Us.textContent=Eo),rl=l(s),w(js.$$.fragment,s),cl=l(s),ks=i(s,"P",{"data-svelte-h":!0}),r(ks)!=="svelte-17wgqng"&&(ks.textContent=Bo),ol=l(s),E=i(s,"P",{});var le=u(E);rr=g(le,"မှန်ကန်မှု"),hl=y(le,!1),gl=g(le," အပေါ်အခြေခံပြီး ထုတ်ပေးထားတဲ့ response တစ်ခုစီကို RM score တစ်ခု သတ်မှတ်ပါ (ဥပမာ- မှန်ကန်တဲ့ response အတွက် 1၊ မှားယွင်းတဲ့ response အတွက် 0)၊ ပြီးရင်"),dl=y(le,!1),yl=g(le," တစ်ခုစီအတွက် အောက်ပါ Advantage value ကို တွက်ချက်ပါ။"),le.forEach(a),ul=l(s),w(Ls.$$.fragment,s),vl=l(s),$s=i(s,"P",{});var wo=u($s);cr=g(wo,`GRPO ရဲ့ အဓိက အမြင်ကတော့ ကျွန်တော်တို့ဟာ အရည်အသွေးရဲ့ absolute measure တွေ မလိုအပ်ပါဘူး — အတူတူ group ထဲက outputs တွေကို နှိုင်းယှဉ်နိုင်ပါတယ်။ ဒါကို standardization ကို အသုံးပြုပြီး လုပ်ဆောင်ပါတယ်- | |
| `),wl=y(wo,!1),wo.forEach(a),Ml=l(s),w(Is.$$.fragment,s),fl=l(s),As=i(s,"P",{"data-svelte-h":!0}),r(As)!=="svelte-nrr76n"&&(As.textContent=Qo),xl=l(s),q=i(s,"TABLE",{});var Qi=u(q);zt=i(Qi,"THEAD",{"data-svelte-h":!0}),r(zt)!=="svelte-1egpqgs"&&(zt.innerHTML=Do),or=l(Qi),I=i(Qi,"TBODY",{});var is=u(I);Hs=i(is,"TR",{});var Di=u(Hs);Jt=i(Di,"TD",{"data-svelte-h":!0}),r(Jt)!=="svelte-175bp0i"&&(Jt.textContent=Zo),hr=l(Di),oe=i(Di,"TD",{});var ld=u(oe);bl=y(ld,!1),ld.forEach(a),Di.forEach(a),gr=l(is),Gs=i(is,"TR",{});var Zi=u(Gs);Ut=i(Zi,"TD",{"data-svelte-h":!0}),r(Ut)!=="svelte-ib0fgo"&&(Ut.textContent=Po),dr=l(Zi),he=i(Zi,"TD",{});var pd=u(he);Tl=y(pd,!1),pd.forEach(a),Zi.forEach(a),yr=l(is),Rs=i(is,"TR",{});var Pi=u(Rs);jt=i(Pi,"TD",{"data-svelte-h":!0}),r(jt)!=="svelte-1dv54pk"&&(jt.textContent=So),ur=l(Pi),ge=i(Pi,"TD",{});var md=u(ge);_l=y(md,!1),md.forEach(a),Pi.forEach(a),vr=l(is),Es=i(is,"TR",{});var Si=u(Es);kt=i(Si,"TD",{"data-svelte-h":!0}),r(kt)!=="svelte-1tmz1sl"&&(kt.textContent=No),wr=l(Si),de=i(Si,"TD",{});var id=u(de);Cl=y(id,!1),id.forEach(a),Si.forEach(a),is.forEach(a),Qi.forEach(a),zl=l(s),w(Bs.$$.fragment,s),Jl=l(s),Qs=i(s,"P",{"data-svelte-h":!0}),r(Qs)!=="svelte-12fov90"&&(Qs.textContent=Xo),Ul=l(s),C=i(s,"P",{});var $=u(C);Mr=g($,"ဒီ standardization (ဆိုလိုသည်မှာ"),jl=y($,!1),kl=g($," weighting) က model ကို response တစ်ခုစီရဲ့ ဆက်စပ်စွမ်းဆောင်ရည်ကို အကဲဖြတ်နိုင်စေပြီး၊ ပိုကောင်းတဲ့ (reward မြင့်မားတဲ့) responses တွေကို optimization process က ဦးတည်စေကာ၊ ပိုဆိုးတဲ့ဟာတွေကို ရှောင်ရှားစေပါတယ်။ ဥပမာအားဖြင့်၊"),Ll=y($,!1),$l=g($," ဖြစ်ရင်၊"),Il=y($,!1),Al=g($," ဟာ သူ့ group အတွင်းက ပျမ်းမျှအဆင့်ထက် ပိုကောင်းတဲ့ response ဖြစ်ပါတယ်၊ ပြီးတော့"),Hl=y($,!1),Gl=g($," ဖြစ်ရင်၊"),Rl=y($,!1),El=g($," ဟာ ပျမ်းမျှထက် နိမ့်တဲ့ အရည်အသွေး (ဆိုလိုသည်မှာ အရည်အသွေးညံ့/စွမ်းဆောင်ရည်ညံ့) ရှိပါတယ်။"),$.forEach(a),Bl=l(s),W=i(s,"P",{});var Ni=u(W);fr=g(Ni,"အထက်ပါ ဥပမာအတွက်၊ အကယ်၍"),Ql=y(Ni,!1),Dl=g(Ni," ဖြစ်ပါက optimization steps များအတွင်း ၎င်း၏ generation probability ကို တိုးမြှင့်ပါလိမ့်မည်။"),Ni.forEach(a),Zl=l(s),Ds=i(s,"P",{"data-svelte-h":!0}),r(Ds)!=="svelte-g0afwt"&&(Ds.textContent=Vo),Pl=l(s),w(Zs.$$.fragment,s),Sl=l(s),Ps=i(s,"P",{"data-svelte-h":!0}),r(Ps)!=="svelte-sojz5p"&&(Ps.textContent=Fo),Nl=l(s),Ss=i(s,"P",{});var Mo=u(Ss);xr=g(Mo,`policy update အတွက် target function က… | |
| `),Xl=y(Mo,!1),Mo.forEach(a),Vl=l(s),Ns=i(s,"P",{"data-svelte-h":!0}),r(Ns)!=="svelte-14zuoyj"&&(Ns.textContent=qo),Fl=l(s),w(Xs.$$.fragment,s),ql=l(s),Vs=i(s,"P",{"data-svelte-h":!0}),r(Vs)!=="svelte-9p91qx"&&(Vs.textContent=Wo),Wl=l(s),w(Fs.$$.fragment,s),Ol=l(s),qs=i(s,"P",{});var fo=u(qs);br=g(fo,`Probability ratio ကို အောက်ပါအတိုင်း သတ်မှတ်ပါတယ်။ | |
| `),Yl=y(fo,!1),fo.forEach(a),Kl=l(s),Ws=i(s,"P",{"data-svelte-h":!0}),r(Ws)!=="svelte-bb8by4"&&(Ws.textContent=Oo),sp=l(s),w(Os.$$.fragment,s),ap=l(s),O=i(s,"UL",{});var Xi=u(O);S=i(Xi,"LI",{});var pe=u(S);Tr=g(pe,"အကယ်၍"),tp=y(pe,!1),np=g(pe," ဖြစ်ရင်၊ new model က response"),ep=y(pe,!1),lp=g(pe," ကို old model ထက် ပိုမြင့်တဲ့ probability သတ်မှတ်ပါတယ်။"),pe.forEach(a),_r=l(Xi),N=i(Xi,"LI",{});var me=u(N);Cr=g(me,"အကယ်၍"),pp=y(me,!1),mp=g(me," ဖြစ်ရင်၊ new model က"),ip=y(me,!1),rp=g(me," ကို ပိုနိမ့်တဲ့ probability သတ်မှတ်ပါတယ်။"),me.forEach(a),Xi.forEach(a),cp=l(s),Ys=i(s,"P",{"data-svelte-h":!0}),r(Ys)!=="svelte-c33d8h"&&(Ys.textContent=Yo),op=l(s),w(Ks.$$.fragment,s),hp=l(s),sa=i(s,"P",{});var xo=u(sa);zr=g(xo,`Clipping function ကို အောက်ပါအတိုင်း သတ်မှတ်ပါတယ်။ | |
| `),gp=y(xo,!1),xo.forEach(a),dp=l(s),Y=i(s,"P",{});var Vi=u(Y);Jr=g(Vi,"အထက်မှာ ဆွေးနွေးခဲ့တဲ့ ratio ကို"),yp=y(Vi,!1),up=g(Vi," အတွင်းမှာ ကန့်သတ်ခြင်းဖြင့် ကြီးမားတဲ့ ပြောင်းလဲမှုတွေ ဒါမှမဟုတ် မမှန်ကန်တဲ့ updates တွေ ဖြစ်ပေါ်တာကို ရှောင်ရှားပြီး old policy ကနေ အဝေးကြီး မသွေဖည်အောင် ထိန်းချုပ်ပါတယ်။ တနည်းအားဖြင့်၊ ဒါက probability ratio ဘယ်လောက်အထိ တိုးမြှင့်နိုင်လဲဆိုတာကို ကန့်သတ်ပြီး new model ကို old model ကနေ အဝေးကြီး မတွန်းပို့မိစေဖို့ တည်ငြိမ်မှုကို ထိန်းသိမ်းရာမှာ ကူညီပေးပါတယ်။"),Vi.forEach(a),vp=l(s),w(aa.$$.fragment,s),wp=l(s),ta=i(s,"P",{"data-svelte-h":!0}),r(ta)!=="svelte-giwd3y"&&(ta.textContent=Ko),Mp=l(s),K=i(s,"UL",{});var Fi=u(K);ss=i(Fi,"LI",{});var fe=u(ss);Lt=i(fe,"STRONG",{"data-svelte-h":!0}),r(Lt)!=="svelte-jiugjv"&&(Lt.textContent=sh),Ur=g(fe,`: အကယ်၍ new policy မှာ သီးခြား response တစ်ခုအတွက် probability 0.9 ရှိပြီး old policy မှာ probability 0.5 ရှိတယ်ဆိုရင်၊ ဒီ response က new policy ကနေ ပိုမြင့်တဲ့ probability ရရှိဖို့ အားဖြည့်ပေးခံရတာကို ဆိုလိုပါတယ်။ ဒါပေမယ့် clipping ဆိုတဲ့ ထိန်းချုပ်ထားတဲ့ ကန့်သတ်ချက် (upper bound limit 1.2) အတွင်းမှာပဲ ဖြစ်ပါတယ် | |
| -`),fp=y(fe,!1),xp=g(fe," (upper bound limit 1.2)"),fe.forEach(a),jr=l(Fi),as=i(Fi,"LI",{});var xe=u(as);$t=i(xe,"STRONG",{"data-svelte-h":!0}),r($t)!=="svelte-v52hoy"&&($t.textContent=ah),kr=g(xe,`: အကယ်၍ new policy က response တစ်ခုကို မထောက်ခံဘူးဆိုရင် (lower probability ဥပမာ- 0.2)၊ ဆိုလိုတာက response က အကျိုးမပြုဘူးဆိုရင် တိုးမြှင့်မှုက မမှန်ကန်နိုင်ပြီး model ကို ပြစ်ဒဏ်ပေးပါလိမ့်မယ်။ | |
| -`),bp=y(xe,!1),Tp=g(xe," (lower bound limit 0.8)"),xe.forEach(a),Fi.forEach(a),_p=l(s),w(na.$$.fragment,s),Cp=l(s),k=i(s,"UL",{});var rs=u(k);It=i(rs,"LI",{"data-svelte-h":!0}),r(It)!=="svelte-100q17p"&&(It.innerHTML=th),Lr=l(rs),ea=i(rs,"LI",{});var qi=u(ea);$r=g(qi,"အကယ်၍ old model က မြင့်မားတဲ့ probability နဲ့ response တစ်ခုကို နှစ်သက်ပြီးသားဆိုရင်၊ new model က အဲဒါကို ဆက်လက်အားဖြည့်နိုင်ပါတယ် "),X=i(qi,"STRONG",{});var ie=u(X);Ir=g(ie,"ဒါပေမယ့် ထိန်းချုပ်ထားတဲ့ ကန့်သတ်ချက်"),zp=y(ie,!1),Jp=g(ie," (ဥပမာ-"),Up=y(ie,!1),jp=g(ie," အတွင်းမှာသာ ဖြစ်ပါတယ်"),ie.forEach(a),Ar=g(qi,"။"),qi.forEach(a),Hr=l(rs),At=i(rs,"LI",{"data-svelte-h":!0}),r(At)!=="svelte-rt9giv"&&(At.innerHTML=nh),Gr=l(rs),la=i(rs,"LI",{});var Wi=u(la);Rr=g(Wi,"ဒါကြောင့်၊ concept အားဖြင့်၊ probability ratio ကို ပေါင်းစပ်ခြင်းဖြင့်၊ objective function က policy အပေါ် updates တွေကို advantage"),kp=y(Wi,!1),Lp=g(Wi," နဲ့ အချိုးကျဖြစ်စေပြီး ကြီးမားတဲ့ ပြောင်းလဲမှုတွေကို ကာကွယ်ပေးပါတယ်။"),Wi.forEach(a),rs.forEach(a),$p=l(s),pa=i(s,"P",{"data-svelte-h":!0}),r(pa)!=="svelte-1f0cg0q"&&(pa.textContent=eh),Ip=l(s),w(ma.$$.fragment,s),Ap=l(s),ia=i(s,"P",{});var bo=u(ia);Er=g(bo,`KL divergence term က | |
| `),Hp=y(bo,!1),bo.forEach(a),Gp=l(s),z=i(s,"P",{});var Z=u(z);Br=g(Z,"KL divergence term မှာ၊"),Rp=y(Z,!1),Ep=g(Z," က အခြေခံအားဖြင့် pre-update model ရဲ့ output ဖြစ်တဲ့ "),Ht=i(Z,"CODE",{"data-svelte-h":!0}),r(Ht)!=="svelte-12zlpbd"&&(Ht.textContent=lh),Qr=g(Z," ဖြစ်ပြီး"),Bp=y(Z,!1),Qp=g(Z," က new model ရဲ့ output ဖြစ်တဲ့ "),Gt=i(Z,"CODE",{"data-svelte-h":!0}),r(Gt)!=="svelte-nsdcke"&&(Gt.textContent=ph),Dr=g(Z," ဖြစ်ပါတယ်။ သီအိုရီအရ၊ KL divergence ကို optimization လုပ်နေစဉ် model က ၎င်းရဲ့ မူရင်း behavior ကနေ အဝေးကြီး မသွေဖည်အောင် ကာကွယ်ဖို့ minimize လုပ်ပါတယ်။ ဒါက reward signal ပေါ် အခြေခံပြီး စွမ်းဆောင်ရည်ကို မြှင့်တင်ခြင်းနဲ့ coherence ကို ထိန်းသိမ်းခြင်းကြား ဟန်ချက်ညီစေဖို့ ကူညီပေးပါတယ်။ ဒီအခြေအနေမှာ၊ KL divergence ကို minimize လုပ်ခြင်းက model က အဓိပ္ပာယ်မရှိတဲ့ စာသားတွေ ဒါမှမဟုတ်၊ သင်္ချာဆိုင်ရာ reasoning မှာဆိုရင်၊ လုံးဝမမှန်ကန်တဲ့ အဖြေတွေကို ထုတ်ပေးနိုင်ခြေကို လျှော့ချပေးပါတယ်။"),Z.forEach(a),Dp=l(s),w(ra.$$.fragment,s),Zp=l(s),ca=i(s,"UL",{"data-svelte-h":!0}),r(ca)!=="svelte-miit2t"&&(ca.innerHTML=mh),Pp=l(s),w(oa.$$.fragment,s),Sp=l(s),ha=i(s,"P",{"data-svelte-h":!0}),r(ha)!=="svelte-185hbfl"&&(ha.textContent=ih),Np=l(s),ts=i(s,"P",{});var Oi=u(ts);Zr=g(Oi,"KL distance ကို အောက်ပါအတိုင်း သတ်မှတ်ကြောင်း ပြန်လည်မှတ်မိပါ။"),Xp=y(Oi,!1),Vp=g(Oi,` | |
| RLHF မှာ၊ စိတ်ဝင်စားတဲ့ distribution နှစ်ခုက မကြာခဏဆိုသလို new model version ရဲ့ distribution ဖြစ်တဲ့ P(x) နဲ့ reference policy ရဲ့ distribution ဖြစ်တဲ့ Q(x) တို့ ဖြစ်ပါတယ်။`),Oi.forEach(a),Fp=l(s),w(ga.$$.fragment,s),qp=l(s),ns=i(s,"P",{});var Yi=u(ns);Pr=g(Yi,"coefficient"),Wp=y(Yi,!1),Op=g(Yi," က KL divergence ကန့်သတ်ချက်ကို ကျွန်တော်တို့ ဘယ်လောက်ပြင်းပြင်းထန်ထန် သတ်မှတ်မလဲဆိုတာ ထိန်းချုပ်ပါတယ်။"),Yi.forEach(a),Yp=l(s),B=i(s,"UL",{});var re=u(B);Rt=i(re,"LI",{"data-svelte-h":!0}),r(Rt)!=="svelte-ncbntt"&&(Rt.innerHTML=rh),Sr=l(re),Et=i(re,"LI",{"data-svelte-h":!0}),r(Et)!=="svelte-1ammrbl"&&(Et.innerHTML=ch),Nr=l(re),Q=i(re,"LI",{});var bt=u(Q);Bt=i(bt,"STRONG",{"data-svelte-h":!0}),r(Bt)!=="svelte-1sfvsn0"&&(Bt.textContent=oh),Xr=l(bt),es=i(bt,"A",{href:!0,rel:!0,"data-svelte-h":!0}),r(es)!=="svelte-14kt0ed"&&(es.textContent=hh),Vr=g(bt," paper က ဒီ"),Kp=y(bt,!1),sm=g(bt," လို့ သတ်မှတ်ထားပါတယ်။"),bt.forEach(a),re.forEach(a),am=l(s),da=i(s,"P",{"data-svelte-h":!0}),r(da)!=="svelte-9n08ys"&&(da.textContent=gh),tm=l(s),w(ya.$$.fragment,s),nm=l(s),ua=i(s,"P",{"data-svelte-h":!0}),r(ua)!=="svelte-16makwl"&&(ua.textContent=dh),em=l(s),w(va.$$.fragment,s),lm=l(s),Qt=i(s,"P",{});var rd=u(Qt);pm=y(rd,!1),rd.forEach(a),mm=l(s),w(wa.$$.fragment,s),im=l(s),Ma=i(s,"P",{"data-svelte-h":!0}),r(Ma)!=="svelte-1tfvkz1"&&(Ma.textContent=yh),rm=l(s),U=i(s,"P",{});var V=u(U);Fr=g(V,"responses"),cm=y(V,!1),om=g(V," ခုကို ထုတ်ပေးပါ၊"),hm=y(V,!1),gm=g(V," ခုက မှန်ကန်တဲ့အဖြေ (\\( 14, \\text{reward=} 1 \\)) ဖြစ်ပြီး"),dm=y(V,!1),ym=g(V,` ခုက မမှန်ကန်ပါ (\\( \\text{reward= 0)} \\))၊ ဒါကြောင့်- | |
| `),um=y(V,!1),V.forEach(a),vm=l(s),w(fa.$$.fragment,s),wm=l(s),xa=i(s,"P",{"data-svelte-h":!0}),r(xa)!=="svelte-ss38cr"&&(xa.textContent=uh),Mm=l(s),ls=i(s,"TABLE",{});var Ki=u(ls);Dt=i(Ki,"THEAD",{"data-svelte-h":!0}),r(Dt)!=="svelte-115zw5e"&&(Dt.innerHTML=vh),qr=l(Ki),A=i(Ki,"TBODY",{});var cs=u(A);ba=i(cs,"TR",{});var sr=u(ba);Zt=i(sr,"TD",{"data-svelte-h":!0}),r(Zt)!=="svelte-175bp0i"&&(Zt.textContent=wh),Wr=l(sr),ye=i(sr,"TD",{});var cd=u(ye);fm=y(cd,!1),cd.forEach(a),sr.forEach(a),Or=l(cs),Ta=i(cs,"TR",{});var ar=u(Ta);Pt=i(ar,"TD",{"data-svelte-h":!0}),r(Pt)!=="svelte-ib0fgo"&&(Pt.textContent=Mh),Yr=l(ar),ue=i(ar,"TD",{});var od=u(ue);xm=y(od,!1),od.forEach(a),ar.forEach(a),Kr=l(cs),_a=i(cs,"TR",{});var tr=u(_a);St=i(tr,"TD",{"data-svelte-h":!0}),r(St)!=="svelte-1dv54pk"&&(St.textContent=fh),sc=l(tr),ve=i(tr,"TD",{});var hd=u(ve);bm=y(hd,!1),hd.forEach(a),tr.forEach(a),ac=l(cs),Ca=i(cs,"TR",{});var nr=u(Ca);Nt=i(nr,"TD",{"data-svelte-h":!0}),r(Nt)!=="svelte-1tmz1sl"&&(Nt.textContent=xh),tc=l(nr),we=i(nr,"TD",{});var gd=u(we);Tm=y(gd,!1),gd.forEach(a),nr.forEach(a),cs.forEach(a),Ki.forEach(a),_m=l(s),w(za.$$.fragment,s),Cm=l(s),Ja=i(s,"P",{"data-svelte-h":!0}),r(Ja)!=="svelte-nttvh1"&&(Ja.textContent=bh),zm=l(s),ps=i(s,"UL",{});var er=u(ps);J=i(er,"LI",{});var H=u(J);nc=g(H,"correct output"),Jm=y(H,!1),Um=g(H," အတွက် old policy (\\( \\pi"),Xt=i(H,"EM",{"data-svelte-h":!0}),r(Xt)!=="svelte-19ssaec"&&(Xt.textContent=Th),ec=g(H,"{old}} \\)) ရဲ့ probability က"),jm=y(H,!1),km=g(H," ဖြစ်ပြီး new policy က"),Lm=y(H,!1),$m=g(H," အထိ တိုးမြှင့်လိုက်တယ်လို့ ယူဆပါက-"),Im=y(H,!1),H.forEach(a),lc=l(er),Ua=i(er,"LI",{});var lr=u(Ua);pc=g(lr,"ထို့နောက် target function ကို ပြန်လည်ချိန်ညှိတဲ့အခါ၊ model က correct output ကို ထုတ်ပေးခြင်းကို အားဖြည့်ဖို့ ကြိုးစားပြီး"),Am=y(lr,!1),Hm=g(lr," က reference policy ကနေ သွေဖည်မှုကို ကန့်သတ်ပါတယ်။"),lr.forEach(a),er.forEach(a),Gm=l(s),ja=i(s,"P",{"data-svelte-h":!0}),r(ja)!=="svelte-1447kgy"&&(ja.textContent=_h),Rm=l(s),w(ka.$$.fragment,s),Em=l(s),La=i(s,"P",{"data-svelte-h":!0}),r(La)!=="svelte-tda4zt"&&(La.textContent=Ch),Bm=l(s),w($a.$$.fragment,s),Qm=l(s),Ia=i(s,"P",{"data-svelte-h":!0}),r(Ia)!=="svelte-17m006q"&&(Ia.textContent=zh),Dm=l(s),w(Aa.$$.fragment,s),Zm=l(s),Ha=i(s,"P",{"data-svelte-h":!0}),r(Ha)!=="svelte-1qchxa2"&&(Ha.textContent=Jh),Pm=l(s),w(Ga.$$.fragment,s),Sm=l(s),w(Ra.$$.fragment,s),Nm=l(s),Ea=i(s,"P",{"data-svelte-h":!0}),r(Ea)!=="svelte-1p9nvdq"&&(Ea.textContent=Uh),Xm=l(s),Ba=i(s,"P",{"data-svelte-h":!0}),r(Ba)!=="svelte-e3ho0p"&&(Ba.innerHTML=jh),Vm=l(s),Qa=i(s,"UL",{"data-svelte-h":!0}),r(Qa)!=="svelte-4kk6wt"&&(Qa.innerHTML=kh),Fm=l(s),Da=i(s,"P",{"data-svelte-h":!0}),r(Da)!=="svelte-jf33td"&&(Da.textContent=Lh),qm=l(s),w(Za.$$.fragment,s),Wm=l(s),Pa=i(s,"P",{"data-svelte-h":!0}),r(Pa)!=="svelte-r71s26"&&(Pa.textContent=$h),Om=l(s),w(Sa.$$.fragment,s),Ym=l(s),Na=i(s,"P",{"data-svelte-h":!0}),r(Na)!=="svelte-1cj2o0a"&&(Na.textContent=Ih),Km=l(s),w(Xa.$$.fragment,s),si=l(s),Va=i(s,"P",{"data-svelte-h":!0}),r(Va)!=="svelte-tx6yfm"&&(Va.textContent=Ah),ai=l(s),w(Fa.$$.fragment,s),ti=l(s),qa=i(s,"P",{"data-svelte-h":!0}),r(qa)!=="svelte-1cj2o0a"&&(qa.textContent=Hh),ni=l(s),w(Wa.$$.fragment,s),ei=l(s),Oa=i(s,"P",{"data-svelte-h":!0}),r(Oa)!=="svelte-8r2fmi"&&(Oa.textContent=Gh),li=l(s),w(Ya.$$.fragment,s),pi=l(s),Ka=i(s,"P",{"data-svelte-h":!0}),r(Ka)!=="svelte-m4nm5q"&&(Ka.innerHTML=Rh),mi=l(s),w(st.$$.fragment,s),ii=l(s),at=i(s,"P",{"data-svelte-h":!0}),r(at)!=="svelte-1cj2o0a"&&(at.textContent=Eh),ri=l(s),w(tt.$$.fragment,s),ci=l(s),nt=i(s,"P",{"data-svelte-h":!0}),r(nt)!=="svelte-bhb74h"&&(nt.textContent=Bh),oi=l(s),w(et.$$.fragment,s),hi=l(s),lt=i(s,"P",{"data-svelte-h":!0}),r(lt)!=="svelte-yi3paw"&&(lt.textContent=Qh),gi=l(s),w(pt.$$.fragment,s),di=l(s),mt=i(s,"P",{"data-svelte-h":!0}),r(mt)!=="svelte-w5yo9w"&&(mt.innerHTML=Dh),yi=l(s),w(it.$$.fragment,s),ui=l(s),rt=i(s,"P",{"data-svelte-h":!0}),r(rt)!=="svelte-dj220b"&&(rt.innerHTML=Zh),vi=l(s),w(ct.$$.fragment,s),wi=l(s),ot=i(s,"P",{"data-svelte-h":!0}),r(ot)!=="svelte-1e2egot"&&(ot.innerHTML=Ph),Mi=l(s),w(ht.$$.fragment,s),fi=l(s),gt=i(s,"P",{"data-svelte-h":!0}),r(gt)!=="svelte-1dgguby"&&(gt.textContent=Sh),xi=l(s),dt=i(s,"P",{"data-svelte-h":!0}),r(dt)!=="svelte-11514b1"&&(dt.textContent=Nh),bi=l(s),yt=i(s,"P",{"data-svelte-h":!0}),r(yt)!=="svelte-1ing07h"&&(yt.textContent=Xh),Ti=l(s),ut=i(s,"P",{"data-svelte-h":!0}),r(ut)!=="svelte-114vvt5"&&(ut.textContent=Vh),_i=l(s),vt=i(s,"P",{"data-svelte-h":!0}),r(vt)!=="svelte-143ddkt"&&(vt.textContent=Fh),Ci=l(s),w(wt.$$.fragment,s),zi=l(s),Mt=i(s,"OL",{"data-svelte-h":!0}),r(Mt)!=="svelte-18yys75"&&(Mt.innerHTML=qh),Ji=l(s),Ui=i(s,"HR",{}),ji=l(s),w(ft.$$.fragment,s),ki=l(s),c=i(s,"UL",{});var o=u(c);Vt=i(o,"LI",{"data-svelte-h":!0}),r(Vt)!=="svelte-a0t5yz"&&(Vt.innerHTML=Wh),mc=l(o),Ft=i(o,"LI",{"data-svelte-h":!0}),r(Ft)!=="svelte-1fuumf6"&&(Ft.innerHTML=Oh),ic=l(o),qt=i(o,"LI",{"data-svelte-h":!0}),r(qt)!=="svelte-3wcjkd"&&(qt.innerHTML=Yh),rc=l(o),Wt=i(o,"LI",{"data-svelte-h":!0}),r(Wt)!=="svelte-154325s"&&(Wt.innerHTML=Kh),cc=l(o),Ot=i(o,"LI",{"data-svelte-h":!0}),r(Ot)!=="svelte-1v4s720"&&(Ot.innerHTML=sg),oc=l(o),Yt=i(o,"LI",{"data-svelte-h":!0}),r(Yt)!=="svelte-14vxxe5"&&(Yt.innerHTML=ag),hc=l(o),Kt=i(o,"LI",{"data-svelte-h":!0}),r(Kt)!=="svelte-59527o"&&(Kt.innerHTML=tg),gc=l(o),sn=i(o,"LI",{"data-svelte-h":!0}),r(sn)!=="svelte-10dbisk"&&(sn.innerHTML=ng),dc=l(o),an=i(o,"LI",{"data-svelte-h":!0}),r(an)!=="svelte-kyq36r"&&(an.innerHTML=eg),yc=l(o),tn=i(o,"LI",{"data-svelte-h":!0}),r(tn)!=="svelte-xotlbe"&&(tn.innerHTML=lg),uc=l(o),nn=i(o,"LI",{"data-svelte-h":!0}),r(nn)!=="svelte-j4i6f6"&&(nn.innerHTML=pg),vc=l(o),en=i(o,"LI",{"data-svelte-h":!0}),r(en)!=="svelte-67jj69"&&(en.innerHTML=mg),wc=l(o),ln=i(o,"LI",{"data-svelte-h":!0}),r(ln)!=="svelte-1r05jdi"&&(ln.innerHTML=ig),Mc=l(o),pn=i(o,"LI",{"data-svelte-h":!0}),r(pn)!=="svelte-18dd7nr"&&(pn.innerHTML=rg),fc=l(o),mn=i(o,"LI",{"data-svelte-h":!0}),r(mn)!=="svelte-5tsxfd"&&(mn.innerHTML=cg),xc=l(o),rn=i(o,"LI",{"data-svelte-h":!0}),r(rn)!=="svelte-1rt1hkn"&&(rn.innerHTML=og),bc=l(o),cn=i(o,"LI",{"data-svelte-h":!0}),r(cn)!=="svelte-18ihopu"&&(cn.innerHTML=hg),Tc=l(o),on=i(o,"LI",{"data-svelte-h":!0}),r(on)!=="svelte-925r2m"&&(on.innerHTML=gg),_c=l(o),hn=i(o,"LI",{"data-svelte-h":!0}),r(hn)!=="svelte-r6oavu"&&(hn.innerHTML=dg),Cc=l(o),D=i(o,"LI",{});var Tt=u(D);gn=i(Tt,"STRONG",{"data-svelte-h":!0}),r(gn)!=="svelte-1un35le"&&(gn.innerHTML=yg),zc=g(Tt,": New policy အောက်တွင် response"),Li=y(Tt,!1),$i=g(Tt," ကို query"),Ii=y(Tt,!1),Ai=g(Tt," အတွက် ထုတ်ပေးနိုင်ခြေ probability။"),Tt.forEach(a),Jc=l(o),ms=i(o,"LI",{});var be=u(ms);dn=i(be,"STRONG",{"data-svelte-h":!0}),r(dn)!=="svelte-1j5l019"&&(dn.innerHTML=ug),Uc=g(be,": Probability ratio ကို သတ်မှတ်ထားသော အတိုင်းအတာ"),Hi=y(be,!1),Gi=g(be," အတွင်း ကန့်သတ်သော clipping function။"),be.forEach(a),jc=l(o),yn=i(o,"LI",{"data-svelte-h":!0}),r(yn)!=="svelte-1m6vq8u"&&(yn.innerHTML=vg),kc=l(o),un=i(o,"LI",{"data-svelte-h":!0}),r(un)!=="svelte-u0fuen"&&(un.innerHTML=wg),Lc=l(o),vn=i(o,"LI",{"data-svelte-h":!0}),r(vn)!=="svelte-ppdrxc"&&(vn.innerHTML=Mg),$c=l(o),wn=i(o,"LI",{"data-svelte-h":!0}),r(wn)!=="svelte-1rkhj13"&&(wn.innerHTML=fg),Ic=l(o),Mn=i(o,"LI",{"data-svelte-h":!0}),r(Mn)!=="svelte-2gqu61"&&(Mn.innerHTML=xg),Ac=l(o),fn=i(o,"LI",{"data-svelte-h":!0}),r(fn)!=="svelte-1umx7j0"&&(fn.innerHTML=bg),Hc=l(o),xn=i(o,"LI",{"data-svelte-h":!0}),r(xn)!=="svelte-og97si"&&(xn.innerHTML=Tg),Gc=l(o),bn=i(o,"LI",{"data-svelte-h":!0}),r(bn)!=="svelte-agiryw"&&(bn.innerHTML=_g),Rc=l(o),Tn=i(o,"LI",{"data-svelte-h":!0}),r(Tn)!=="svelte-1dpvqfo"&&(Tn.innerHTML=Cg),Ec=l(o),_n=i(o,"LI",{"data-svelte-h":!0}),r(_n)!=="svelte-fy357x"&&(_n.innerHTML=zg),Bc=l(o),Cn=i(o,"LI",{"data-svelte-h":!0}),r(Cn)!=="svelte-1che2x5"&&(Cn.innerHTML=Jg),Qc=l(o),zn=i(o,"LI",{"data-svelte-h":!0}),r(zn)!=="svelte-xwlo63"&&(zn.innerHTML=Ug),Dc=l(o),Jn=i(o,"LI",{"data-svelte-h":!0}),r(Jn)!=="svelte-1gi9adu"&&(Jn.innerHTML=jg),Zc=l(o),Un=i(o,"LI",{"data-svelte-h":!0}),r(Un)!=="svelte-1ay5uxg"&&(Un.innerHTML=kg),Pc=l(o),jn=i(o,"LI",{"data-svelte-h":!0}),r(jn)!=="svelte-i3u35o"&&(jn.innerHTML=Lg),Sc=l(o),kn=i(o,"LI",{"data-svelte-h":!0}),r(kn)!=="svelte-4gsj4s"&&(kn.innerHTML=$g),Nc=l(o),Ln=i(o,"LI",{"data-svelte-h":!0}),r(Ln)!=="svelte-1jtrdu8"&&(Ln.innerHTML=Ig),Xc=l(o),$n=i(o,"LI",{"data-svelte-h":!0}),r($n)!=="svelte-1whrrfd"&&($n.innerHTML=Ag),Vc=l(o),In=i(o,"LI",{"data-svelte-h":!0}),r(In)!=="svelte-av500o"&&(In.innerHTML=Hg),Fc=l(o),An=i(o,"LI",{"data-svelte-h":!0}),r(An)!=="svelte-1yvaxqr"&&(An.innerHTML=Gg),qc=l(o),Hn=i(o,"LI",{"data-svelte-h":!0}),r(Hn)!=="svelte-yry7ha"&&(Hn.innerHTML=Rg),Wc=l(o),Gn=i(o,"LI",{"data-svelte-h":!0}),r(Gn)!=="svelte-1cj397z"&&(Gn.innerHTML=Eg),Oc=l(o),Rn=i(o,"LI",{"data-svelte-h":!0}),r(Rn)!=="svelte-1qjnw0c"&&(Rn.innerHTML=Bg),Yc=l(o),En=i(o,"LI",{"data-svelte-h":!0}),r(En)!=="svelte-ado0if"&&(En.innerHTML=Qg),Kc=l(o),Bn=i(o,"LI",{"data-svelte-h":!0}),r(Bn)!=="svelte-hpsyj8"&&(Bn.innerHTML=Dg),so=l(o),Qn=i(o,"LI",{"data-svelte-h":!0}),r(Qn)!=="svelte-1bureh0"&&(Qn.innerHTML=Zg),ao=l(o),Dn=i(o,"LI",{"data-svelte-h":!0}),r(Dn)!=="svelte-1y4yrod"&&(Dn.innerHTML=Pg),to=l(o),Zn=i(o,"LI",{"data-svelte-h":!0}),r(Zn)!=="svelte-16p9qrq"&&(Zn.innerHTML=Sg),no=l(o),Pn=i(o,"LI",{"data-svelte-h":!0}),r(Pn)!=="svelte-s7yhu4"&&(Pn.innerHTML=Ng),eo=l(o),Sn=i(o,"LI",{"data-svelte-h":!0}),r(Sn)!=="svelte-47wfgq"&&(Sn.innerHTML=Xg),lo=l(o),Nn=i(o,"LI",{"data-svelte-h":!0}),r(Nn)!=="svelte-1sdijj7"&&(Nn.innerHTML=Vg),po=l(o),Xn=i(o,"LI",{"data-svelte-h":!0}),r(Xn)!=="svelte-13pqyqi"&&(Xn.innerHTML=Fg),mo=l(o),Vn=i(o,"LI",{"data-svelte-h":!0}),r(Vn)!=="svelte-rsa6kv"&&(Vn.innerHTML=qg),io=l(o),Fn=i(o,"LI",{"data-svelte-h":!0}),r(Fn)!=="svelte-ow2jeg"&&(Fn.innerHTML=Wg),ro=l(o),qn=i(o,"LI",{"data-svelte-h":!0}),r(qn)!=="svelte-wy6y24"&&(qn.innerHTML=Og),co=l(o),Wn=i(o,"LI",{"data-svelte-h":!0}),r(Wn)!=="svelte-1go82my"&&(Wn.innerHTML=Yg),oo=l(o),On=i(o,"LI",{"data-svelte-h":!0}),r(On)!=="svelte-1br9amv"&&(On.innerHTML=Kg),ho=l(o),Yn=i(o,"LI",{"data-svelte-h":!0}),r(Yn)!=="svelte-z25jni"&&(Yn.innerHTML=sd),go=l(o),Kn=i(o,"LI",{"data-svelte-h":!0}),r(Kn)!=="svelte-f3mpij"&&(Kn.innerHTML=ad),yo=l(o),se=i(o,"LI",{"data-svelte-h":!0}),r(se)!=="svelte-4oo5pz"&&(se.innerHTML=td),uo=l(o),ae=i(o,"LI",{"data-svelte-h":!0}),r(ae)!=="svelte-fpvhvt"&&(ae.innerHTML=nd),vo=l(o),te=i(o,"LI",{"data-svelte-h":!0}),r(te)!=="svelte-xbwlrq"&&(te.innerHTML=ed),o.forEach(a),Ri=l(s),w(xt.$$.fragment,s),Ei=l(s),Me=i(s,"P",{}),u(Me).forEach(a),this.h()},h(){Te(P,"name","hf:doc:metadata"),Te(P,"content",J0),Te(F,"class","tip"),Qe.a=De,Ze.a=Pe,Se.a=Ne,Xe.a=Ve,Fe.a=qe,Ke.a=sl,al.a=null,nl.a=el,ll.a=null,hl.a=gl,dl.a=yl,wl.a=null,bl.a=null,Tl.a=null,_l.a=null,Cl.a=null,jl.a=kl,Ll.a=$l,Il.a=Al,Hl.a=Gl,Rl.a=El,Ql.a=Dl,Xl.a=null,Yl.a=null,tp.a=np,ep.a=lp,pp.a=mp,ip.a=rp,gp.a=null,yp.a=up,fp.a=xp,bp.a=Tp,zp.a=Jp,Up.a=jp,kp.a=Lp,Hp.a=null,Rp.a=Ep,Bp.a=Qp,Xp.a=Vp,Wp.a=Op,Te(es,"href","https://arxiv.org/abs/2402.03300"),Te(es,"rel","nofollow"),Kp.a=sm,pm.a=null,cm.a=om,hm.a=gm,dm.a=ym,um.a=null,fm.a=null,xm.a=null,bm.a=null,Tm.a=null,Jm.a=Um,jm.a=km,Lm.a=$m,Im.a=null,Am.a=Hm,Li.a=$i,Ii.a=Ai,Hi.a=Gi},m(s,n){t(document.head,P),p(s,_e,n),p(s,ce,n),p(s,Ce,n),M(os,s,n),p(s,ze,n),M(hs,s,n),p(s,Je,n),p(s,F,n),p(s,Ue,n),p(s,gs,n),p(s,je,n),p(s,ds,n),p(s,ke,n),p(s,ys,n),p(s,Le,n),p(s,us,n),p(s,$e,n),p(s,vs,n),p(s,Ie,n),p(s,ws,n),p(s,Ae,n),M(Ms,s,n),p(s,He,n),p(s,fs,n),p(s,Ge,n),p(s,xs,n),p(s,Re,n),M(bs,s,n),p(s,Ee,n),p(s,Ts,n),p(s,Be,n),p(s,_,n),t(_,pr),Qe.m(dd,_),t(_,De),Ze.m(yd,_),t(_,Pe),Se.m(ud,_),t(_,Ne),Xe.m(vd,_),t(_,Ve),Fe.m(wd,_),t(_,qe),p(s,We,n),M(_s,s,n),p(s,Oe,n),p(s,Cs,n),p(s,Ye,n),p(s,G,n),t(G,_t),t(G,mr),Ke.m(Md,G),t(G,sl),al.m(fd,G),p(s,tl,n),p(s,R,n),t(R,Ct),t(R,ir),nl.m(xd,R),t(R,el),ll.m(bd,R),p(s,pl,n),p(s,zs,n),p(s,ml,n),M(Js,s,n),p(s,il,n),p(s,Us,n),p(s,rl,n),M(js,s,n),p(s,cl,n),p(s,ks,n),p(s,ol,n),p(s,E,n),t(E,rr),hl.m(Td,E),t(E,gl),dl.m(_d,E),t(E,yl),p(s,ul,n),M(Ls,s,n),p(s,vl,n),p(s,$s,n),t($s,cr),wl.m(Cd,$s),p(s,Ml,n),M(Is,s,n),p(s,fl,n),p(s,As,n),p(s,xl,n),p(s,q,n),t(q,zt),t(q,or),t(q,I),t(I,Hs),t(Hs,Jt),t(Hs,hr),t(Hs,oe),bl.m(zd,oe),t(I,gr),t(I,Gs),t(Gs,Ut),t(Gs,dr),t(Gs,he),Tl.m(Jd,he),t(I,yr),t(I,Rs),t(Rs,jt),t(Rs,ur),t(Rs,ge),_l.m(Ud,ge),t(I,vr),t(I,Es),t(Es,kt),t(Es,wr),t(Es,de),Cl.m(jd,de),p(s,zl,n),M(Bs,s,n),p(s,Jl,n),p(s,Qs,n),p(s,Ul,n),p(s,C,n),t(C,Mr),jl.m(kd,C),t(C,kl),Ll.m(Ld,C),t(C,$l),Il.m($d,C),t(C,Al),Hl.m(Id,C),t(C,Gl),Rl.m(Ad,C),t(C,El),p(s,Bl,n),p(s,W,n),t(W,fr),Ql.m(Hd,W),t(W,Dl),p(s,Zl,n),p(s,Ds,n),p(s,Pl,n),M(Zs,s,n),p(s,Sl,n),p(s,Ps,n),p(s,Nl,n),p(s,Ss,n),t(Ss,xr),Xl.m(Gd,Ss),p(s,Vl,n),p(s,Ns,n),p(s,Fl,n),M(Xs,s,n),p(s,ql,n),p(s,Vs,n),p(s,Wl,n),M(Fs,s,n),p(s,Ol,n),p(s,qs,n),t(qs,br),Yl.m(Rd,qs),p(s,Kl,n),p(s,Ws,n),p(s,sp,n),M(Os,s,n),p(s,ap,n),p(s,O,n),t(O,S),t(S,Tr),tp.m(Ed,S),t(S,np),ep.m(Bd,S),t(S,lp),t(O,_r),t(O,N),t(N,Cr),pp.m(Qd,N),t(N,mp),ip.m(Dd,N),t(N,rp),p(s,cp,n),p(s,Ys,n),p(s,op,n),M(Ks,s,n),p(s,hp,n),p(s,sa,n),t(sa,zr),gp.m(Zd,sa),p(s,dp,n),p(s,Y,n),t(Y,Jr),yp.m(Pd,Y),t(Y,up),p(s,vp,n),M(aa,s,n),p(s,wp,n),p(s,ta,n),p(s,Mp,n),p(s,K,n),t(K,ss),t(ss,Lt),t(ss,Ur),fp.m(Sd,ss),t(ss,xp),t(K,jr),t(K,as),t(as,$t),t(as,kr),bp.m(Nd,as),t(as,Tp),p(s,_p,n),M(na,s,n),p(s,Cp,n),p(s,k,n),t(k,It),t(k,Lr),t(k,ea),t(ea,$r),t(ea,X),t(X,Ir),zp.m(Xd,X),t(X,Jp),Up.m(Vd,X),t(X,jp),t(ea,Ar),t(k,Hr),t(k,At),t(k,Gr),t(k,la),t(la,Rr),kp.m(Fd,la),t(la,Lp),p(s,$p,n),p(s,pa,n),p(s,Ip,n),M(ma,s,n),p(s,Ap,n),p(s,ia,n),t(ia,Er),Hp.m(qd,ia),p(s,Gp,n),p(s,z,n),t(z,Br),Rp.m(Wd,z),t(z,Ep),t(z,Ht),t(z,Qr),Bp.m(Od,z),t(z,Qp),t(z,Gt),t(z,Dr),p(s,Dp,n),M(ra,s,n),p(s,Zp,n),p(s,ca,n),p(s,Pp,n),M(oa,s,n),p(s,Sp,n),p(s,ha,n),p(s,Np,n),p(s,ts,n),t(ts,Zr),Xp.m(Yd,ts),t(ts,Vp),p(s,Fp,n),M(ga,s,n),p(s,qp,n),p(s,ns,n),t(ns,Pr),Wp.m(Kd,ns),t(ns,Op),p(s,Yp,n),p(s,B,n),t(B,Rt),t(B,Sr),t(B,Et),t(B,Nr),t(B,Q),t(Q,Bt),t(Q,Xr),t(Q,es),t(Q,Vr),Kp.m(s0,Q),t(Q,sm),p(s,am,n),p(s,da,n),p(s,tm,n),M(ya,s,n),p(s,nm,n),p(s,ua,n),p(s,em,n),M(va,s,n),p(s,lm,n),p(s,Qt,n),pm.m(a0,Qt),p(s,mm,n),M(wa,s,n),p(s,im,n),p(s,Ma,n),p(s,rm,n),p(s,U,n),t(U,Fr),cm.m(t0,U),t(U,om),hm.m(n0,U),t(U,gm),dm.m(e0,U),t(U,ym),um.m(l0,U),p(s,vm,n),M(fa,s,n),p(s,wm,n),p(s,xa,n),p(s,Mm,n),p(s,ls,n),t(ls,Dt),t(ls,qr),t(ls,A),t(A,ba),t(ba,Zt),t(ba,Wr),t(ba,ye),fm.m(p0,ye),t(A,Or),t(A,Ta),t(Ta,Pt),t(Ta,Yr),t(Ta,ue),xm.m(m0,ue),t(A,Kr),t(A,_a),t(_a,St),t(_a,sc),t(_a,ve),bm.m(i0,ve),t(A,ac),t(A,Ca),t(Ca,Nt),t(Ca,tc),t(Ca,we),Tm.m(r0,we),p(s,_m,n),M(za,s,n),p(s,Cm,n),p(s,Ja,n),p(s,zm,n),p(s,ps,n),t(ps,J),t(J,nc),Jm.m(c0,J),t(J,Um),t(J,Xt),t(J,ec),jm.m(o0,J),t(J,km),Lm.m(h0,J),t(J,$m),Im.m(g0,J),t(ps,lc),t(ps,Ua),t(Ua,pc),Am.m(d0,Ua),t(Ua,Hm),p(s,Gm,n),p(s,ja,n),p(s,Rm,n),M(ka,s,n),p(s,Em,n),p(s,La,n),p(s,Bm,n),M($a,s,n),p(s,Qm,n),p(s,Ia,n),p(s,Dm,n),M(Aa,s,n),p(s,Zm,n),p(s,Ha,n),p(s,Pm,n),M(Ga,s,n),p(s,Sm,n),M(Ra,s,n),p(s,Nm,n),p(s,Ea,n),p(s,Xm,n),p(s,Ba,n),p(s,Vm,n),p(s,Qa,n),p(s,Fm,n),p(s,Da,n),p(s,qm,n),M(Za,s,n),p(s,Wm,n),p(s,Pa,n),p(s,Om,n),M(Sa,s,n),p(s,Ym,n),p(s,Na,n),p(s,Km,n),M(Xa,s,n),p(s,si,n),p(s,Va,n),p(s,ai,n),M(Fa,s,n),p(s,ti,n),p(s,qa,n),p(s,ni,n),M(Wa,s,n),p(s,ei,n),p(s,Oa,n),p(s,li,n),M(Ya,s,n),p(s,pi,n),p(s,Ka,n),p(s,mi,n),M(st,s,n),p(s,ii,n),p(s,at,n),p(s,ri,n),M(tt,s,n),p(s,ci,n),p(s,nt,n),p(s,oi,n),M(et,s,n),p(s,hi,n),p(s,lt,n),p(s,gi,n),M(pt,s,n),p(s,di,n),p(s,mt,n),p(s,yi,n),M(it,s,n),p(s,ui,n),p(s,rt,n),p(s,vi,n),M(ct,s,n),p(s,wi,n),p(s,ot,n),p(s,Mi,n),M(ht,s,n),p(s,fi,n),p(s,gt,n),p(s,xi,n),p(s,dt,n),p(s,bi,n),p(s,yt,n),p(s,Ti,n),p(s,ut,n),p(s,_i,n),p(s,vt,n),p(s,Ci,n),M(wt,s,n),p(s,zi,n),p(s,Mt,n),p(s,Ji,n),p(s,Ui,n),p(s,ji,n),M(ft,s,n),p(s,ki,n),p(s,c,n),t(c,Vt),t(c,mc),t(c,Ft),t(c,ic),t(c,qt),t(c,rc),t(c,Wt),t(c,cc),t(c,Ot),t(c,oc),t(c,Yt),t(c,hc),t(c,Kt),t(c,gc),t(c,sn),t(c,dc),t(c,an),t(c,yc),t(c,tn),t(c,uc),t(c,nn),t(c,vc),t(c,en),t(c,wc),t(c,ln),t(c,Mc),t(c,pn),t(c,fc),t(c,mn),t(c,xc),t(c,rn),t(c,bc),t(c,cn),t(c,Tc),t(c,on),t(c,_c),t(c,hn),t(c,Cc),t(c,D),t(D,gn),t(D,zc),Li.m(y0,D),t(D,$i),Ii.m(u0,D),t(D,Ai),t(c,Jc),t(c,ms),t(ms,dn),t(ms,Uc),Hi.m(v0,ms),t(ms,Gi),t(c,jc),t(c,yn),t(c,kc),t(c,un),t(c,Lc),t(c,vn),t(c,$c),t(c,wn),t(c,Ic),t(c,Mn),t(c,Ac),t(c,fn),t(c,Hc),t(c,xn),t(c,Gc),t(c,bn),t(c,Rc),t(c,Tn),t(c,Ec),t(c,_n),t(c,Bc),t(c,Cn),t(c,Qc),t(c,zn),t(c,Dc),t(c,Jn),t(c,Zc),t(c,Un),t(c,Pc),t(c,jn),t(c,Sc),t(c,kn),t(c,Nc),t(c,Ln),t(c,Xc),t(c,$n),t(c,Vc),t(c,In),t(c,Fc),t(c,An),t(c,qc),t(c,Hn),t(c,Wc),t(c,Gn),t(c,Oc),t(c,Rn),t(c,Yc),t(c,En),t(c,Kc),t(c,Bn),t(c,so),t(c,Qn),t(c,ao),t(c,Dn),t(c,to),t(c,Zn),t(c,no),t(c,Pn),t(c,eo),t(c,Sn),t(c,lo),t(c,Nn),t(c,po),t(c,Xn),t(c,mo),t(c,Vn),t(c,io),t(c,Fn),t(c,ro),t(c,qn),t(c,co),t(c,Wn),t(c,oo),t(c,On),t(c,ho),t(c,Yn),t(c,go),t(c,Kn),t(c,yo),t(c,se),t(c,uo),t(c,ae),t(c,vo),t(c,te),p(s,Ri,n),M(xt,s,n),p(s,Ei,n),p(s,Me,n),Bi=!0},p:M0,i(s){Bi||(f(os.$$.fragment,s),f(hs.$$.fragment,s),f(Ms.$$.fragment,s),f(bs.$$.fragment,s),f(_s.$$.fragment,s),f(Js.$$.fragment,s),f(js.$$.fragment,s),f(Ls.$$.fragment,s),f(Is.$$.fragment,s),f(Bs.$$.fragment,s),f(Zs.$$.fragment,s),f(Xs.$$.fragment,s),f(Fs.$$.fragment,s),f(Os.$$.fragment,s),f(Ks.$$.fragment,s),f(aa.$$.fragment,s),f(na.$$.fragment,s),f(ma.$$.fragment,s),f(ra.$$.fragment,s),f(oa.$$.fragment,s),f(ga.$$.fragment,s),f(ya.$$.fragment,s),f(va.$$.fragment,s),f(wa.$$.fragment,s),f(fa.$$.fragment,s),f(za.$$.fragment,s),f(ka.$$.fragment,s),f($a.$$.fragment,s),f(Aa.$$.fragment,s),f(Ga.$$.fragment,s),f(Ra.$$.fragment,s),f(Za.$$.fragment,s),f(Sa.$$.fragment,s),f(Xa.$$.fragment,s),f(Fa.$$.fragment,s),f(Wa.$$.fragment,s),f(Ya.$$.fragment,s),f(st.$$.fragment,s),f(tt.$$.fragment,s),f(et.$$.fragment,s),f(pt.$$.fragment,s),f(it.$$.fragment,s),f(ct.$$.fragment,s),f(ht.$$.fragment,s),f(wt.$$.fragment,s),f(ft.$$.fragment,s),f(xt.$$.fragment,s),Bi=!0)},o(s){x(os.$$.fragment,s),x(hs.$$.fragment,s),x(Ms.$$.fragment,s),x(bs.$$.fragment,s),x(_s.$$.fragment,s),x(Js.$$.fragment,s),x(js.$$.fragment,s),x(Ls.$$.fragment,s),x(Is.$$.fragment,s),x(Bs.$$.fragment,s),x(Zs.$$.fragment,s),x(Xs.$$.fragment,s),x(Fs.$$.fragment,s),x(Os.$$.fragment,s),x(Ks.$$.fragment,s),x(aa.$$.fragment,s),x(na.$$.fragment,s),x(ma.$$.fragment,s),x(ra.$$.fragment,s),x(oa.$$.fragment,s),x(ga.$$.fragment,s),x(ya.$$.fragment,s),x(va.$$.fragment,s),x(wa.$$.fragment,s),x(fa.$$.fragment,s),x(za.$$.fragment,s),x(ka.$$.fragment,s),x($a.$$.fragment,s),x(Aa.$$.fragment,s),x(Ga.$$.fragment,s),x(Ra.$$.fragment,s),x(Za.$$.fragment,s),x(Sa.$$.fragment,s),x(Xa.$$.fragment,s),x(Fa.$$.fragment,s),x(Wa.$$.fragment,s),x(Ya.$$.fragment,s),x(st.$$.fragment,s),x(tt.$$.fragment,s),x(et.$$.fragment,s),x(pt.$$.fragment,s),x(it.$$.fragment,s),x(ct.$$.fragment,s),x(ht.$$.fragment,s),x(wt.$$.fragment,s),x(ft.$$.fragment,s),x(xt.$$.fragment,s),Bi=!1},d(s){s&&(a(_e),a(ce),a(Ce),a(ze),a(Je),a(F),a(Ue),a(gs),a(je),a(ds),a(ke),a(ys),a(Le),a(us),a($e),a(vs),a(Ie),a(ws),a(Ae),a(He),a(fs),a(Ge),a(xs),a(Re),a(Ee),a(Ts),a(Be),a(_),a(We),a(Oe),a(Cs),a(Ye),a(G),a(tl),a(R),a(pl),a(zs),a(ml),a(il),a(Us),a(rl),a(cl),a(ks),a(ol),a(E),a(ul),a(vl),a($s),a(Ml),a(fl),a(As),a(xl),a(q),a(zl),a(Jl),a(Qs),a(Ul),a(C),a(Bl),a(W),a(Zl),a(Ds),a(Pl),a(Sl),a(Ps),a(Nl),a(Ss),a(Vl),a(Ns),a(Fl),a(ql),a(Vs),a(Wl),a(Ol),a(qs),a(Kl),a(Ws),a(sp),a(ap),a(O),a(cp),a(Ys),a(op),a(hp),a(sa),a(dp),a(Y),a(vp),a(wp),a(ta),a(Mp),a(K),a(_p),a(Cp),a(k),a($p),a(pa),a(Ip),a(Ap),a(ia),a(Gp),a(z),a(Dp),a(Zp),a(ca),a(Pp),a(Sp),a(ha),a(Np),a(ts),a(Fp),a(qp),a(ns),a(Yp),a(B),a(am),a(da),a(tm),a(nm),a(ua),a(em),a(lm),a(Qt),a(mm),a(im),a(Ma),a(rm),a(U),a(vm),a(wm),a(xa),a(Mm),a(ls),a(_m),a(Cm),a(Ja),a(zm),a(ps),a(Gm),a(ja),a(Rm),a(Em),a(La),a(Bm),a(Qm),a(Ia),a(Dm),a(Zm),a(Ha),a(Pm),a(Sm),a(Nm),a(Ea),a(Xm),a(Ba),a(Vm),a(Qa),a(Fm),a(Da),a(qm),a(Wm),a(Pa),a(Om),a(Ym),a(Na),a(Km),a(si),a(Va),a(ai),a(ti),a(qa),a(ni),a(ei),a(Oa),a(li),a(pi),a(Ka),a(mi),a(ii),a(at),a(ri),a(ci),a(nt),a(oi),a(hi),a(lt),a(gi),a(di),a(mt),a(yi),a(ui),a(rt),a(vi),a(wi),a(ot),a(Mi),a(fi),a(gt),a(xi),a(dt),a(bi),a(yt),a(Ti),a(ut),a(_i),a(vt),a(Ci),a(zi),a(Mt),a(Ji),a(Ui),a(ji),a(ki),a(c),a(Ri),a(Ei),a(Me)),a(P),b(os,s),b(hs,s),b(Ms,s),b(bs,s),b(_s,s),b(Js,s),b(js,s),b(Ls,s),b(Is,s),b(Bs,s),b(Zs,s),b(Xs,s),b(Fs,s),b(Os,s),b(Ks,s),b(aa,s),b(na,s),b(ma,s),b(ra,s),b(oa,s),b(ga,s),b(ya,s),b(va,s),b(wa,s),b(fa,s),b(za,s),b(ka,s),b($a,s),b(Aa,s),b(Ga,s),b(Ra,s),b(Za,s),b(Sa,s),b(Xa,s),b(Fa,s),b(Wa,s),b(Ya,s),b(st,s),b(tt,s),b(et,s),b(pt,s),b(it,s),b(ct,s),b(ht,s),b(wt,s),b(ft,s),b(xt,s)}}}const J0='{"title":"DeepSeekMath တွင် Group Relative Policy Optimization (GRPO) ကို အဆင့်မြင့် နားလည်ခြင်း","local":"advanced-understanding-of-group-relative-policy-optimization-grpo-in-deepseekmath","sections":[{"title":"GRPO Algorithm","local":"grpo-algorithm","sections":[{"title":"အဆင့် ၁: Group Sampling","local":"အဆင-၁-group-sampling","sections":[{"title":"ဥပမာ","local":"ဥပမ","sections":[],"depth":4}],"depth":3},{"title":"အဆင့် ၂: Advantage Calculation","local":"အဆင-၂-advantage-calculation","sections":[{"title":"Reward Distribution","local":"reward-distribution","sections":[],"depth":4},{"title":"Advantage Value Formula","local":"advantage-value-formula","sections":[],"depth":4},{"title":"ဥပမာ","local":"ဥပမ","sections":[],"depth":4},{"title":"အဓိပ္ပာယ်ဖွင့်ဆိုချက်","local":"အဓပပယဖငဆခက","sections":[],"depth":4}],"depth":3},{"title":"အဆင့် ၃: Policy Update","local":"အဆင-၃-policy-update","sections":[],"depth":3}],"depth":2},{"title":"Target Function ၏ အဓိက အစိတ်အပိုင်းများ","local":"target-function--အဓက-အစတအပငမ","sections":[{"title":"၁။ Probability Ratio","local":"၁-probability-ratio","sections":[{"title":"အဓိပ္ပာယ်ဖွင့်ဆိုချက်","local":"အဓပပယဖငဆခက","sections":[],"depth":4}],"depth":3},{"title":"၂။ Clip Function","local":"၂-clip-function","sections":[{"title":"ဥပမာ (ε = 0.2)","local":"ဥပမ-ε--02","sections":[],"depth":4},{"title":"အဓိပ္ပာယ်ဖွင့်ဆိုချက်","local":"အဓပပယဖငဆခက","sections":[],"depth":4}],"depth":3},{"title":"၃။ KL Divergence","local":"၃-kl-divergence","sections":[{"title":"အဓိပ္ပာယ်ဖွင့်ဆိုချက်","local":"အဓပပယဖငဆခက","sections":[],"depth":4},{"title":"သင်္ချာဆိုင်ရာ အဓိပ္ပာယ်ဖွင့်ဆိုချက်","local":"သငခဆငရ-အဓပပယဖငဆခက","sections":[],"depth":4},{"title":"β Parameter ၏ အခန်းကဏ္ဍ","local":"β-parameter--အခနကဏဍ","sections":[],"depth":4}],"depth":3}],"depth":2},{"title":"GRPO ဖြင့် လုပ်ဆောင်ခဲ့သော ဥပမာ","local":"grpo-ဖင-လပဆငခသ-ဥပမ","sections":[{"title":"ဥပမာ ပြဿနာ","local":"ဥပမ-ပဿန","sections":[],"depth":3},{"title":"အဆင့် ၁: Group Sampling","local":"အဆင-၁-group-sampling","sections":[],"depth":3},{"title":"အဆင့် ၂: Advantage Calculation","local":"အဆင-၂-advantage-calculation","sections":[],"depth":3},{"title":"အဆင့် ၃: Policy Update","local":"အဆင-၃-policy-update","sections":[],"depth":3}],"depth":2},{"title":"Implementation Example","local":"implementation-example","sections":[{"title":"၁။ Model ကို Loading လုပ်ခြင်းနှင့် Responses များ ထုတ်ပေးခြင်း","local":"၁-model-က-loading-လပခငနင-responses-မ-ထတပခင","sections":[],"depth":3},{"title":"၂။ Rewards များကို တွက်ချက်ခြင်း","local":"၂-rewards-မက-တကခကခင","sections":[],"depth":3},{"title":"၃။ Policy ကို Update လုပ်ခြင်း","local":"၃-policy-က-update-လပခင","sections":[],"depth":3}],"depth":2},{"title":"အနှစ်ချုပ်နှင့် နောက်ထပ် အဆင့်များ","local":"အနစခပနင-နကထပ-အဆငမ","sections":[],"depth":2},{"title":"References","local":"references","sections":[],"depth":2},{"title":"ဝေါဟာရ ရှင်းလင်းချက် (Glossary)","local":"ဝဟရ-ရငလငခက-glossary","sections":[],"depth":2}],"depth":1}';function U0(To){return f0(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class I0 extends x0{constructor(P){super(),b0(this,P,U0,z0,w0,{})}}export{I0 as component}; | |
Xet Storage Details
- Size:
- 248 kB
- Xet hash:
- e9c898ddc30f5fc450ee64d6b48911efccde960ccbc8141b1f592a72956bff46
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.