Hugging Face's logo

Buckets:
hf-doc-build
/
doc-dev

Files
xet
hf-doc-build/doc-dev / trl /pr_5313 /en /_app /immutable /nodes /29.pcworu2R.js
HuggingFaceDocBuilder's picture
HuggingFaceDocBuilder
download
raw
57.1 kB
import"../chunks/DsnmJJEf.js";import{i as as,h as ns,C as ts,H as i,D as o,b as g,E as es,s as ls,a as c}from"../chunks/ChUdhzCR.js";import{p as ps,o as rs,s,f as m,a as r,b as is,c as e,d as U,r as l,n as E}from"../chunks/BxrqFSCq.js";import{E as h}from"../chunks/6U88GyHj.js";const ms='{"title":"Reward Functions","local":"reward-functions","sections":[{"title":"accuracy_reward","local":"trl.rewards.accuracy_reward","sections":[],"depth":2},{"title":"reasoning_accuracy_reward","local":"trl.rewards.reasoning_accuracy_reward","sections":[],"depth":2},{"title":"get_cosine_scaled_reward","local":"trl.rewards.get_cosine_scaled_reward","sections":[],"depth":2},{"title":"think_format_reward","local":"trl.rewards.think_format_reward","sections":[],"depth":2},{"title":"get_repetition_penalty_reward","local":"trl.rewards.get_repetition_penalty_reward","sections":[],"depth":2},{"title":"get_soft_overlong_punishment","local":"trl.rewards.get_soft_overlong_punishment","sections":[],"depth":2}],"depth":1}';var os=U('<meta name="hf:doc:metadata"/>'),d=U("<p>Example:</p> <!>",1),cs=U(`<p></p> <!> <!> <p>This module contains some useful reward functions, primarily intended for use with the <a href="/docs/trl/pr_5313/en/gspo_token#trl.GRPOTrainer">GRPOTrainer</a> and <a href="/docs/trl/pr_5313/en/rloo_trainer#trl.RLOOTrainer">RLOOTrainer</a>.</p> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Reward function that checks if the completion matches the ground truth.</p> <ul><li>If both gold and prediction are parseable → use math verification.</li> <li>If gold is not parseable → return <code>None</code> to skip the example.</li></ul> <!></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Reward function that removes the reasoning content and checks if the final answer matches the ground truth.</p> <ul><li>If both gold and prediction are parseable → use math verification.</li> <li>If gold is not parseable → return <code>None</code> to skip the example.</li></ul> <!></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Reward function that scales a correctness reward by the completion length following a cosine schedule, to favor
concise reasoning. Reference: Appendix C.1 of the “Demystifying Long Chain-of-Thought Reasoning” paper
(<a href="https://huggingface.co/papers/2502.03373" rel="nofollow">https://huggingface.co/papers/2502.03373</a>).</p> <p>Correctness is determined by math verification (as in <a href="/docs/trl/pr_5313/en/rewards#trl.rewards.accuracy_reward">accuracy_reward()</a>), and the length is the number
of completion tokens. The reward interpolates along a cosine schedule between a short-completion and a
long-completion bound: <!></p> <p>For a <strong>correct</strong> completion, $(v<em>&#123;\\min}, v</em>&#123;\\max}) = (\\texttt&#123;min_value_correct}, \\texttt&#123;max_value_correct})$,
so a shorter completion is rewarded more. For a <strong>wrong</strong> completion, the bounds are swapped to
$(v<em>&#123;\\min}, v</em>&#123;\\max}) = (\\texttt&#123;max_value_wrong}, \\texttt&#123;min_value_wrong})$, so a shorter completion is
penalized more (a longer wrong completion is penalized less, preserving exploration). When the gold solution is not
parseable, the example is skipped (reward <code>None</code>), as in <a href="/docs/trl/pr_5313/en/rewards#trl.rewards.accuracy_reward">accuracy_reward()</a>.</p> <!></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Reward function that checks if the reasoning process is enclosed within <code>"&lt;think&gt;"</code> and <code>"&lt;/think&gt;"</code> tags. The
function returns a reward of 1.0 if the format is correct, otherwise 0.0.</p> <!></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Reward function that penalizes repeated n-grams in a completion, used to discourage degenerate, repetitive text
(a common failure mode and reward-hacking strategy when length- or format-shaping rewards are used). Reference:
Appendix C.2 of the “Demystifying Long Chain-of-Thought Reasoning” paper (<a href="https://huggingface.co/papers/2502.03373" rel="nofollow">https://huggingface.co/papers/2502.03373</a>).</p> <p>The penalty is proportional to the fraction of repeated n-grams in the completion: <!></p> <p>where <!> is <code>max_penalty</code>. A completion with no repeated n-gram gets a reward of <code>0.0</code>, while a fully repetitive
one approaches <code>max_penalty</code>. The n-grams are computed over the completion token ids (the paper applies the penalty
to repeated tokens), so the reward is tokenizer-defined and language-agnostic. Completions with fewer than <code>ngram_size</code> tokens get a reward of <code>0.0</code>.</p> <!></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Reward function that penalizes overlong completions. It is used to penalize overlong completions, but not to reward
shorter completions. Reference: Eq. (13) from the DAPO paper (<a href="https://huggingface.co/papers/2503.14476" rel="nofollow">https://huggingface.co/papers/2503.14476</a>) <!></p> <!></div> <!> <p></p>`,1);function us(V,B){ps(B,!1),rs(()=>{new URLSearchParams(window.location.search).get("fw")}),as();var T=cs();ns("idkh17",n=>{var p=os();ls(p,"content",ms),r(n,p)});var x=s(m(T),2);ts(x,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var I=s(x,2);i(I,{title:"Reward Functions",local:"reward-functions",headingTag:"h1"});var C=s(I,4);i(C,{title:"accuracy_reward",local:"trl.rewards.accuracy_reward",headingTag:"h2"});var y=s(C,2),z=e(y);o(z,{name:"trl.rewards.accuracy_reward",anchor:"trl.rewards.accuracy_reward",source:"https://github.com/huggingface/trl/blob/vr_5313/trl/rewards/accuracy_rewards.py#L28",parameters:[{name:"completions",val:": list"},{name:"solution",val:": list"},{name:"log_extra",val:": collections.abc.Callable[[str, list], None] | None = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"trl.rewards.accuracy_reward.completions",description:`<strong>completions</strong> (<code>list[list[dict[str, str]]]</code>) &#x2014;
List of completions to be evaluated. Each completion must be a list of one message, i.e. a dictionary
containing the key <code>&quot;content&quot;</code> with the value being the text of the completion.`,name:"completions"},{anchor:"trl.rewards.accuracy_reward.solution",description:`<strong>solution</strong> (<code>list[str]</code>) &#x2014;
List of the raw-text solutions to the questions/problems/prompts.`,name:"solution"},{anchor:"trl.rewards.accuracy_reward.log_extra",description:`<strong>log_extra</strong> (<code>callable</code>, <em>optional</em>) &#x2014;
Callable to log extra columns to the completions table, provided automatically by the trainer. Defaults to
<code>None</code> to allow calling the function directly outside of a trainer (e.g., for testing).`,name:"log_extra"},{anchor:"trl.rewards.accuracy_reward.*kwargs",description:`*<strong>*kwargs</strong> &#x2014;
Additional keyword arguments. This function does not use them, but they are required in the function
signature to ensure compatibility with trainers like <a href="/docs/trl/pr_5313/en/gspo_token#trl.GRPOTrainer">GRPOTrainer</a>.`,name:"*kwargs"}]});var L=s(z,6);h(L,{anchor:"trl.rewards.accuracy_reward.example",children:(n,p)=>{var a=d(),t=s(m(a),2);c(t,{code:"ZnJvbSUyMHRybC5yZXdhcmRzJTIwaW1wb3J0JTIwYWNjdXJhY3lfcmV3YXJkJTBBJTBBc29sdXRpb25zJTIwJTNEJTIwJTVCciUyMiU1Q2ZyYWMlN0IxJTdEJTdCMyU3RCUyMiUyQyUyMHIlMjIlNUNmcmFjJTdCMSU3RCU3QjMlN0QlMjIlNUQlMEFjb21wbGV0aW9ucyUyMCUzRCUyMCU1QiUwQSUyMCUyMCUyMCUyMCU1QiU3QiUyMnJvbGUlMjIlM0ElMjAlMjJhc3Npc3RhbnQlMjIlMkMlMjAlMjJjb250ZW50JTIyJTNBJTIwciUyMk15JTIwYW5zd2VyJTIwaXMlMjAlNUNib3hlZCU3QiU1Q2ZyYWMlN0IxJTdEJTdCMyU3RCU3RCUyMiU3RCU1RCUyQyUwQSUyMCUyMCUyMCUyMCU1QiU3QiUyMnJvbGUlMjIlM0ElMjAlMjJhc3Npc3RhbnQlMjIlMkMlMjAlMjJjb250ZW50JTIyJTNBJTIwciUyMk15JTIwYW5zd2VyJTIwaXMlMjAlNUNib3hlZCU3QiU1Q2ZyYWMlN0IxJTdEJTdCMiU3RCU3RCUyMiU3RCU1RCUyQyUwQSU1RCUwQWFjY3VyYWN5X3Jld2FyZChjb21wbGV0aW9ucyUyQyUyMHNvbHV0aW9ucyk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> trl.rewards <span class="hljs-keyword">import</span> accuracy_reward
<span class="hljs-meta">&gt;&gt;&gt; </span>solutions = [<span class="hljs-string">r&quot;\\frac{1}{3}&quot;</span>, <span class="hljs-string">r&quot;\\frac{1}{3}&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>completions = [
<span class="hljs-meta">... </span> [{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;assistant&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">r&quot;My answer is \\boxed{\\frac{1}{3}}&quot;</span>}],
<span class="hljs-meta">... </span> [{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;assistant&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">r&quot;My answer is \\boxed{\\frac{1}{2}}&quot;</span>}],
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>accuracy_reward(completions, solutions)
[<span class="hljs-number">1.0</span>, <span class="hljs-number">0.0</span>]`,lang:"python",wrap:!1}),r(n,a)},$$slots:{default:!0}}),l(y);var N=s(y,2);i(N,{title:"reasoning_accuracy_reward",local:"trl.rewards.reasoning_accuracy_reward",headingTag:"h2"});var u=s(N,2),k=e(u);o(k,{name:"trl.rewards.reasoning_accuracy_reward",anchor:"trl.rewards.reasoning_accuracy_reward",source:"https://github.com/huggingface/trl/blob/vr_5313/trl/rewards/accuracy_rewards.py#L223",parameters:[{name:"completions",val:": list"},{name:"solution",val:": list"},{name:"reasoning_delimiters",val:": list[str] | None = None"},{name:"log_extra",val:": collections.abc.Callable[[str, list], None] | None = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"trl.rewards.reasoning_accuracy_reward.completions",description:`<strong>completions</strong> (<code>list[list[dict[str, str]]]</code>) &#x2014;
List of completions to be evaluated. Each completion must be a list of one message, i.e. a dictionary
containing the key <code>&quot;content&quot;</code> with the value being the text of the completion.`,name:"completions"},{anchor:"trl.rewards.reasoning_accuracy_reward.solution",description:`<strong>solution</strong> (<code>list[str]</code>) &#x2014;
List of the raw-text solutions to the questions/problems/prompts.`,name:"solution"},{anchor:"trl.rewards.reasoning_accuracy_reward.reasoning_delimiters",description:`<strong>reasoning_delimiters</strong> (<code>list[str]]</code>, <em>optional</em>) &#x2014;
List of strings indicating where the reasoning content ends. The final answer is assumed to be after the
last occurrence of any of these delimiters. If <code>None</code>, defaults to <code>[&quot;&lt;/think&gt;&quot;]</code>.`,name:"reasoning_delimiters"},{anchor:"trl.rewards.reasoning_accuracy_reward.log_extra",description:`<strong>log_extra</strong> (<code>callable</code>, <em>optional</em>) &#x2014;
Callable to log extra columns to the completions table, provided automatically by the trainer. Defaults to
<code>None</code> to allow calling the function directly outside of a trainer (e.g., for testing).`,name:"log_extra"},{anchor:"trl.rewards.reasoning_accuracy_reward.*kwargs",description:`*<strong>*kwargs</strong> &#x2014;
Additional keyword arguments. This function does not use them, but they are required in the function
signature to ensure compatibility with trainers like <a href="/docs/trl/pr_5313/en/gspo_token#trl.GRPOTrainer">GRPOTrainer</a>.`,name:"*kwargs"}]});var F=s(k,6);h(F,{anchor:"trl.rewards.reasoning_accuracy_reward.example",children:(n,p)=>{var a=d(),t=s(m(a),2);c(t,{code:"ZnJvbSUyMHRybC5yZXdhcmRzJTIwaW1wb3J0JTIwcmVhc29uaW5nX2FjY3VyYWN5X3Jld2FyZCUwQSUwQXJlYXNvbmluZ19kZWxpbWl0ZXJzJTIwJTNEJTIwJTVCJTIyJTNDJTJGdGhpbmslM0UlMjIlNUQlMEFzb2x1dGlvbnMlMjAlM0QlMjAlNUJyJTIyJTVDZnJhYyU3QjElN0QlN0IzJTdEJTIyJTJDJTIwciUyMiU1Q2ZyYWMlN0IxJTdEJTdCMyU3RCUyMiUyQyUyMHIlMjIlNUNmcmFjJTdCMSU3RCU3QjMlN0QlMjIlNUQlMEFjb21wbGV0aW9ucyUyMCUzRCUyMCU1QiUwQSUyMCUyMCUyMCUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJvbGUlMjIlM0ElMjAlMjJhc3Npc3RhbnQlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJjb250ZW50JTIyJTNBJTIwciUyMiUzQ3RoaW5rJTNFJTIwUmVhc29uaW5nJTIwY29udGVudCUyMCUzQyUyRnRoaW5rJTNFJTIwVGhlJTIwZmluYWwlMjBhbnN3ZXIlMjBpcyUyMCU1Q2JveGVkJTdCJTVDZnJhYyU3QjElN0QlN0IzJTdEJTdEJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTBBJTIwJTIwJTIwJTIwJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycm9sZSUyMiUzQSUyMCUyMmFzc2lzdGFudCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmNvbnRlbnQlMjIlM0ElMjByJTIyJTNDdGhpbmslM0UlMjBSZWFzb25pbmclMjBjb250ZW50JTIwJTNDJTJGdGhpbmslM0UlMjBUaGUlMjBmaW5hbCUyMGFuc3dlciUyMGlzJTIwJTVDYm94ZWQlN0IlNUNmcmFjJTdCMSU3RCU3QjIlN0QlN0QlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0QlMEElMjAlMjAlMjAlMjAlNUQlMkMlMEElMjAlMjAlMjAlMjAlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJyb2xlJTIyJTNBJTIwJTIyYXNzaXN0YW50JTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGVudCUyMiUzQSUyMHIlMjIlM0N0aGluayUzRSUyMFJlYXNvbmluZyUyMGNvbnRlbnQlMjB3aXRoJTIwcGFydGlhbCUyMGFuc3dlcnMlMjAlNUNib3hlZCU3QiU1Q2ZyYWMlN0IxJTdEJTdCMyU3RCU3RCUyMGJ1dCUyMG5vJTIwZmluYWwlMjBhbnN3ZXIlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0QlMEElMjAlMjAlMjAlMjAlNUQlMkMlMEElNUQlMEFyZWFzb25pbmdfYWNjdXJhY3lfcmV3YXJkKGNvbXBsZXRpb25zJTJDJTIwc29sdXRpb25zJTJDJTIwcmVhc29uaW5nX2RlbGltaXRlcnMlM0RyZWFzb25pbmdfZGVsaW1pdGVycyk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> trl.rewards <span class="hljs-keyword">import</span> reasoning_accuracy_reward
<span class="hljs-meta">&gt;&gt;&gt; </span>reasoning_delimiters = [<span class="hljs-string">&quot;&lt;/think&gt;&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>solutions = [<span class="hljs-string">r&quot;\\frac{1}{3}&quot;</span>, <span class="hljs-string">r&quot;\\frac{1}{3}&quot;</span>, <span class="hljs-string">r&quot;\\frac{1}{3}&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>completions = [
<span class="hljs-meta">... </span> [
<span class="hljs-meta">... </span> {
<span class="hljs-meta">... </span> <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;assistant&quot;</span>,
<span class="hljs-meta">... </span> <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">r&quot;&lt;think&gt; Reasoning content &lt;/think&gt; The final answer is \\boxed{\\frac{1}{3}}&quot;</span>,
<span class="hljs-meta">... </span> }
<span class="hljs-meta">... </span> ],
<span class="hljs-meta">... </span> [
<span class="hljs-meta">... </span> {
<span class="hljs-meta">... </span> <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;assistant&quot;</span>,
<span class="hljs-meta">... </span> <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">r&quot;&lt;think&gt; Reasoning content &lt;/think&gt; The final answer is \\boxed{\\frac{1}{2}}&quot;</span>,
<span class="hljs-meta">... </span> }
<span class="hljs-meta">... </span> ],
<span class="hljs-meta">... </span> [
<span class="hljs-meta">... </span> {
<span class="hljs-meta">... </span> <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;assistant&quot;</span>,
<span class="hljs-meta">... </span> <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">r&quot;&lt;think&gt; Reasoning content with partial answers \\boxed{\\frac{1}{3}} but no final answer&quot;</span>,
<span class="hljs-meta">... </span> }
<span class="hljs-meta">... </span> ],
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>reasoning_accuracy_reward(completions, solutions, reasoning_delimiters=reasoning_delimiters)
[<span class="hljs-number">1.0</span>, <span class="hljs-number">0.0</span>, <span class="hljs-number">0.0</span>]`,lang:"python",wrap:!1}),r(n,a)},$$slots:{default:!0}}),l(u);var R=s(u,2);i(R,{title:"get_cosine_scaled_reward",local:"trl.rewards.get_cosine_scaled_reward",headingTag:"h2"});var w=s(R,2),Q=e(w);o(Q,{name:"trl.rewards.get_cosine_scaled_reward",anchor:"trl.rewards.get_cosine_scaled_reward",source:"https://github.com/huggingface/trl/blob/vr_5313/trl/rewards/accuracy_rewards.py#L118",parameters:[{name:"max_len",val:": int"},{name:"min_value_wrong",val:": float = -1.0"},{name:"max_value_wrong",val:": float = -0.5"},{name:"min_value_correct",val:": float = 0.5"},{name:"max_value_correct",val:": float = 1.0"}],parametersDescription:[{anchor:"trl.rewards.get_cosine_scaled_reward.max_len",description:`<strong>max_len</strong> (<code>int</code>) &#x2014;
Maximum completion length (in tokens) used to normalize the cosine schedule, $L_{\\max}$.`,name:"max_len"},{anchor:"trl.rewards.get_cosine_scaled_reward.min_value_wrong",description:`<strong>min_value_wrong</strong> (<code>float</code>, <em>optional</em>, defaults to <code>-1.0</code>) &#x2014;
Reward of a wrong completion at the shortest length.`,name:"min_value_wrong"},{anchor:"trl.rewards.get_cosine_scaled_reward.max_value_wrong",description:`<strong>max_value_wrong</strong> (<code>float</code>, <em>optional</em>, defaults to <code>-0.5</code>) &#x2014;
Reward of a wrong completion at the longest length.`,name:"max_value_wrong"},{anchor:"trl.rewards.get_cosine_scaled_reward.min_value_correct",description:`<strong>min_value_correct</strong> (<code>float</code>, <em>optional</em>, defaults to <code>0.5</code>) &#x2014;
Reward of a correct completion at the longest length.`,name:"min_value_correct"},{anchor:"trl.rewards.get_cosine_scaled_reward.max_value_correct",description:`<strong>max_value_correct</strong> (<code>float</code>, <em>optional</em>, defaults to <code>1.0</code>) &#x2014;
Reward of a correct completion at the shortest length.`,name:"max_value_correct"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>
<p>A reward function that takes completions, their solutions and token ids, and returns a list of rewards
(<code>None</code> for examples with an unparseable gold solution).</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>
<p><code>Callable</code></p>
`});var v=s(Q,4),Y=s(e(v),3);g(Y,()=>`<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi>R</mi><mtext>cosine</mtext></msub><mo stretchy="false">(</mo><mi>y</mi><mo stretchy="false">)</mo><mo>=</mo><msub><mi>v</mi><mi>min</mi><mo>⁡</mo></msub><mo>+</mo><mfrac><mn>1</mn><mn>2</mn></mfrac><mo stretchy="false">(</mo><msub><mi>v</mi><mi>max</mi><mo>⁡</mo></msub><mo>−</mo><msub><mi>v</mi><mi>min</mi><mo>⁡</mo></msub><mo stretchy="false">)</mo><mrow><mo fence="true">(</mo><mn>1</mn><mo>+</mo><mi>cos</mi><mo>⁡</mo><mrow><mo fence="true">(</mo><mfrac><mrow><mi mathvariant="normal">∣</mi><mi>y</mi><mi mathvariant="normal">∣</mi></mrow><msub><mi>L</mi><mi>max</mi><mo>⁡</mo></msub></mfrac><mi>π</mi><mo fence="true">)</mo></mrow><mo fence="true">)</mo></mrow></mrow><annotation encoding="application/x-tex">
R_{\\text{cosine}}(y) = v_{\\min} + \\frac{1}{2}(v_{\\max} - v_{\\min})\\left(1 + \\cos\\left(\\frac{|y|}{L_{\\max}}\\pi\\right)\\right)
</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0077em;">R</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3175em;"><span style="top:-2.55em;margin-left:-0.0077em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord text mtight"><span class="mord mtight">cosine</span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.0359em;">y</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.7333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0359em;">v</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3175em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mop mtight"><span class="mtight">m</span><span class="mtight">i</span><span class="mtight">n</span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">2</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">1</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0359em;">v</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.1514em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mop mtight"><span class="mtight">m</span><span class="mtight">a</span><span class="mtight">x</span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:2.4em;vertical-align:-0.95em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0359em;">v</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3175em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mop mtight"><span class="mtight">m</span><span class="mtight">i</span><span class="mtight">n</span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mclose">)</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="minner"><span class="mopen delimcenter" style="top:0em;"><span class="delimsizing size3">(</span></span><span class="mord">1</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mop">cos</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="minner"><span class="mopen delimcenter" style="top:0em;"><span class="delimsizing size3">(</span></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.427em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal">L</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.1514em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mop mtight"><span class="mtight">m</span><span class="mtight">a</span><span class="mtight">x</span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">∣</span><span class="mord mathnormal" style="margin-right:0.0359em;">y</span><span class="mord">∣</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.836em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mord mathnormal" style="margin-right:0.0359em;">π</span><span class="mclose delimcenter" style="top:0em;"><span class="delimsizing size3">)</span></span></span><span class="mclose delimcenter" style="top:0em;"><span class="delimsizing size3">)</span></span></span></span></span></span></span>`),l(v);var D=s(v,4);h(D,{anchor:"trl.rewards.get_cosine_scaled_reward.example",children:(n,p)=>{var a=d(),t=s(m(a),2);c(t,{code:"ZnJvbSUyMHRybC5yZXdhcmRzJTIwaW1wb3J0JTIwZ2V0X2Nvc2luZV9zY2FsZWRfcmV3YXJkJTBBJTBBY29zaW5lX3NjYWxlZF9yZXdhcmQlMjAlM0QlMjBnZXRfY29zaW5lX3NjYWxlZF9yZXdhcmQobWF4X2xlbiUzRDEwMCklMEFjb21wbGV0aW9ucyUyMCUzRCUyMCU1QiU1QiU3QiUyMmNvbnRlbnQlMjIlM0ElMjByJTIyJTVDYm94ZWQlN0IlNUNmcmFjJTdCMSU3RCU3QjMlN0QlN0QlMjIlN0QlNUQlMkMlMjAlNUIlN0IlMjJjb250ZW50JTIyJTNBJTIwciUyMiU1Q2JveGVkJTdCJTVDZnJhYyU3QjElN0QlN0IyJTdEJTdEJTIyJTdEJTVEJTVEJTBBc29sdXRpb24lMjAlM0QlMjAlNUJyJTIyJTVDZnJhYyU3QjElN0QlN0IzJTdEJTIyJTJDJTIwciUyMiU1Q2ZyYWMlN0IxJTdEJTdCMyU3RCUyMiU1RCUwQWNvbXBsZXRpb25faWRzJTIwJTNEJTIwJTVCJTVCMSU1RCUyMColMjA1MCUyQyUyMCU1QjElNUQlMjAqJTIwNTAlNUQlMjAlMjAlMjMlMjBib3RoJTIwY29tcGxldGlvbnMlMjBhcmUlMjA1MCUyMHRva2VucyUyQyUyMGhhbGYlMjBvZiUyMG1heF9sZW4lMEFjb3NpbmVfc2NhbGVkX3Jld2FyZChjb21wbGV0aW9ucyUyQyUyMHNvbHV0aW9uJTJDJTIwY29tcGxldGlvbl9pZHMp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> trl.rewards <span class="hljs-keyword">import</span> get_cosine_scaled_reward
<span class="hljs-meta">&gt;&gt;&gt; </span>cosine_scaled_reward = get_cosine_scaled_reward(max_len=<span class="hljs-number">100</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>completions = [[{<span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">r&quot;\\boxed{\\frac{1}{3}}&quot;</span>}], [{<span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">r&quot;\\boxed{\\frac{1}{2}}&quot;</span>}]]
<span class="hljs-meta">&gt;&gt;&gt; </span>solution = [<span class="hljs-string">r&quot;\\frac{1}{3}&quot;</span>, <span class="hljs-string">r&quot;\\frac{1}{3}&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>completion_ids = [[<span class="hljs-number">1</span>] * <span class="hljs-number">50</span>, [<span class="hljs-number">1</span>] * <span class="hljs-number">50</span>] <span class="hljs-comment"># both completions are 50 tokens, half of max_len</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>cosine_scaled_reward(completions, solution, completion_ids)
[<span class="hljs-number">0.75</span>, -<span class="hljs-number">0.75</span>]`,lang:"python",wrap:!1}),r(n,a)},$$slots:{default:!0}}),l(w);var A=s(w,2);i(A,{title:"think_format_reward",local:"trl.rewards.think_format_reward",headingTag:"h2"});var M=s(A,2),Z=e(M);o(Z,{name:"trl.rewards.think_format_reward",anchor:"trl.rewards.think_format_reward",source:"https://github.com/huggingface/trl/blob/vr_5313/trl/rewards/format_rewards.py#L18",parameters:[{name:"completions",val:": list"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"trl.rewards.think_format_reward.completions",description:`<strong>completions</strong> (<code>list[list[dict[str, str]]]</code>) &#x2014;
List of completions to be evaluated. Each completion must be a list of one message, i.e. a dictionary
containing the key <code>&quot;content&quot;</code> with the value being the text of the completion.`,name:"completions"},{anchor:"trl.rewards.think_format_reward.*kwargs",description:`*<strong>*kwargs</strong> &#x2014;
Additional keyword arguments. This function does not use them, but they are required in the function
signature to ensure compatibility with trainers like <a href="/docs/trl/pr_5313/en/gspo_token#trl.GRPOTrainer">GRPOTrainer</a>.`,name:"*kwargs"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>
<p>A list of rewards, where each reward is 1.0 if the completion matches the expected format, otherwise 0.0.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>
<p><code>list[float]</code></p>
`});var S=s(Z,4);h(S,{anchor:"trl.rewards.think_format_reward.example",children:(n,p)=>{var a=d(),t=s(m(a),2);c(t,{code:"ZnJvbSUyMHRybC5yZXdhcmRzJTIwaW1wb3J0JTIwdGhpbmtfZm9ybWF0X3Jld2FyZCUwQSUwQWNvbXBsZXRpb25zJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTVCJTdCJTIyY29udGVudCUyMiUzQSUyMCUyMiUzQ3RoaW5rJTNFJTVDblRoaXMlMjBpcyUyMG15JTIwcmVhc29uaW5nLiU1Q24lM0MlMkZ0aGluayUzRSU1Q25UaGlzJTIwaXMlMjBteSUyMGFuc3dlci4lMjIlN0QlNUQlMkMlMEElMjAlMjAlMjAlMjAlNUIlN0IlMjJjb250ZW50JTIyJTNBJTIwJTIyJTNDdGhpbmslM0UlNUNuVGhpcyUyMGlzJTIwbXklMjByZWFzb25pbmcuJTVDblRoaXMlMjBpcyUyMG15JTIwYW5zd2VyLiUyMiU3RCU1RCUyQyUwQSU1RCUwQXRoaW5rX2Zvcm1hdF9yZXdhcmQoY29tcGxldGlvbnMp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> trl.rewards <span class="hljs-keyword">import</span> think_format_reward
<span class="hljs-meta">&gt;&gt;&gt; </span>completions = [
<span class="hljs-meta">... </span> [{<span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;&lt;think&gt;\\nThis is my reasoning.\\n&lt;/think&gt;\\nThis is my answer.&quot;</span>}],
<span class="hljs-meta">... </span> [{<span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;&lt;think&gt;\\nThis is my reasoning.\\nThis is my answer.&quot;</span>}],
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>think_format_reward(completions)
[<span class="hljs-number">1.0</span>, <span class="hljs-number">0.0</span>]`,lang:"python",wrap:!1}),r(n,a)},$$slots:{default:!0}}),l(M);var q=s(M,2);i(q,{title:"get_repetition_penalty_reward",local:"trl.rewards.get_repetition_penalty_reward",headingTag:"h2"});var _=s(q,2),G=e(_);o(G,{name:"trl.rewards.get_repetition_penalty_reward",anchor:"trl.rewards.get_repetition_penalty_reward",source:"https://github.com/huggingface/trl/blob/vr_5313/trl/rewards/other_rewards.py#L18",parameters:[{name:"ngram_size",val:": int = 3"},{name:"max_penalty",val:": float = -1.0"}],parametersDescription:[{anchor:"trl.rewards.get_repetition_penalty_reward.ngram_size",description:`<strong>ngram_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>3</code>) &#x2014;
Size of the token n-grams to consider.`,name:"ngram_size"},{anchor:"trl.rewards.get_repetition_penalty_reward.max_penalty",description:`<strong>max_penalty</strong> (<code>float</code>, <em>optional</em>, defaults to <code>-1.0</code>) &#x2014;
Most negative penalty, applied to a fully repetitive completion. Must be non-positive.`,name:"max_penalty"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>
<p>A reward function that takes a list of completion token ids and returns a list of penalties (each in
<code>[max_penalty, 0.0]</code>).</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>
<p><code>Callable</code></p>
`});var b=s(G,4),H=s(e(b));g(H,()=>`<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi>R</mi><mtext>repetition</mtext></msub><mo stretchy="false">(</mo><mi>y</mi><mo stretchy="false">)</mo><mo>=</mo><mrow><mo fence="true">(</mo><mn>1</mn><mo>−</mo><mfrac><mrow><mi mathvariant="normal">#</mi><mtext> </mtext><mtext>unique </mtext><mi>n</mi><mtext>-grams</mtext></mrow><mrow><mi mathvariant="normal">#</mi><mtext> </mtext><mtext>total </mtext><mi>n</mi><mtext>-grams</mtext></mrow></mfrac><mo fence="true">)</mo></mrow><mo>×</mo><mi>p</mi></mrow><annotation encoding="application/x-tex">
R_{\\text{repetition}}(y) = \\left(1 - \\frac{\\#\\,\\text{unique } n\\text{-grams}}{\\#\\,\\text{total } n\\text{-grams}}\\right) \\times p
</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1.0361em;vertical-align:-0.2861em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0077em;">R</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3175em;"><span style="top:-2.55em;margin-left:-0.0077em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord text mtight"><span class="mord mtight">repetition</span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.2861em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.0359em;">y</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.4em;vertical-align:-0.95em;"></span><span class="minner"><span class="mopen delimcenter" style="top:0em;"><span class="delimsizing size3">(</span></span><span class="mord">1</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3714em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">#</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord text"><span class="mord">total </span></span><span class="mord mathnormal">n</span><span class="mord text"><span class="mord">-grams</span></span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">#</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord text"><span class="mord">unique </span></span><span class="mord mathnormal">n</span><span class="mord text"><span class="mord">-grams</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.8804em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mclose delimcenter" style="top:0em;"><span class="delimsizing size3">)</span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">×</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.625em;vertical-align:-0.1944em;"></span><span class="mord mathnormal">p</span></span></span></span></span>`),l(b);var f=s(b,2),$=s(e(f));g($,()=>'<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi></mrow><annotation encoding="application/x-tex">p</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.625em;vertical-align:-0.1944em;"></span><span class="mord mathnormal">p</span></span></span></span>'),E(11),l(f);var P=s(f,2);h(P,{anchor:"trl.rewards.get_repetition_penalty_reward.example",children:(n,p)=>{var a=d(),t=s(m(a),2);c(t,{code:"ZnJvbSUyMHRybC5yZXdhcmRzJTIwaW1wb3J0JTIwZ2V0X3JlcGV0aXRpb25fcGVuYWx0eV9yZXdhcmQlMEElMEFyZXBldGl0aW9uX3BlbmFsdHklMjAlM0QlMjBnZXRfcmVwZXRpdGlvbl9wZW5hbHR5X3Jld2FyZChuZ3JhbV9zaXplJTNEMiUyQyUyMG1heF9wZW5hbHR5JTNELTEuMCklMEFjb21wbGV0aW9uX2lkcyUyMCUzRCUyMCU1QiU1QjElMkMlMjAyJTJDJTIwMyUyQyUyMDQlNUQlMkMlMjAlNUI1JTJDJTIwNSUyQyUyMDUlMkMlMjA1JTJDJTIwNSU1RCU1RCUwQXJlcGV0aXRpb25fcGVuYWx0eShjb21wbGV0aW9uX2lkcyk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> trl.rewards <span class="hljs-keyword">import</span> get_repetition_penalty_reward
<span class="hljs-meta">&gt;&gt;&gt; </span>repetition_penalty = get_repetition_penalty_reward(ngram_size=<span class="hljs-number">2</span>, max_penalty=-<span class="hljs-number">1.0</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>completion_ids = [[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">4</span>], [<span class="hljs-number">5</span>, <span class="hljs-number">5</span>, <span class="hljs-number">5</span>, <span class="hljs-number">5</span>, <span class="hljs-number">5</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>repetition_penalty(completion_ids)
[<span class="hljs-number">0.0</span>, -<span class="hljs-number">0.75</span>]`,lang:"python",wrap:!1}),r(n,a)},$$slots:{default:!0}}),l(_);var X=s(_,2);i(X,{title:"get_soft_overlong_punishment",local:"trl.rewards.get_soft_overlong_punishment",headingTag:"h2"});var j=s(X,2),W=e(j);o(W,{name:"trl.rewards.get_soft_overlong_punishment",anchor:"trl.rewards.get_soft_overlong_punishment",source:"https://github.com/huggingface/trl/blob/vr_5313/trl/rewards/other_rewards.py#L83",parameters:[{name:"max_completion_len",val:": int"},{name:"soft_punish_cache",val:": int"}],parametersDescription:[{anchor:"trl.rewards.get_soft_overlong_punishment.max_completion_len",description:`<strong>max_completion_len</strong> (<code>int</code>) &#x2014;
Maximum length of the completion, ( L_{\\max} ).`,name:"max_completion_len"},{anchor:"trl.rewards.get_soft_overlong_punishment.soft_punish_cache",description:`<strong>soft_punish_cache</strong> (<code>int</code>) &#x2014;
Minimum length of the completion, ( L_{\\text{cache}&#x200C;} ). If set to <code>0</code>, no minimum length is applied.`,name:"soft_punish_cache"}]});var J=s(W,2),O=s(e(J),3);g(O,()=>`<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi>R</mi><mtext>length</mtext></msub><mo stretchy="false">(</mo><mi>y</mi><mo stretchy="false">)</mo><mo>=</mo><mrow><mo fence="true">{</mo><mtable rowspacing="0.36em" columnalign="left left" columnspacing="1em"><mtr><mtd><mstyle scriptlevel="0" displaystyle="false"><mrow><mn>0</mn><mo separator="true">,</mo></mrow></mstyle></mtd><mtd><mstyle scriptlevel="0" displaystyle="false"><mrow><mi mathvariant="normal">∣</mi><mi>y</mi><mi mathvariant="normal">∣</mi><mo>≤</mo><msub><mi>L</mi><mi>max</mi><mo>⁡</mo></msub><mo>−</mo><msub><mi>L</mi><mtext>cache</mtext></msub></mrow></mstyle></mtd></mtr><mtr><mtd><mstyle scriptlevel="0" displaystyle="false"><mrow><mstyle scriptlevel="0" displaystyle="true"><mfrac><mrow><mo stretchy="false">(</mo><msub><mi>L</mi><mi>max</mi><mo>⁡</mo></msub><mo>−</mo><msub><mi>L</mi><mtext>cache</mtext></msub><mo stretchy="false">)</mo><mo>−</mo><mi mathvariant="normal">∣</mi><mi>y</mi><mi mathvariant="normal">∣</mi></mrow><msub><mi>L</mi><mtext>cache</mtext></msub></mfrac></mstyle><mo separator="true">,</mo></mrow></mstyle></mtd><mtd><mstyle scriptlevel="0" displaystyle="false"><mrow><msub><mi>L</mi><mi>max</mi><mo>⁡</mo></msub><mo>−</mo><msub><mi>L</mi><mtext>cache</mtext></msub><mo>&lt;</mo><mi mathvariant="normal">∣</mi><mi>y</mi><mi mathvariant="normal">∣</mi><mo>≤</mo><msub><mi>L</mi><mi>max</mi><mo>⁡</mo></msub></mrow></mstyle></mtd></mtr><mtr><mtd><mstyle scriptlevel="0" displaystyle="false"><mrow><mo>−</mo><mn>1</mn><mo separator="true">,</mo></mrow></mstyle></mtd><mtd><mstyle scriptlevel="0" displaystyle="false"><mrow><msub><mi>L</mi><mi>max</mi><mo>⁡</mo></msub><mo>&lt;</mo><mi mathvariant="normal">∣</mi><mi>y</mi><mi mathvariant="normal">∣</mi></mrow></mstyle></mtd></mtr></mtable></mrow></mrow><annotation encoding="application/x-tex">
R_{\\text{length}}(y) = \\begin{cases}
0, &amp; |y| \\le L_{\\max} - L_{\\text{cache}} \\\\
\\dfrac{(L_{\\max} - L_{\\text{cache}}) - |y|}{L_{\\text{cache}}}, &amp; L_{\\max} - L_{\\text{cache}} &lt; |y| \\le L_{\\max} \\\\
-1, &amp; L_{\\max} &lt; |y|
\\end{cases}
</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1.0361em;vertical-align:-0.2861em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0077em;">R</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0077em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord text mtight"><span class="mord mtight">length</span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.2861em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.0359em;">y</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:5.143em;vertical-align:-2.3215em;"></span><span class="minner"><span class="mopen"><span class="delimsizing mult"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:2.65em;"><span style="top:-1.9em;"><span class="pstrut" style="height:3.15em;"></span><span class="delimsizinginner delim-size4"><span>⎩</span></span></span><span style="top:-1.892em;"><span class="pstrut" style="height:3.15em;"></span><span style="height:0.616em;width:0.8889em;"><svg xmlns="http://www.w3.org/2000/svg" width="0.8889em" height="0.616em" style="width:0.8889em" viewBox="0 0 888.89 616" preserveAspectRatio="xMinYMin"><path d="M384 0 H504 V616 H384z M384 0 H504 V616 H384z"/></svg></span></span><span style="top:-3.15em;"><span class="pstrut" style="height:3.15em;"></span><span class="delimsizinginner delim-size4"><span>⎨</span></span></span><span style="top:-4.292em;"><span class="pstrut" style="height:3.15em;"></span><span style="height:0.616em;width:0.8889em;"><svg xmlns="http://www.w3.org/2000/svg" width="0.8889em" height="0.616em" style="width:0.8889em" viewBox="0 0 888.89 616" preserveAspectRatio="xMinYMin"><path d="M384 0 H504 V616 H384z M384 0 H504 V616 H384z"/></svg></span></span><span style="top:-4.9em;"><span class="pstrut" style="height:3.15em;"></span><span class="delimsizinginner delim-size4"><span>⎧</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:2.15em;"><span></span></span></span></span></span></span><span class="mord"><span class="mtable"><span class="col-align-l"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:2.8215em;"><span style="top:-5.2405em;"><span class="pstrut" style="height:3.427em;"></span><span class="mord"><span class="mord">0</span><span class="mpunct">,</span></span></span><span style="top:-3.3815em;"><span class="pstrut" style="height:3.427em;"></span><span class="mord"><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.427em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal">L</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord text mtight"><span class="mord mtight">cache</span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">L</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.1514em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mop mtight"><span class="mtight">m</span><span class="mtight">a</span><span class="mtight">x</span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord"><span class="mord mathnormal">L</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord text mtight"><span class="mord mtight">cache</span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord">∣</span><span class="mord mathnormal" style="margin-right:0.0359em;">y</span><span class="mord">∣</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.836em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mpunct">,</span></span></span><span style="top:-1.5375em;"><span class="pstrut" style="height:3.427em;"></span><span class="mord"><span class="mord">−</span><span class="mord">1</span><span class="mpunct">,</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:2.3215em;"><span></span></span></span></span></span><span class="arraycolsep" style="width:1em;"></span><span class="col-align-l"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:2.8215em;"><span style="top:-5.2405em;"><span class="pstrut" style="height:3.427em;"></span><span class="mord"><span class="mord">∣</span><span class="mord mathnormal" style="margin-right:0.0359em;">y</span><span class="mord">∣</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">≤</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mord"><span class="mord mathnormal">L</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.1514em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mop mtight"><span class="mtight">m</span><span class="mtight">a</span><span class="mtight">x</span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord"><span class="mord mathnormal">L</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord text mtight"><span class="mord mtight">cache</span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span><span style="top:-3.3815em;"><span class="pstrut" style="height:3.427em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal">L</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.1514em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mop mtight"><span class="mtight">m</span><span class="mtight">a</span><span class="mtight">x</span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord"><span class="mord mathnormal">L</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord text mtight"><span class="mord mtight">cache</span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">&lt;</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mord">∣</span><span class="mord mathnormal" style="margin-right:0.0359em;">y</span><span class="mord">∣</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">≤</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mord"><span class="mord mathnormal">L</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.1514em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mop mtight"><span class="mtight">m</span><span class="mtight">a</span><span class="mtight">x</span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span><span style="top:-1.5375em;"><span class="pstrut" style="height:3.427em;"></span><span class="mord"><span class="mord"><span class="mord mathnormal">L</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.1514em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mop mtight"><span class="mtight">m</span><span class="mtight">a</span><span class="mtight">x</span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">&lt;</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mord">∣</span><span class="mord mathnormal" style="margin-right:0.0359em;">y</span><span class="mord">∣</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:2.3215em;"><span></span></span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span>`),l(J);var K=s(J,2);h(K,{anchor:"trl.rewards.get_soft_overlong_punishment.example",children:(n,p)=>{var a=d(),t=s(m(a),2);c(t,{code:"ZnJvbSUyMHRybC5yZXdhcmRzJTIwaW1wb3J0JTIwZ2V0X3NvZnRfb3ZlcmxvbmdfcHVuaXNobWVudCUwQSUwQXNvZnRfb3ZlcmxvbmdfcHVuaXNobWVudCUyMCUzRCUyMGdldF9zb2Z0X292ZXJsb25nX3B1bmlzaG1lbnQobWF4X2NvbXBsZXRpb25fbGVuJTNEMTAwJTJDJTIwc29mdF9wdW5pc2hfY2FjaGUlM0QyMCklMEFjb21wbGV0aW9uX2lkcyUyMCUzRCUyMCU1QiU1QjElNUQlMjAqJTIwOTAlNUQlMjAlMjAlMjMlMjBzaW11bGF0aW5nJTIwYSUyMGNvbXBsZXRpb24lMjB3aXRoJTIwOTAlMjB0b2tlbnMuJTIwOTAlMjBpcyUyMGJldHdlZW4lMjA4MCUyMGFuZCUyMDEwMC4lMEFzb2Z0X292ZXJsb25nX3B1bmlzaG1lbnQoY29tcGxldGlvbl9pZHMpJTBBJTVCLTAuNSU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> trl.rewards <span class="hljs-keyword">import</span> get_soft_overlong_punishment
<span class="hljs-meta">&gt;&gt;&gt; </span>soft_overlong_punishment = get_soft_overlong_punishment(max_completion_len=<span class="hljs-number">100</span>, soft_punish_cache=<span class="hljs-number">20</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>completion_ids = [[<span class="hljs-number">1</span>] * <span class="hljs-number">90</span>] <span class="hljs-comment"># simulating a completion with 90 tokens. 90 is between 80 and 100.</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>soft_overlong_punishment(completion_ids)
<span class="hljs-meta">&gt;&gt;&gt; </span>[-<span class="hljs-number">0.5</span>]`,lang:"python",wrap:!1}),r(n,a)},$$slots:{default:!0}}),l(j);var ss=s(j,2);es(ss,{source:"https://github.com/huggingface/trl/blob/main/docs/source/rewards.md"}),E(2),r(V,T),is()}export{us as component};

Xet Storage Details

Size:
57.1 kB
·
Xet hash:
b338ca013d6ff8354f09aee87b020de78f920a5c2a7567f131f33c4b6eb38110

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.