trl/pr_5607/en/_app/immutable/nodes/23.a541ead1.js

import{s as Fe,n as je,o as Ie}from"../chunks/scheduler.7b731bd4.js";import{S as We,i as Ze,e as s,s as n,c as m,h as Pe,a as i,d as t,b as l,f as T,g as p,j as X,k as M,l as o,m as r,n as _,t as c,o as d,p as f}from"../chunks/index.cc268345.js";import{C as Xe,H as q,E as Ee}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.f0d99f98.js";import{D as A}from"../chunks/Docstring.03f7b462.js";import{C as Oe}from"../chunks/CodeBlock.169a125f.js";function ze(Re){let v,D,Y,K,$,ee,R,ae,J,Je="This experimental trainer, trains a model with GRPO but replaces groups (and corresponding completions) that have 0 standard deviation with groups with high rewards and standard deviation that’ve been used to train a model in prior batches.",te,U,ne,x,le,k,re,u,C,ve,y,B,he,E,Ue="Main training entry point.",be,g,G,ye,O,xe="Will save the model, so you can reload it using <code>from_pretrained()</code>.",we,z,ke="Will only save from the main process.",Ne,w,F,Te,V,Ce="Upload <code>self.model</code> and <code>self.processing_class</code> to the 🤗 model hub on the repo <code>self.args.hub_model_id</code>.",oe,j,se,h,I,Me,Q,Be=`New Parameters:
replay_buffer_size (<code>int</code>, <em>optional</em>, defaults to <code>0</code>):
A cache that stores the rollouts with the highest advantage scores and variance per group. If a new
group has 0 variance, it is replaced with a group sampled from the replay buffer.`,ie,W,me,b,Z,$e,S,Ge="A simple replay buffer to store and sample previously seen rollouts.",pe,P,_e,L,ce;return $=new Xe({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),R=new q({props:{title:"GRPO With Replay Buffer",local:"grpo-with-replay-buffer",headingTag:"h1"}}),U=new q({props:{title:"Usage",local:"usage",headingTag:"h2"}}),x=new Oe({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJsLmV4cGVyaW1lbnRhbC5ncnBvX3dpdGhfcmVwbGF5X2J1ZmZlciUyMGltcG9ydCUyMEdSUE9XaXRoUmVwbGF5QnVmZmVyQ29uZmlnJTJDJTIwR1JQT1dpdGhSZXBsYXlCdWZmZXJUcmFpbmVyJTBBZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJ0cmwtaW50ZXJuYWwtdGVzdGluZyUyRnplbiUyMiUyQyUyMCUyMnN0YW5kYXJkX3Byb21wdF9vbmx5JTIyJTJDJTIwc3BsaXQlM0QlMjJ0cmFpbiUyMiklMEElMEElMjMlMjBHdWFyYW50ZWUlMjB0aGF0JTIwc29tZSUyMHJld2FyZHMlMjBoYXZlJTIwMCUyMHN0ZCUwQWRlZiUyMGN1c3RvbV9yZXdhcmRfZnVuYyhjb21wbGV0aW9ucyUyQyUyMCoqa3dhcmdzKSUzQSUwQSUyMCUyMCUyMCUyMGlmJTIwdG9yY2gucmFuZCgxKS5pdGVtKCklMjAlM0MlMjAwLjI1JTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwJTVCMCU1RCUyMColMjBsZW4oY29tcGxldGlvbnMpJTIwJTIwJTIzJTIwc2ltdWxhdGUlMjBzb21lJTIwTm9uZSUyMHJld2FyZHMlMEElMjAlMjAlMjAlMjBlbHNlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwdG9yY2gucmFuZChsZW4oY29tcGxldGlvbnMpKS50b2xpc3QoKSUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBHUlBPV2l0aFJlcGxheUJ1ZmZlckNvbmZpZyglMEElMjAlMjAlMjAlMjBvdXRwdXRfZGlyJTNEJTIyLiUyRnRtcCUyMiUyQyUwQSUyMCUyMCUyMCUyMGxlYXJuaW5nX3JhdGUlM0QxZS00JTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTNENCUyQyUwQSUyMCUyMCUyMCUyMG51bV9nZW5lcmF0aW9ucyUzRDQlMkMlMEElMjAlMjAlMjAlMjBtYXhfY29tcGxldGlvbl9sZW5ndGglM0Q4JTJDJTBBJTIwJTIwJTIwJTIwcmVwbGF5X2J1ZmZlcl9zaXplJTNEOCUyQyUwQSUyMCUyMCUyMCUyMHJlcG9ydF90byUzRCUyMm5vbmUlMjIlMkMlMEEpJTBBJTBBdHJhaW5lciUyMCUzRCUyMEdSUE9XaXRoUmVwbGF5QnVmZmVyVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRCUyMnRybC1pbnRlcm5hbC10ZXN0aW5nJTJGdGlueS1Rd2VuMkZvckNhdXNhbExNLTIuNSUyMiUyQyUwQSUyMCUyMCUyMCUyMHJld2FyZF9mdW5jcyUzRCU1QmN1c3RvbV9yZXdhcmRfZnVuYyU1RCUyQyUwQSUyMCUyMCUyMCUyMGFyZ3MlM0R0cmFpbmluZ19hcmdzJTJDJTBBJTIwJTIwJTIwJTIwdHJhaW5fZGF0YXNldCUzRGRhdGFzZXQlMkMlMEEpJTBBJTBBcHJldmlvdXNfdHJhaW5hYmxlX3BhcmFtcyUyMCUzRCUyMCU3Qm4lM0ElMjBwYXJhbS5jbG9uZSgpJTIwZm9yJTIwbiUyQyUyMHBhcmFtJTIwaW4lMjB0cmFpbmVyLm1vZGVsLm5hbWVkX3BhcmFtZXRlcnMoKSU3RCUwQSUwQXRyYWluZXIudHJhaW4oKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> trl.experimental.grpo_with_replay_buffer <span class="hljs-keyword">import</span> GRPOWithReplayBufferConfig, GRPOWithReplayBufferTrainer
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

dataset = load_dataset(<span class="hljs-string">&quot;trl-internal-testing/zen&quot;</span>, <span class="hljs-string">&quot;standard_prompt_only&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)

<span class="hljs-comment"># Guarantee that some rewards have 0 std</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">custom_reward_func</span>(<span class="hljs-params">completions, **kwargs</span>):
    <span class="hljs-keyword">if</span> torch.rand(<span class="hljs-number">1</span>).item() &lt; <span class="hljs-number">0.25</span>:
        <span class="hljs-keyword">return</span> [<span class="hljs-number">0</span>] * <span class="hljs-built_in">len</span>(completions)  <span class="hljs-comment"># simulate some None rewards</span>
    <span class="hljs-keyword">else</span>:
        <span class="hljs-keyword">return</span> torch.rand(<span class="hljs-built_in">len</span>(completions)).tolist()

training_args = GRPOWithReplayBufferConfig(
    output_dir=<span class="hljs-string">&quot;./tmp&quot;</span>,
    learning_rate=<span class="hljs-number">1e-4</span>,
    per_device_train_batch_size=<span class="hljs-number">4</span>,
    num_generations=<span class="hljs-number">4</span>,
    max_completion_length=<span class="hljs-number">8</span>,
    replay_buffer_size=<span class="hljs-number">8</span>,
    report_to=<span class="hljs-string">&quot;none&quot;</span>,
)

trainer = GRPOWithReplayBufferTrainer(
    model=<span class="hljs-string">&quot;trl-internal-testing/tiny-Qwen2ForCausalLM-2.5&quot;</span>,
    reward_funcs=[custom_reward_func],
    args=training_args,
    train_dataset=dataset,
)

previous_trainable_params = {n: param.clone() <span class="hljs-keyword">for</span> n, param <span class="hljs-keyword">in</span> trainer.model.named_parameters()}

trainer.train()`,wrap:!1}}),k=new q({props:{title:"GRPOWithReplayBufferTrainer",local:"trl.experimental.grpo_with_replay_buffer.GRPOWithReplayBufferTrainer",headingTag:"h2"}}),C=new A({props:{name:"class trl.experimental.grpo_with_replay_buffer.GRPOWithReplayBufferTrainer",anchor:"trl.experimental.grpo_with_replay_buffer.GRPOWithReplayBufferTrainer",parameters:[{name:"args",val:": trl.experimental.grpo_with_replay_buffer.grpo_with_replay_buffer_config.GRPOWithReplayBufferConfig | None = None"},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/trl/blob/vr_5607/trl/experimental/grpo_with_replay_buffer/grpo_with_replay_buffer_trainer.py#L60"}}),B=new A({props:{name:"train",anchor:"trl.experimental.grpo_with_replay_buffer.GRPOWithReplayBufferTrainer.train",parameters:[{name:"resume_from_checkpoint",val:": str | bool | None = None"},{name:"trial",val:": optuna.Trial | dict[str, Any] | None = None"},{name:"ignore_keys_for_eval",val:": list[str] | None = None"}],parametersDescription:[{anchor:"trl.experimental.grpo_with_replay_buffer.GRPOWithReplayBufferTrainer.train.resume_from_checkpoint",description:`<strong>resume_from_checkpoint</strong> (<code>str</code> or <code>bool</code>, <em>optional</em>) &#x2014;
If a <code>str</code>, local path to a saved checkpoint as saved by a previous instance of <code>Trainer</code>. If a
<code>bool</code> and equals <code>True</code>, load the last checkpoint in <em>args.output_dir</em> as saved by a previous instance
of <code>Trainer</code>. If present, training will resume from the model/optimizer/scheduler states loaded here.`,name:"resume_from_checkpoint"},{anchor:"trl.experimental.grpo_with_replay_buffer.GRPOWithReplayBufferTrainer.train.trial",description:`<strong>trial</strong> (<code>optuna.Trial</code> or <code>dict[str, Any]</code>, <em>optional</em>) &#x2014;
The trial run or the hyperparameter dictionary for hyperparameter search.`,name:"trial"},{anchor:"trl.experimental.grpo_with_replay_buffer.GRPOWithReplayBufferTrainer.train.ignore_keys_for_eval",description:`<strong>ignore_keys_for_eval</strong> (<code>list[str]</code>, <em>optional</em>) &#x2014;
A list of keys in the output of your model (if it is a dictionary) that should be ignored when
gathering predictions for evaluation during the training.`,name:"ignore_keys_for_eval"}],source:"https://github.com/huggingface/trl/blob/vr_5607/transformers/trainer.py#L1323",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>Object containing the global step count, training loss, and metrics.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>~trainer_utils.TrainOutput</code></p>
`}}),G=new A({props:{name:"save_model",anchor:"trl.experimental.grpo_with_replay_buffer.GRPOWithReplayBufferTrainer.save_model",parameters:[{name:"output_dir",val:": str | None = None"},{name:"_internal_call",val:": bool = False"}],source:"https://github.com/huggingface/trl/blob/vr_5607/transformers/trainer.py#L3746"}}),F=new A({props:{name:"push_to_hub",anchor:"trl.experimental.grpo_with_replay_buffer.GRPOWithReplayBufferTrainer.push_to_hub",parameters:[{name:"commit_message",val:": str | None = 'End of training'"},{name:"blocking",val:": bool = True"},{name:"token",val:": str | None = None"},{name:"revision",val:": str | None = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"trl.experimental.grpo_with_replay_buffer.GRPOWithReplayBufferTrainer.push_to_hub.commit_message",description:`<strong>commit_message</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;End of training&quot;</code>) &#x2014;
Message to commit while pushing.`,name:"commit_message"},{anchor:"trl.experimental.grpo_with_replay_buffer.GRPOWithReplayBufferTrainer.push_to_hub.blocking",description:`<strong>blocking</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether the function should return only when the <code>git push</code> has finished.`,name:"blocking"},{anchor:"trl.experimental.grpo_with_replay_buffer.GRPOWithReplayBufferTrainer.push_to_hub.token",description:`<strong>token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Token with write permission to overwrite Trainer&#x2019;s original args.`,name:"token"},{anchor:"trl.experimental.grpo_with_replay_buffer.GRPOWithReplayBufferTrainer.push_to_hub.revision",description:`<strong>revision</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The git revision to commit from. Defaults to the head of the &#x201C;main&#x201D; branch.`,name:"revision"},{anchor:"trl.experimental.grpo_with_replay_buffer.GRPOWithReplayBufferTrainer.push_to_hub.kwargs",description:`<strong>kwargs</strong> (<code>dict[str, Any]</code>, <em>optional</em>) &#x2014;
Additional keyword arguments passed along to <code>~Trainer.create_model_card</code>.`,name:"kwargs"}],source:"https://github.com/huggingface/trl/blob/vr_5607/transformers/trainer.py#L3993",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The URL of the repository where the model was pushed if <code>blocking=False</code>, or a <code>Future</code> object tracking the
progress of the commit if <code>blocking=True</code>.</p>
`}}),j=new q({props:{title:"GRPOWithReplayBufferConfig",local:"trl.experimental.grpo_with_replay_buffer.GRPOWithReplayBufferConfig",headingTag:"h2"}}),I=new A({props:{name:"class trl.experimental.grpo_with_replay_buffer.GRPOWithReplayBufferConfig",anchor:"trl.experimental.grpo_with_replay_buffer.GRPOWithReplayBufferConfig",parameters:[{name:"output_dir",val:": str | None = None"},{name:"per_device_train_batch_size",val:": int = 8"},{name:"num_train_epochs",val:": float = 3.0"},{name:"max_steps",val:": int = -1"},{name:"learning_rate",val:": float = 1e-06"},{name:"lr_scheduler_type",val:": transformers.trainer_utils.SchedulerType | str = 'linear'"},{name:"lr_scheduler_kwargs",val:": dict | str | None = None"},{name:"warmup_steps",val:": float = 0"},{name:"optim",val:": transformers.training_args.OptimizerNames | str = 'adamw_torch_fused'"},{name:"optim_args",val:": str | None = None"},{name:"weight_decay",val:": float = 0.0"},{name:"adam_beta1",val:": float = 0.9"},{name:"adam_beta2",val:": float = 0.999"},{name:"adam_epsilon",val:": float = 1e-08"},{name:"optim_target_modules",val:": None | str | list[str] = None"},{name:"gradient_accumulation_steps",val:": int = 1"},{name:"average_tokens_across_devices",val:": bool = True"},{name:"max_grad_norm",val:": float = 1.0"},{name:"label_smoothing_factor",val:": float = 0.0"},{name:"bf16",val:": bool | None = None"},{name:"fp16",val:": bool = False"},{name:"bf16_full_eval",val:": bool = False"},{name:"fp16_full_eval",val:": bool = False"},{name:"tf32",val:": bool | None = None"},{name:"gradient_checkpointing",val:": bool = True"},{name:"gradient_checkpointing_kwargs",val:": dict[str, typing.Any] | str | None = None"},{name:"torch_compile",val:": bool = False"},{name:"torch_compile_backend",val:": str | None = None"},{name:"torch_compile_mode",val:": str | None = None"},{name:"use_liger_kernel",val:": bool = False"},{name:"liger_kernel_config",val:": dict[str, bool] | None = None"},{name:"use_cache",val:": bool = False"},{name:"neftune_noise_alpha",val:": float | None = None"},{name:"torch_empty_cache_steps",val:": int | None = None"},{name:"auto_find_batch_size",val:": bool = False"},{name:"logging_strategy",val:": transformers.trainer_utils.IntervalStrategy | str = 'steps'"},{name:"logging_steps",val:": float = 10"},{name:"logging_first_step",val:": bool = False"},{name:"log_on_each_node",val:": bool = True"},{name:"logging_nan_inf_filter",val:": bool = True"},{name:"include_num_input_tokens_seen",val:": str | bool = 'no'"},{name:"log_level",val:": str = 'passive'"},{name:"log_level_replica",val:": str = 'warning'"},{name:"disable_tqdm",val:": bool | None = None"},{name:"report_to",val:": None | str | list[str] = 'none'"},{name:"run_name",val:": str | None = None"},{name:"project",val:": str = 'huggingface'"},{name:"trackio_space_id",val:": str | None = 'trackio'"},{name:"eval_strategy",val:": transformers.trainer_utils.IntervalStrategy | str = 'no'"},{name:"eval_steps",val:": float | None = None"},{name:"eval_delay",val:": float = 0"},{name:"per_device_eval_batch_size",val:": int = 8"},{name:"prediction_loss_only",val:": bool = False"},{name:"eval_on_start",val:": bool = False"},{name:"eval_do_concat_batches",val:": bool = True"},{name:"eval_use_gather_object",val:": bool = False"},{name:"eval_accumulation_steps",val:": int | None = None"},{name:"include_for_metrics",val:": list = <factory>"},{name:"batch_eval_metrics",val:": bool = False"},{name:"save_only_model",val:": bool = False"},{name:"save_strategy",val:": transformers.trainer_utils.SaveStrategy | str = 'steps'"},{name:"save_steps",val:": float = 500"},{name:"save_on_each_node",val:": bool = False"},{name:"save_total_limit",val:": int | None = None"},{name:"enable_jit_checkpoint",val:": bool = False"},{name:"push_to_hub",val:": bool = False"},{name:"hub_token",val:": str | None = None"},{name:"hub_private_repo",val:": bool | None = None"},{name:"hub_model_id",val:": str | None = None"},{name:"hub_strategy",val:": transformers.trainer_utils.HubStrategy | str = 'every_save'"},{name:"hub_always_push",val:": bool = False"},{name:"hub_revision",val:": str | None = None"},{name:"load_best_model_at_end",val:": bool = False"},{name:"metric_for_best_model",val:": str | None = None"},{name:"greater_is_better",val:": bool | None = None"},{name:"ignore_data_skip",val:": bool = False"},{name:"restore_callback_states_from_checkpoint",val:": bool = False"},{name:"full_determinism",val:": bool = False"},{name:"seed",val:": int = 42"},{name:"data_seed",val:": int | None = None"},{name:"use_cpu",val:": bool = False"},{name:"accelerator_config",val:": dict | str | None = None"},{name:"parallelism_config",val:": accelerate.parallelism_config.ParallelismConfig | None = None"},{name:"dataloader_drop_last",val:": bool = False"},{name:"dataloader_num_workers",val:": int = 0"},{name:"dataloader_pin_memory",val:": bool = True"},{name:"dataloader_persistent_workers",val:": bool = False"},{name:"dataloader_prefetch_factor",val:": int | None = None"},{name:"remove_unused_columns",val:": bool | None = False"},{name:"label_names",val:": list[str] | None = None"},{name:"train_sampling_strategy",val:": str = 'random'"},{name:"length_column_name",val:": str = 'length'"},{name:"ddp_find_unused_parameters",val:": bool | None = None"},{name:"ddp_bucket_cap_mb",val:": int | None = None"},{name:"ddp_broadcast_buffers",val:": bool | None = None"},{name:"ddp_backend",val:": str | None = None"},{name:"ddp_timeout",val:": int = 1800"},{name:"fsdp",val:": list[transformers.trainer_utils.FSDPOption] | str | None = None"},{name:"fsdp_config",val:": dict[str, typing.Any] | str | None = None"},{name:"deepspeed",val:": dict | str | None = None"},{name:"debug",val:": str | list[transformers.debug_utils.DebugOption] = ''"},{name:"skip_memory_metrics",val:": bool = True"},{name:"do_train",val:": bool = False"},{name:"do_eval",val:": bool = False"},{name:"do_predict",val:": bool = False"},{name:"resume_from_checkpoint",val:": str | None = None"},{name:"warmup_ratio",val:": float | None = None"},{name:"logging_dir",val:": str | None = None"},{name:"local_rank",val:": int = -1"},{name:"model_init_kwargs",val:": dict[str, typing.Any] | str | None = None"},{name:"disable_dropout",val:": bool = False"},{name:"cast_lm_head_to_fp32",val:": bool = False"},{name:"num_generations",val:": int | None = 8"},{name:"num_generations_eval",val:": int | None = None"},{name:"max_completion_length",val:": int | None = 256"},{name:"ds3_gather_for_generation",val:": bool = True"},{name:"shuffle_dataset",val:": bool | None = True"},{name:"pad_to_multiple_of",val:": int | None = None"},{name:"generation_batch_size",val:": int | None = None"},{name:"steps_per_generation",val:": int | None = None"},{name:"temperature",val:": float = 1.0"},{name:"top_p",val:": float = 1.0"},{name:"top_k",val:": int = 0"},{name:"min_p",val:": float | None = None"},{name:"generation_kwargs",val:": dict | None = None"},{name:"chat_template_kwargs",val:": dict | None = None"},{name:"repetition_penalty",val:": float = 1.0"},{name:"cache_implementation",val:": str | None = None"},{name:"use_vllm",val:": bool = False"},{name:"vllm_mode",val:": str = 'colocate'"},{name:"vllm_model_impl",val:": str = 'vllm'"},{name:"vllm_enable_sleep_mode",val:": bool = False"},{name:"vllm_structured_outputs_regex",val:": str | None = None"},{name:"vllm_server_base_url",val:": str | None = None"},{name:"vllm_server_host",val:": str = '0.0.0.0'"},{name:"vllm_server_port",val:": int = 8000"},{name:"vllm_server_timeout",val:": float = 240.0"},{name:"vllm_group_port",val:": int = 51216"},{name:"vllm_gpu_memory_utilization",val:": float = 0.3"},{name:"vllm_max_model_length",val:": int | None = None"},{name:"vllm_tensor_parallel_size",val:": int = 1"},{name:"beta",val:": float = 0.0"},{name:"num_iterations",val:": int = 1"},{name:"epsilon",val:": float = 0.2"},{name:"delta",val:": float | None = None"},{name:"epsilon_high",val:": float | None = None"},{name:"sapo_temperature_neg",val:": float = 1.05"},{name:"sapo_temperature_pos",val:": float = 1.0"},{name:"vespo_k_pos",val:": float = 2.0"},{name:"vespo_lambda_pos",val:": float = 3.0"},{name:"vespo_k_neg",val:": float = 3.0"},{name:"vespo_lambda_neg",val:": float = 2.0"},{name:"importance_sampling_level",val:": str = 'token'"},{name:"reward_weights",val:": list[float] | None = None"},{name:"multi_objective_aggregation",val:": str = 'sum_then_normalize'"},{name:"scale_rewards",val:": str = 'group'"},{name:"loss_type",val:": str = 'dapo'"},{name:"mask_truncated_completions",val:": bool = False"},{name:"sync_ref_model",val:": bool = False"},{name:"ref_model_mixup_alpha",val:": float = 0.6"},{name:"ref_model_sync_steps",val:": int = 512"},{name:"top_entropy_quantile",val:": float = 1.0"},{name:"max_tool_calling_iterations",val:": int | None = None"},{name:"vllm_importance_sampling_correction",val:": bool = True"},{name:"vllm_importance_sampling_mode",val:": str = 'sequence_mask'"},{name:"vllm_importance_sampling_cap",val:": float = 3.0"},{name:"off_policy_mask_threshold",val:": float | None = None"},{name:"use_bias_correction_kl",val:": bool = False"},{name:"log_completions",val:": bool = False"},{name:"num_completions_to_print",val:": int | None = None"},{name:"log_unique_prompts",val:": bool = False"},{name:"log_completions_hub_repo",val:": str | None = None"},{name:"use_transformers_paged",val:": bool = False"},{name:"replay_buffer_size",val:": int = 64"}],source:"https://github.com/huggingface/trl/blob/vr_5607/trl/experimental/grpo_with_replay_buffer/grpo_with_replay_buffer_config.py#L21"}}),W=new q({props:{title:"ReplayBuffer",local:"trl.experimental.grpo_with_replay_buffer.ReplayBuffer",headingTag:"h2"}}),Z=new A({props:{name:"class trl.experimental.grpo_with_replay_buffer.ReplayBuffer",anchor:"trl.experimental.grpo_with_replay_buffer.ReplayBuffer",parameters:[{name:"max_size",val:": int"}],source:"https://github.com/huggingface/trl/blob/vr_5607/trl/experimental/grpo_with_replay_buffer/grpo_with_replay_buffer_trainer.py#L28"}}),P=new Ee({props:{source:"https://github.com/huggingface/trl/blob/main/docs/source/grpo_with_replay_buffer.md"}}),{c(){v=s("meta"),D=n(),Y=s("p"),K=n(),m($.$$.fragment),ee=n(),m(R.$$.fragment),ae=n(),J=s("p"),J.textContent=Je,te=n(),m(U.$$.fragment),ne=n(),m(x.$$.fragment),le=n(),m(k.$$.fragment),re=n(),u=s("div"),m(C.$$.fragment),ve=n(),y=s("div"),m(B.$$.fragment),he=n(),E=s("p"),E.textContent=Ue,be=n(),g=s("div"),m(G.$$.fragment),ye=n(),O=s("p"),O.innerHTML=xe,we=n(),z=s("p"),z.textContent=ke,Ne=n(),w=s("div"),m(F.$$.fragment),Te=n(),V=s("p"),V.innerHTML=Ce,oe=n(),m(j.$$.fragment),se=n(),h=s("div"),m(I.$$.fragment),Me=n(),Q=s("p"),Q.innerHTML=Be,ie=n(),m(W.$$.fragment),me=n(),b=s("div"),m(Z.$$.fragment),$e=n(),S=s("p"),S.textContent=Ge,pe=n(),m(P.$$.fragment),_e=n(),L=s("p"),this.h()},l(e){const a=Pe("svelte-u9bgzb",document.head);v=i(a,"META",{name:!0,content:!0}),a.forEach(t),D=l(e),Y=i(e,"P",{}),T(Y).forEach(t),K=l(e),p($.$$.fragment,e),ee=l(e),p(R.$$.fragment,e),ae=l(e),J=i(e,"P",{"data-svelte-h":!0}),X(J)!=="svelte-avq1w"&&(J.textContent=Je),te=l(e),p(U.$$.fragment,e),ne=l(e),p(x.$$.fragment,e),le=l(e),p(k.$$.fragment,e),re=l(e),u=i(e,"DIV",{class:!0});var N=T(u);p(C.$$.fragment,N),ve=l(N),y=i(N,"DIV",{class:!0});var de=T(y);p(B.$$.fragment,de),he=l(de),E=i(de,"P",{"data-svelte-h":!0}),X(E)!=="svelte-1cilnet"&&(E.textContent=Ue),de.forEach(t),be=l(N),g=i(N,"DIV",{class:!0});var H=T(g);p(G.$$.fragment,H),ye=l(H),O=i(H,"P",{"data-svelte-h":!0}),X(O)!=="svelte-r8h4ov"&&(O.innerHTML=xe),we=l(H),z=i(H,"P",{"data-svelte-h":!0}),X(z)!=="svelte-1e6bius"&&(z.textContent=ke),H.forEach(t),Ne=l(N),w=i(N,"DIV",{class:!0});var fe=T(w);p(F.$$.fragment,fe),Te=l(fe),V=i(fe,"P",{"data-svelte-h":!0}),X(V)!=="svelte-8tudwd"&&(V.innerHTML=Ce),fe.forEach(t),N.forEach(t),oe=l(e),p(j.$$.fragment,e),se=l(e),h=i(e,"DIV",{class:!0});var ue=T(h);p(I.$$.fragment,ue),Me=l(ue),Q=i(ue,"P",{"data-svelte-h":!0}),X(Q)!=="svelte-uchgs5"&&(Q.innerHTML=Be),ue.forEach(t),ie=l(e),p(W.$$.fragment,e),me=l(e),b=i(e,"DIV",{class:!0});var ge=T(b);p(Z.$$.fragment,ge),$e=l(ge),S=i(ge,"P",{"data-svelte-h":!0}),X(S)!=="svelte-1a3zlxw"&&(S.textContent=Ge),ge.forEach(t),pe=l(e),p(P.$$.fragment,e),_e=l(e),L=i(e,"P",{}),T(L).forEach(t),this.h()},h(){M(v,"name","hf:doc:metadata"),M(v,"content",Ve),M(y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),M(g,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),M(w,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),M(u,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),M(h,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),M(b,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,a){o(document.head,v),r(e,D,a),r(e,Y,a),r(e,K,a),_($,e,a),r(e,ee,a),_(R,e,a),r(e,ae,a),r(e,J,a),r(e,te,a),_(U,e,a),r(e,ne,a),_(x,e,a),r(e,le,a),_(k,e,a),r(e,re,a),r(e,u,a),_(C,u,null),o(u,ve),o(u,y),_(B,y,null),o(y,he),o(y,E),o(u,be),o(u,g),_(G,g,null),o(g,ye),o(g,O),o(g,we),o(g,z),o(u,Ne),o(u,w),_(F,w,null),o(w,Te),o(w,V),r(e,oe,a),_(j,e,a),r(e,se,a),r(e,h,a),_(I,h,null),o(h,Me),o(h,Q),r(e,ie,a),_(W,e,a),r(e,me,a),r(e,b,a),_(Z,b,null),o(b,$e),o(b,S),r(e,pe,a),_(P,e,a),r(e,_e,a),r(e,L,a),ce=!0},p:je,i(e){ce||(c($.$$.fragment,e),c(R.$$.fragment,e),c(U.$$.fragment,e),c(x.$$.fragment,e),c(k.$$.fragment,e),c(C.$$.fragment,e),c(B.$$.fragment,e),c(G.$$.fragment,e),c(F.$$.fragment,e),c(j.$$.fragment,e),c(I.$$.fragment,e),c(W.$$.fragment,e),c(Z.$$.fragment,e),c(P.$$.fragment,e),ce=!0)},o(e){d($.$$.fragment,e),d(R.$$.fragment,e),d(U.$$.fragment,e),d(x.$$.fragment,e),d(k.$$.fragment,e),d(C.$$.fragment,e),d(B.$$.fragment,e),d(G.$$.fragment,e),d(F.$$.fragment,e),d(j.$$.fragment,e),d(I.$$.fragment,e),d(W.$$.fragment,e),d(Z.$$.fragment,e),d(P.$$.fragment,e),ce=!1},d(e){e&&(t(D),t(Y),t(K),t(ee),t(ae),t(J),t(te),t(ne),t(le),t(re),t(u),t(oe),t(se),t(h),t(ie),t(me),t(b),t(pe),t(_e),t(L)),t(v),f($,e),f(R,e),f(U,e),f(x,e),f(k,e),f(C),f(B),f(G),f(F),f(j,e),f(I),f(W,e),f(Z),f(P,e)}}}const Ve='{"title":"GRPO With Replay Buffer","local":"grpo-with-replay-buffer","sections":[{"title":"Usage","local":"usage","sections":[],"depth":2},{"title":"GRPOWithReplayBufferTrainer","local":"trl.experimental.grpo_with_replay_buffer.GRPOWithReplayBufferTrainer","sections":[],"depth":2},{"title":"GRPOWithReplayBufferConfig","local":"trl.experimental.grpo_with_replay_buffer.GRPOWithReplayBufferConfig","sections":[],"depth":2},{"title":"ReplayBuffer","local":"trl.experimental.grpo_with_replay_buffer.ReplayBuffer","sections":[],"depth":2}],"depth":1}';function Qe(Re){return Ie(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class qe extends We{constructor(v){super(),Ze(this,v,Qe,ze,Fe,{})}}export{qe as component};