transformers/pr_48314/en/_app/immutable/nodes/110.DTCVSefe.js

import"../chunks/DsnmJJEf.js";import{i as Je,h as je,C as we,H as n,a as c,D as t,E as ke,s as Ie}from"../chunks/CD2rhSaz.js";import{p as ve,o as qe,s as e,f as m,a as i,b as Ze,c as a,d as h,r,n as u}from"../chunks/DmjbnfDo.js";import{T as p}from"../chunks/B2suExpn.js";import{E as M}from"../chunks/B5qO6y4i.js";const Ne='{"title":"Csm","local":"csm","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Usage Tips","local":"usage-tips","sections":[{"title":"Without Conversational Context","local":"without-conversational-context","sections":[],"depth":3},{"title":"With Conversational Context","local":"with-conversational-context","sections":[],"depth":3},{"title":"Batched Inference","local":"batched-inference","sections":[],"depth":3},{"title":"Making The Model Go Brrr","local":"making-the-model-go-brrr","sections":[],"depth":3},{"title":"Training","local":"training","sections":[],"depth":3}],"depth":2},{"title":"CsmConfig","local":"transformers.CsmConfig","sections":[],"depth":2},{"title":"CsmDepthDecoderConfig","local":"transformers.CsmDepthDecoderConfig","sections":[],"depth":2},{"title":"CsmProcessor","local":"transformers.CsmProcessor","sections":[],"depth":2},{"title":"CsmForConditionalGeneration","local":"transformers.CsmForConditionalGeneration","sections":[],"depth":2},{"title":"CsmDepthDecoderForCausalLM","local":"transformers.CsmDepthDecoderForCausalLM","sections":[],"depth":2},{"title":"CsmDepthDecoderModel","local":"transformers.CsmDepthDecoderModel","sections":[],"depth":2},{"title":"CsmBackboneModel","local":"transformers.CsmBackboneModel","sections":[],"depth":2}],"depth":1}';var Be=h('<meta name="hf:doc:metadata"/>'),j=h("<p>Example:</p> <!>",1),g=h(`<p>Although the recipe for forward pass needs to be defined within this function, one should call the <code>Module</code> instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.</p>`),Qe=h(`<p>Most generation-controlling parameters are set in <code>generation_config</code> which, if not passed, will be set to the
model’s default generation configuration. You can override any <code>generation_config</code> by passing the corresponding
parameters to generate(), e.g. <code>.generate(inputs, do_sample=True)</code>.</p>`),Ge=h(`<p></p> <p><em>This model was contributed to Hugging Face Transformers on 2025-05-07.</em></p> <!> <!> <!> <p>The Conversational Speech Model (CSM) is the first open-source contextual text-to-speech model <a href="https://www.sesame.com/research/crossing_the_uncanny_valley_of_voice" rel="nofollow">released by Sesame</a>. It is designed to generate natural-sounding speech with or without conversational context. This context typically consists of multi-turn dialogue between speakers, represented as sequences of text and corresponding spoken audio.</p> <p><strong>Model Architecture:</strong> CSM is composed of two LLaMA-style auto-regressive transformer decoders: a backbone decoder that predicts the first codebook token and a depth decoder that generates the remaining tokens. It uses the pretrained codec model <a href="./mimi">Mimi</a>, introduced by Kyutai, to encode speech into discrete codebook tokens and decode them back into audio.</p> <p>The original csm-1b checkpoint is available under the <a href="https://huggingface.co/sesame/csm-1b" rel="nofollow">Sesame</a> organization on Hugging Face.</p> <div class="flex justify-center"><img src="https://huggingface.co/datasets/eustlb/documentation-images/resolve/main/csm_architecture.png"/></div> <blockquote class="tip"><p>Set <code>use_kernels=True</code> in <a href="/docs/transformers/pr_48314/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> to replace supported layers with optimized kernels from the Hub. Refer to <a href="../kernel_doc/loading_kernels">Loading kernels</a> to learn more.</p></blockquote> <!> <!> <p>CSM can be used to simply generate speech from a text prompt:</p> <!> <!> <p>CSM can be used to generate speech given a conversation, allowing consistency in the voices and content-aware generation:</p> <!> <!> <p>CSM supports batched inference!</p> <!> <!> <p>CSM supports full-graph compilation with CUDA graphs!</p> <!> <!> <p>CSM Transformers integration supports training!</p> <!> <p>This model was contributed by <a href="https://huggingface.co/eustlb" rel="nofollow">Eustache Le Bihan</a>.
The original code can be found <a href="https://github.com/SesameAILabs/csm" rel="nofollow">here</a>.</p> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>This is the configuration class to store the configuration of a CsmForConditionalGeneration. It is used to instantiate a Csm
model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
defaults will yield a similar configuration to that of the <a href="https://huggingface.co/sesame/csm-1b" rel="nofollow">sesame/csm-1b</a></p> <p>Configuration objects inherit from <a href="/docs/transformers/pr_48314/en/main_classes/configuration#transformers.PreTrainedConfig">PreTrainedConfig</a> and can be used to control the model outputs. Read the
documentation from <a href="/docs/transformers/pr_48314/en/main_classes/configuration#transformers.PreTrainedConfig">PreTrainedConfig</a> for more information.</p> <!></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>This is the configuration class to store the configuration of a CsmForConditionalGeneration. It is used to instantiate a Csm
model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
defaults will yield a similar configuration to that of the <a href="https://huggingface.co/sesame/csm-1b" rel="nofollow">sesame/csm-1b</a></p> <p>Configuration objects inherit from <a href="/docs/transformers/pr_48314/en/main_classes/configuration#transformers.PreTrainedConfig">PreTrainedConfig</a> and can be used to control the model outputs. Read the
documentation from <a href="/docs/transformers/pr_48314/en/main_classes/configuration#transformers.PreTrainedConfig">PreTrainedConfig</a> for more information.</p> <!></div> <!> <div class="flex justify-center"><img src="https://huggingface.co/datasets/eustlb/documentation-images/resolve/main/fig1.jpg"/></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Constructs a CsmProcessor which wraps a feature extractor and a tokenizer into a single processor.</p> <p><a href="/docs/transformers/pr_48314/en/model_doc/csm#transformers.CsmProcessor">CsmProcessor</a> offers all the functionalities of <a href="/docs/transformers/pr_48314/en/model_doc/encodec#transformers.EncodecFeatureExtractor">EncodecFeatureExtractor</a> and <code>tokenizer_class</code>. See the <a href="/docs/transformers/pr_48314/en/model_doc/encodec#transformers.EncodecFeatureExtractor">~EncodecFeatureExtractor</a> and <code>~tokenizer_class</code> for more information.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The Csm model consists of two llama-like auto-regressive transformer models: a backbone model that predicts the first codebook token and a depth decoder that predicts the other codebook tokens.</p> <p>This model inherits from <a href="/docs/transformers/pr_48314/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)</p> <p>This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The <a href="/docs/transformers/pr_48314/en/model_doc/csm#transformers.CsmForConditionalGeneration">CsmForConditionalGeneration</a> forward method, overrides the <code>__call__</code> special method.</p> <!> <ul><li><p><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) — Language modeling loss (for next-token prediction).</p></li> <li><p><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) — Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).</p></li> <li><p><strong>past_key_values</strong> (<code>Cache</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) — It is a <a href="/docs/transformers/pr_48314/en/internal/generation_utils#transformers.Cache">Cache</a> instance. For more details, see our <a href="https://huggingface.co/docs/transformers/en/kv_cache" rel="nofollow">kv cache guide</a>.</p> <p>Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see <code>past_key_values</code> input) to speed up sequential decoding.</p></li> <li><p><strong>hidden_states</strong> (<code>tuple[torch.FloatTensor, ...]</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p> <p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p></li> <li><p><strong>attentions</strong> (<code>tuple[torch.FloatTensor, ...]</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p> <p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.</p></li> <li><p><strong>depth_decoder_loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) — Language modeling loss (for next-token prediction) of the depth decoder model.</p></li> <li><p><strong>depth_decoder_logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) — Prediction scores of the depth decoder (scores for each vocabulary token before SoftMax).</p></li> <li><p><strong>depth_decoder_past_key_values</strong> (<code>Cache</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) — It is a <a href="/docs/transformers/pr_48314/en/internal/generation_utils#transformers.Cache">Cache</a> instance. For more details, see our <a href="https://huggingface.co/docs/transformers/en/kv_cache" rel="nofollow">kv cache guide</a>.</p></li> <li><p><strong>depth_decoder_hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p> <p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p></li> <li><p><strong>depth_decoder_attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p></li> <li><p><strong>backbone_loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) — Language modeling loss (for next-token prediction) of the backbone model.</p></li></ul> <!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>This method overrides <a href="/docs/transformers/pr_48314/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a> to match the specifics of the Csm model.
Indeed, Csm model requires a custom generation sampling step:</p> <ol><li>Infer the backbone model to sample the first codebook token</li> <li>Call generate on the depth decoder with the first codebook token as <code>input_ids</code> to sample the next codebook tokens</li> <li>Use these generated codebook tokens as <code>input_ids</code> to sample the next first codebook token using the backbone model</li> <li>Repeat until stopping criteria is met</li></ol> <!> <!></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The CsmDepthDecoder Model transformer, with a <code>CsmCodebooksHead</code> on top,
which can be seen a position-specific language modeling head, allowing to use a different linear layer for each codebook
(e.g. position 0 is the first codebook and uses the first codebook head, etc.)</p> <p>This model inherits from <a href="/docs/transformers/pr_48314/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)</p> <p>This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The <a href="/docs/transformers/pr_48314/en/model_doc/csm#transformers.CsmDepthDecoderForCausalLM">CsmDepthDecoderForCausalLM</a> forward method, overrides the <code>__call__</code> special method.</p> <!> <ul><li><p><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) — Language modeling loss (for next-token prediction).</p></li> <li><p><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) — Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).</p></li> <li><p><strong>past_key_values</strong> (<code>Cache</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) — It is a <a href="/docs/transformers/pr_48314/en/internal/generation_utils#transformers.Cache">Cache</a> instance. For more details, see our <a href="https://huggingface.co/docs/transformers/en/kv_cache" rel="nofollow">kv cache guide</a>.</p> <p>Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see <code>past_key_values</code> input) to speed up sequential decoding.</p></li> <li><p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p> <p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p></li> <li><p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p> <p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.</p></li></ul></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The bare Csm Model outputting raw hidden-states without any specific head on top.</p> <p>This model inherits from <a href="/docs/transformers/pr_48314/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)</p> <p>This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The <a href="/docs/transformers/pr_48314/en/model_doc/csm#transformers.CsmDepthDecoderModel">CsmDepthDecoderModel</a> forward method, overrides the <code>__call__</code> special method.</p> <!> <ul><li><p><strong>last_hidden_state</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>) — Sequence of hidden-states at the output of the last layer of the model.</p> <p>If <code>past_key_values</code> is used only the last hidden-state of the sequences of shape <code>(batch_size, 1, hidden_size)</code> is output.</p></li> <li><p><strong>past_key_values</strong> (<code>Cache</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) — It is a <a href="/docs/transformers/pr_48314/en/internal/generation_utils#transformers.Cache">Cache</a> instance. For more details, see our <a href="https://huggingface.co/docs/transformers/en/kv_cache" rel="nofollow">kv cache guide</a>.</p> <p>Contains pre-computed hidden-states (key and values in the self-attention blocks and optionally if <code>config.is_encoder_decoder=True</code> in the cross-attention blocks) that can be used (see <code>past_key_values</code> input) to speed up sequential decoding.</p></li> <li><p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p> <p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p></li> <li><p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p> <p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.</p></li></ul></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The bare Csm Model outputting raw hidden-states without any specific head on top.</p> <p>This model inherits from <a href="/docs/transformers/pr_48314/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)</p> <p>This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The <a href="/docs/transformers/pr_48314/en/model_doc/csm#transformers.CsmBackboneModel">CsmBackboneModel</a> forward method, overrides the <code>__call__</code> special method.</p> <!> <ul><li><p><strong>last_hidden_state</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>) — Sequence of hidden-states at the output of the last layer of the model.</p> <p>If <code>past_key_values</code> is used only the last hidden-state of the sequences of shape <code>(batch_size, 1, hidden_size)</code> is output.</p></li> <li><p><strong>past_key_values</strong> (<code>Cache</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) — It is a <a href="/docs/transformers/pr_48314/en/internal/generation_utils#transformers.Cache">Cache</a> instance. For more details, see our <a href="https://huggingface.co/docs/transformers/en/kv_cache" rel="nofollow">kv cache guide</a>.</p> <p>Contains pre-computed hidden-states (key and values in the self-attention blocks and optionally if <code>config.is_encoder_decoder=True</code> in the cross-attention blocks) that can be used (see <code>past_key_values</code> input) to speed up sequential decoding.</p></li> <li><p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p> <p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p></li> <li><p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p> <p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.</p></li></ul></div></div> <!> <p></p>`,1);function xe(me,ue){ve(ue,!1),qe(()=>{new URLSearchParams(window.location.search).get("fw")}),Je();var w=Ge();je("x7bnsp",s=>{var l=Be();Ie(l,"content",Ne),i(s,l)});var k=e(m(w),4);we(k,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var I=e(k,2);n(I,{title:"Csm",local:"csm",headingTag:"h1"});var v=e(I,2);n(v,{title:"Overview",local:"overview",headingTag:"h2"});var q=e(v,12);n(q,{title:"Usage Tips",local:"usage-tips",headingTag:"h2"});var Z=e(q,2);n(Z,{title:"Without Conversational Context",local:"without-conversational-context",headingTag:"h3"});var N=e(Z,4);c(N,{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Qcm9jZXNzb3IlMkMlMjBDc21Gb3JDb25kaXRpb25hbEdlbmVyYXRpb24lMEElMEElMEFtb2RlbF9pZCUyMCUzRCUyMCUyMnNlc2FtZSUyRmNzbS0xYiUyMiUwQSUwQSUyMyUyMGxvYWQlMjB0aGUlMjBtb2RlbCUyMGFuZCUyMHRoZSUyMHByb2Nlc3NvciUwQXByb2Nlc3NvciUyMCUzRCUyMEF1dG9Qcm9jZXNzb3IuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQW1vZGVsJTIwJTNEJTIwQ3NtRm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uLmZyb21fcHJldHJhaW5lZChtb2RlbF9pZCUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyKSUwQSUwQSUyMyUyMHByZXBhcmUlMjB0aGUlMjBpbnB1dHMlMEF0ZXh0JTIwJTNEJTIwJTIyJTVCMCU1RFRoZSUyMHBhc3QlMjBpcyUyMGp1c3QlMjBhJTIwc3RvcnklMjB3ZSUyMHRlbGwlMjBvdXJzZWx2ZXMuJTIyJTIwJTIzJTIwJTYwJTVCMCU1RCU2MCUyMGZvciUyMHNwZWFrZXIlMjBpZCUyMDAlMEFpbnB1dHMlMjAlM0QlMjBwcm9jZXNzb3IodGV4dCUyQyUyMGFkZF9zcGVjaWFsX3Rva2VucyUzRFRydWUpLnRvKG1vZGVsLmRldmljZSklMEElMEElMjMlMjBhbm90aGVyJTIwZXF1aXZhbGVudCUyMHdheSUyMHRvJTIwcHJlcGFyZSUyMHRoZSUyMGlucHV0cyUwQWNvbnZlcnNhdGlvbiUyMCUzRCUyMCU1QiUwQSUyMCUyMCUyMCUyMCU3QiUyMnJvbGUlMjIlM0ElMjAlMjIwJTIyJTJDJTIwJTIyY29udGVudCUyMiUzQSUyMCU1QiU3QiUyMnR5cGUlMjIlM0ElMjAlMjJ0ZXh0JTIyJTJDJTIwJTIydGV4dCUyMiUzQSUyMCUyMlRoZSUyMHBhc3QlMjBpcyUyMGp1c3QlMjBhJTIwc3RvcnklMjB3ZSUyMHRlbGwlMjBvdXJzZWx2ZXMuJTIyJTdEJTVEJTdEJTJDJTBBJTVEJTBBaW5wdXRzJTIwJTNEJTIwcHJvY2Vzc29yLmFwcGx5X2NoYXRfdGVtcGxhdGUoJTBBJTIwJTIwJTIwJTIwY29udmVyc2F0aW9uJTJDJTBBJTIwJTIwJTIwJTIwdG9rZW5pemUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwcmV0dXJuX2RpY3QlM0RUcnVlJTJDJTBBKS50byhtb2RlbC5kZXZpY2UpJTBBJTBBJTIzJTIwaW5mZXIlMjB0aGUlMjBtb2RlbCUwQWF1ZGlvJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBvdXRwdXRfYXVkaW8lM0RUcnVlKSUwQXByb2Nlc3Nvci5zYXZlX2F1ZGlvKGF1ZGlvJTJDJTIwJTIyZXhhbXBsZV93aXRob3V0X2NvbnRleHQud2F2JTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor, CsmForConditionalGeneration


model_id = <span class="hljs-string">&quot;sesame/csm-1b&quot;</span>

<span class="hljs-comment"># load the model and the processor</span>
processor = AutoProcessor.from_pretrained(model_id)
model = CsmForConditionalGeneration.from_pretrained(model_id, device_map=<span class="hljs-string">&quot;auto&quot;</span>)

<span class="hljs-comment"># prepare the inputs</span>
text = <span class="hljs-string">&quot;[0]The past is just a story we tell ourselves.&quot;</span> <span class="hljs-comment"># \`[0]\` for speaker id 0</span>
inputs = processor(text, add_special_tokens=<span class="hljs-literal">True</span>).to(model.device)

<span class="hljs-comment"># another equivalent way to prepare the inputs</span>
conversation = [
    {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;0&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: [{<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: <span class="hljs-string">&quot;The past is just a story we tell ourselves.&quot;</span>}]},
]
inputs = processor.apply_chat_template(
    conversation,
    tokenize=<span class="hljs-literal">True</span>,
    return_dict=<span class="hljs-literal">True</span>,
).to(model.device)

<span class="hljs-comment"># infer the model</span>
audio = model.generate(**inputs, output_audio=<span class="hljs-literal">True</span>)
processor.save_audio(audio, <span class="hljs-string">&quot;example_without_context.wav&quot;</span>)`,lang:"python",wrap:!1});var B=e(N,2);n(B,{title:"With Conversational Context",local:"with-conversational-context",headingTag:"h3"});var Q=e(B,4);c(Q,{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwQXVkaW8lMkMlMjBsb2FkX2RhdGFzZXQlMEElMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Byb2Nlc3NvciUyQyUyMENzbUZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbiUwQSUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyc2VzYW1lJTJGY3NtLTFiJTIyJTBBJTBBJTIzJTIwbG9hZCUyMHRoZSUyMG1vZGVsJTIwYW5kJTIwdGhlJTIwcHJvY2Vzc29yJTBBcHJvY2Vzc29yJTIwJTNEJTIwQXV0b1Byb2Nlc3Nvci5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQpJTBBbW9kZWwlMjAlM0QlMjBDc21Gb3JDb25kaXRpb25hbEdlbmVyYXRpb24uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIpJTBBJTBBJTIzJTIwcHJlcGFyZSUyMHRoZSUyMGlucHV0cyUwQWRzJTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMmhmLWludGVybmFsLXRlc3RpbmclMkZkYWlseXRhbGstZHVtbXklMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKSUwQSUyMyUyMGVuc3VyZSUyMHRoZSUyMGF1ZGlvJTIwaXMlMjAyNGtIeiUwQWRzJTIwJTNEJTIwZHMuY2FzdF9jb2x1bW4oJTIyYXVkaW8lMjIlMkMlMjBBdWRpbyhzYW1wbGluZ19yYXRlJTNEMjQwMDApKSUwQWNvbnZlcnNhdGlvbiUyMCUzRCUyMCU1QiU1RCUwQSUwQSUyMyUyMDEuJTIwY29udGV4dCUwQWZvciUyMHRleHQlMkMlMjBhdWRpbyUyQyUyMHNwZWFrZXJfaWQlMjBpbiUyMHppcChkcyU1QiUzQTQlNUQlNUIlMjJ0ZXh0JTIyJTVEJTJDJTIwZHMlNUIlM0E0JTVEJTVCJTIyYXVkaW8lMjIlNUQlMkMlMjBkcyU1QiUzQTQlNUQlNUIlMjJzcGVha2VyX2lkJTIyJTVEKSUzQSUwQSUyMCUyMCUyMCUyMGNvbnZlcnNhdGlvbi5hcHBlbmQoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycm9sZSUyMiUzQSUyMGYlMjIlN0JzcGVha2VyX2lkJTdEJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGVudCUyMiUzQSUyMCU1QiU3QiUyMnR5cGUlMjIlM0ElMjAlMjJ0ZXh0JTIyJTJDJTIwJTIydGV4dCUyMiUzQSUyMHRleHQlN0QlMkMlMjAlN0IlMjJ0eXBlJTIyJTNBJTIwJTIyYXVkaW8lMjIlMkMlMjAlMjJwYXRoJTIyJTNBJTIwYXVkaW8lNUIlMjJhcnJheSUyMiU1RCU3RCU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3RCUwQSUyMCUyMCUyMCUyMCklMEElMEElMjMlMjAyLiUyMHRleHQlMjBwcm9tcHQlMEFjb252ZXJzYXRpb24uYXBwZW5kKCU3QiUyMnJvbGUlMjIlM0ElMjBmJTIyJTdCZHMlNUI0JTVEJTVCJ3NwZWFrZXJfaWQnJTVEJTdEJTIyJTJDJTIwJTIyY29udGVudCUyMiUzQSUyMCU1QiU3QiUyMnR5cGUlMjIlM0ElMjAlMjJ0ZXh0JTIyJTJDJTIwJTIydGV4dCUyMiUzQSUyMGRzJTVCNCU1RCU1QiUyMnRleHQlMjIlNUQlN0QlNUQlN0QpJTBBJTBBaW5wdXRzJTIwJTNEJTIwcHJvY2Vzc29yLmFwcGx5X2NoYXRfdGVtcGxhdGUoJTBBJTIwJTIwJTIwJTIwY29udmVyc2F0aW9uJTJDJTBBJTIwJTIwJTIwJTIwdG9rZW5pemUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwcmV0dXJuX2RpY3QlM0RUcnVlJTJDJTBBKS50byhtb2RlbC5kZXZpY2UpJTBBJTBBJTIzJTIwaW5mZXIlMjB0aGUlMjBtb2RlbCUwQWF1ZGlvJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBvdXRwdXRfYXVkaW8lM0RUcnVlKSUwQXByb2Nlc3Nvci5zYXZlX2F1ZGlvKGF1ZGlvJTJDJTIwJTIyZXhhbXBsZV93aXRoX2NvbnRleHQud2F2JTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Audio, load_dataset

<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor, CsmForConditionalGeneration


model_id = <span class="hljs-string">&quot;sesame/csm-1b&quot;</span>

<span class="hljs-comment"># load the model and the processor</span>
processor = AutoProcessor.from_pretrained(model_id)
model = CsmForConditionalGeneration.from_pretrained(model_id, device_map=<span class="hljs-string">&quot;auto&quot;</span>)

<span class="hljs-comment"># prepare the inputs</span>
ds = load_dataset(<span class="hljs-string">&quot;hf-internal-testing/dailytalk-dummy&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
<span class="hljs-comment"># ensure the audio is 24kHz</span>
ds = ds.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">24000</span>))
conversation = []

<span class="hljs-comment"># 1. context</span>
<span class="hljs-keyword">for</span> text, audio, speaker_id <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(ds[:<span class="hljs-number">4</span>][<span class="hljs-string">&quot;text&quot;</span>], ds[:<span class="hljs-number">4</span>][<span class="hljs-string">&quot;audio&quot;</span>], ds[:<span class="hljs-number">4</span>][<span class="hljs-string">&quot;speaker_id&quot;</span>]):
    conversation.append(
        {
            <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">f&quot;<span class="hljs-subst">{speaker_id}</span>&quot;</span>,
            <span class="hljs-string">&quot;content&quot;</span>: [{<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: text}, {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;audio&quot;</span>, <span class="hljs-string">&quot;path&quot;</span>: audio[<span class="hljs-string">&quot;array&quot;</span>]}],
        }
    )

<span class="hljs-comment"># 2. text prompt</span>
conversation.append({<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">f&quot;<span class="hljs-subst">{ds[<span class="hljs-number">4</span>][<span class="hljs-string">&#x27;speaker_id&#x27;</span>]}</span>&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: [{<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: ds[<span class="hljs-number">4</span>][<span class="hljs-string">&quot;text&quot;</span>]}]})

inputs = processor.apply_chat_template(
    conversation,
    tokenize=<span class="hljs-literal">True</span>,
    return_dict=<span class="hljs-literal">True</span>,
).to(model.device)

<span class="hljs-comment"># infer the model</span>
audio = model.generate(**inputs, output_audio=<span class="hljs-literal">True</span>)
processor.save_audio(audio, <span class="hljs-string">&quot;example_with_context.wav&quot;</span>)`,lang:"python",wrap:!1});var G=e(Q,2);n(G,{title:"Batched Inference",local:"batched-inference",headingTag:"h3"});var R=e(G,4);c(R,{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwQXVkaW8lMkMlMjBsb2FkX2RhdGFzZXQlMEElMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Byb2Nlc3NvciUyQyUyMENzbUZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbiUwQSUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyc2VzYW1lJTJGY3NtLTFiJTIyJTBBJTBBJTIzJTIwbG9hZCUyMHRoZSUyMG1vZGVsJTIwYW5kJTIwdGhlJTIwcHJvY2Vzc29yJTBBcHJvY2Vzc29yJTIwJTNEJTIwQXV0b1Byb2Nlc3Nvci5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQpJTBBbW9kZWwlMjAlM0QlMjBDc21Gb3JDb25kaXRpb25hbEdlbmVyYXRpb24uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIpJTBBJTBBJTIzJTIwcHJlcGFyZSUyMHRoZSUyMGlucHV0cyUwQWRzJTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMmhmLWludGVybmFsLXRlc3RpbmclMkZkYWlseXRhbGstZHVtbXklMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKSUwQSUyMyUyMGVuc3VyZSUyMHRoZSUyMGF1ZGlvJTIwaXMlMjAyNGtIeiUwQWRzJTIwJTNEJTIwZHMuY2FzdF9jb2x1bW4oJTIyYXVkaW8lMjIlMkMlMjBBdWRpbyhzYW1wbGluZ19yYXRlJTNEMjQwMDApKSUwQSUyMyUyMGhlcmUlMjBhJTIwYmF0Y2glMjB3aXRoJTIwdHdvJTIwcHJvbXB0cyUwQWNvbnZlcnNhdGlvbiUyMCUzRCUyMCU1QiUwQSUyMCUyMCUyMCUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJvbGUlMjIlM0ElMjBmJTIyJTdCZHMlNUIwJTVEJTVCJ3NwZWFrZXJfaWQnJTVEJTdEJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGVudCUyMiUzQSUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUyMnR5cGUlMjIlM0ElMjAlMjJ0ZXh0JTIyJTJDJTIwJTIydGV4dCUyMiUzQSUyMGRzJTVCMCU1RCU1QiUyMnRleHQlMjIlNUQlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJ0eXBlJTIyJTNBJTIwJTIyYXVkaW8lMjIlMkMlMjAlMjJwYXRoJTIyJTNBJTIwZHMlNUIwJTVEJTVCJTIyYXVkaW8lMjIlNUQlNUIlMjJhcnJheSUyMiU1RCU3RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJvbGUlMjIlM0ElMjBmJTIyJTdCZHMlNUIxJTVEJTVCJ3NwZWFrZXJfaWQnJTVEJTdEJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGVudCUyMiUzQSUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUyMnR5cGUlMjIlM0ElMjAlMjJ0ZXh0JTIyJTJDJTIwJTIydGV4dCUyMiUzQSUyMGRzJTVCMSU1RCU1QiUyMnRleHQlMjIlNUQlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0QlMkMlMEElMjAlMjAlMjAlMjAlNUQlMkMlMEElMjAlMjAlMjAlMjAlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJyb2xlJTIyJTNBJTIwZiUyMiU3QmRzJTVCMCU1RCU1QidzcGVha2VyX2lkJyU1RCU3RCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJ0eXBlJTIyJTNBJTIwJTIydGV4dCUyMiUyQyUyMCUyMnRleHQlMjIlM0ElMjBkcyU1QjAlNUQlNUIlMjJ0ZXh0JTIyJTVEJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTBBJTIwJTIwJTIwJTIwJTVEJTJDJTBBJTVEJTBBaW5wdXRzJTIwJTNEJTIwcHJvY2Vzc29yLmFwcGx5X2NoYXRfdGVtcGxhdGUoJTBBJTIwJTIwJTIwJTIwY29udmVyc2F0aW9uJTJDJTBBJTIwJTIwJTIwJTIwdG9rZW5pemUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwcmV0dXJuX2RpY3QlM0RUcnVlJTJDJTBBKS50byhtb2RlbC5kZXZpY2UpJTBBJTBBYXVkaW8lMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0cyUyQyUyMG91dHB1dF9hdWRpbyUzRFRydWUpJTBBcHJvY2Vzc29yLnNhdmVfYXVkaW8oYXVkaW8lMkMlMjAlNUJmJTIyc3BlZWNoX2JhdGNoX2lkeF8lN0JpJTdELndhdiUyMiUyMGZvciUyMGklMjBpbiUyMHJhbmdlKGxlbihhdWRpbykpJTVEKQ==",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Audio, load_dataset

<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor, CsmForConditionalGeneration


model_id = <span class="hljs-string">&quot;sesame/csm-1b&quot;</span>

<span class="hljs-comment"># load the model and the processor</span>
processor = AutoProcessor.from_pretrained(model_id)
model = CsmForConditionalGeneration.from_pretrained(model_id, device_map=<span class="hljs-string">&quot;auto&quot;</span>)

<span class="hljs-comment"># prepare the inputs</span>
ds = load_dataset(<span class="hljs-string">&quot;hf-internal-testing/dailytalk-dummy&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
<span class="hljs-comment"># ensure the audio is 24kHz</span>
ds = ds.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">24000</span>))
<span class="hljs-comment"># here a batch with two prompts</span>
conversation = [
    [
        {
            <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">f&quot;<span class="hljs-subst">{ds[<span class="hljs-number">0</span>][<span class="hljs-string">&#x27;speaker_id&#x27;</span>]}</span>&quot;</span>,
            <span class="hljs-string">&quot;content&quot;</span>: [
                {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: ds[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;text&quot;</span>]},
                {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;audio&quot;</span>, <span class="hljs-string">&quot;path&quot;</span>: ds[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>]},
            ],
        },
        {
            <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">f&quot;<span class="hljs-subst">{ds[<span class="hljs-number">1</span>][<span class="hljs-string">&#x27;speaker_id&#x27;</span>]}</span>&quot;</span>,
            <span class="hljs-string">&quot;content&quot;</span>: [
                {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: ds[<span class="hljs-number">1</span>][<span class="hljs-string">&quot;text&quot;</span>]},
            ],
        },
    ],
    [
        {
            <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">f&quot;<span class="hljs-subst">{ds[<span class="hljs-number">0</span>][<span class="hljs-string">&#x27;speaker_id&#x27;</span>]}</span>&quot;</span>,
            <span class="hljs-string">&quot;content&quot;</span>: [
                {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: ds[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;text&quot;</span>]},
            ],
        }
    ],
]
inputs = processor.apply_chat_template(
    conversation,
    tokenize=<span class="hljs-literal">True</span>,
    return_dict=<span class="hljs-literal">True</span>,
).to(model.device)

audio = model.generate(**inputs, output_audio=<span class="hljs-literal">True</span>)
processor.save_audio(audio, [<span class="hljs-string">f&quot;speech_batch_idx_<span class="hljs-subst">{i}</span>.wav&quot;</span> <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(audio))])`,lang:"python",wrap:!1});var z=e(R,2);n(z,{title:"Making The Model Go Brrr",local:"making-the-model-go-brrr",headingTag:"h3"});var W=e(z,4);c(W,{code:"JTBBaW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGF0YXNldHMlMjBpbXBvcnQlMjBsb2FkX2RhdGFzZXQlMEElMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Byb2Nlc3NvciUyQyUyMENzbUZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbiUwQSUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyc2VzYW1lJTJGY3NtLTFiJTIyJTBBJTBBJTIzJTIwc2V0JTIwbG9ncyUyMHRvJTIwZW5zdXJlJTIwbm8lMjByZWNvbXBpbGF0aW9uJTIwYW5kJTIwZ3JhcGglMjBicmVha3MlMEF0b3JjaC5fbG9nZ2luZy5zZXRfbG9ncyhncmFwaF9icmVha3MlM0RUcnVlJTJDJTIwcmVjb21waWxlcyUzRFRydWUlMkMlMjBjdWRhZ3JhcGhzJTNEVHJ1ZSklMEElMEElMjMlMjBsb2FkJTIwdGhlJTIwbW9kZWwlMjBhbmQlMjB0aGUlMjBwcm9jZXNzb3IlMEFwcm9jZXNzb3IlMjAlM0QlMjBBdXRvUHJvY2Vzc29yLmZyb21fcHJldHJhaW5lZChtb2RlbF9pZCklMEFtb2RlbCUyMCUzRCUyMENzbUZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbi5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiklMEElMEElMjMlMjB1c2UlMjBzdGF0aWMlMjBjYWNoZSUyQyUyMGVuYWJsaW5nJTIwYXV0b21hdGljYWxseSUyMHRvcmNoJTIwY29tcGlsZSUyMHdpdGglMjBmdWxsZ3JhcGglMjBhbmQlMjByZWR1Y2Utb3ZlcmhlYWQlMEFtb2RlbC5nZW5lcmF0aW9uX2NvbmZpZy5tYXhfbGVuZ3RoJTIwJTNEJTIwMjUwJTIwJTIzJTIwYmlnJTIwZW5vdWdoJTIwdG8lMjBhdm9pZCUyMHJlY29tcGlsYXRpb24lMEFtb2RlbC5nZW5lcmF0aW9uX2NvbmZpZy5tYXhfbmV3X3Rva2VucyUyMCUzRCUyME5vbmUlMjAlMjMlMjB3b3VsZCUyMHRha2UlMjBwcmVjZWRlbmNlJTIwb3ZlciUyMG1heF9sZW5ndGglMEFtb2RlbC5nZW5lcmF0aW9uX2NvbmZpZy5jYWNoZV9pbXBsZW1lbnRhdGlvbiUyMCUzRCUyMCUyMnN0YXRpYyUyMiUwQW1vZGVsLmRlcHRoX2RlY29kZXIuZ2VuZXJhdGlvbl9jb25maWcuY2FjaGVfaW1wbGVtZW50YXRpb24lMjAlM0QlMjAlMjJzdGF0aWMlMjIlMEElMEElMjMlMjBnZW5lcmF0aW9uJTIwa3dhcmdzJTBBZ2VuX2t3YXJncyUyMCUzRCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMmRvX3NhbXBsZSUyMiUzQSUyMEZhbHNlJTJDJTBBJTIwJTIwJTIwJTIwJTIyZGVwdGhfZGVjb2Rlcl9kb19zYW1wbGUlMjIlM0ElMjBGYWxzZSUyQyUwQSUyMCUyMCUyMCUyMCUyMnRlbXBlcmF0dXJlJTIyJTNBJTIwMS4wJTJDJTBBJTIwJTIwJTIwJTIwJTIyZGVwdGhfZGVjb2Rlcl90ZW1wZXJhdHVyZSUyMiUzQSUyMDEuMCUyQyUwQSU3RCUwQSUwQSUyMyUyMERlZmluZSUyMGElMjB0aW1pbmclMjBkZWNvcmF0b3IlMEFjbGFzcyUyMFRpbWVyQ29udGV4dCUzQSUwQSUyMCUyMCUyMCUyMGRlZiUyMF9faW5pdF9fKHNlbGYlMkMlMjBuYW1lJTNEJTIyRXhlY3V0aW9uJTIyKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNlbGYubmFtZSUyMCUzRCUyMG5hbWUlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLnN0YXJ0X2V2ZW50JTIwJTNEJTIwTm9uZSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNlbGYuZW5kX2V2ZW50JTIwJTNEJTIwTm9uZSUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMF9fZW50ZXJfXyhzZWxmKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMFVzZSUyMENVREElMjBldmVudHMlMjBmb3IlMjBtb3JlJTIwYWNjdXJhdGUlMjBHUFUlMjB0aW1pbmclMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLnN0YXJ0X2V2ZW50JTIwJTNEJTIwdG9yY2guY3VkYS5FdmVudChlbmFibGVfdGltaW5nJTNEVHJ1ZSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLmVuZF9ldmVudCUyMCUzRCUyMHRvcmNoLmN1ZGEuRXZlbnQoZW5hYmxlX3RpbWluZyUzRFRydWUpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5zdGFydF9ldmVudC5yZWNvcmQoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMHNlbGYlMEElMEElMjAlMjAlMjAlMjBkZWYlMjBfX2V4aXRfXyhzZWxmJTJDJTIwKmFyZ3MpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5lbmRfZXZlbnQucmVjb3JkKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0b3JjaC5jdWRhLnN5bmNocm9uaXplKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbGFwc2VkX3RpbWUlMjAlM0QlMjBzZWxmLnN0YXJ0X2V2ZW50LmVsYXBzZWRfdGltZShzZWxmLmVuZF9ldmVudCklMjAlMkYlMjAxMDAwLjAlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmludChmJTIyJTdCc2VsZi5uYW1lJTdEJTIwdGltZSUzQSUyMCU3QmVsYXBzZWRfdGltZSUzQS40ZiU3RCUyMHNlY29uZHMlMjIpJTBBJTBBJTIzJTIwcHJlcGFyZSUyMHRoZSUyMGlucHV0cyUwQWRzJTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMmhmLWludGVybmFsLXRlc3RpbmclMkZkYWlseXRhbGstZHVtbXklMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKSUwQSUwQWNvbnZlcnNhdGlvbiUyMCUzRCUyMCU1QiUwQSUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJvbGUlMjIlM0ElMjBmJTIyJTdCZHMlNUIwJTVEJTVCJ3NwZWFrZXJfaWQnJTVEJTdEJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGVudCUyMiUzQSUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUyMnR5cGUlMjIlM0ElMjAlMjJ0ZXh0JTIyJTJDJTIwJTIydGV4dCUyMiUzQSUyMGRzJTVCMCU1RCU1QiUyMnRleHQlMjIlNUQlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJ0eXBlJTIyJTNBJTIwJTIyYXVkaW8lMjIlMkMlMjAlMjJwYXRoJTIyJTNBJTIwZHMlNUIwJTVEJTVCJTIyYXVkaW8lMjIlNUQlNUIlMjJhcnJheSUyMiU1RCU3RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU1RCUyQyUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJvbGUlMjIlM0ElMjBmJTIyJTdCZHMlNUIxJTVEJTVCJ3NwZWFrZXJfaWQnJTVEJTdEJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGVudCUyMiUzQSUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUyMnR5cGUlMjIlM0ElMjAlMjJ0ZXh0JTIyJTJDJTIwJTIydGV4dCUyMiUzQSUyMGRzJTVCMSU1RCU1QiUyMnRleHQlMjIlNUQlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJ0eXBlJTIyJTNBJTIwJTIyYXVkaW8lMjIlMkMlMjAlMjJwYXRoJTIyJTNBJTIwZHMlNUIxJTVEJTVCJTIyYXVkaW8lMjIlNUQlNUIlMjJhcnJheSUyMiU1RCU3RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU1RCUyQyUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJvbGUlMjIlM0ElMjBmJTIyJTdCZHMlNUIyJTVEJTVCJ3NwZWFrZXJfaWQnJTVEJTdEJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGVudCUyMiUzQSUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUyMnR5cGUlMjIlM0ElMjAlMjJ0ZXh0JTIyJTJDJTIwJTIydGV4dCUyMiUzQSUyMGRzJTVCMiU1RCU1QiUyMnRleHQlMjIlNUQlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUQlMkMlMEElMjAlMjAlMjAlMjAlN0QlMkMlMEElNUQlMEElMEFwYWRkZWRfaW5wdXRzXzElMjAlM0QlMjBwcm9jZXNzb3IuYXBwbHlfY2hhdF90ZW1wbGF0ZSglMEElMjAlMjAlMjAlMjBjb252ZXJzYXRpb24lMkMlMEElMjAlMjAlMjAlMjB0b2tlbml6ZSUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjByZXR1cm5fZGljdCUzRFRydWUlMkMlMEEpLnRvKG1vZGVsLmRldmljZSklMEElMEFwcmludCglMjIlNUNuJTIyJTIwJTJCJTIwJTIyJTNEJTIyKjUwKSUwQXByaW50KCUyMkZpcnN0JTIwZ2VuZXJhdGlvbiUyMC0lMjBjb21waWxpbmclMjBhbmQlMjByZWNvcmRpbmclMjBDVURBJTIwZ3JhcGhzLi4uJTIyKSUwQXdpdGglMjBUaW1lckNvbnRleHQoJTIyRmlyc3QlMjBnZW5lcmF0aW9uJTIyKSUzQSUwQSUyMCUyMCUyMCUyMF8lMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKnBhZGRlZF9pbnB1dHNfMSUyQyUyMCoqZ2VuX2t3YXJncyklMEFwcmludCglMjIlM0QlMjIqNTApJTBBJTBBcHJpbnQoJTIyJTVDbiUyMiUyMCUyQiUyMCUyMiUzRCUyMio1MCklMEFwcmludCglMjJTZWNvbmQlMjBnZW5lcmF0aW9uJTIwLSUyMGZhc3QlMjAhISElMjIpJTBBd2l0aCUyMFRpbWVyQ29udGV4dCglMjJTZWNvbmQlMjBnZW5lcmF0aW9uJTIyKSUzQSUwQSUyMCUyMCUyMCUyMF8lMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKnBhZGRlZF9pbnB1dHNfMSUyQyUyMCoqZ2VuX2t3YXJncyklMEFwcmludCglMjIlM0QlMjIqNTApJTBBJTBBJTIzJTIwbm93JTIwd2l0aCUyMGRpZmZlcmVudCUyMGlucHV0cyUwQWNvbnZlcnNhdGlvbiUyMCUzRCUyMCU1QiUwQSUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJvbGUlMjIlM0ElMjBmJTIyJTdCZHMlNUIwJTVEJTVCJ3NwZWFrZXJfaWQnJTVEJTdEJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGVudCUyMiUzQSUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUyMnR5cGUlMjIlM0ElMjAlMjJ0ZXh0JTIyJTJDJTIwJTIydGV4dCUyMiUzQSUyMGRzJTVCMiU1RCU1QiUyMnRleHQlMjIlNUQlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJ0eXBlJTIyJTNBJTIwJTIyYXVkaW8lMjIlMkMlMjAlMjJwYXRoJTIyJTNBJTIwZHMlNUIyJTVEJTVCJTIyYXVkaW8lMjIlNUQlNUIlMjJhcnJheSUyMiU1RCU3RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU1RCUyQyUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJvbGUlMjIlM0ElMjBmJTIyJTdCZHMlNUIxJTVEJTVCJ3NwZWFrZXJfaWQnJTVEJTdEJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGVudCUyMiUzQSUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUyMnR5cGUlMjIlM0ElMjAlMjJ0ZXh0JTIyJTJDJTIwJTIydGV4dCUyMiUzQSUyMGRzJTVCMyU1RCU1QiUyMnRleHQlMjIlNUQlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJ0eXBlJTIyJTNBJTIwJTIyYXVkaW8lMjIlMkMlMjAlMjJwYXRoJTIyJTNBJTIwZHMlNUIzJTVEJTVCJTIyYXVkaW8lMjIlNUQlNUIlMjJhcnJheSUyMiU1RCU3RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU1RCUyQyUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJvbGUlMjIlM0ElMjBmJTIyJTdCZHMlNUIyJTVEJTVCJ3NwZWFrZXJfaWQnJTVEJTdEJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGVudCUyMiUzQSUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUyMnR5cGUlMjIlM0ElMjAlMjJ0ZXh0JTIyJTJDJTIwJTIydGV4dCUyMiUzQSUyMGRzJTVCNCU1RCU1QiUyMnRleHQlMjIlNUQlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUQlMkMlMEElMjAlMjAlMjAlMjAlN0QlMkMlMEElNUQlMEFwYWRkZWRfaW5wdXRzXzIlMjAlM0QlMjBwcm9jZXNzb3IuYXBwbHlfY2hhdF90ZW1wbGF0ZSglMEElMjAlMjAlMjAlMjBjb252ZXJzYXRpb24lMkMlMEElMjAlMjAlMjAlMjB0b2tlbml6ZSUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjByZXR1cm5fZGljdCUzRFRydWUlMkMlMEEpLnRvKG1vZGVsLmRldmljZSklMEElMEFwcmludCglMjIlNUNuJTIyJTIwJTJCJTIwJTIyJTNEJTIyKjUwKSUwQXByaW50KCUyMkdlbmVyYXRpb24lMjB3aXRoJTIwb3RoZXIlMjBpbnB1dHMhJTIyKSUwQXdpdGglMjBUaW1lckNvbnRleHQoJTIyR2VuZXJhdGlvbiUyMHdpdGglMjBkaWZmZXJlbnQlMjBpbnB1dHMlMjIpJTNBJTBBJTIwJTIwJTIwJTIwXyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqcGFkZGVkX2lucHV0c18yJTJDJTIwKipnZW5fa3dhcmdzKSUwQXByaW50KCUyMiUzRCUyMio1MCk=",highlighted:`
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor, CsmForConditionalGeneration


model_id = <span class="hljs-string">&quot;sesame/csm-1b&quot;</span>

<span class="hljs-comment"># set logs to ensure no recompilation and graph breaks</span>
torch._logging.set_logs(graph_breaks=<span class="hljs-literal">True</span>, recompiles=<span class="hljs-literal">True</span>, cudagraphs=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># load the model and the processor</span>
processor = AutoProcessor.from_pretrained(model_id)
model = CsmForConditionalGeneration.from_pretrained(model_id, device_map=<span class="hljs-string">&quot;auto&quot;</span>)

<span class="hljs-comment"># use static cache, enabling automatically torch compile with fullgraph and reduce-overhead</span>
model.generation_config.max_length = <span class="hljs-number">250</span> <span class="hljs-comment"># big enough to avoid recompilation</span>
model.generation_config.max_new_tokens = <span class="hljs-literal">None</span> <span class="hljs-comment"># would take precedence over max_length</span>
model.generation_config.cache_implementation = <span class="hljs-string">&quot;static&quot;</span>
model.depth_decoder.generation_config.cache_implementation = <span class="hljs-string">&quot;static&quot;</span>

<span class="hljs-comment"># generation kwargs</span>
gen_kwargs = {
    <span class="hljs-string">&quot;do_sample&quot;</span>: <span class="hljs-literal">False</span>,
    <span class="hljs-string">&quot;depth_decoder_do_sample&quot;</span>: <span class="hljs-literal">False</span>,
    <span class="hljs-string">&quot;temperature&quot;</span>: <span class="hljs-number">1.0</span>,
    <span class="hljs-string">&quot;depth_decoder_temperature&quot;</span>: <span class="hljs-number">1.0</span>,
}

<span class="hljs-comment"># Define a timing decorator</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">TimerContext</span>:
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, name=<span class="hljs-string">&quot;Execution&quot;</span></span>):
        <span class="hljs-variable language_">self</span>.name = name
        <span class="hljs-variable language_">self</span>.start_event = <span class="hljs-literal">None</span>
        <span class="hljs-variable language_">self</span>.end_event = <span class="hljs-literal">None</span>

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__enter__</span>(<span class="hljs-params">self</span>):
        <span class="hljs-comment"># Use CUDA events for more accurate GPU timing</span>
        <span class="hljs-variable language_">self</span>.start_event = torch.cuda.Event(enable_timing=<span class="hljs-literal">True</span>)
        <span class="hljs-variable language_">self</span>.end_event = torch.cuda.Event(enable_timing=<span class="hljs-literal">True</span>)
        <span class="hljs-variable language_">self</span>.start_event.record()
        <span class="hljs-keyword">return</span> <span class="hljs-variable language_">self</span>

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__exit__</span>(<span class="hljs-params">self, *args</span>):
        <span class="hljs-variable language_">self</span>.end_event.record()
        torch.cuda.synchronize()
        elapsed_time = <span class="hljs-variable language_">self</span>.start_event.elapsed_time(<span class="hljs-variable language_">self</span>.end_event) / <span class="hljs-number">1000.0</span>
        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{self.name}</span> time: <span class="hljs-subst">{elapsed_time:<span class="hljs-number">.4</span>f}</span> seconds&quot;</span>)

<span class="hljs-comment"># prepare the inputs</span>
ds = load_dataset(<span class="hljs-string">&quot;hf-internal-testing/dailytalk-dummy&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)

conversation = [
    {
        <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">f&quot;<span class="hljs-subst">{ds[<span class="hljs-number">0</span>][<span class="hljs-string">&#x27;speaker_id&#x27;</span>]}</span>&quot;</span>,
        <span class="hljs-string">&quot;content&quot;</span>: [
            {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: ds[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;text&quot;</span>]},
            {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;audio&quot;</span>, <span class="hljs-string">&quot;path&quot;</span>: ds[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>]},
        ],
    },
    {
        <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">f&quot;<span class="hljs-subst">{ds[<span class="hljs-number">1</span>][<span class="hljs-string">&#x27;speaker_id&#x27;</span>]}</span>&quot;</span>,
        <span class="hljs-string">&quot;content&quot;</span>: [
            {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: ds[<span class="hljs-number">1</span>][<span class="hljs-string">&quot;text&quot;</span>]},
            {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;audio&quot;</span>, <span class="hljs-string">&quot;path&quot;</span>: ds[<span class="hljs-number">1</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>]},
        ],
    },
    {
        <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">f&quot;<span class="hljs-subst">{ds[<span class="hljs-number">2</span>][<span class="hljs-string">&#x27;speaker_id&#x27;</span>]}</span>&quot;</span>,
        <span class="hljs-string">&quot;content&quot;</span>: [
            {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: ds[<span class="hljs-number">2</span>][<span class="hljs-string">&quot;text&quot;</span>]},
        ],
    },
]

padded_inputs_1 = processor.apply_chat_template(
    conversation,
    tokenize=<span class="hljs-literal">True</span>,
    return_dict=<span class="hljs-literal">True</span>,
).to(model.device)

<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;\\n&quot;</span> + <span class="hljs-string">&quot;=&quot;</span>*<span class="hljs-number">50</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;First generation - compiling and recording CUDA graphs...&quot;</span>)
<span class="hljs-keyword">with</span> TimerContext(<span class="hljs-string">&quot;First generation&quot;</span>):
    _ = model.generate(**padded_inputs_1, **gen_kwargs)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;=&quot;</span>*<span class="hljs-number">50</span>)

<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;\\n&quot;</span> + <span class="hljs-string">&quot;=&quot;</span>*<span class="hljs-number">50</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Second generation - fast !!!&quot;</span>)
<span class="hljs-keyword">with</span> TimerContext(<span class="hljs-string">&quot;Second generation&quot;</span>):
    _ = model.generate(**padded_inputs_1, **gen_kwargs)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;=&quot;</span>*<span class="hljs-number">50</span>)

<span class="hljs-comment"># now with different inputs</span>
conversation = [
    {
        <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">f&quot;<span class="hljs-subst">{ds[<span class="hljs-number">0</span>][<span class="hljs-string">&#x27;speaker_id&#x27;</span>]}</span>&quot;</span>,
        <span class="hljs-string">&quot;content&quot;</span>: [
            {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: ds[<span class="hljs-number">2</span>][<span class="hljs-string">&quot;text&quot;</span>]},
            {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;audio&quot;</span>, <span class="hljs-string">&quot;path&quot;</span>: ds[<span class="hljs-number">2</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>]},
        ],
    },
    {
        <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">f&quot;<span class="hljs-subst">{ds[<span class="hljs-number">1</span>][<span class="hljs-string">&#x27;speaker_id&#x27;</span>]}</span>&quot;</span>,
        <span class="hljs-string">&quot;content&quot;</span>: [
            {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: ds[<span class="hljs-number">3</span>][<span class="hljs-string">&quot;text&quot;</span>]},
            {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;audio&quot;</span>, <span class="hljs-string">&quot;path&quot;</span>: ds[<span class="hljs-number">3</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>]},
        ],
    },
    {
        <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">f&quot;<span class="hljs-subst">{ds[<span class="hljs-number">2</span>][<span class="hljs-string">&#x27;speaker_id&#x27;</span>]}</span>&quot;</span>,
        <span class="hljs-string">&quot;content&quot;</span>: [
            {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: ds[<span class="hljs-number">4</span>][<span class="hljs-string">&quot;text&quot;</span>]},
        ],
    },
]
padded_inputs_2 = processor.apply_chat_template(
    conversation,
    tokenize=<span class="hljs-literal">True</span>,
    return_dict=<span class="hljs-literal">True</span>,
).to(model.device)

<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;\\n&quot;</span> + <span class="hljs-string">&quot;=&quot;</span>*<span class="hljs-number">50</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Generation with other inputs!&quot;</span>)
<span class="hljs-keyword">with</span> TimerContext(<span class="hljs-string">&quot;Generation with different inputs&quot;</span>):
    _ = model.generate(**padded_inputs_2, **gen_kwargs)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;=&quot;</span>*<span class="hljs-number">50</span>)`,lang:"python",wrap:!1});var F=e(W,2);n(F,{title:"Training",local:"training",headingTag:"h3"});var V=e(F,4);c(V,{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwQXVkaW8lMkMlMjBsb2FkX2RhdGFzZXQlMEElMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Byb2Nlc3NvciUyQyUyMENzbUZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbiUwQSUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyc2VzYW1lJTJGY3NtLTFiJTIyJTBBJTBBJTIzJTIwbG9hZCUyMHRoZSUyMG1vZGVsJTIwYW5kJTIwdGhlJTIwcHJvY2Vzc29yJTBBcHJvY2Vzc29yJTIwJTNEJTIwQXV0b1Byb2Nlc3Nvci5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQpJTBBbW9kZWwlMjAlM0QlMjBDc21Gb3JDb25kaXRpb25hbEdlbmVyYXRpb24uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIpJTBBbW9kZWwudHJhaW4oKSUwQW1vZGVsLmNvZGVjX21vZGVsLmV2YWwoKSUwQSUwQWRzJTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMmhmLWludGVybmFsLXRlc3RpbmclMkZkYWlseXRhbGstZHVtbXklMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKSUwQSUyMyUyMGVuc3VyZSUyMHRoZSUyMGF1ZGlvJTIwaXMlMjAyNGtIeiUwQWRzJTIwJTNEJTIwZHMuY2FzdF9jb2x1bW4oJTIyYXVkaW8lMjIlMkMlMjBBdWRpbyhzYW1wbGluZ19yYXRlJTNEMjQwMDApKSUwQWNvbnZlcnNhdGlvbiUyMCUzRCUyMCU1QiU1RCUwQSUwQSUyMyUyMGNvbnRleHQlMEFmb3IlMjB0ZXh0JTJDJTIwYXVkaW8lMkMlMjBzcGVha2VyX2lkJTIwaW4lMjB6aXAoZHMlNUIlM0E0JTVEJTVCJTIydGV4dCUyMiU1RCUyQyUyMGRzJTVCJTNBNCU1RCU1QiUyMmF1ZGlvJTIyJTVEJTJDJTIwZHMlNUIlM0E0JTVEJTVCJTIyc3BlYWtlcl9pZCUyMiU1RCklM0ElMEElMjAlMjAlMjAlMjBjb252ZXJzYXRpb24uYXBwZW5kKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJvbGUlMjIlM0ElMjBmJTIyJTdCc3BlYWtlcl9pZCU3RCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlNUIlN0IlMjJ0eXBlJTIyJTNBJTIwJTIydGV4dCUyMiUyQyUyMCUyMnRleHQlMjIlM0ElMjB0ZXh0JTdEJTJDJTIwJTdCJTIydHlwZSUyMiUzQSUyMCUyMmF1ZGlvJTIyJTJDJTIwJTIycGF0aCUyMiUzQSUyMGF1ZGlvJTVCJTIyYXJyYXklMjIlNUQlN0QlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0QlMEElMjAlMjAlMjAlMjApJTBBJTBBaW5wdXRzJTIwJTNEJTIwcHJvY2Vzc29yLmFwcGx5X2NoYXRfdGVtcGxhdGUoJTBBJTIwJTIwJTIwJTIwY29udmVyc2F0aW9uJTJDJTBBJTIwJTIwJTIwJTIwdG9rZW5pemUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwcmV0dXJuX2RpY3QlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2xhYmVscyUzRFRydWUlMkMlMEEpLnRvKG1vZGVsLmRldmljZSklMEElMEFvdXQlMjAlM0QlMjBtb2RlbCgqKmlucHV0cyklMEFvdXQubG9zcy5iYWNrd2FyZCgp",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Audio, load_dataset

<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor, CsmForConditionalGeneration


model_id = <span class="hljs-string">&quot;sesame/csm-1b&quot;</span>

<span class="hljs-comment"># load the model and the processor</span>
processor = AutoProcessor.from_pretrained(model_id)
model = CsmForConditionalGeneration.from_pretrained(model_id, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
model.train()
model.codec_model.<span class="hljs-built_in">eval</span>()

ds = load_dataset(<span class="hljs-string">&quot;hf-internal-testing/dailytalk-dummy&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
<span class="hljs-comment"># ensure the audio is 24kHz</span>
ds = ds.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">24000</span>))
conversation = []

<span class="hljs-comment"># context</span>
<span class="hljs-keyword">for</span> text, audio, speaker_id <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(ds[:<span class="hljs-number">4</span>][<span class="hljs-string">&quot;text&quot;</span>], ds[:<span class="hljs-number">4</span>][<span class="hljs-string">&quot;audio&quot;</span>], ds[:<span class="hljs-number">4</span>][<span class="hljs-string">&quot;speaker_id&quot;</span>]):
    conversation.append(
        {
            <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">f&quot;<span class="hljs-subst">{speaker_id}</span>&quot;</span>,
            <span class="hljs-string">&quot;content&quot;</span>: [{<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: text}, {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;audio&quot;</span>, <span class="hljs-string">&quot;path&quot;</span>: audio[<span class="hljs-string">&quot;array&quot;</span>]}],
        }
    )

inputs = processor.apply_chat_template(
    conversation,
    tokenize=<span class="hljs-literal">True</span>,
    return_dict=<span class="hljs-literal">True</span>,
    output_labels=<span class="hljs-literal">True</span>,
).to(model.device)

out = model(**inputs)
out.loss.backward()`,lang:"python",wrap:!1});var x=e(V,4);n(x,{title:"CsmConfig",local:"transformers.CsmConfig",headingTag:"h2"});var y=e(x,2),A=a(y);t(A,{name:"class transformers.CsmConfig",anchor:"transformers.CsmConfig",source:"https://github.com/huggingface/transformers/blob/vr_48314/src/transformers/models/csm/configuration_csm.py#L90",parameters:[{name:"transformers_version",val:": str | None = None"},{name:"architectures",val:": list[str] | None = None"},{name:"output_hidden_states",val:": bool | None = False"},{name:"return_dict",val:": bool | None = True"},{name:"dtype",val:": typing.Union[str, ForwardRef('torch.dtype'), NoneType] = None"},{name:"chunk_size_feed_forward",val:": int = 0"},{name:"is_encoder_decoder",val:": bool = False"},{name:"id2label",val:": dict[int, str] | dict[str, str] | None = None"},{name:"label2id",val:": dict[str, int] | dict[str, str] | None = None"},{name:"problem_type",val:": typing.Optional[typing.Literal['regression', 'single_label_classification', 'multi_label_classification']] = None"},{name:"num_codebooks",val:": int | None = 32"},{name:"vocab_size",val:": int = 2051"},{name:"text_vocab_size",val:": int = 128256"},{name:"hidden_size",val:": int = 2048"},{name:"intermediate_size",val:": int = 8192"},{name:"num_hidden_layers",val:": int = 16"},{name:"num_attention_heads",val:": int = 32"},{name:"num_key_value_heads",val:": int | None = 8"},{name:"hidden_act",val:": str = 'silu'"},{name:"max_position_embeddings",val:": int = 2048"},{name:"initializer_range",val:": float = 0.02"},{name:"rms_norm_eps",val:": float = 1e-05"},{name:"use_cache",val:": bool = True"},{name:"pad_token_id",val:": int | None = 128002"},{name:"codebook_pad_token_id",val:": int | None = 2050"},{name:"codebook_eos_token_id",val:": int | list[int] | None = 0"},{name:"bos_token_id",val:": int | None = 128000"},{name:"eos_token_id",val:": int | list[int] | None = None"},{name:"audio_token_id",val:": int | None = 128002"},{name:"audio_eos_token_id",val:": int | list[int] | None = 128003"},{name:"rope_parameters",val:": transformers.modeling_rope_utils.RopeParameters | dict | None = None"},{name:"attention_bias",val:": bool = False"},{name:"attention_dropout",val:": float | int | None = 0.0"},{name:"mlp_bias",val:": bool = False"},{name:"head_dim",val:": int | None = None"},{name:"tie_codebooks_embeddings",val:": bool | None = True"},{name:"depth_decoder_config",val:": dict | transformers.configuration_utils.PreTrainedConfig | None = None"},{name:"codec_config",val:": dict | transformers.configuration_utils.PreTrainedConfig | None = None"}],parametersDescription:[{anchor:"transformers.CsmConfig.num_codebooks",description:`<strong>num_codebooks</strong> (<code>int</code>, <em>optional</em>, defaults to <code>32</code>) &#x2014;
The number of parallel codebooks used by the model.`,name:"num_codebooks"},{anchor:"transformers.CsmConfig.vocab_size",description:`<strong>vocab_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>2051</code>) &#x2014;
Vocabulary size of the model. Defines the number of different tokens that can be represented by the <code>input_ids</code>.`,name:"vocab_size"},{anchor:"transformers.CsmConfig.text_vocab_size",description:`<strong>text_vocab_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>128256</code>) &#x2014;
Vocabulary size of the model. Defines the number of different tokens that can be represented by the <code>input_ids</code>.`,name:"text_vocab_size"},{anchor:"transformers.CsmConfig.hidden_size",description:`<strong>hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>2048</code>) &#x2014;
Dimension of the hidden representations.`,name:"hidden_size"},{anchor:"transformers.CsmConfig.intermediate_size",description:`<strong>intermediate_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>8192</code>) &#x2014;
Dimension of the MLP representations.`,name:"intermediate_size"},{anchor:"transformers.CsmConfig.num_hidden_layers",description:`<strong>num_hidden_layers</strong> (<code>int</code>, <em>optional</em>, defaults to <code>16</code>) &#x2014;
Number of hidden layers in the Transformer decoder.`,name:"num_hidden_layers"},{anchor:"transformers.CsmConfig.num_attention_heads",description:`<strong>num_attention_heads</strong> (<code>int</code>, <em>optional</em>, defaults to <code>32</code>) &#x2014;
Number of attention heads for each attention layer in the Transformer decoder.`,name:"num_attention_heads"},{anchor:"transformers.CsmConfig.num_key_value_heads",description:`<strong>num_key_value_heads</strong> (<code>int</code>, <em>optional</em>, defaults to <code>8</code>) &#x2014;
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
<code>num_key_value_heads=num_attention_heads</code>, the model will use Multi Head Attention (MHA), if
<code>num_key_value_heads=1</code> the model will use Multi Query Attention (MQA) otherwise GQA is used. When
converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
by meanpooling all the original heads within that group. For more details, check out <a href="https://huggingface.co/papers/2305.13245" rel="nofollow">this
paper</a>. If it is not specified, will default to
<code>num_attention_heads</code>.`,name:"num_key_value_heads"},{anchor:"transformers.CsmConfig.hidden_act",description:`<strong>hidden_act</strong> (<code>str</code>, <em>optional</em>, defaults to <code>silu</code>) &#x2014;
The non-linear activation function (function or string) in the decoder. For example, <code>&quot;gelu&quot;</code>,
<code>&quot;relu&quot;</code>, <code>&quot;silu&quot;</code>, etc.`,name:"hidden_act"},{anchor:"transformers.CsmConfig.max_position_embeddings",description:`<strong>max_position_embeddings</strong> (<code>int</code>, <em>optional</em>, defaults to <code>2048</code>) &#x2014;
The maximum sequence length that this model might ever be used with.`,name:"max_position_embeddings"},{anchor:"transformers.CsmConfig.initializer_range",description:`<strong>initializer_range</strong> (<code>float</code>, <em>optional</em>, defaults to <code>0.02</code>) &#x2014;
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.`,name:"initializer_range"},{anchor:"transformers.CsmConfig.rms_norm_eps",description:`<strong>rms_norm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to <code>1e-05</code>) &#x2014;
The epsilon used by the rms normalization layers.`,name:"rms_norm_eps"},{anchor:"transformers.CsmConfig.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not the model should return the last key/values attentions (not used by all models). Only
relevant if <code>config.is_decoder=True</code> or when the model is a decoder-only generative model.`,name:"use_cache"},{anchor:"transformers.CsmConfig.pad_token_id",description:`<strong>pad_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to <code>128002</code>) &#x2014;
Token id used for padding in the vocabulary.`,name:"pad_token_id"},{anchor:"transformers.CsmConfig.codebook_pad_token_id",description:`<strong>codebook_pad_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to 2050) &#x2014;
Padding token id for codebook tokens.`,name:"codebook_pad_token_id"},{anchor:"transformers.CsmConfig.codebook_eos_token_id",description:`<strong>codebook_eos_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to 0) &#x2014;
End of stream token id for codebook tokens.`,name:"codebook_eos_token_id"},{anchor:"transformers.CsmConfig.bos_token_id",description:`<strong>bos_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to <code>128000</code>) &#x2014;
Token id used for beginning-of-stream in the vocabulary.`,name:"bos_token_id"},{anchor:"transformers.CsmConfig.eos_token_id",description:`<strong>eos_token_id</strong> (<code>Union[int, list[int]]</code>, <em>optional</em>) &#x2014;
Token id used for end-of-stream in the vocabulary.`,name:"eos_token_id"},{anchor:"transformers.CsmConfig.audio_token_id",description:`<strong>audio_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to 128002) &#x2014;
Audio token id in the text input.`,name:"audio_token_id"},{anchor:"transformers.CsmConfig.audio_eos_token_id",description:`<strong>audio_eos_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to 128003) &#x2014;
End of stream token id for audio in the text input.`,name:"audio_eos_token_id"},{anchor:"transformers.CsmConfig.rope_parameters",description:`<strong>rope_parameters</strong> (<code>Union[~modeling_rope_utils.RopeParameters, dict]</code>, <em>optional</em>) &#x2014;
Dictionary containing the configuration parameters for the RoPE embeddings. The dictionary should contain
a value for <code>rope_theta</code> and optionally parameters used for scaling in case you want to use RoPE
with longer <code>max_position_embeddings</code>.`,name:"rope_parameters"},{anchor:"transformers.CsmConfig.attention_bias",description:`<strong>attention_bias</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to use a bias in the query, key, value and output projection layers during self-attention.`,name:"attention_bias"},{anchor:"transformers.CsmConfig.attention_dropout",description:`<strong>attention_dropout</strong> (<code>Union[float, int]</code>, <em>optional</em>, defaults to <code>0.0</code>) &#x2014;
The dropout ratio for the attention probabilities.`,name:"attention_dropout"},{anchor:"transformers.CsmConfig.mlp_bias",description:`<strong>mlp_bias</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to use a bias in up_proj, down_proj and gate_proj layers in the MLP layers.`,name:"mlp_bias"},{anchor:"transformers.CsmConfig.head_dim",description:`<strong>head_dim</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The attention head dimension. If None, it will default to hidden_size // num_attention_heads`,name:"head_dim"},{anchor:"transformers.CsmConfig.tie_codebooks_embeddings",description:`<strong>tie_codebooks_embeddings</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to tie the codebook tokens embeddings of the backbone model to the codebook tokens embeddings of the depth decoder.`,name:"tie_codebooks_embeddings"},{anchor:"transformers.CsmConfig.depth_decoder_config",description:`<strong>depth_decoder_config</strong> (<code>CsmDepthDecoderConfig</code>, <em>optional</em>) &#x2014;
Configuration for the depth decoder.`,name:"depth_decoder_config"},{anchor:"transformers.CsmConfig.codec_config",description:`<strong>codec_config</strong> (<code>PreTrainedConfig</code>, <em>optional</em>) &#x2014;
Configuration for the codec.`,name:"codec_config"}]});var Me=e(A,6);M(Me,{anchor:"transformers.CsmConfig.example",children:(s,l)=>{c(s,{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMENzbUZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbiUyQyUyMENzbUNvbmZpZyUwQSUwQSUyMyUyMEluaXRpYWxpemluZyUyMGElMjBDc21Db25maWclMEFjb25maWd1cmF0aW9uJTIwJTNEJTIwQ3NtQ29uZmlnKCklMEElMEElMjMlMjBJbml0aWFsaXppbmclMjBhJTIwbW9kZWwlMEFtb2RlbCUyMCUzRCUyMENzbUZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbihjb25maWd1cmF0aW9uKSUwQSUwQSUyMyUyMEFjY2Vzc2luZyUyMHRoZSUyMG1vZGVsJTIwY29uZmlndXJhdGlvbiUwQWNvbmZpZ3VyYXRpb24lMjAlM0QlMjBtb2RlbC5jb25maWc=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> CsmForConditionalGeneration, CsmConfig

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a CsmConfig</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = CsmConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a model</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = CsmForConditionalGeneration(configuration)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config`,lang:"python",wrap:!1})},$$slots:{default:!0}}),r(y);var E=e(y,2);n(E,{title:"CsmDepthDecoderConfig",local:"transformers.CsmDepthDecoderConfig",headingTag:"h2"});var f=e(E,2),X=a(f);t(X,{name:"class transformers.CsmDepthDecoderConfig",anchor:"transformers.CsmDepthDecoderConfig",source:"https://github.com/huggingface/transformers/blob/vr_48314/src/transformers/models/csm/configuration_csm.py#L29",parameters:[{name:"transformers_version",val:": str | None = None"},{name:"architectures",val:": list[str] | None = None"},{name:"output_hidden_states",val:": bool | None = False"},{name:"return_dict",val:": bool | None = True"},{name:"dtype",val:": typing.Union[str, ForwardRef('torch.dtype'), NoneType] = None"},{name:"chunk_size_feed_forward",val:": int = 0"},{name:"is_encoder_decoder",val:": bool = False"},{name:"id2label",val:": dict[int, str] | dict[str, str] | None = None"},{name:"label2id",val:": dict[str, int] | dict[str, str] | None = None"},{name:"problem_type",val:": typing.Optional[typing.Literal['regression', 'single_label_classification', 'multi_label_classification']] = None"},{name:"num_codebooks",val:": int | None = 32"},{name:"backbone_hidden_size",val:": int = 2048"},{name:"vocab_size",val:": int = 2051"},{name:"hidden_size",val:": int = 1024"},{name:"intermediate_size",val:": int = 8192"},{name:"num_hidden_layers",val:": int = 4"},{name:"num_attention_heads",val:": int = 8"},{name:"num_key_value_heads",val:": int | None = 2"},{name:"hidden_act",val:": str = 'silu'"},{name:"max_position_embeddings",val:": int = 33"},{name:"initializer_range",val:": float = 0.02"},{name:"rms_norm_eps",val:": float = 1e-05"},{name:"use_cache",val:": bool = True"},{name:"pad_token_id",val:": int | None = None"},{name:"bos_token_id",val:": int | None = None"},{name:"eos_token_id",val:": int | list[int] | None = None"},{name:"rope_parameters",val:": transformers.modeling_rope_utils.RopeParameters | dict | None = None"},{name:"attention_bias",val:": bool = False"},{name:"attention_dropout",val:": float | int | None = 0.0"},{name:"mlp_bias",val:": bool = False"},{name:"head_dim",val:": int | None = None"}],parametersDescription:[{anchor:"transformers.CsmDepthDecoderConfig.num_codebooks",description:`<strong>num_codebooks</strong> (<code>int</code>, <em>optional</em>, defaults to <code>32</code>) &#x2014;
The number of parallel codebooks used by the model.`,name:"num_codebooks"},{anchor:"transformers.CsmDepthDecoderConfig.backbone_hidden_size",description:`<strong>backbone_hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to 2048) &#x2014;
Dimension of the hidden representations of the backbone model used with this depth decoder.`,name:"backbone_hidden_size"},{anchor:"transformers.CsmDepthDecoderConfig.vocab_size",description:`<strong>vocab_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>2051</code>) &#x2014;
Vocabulary size of the model. Defines the number of different tokens that can be represented by the <code>input_ids</code>.`,name:"vocab_size"},{anchor:"transformers.CsmDepthDecoderConfig.hidden_size",description:`<strong>hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>1024</code>) &#x2014;
Dimension of the hidden representations.`,name:"hidden_size"},{anchor:"transformers.CsmDepthDecoderConfig.intermediate_size",description:`<strong>intermediate_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>8192</code>) &#x2014;
Dimension of the MLP representations.`,name:"intermediate_size"},{anchor:"transformers.CsmDepthDecoderConfig.num_hidden_layers",description:`<strong>num_hidden_layers</strong> (<code>int</code>, <em>optional</em>, defaults to <code>4</code>) &#x2014;
Number of hidden layers in the Transformer decoder.`,name:"num_hidden_layers"},{anchor:"transformers.CsmDepthDecoderConfig.num_attention_heads",description:`<strong>num_attention_heads</strong> (<code>int</code>, <em>optional</em>, defaults to <code>8</code>) &#x2014;
Number of attention heads for each attention layer in the Transformer decoder.`,name:"num_attention_heads"},{anchor:"transformers.CsmDepthDecoderConfig.num_key_value_heads",description:`<strong>num_key_value_heads</strong> (<code>int</code>, <em>optional</em>, defaults to <code>2</code>) &#x2014;
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
<code>num_key_value_heads=num_attention_heads</code>, the model will use Multi Head Attention (MHA), if
<code>num_key_value_heads=1</code> the model will use Multi Query Attention (MQA) otherwise GQA is used. When
converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
by meanpooling all the original heads within that group. For more details, check out <a href="https://huggingface.co/papers/2305.13245" rel="nofollow">this
paper</a>. If it is not specified, will default to
<code>num_attention_heads</code>.`,name:"num_key_value_heads"},{anchor:"transformers.CsmDepthDecoderConfig.hidden_act",description:`<strong>hidden_act</strong> (<code>str</code>, <em>optional</em>, defaults to <code>silu</code>) &#x2014;
The non-linear activation function (function or string) in the decoder. For example, <code>&quot;gelu&quot;</code>,
<code>&quot;relu&quot;</code>, <code>&quot;silu&quot;</code>, etc.`,name:"hidden_act"},{anchor:"transformers.CsmDepthDecoderConfig.max_position_embeddings",description:`<strong>max_position_embeddings</strong> (<code>int</code>, <em>optional</em>, defaults to <code>33</code>) &#x2014;
The maximum sequence length that this model might ever be used with.`,name:"max_position_embeddings"},{anchor:"transformers.CsmDepthDecoderConfig.initializer_range",description:`<strong>initializer_range</strong> (<code>float</code>, <em>optional</em>, defaults to <code>0.02</code>) &#x2014;
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.`,name:"initializer_range"},{anchor:"transformers.CsmDepthDecoderConfig.rms_norm_eps",description:`<strong>rms_norm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to <code>1e-05</code>) &#x2014;
The epsilon used by the rms normalization layers.`,name:"rms_norm_eps"},{anchor:"transformers.CsmDepthDecoderConfig.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not the model should return the last key/values attentions (not used by all models). Only
relevant if <code>config.is_decoder=True</code> or when the model is a decoder-only generative model.`,name:"use_cache"},{anchor:"transformers.CsmDepthDecoderConfig.pad_token_id",description:`<strong>pad_token_id</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Token id used for padding in the vocabulary.`,name:"pad_token_id"},{anchor:"transformers.CsmDepthDecoderConfig.bos_token_id",description:`<strong>bos_token_id</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Token id used for beginning-of-stream in the vocabulary.`,name:"bos_token_id"},{anchor:"transformers.CsmDepthDecoderConfig.eos_token_id",description:`<strong>eos_token_id</strong> (<code>Union[int, list[int]]</code>, <em>optional</em>) &#x2014;
Token id used for end-of-stream in the vocabulary.`,name:"eos_token_id"},{anchor:"transformers.CsmDepthDecoderConfig.rope_parameters",description:`<strong>rope_parameters</strong> (<code>Union[~modeling_rope_utils.RopeParameters, dict]</code>, <em>optional</em>) &#x2014;
Dictionary containing the configuration parameters for the RoPE embeddings. The dictionary should contain
a value for <code>rope_theta</code> and optionally parameters used for scaling in case you want to use RoPE
with longer <code>max_position_embeddings</code>.`,name:"rope_parameters"},{anchor:"transformers.CsmDepthDecoderConfig.attention_bias",description:`<strong>attention_bias</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to use a bias in the query, key, value and output projection layers during self-attention.`,name:"attention_bias"},{anchor:"transformers.CsmDepthDecoderConfig.attention_dropout",description:`<strong>attention_dropout</strong> (<code>Union[float, int]</code>, <em>optional</em>, defaults to <code>0.0</code>) &#x2014;
The dropout ratio for the attention probabilities.`,name:"attention_dropout"},{anchor:"transformers.CsmDepthDecoderConfig.mlp_bias",description:`<strong>mlp_bias</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to use a bias in up_proj, down_proj and gate_proj layers in the MLP layers.`,name:"mlp_bias"},{anchor:"transformers.CsmDepthDecoderConfig.head_dim",description:`<strong>head_dim</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The attention head dimension. If None, it will default to hidden_size // num_attention_heads`,name:"head_dim"}]});var ge=e(X,6);M(ge,{anchor:"transformers.CsmDepthDecoderConfig.example",children:(s,l)=>{var o=j(),d=e(m(o),2);c(d,{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMENzbURlcHRoRGVjb2RlciUyQyUyMENzbURlcHRoRGVjb2RlckNvbmZpZyUwQSUwQSUyMyUyMEluaXRpYWxpemluZyUyMGElMjBDc21EZXB0aERlY29kZXIlMEFjb25maWd1cmF0aW9uJTIwJTNEJTIwQ3NtRGVwdGhEZWNvZGVyQ29uZmlnKCklMEFtb2RlbCUyMCUzRCUyMENzbURlcHRoRGVjb2Rlck1vZGVsKGNvbmZpZ3VyYXRpb24pJTBBJTBBJTIzJTIwQWNjZXNzaW5nJTIwdGhlJTIwbW9kZWwlMjBjb25maWd1cmF0aW9uJTBBY29uZmlndXJhdGlvbiUyMCUzRCUyMG1vZGVsLmNvbmZpZw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> CsmDepthDecoder, CsmDepthDecoderConfig

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a CsmDepthDecoder</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = CsmDepthDecoderConfig()
<span class="hljs-meta">&gt;&gt;&gt; </span>model = CsmDepthDecoderModel(configuration)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config`,lang:"python",wrap:!1}),i(s,o)},$$slots:{default:!0}}),r(f);var S=e(f,2);n(S,{title:"CsmProcessor",local:"transformers.CsmProcessor",headingTag:"h2"});var _=e(S,4),D=a(_);t(D,{name:"class transformers.CsmProcessor",anchor:"transformers.CsmProcessor",source:"https://github.com/huggingface/transformers/blob/vr_48314/src/transformers/models/csm/processing_csm.py#L72",parameters:[{name:"feature_extractor",val:""},{name:"tokenizer",val:""},{name:"chat_template",val:" = None"}],parametersDescription:[{anchor:"transformers.CsmProcessor.feature_extractor",description:`<strong>feature_extractor</strong> (<code>EncodecFeatureExtractor</code>) &#x2014;
The feature extractor is a required input.`,name:"feature_extractor"},{anchor:"transformers.CsmProcessor.tokenizer",description:`<strong>tokenizer</strong> (<code>tokenizer_class</code>) &#x2014;
The tokenizer is a required input.`,name:"tokenizer"},{anchor:"transformers.CsmProcessor.chat_template",description:`<strong>chat_template</strong> (<code>str</code>) &#x2014;
A Jinja template to convert lists of messages in a chat into a tokenizable string.`,name:"chat_template"}]});var Y=e(D,6),ye=a(Y);t(ye,{name:"__call__",anchor:"transformers.CsmProcessor.__call__",source:"https://github.com/huggingface/transformers/blob/vr_48314/src/transformers/models/csm/processing_csm.py#L166",parameters:[{name:"text",val:": str | list[str] | list[list[str]] | None"},{name:"audio",val:": typing.Union[numpy.ndarray, ForwardRef('torch.Tensor'), collections.abc.Sequence[numpy.ndarray], collections.abc.Sequence['torch.Tensor'], NoneType] = None"},{name:"output_labels",val:": bool | None = False"},{name:"depth_decoder_labels_ratio",val:": float | None = 1.0"},{name:"**kwargs",val:": Unpack"}],parametersDescription:[{anchor:"transformers.CsmProcessor.__call__.text",description:`<strong>text</strong> (<code>Union[str, list[str], list[list[str]]]</code>, <em>optional</em>) &#x2014;
The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
(pretokenized string). If you pass a pretokenized input, set <code>is_split_into_words=True</code> to avoid ambiguity with batched inputs.`,name:"text"},{anchor:"transformers.CsmProcessor.__call__.audio",description:`<strong>audio</strong> (<code>Union[numpy.ndarray, torch.Tensor, collections.abc.Sequence[numpy.ndarray], collections.abc.Sequence[torch.Tensor]]</code>, <em>optional</em>) &#x2014;
The audio or batch of audios to be prepared. Each audio can be a NumPy array or PyTorch tensor.
In case of a NumPy array/PyTorch tensor, each audio should be of shape (C, T), where C is a number of channels,
and T is the sample length of the audio.`,name:"audio"},{anchor:"transformers.CsmProcessor.__call__.output_labels",description:`<strong>output_labels</strong> (<code>bool</code>, <em>optional</em>, default=False) &#x2014;
Whether to return labels for training. Indices will be in <code>[config.audio_token_id, -100, -101]</code>.</p>
<ul>
<li><code>config.audio_token_id</code> indicates an audio frame (considering sequence length elements as frames)</li>
<li><code>-100</code> will be ignored in the loss computation</li>
<li><code>-101</code> indicates the audio frame will be used only for the backbone model (using the first codebook token as labels)</li>
</ul>`,name:"output_labels"},{anchor:"transformers.CsmProcessor.__call__.depth_decoder_labels_ratio",description:`<strong>depth_decoder_labels_ratio</strong> (<code>float</code>, <em>optional</em>, default=1.0) &#x2014;
The ratio of audio frames to keep for the depth decoder labels.`,name:"depth_decoder_labels_ratio"},{anchor:"transformers.CsmProcessor.__call__.encoded_length_kwargs",description:`<strong>encoded_length_kwargs</strong> (<code>dict[str, *kwargs*, Any]</code>, <em>optional</em>) &#x2014;
Dictionary of keyword arguments used to compute the encoded audio sequence length. This includes parameters
such as <code>kernel_sizes</code>, <code>strides</code>, <code>dilations</code>, and <code>use_causal_conv</code> that define the convolutional layers
used in audio encoding. The encoded length is used to determine how many audio tokens to generate for each
audio input in the text sequence.`,name:"encoded_length_kwargs"},{anchor:"transformers.CsmProcessor.__call__.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code> or <a href="/docs/transformers/pr_48314/en/internal/file_utils#transformers.TensorType">TensorType</a>, <em>optional</em>) &#x2014;
If set, will return tensors of a particular framework. Acceptable values are:</p>
<ul>
<li><code>&apos;pt&apos;</code>: Return PyTorch <code>torch.Tensor</code> objects.</li>
<li><code>&apos;np&apos;</code>: Return NumPy <code>np.ndarray</code> objects.</li>
</ul>`,name:"return_tensors"},{anchor:"transformers.CsmProcessor.__call__.*kwargs",description:`*<strong>*kwargs</strong> (<a href="/docs/transformers/pr_48314/en/main_classes/processors#transformers.ProcessingKwargs">ProcessingKwargs</a>, <em>optional</em>) &#x2014;
Additional processing options for each modality (text, images, videos, audio). Model-specific parameters
are listed above; see the TypedDict class for the complete list of supported arguments.`,name:"*kwargs"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <a
  href="/docs/transformers/pr_48314/en/main_classes/image_processor#transformers.BatchFeature"
>BatchFeature</a> with the following fields:</p>
<ul>
<li><strong>input_ids</strong> — List of token ids to be fed to a model. Returned when <code>text</code> is not <code>None</code>.</li>
<li><strong>input_values</strong> — List of audio values to be fed to a model. Returned when <code>audio</code> is not <code>None</code>.</li>
<li><strong>attention_mask</strong> — List of indices specifying which tokens should be attended to by the model (when
<code>return_attention_mask=True</code> or if <em>“attention_mask”</em> is in <code>self.model_input_names</code> and if <code>text</code> is not
<code>None</code>).</li>
<li><strong>labels</strong> — List of labels for the audio frames. Returned when <code>output_labels=True</code>.</li>
</ul>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><a
  href="/docs/transformers/pr_48314/en/main_classes/image_processor#transformers.BatchFeature"
>BatchFeature</a></p>
`}),r(Y),r(_);var H=e(_,2);n(H,{title:"CsmForConditionalGeneration",local:"transformers.CsmForConditionalGeneration",headingTag:"h2"});var U=e(H,2),P=a(U);t(P,{name:"class transformers.CsmForConditionalGeneration",anchor:"transformers.CsmForConditionalGeneration",source:"https://github.com/huggingface/transformers/blob/vr_48314/src/transformers/models/csm/modeling_csm.py#L752",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.CsmForConditionalGeneration.config",description:`<strong>config</strong> (<a href="/docs/transformers/pr_48314/en/model_doc/csm#transformers.CsmForConditionalGeneration">CsmForConditionalGeneration</a>) &#x2014;
Model configuration class with all the parameters of the model. Initializing with a config file does not
load the weights associated with the model, only the configuration. Check out the
<a href="/docs/transformers/pr_48314/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}]});var T=e(P,8),L=a(T);t(L,{name:"forward",anchor:"transformers.CsmForConditionalGeneration.forward",source:"https://github.com/huggingface/transformers/blob/vr_48314/src/transformers/models/csm/modeling_csm.py#L923",parameters:[{name:"input_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"input_values",val:": typing.Optional[torch.Tensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.Tensor] = None"},{name:"input_values_cutoffs",val:": typing.Optional[torch.Tensor] = None"},{name:"position_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"past_key_values",val:": transformers.cache_utils.Cache | None = None"},{name:"inputs_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"labels",val:": typing.Optional[torch.LongTensor] = None"},{name:"use_cache",val:": bool | None = None"},{name:"logits_to_keep",val:": typing.Union[int, torch.Tensor] = 0"},{name:"**kwargs",val:": Unpack"}],parametersDescription:[{anchor:"transformers.CsmForConditionalGeneration.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length, num_codebooks) or (batch_size, sequence_length)</code>) &#x2014;</p>
<ol>
<li>
<p>(batch_size, sequence_length): corresponds to the input sequence prepared with the processor from the text prompt. Such input
requires <code>input_values</code> to be provided so that audio can be encoded in codebook tokens and then merged with the text tokens.</p>
</li>
<li>
<p>(batch_size, sequence_length, num_codebooks): codebook tokens generated during the autoregressive decoding. Such input is not meant to be used by end users.</p>
</li>
</ol>
<p>Indices can be obtained using <a href="/docs/transformers/pr_48314/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/pr_48314/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/pr_48314/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.CsmForConditionalGeneration.forward.input_values",description:`<strong>input_values</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Float values of input raw speech waveform. Values can be obtained by loading a <code>.flac</code> or <code>.wav</code> audio file
into an array of type <code>list[float]</code>, a <code>numpy.ndarray</code> or a <code>torch.Tensor</code>, <em>e.g.</em> via the torchcodec library
(<code>pip install torchcodec</code>) or the soundfile library (<code>pip install soundfile</code>).
To prepare the array into <code>input_values</code>, the <a href="/docs/transformers/pr_48314/en/model_doc/auto#transformers.AutoProcessor">AutoProcessor</a> should be used for padding and conversion
into a tensor of type <code>torch.FloatTensor</code>. See <a href="/docs/transformers/pr_48314/en/model_doc/csm#transformers.CsmProcessor.__call__">CsmProcessor.<strong>call</strong>()</a> for details.`,name:"input_values"},{anchor:"transformers.CsmForConditionalGeneration.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.CsmForConditionalGeneration.forward.input_values_cutoffs",description:`<strong>input_values_cutoffs</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, max_num_audio)</code>, <em>optional</em>) &#x2014;
Specify the end positions of audio segments within each batch entry, relative to the concatenated audio input.
If a batch entry has fewer segments than the maximum, it is padded with -1. For example, in a batch of 2 sequences
where the first contains 2 audio segments of length l1, and the second contains 1 audio segment of length l2,
the input_values_cutoffs would be: [[l1, 2 * l1], [l2, -1]].`,name:"input_values_cutoffs"},{anchor:"transformers.CsmForConditionalGeneration.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>.</p>
<p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.CsmForConditionalGeneration.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>~cache_utils.Cache</code>, <em>optional</em>) &#x2014;
Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used to speed up sequential decoding. This typically consists in the <code>past_key_values</code>
returned by the model at a previous stage of decoding, when <code>use_cache=True</code> or <code>config.use_cache=True</code>.</p>
<p>Only <a href="/docs/transformers/pr_48314/en/internal/generation_utils#transformers.Cache">Cache</a> instance is allowed as input, see our <a href="https://huggingface.co/docs/transformers/en/kv_cache" rel="nofollow">kv cache guide</a>.
If no <code>past_key_values</code> are passed, <a href="/docs/transformers/pr_48314/en/internal/generation_utils#transformers.DynamicCache">DynamicCache</a> will be initialized by default.</p>
<p>The model will output the same cache format that is fed as input.</p>
<p>If <code>past_key_values</code> are used, the user is expected to input only unprocessed <code>input_ids</code> (those that don&#x2019;t
have their past key value states given to this model) of shape <code>(batch_size, unprocessed_length)</code> instead of all <code>input_ids</code>
of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.CsmForConditionalGeneration.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.CsmForConditionalGeneration.forward.labels",description:`<strong>labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Labels for computing the masked language modeling loss. Indices should be in <code>[config.audio_token_id, -100, -101]</code>.
Requires targeted <code>input_values</code> to be provided as audio tokens will be inferred from it using the <code>codec_model</code>.</p>
<ul>
<li><code>config.audio_token_id</code> indicates an audio frames (considering sequence length elements as frames)</li>
<li><code>-100</code> will be ignored in the loss computation</li>
<li><code>-101</code> indicates the audio frame will be used only for the backbone model (using the first codebook token as labels)</li>
</ul>
<p>Such labels can be prepared using <code>output_labels=True</code> when calling <a href="/docs/transformers/pr_48314/en/model_doc/csm#transformers.CsmProcessor">CsmProcessor</a>.`,name:"labels"},{anchor:"transformers.CsmForConditionalGeneration.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"},{anchor:"transformers.CsmForConditionalGeneration.forward.logits_to_keep",description:`<strong>logits_to_keep</strong> (<code>int</code> or <code>torch.Tensor</code>, <em>optional</em>) &#x2014;
Kept for compatibility. Does not support another value than:</p>
<ol>
<li><code>0</code>, which is equivalent to keeping all logits, used in the training regime</li>
<li><code>1</code>, which is equivalent to keeping only the last logit, used in the generation regime</li>
</ol>`,name:"logits_to_keep"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <code>CsmOutputWithPast</code> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/pr_48314/en/model_doc/csm#transformers.CsmConfig"
>CsmConfig</a>) and inputs.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>CsmOutputWithPast</code> or <code>tuple(torch.FloatTensor)</code></p>
`});var K=e(L,4);p(K,{children:(s,l)=>{var o=g();i(s,o)},$$slots:{default:!0}});var fe=e(K,4);M(fe,{anchor:"transformers.CsmForConditionalGeneration.forward.example",children:(s,l)=>{var o=j(),d=e(m(o),2);c(d,{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQ3NtRm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uJTJDJTIwQXV0b1Byb2Nlc3NvciUwQWZyb20lMjBkYXRhc2V0cyUyMGltcG9ydCUyMGxvYWRfZGF0YXNldCUyQyUyMEF1ZGlvJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJzZXNhbWUlMkZjc20tMWIlMjIlMEF0b3JjaF9kZXZpY2UlMjAlM0QlMjAlMjJjdWRhJTIyJTIwaWYlMjB0b3JjaC5jdWRhLmlzX2F2YWlsYWJsZSgpJTIwZWxzZSUyMCUyMmNwdSUyMiUwQSUwQXByb2Nlc3NvciUyMCUzRCUyMEF1dG9Qcm9jZXNzb3IuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQSUwQWRzJTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMmhmLWludGVybmFsLXRlc3RpbmclMkZkYWlseXRhbGstZHVtbXklMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKSUwQSUyMyUyMGVuc3VyZSUyMHRoZSUyMGF1ZGlvJTIwaXMlMjAyNGtIeiUwQWRzJTIwJTNEJTIwZHMuY2FzdF9jb2x1bW4oJTIyYXVkaW8lMjIlMkMlMjBBdWRpbyhzYW1wbGluZ19yYXRlJTNEMjQwMDApKSUwQSUwQWNvbnZlcnNhdGlvbiUyMCUzRCUyMCU1QiU1RCUwQSUyMyUyMHByZXBhcmUlMjBhJTIwY29udmVyc2F0aW9uJTIwd2l0aCUyMHRleHQlMjBhbmQlMjBjb3JyZXNwb25kaW5nJTIwYXVkaW8lMEFmb3IlMjB0ZXh0JTJDJTIwYXVkaW8lMkMlMjBzcGVha2VyX2lkJTIwaW4lMjB6aXAoZHMlNUIlM0E0JTVEJTVCJTIydGV4dCUyMiU1RCUyQyUyMGRzJTVCJTNBNCU1RCU1QiUyMmF1ZGlvJTIyJTVEJTJDJTIwZHMlNUIlM0E0JTVEJTVCJTIyc3BlYWtlcl9pZCUyMiU1RCklM0ElMEElMjAlMjAlMjAlMjBjb252ZXJzYXRpb24uYXBwZW5kKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJvbGUlMjIlM0ElMjBmJTIyJTdCc3BlYWtlcl9pZCU3RCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlNUIlN0IlMjJ0eXBlJTIyJTNBJTIwJTIydGV4dCUyMiUyQyUyMCUyMnRleHQlMjIlM0ElMjB0ZXh0JTdEJTJDJTIwJTdCJTIydHlwZSUyMiUzQSUyMCUyMmF1ZGlvJTIyJTJDJTIwJTIycGF0aCUyMiUzQSUyMGF1ZGlvJTVCJTIyYXJyYXklMjIlNUQlN0QlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0QlMEElMjAlMjAlMjAlMjApJTBBJTBBaW5wdXRzJTIwJTNEJTIwcHJvY2Vzc29yLmFwcGx5X2NoYXRfdGVtcGxhdGUoJTBBJTIwJTIwJTIwJTIwY29udmVyc2F0aW9uJTJDJTBBJTIwJTIwJTIwJTIwdG9rZW5pemUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwcmV0dXJuX2RpY3QlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2xhYmVscyUzRFRydWUlMkMlMEEpLnRvKHRvcmNoX2RldmljZSklMEElMEFtb2RlbCUyMCUzRCUyMENzbUZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbi5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQlMkMlMjBkZXZpY2VfbWFwJTNEdG9yY2hfZGV2aWNlKSUwQW91dHB1dCUyMCUzRCUyMG1vZGVsKCoqaW5wdXRzKSUwQW91dHB1dC5sb3NzLmJhY2t3YXJkKCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> CsmForConditionalGeneration, AutoProcessor
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, Audio

<span class="hljs-meta">&gt;&gt;&gt; </span>model_id = <span class="hljs-string">&quot;sesame/csm-1b&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>torch_device = <span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>processor = AutoProcessor.from_pretrained(model_id)

<span class="hljs-meta">&gt;&gt;&gt; </span>ds = load_dataset(<span class="hljs-string">&quot;hf-internal-testing/dailytalk-dummy&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># ensure the audio is 24kHz</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">24000</span>))

<span class="hljs-meta">&gt;&gt;&gt; </span>conversation = []
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># prepare a conversation with text and corresponding audio</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> text, audio, speaker_id <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(ds[:<span class="hljs-number">4</span>][<span class="hljs-string">&quot;text&quot;</span>], ds[:<span class="hljs-number">4</span>][<span class="hljs-string">&quot;audio&quot;</span>], ds[:<span class="hljs-number">4</span>][<span class="hljs-string">&quot;speaker_id&quot;</span>]):
<span class="hljs-meta">... </span>    conversation.append(
<span class="hljs-meta">... </span>        {
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">f&quot;<span class="hljs-subst">{speaker_id}</span>&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;content&quot;</span>: [{<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: text}, {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;audio&quot;</span>, <span class="hljs-string">&quot;path&quot;</span>: audio[<span class="hljs-string">&quot;array&quot;</span>]}],
<span class="hljs-meta">... </span>        }
<span class="hljs-meta">... </span>    )

<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = processor.apply_chat_template(
<span class="hljs-meta">... </span>    conversation,
<span class="hljs-meta">... </span>    tokenize=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    return_dict=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    output_labels=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>).to(torch_device)

<span class="hljs-meta">&gt;&gt;&gt; </span>model = CsmForConditionalGeneration.from_pretrained(model_id, device_map=torch_device)
<span class="hljs-meta">&gt;&gt;&gt; </span>output = model(**inputs)
<span class="hljs-meta">&gt;&gt;&gt; </span>output.loss.backward()`,lang:"python",wrap:!1}),i(s,o)},$$slots:{default:!0}}),r(T);var O=e(T,2),$=a(O);t($,{name:"generate",anchor:"transformers.CsmForConditionalGeneration.generate",source:"https://github.com/huggingface/transformers/blob/vr_48314/src/transformers/models/csm/generation_csm.py#L334",parameters:[{name:"input_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"input_values",val:": typing.Optional[torch.Tensor] = None"},{name:"input_values_cutoffs",val:": typing.Optional[torch.Tensor] = None"},{name:"generation_config",val:": transformers.generation.configuration_utils.GenerationConfig | None = None"},{name:"logits_processor",val:": transformers.generation.logits_process.LogitsProcessorList | None = None"},{name:"stopping_criteria",val:": transformers.generation.stopping_criteria.StoppingCriteriaList | None = None"},{name:"synced_gpus",val:": bool | None = None"},{name:"streamer",val:": typing.Optional[ForwardRef('BaseStreamer')] = None"},{name:"output_audio",val:": bool | None = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.CsmForConditionalGeneration.generate.inputs_ids",description:`<strong>inputs_ids</strong> (<code>torch.Tensor</code> of shape (batch_size, seq_length), <em>optional</em>) &#x2014;
The sequence used as a prompt for the backbone model.`,name:"inputs_ids"},{anchor:"transformers.CsmForConditionalGeneration.generate.input_values",description:`<strong>input_values</strong> (<code>torch.Tensor</code> of shape (batch_size, channels, max_concatenated_audio_length), <em>optional</em>) &#x2014;
The batched audio input values, where each batch entry contains the concatenation of all audio segments for that entry.
These values will be encoded into codebook tokens using the codec model and merged with the text input ids provided in <code>input_ids</code>.`,name:"input_values"},{anchor:"transformers.CsmForConditionalGeneration.generate.input_values_cutoffs",description:`<strong>input_values_cutoffs</strong> (<code>torch.Tensor</code> of shape (batch_size, max_num_audio), <em>optional</em>) &#x2014;
Specify the end positions of audio segments within each batch entry, relative to the concatenated audio input.
If a batch entry has fewer segments than the maximum, it is padded with -1. For example, in a batch of 2 sequences
where the first contains 2 audio segments of length l1, and the second contains 1 audio segment of length l2,
the input_values_cutoffs would be: [[l1, 2 * l1], [l2, -1]].`,name:"input_values_cutoffs"},{anchor:"transformers.CsmForConditionalGeneration.generate.generation_config",description:`<strong>generation_config</strong> (<a href="/docs/transformers/pr_48314/en/main_classes/text_generation#transformers.GenerationConfig">GenerationConfig</a>, <em>optional</em>) &#x2014;
The generation configuration to be used as base parametrization for the generation call. <code>**kwargs</code>
passed to generate matching the attributes of <code>generation_config</code> will override them. If
<code>generation_config</code> is not provided, the default will be used, which has the following loading
priority: 1) from the <code>generation_config.json</code> model file, if it exists; 2) from the model
configuration. Please note that unspecified parameters will inherit <a href="/docs/transformers/pr_48314/en/main_classes/text_generation#transformers.GenerationConfig">GenerationConfig</a>&#x2019;s
default values, whose documentation should be checked to parameterize generation.`,name:"generation_config"},{anchor:"transformers.CsmForConditionalGeneration.generate.logits_processor",description:`<strong>logits_processor</strong> (<code>LogitsProcessorList</code>, <em>optional</em>) &#x2014;
Custom logits processors that complement the default logits processors built from arguments and
generation config. If a logit processor is passed that is already created with the arguments or a
generation config an error is thrown. This feature is intended for advanced users.`,name:"logits_processor"},{anchor:"transformers.CsmForConditionalGeneration.generate.stopping_criteria",description:`<strong>stopping_criteria</strong> (<code>StoppingCriteriaList</code>, <em>optional</em>) &#x2014;
Custom stopping criteria that complements the default stopping criteria built from arguments and a
generation config. If a stopping criteria is passed that is already created with the arguments or a
generation config an error is thrown. If your stopping criteria depends on the <code>scores</code> input, make
sure you pass <code>return_dict_in_generate=True, output_scores=True</code> to <code>generate</code>. This feature is
intended for advanced users.`,name:"stopping_criteria"},{anchor:"transformers.CsmForConditionalGeneration.generate.synced_gpus",description:`<strong>synced_gpus</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether to continue running the while loop until max_length. Unless overridden, this flag will be set
to <code>True</code> if using <code>FullyShardedDataParallel</code> or DeepSpeed ZeRO Stage 3 with multiple GPUs to avoid
deadlocking if one GPU finishes generating before other GPUs. Otherwise, defaults to <code>False</code>.`,name:"synced_gpus"},{anchor:"transformers.CsmForConditionalGeneration.generate.streamer",description:`<strong>streamer</strong> (<code>BaseStreamer</code>, <em>optional</em>) &#x2014;
Streamer object that will be used to stream the generated sequences. Generated tokens are passed
through <code>streamer.put(token_ids)</code> and the streamer is responsible for any further processing.`,name:"streamer"},{anchor:"transformers.CsmForConditionalGeneration.generate.output_audio",description:`<strong>output_audio</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether to return the generated audio.`,name:"output_audio"},{anchor:"transformers.CsmForConditionalGeneration.generate.kwargs",description:`<strong>kwargs</strong> (<code>dict[str, Any]</code>, <em>optional</em>) &#x2014;
Ad hoc parametrization of <code>generation_config</code> and/or additional model-specific kwargs that will be
forwarded to the <code>forward</code> function of the model. Depth decoder specific kwargs should be prefixed with <em>depth<em>decoder</em></em>.`,name:"kwargs"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <code>CsmGenerateOutput</code>
(if <code>return_dict_in_generate=True</code> or when <code>config.return_dict_in_generate=True</code>) or a <code>torch.LongTensor</code> when <code>output_audio=False</code>
or a <code>list[torch.FloatTensor]</code> otherwise.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>CsmGenerateOutput</code> or <code>torch.LongTensor</code> or <code>list[torch.FloatTensor]</code></p>
`});var ee=e($,6);p(ee,{warning:!0,children:(s,l)=>{var o=Qe();i(s,o)},$$slots:{default:!0}});var _e=e(ee,2);M(_e,{anchor:"transformers.CsmForConditionalGeneration.generate.example",children:(s,l)=>{var o=j(),d=e(m(o),2);c(d,{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMENzbVByb2Nlc3NvciUyQyUyMENzbUZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbiUwQWZyb20lMjBkYXRhc2V0cyUyMGltcG9ydCUyMGxvYWRfZGF0YXNldCUyQyUyMEF1ZGlvJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJzZXNhbWUlMkZjc20tMWIlMjIlMEF0b3JjaF9kZXZpY2UlMjAlM0QlMjAlMjJjdWRhJTIyJTIwaWYlMjB0b3JjaC5jdWRhLmlzX2F2YWlsYWJsZSgpJTIwZWxzZSUyMCUyMmNwdSUyMiUwQSUwQXByb2Nlc3NvciUyMCUzRCUyMEF1dG9Qcm9jZXNzb3IuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQSUwQWRzJTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMmhmLWludGVybmFsLXRlc3RpbmclMkZkYWlseXRhbGstZHVtbXklMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKSUwQSUyMyUyMGVuc3VyZSUyMHRoZSUyMGF1ZGlvJTIwaXMlMjAyNGtIeiUwQWRzJTIwJTNEJTIwZHMuY2FzdF9jb2x1bW4oJTIyYXVkaW8lMjIlMkMlMjBBdWRpbyhzYW1wbGluZ19yYXRlJTNEMjQwMDApKSUwQSUwQWNvbnZlcnNhdGlvbiUyMCUzRCUyMCU1QiU1RCUwQSUyMyUyMHByZXBhcmUlMjBhJTIwY29udmVyc2F0aW9uJTIwd2l0aCUyMHRleHQlMjBhbmQlMjBjb3JyZXNwb25kaW5nJTIwYXVkaW8lMEFmb3IlMjB0ZXh0JTJDJTIwYXVkaW8lMkMlMjBzcGVha2VyX2lkJTIwaW4lMjB6aXAoZHMlNUIlM0E0JTVEJTVCJTIydGV4dCUyMiU1RCUyQyUyMGRzJTVCJTNBNCU1RCU1QiUyMmF1ZGlvJTIyJTVEJTJDJTIwZHMlNUIlM0E0JTVEJTVCJTIyc3BlYWtlcl9pZCUyMiU1RCklM0ElMEElMjAlMjAlMjAlMjBjb252ZXJzYXRpb24uYXBwZW5kKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJvbGUlMjIlM0ElMjBmJTIyJTdCc3BlYWtlcl9pZCU3RCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlNUIlN0IlMjJ0eXBlJTIyJTNBJTIwJTIydGV4dCUyMiUyQyUyMCUyMnRleHQlMjIlM0ElMjB0ZXh0JTdEJTJDJTIwJTdCJTIydHlwZSUyMiUzQSUyMCUyMmF1ZGlvJTIyJTJDJTIwJTIycGF0aCUyMiUzQSUyMGF1ZGlvJTVCJTIyYXJyYXklMjIlNUQlN0QlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0QlMEElMjAlMjAlMjAlMjApJTBBJTBBJTIzJTIwdGV4dCUyMHByb21wdCUwQWNvbnZlcnNhdGlvbi5hcHBlbmQoJTdCJTIycm9sZSUyMiUzQSUyMGYlMjIlN0JkcyU1QjQlNUQlNUInc3BlYWtlcl9pZCclNUQlN0QlMjIlMkMlMjAlMjJjb250ZW50JTIyJTNBJTIwJTVCJTdCJTIydHlwZSUyMiUzQSUyMCUyMnRleHQlMjIlMkMlMjAlMjJ0ZXh0JTIyJTNBJTIwZHMlNUI0JTVEJTVCJTIydGV4dCUyMiU1RCU3RCU1RCU3RCklMEElMEFpbnB1dHMlMjAlM0QlMjBwcm9jZXNzb3IuYXBwbHlfY2hhdF90ZW1wbGF0ZSglMEElMjAlMjAlMjAlMjBjb252ZXJzYXRpb24lMkMlMEElMjAlMjAlMjAlMjB0b2tlbml6ZSUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjByZXR1cm5fZGljdCUzRFRydWUlMkMlMEEpLnRvKHRvcmNoX2RldmljZSklMEElMEFtb2RlbCUyMCUzRCUyMENzbUZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbi5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQlMkMlMjBkZXZpY2VfbWFwJTNEdG9yY2hfZGV2aWNlKSUwQWF1ZGlvJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBvdXRwdXRfYXVkaW8lM0RUcnVlKSUwQXByb2Nlc3Nvci5zYXZlX2F1ZGlvKGF1ZGlvJTJDJTIwJTIyb3V0cHV0LndhdiUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> CsmProcessor, CsmForConditionalGeneration
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, Audio

<span class="hljs-meta">&gt;&gt;&gt; </span>model_id = <span class="hljs-string">&quot;sesame/csm-1b&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>torch_device = <span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>processor = AutoProcessor.from_pretrained(model_id)

<span class="hljs-meta">&gt;&gt;&gt; </span>ds = load_dataset(<span class="hljs-string">&quot;hf-internal-testing/dailytalk-dummy&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># ensure the audio is 24kHz</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">24000</span>))

<span class="hljs-meta">&gt;&gt;&gt; </span>conversation = []
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># prepare a conversation with text and corresponding audio</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> text, audio, speaker_id <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(ds[:<span class="hljs-number">4</span>][<span class="hljs-string">&quot;text&quot;</span>], ds[:<span class="hljs-number">4</span>][<span class="hljs-string">&quot;audio&quot;</span>], ds[:<span class="hljs-number">4</span>][<span class="hljs-string">&quot;speaker_id&quot;</span>]):
<span class="hljs-meta">... </span>    conversation.append(
<span class="hljs-meta">... </span>        {
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">f&quot;<span class="hljs-subst">{speaker_id}</span>&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;content&quot;</span>: [{<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: text}, {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;audio&quot;</span>, <span class="hljs-string">&quot;path&quot;</span>: audio[<span class="hljs-string">&quot;array&quot;</span>]}],
<span class="hljs-meta">... </span>        }
<span class="hljs-meta">... </span>    )

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># text prompt</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>conversation.append({<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">f&quot;<span class="hljs-subst">{ds[<span class="hljs-number">4</span>][<span class="hljs-string">&#x27;speaker_id&#x27;</span>]}</span>&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: [{<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: ds[<span class="hljs-number">4</span>][<span class="hljs-string">&quot;text&quot;</span>]}]})

<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = processor.apply_chat_template(
<span class="hljs-meta">... </span>    conversation,
<span class="hljs-meta">... </span>    tokenize=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    return_dict=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>).to(torch_device)

<span class="hljs-meta">&gt;&gt;&gt; </span>model = CsmForConditionalGeneration.from_pretrained(model_id, device_map=torch_device)
<span class="hljs-meta">&gt;&gt;&gt; </span>audio = model.generate(**inputs, output_audio=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>processor.save_audio(audio, <span class="hljs-string">&quot;output.wav&quot;</span>)`,lang:"python",wrap:!1}),i(s,o)},$$slots:{default:!0}}),r(O),r(U);var se=e(U,2);n(se,{title:"CsmDepthDecoderForCausalLM",local:"transformers.CsmDepthDecoderForCausalLM",headingTag:"h2"});var C=e(se,2),oe=a(C);t(oe,{name:"class transformers.CsmDepthDecoderForCausalLM",anchor:"transformers.CsmDepthDecoderForCausalLM",source:"https://github.com/huggingface/transformers/blob/vr_48314/src/transformers/models/csm/modeling_csm.py#L541",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.CsmDepthDecoderForCausalLM.config",description:`<strong>config</strong> (<a href="/docs/transformers/pr_48314/en/model_doc/csm#transformers.CsmDepthDecoderForCausalLM">CsmDepthDecoderForCausalLM</a>) &#x2014;
Model configuration class with all the parameters of the model. Initializing with a config file does not
load the weights associated with the model, only the configuration. Check out the
<a href="/docs/transformers/pr_48314/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}]});var ne=e(oe,8),te=a(ne);t(te,{name:"forward",anchor:"transformers.CsmDepthDecoderForCausalLM.forward",source:"https://github.com/huggingface/transformers/blob/vr_48314/src/transformers/models/csm/modeling_csm.py#L556",parameters:[{name:"input_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"backbone_last_hidden_state",val:": typing.Optional[torch.FloatTensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.Tensor] = None"},{name:"position_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"past_key_values",val:": transformers.cache_utils.Cache | None = None"},{name:"inputs_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"labels",val:": typing.Optional[torch.LongTensor] = None"},{name:"use_cache",val:": bool | None = None"},{name:"logits_to_keep",val:": typing.Union[int, torch.Tensor] = 0"},{name:"**kwargs",val:": Unpack"}],parametersDescription:[{anchor:"transformers.CsmDepthDecoderForCausalLM.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default.</p>
<p>Indices can be obtained using <a href="/docs/transformers/pr_48314/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/pr_48314/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/pr_48314/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.CsmDepthDecoderForCausalLM.forward.backbone_last_hidden_state",description:`<strong>backbone_last_hidden_state</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, backbone_hidden_size)</code>, <em>optional</em>) &#x2014;
The last hidden state of the backbone model. Such input is required when the first codebook token (the one generated by the backbone model)
is provided in the <code>input_ids</code> argument.`,name:"backbone_last_hidden_state"},{anchor:"transformers.CsmDepthDecoderForCausalLM.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.CsmDepthDecoderForCausalLM.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>.</p>
<p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.CsmDepthDecoderForCausalLM.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>~cache_utils.Cache</code>, <em>optional</em>) &#x2014;
Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used to speed up sequential decoding. This typically consists in the <code>past_key_values</code>
returned by the model at a previous stage of decoding, when <code>use_cache=True</code> or <code>config.use_cache=True</code>.</p>
<p>Only <a href="/docs/transformers/pr_48314/en/internal/generation_utils#transformers.Cache">Cache</a> instance is allowed as input, see our <a href="https://huggingface.co/docs/transformers/en/kv_cache" rel="nofollow">kv cache guide</a>.
If no <code>past_key_values</code> are passed, <a href="/docs/transformers/pr_48314/en/internal/generation_utils#transformers.DynamicCache">DynamicCache</a> will be initialized by default.</p>
<p>The model will output the same cache format that is fed as input.</p>
<p>If <code>past_key_values</code> are used, the user is expected to input only unprocessed <code>input_ids</code> (those that don&#x2019;t
have their past key value states given to this model) of shape <code>(batch_size, unprocessed_length)</code> instead of all <code>input_ids</code>
of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.CsmDepthDecoderForCausalLM.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.CsmDepthDecoderForCausalLM.forward.labels",description:`<strong>labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Labels for computing the masked language modeling loss. Indices should either be in <code>[0, ..., config.vocab_size]</code> or -100 (see <code>input_ids</code> docstring). Tokens with indices set to <code>-100</code> are ignored
(masked), the loss is only computed for the tokens with labels in <code>[0, ..., config.vocab_size]</code>.`,name:"labels"},{anchor:"transformers.CsmDepthDecoderForCausalLM.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"},{anchor:"transformers.CsmDepthDecoderForCausalLM.forward.logits_to_keep",description:`<strong>logits_to_keep</strong> (<code>Union[int, torch.Tensor]</code>, <em>optional</em>, defaults to <code>0</code>) &#x2014;
If an <code>int</code>, compute logits for the last <code>logits_to_keep</code> tokens. If <code>0</code>, calculate logits for all
<code>input_ids</code> (special case). Only last token logits are needed for generation, and calculating them only for that
token can save memory, which becomes pretty significant for long sequences or large vocabulary size.
If a <code>torch.Tensor</code>, must be 1D corresponding to the indices to keep in the sequence length dimension.
This is useful when using packed tensor format (single dimension for batch and sequence length).`,name:"logits_to_keep"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <a
  href="/docs/transformers/pr_48314/en/main_classes/output#transformers.modeling_outputs.CausalLMOutputWithPast"
>CausalLMOutputWithPast</a> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/pr_48314/en/model_doc/csm#transformers.CsmConfig"
>CsmConfig</a>) and inputs.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><a
  href="/docs/transformers/pr_48314/en/main_classes/output#transformers.modeling_outputs.CausalLMOutputWithPast"
>CausalLMOutputWithPast</a> or <code>tuple(torch.FloatTensor)</code></p>
`});var Ue=e(te,4);p(Ue,{children:(s,l)=>{var o=g();i(s,o)},$$slots:{default:!0}}),u(2),r(ne),r(C);var ae=e(C,2);n(ae,{title:"CsmDepthDecoderModel",local:"transformers.CsmDepthDecoderModel",headingTag:"h2"});var b=e(ae,2),re=a(b);t(re,{name:"class transformers.CsmDepthDecoderModel",anchor:"transformers.CsmDepthDecoderModel",source:"https://github.com/huggingface/transformers/blob/vr_48314/src/transformers/models/csm/modeling_csm.py#L411",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.CsmDepthDecoderModel.config",description:`<strong>config</strong> (<a href="/docs/transformers/pr_48314/en/model_doc/csm#transformers.CsmDepthDecoderModel">CsmDepthDecoderModel</a>) &#x2014;
Model configuration class with all the parameters of the model. Initializing with a config file does not
load the weights associated with the model, only the configuration. Check out the
<a href="/docs/transformers/pr_48314/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}]});var le=e(re,8),ie=a(le);t(ie,{name:"forward",anchor:"transformers.CsmDepthDecoderModel.forward",source:"https://github.com/huggingface/transformers/blob/vr_48314/src/transformers/models/csm/modeling_csm.py#L430",parameters:[{name:"input_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"backbone_last_hidden_state",val:": typing.Optional[torch.FloatTensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.Tensor] = None"},{name:"position_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"past_key_values",val:": transformers.cache_utils.Cache | None = None"},{name:"inputs_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"use_cache",val:": bool | None = None"},{name:"**kwargs",val:": Unpack"}],parametersDescription:[{anchor:"transformers.CsmDepthDecoderModel.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default.</p>
<p>Indices can be obtained using <a href="/docs/transformers/pr_48314/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/pr_48314/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/pr_48314/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.CsmDepthDecoderModel.forward.backbone_last_hidden_state",description:`<strong>backbone_last_hidden_state</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, backbone_hidden_size)</code>, <em>optional</em>) &#x2014;
The last hidden state of the backbone model. Such input is required when the first codebook token (the one generated by the backbone model)
is provided in the <code>input_ids</code> argument.`,name:"backbone_last_hidden_state"},{anchor:"transformers.CsmDepthDecoderModel.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.CsmDepthDecoderModel.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>.</p>
<p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.CsmDepthDecoderModel.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>~cache_utils.Cache</code>, <em>optional</em>) &#x2014;
Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used to speed up sequential decoding. This typically consists in the <code>past_key_values</code>
returned by the model at a previous stage of decoding, when <code>use_cache=True</code> or <code>config.use_cache=True</code>.</p>
<p>Only <a href="/docs/transformers/pr_48314/en/internal/generation_utils#transformers.Cache">Cache</a> instance is allowed as input, see our <a href="https://huggingface.co/docs/transformers/en/kv_cache" rel="nofollow">kv cache guide</a>.
If no <code>past_key_values</code> are passed, <a href="/docs/transformers/pr_48314/en/internal/generation_utils#transformers.DynamicCache">DynamicCache</a> will be initialized by default.</p>
<p>The model will output the same cache format that is fed as input.</p>
<p>If <code>past_key_values</code> are used, the user is expected to input only unprocessed <code>input_ids</code> (those that don&#x2019;t
have their past key value states given to this model) of shape <code>(batch_size, unprocessed_length)</code> instead of all <code>input_ids</code>
of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.CsmDepthDecoderModel.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.CsmDepthDecoderModel.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <a
  href="/docs/transformers/pr_48314/en/main_classes/output#transformers.modeling_outputs.BaseModelOutputWithPast"
>BaseModelOutputWithPast</a> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/pr_48314/en/model_doc/csm#transformers.CsmConfig"
>CsmConfig</a>) and inputs.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><a
  href="/docs/transformers/pr_48314/en/main_classes/output#transformers.modeling_outputs.BaseModelOutputWithPast"
>BaseModelOutputWithPast</a> or <code>tuple(torch.FloatTensor)</code></p>
`});var Te=e(ie,4);p(Te,{children:(s,l)=>{var o=g();i(s,o)},$$slots:{default:!0}}),u(2),r(le),r(b);var ce=e(b,2);n(ce,{title:"CsmBackboneModel",local:"transformers.CsmBackboneModel",headingTag:"h2"});var J=e(ce,2),de=a(J);t(de,{name:"class transformers.CsmBackboneModel",anchor:"transformers.CsmBackboneModel",source:"https://github.com/huggingface/transformers/blob/vr_48314/src/transformers/models/csm/modeling_csm.py#L663",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.CsmBackboneModel.config",description:`<strong>config</strong> (<a href="/docs/transformers/pr_48314/en/model_doc/csm#transformers.CsmBackboneModel">CsmBackboneModel</a>) &#x2014;
Model configuration class with all the parameters of the model. Initializing with a config file does not
load the weights associated with the model, only the configuration. Check out the
<a href="/docs/transformers/pr_48314/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}]});var pe=e(de,8),he=a(pe);t(he,{name:"forward",anchor:"transformers.CsmBackboneModel.forward",source:"https://github.com/huggingface/transformers/blob/vr_48314/src/transformers/models/csm/modeling_csm.py#L679",parameters:[{name:"input_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.Tensor] = None"},{name:"position_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"past_key_values",val:": transformers.cache_utils.Cache | None = None"},{name:"inputs_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"use_cache",val:": bool | None = None"},{name:"**kwargs",val:": Unpack"}],parametersDescription:[{anchor:"transformers.CsmBackboneModel.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length, num_codebooks) or (batch_size, sequence_length)</code>) &#x2014;</p>
<ol>
<li>
<p>(batch_size, sequence_length): corresponds to the input sequence prepared with the processor from the text prompt. Such input
requires <code>input_values</code> to be provided so that audio can be encoded in codebook tokens and then merged with the text tokens.</p>
</li>
<li>
<p>(batch_size, sequence_length, num_codebooks): codebook tokens generated during the autoregressive decoding. Such input is not meant to be used by end users.</p>
</li>
</ol>
<p>Indices can be obtained using <a href="/docs/transformers/pr_48314/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/pr_48314/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/pr_48314/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.CsmBackboneModel.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.CsmBackboneModel.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>.</p>
<p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.CsmBackboneModel.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>~cache_utils.Cache</code>, <em>optional</em>) &#x2014;
Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used to speed up sequential decoding. This typically consists in the <code>past_key_values</code>
returned by the model at a previous stage of decoding, when <code>use_cache=True</code> or <code>config.use_cache=True</code>.</p>
<p>Only <a href="/docs/transformers/pr_48314/en/internal/generation_utils#transformers.Cache">Cache</a> instance is allowed as input, see our <a href="https://huggingface.co/docs/transformers/en/kv_cache" rel="nofollow">kv cache guide</a>.
If no <code>past_key_values</code> are passed, <a href="/docs/transformers/pr_48314/en/internal/generation_utils#transformers.DynamicCache">DynamicCache</a> will be initialized by default.</p>
<p>The model will output the same cache format that is fed as input.</p>
<p>If <code>past_key_values</code> are used, the user is expected to input only unprocessed <code>input_ids</code> (those that don&#x2019;t
have their past key value states given to this model) of shape <code>(batch_size, unprocessed_length)</code> instead of all <code>input_ids</code>
of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.CsmBackboneModel.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.CsmBackboneModel.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <a
  href="/docs/transformers/pr_48314/en/main_classes/output#transformers.modeling_outputs.BaseModelOutputWithPast"
>BaseModelOutputWithPast</a> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/pr_48314/en/model_doc/csm#transformers.CsmConfig"
>CsmConfig</a>) and inputs.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><a
  href="/docs/transformers/pr_48314/en/main_classes/output#transformers.modeling_outputs.BaseModelOutputWithPast"
>BaseModelOutputWithPast</a> or <code>tuple(torch.FloatTensor)</code></p>
`});var Ce=e(he,4);p(Ce,{children:(s,l)=>{var o=g();i(s,o)},$$slots:{default:!0}}),u(2),r(pe),r(J);var be=e(J,2);ke(be,{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/model_doc/csm.md"}),u(2),i(me,w),Ze()}export{xe as component};