Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / transformers /pr_43265 /en /tasks /audio_text_to_text.html

HuggingFaceDocBuilder

about 2 months ago

download

raw

69.9 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Audio-text-to-text","local":"audio-text-to-text","sections":[{"title":"Load AudioCaps dataset","local":"load-audiocaps-dataset","sections":[],"depth":2},{"title":"Preprocess","local":"preprocess","sections":[],"depth":2},{"title":"Train","local":"train","sections":[{"title":"Configure LoRA","local":"configure-lora","sections":[],"depth":3},{"title":"Setup training","local":"setup-training","sections":[],"depth":3}],"depth":2},{"title":"Inference","local":"inference","sections":[],"depth":2},{"title":"Pipeline","local":"pipeline","sections":[],"depth":2},{"title":"Further Reading","local":"further-reading","sections":[],"depth":2}],"depth":1}">
	<link href="/docs/transformers/pr_43265/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/entry/start.1dae3980.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/scheduler.31fdf58d.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/singletons.5fcb8d34.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/index.252883d5.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/paths.e648cef5.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/entry/app.66dba7a3.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/preload-helper.5b8be175.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/index.2f76fdf0.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/nodes/0.4837e553.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/nodes/674.90fbf8cf.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/CopyLLMTxtMenu.53b607bf.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.08750ec0.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/IconCopy.ac192424.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/CodeBlock.e52df5d6.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/DocNotebookDropdown.dd28433e.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Audio-text-to-text","local":"audio-text-to-text","sections":[{"title":"Load AudioCaps dataset","local":"load-audiocaps-dataset","sections":[],"depth":2},{"title":"Preprocess","local":"preprocess","sections":[],"depth":2},{"title":"Train","local":"train","sections":[{"title":"Configure LoRA","local":"configure-lora","sections":[],"depth":3},{"title":"Setup training","local":"setup-training","sections":[],"depth":3}],"depth":2},{"title":"Inference","local":"inference","sections":[],"depth":2},{"title":"Pipeline","local":"pipeline","sections":[],"depth":2},{"title":"Further Reading","local":"further-reading","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <div class="flex space-x-1 " style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"> <div class="relative colab-dropdown "> <button class=" " type="button"> <img alt="Open In Colab" class="!m-0" src="https://colab.research.google.com/assets/colab-badge.svg"> </button> </div> <div class="relative colab-dropdown "> <button class=" " type="button"> <img alt="Open In Studio Lab" class="!m-0" src="https://studiolab.sagemaker.aws/studiolab.svg"> </button> </div></div> <h1 class="relative group"><a id="audio-text-to-text" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#audio-text-to-text"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Audio-text-to-text</span></h1> <p data-svelte-h="svelte-1h47o0u">Audio-text-to-text models accept both audio and text as inputs and generate text as output. They combine audio understanding with language generation, enabling tasks like audio question answering (e.g., “What is being said in this clip?”), audio reasoning (e.g., “What emotion does the speaker convey?”), and spoken dialogue understanding. Unlike traditional automatic speech recognition (ASR) models that only transcribe speech into text, audio-text-to-text models can reason about the audio content, follow complex instructions, and produce contextual responses based on what they hear.</p> <p data-svelte-h="svelte-1nchqjm">The example below shows how to load a model and processor, pass an audio file with a text prompt, and generate a response. In this case, we ask the model to transcribe a speech recording.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AudioFlamingo3ForConditionalGeneration, AutoProcessor

	model_id = <span class="hljs-string">"nvidia/audio-flamingo-3-hf"</span>
	processor = AutoProcessor.from_pretrained(model_id)
	model = AudioFlamingo3ForConditionalGeneration.from_pretrained(model_id, device_map=<span class="hljs-string">"auto"</span>)

	conversation = [
	{
	<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>,
	<span class="hljs-string">"content"</span>: [
	{<span class="hljs-string">"type"</span>: <span class="hljs-string">"text"</span>, <span class="hljs-string">"text"</span>: <span class="hljs-string">"Transcribe the input speech."</span>},
	{<span class="hljs-string">"type"</span>: <span class="hljs-string">"audio"</span>, <span class="hljs-string">"path"</span>: <span class="hljs-string">"https://huggingface.co/datasets/nvidia/AudioSkills/resolve/main/assets/WhDJDIviAOg_120_10.mp3"</span>},
	],
	}
	]

	inputs = processor.apply_chat_template(
	conversation,
	tokenize=<span class="hljs-literal">True</span>,
	add_generation_prompt=<span class="hljs-literal">True</span>,
	return_dict=<span class="hljs-literal">True</span>,
	).to(model.device)

	outputs = model.generate(**inputs, max_new_tokens=<span class="hljs-number">500</span>)

	decoded_outputs = processor.batch_decode(outputs[:, inputs.input_ids.shape[<span class="hljs-number">1</span>]:], skip_special_tokens=<span class="hljs-literal">True</span>)
	<span class="hljs-built_in">print</span>(decoded_outputs)
	<span class="hljs-comment">## ["The transcription of the audio is 'summer follows spring the days grow longer and the nights are warm'."]</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1aff4p7">This guide will show you how to:</p> <ol data-svelte-h="svelte-1gg247d"><li>Fine-tune <a href="https://huggingface.co/nvidia/audio-flamingo-3-hf" rel="nofollow">Audio Flamingo 3</a> on the <a href="https://huggingface.co/datasets/OpenSound/AudioCaps" rel="nofollow">AudioCaps</a> dataset for audio captioning using LoRA.</li> <li>Use your fine-tuned model for inference.</li></ol> <blockquote class="tip" data-svelte-h="svelte-r8zwt1"><p>To see all architectures and checkpoints compatible with this task, we recommend checking the <a href="https://huggingface.co/tasks/audio-text-to-text" rel="nofollow">task-page</a>.</p></blockquote> <p data-svelte-h="svelte-1c9nexd">Before you begin, make sure you have all the necessary libraries installed:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START -->pip install transformers datasets peft accelerate<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-k76o1m">We encourage you to login to your Hugging Face account so you can upload and share your model with the community. When prompted, enter your token to login:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login
	<span class="hljs-meta">>>> </span>notebook_login()<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="load-audiocaps-dataset" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#load-audiocaps-dataset"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Load AudioCaps dataset</span></h2> <p data-svelte-h="svelte-896rl0">Start by loading the <a href="https://huggingface.co/datasets/OpenSound/AudioCaps" rel="nofollow">AudioCaps</a> dataset from the 🤗 Datasets library in streaming mode. This dataset contains audio clips with descriptive captions, perfect for audio captioning tasks.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, Audio
	<span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">"OpenSound/AudioCaps"</span>, split=<span class="hljs-string">"train"</span>, streaming=<span class="hljs-literal">True</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1m00aez">Cast the audio column to 16kHz, which is required by Audio Flamingo’s Whisper feature extractor:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>dataset = dataset.cast_column(<span class="hljs-string">"audio"</span>, Audio(sampling_rate=<span class="hljs-number">16000</span>))<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-19v1ny2">Split the dataset into train and test sets using <code>.take()</code> and <code>.skip()</code> for streaming datasets:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>train_dataset = dataset.take(<span class="hljs-number">1000</span>)
	<span class="hljs-meta">>>> </span>eval_dataset = dataset.skip(<span class="hljs-number">1000</span>).take(<span class="hljs-number">100</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-13i9wub">Take a look at an example:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(train_dataset))
	{<span class="hljs-string">'audio'</span>: {<span class="hljs-string">'array'</span>: array([...], dtype=float32),
	<span class="hljs-string">'path'</span>: <span class="hljs-string">'...'</span>,
	<span class="hljs-string">'sampling_rate'</span>: <span class="hljs-number">16000</span>},
	<span class="hljs-string">'caption'</span>: <span class="hljs-string">'A man speaks followed by footsteps'</span>}<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-91kryu">The dataset contains:</p> <ul data-svelte-h="svelte-1a8r0ok"><li><code>audio</code>: the audio waveform</li> <li><code>caption</code>: the descriptive text caption for the audio</li></ul> <h2 class="relative group"><a id="preprocess" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#preprocess"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Preprocess</span></h2> <p data-svelte-h="svelte-j1cww1">Load the Audio Flamingo processor to handle both audio and text inputs:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor
	<span class="hljs-meta">>>> </span>processor = AutoProcessor.from_pretrained(<span class="hljs-string">"nvidia/audio-flamingo-3-hf"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-kdlkuq">Create a data collator that processes audio-text pairs into the format expected by Audio Flamingo. The collator uses the chat template format with direct audio arrays:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">class</span> <span class="hljs-title class_">AudioFlamingo3DataCollator</span>:
	<span class="hljs-meta">... </span> <span class="hljs-string">"""Data collator for Audio Flamingo 3 audio captioning training."""</span>
	...
	<span class="hljs-meta">... </span> <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, processor</span>):
	<span class="hljs-meta">... </span> self.processor = processor
	...
	<span class="hljs-meta">... </span> <span class="hljs-keyword">def</span> <span class="hljs-title function_">__call__</span>(<span class="hljs-params">self, features</span>):
	<span class="hljs-meta">... </span> conversations = []
	...
	<span class="hljs-meta">... </span> <span class="hljs-keyword">for</span> feature <span class="hljs-keyword">in</span> features:
	<span class="hljs-meta">... </span> <span class="hljs-comment"># Build conversation format for Audio Flamingo</span>
	<span class="hljs-meta">... </span> <span class="hljs-comment"># Audio is passed directly as an array, no base64 encoding needed</span>
	<span class="hljs-meta">... </span> sample = [
	<span class="hljs-meta">... </span> {
	<span class="hljs-meta">... </span> <span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>,
	<span class="hljs-meta">... </span> <span class="hljs-string">"content"</span>: [
	<span class="hljs-meta">... </span> {<span class="hljs-string">"type"</span>: <span class="hljs-string">"text"</span>, <span class="hljs-string">"text"</span>: <span class="hljs-string">"Describe the audio."</span>},
	<span class="hljs-meta">... </span> {<span class="hljs-string">"type"</span>: <span class="hljs-string">"audio"</span>, <span class="hljs-string">"audio"</span>: feature[<span class="hljs-string">"audio"</span>][<span class="hljs-string">"array"</span>]},
	<span class="hljs-meta">... </span> ],
	<span class="hljs-meta">... </span> },
	<span class="hljs-meta">... </span> {
	<span class="hljs-meta">... </span> <span class="hljs-string">"role"</span>: <span class="hljs-string">"assistant"</span>,
	<span class="hljs-meta">... </span> <span class="hljs-string">"content"</span>: [{<span class="hljs-string">"type"</span>: <span class="hljs-string">"text"</span>, <span class="hljs-string">"text"</span>: feature[<span class="hljs-string">"caption"</span>]}],
	<span class="hljs-meta">... </span> }
	<span class="hljs-meta">... </span> ]
	<span class="hljs-meta">... </span> conversations.append(sample)
	...
	<span class="hljs-meta">... </span> <span class="hljs-comment"># Apply chat template and format labels for training</span>
	<span class="hljs-meta">... </span> <span class="hljs-keyword">return</span> self.processor.apply_chat_template(
	<span class="hljs-meta">... </span> conversations,
	<span class="hljs-meta">... </span> tokenize=<span class="hljs-literal">True</span>,
	<span class="hljs-meta">... </span> add_generation_prompt=<span class="hljs-literal">False</span>,
	<span class="hljs-meta">... </span> return_dict=<span class="hljs-literal">True</span>,
	<span class="hljs-meta">... </span> output_labels=<span class="hljs-literal">True</span>, <span class="hljs-comment"># Automatically creates labels for training</span>
	<span class="hljs-meta">... </span> )<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1jv3dyf">Instantiate the data collator:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>data_collator = AudioFlamingo3DataCollator(processor)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="train" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#train"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Train</span></h2> <blockquote class="tip" data-svelte-h="svelte-1vsly5i"><p>If you aren’t familiar with fine-tuning a model with the <a href="/docs/transformers/pr_43265/en/main_classes/trainer#transformers.Trainer">Trainer</a>, take a look at the basic tutorial <a href="../training">here</a>!</p></blockquote> <p data-svelte-h="svelte-1p74rdv">Load the Audio Flamingo model. We use <code>bfloat16</code> precision and <code>device_map="auto"</code> for efficient memory usage:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AudioFlamingo3ForConditionalGeneration
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> torch
	<span class="hljs-meta">>>> </span>model = AudioFlamingo3ForConditionalGeneration.from_pretrained(
	<span class="hljs-meta">... </span> <span class="hljs-string">"nvidia/audio-flamingo-3-hf"</span>,
	<span class="hljs-meta">... </span> torch_dtype=torch.bfloat16,
	<span class="hljs-meta">... </span> device_map=<span class="hljs-string">"auto"</span>,
	<span class="hljs-meta">... </span>)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="configure-lora" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#configure-lora"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Configure LoRA</span></h3> <p data-svelte-h="svelte-s96d6t"><a href="https://huggingface.co/docs/peft/conceptual_guides/adapter#low-rank-adaptation-lora" rel="nofollow">LoRA</a> (Low-Rank Adaptation) enables efficient fine-tuning by only training a small number of additional parameters. Configure LoRA to target the language model’s attention and feed-forward layers:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> LoraConfig, get_peft_model
	<span class="hljs-meta">>>> </span>lora_config = LoraConfig(
	<span class="hljs-meta">... </span> r=<span class="hljs-number">16</span>, <span class="hljs-comment"># LoRA rank</span>
	<span class="hljs-meta">... </span> lora_alpha=<span class="hljs-number">32</span>, <span class="hljs-comment"># LoRA scaling factor</span>
	<span class="hljs-meta">... </span> target_modules=[
	<span class="hljs-meta">... </span> <span class="hljs-comment"># Language model attention</span>
	<span class="hljs-meta">... </span> <span class="hljs-string">"q_proj"</span>,
	<span class="hljs-meta">... </span> <span class="hljs-string">"k_proj"</span>,
	<span class="hljs-meta">... </span> <span class="hljs-string">"v_proj"</span>,
	<span class="hljs-meta">... </span> <span class="hljs-string">"o_proj"</span>,
	<span class="hljs-meta">... </span> <span class="hljs-comment"># Feed-forward layers</span>
	<span class="hljs-meta">... </span> <span class="hljs-string">"gate_proj"</span>,
	<span class="hljs-meta">... </span> <span class="hljs-string">"up_proj"</span>,
	<span class="hljs-meta">... </span> <span class="hljs-string">"down_proj"</span>,
	<span class="hljs-meta">... </span> ],
	<span class="hljs-meta">... </span> lora_dropout=<span class="hljs-number">0.05</span>,
	<span class="hljs-meta">... </span> bias=<span class="hljs-string">"none"</span>,
	<span class="hljs-meta">... </span> task_type=<span class="hljs-string">"CAUSAL_LM"</span>,
	<span class="hljs-meta">... </span>)
	<span class="hljs-meta">>>> </span>model = get_peft_model(model, lora_config)
	<span class="hljs-meta">>>> </span>model.print_trainable_parameters()<!-- HTML_TAG_END --></pre></div> <blockquote data-svelte-h="svelte-183lqxp"><p>[!TIP][LoRA](<a href="https://huggingface.co/docs/peft/main/conceptual_guides/lora" rel="nofollow">https://huggingface.co/docs/peft/main/conceptual_guides/lora</a>) significantly reduces memory usage and training time by only updating a small number of adapter parameters instead of the full model. This configuration targets the language model’s attention and feed-forward layers while keeping the audio encoder frozen, making it possible to fine-tune on a single GPU.</p></blockquote> <h3 class="relative group"><a id="setup-training" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#setup-training"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Setup training</span></h3> <p data-svelte-h="svelte-1vyhm8p">Define training hyperparameters in <a href="/docs/transformers/pr_43265/en/main_classes/trainer#transformers.TrainingArguments">TrainingArguments</a>. Note that we use <code>max_steps</code> instead of epochs since we’re using a streaming dataset:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments, Trainer
	<span class="hljs-meta">>>> </span>training_args = TrainingArguments(
	<span class="hljs-meta">... </span> output_dir=<span class="hljs-string">"audio-flamingo-3-hf-lora-finetuned"</span>,
	<span class="hljs-meta">... </span> per_device_train_batch_size=<span class="hljs-number">4</span>,
	<span class="hljs-meta">... </span> per_device_eval_batch_size=<span class="hljs-number">4</span>,
	<span class="hljs-meta">... </span> gradient_accumulation_steps=<span class="hljs-number">4</span>,
	<span class="hljs-meta">... </span> learning_rate=<span class="hljs-number">1e-4</span>,
	<span class="hljs-meta">... </span> max_steps=<span class="hljs-number">500</span>, <span class="hljs-comment"># Use max_steps with streaming datasets</span>
	<span class="hljs-meta">... </span> bf16=<span class="hljs-literal">True</span>,
	<span class="hljs-meta">... </span> logging_steps=<span class="hljs-number">10</span>,
	<span class="hljs-meta">... </span> eval_steps=<span class="hljs-number">100</span>,
	<span class="hljs-meta">... </span> save_steps=<span class="hljs-number">250</span>,
	<span class="hljs-meta">... </span> save_total_limit=<span class="hljs-number">2</span>, <span class="hljs-comment"># Keep only the latest 2 checkpoints</span>
	<span class="hljs-meta">... </span> save_only_model=<span class="hljs-literal">True</span>, <span class="hljs-comment"># Skip saving optimizer state to save disk space</span>
	<span class="hljs-meta">... </span> eval_strategy=<span class="hljs-string">"steps"</span>,
	<span class="hljs-meta">... </span> save_strategy=<span class="hljs-string">"steps"</span>,
	<span class="hljs-meta">... </span> remove_unused_columns=<span class="hljs-literal">False</span>,
	<span class="hljs-meta">... </span> dataloader_num_workers=<span class="hljs-number">0</span>, <span class="hljs-comment"># Must be 0 for streaming datasets</span>
	<span class="hljs-meta">... </span> gradient_checkpointing=<span class="hljs-literal">True</span>,
	<span class="hljs-meta">... </span> report_to=<span class="hljs-string">"none"</span>,
	<span class="hljs-meta">... </span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-gwany4">Pass the training arguments to <a href="/docs/transformers/pr_43265/en/main_classes/trainer#transformers.Trainer">Trainer</a> along with the model, datasets, and data collator:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>trainer = Trainer(
	<span class="hljs-meta">... </span> model=model,
	<span class="hljs-meta">... </span> args=training_args,
	<span class="hljs-meta">... </span> train_dataset=train_dataset,
	<span class="hljs-meta">... </span> eval_dataset=eval_dataset,
	<span class="hljs-meta">... </span> data_collator=data_collator,
	<span class="hljs-meta">... </span>)
	<span class="hljs-meta">>>> </span>trainer.train()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-972uga">Save the LoRA adapter and processor:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>trainer.save_model()
	<span class="hljs-meta">>>> </span>processor.save_pretrained(<span class="hljs-string">"audio-flamingo-3-hf-lora-finetuned"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-si4y14">Once training is completed, share your model to the Hub:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>trainer.push_to_hub()<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="inference" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#inference"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Inference</span></h2> <p data-svelte-h="svelte-ipr7q5">Now that you’ve fine-tuned the model, you can use it for audio captioning.</p> <p data-svelte-h="svelte-18y1pko">Load the fine-tuned model and processor:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AudioFlamingo3ForConditionalGeneration, AutoProcessor
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> PeftModel
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> torch
	<span class="hljs-meta">>>> </span>base_model = AudioFlamingo3ForConditionalGeneration.from_pretrained(
	<span class="hljs-meta">... </span> <span class="hljs-string">"nvidia/audio-flamingo-3-hf"</span>,
	<span class="hljs-meta">... </span> torch_dtype=torch.bfloat16,
	<span class="hljs-meta">... </span> device_map=<span class="hljs-string">"auto"</span>,
	<span class="hljs-meta">... </span>)
	<span class="hljs-meta">>>> </span>model = PeftModel.from_pretrained(base_model, <span class="hljs-string">"audio-flamingo-3-hf-lora-finetuned"</span>)
	<span class="hljs-meta">>>> </span>processor = AutoProcessor.from_pretrained(<span class="hljs-string">"audio-flamingo-3-hf-lora-finetuned"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-5ax06d">Load an audio sample for inference:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, Audio
	<span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">"OpenSound/AudioCaps"</span>, split=<span class="hljs-string">"test"</span>, streaming=<span class="hljs-literal">True</span>)
	<span class="hljs-meta">>>> </span>dataset = dataset.cast_column(<span class="hljs-string">"audio"</span>, Audio(sampling_rate=<span class="hljs-number">16000</span>))
	<span class="hljs-meta">>>> </span>sample = <span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(dataset))<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-x2pm4p">Prepare the input with a conversation format:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>messages = [
	<span class="hljs-meta">... </span> {
	<span class="hljs-meta">... </span> <span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>,
	<span class="hljs-meta">... </span> <span class="hljs-string">"content"</span>: [
	<span class="hljs-meta">... </span> {<span class="hljs-string">"type"</span>: <span class="hljs-string">"text"</span>, <span class="hljs-string">"text"</span>: <span class="hljs-string">"Describe the audio."</span>},
	<span class="hljs-meta">... </span> {<span class="hljs-string">"type"</span>: <span class="hljs-string">"audio"</span>, <span class="hljs-string">"audio"</span>: sample[<span class="hljs-string">"audio"</span>][<span class="hljs-string">"array"</span>]},
	<span class="hljs-meta">... </span> ],
	<span class="hljs-meta">... </span> }
	<span class="hljs-meta">... </span>]
	<span class="hljs-meta">>>> </span>inputs = processor.apply_chat_template(
	<span class="hljs-meta">... </span> messages,
	<span class="hljs-meta">... </span> tokenize=<span class="hljs-literal">True</span>,
	<span class="hljs-meta">... </span> add_generation_prompt=<span class="hljs-literal">True</span>,
	<span class="hljs-meta">... </span> return_dict=<span class="hljs-literal">True</span>,
	<span class="hljs-meta">... </span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1q2po9l">Generate a response:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">with</span> torch.no_grad():
	<span class="hljs-meta">... </span> output_ids = model.generate(**inputs, max_new_tokens=<span class="hljs-number">100</span>)
	<span class="hljs-meta">>>> </span><span class="hljs-comment"># Decode only the generated tokens</span>
	<span class="hljs-meta">>>> </span>input_len = inputs[<span class="hljs-string">"input_ids"</span>].shape[<span class="hljs-number">1</span>]
	<span class="hljs-meta">>>> </span>response = processor.tokenizer.decode(output_ids[<span class="hljs-number">0</span>][input_len:], skip_special_tokens=<span class="hljs-literal">True</span>)
	<span class="hljs-meta">>>> </span><span class="hljs-built_in">print</span>(response)
	<span class="hljs-comment">## A sewing machine is running while people are talking</span><!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="pipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#pipeline"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Pipeline</span></h2> <p data-svelte-h="svelte-fa1cha">You can also use the <a href="/docs/transformers/pr_43265/en/main_classes/pipelines#transformers.Pipeline">Pipeline</a> API for quick inference. First, merge the LoRA adapter with the base model, then create a pipeline:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline
	<span class="hljs-meta">>>> </span><span class="hljs-comment"># Merge LoRA adapter for pipeline use</span>
	<span class="hljs-meta">>>> </span>merged_model = model.merge_and_unload()
	<span class="hljs-meta">>>> </span>pipe = pipeline(
	<span class="hljs-meta">... </span> <span class="hljs-string">"audio-text-to-text"</span>,
	<span class="hljs-meta">... </span> model=merged_model,
	<span class="hljs-meta">... </span> processor=processor,
	<span class="hljs-meta">... </span>)
	<span class="hljs-meta">>>> </span>result = pipe(
	<span class="hljs-meta">... </span> sample[<span class="hljs-string">"audio"</span>][<span class="hljs-string">"array"</span>],
	<span class="hljs-meta">... </span> generate_kwargs={<span class="hljs-string">"max_new_tokens"</span>: <span class="hljs-number">100</span>},
	<span class="hljs-meta">... </span>)
	<span class="hljs-meta">>>> </span><span class="hljs-built_in">print</span>(result[<span class="hljs-number">0</span>][<span class="hljs-string">"generated_text"</span>])<!-- HTML_TAG_END --></pre></div> <blockquote class="tip" data-svelte-h="svelte-19xc9ji"><p>For more advanced use cases like multi-turn conversations with audio, you can structure your messages with alternating user and assistant roles, similar to <a href="./image_text_to_text">image-text-to-text</a> models.</p></blockquote> <h2 class="relative group"><a id="further-reading" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#further-reading"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Further Reading</span></h2> <ul data-svelte-h="svelte-10ai899"><li><a href="https://huggingface.co/tasks/audio-text-to-text" rel="nofollow">Audio-text-to-text task page</a> covers model types, use cases, and datasets.</li> <li><a href="https://huggingface.co/docs/peft" rel="nofollow">PEFT documentation</a> for more LoRA configuration options and other adapter methods.</li> <li><a href="https://huggingface.co/nvidia/audio-flamingo-3-hf" rel="nofollow">Audio Flamingo 3 model card</a> for model-specific details and capabilities.</li></ul> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers/blob/main/docs/source/en/tasks/audio_text_to_text.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_1gaarxw = {
	assets: "/docs/transformers/pr_43265/en",
	base: "/docs/transformers/pr_43265/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/transformers/pr_43265/en/_app/immutable/entry/start.1dae3980.js"),
	import("/docs/transformers/pr_43265/en/_app/immutable/entry/app.66dba7a3.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 674],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 69.9 kB
Xet hash:: 5e4e565a8d18b6c81bb29e25f89278612f79f351018c00ca62fc33326c0624ee

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.