Hugging Face's logo

Buckets:
hf-doc-build
/
doc-dev

Files
xet
hf-doc-build/doc-dev / peft /pr_3207 /en /task_guides /prompt_based_methods.html
HuggingFaceDocBuilder's picture
HuggingFaceDocBuilder
download
raw
50.3 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Prompt-based methods&quot;,&quot;local&quot;:&quot;prompt-based-methods&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Dataset&quot;,&quot;local&quot;:&quot;dataset&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Model&quot;,&quot;local&quot;:&quot;model&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;PEFT configuration and model&quot;,&quot;local&quot;:&quot;peft-configuration-and-model&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Training&quot;,&quot;local&quot;:&quot;training&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Share your model&quot;,&quot;local&quot;:&quot;share-your-model&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Inference&quot;,&quot;local&quot;:&quot;inference&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}">
 <link href="/docs/peft/pr_3207/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
 <link rel="modulepreload" href="/docs/peft/pr_3207/en/_app/immutable/entry/start.dbacf4b8.js">
 <link rel="modulepreload" href="/docs/peft/pr_3207/en/_app/immutable/chunks/scheduler.78382b47.js">
 <link rel="modulepreload" href="/docs/peft/pr_3207/en/_app/immutable/chunks/singletons.2397427a.js">
 <link rel="modulepreload" href="/docs/peft/pr_3207/en/_app/immutable/chunks/index.fadd215c.js">
 <link rel="modulepreload" href="/docs/peft/pr_3207/en/_app/immutable/chunks/paths.6fea2ad6.js">
 <link rel="modulepreload" href="/docs/peft/pr_3207/en/_app/immutable/entry/app.cb03e0d4.js">
 <link rel="modulepreload" href="/docs/peft/pr_3207/en/_app/immutable/chunks/preload-helper.2f407600.js">
 <link rel="modulepreload" href="/docs/peft/pr_3207/en/_app/immutable/chunks/index.6dd35eb6.js">
 <link rel="modulepreload" href="/docs/peft/pr_3207/en/_app/immutable/nodes/0.18735116.js">
 <link rel="modulepreload" href="/docs/peft/pr_3207/en/_app/immutable/chunks/each.e59479a4.js">
 <link rel="modulepreload" href="/docs/peft/pr_3207/en/_app/immutable/nodes/72.047b539a.js">
 <link rel="modulepreload" href="/docs/peft/pr_3207/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.d25d6883.js">
 <link rel="modulepreload" href="/docs/peft/pr_3207/en/_app/immutable/chunks/CodeBlock.147ab5db.js">
 <link rel="modulepreload" href="/docs/peft/pr_3207/en/_app/immutable/chunks/HfOption.a1db6210.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Prompt-based methods&quot;,&quot;local&quot;:&quot;prompt-based-methods&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Dataset&quot;,&quot;local&quot;:&quot;dataset&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Model&quot;,&quot;local&quot;:&quot;model&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;PEFT configuration and model&quot;,&quot;local&quot;:&quot;peft-configuration-and-model&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Training&quot;,&quot;local&quot;:&quot;training&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Share your model&quot;,&quot;local&quot;:&quot;share-your-model&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Inference&quot;,&quot;local&quot;:&quot;inference&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="prompt-based-methods" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#prompt-based-methods"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Prompt-based methods</span></h1> <p data-svelte-h="svelte-1nma30k">A prompt can describe a task or provide an example of a task you want the model to learn. Instead of manually creating these prompts, soft prompting methods add learnable parameters to the input embeddings that can be optimized for a specific task while keeping the pretrained model’s parameters frozen. This makes it both faster and easier to finetune large language models (LLMs) for new downstream tasks.</p> <p data-svelte-h="svelte-qxepz2">The PEFT library supports several types of prompting methods (p-tuning, prefix tuning, prompt tuning) and you can learn more about how these methods work conceptually in the <a href="../conceptual_guides/prompting">Soft prompts</a> guide. If you’re interested in applying these methods to other tasks and use cases, take a look at our <a href="https://huggingface.co/spaces/PEFT/soft-prompting" rel="nofollow">notebook collection</a>!</p> <p data-svelte-h="svelte-10gyoze">This guide will show you how to train a causal language model - with a soft prompting method - to <em>generate a classification</em> for whether a tweet is a complaint or not.</p> <blockquote class="tip" data-svelte-h="svelte-1p5rdzu"><p>Some familiarity with the general process of training a causal language model would be really helpful and allow you to focus on the soft prompting methods. If you’re new, we recommend taking a look at the <a href="https://huggingface.co/docs/transformers/tasks/language_modeling" rel="nofollow">Causal language modeling</a> guide first from the Transformers documentation. When you’re ready, come back and see how easy it is to drop PEFT in to your training!</p></blockquote> <p data-svelte-h="svelte-1rdzhb1">Before you begin, make sure you have all the necessary libraries installed.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->pip install -q peft transformers datasets<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="dataset" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#dataset"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Dataset</span></h2> <p data-svelte-h="svelte-b49pm">For this guide, you’ll use the <code>twitter_complaints</code> subset of the <a href="https://huggingface.co/datasets/ought/raft" rel="nofollow">RAFT</a> dataset. The <code>twitter_complaints</code> subset contains tweets labeled as <code>complaint</code> and <code>no complaint</code> and you can check out the <a href="https://huggingface.co/datasets/ought/raft/viewer/twitter_complaints" rel="nofollow">dataset viewer</a> for a better idea of what the data looks like.</p> <p data-svelte-h="svelte-5j9yqq">Use the <a href="https://huggingface.co/docs/datasets/main/en/package_reference/loading_methods#datasets.load_dataset" rel="nofollow">load_dataset</a> function to load the dataset and create a new <code>text_label</code> column so it is easier to understand what the <code>Label</code> values, <code>1</code> and <code>2</code> mean.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
ds = load_dataset(
 <span class="hljs-string">&quot;parquet&quot;</span>,
 data_files={
 <span class="hljs-string">&quot;train&quot;</span>: <span class="hljs-string">&quot;hf://datasets/ought/raft@refs/convert/parquet/twitter_complaints/train/0000.parquet&quot;</span>,
 <span class="hljs-string">&quot;test&quot;</span>: <span class="hljs-string">&quot;hf://datasets/ought/raft@refs/convert/parquet/twitter_complaints/test/0000.parquet&quot;</span>
 }
)
classes = [k.replace(<span class="hljs-string">&quot;_&quot;</span>, <span class="hljs-string">&quot; &quot;</span>) <span class="hljs-keyword">for</span> k <span class="hljs-keyword">in</span> ds[<span class="hljs-string">&quot;train&quot;</span>].features[<span class="hljs-string">&quot;Label&quot;</span>].names]
ds = ds.<span class="hljs-built_in">map</span>(
 <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;text_label&quot;</span>: [classes[label] <span class="hljs-keyword">for</span> label <span class="hljs-keyword">in</span> x[<span class="hljs-string">&quot;Label&quot;</span>]]},
 batched=<span class="hljs-literal">True</span>,
 num_proc=<span class="hljs-number">1</span>,
)
ds[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&quot;Tweet text&quot;</span>: <span class="hljs-string">&quot;@HMRCcustomers No this is my first job&quot;</span>, <span class="hljs-string">&quot;ID&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;Label&quot;</span>: <span class="hljs-number">2</span>, <span class="hljs-string">&quot;text_label&quot;</span>: <span class="hljs-string">&quot;no complaint&quot;</span>}<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-gjlxro">Load a tokenizer, define the padding token to use, and determine the maximum length of the tokenized label.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bigscience/bloomz-560m&quot;</span>)
<span class="hljs-keyword">if</span> tokenizer.pad_token_id <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
 tokenizer.pad_token_id = tokenizer.eos_token_id
target_max_length = <span class="hljs-built_in">max</span>([<span class="hljs-built_in">len</span>(tokenizer(class_label)[<span class="hljs-string">&quot;input_ids&quot;</span>]) <span class="hljs-keyword">for</span> class_label <span class="hljs-keyword">in</span> classes])
<span class="hljs-built_in">print</span>(target_max_length)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-s6tghk">Create a preprocessing function that tokenizes the tweet text and labels, pad the inputs and labels in each batch, create an attention mask, and truncate sequences to the <code>max_length</code>. Then convert the <code>input_ids</code>, <code>attention_mask</code>, and <code>labels</code> to PyTorch tensors.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> torch
max_length = <span class="hljs-number">64</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples, text_column=<span class="hljs-string">&quot;Tweet text&quot;</span>, label_column=<span class="hljs-string">&quot;text_label&quot;</span></span>):
 batch_size = <span class="hljs-built_in">len</span>(examples[text_column])
 inputs = [<span class="hljs-string">f&quot;<span class="hljs-subst">{text_column}</span> : <span class="hljs-subst">{x}</span> Label : &quot;</span> <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> examples[text_column]]
 targets = [<span class="hljs-built_in">str</span>(x) <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> examples[label_column]]
 model_inputs = tokenizer(inputs)
 labels = tokenizer(targets)
 classes = [k.replace(<span class="hljs-string">&quot;_&quot;</span>, <span class="hljs-string">&quot; &quot;</span>) <span class="hljs-keyword">for</span> k <span class="hljs-keyword">in</span> ds[<span class="hljs-string">&quot;train&quot;</span>].features[<span class="hljs-string">&quot;Label&quot;</span>].names]
 <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(batch_size):
 sample_input_ids = model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>][i]
 label_input_ids = labels[<span class="hljs-string">&quot;input_ids&quot;</span>][i]
 model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>][i] = [tokenizer.pad_token_id] * (
 max_length - <span class="hljs-built_in">len</span>(sample_input_ids)
 ) + sample_input_ids
 model_inputs[<span class="hljs-string">&quot;attention_mask&quot;</span>][i] = [<span class="hljs-number">0</span>] * (max_length - <span class="hljs-built_in">len</span>(sample_input_ids)) + model_inputs[
 <span class="hljs-string">&quot;attention_mask&quot;</span>
 ][i]
 labels[<span class="hljs-string">&quot;input_ids&quot;</span>][i] = [-<span class="hljs-number">100</span>] * (max_length - <span class="hljs-built_in">len</span>(label_input_ids)) + label_input_ids
 model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>][i] = torch.tensor(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>][i][:max_length])
 model_inputs[<span class="hljs-string">&quot;attention_mask&quot;</span>][i] = torch.tensor(model_inputs[<span class="hljs-string">&quot;attention_mask&quot;</span>][i][:max_length])
 labels[<span class="hljs-string">&quot;input_ids&quot;</span>][i] = torch.tensor(labels[<span class="hljs-string">&quot;input_ids&quot;</span>][i][:max_length])
 model_inputs[<span class="hljs-string">&quot;labels&quot;</span>] = labels[<span class="hljs-string">&quot;input_ids&quot;</span>]
 <span class="hljs-keyword">return</span> model_inputs<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-fdx4gc">Apply the preprocessing function to the entire dataset with the <a href="https://huggingface.co/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.map" rel="nofollow">map</a> function, and remove the unprocessed columns because the model won’t need them.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->processed_ds = ds.<span class="hljs-built_in">map</span>(
 preprocess_function,
 batched=<span class="hljs-literal">True</span>,
 num_proc=<span class="hljs-number">1</span>,
 remove_columns=ds[<span class="hljs-string">&quot;train&quot;</span>].column_names,
 load_from_cache_file=<span class="hljs-literal">False</span>,
 desc=<span class="hljs-string">&quot;Running tokenizer on dataset&quot;</span>,
)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-zejkx4">Finally, create a training and evaluation <a href="https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader" rel="nofollow"><code>DataLoader</code></a>. You can set <code>pin_memory=True</code> to speed up the data transfer to the GPU during training if the samples in your dataset are on a CPU.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> default_data_collator
train_ds = processed_ds[<span class="hljs-string">&quot;train&quot;</span>]
eval_ds = processed_ds[<span class="hljs-string">&quot;test&quot;</span>]
batch_size = <span class="hljs-number">16</span>
train_dataloader = DataLoader(train_ds, shuffle=<span class="hljs-literal">True</span>, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=<span class="hljs-literal">True</span>)
eval_dataloader = DataLoader(eval_ds, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=<span class="hljs-literal">True</span>)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="model" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#model"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Model</span></h2> <p data-svelte-h="svelte-bbm736">Now let’s load a pretrained model to use as the base model for the soft prompt method. This guide uses the <a href="https://huggingface.co/bigscience/bloomz-560m" rel="nofollow">bigscience/bloomz-560m</a> model, but you can use any causal language model you want.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;bigscience/bloomz-560m&quot;</span>)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="peft-configuration-and-model" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#peft-configuration-and-model"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>PEFT configuration and model</span></h3> <p data-svelte-h="svelte-1joqw7b">For any PEFT method, you’ll need to create a configuration which contains all the parameters that specify how the PEFT method should be applied. Once the configuration is setup, pass it to the <a href="/docs/peft/pr_3207/en/package_reference/peft_model#peft.get_peft_model">get_peft_model()</a> function along with the base model to create a trainable <a href="/docs/peft/pr_3207/en/package_reference/peft_model#peft.PeftModel">PeftModel</a>.</p> <blockquote class="tip" data-svelte-h="svelte-1gmi3mo"><p>Call the <a href="/docs/peft/pr_3207/en/package_reference/peft_model#peft.PeftModel.print_trainable_parameters">print_trainable_parameters()</a> method to compare the number of trainable parameters of <a href="/docs/peft/pr_3207/en/package_reference/peft_model#peft.PeftModel">PeftModel</a> versus the number of parameters in the base model!</p></blockquote> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">p-tuning </div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">prefix tuning </div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">prompt tuning </div></div> <div class="language-select"><p data-svelte-h="svelte-hbefu2"><a href="../conceptual_guides/prompting#p-tuning">P-tuning</a> adds a trainable embedding tensor where the prompt tokens can be added anywhere in the input sequence. Create a <a href="/docs/peft/pr_3207/en/package_reference/p_tuning#peft.PromptEncoderConfig">PromptEncoderConfig</a> with the task type, the number of virtual tokens to add and learn, and the hidden size of the encoder for learning the prompt parameters.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> PromptEncoderConfig, get_peft_model
peft_config = PromptEncoderConfig(task_type=<span class="hljs-string">&quot;CAUSAL_LM&quot;</span>, num_virtual_tokens=<span class="hljs-number">20</span>, encoder_hidden_size=<span class="hljs-number">128</span>)
model = get_peft_model(model, peft_config)
model.print_trainable_parameters()
<span class="hljs-string">&quot;trainable params: 300,288 || all params: 559,514,880 || trainable%: 0.05366935013417338&quot;</span><!-- HTML_TAG_END --></pre></div> </div> <h3 class="relative group"><a id="training" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#training"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Training</span></h3> <p data-svelte-h="svelte-tlkvop">Set up an optimizer and learning rate scheduler.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> get_linear_schedule_with_warmup
lr = <span class="hljs-number">3e-2</span>
num_epochs = <span class="hljs-number">50</span>
optimizer = torch.optim.AdamW(model.parameters(), lr=lr)
lr_scheduler = get_linear_schedule_with_warmup(
 optimizer=optimizer,
 num_warmup_steps=<span class="hljs-number">0</span>,
 num_training_steps=(<span class="hljs-built_in">len</span>(train_dataloader) * num_epochs),
)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-bqhytl">Move the model to the GPU and create a training loop that reports the loss and perplexity for each epoch.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> tqdm <span class="hljs-keyword">import</span> tqdm
device = <span class="hljs-string">&quot;cuda&quot;</span>
model = model.to(device)
<span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(num_epochs):
 model.train()
 total_loss = <span class="hljs-number">0</span>
 <span class="hljs-keyword">for</span> step, batch <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(tqdm(train_dataloader)):
 batch = {k: v.to(device) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> batch.items()}
 outputs = model(**batch)
 loss = outputs.loss
 total_loss += loss.detach().<span class="hljs-built_in">float</span>()
 loss.backward()
 optimizer.step()
 lr_scheduler.step()
 optimizer.zero_grad()
model.<span class="hljs-built_in">eval</span>()
 eval_loss = <span class="hljs-number">0</span>
<span class="hljs-keyword">for</span> step, batch <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(tqdm(eval_dataloader)):
 batch = {k: v.to(device) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> batch.items()}
 <span class="hljs-keyword">with</span> torch.no_grad():
 outputs = model(**batch)
 loss = outputs.loss
 eval_loss += loss.detach().<span class="hljs-built_in">float</span>()
eval_epoch_loss = eval_loss / <span class="hljs-built_in">len</span>(eval_dataloader)
 eval_ppl = torch.exp(eval_epoch_loss)
 train_epoch_loss = total_loss / <span class="hljs-built_in">len</span>(train_dataloader)
 train_ppl = torch.exp(train_epoch_loss)
 <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{epoch=}</span>: <span class="hljs-subst">{train_ppl=}</span> <span class="hljs-subst">{train_epoch_loss=}</span> <span class="hljs-subst">{eval_ppl=}</span> <span class="hljs-subst">{eval_epoch_loss=}</span>&quot;</span>)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="share-your-model" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#share-your-model"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Share your model</span></h2> <p data-svelte-h="svelte-vaxac6">Once training is complete, you can upload your model to the Hub with the <a href="https://huggingface.co/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.push_to_hub" rel="nofollow">push_to_hub</a> method. You’ll need to login to your Hugging Face account first and enter your token when prompted.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login
account = &lt;your-hf-account-name&gt;
peft_model_id = <span class="hljs-string">f&quot;<span class="hljs-subst">{account}</span>/bloomz-560-m-peft-method&quot;</span>
model.push_to_hub(peft_model_id)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-b4rki0">If you check the model file size in the repository, you’ll see that it is a lot smaller than a full sized model!</p> <div class="flex flex-col justify-center" data-svelte-h="svelte-i5cvgh"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/PEFT-hub-screenshot.png"> <figcaption class="text-center">For example, the adapter weights for a opt-350m model stored on the Hub are only ~6MB compared to the full model size which can be ~700MB.</figcaption></div> <h2 class="relative group"><a id="inference" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#inference"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Inference</span></h2> <p data-svelte-h="svelte-19ver11">Let’s load the model for inference and test it out on a tweet!</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> AutoPeftModelForCausalLM
model = AutoPeftModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;peft_model_id&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bigscience/bloomz-560m&quot;</span>)
i = <span class="hljs-number">15</span>
inputs = tokenizer(<span class="hljs-string">f&#x27;<span class="hljs-subst">{text_column}</span> : <span class="hljs-subst">{ds[<span class="hljs-string">&quot;test&quot;</span>][i][<span class="hljs-string">&quot;Tweet text&quot;</span>]}</span> Label : &#x27;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-built_in">print</span>(ds[<span class="hljs-string">&quot;test&quot;</span>][i][<span class="hljs-string">&quot;Tweet text&quot;</span>])
<span class="hljs-string">&quot;@NYTsupport i have complained a dozen times &amp;amp; yet my papers are still thrown FAR from my door. Why is this so hard to resolve?&quot;</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1gths0y">Call the <a href="https://huggingface.co/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate" rel="nofollow">generate</a> method to generate the predicted classification label.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">with</span> torch.no_grad():
 inputs = {k: v.to(device) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> inputs.items()}
 outputs = model.generate(input_ids=inputs[<span class="hljs-string">&quot;input_ids&quot;</span>], max_new_tokens=<span class="hljs-number">10</span>)
 <span class="hljs-built_in">print</span>(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=<span class="hljs-literal">True</span>))
<span class="hljs-string">&quot;[&#x27;Tweet text : @NYTsupport i have complained a dozen times &amp;amp; yet my papers are still thrown FAR from my door. Why is this so hard to resolve? Label : complaint&#x27;]&quot;</span><!-- HTML_TAG_END --></pre></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/peft/blob/main/docs/source/task_guides/prompt_based_methods.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p>
<script>
 {
 __sveltekit_1wz1iqk = {
 assets: "/docs/peft/pr_3207/en",
 base: "/docs/peft/pr_3207/en",
 env: {}
 };
 const element = document.currentScript.parentElement;
 const data = [null,null];
 Promise.all([
 import("/docs/peft/pr_3207/en/_app/immutable/entry/start.dbacf4b8.js"),
 import("/docs/peft/pr_3207/en/_app/immutable/entry/app.cb03e0d4.js")
 ]).then(([kit, app]) => {
 kit.start(app, element, {
 node_ids: [0, 72],
 data,
 form: null,
 error: null
 });
 });
 }
 </script>

Xet Storage Details

Size:
50.3 kB
·
Xet hash:
d5edcb457eae9567d1b6efb7d1532441b1ab376126f3d852b7c3bec57cfd495e

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.