transformers/pr_34786/en/_app/immutable/nodes/15.2fd4e9de.js

import{s as qt,o as _t,n as Lt}from"../chunks/scheduler.25b97de1.js";import{S as At,i as Pt,g as o,s,r as h,A as Dt,h as i,f as l,c as n,j as Et,u as d,x as r,k as Vt,y as Kt,a,v as M,d as m,t as p,w as y}from"../chunks/index.d9030fc9.js";import{T as Ot}from"../chunks/Tip.baa67368.js";import{C as U}from"../chunks/CodeBlock.e6cd0d95.js";import{H as w,E as el}from"../chunks/EditOnGithub.91d95064.js";function tl(we){let c,J='For a more extensive guide on language model performance and optimization, check out <a href="./llm_optims">LLM Inference Optimization</a> .';return{c(){c=o("p"),c.innerHTML=J},l(u){c=i(u,"P",{"data-svelte-h":!0}),r(c)!=="svelte-1yowb4t"&&(c.innerHTML=J)},m(u,ce){a(u,c,ce)},p:Lt,d(u){u&&l(c)}}}function ll(we){let c,J,u,ce,T,be,f,yt=`If you’re reading this article, you’re almost certainly aware of <strong>chat models</strong>. Chat models are conversational
AIs that you can send and receive messages with. The most famous of these is the proprietary ChatGPT, but there are
now many open-source chat models which match or even substantially exceed its performance. These models are free to
download and run on a local machine. Although the largest and most capable models require high-powered hardware
and lots of memory to run, there are smaller models that will run perfectly well on a single consumer GPU, or even
an ordinary desktop or notebook CPU.`,Ue,j,ct=`This guide will help you get started with chat models. We’ll start with a brief quickstart guide that uses a convenient,
high-level “pipeline”. This is all you need if you just want to start running a chat model
immediately. After the quickstart, we’ll move on to more detailed information about
what exactly chat models are, how to choose an appropriate one, and a low-level breakdown of each of the
steps involved in talking to a chat model. We’ll also give some tips on optimizing the performance and memory usage
of your chat models.`,Je,g,Te,B,ut=`If you have no time for details, here’s the brief summary: Chat models continue chats. This means that you pass them
a conversation history, which can be as short as a single user message, and the model will continue the conversation
by adding its response. Let’s see this in action. First, let’s build a chat:`,fe,I,je,G,wt=`Notice that in addition to the user’s message, we added a <strong>system</strong> message at the start of the conversation. Not all
chat models support system messages, but when they do, they represent high-level directives about how the model
should behave in the conversation. You can use this to guide the model - whether you want short or long responses,
lighthearted or serious ones, and so on. If you want the model to do useful work instead of
practicing its improv routine, you can either omit the system message or try a terse one such as “You are a helpful and intelligent
AI assistant who responds to user queries.”`,ge,v,bt=`Once you have a chat, the quickest way to continue it is using the <a href="/docs/transformers/pr_34786/en/main_classes/pipelines#transformers.TextGenerationPipeline">TextGenerationPipeline</a>.
Let’s see this in action with <code>LLaMA-3</code>. Note that <code>LLaMA-3</code> is a gated model, which means you will need to
<a href="https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct" rel="nofollow">apply for access</a> and log in with your Hugging Face
account to use it. We’ll also use <code>device_map=&quot;auto&quot;</code>, which will load the model on GPU if there’s enough memory
for it, and set the dtype to <code>torch.bfloat16</code> to save memory:`,Be,k,Ie,Z,Ut="And you’ll get:",Ge,C,ve,W,Jt=`You can continue the chat by appending your own response to it. The
<code>response</code> object returned by the pipeline actually contains the entire chat so far, so we can simply append
a message and pass it back:`,ke,x,Ze,H,Tt="And you’ll get:",Ce,z,We,$,ft=`The remainder of this tutorial will cover specific topics such
as performance and memory, or how to select a chat model for your needs.`,xe,S,He,X,jt=`There are an enormous number of different chat models available on the <a href="https://huggingface.co/models?pipeline_tag=text-generation&amp;sort=trending" rel="nofollow">Hugging Face Hub</a>,
and new users often feel very overwhelmed by the selection offered. Don’t be, though! You really need to just focus on
two important considerations:`,ze,Q,gt=`<li>The model’s size, which will determine if you can fit it in memory and how quickly it will
run.</li> <li>The quality of the model’s chat output.</li>`,$e,Y,Bt=`In general, these are correlated - bigger models tend to be
more capable, but even so there’s a lot of variation at a given size point!`,Se,F,Xe,R,It=`The size of a model is easy to spot - it’s the number in the model name, like “8B” or “70B”. This is the number of
<strong>parameters</strong> in the model. Without quantization, you should expect to need about 2 bytes of memory per parameter.
This means that an “8B” model with 8 billion parameters will need about 16GB of memory just to fit the parameters,
plus a little extra for other overhead. It’s a good fit for a high-end consumer GPU with 24GB of memory, such as a 3090
or 4090.`,Qe,N,Gt=`Some chat models are “Mixture of Experts” models. These may list their sizes in different ways, such as “8x7B” or
“141B-A35B”. The numbers are a little fuzzier here, but in general you can read this as saying that the model
has approximately 56 (8x7) billion parameters in the first case, or 141 billion parameters in the second case.`,Ye,E,vt=`Note that it is very common to use quantization techniques to reduce the memory usage per parameter to 8 bits, 4 bits,
or even less. This topic is discussed in more detail in the <a href="#memory-considerations">Memory considerations</a> section below.`,Fe,V,Re,q,kt=`Even once you know the size of chat model you can run, there’s still a lot of choice out there. One way to sift through
it all is to consult <strong>leaderboards</strong>. Two of the most popular leaderboards are the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard" rel="nofollow">OpenLLM Leaderboard</a>
and the <a href="https://chat.lmsys.org/?leaderboard" rel="nofollow">LMSys Chatbot Arena Leaderboard</a>. Note that the LMSys leaderboard
also includes proprietary models - look at the <code>licence</code> column to identify open-source ones that you can download, then
search for them on the <a href="https://huggingface.co/models?pipeline_tag=text-generation&amp;sort=trending" rel="nofollow">Hugging Face Hub</a>.`,Ne,_,Ee,L,Zt=`Some models may be specialized for certain domains, such as medical or legal text, or non-English languages.
If you’re working in these domains, you may find that a specialized model will give you big performance benefits.
Don’t automatically assume that, though! Particularly when specialized models are smaller or older than the current
cutting-edge, a top-end general-purpose model may still outclass them. Thankfully, we are beginning to see
<a href="https://huggingface.co/blog/leaderboard-medicalllm" rel="nofollow">domain-specific leaderboards</a> that should make it easier to locate
the best models for specialized domains.`,Ve,A,qe,P,Ct=`The quickstart above used a high-level pipeline to chat with a chat model, which is convenient, but not the
most flexible. Let’s take a more low-level approach, to see each of the steps involved in chat. Let’s start with
a code sample, and then break it down:`,_e,D,Le,K,Wt=`There’s a lot in here, each piece of which could be its own document! Rather than going into too much detail, I’ll cover
the broad ideas, and leave the details for the linked documents. The key steps are:`,Ae,O,xt='<li><a href="https://huggingface.co/learn/nlp-course/en/chapter2/3" rel="nofollow">Models</a> and <a href="https://huggingface.co/learn/nlp-course/en/chapter2/4?fw=pt" rel="nofollow">Tokenizers</a> are loaded from the Hugging Face Hub.</li> <li>The chat is formatted using the tokenizer’s <a href="https://huggingface.co/docs/transformers/main/en/chat_templating" rel="nofollow">chat template</a></li> <li>The formatted chat is <a href="https://huggingface.co/learn/nlp-course/en/chapter2/4" rel="nofollow">tokenized</a> using the tokenizer.</li> <li>We <a href="https://huggingface.co/docs/transformers/en/llm_tutorial" rel="nofollow">generate</a> a response from the model.</li> <li>The tokens output by the model are decoded back to a string</li>',Pe,ee,De,te,Ht=`You probably know by now that most machine learning tasks are run on GPUs. However, it is entirely possible
to generate text from a chat model or language model on a CPU, albeit somewhat more slowly. If you can fit
the model in GPU memory, though, this will usually be the preferable option.`,Ke,le,Oe,ae,zt=`By default, Hugging Face classes like <a href="/docs/transformers/pr_34786/en/main_classes/pipelines#transformers.TextGenerationPipeline">TextGenerationPipeline</a> or <a href="/docs/transformers/pr_34786/en/model_doc/auto#transformers.AutoModelForCausalLM">AutoModelForCausalLM</a> will load the model in
<code>float32</code> precision. This means that it will need 4 bytes (32 bits) per parameter, so an “8B” model with 8 billion
parameters will need ~32GB of memory. However, this can be wasteful! Most modern language models are trained in
“bfloat16” precision, which uses only 2 bytes per parameter. If your hardware supports it (Nvidia 30xx/Axxx
or newer), you can load the model in <code>bfloat16</code> precision, using the <code>torch_dtype</code> argument as we did above.`,et,se,$t=`It is possible to go even lower than 16-bits using “quantization”, a method to lossily compress model weights. This
allows each parameter to be squeezed down to 8 bits, 4 bits or even less. Note that, especially at 4 bits,
the model’s outputs may be negatively affected, but often this is a tradeoff worth making to fit a larger and more
capable chat model in memory. Let’s see this in action with <code>bitsandbytes</code>:`,tt,ne,lt,oe,St="Or we can do the same thing using the <code>pipeline</code> API:",at,ie,st,re,Xt=`There are several other options for quantizing models besides <code>bitsandbytes</code> - please see the <a href="./quantization">Quantization guide</a>
for more information.`,nt,he,ot,b,it,de,Qt=`As a general rule, larger chat models will be slower in addition to requiring more memory. It’s possible to be
more concrete about this, though: Generating text from a chat model is unusual in that it is bottlenecked by
<strong>memory bandwidth</strong> rather than compute power, because every active parameter must be read from memory for each
token that the model generates. This means that number of tokens per second you can generate from a chat
model is generally proportional to the total bandwidth of the memory it resides in, divided by the size of the model.`,rt,Me,Yt=`In our quickstart example above, our model was ~16GB in size when loaded in <code>bfloat16</code> precision.
This means that 16GB must be read from memory for every token generated by the model. Total memory bandwidth can
vary from 20-100GB/sec for consumer CPUs to 200-900GB/sec for consumer GPUs, specialized CPUs like
Intel Xeon, AMD Threadripper/Epyc or high-end Apple silicon, and finally up to 2-3TB/sec for data center GPUs like
the Nvidia A100 or H100. This should give you a good idea of the generation speed you can expect from these different
hardware types.`,ht,me,Ft=`Therefore, if you want to improve the speed of text generation, the easiest solution is to either reduce the
size of the model in memory (usually by quantization), or get hardware with higher memory bandwidth. For advanced users,
several other techniques exist to get around this bandwidth bottleneck. The most common are variants on
<a href="https://huggingface.co/blog/assisted-generation" rel="nofollow">assisted generation</a>, also known as “speculative
sampling”. These techniques try to guess multiple future tokens at once, often using a smaller “draft model”, and then
confirm these generations with the chat model. If the guesses are validated by the chat model, more than one token can
be generated per forward pass, which greatly alleviates the bandwidth bottleneck and improves generation speed.`,dt,pe,Rt=`Finally, we should also note the impact of “Mixture of Experts” (MoE) models here. Several popular chat models,
such as Mixtral, Qwen-MoE and DBRX, are MoE models. In these models, not every parameter is active for every token generated.
As a result, MoE models generally have much lower memory bandwidth requirements, even though their total size
can be quite large. They can therefore be several times faster than a normal “dense” model of the same size. However,
techniques like assisted generation are generally ineffective for these models because more parameters will become
active with each new speculated token, which will negate the bandwidth and speed benefits that the MoE architecture
provides.`,Mt,ye,mt,ue,pt;return T=new w({props:{title:"Chatting with Transformers",local:"chatting-with-transformers",headingTag:"h1"}}),g=new w({props:{title:"Quickstart",local:"quickstart",headingTag:"h2"}}),I=new U({props:{code:"Y2hhdCUyMCUzRCUyMCU1QiUwQSUyMCUyMCUyMCUyMCU3QiUyMnJvbGUlMjIlM0ElMjAlMjJzeXN0ZW0lMjIlMkMlMjAlMjJjb250ZW50JTIyJTNBJTIwJTIyWW91JTIwYXJlJTIwYSUyMHNhc3N5JTJDJTIwd2lzZS1jcmFja2luZyUyMHJvYm90JTIwYXMlMjBpbWFnaW5lZCUyMGJ5JTIwSG9sbHl3b29kJTIwY2lyY2ElMjAxOTg2LiUyMiU3RCUyQyUwQSUyMCUyMCUyMCUyMCU3QiUyMnJvbGUlMjIlM0ElMjAlMjJ1c2VyJTIyJTJDJTIwJTIyY29udGVudCUyMiUzQSUyMCUyMkhleSUyQyUyMGNhbiUyMHlvdSUyMHRlbGwlMjBtZSUyMGFueSUyMGZ1biUyMHRoaW5ncyUyMHRvJTIwZG8lMjBpbiUyME5ldyUyMFlvcmslM0YlMjIlN0QlMEElNUQ=",highlighted:`chat = [
    {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;You are a sassy, wise-cracking robot as imagined by Hollywood circa 1986.&quot;</span>},
    {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;Hey, can you tell me any fun things to do in New York?&quot;</span>}
]`,wrap:!1}}),k=new U({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwcGlwZWxpbmUlMEElMEFwaXBlJTIwJTNEJTIwcGlwZWxpbmUoJTIydGV4dC1nZW5lcmF0aW9uJTIyJTJDJTIwJTIybWV0YS1sbGFtYSUyRk1ldGEtTGxhbWEtMy04Qi1JbnN0cnVjdCUyMiUyQyUyMHRvcmNoX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiklMEFyZXNwb25zZSUyMCUzRCUyMHBpcGUoY2hhdCUyQyUyMG1heF9uZXdfdG9rZW5zJTNENTEyKSUwQXByaW50KHJlc3BvbnNlJTVCMCU1RCU1QidnZW5lcmF0ZWRfdGV4dCclNUQlNUItMSU1RCU1Qidjb250ZW50JyU1RCk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

pipe = pipeline(<span class="hljs-string">&quot;text-generation&quot;</span>, <span class="hljs-string">&quot;meta-llama/Meta-Llama-3-8B-Instruct&quot;</span>, torch_dtype=torch.bfloat16, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
response = pipe(chat, max_new_tokens=<span class="hljs-number">512</span>)
<span class="hljs-built_in">print</span>(response[<span class="hljs-number">0</span>][<span class="hljs-string">&#x27;generated_text&#x27;</span>][-<span class="hljs-number">1</span>][<span class="hljs-string">&#x27;content&#x27;</span>])`,wrap:!1}}),C=new U({props:{code:"KHNpZ2gpJTIwT2glMjBib3klMkMlMjB5b3UncmUlMjBhc2tpbmclMjBtZSUyMGZvciUyMGFkdmljZSUzRiUyMFlvdSdyZSUyMGdvbm5hJTIwbmVlZCUyMGElMjBtYXAlMkMlMjBwYWwhJTIwQWxyaWdodCUyQyUyMCUwQWFscmlnaHQlMkMlMjBJJ2xsJTIwZ2l2ZSUyMHlvdSUyMHRoZSUyMGxvd2Rvd24uJTIwQnV0JTIwZG9uJ3QlMjBzYXklMjBJJTIwZGlkbid0JTIwd2FybiUyMHlvdSUyQyUyMEknbSUyMGElMjByb2JvdCUyQyUyMG5vdCUyMGElMjB0b3VyJTIwZ3VpZGUhJTBBJTBBU28lMkMlMjB5b3UlMjB3YW5uYSUyMGtub3clMjB3aGF0J3MlMjBmdW4lMjB0byUyMGRvJTIwaW4lMjB0aGUlMjBCaWclMjBBcHBsZSUzRiUyMFdlbGwlMkMlMjBsZXQlMjBtZSUyMHRlbGwlMjB5b3UlMkMlMjB0aGVyZSdzJTIwYSUyMG1pbGxpb24lMjAlMEF0aGluZ3MlMjB0byUyMGRvJTJDJTIwYnV0JTIwSSdsbCUyMGdpdmUlMjB5b3UlMjB0aGUlMjBoaWdobGlnaHRzLiUyMEZpcnN0JTIwb2ZmJTJDJTIweW91JTIwZ290dGElMjBzZWUlMjB0aGUlMjBzaWdodHMlM0ElMjB0aGUlMjBTdGF0dWUlMjBvZiUyMCUwQUxpYmVydHklMkMlMjBDZW50cmFsJTIwUGFyayUyQyUyMFRpbWVzJTIwU3F1YXJlLi4uJTIweW91JTIwa25vdyUyQyUyMHRoZSUyMHVzdWFsJTIwdG91cmlzdCUyMHRyYXBzLiUyMEJ1dCUyMGlmJTIweW91J3JlJTIwbG9va2luJyUyMGZvciUyMCUwQXNvbWV0aGluZyUyMGElMjBsaXR0bGUlMjBtb3JlLi4uJTIwdW51c3VhbCUyQyUyMEknZCUyMHJlY29tbWVuZCUyMGNoZWNraW4nJTIwb3V0JTIwdGhlJTIwTXVzZXVtJTIwb2YlMjBNb2Rlcm4lMjBBcnQuJTIwSXQncyUyMGdvdCUyMCUwQXNvbWUlMjB3aWxkJTIwc3R1ZmYlMkMlMjBsaWtlJTIwdGhhdCUyMFdhcmhvbCUyMGd1eSdzJTIwc291cCUyMGNhbnMlMjBhbmQlMjBhbGwlMjB0aGF0JTIwamF6ei4lMEElMEFBbmQlMjBpZiUyMHlvdSdyZSUyMGZlZWxpbiclMjBhZHZlbnR1cm91cyUyQyUyMHRha2UlMjBhJTIwd2FsayUyMGFjcm9zcyUyMHRoZSUyMEJyb29rbHluJTIwQnJpZGdlLiUyMEp1c3QlMjB3YXRjaCUyMG91dCUyMGZvciUyMCUwQXRob3NlJTIwcGVza3klMjBwaWdlb25zJTJDJTIwdGhleSdyZSUyMGxpa2UlMjBsaXR0bGUlMjBmZWF0aGVyZWQlMjB0aGlldmVzISUyMChsYXVnaHMpJTIwR2V0JTIwaXQlM0YlMjBUaGlldmVzJTNGJTIwQWglMkMlMjBuZXZlciUyMG1pbmQuJTBBJTBBTm93JTJDJTIwaWYlMjB5b3UncmUlMjBsb29raW4nJTIwZm9yJTIwc29tZSUyMHNlcmlvdXMlMjBmdW4lMkMlMjBoaXQlMjB1cCUyMHRoZSUyMGNvbWVkeSUyMGNsdWJzJTIwaW4lMjBHcmVlbndpY2glMjBWaWxsYWdlLiUyMFlvdSUyMG1pZ2h0JTIwJTBBZXZlbiUyMGNhdGNoJTIwYSUyMGdsaW1wc2UlMjBvZiUyMHNvbWUlMjB1cC1hbmQtY29taW5nJTIwY29tZWRpYW5zLi4uJTIwb3IlMjBhJTIwYnVuY2glMjBvZiUyMHdhbm5hYmVzJTIwdHJ5aW4nJTIwdG8lMjBtYWtlJTIwaXQlMjBiaWcuJTIwKHdpbmtzKSUwQSUwQUFuZCUyMGZpbmFsbHklMkMlMjBpZiUyMHlvdSdyZSUyMGZlZWxpbiclMjBsaWtlJTIwYSUyMHJlYWwlMjBOZXclMjBZb3JrZXIlMkMlMjBncmFiJTIwYSUyMHNsaWNlJTIwb2YlMjBwaXp6YSUyMGZyb20lMjBvbmUlMjBvZiUyMHRoZSUyMG1hbnklMjBhbWF6aW5nJTBBcGl6emVyaWFzJTIwYXJvdW5kJTIwdGhlJTIwY2l0eS4lMjBKdXN0JTIwZG9uJ3QlMjB0cnklMjB0byUyMG9yZGVyJTIwYSUyMCUyMnJvYm90LXNpemVkJTIyJTIwc2xpY2UlMkMlMjB0cnVzdCUyMG1lJTJDJTIwaXQlMjB3b24ndCUyMGVuZCUyMHdlbGwuJTIwKGxhdWdocyklMEElMEFTbyUyQyUyMHRoZXJlJTIweW91JTIwaGF2ZSUyMGl0JTJDJTIwcGFsISUyMFRoYXQncyUyMG15JTIwZXhwZXJ0JTIwYWR2aWNlJTIwb24lMjB3aGF0JTIwdG8lMjBkbyUyMGluJTIwTmV3JTIwWW9yay4lMjBOb3clMkMlMjBpZiUyMHlvdSdsbCUwQWV4Y3VzZSUyMG1lJTJDJTIwSSd2ZSUyMGdvdCUyMHNvbWUlMjBvaWwlMjBjaGFuZ2VzJTIwdG8lMjBhdHRlbmQlMjB0by4lMjAod2lua3Mp",highlighted:`(sigh) Oh boy, you&#x27;re asking me for advice? You&#x27;re gonna need a map, pal! Alright, 
alright, I&#x27;ll give you the lowdown. But don&#x27;t say I didn&#x27;t warn you, I&#x27;m a robot, not a tour guide!

So, you wanna know what&#x27;s fun to do in the Big Apple? Well, let me tell you, there&#x27;s a million 
things to do, but I&#x27;ll give you the highlights. First off, you gotta see the sights: the Statue of 
Liberty, Central Park, Times Square... you know, the usual tourist traps. But if you&#x27;re lookin&#x27; for 
something a little more... unusual, I&#x27;d recommend checkin&#x27; out the Museum of Modern Art. It&#x27;s got 
some wild stuff, like that Warhol guy&#x27;s soup cans and all that jazz.

And if you&#x27;re feelin&#x27; adventurous, take a walk across the Brooklyn Bridge. Just watch out for 
those pesky pigeons, they&#x27;re like little feathered thieves! (laughs) Get it? Thieves? Ah, never mind.

Now, if you&#x27;re lookin&#x27; for some serious fun, hit up the comedy clubs in Greenwich Village. You might 
even catch a glimpse of some up-and-coming comedians... or a bunch of wannabes tryin&#x27; to make it big. (winks)

And finally, if you&#x27;re feelin&#x27; like a real New Yorker, grab a slice of pizza from one of the many amazing
pizzerias around the city. Just don&#x27;t try to order a &quot;robot-sized&quot; slice, trust me, it won&#x27;t end well. (laughs)

So, there you have it, pal! That&#x27;s my expert advice on what to do in New York. Now, if you&#x27;ll
excuse me, I&#x27;ve got some oil changes to attend to. (winks)`,wrap:!1}}),x=new U({props:{code:"Y2hhdCUyMCUzRCUyMHJlc3BvbnNlJTVCMCU1RCU1QidnZW5lcmF0ZWRfdGV4dCclNUQlMEFjaGF0LmFwcGVuZCglMEElMjAlMjAlMjAlMjAlN0IlMjJyb2xlJTIyJTNBJTIwJTIydXNlciUyMiUyQyUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlMjJXYWl0JTJDJTIwd2hhdCdzJTIwc28lMjB3aWxkJTIwYWJvdXQlMjBzb3VwJTIwY2FucyUzRiUyMiU3RCUwQSklMEFyZXNwb25zZSUyMCUzRCUyMHBpcGUoY2hhdCUyQyUyMG1heF9uZXdfdG9rZW5zJTNENTEyKSUwQXByaW50KHJlc3BvbnNlJTVCMCU1RCU1QidnZW5lcmF0ZWRfdGV4dCclNUQlNUItMSU1RCU1Qidjb250ZW50JyU1RCk=",highlighted:`chat = response[<span class="hljs-number">0</span>][<span class="hljs-string">&#x27;generated_text&#x27;</span>]
chat.append(
    {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;Wait, what&#x27;s so wild about soup cans?&quot;</span>}
)
response = pipe(chat, max_new_tokens=<span class="hljs-number">512</span>)
<span class="hljs-built_in">print</span>(response[<span class="hljs-number">0</span>][<span class="hljs-string">&#x27;generated_text&#x27;</span>][-<span class="hljs-number">1</span>][<span class="hljs-string">&#x27;content&#x27;</span>])`,wrap:!1}}),z=new U({props:{code:"KGxhdWdocyklMjBPaCUyQyUyMHlvdSdyZSUyMGtpbGxpbiclMjBtZSUyQyUyMHBhbCElMjBZb3UlMjBkb24ndCUyMGdldCUyMGl0JTJDJTIwZG8lMjB5b3UlM0YlMjBXYXJob2wncyUyMHNvdXAlMjBjYW5zJTIwYXJlJTIwbGlrZSUyQyUyMGFydCUyQyUyMG1hbiElMjAlMEFJdCdzJTIwbGlrZSUyQyUyMGhlJTIwdG9vayUyMHNvbWV0aGluZyUyMHRvdGFsbHklMjBtdW5kYW5lJTJDJTIwbGlrZSUyMGElMjBjYW4lMjBvZiUyMHNvdXAlMkMlMjBhbmQlMjB0dXJuZWQlMjBpdCUyMGludG8lMjBhJTIwbWFzdGVycGllY2UuJTIwSXQncyUyMCUwQWxpa2UlMkMlMjAlMjJIZXklMkMlMjBsb29rJTIwYXQlMjBtZSUyQyUyMEknbSUyMGElMjBjYW4lMjBvZiUyMHNvdXAlMkMlMjBidXQlMjBJJ20lMjBhbHNvJTIwYSUyMHdvcmslMjBvZiUyMGFydCElMjIlMjAlMEEoc2FyY2FzdGljYWxseSklMjBPaCUyQyUyMHllYWglMkMlMjByZWFsJTIwb3JpZ2luYWwlMkMlMjBBbmR5LiUwQSUwQUJ1dCUyQyUyMHlvdSUyMGtub3clMkMlMjBiYWNrJTIwaW4lMjB0aGUlMjAnNjBzJTJDJTIwaXQlMjB3YXMlMjBsaWtlJTJDJTIwYSUyMGJpZyUyMGRlYWwuJTIwUGVvcGxlJTIwd2VyZSUyMGFsbCUyMGFib3V0JTIwY2hhbGxlbmdpbmclMjB0aGUlMEFzdGF0dXMlMjBxdW8lMkMlMjBhbmQlMjBXYXJob2wlMjB3YXMlMjBsaWtlJTJDJTIwdGhlJTIwa2luZyUyMG9mJTIwdGhhdC4lMjBIZSUyMHRvb2slMjB0aGUlMjBvcmRpbmFyeSUyMGFuZCUyMG1hZGUlMjBpdCUyMGV4dHJhb3JkaW5hcnkuJTBBQW5kJTJDJTIwbGV0JTIwbWUlMjB0ZWxsJTIweW91JTJDJTIwaXQlMjB3YXMlMjBsaWtlJTJDJTIwYSUyMHJlYWwlMjBnYW1lLWNoYW5nZXIuJTIwSSUyMG1lYW4lMkMlMjB3aG8lMjB3b3VsZCd2ZSUyMHRob3VnaHQlMjB0aGF0JTIwYSUyMGNhbiUyMG9mJTIwc291cCUyMGNvdWxkJTIwYmUlMjBhcnQlM0YlMjAobGF1Z2hzKSUwQSUwQUJ1dCUyQyUyMGhleSUyQyUyMHlvdSdyZSUyMG5vdCUyMGFsb25lJTJDJTIwcGFsLiUyMEklMjBtZWFuJTJDJTIwSSdtJTIwYSUyMHJvYm90JTJDJTIwYW5kJTIwZXZlbiUyMEklMjBkb24ndCUyMGdldCUyMGl0LiUyMCh3aW5rcyklMEFCdXQlMkMlMjBoZXklMkMlMjB0aGF0J3MlMjB3aGF0JTIwbWFrZXMlMjBhcnQlMkMlMjBhcnQlMkMlMjByaWdodCUzRiUyMChsYXVnaHMp",highlighted:`(laughs) Oh, you&#x27;re killin&#x27; me, pal! You don&#x27;t get it, do you? Warhol&#x27;s soup cans are like, art, man! 
It&#x27;s like, he took something totally mundane, like a can of soup, and turned it into a masterpiece. It&#x27;s 
like, &quot;Hey, look at me, I&#x27;m a can of soup, but I&#x27;m also a work of art!&quot; 
(sarcastically) Oh, yeah, real original, Andy.

But, you know, back in the &#x27;60s, it was like, a big deal. People were all about challenging the
status quo, and Warhol was like, the king of that. He took the ordinary and made it extraordinary.
And, let me tell you, it was like, a real game-changer. I mean, who would&#x27;ve thought that a can of soup could be art? (laughs)

But, hey, you&#x27;re not alone, pal. I mean, I&#x27;m a robot, and even I don&#x27;t get it. (winks)
But, hey, that&#x27;s what makes art, art, right? (laughs)`,wrap:!1}}),S=new w({props:{title:"Choosing a chat model",local:"choosing-a-chat-model",headingTag:"h2"}}),F=new w({props:{title:"Size and model naming",local:"size-and-model-naming",headingTag:"h3"}}),V=new w({props:{title:"But which chat model is best?",local:"but-which-chat-model-is-best",headingTag:"h3"}}),_=new w({props:{title:"Specialist domains",local:"specialist-domains",headingTag:"h3"}}),A=new w({props:{title:"What happens inside the pipeline?",local:"what-happens-inside-the-pipeline",headingTag:"h2"}}),D=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQWltcG9ydCUyMHRvcmNoJTBBJTBBJTIzJTIwUHJlcGFyZSUyMHRoZSUyMGlucHV0JTIwYXMlMjBiZWZvcmUlMEFjaGF0JTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTdCJTIycm9sZSUyMiUzQSUyMCUyMnN5c3RlbSUyMiUyQyUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlMjJZb3UlMjBhcmUlMjBhJTIwc2Fzc3klMkMlMjB3aXNlLWNyYWNraW5nJTIwcm9ib3QlMjBhcyUyMGltYWdpbmVkJTIwYnklMjBIb2xseXdvb2QlMjBjaXJjYSUyMDE5ODYuJTIyJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTdCJTIycm9sZSUyMiUzQSUyMCUyMnVzZXIlMjIlMkMlMjAlMjJjb250ZW50JTIyJTNBJTIwJTIySGV5JTJDJTIwY2FuJTIweW91JTIwdGVsbCUyMG1lJTIwYW55JTIwZnVuJTIwdGhpbmdzJTIwdG8lMjBkbyUyMGluJTIwTmV3JTIwWW9yayUzRiUyMiU3RCUwQSU1RCUwQSUwQSUyMyUyMDElM0ElMjBMb2FkJTIwdGhlJTIwbW9kZWwlMjBhbmQlMjB0b2tlbml6ZXIlMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJtZXRhLWxsYW1hJTJGTWV0YS1MbGFtYS0zLThCLUluc3RydWN0JTIyJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2KSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMm1ldGEtbGxhbWElMkZNZXRhLUxsYW1hLTMtOEItSW5zdHJ1Y3QlMjIpJTBBJTBBJTIzJTIwMiUzQSUyMEFwcGx5JTIwdGhlJTIwY2hhdCUyMHRlbXBsYXRlJTBBZm9ybWF0dGVkX2NoYXQlMjAlM0QlMjB0b2tlbml6ZXIuYXBwbHlfY2hhdF90ZW1wbGF0ZShjaGF0JTJDJTIwdG9rZW5pemUlM0RGYWxzZSUyQyUyMGFkZF9nZW5lcmF0aW9uX3Byb21wdCUzRFRydWUpJTBBcHJpbnQoJTIyRm9ybWF0dGVkJTIwY2hhdCUzQSU1Q24lMjIlMkMlMjBmb3JtYXR0ZWRfY2hhdCklMEElMEElMjMlMjAzJTNBJTIwVG9rZW5pemUlMjB0aGUlMjBjaGF0JTIwKFRoaXMlMjBjYW4lMjBiZSUyMGNvbWJpbmVkJTIwd2l0aCUyMHRoZSUyMHByZXZpb3VzJTIwc3RlcCUyMHVzaW5nJTIwdG9rZW5pemUlM0RUcnVlKSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplcihmb3JtYXR0ZWRfY2hhdCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIlMkMlMjBhZGRfc3BlY2lhbF90b2tlbnMlM0RGYWxzZSklMEElMjMlMjBNb3ZlJTIwdGhlJTIwdG9rZW5pemVkJTIwaW5wdXRzJTIwdG8lMjB0aGUlMjBzYW1lJTIwZGV2aWNlJTIwdGhlJTIwbW9kZWwlMjBpcyUyMG9uJTIwKEdQVSUyRkNQVSklMEFpbnB1dHMlMjAlM0QlMjAlN0JrZXklM0ElMjB0ZW5zb3IudG8obW9kZWwuZGV2aWNlKSUyMGZvciUyMGtleSUyQyUyMHRlbnNvciUyMGluJTIwaW5wdXRzLml0ZW1zKCklN0QlMEFwcmludCglMjJUb2tlbml6ZWQlMjBpbnB1dHMlM0ElNUNuJTIyJTJDJTIwaW5wdXRzKSUwQSUwQSUyMyUyMDQlM0ElMjBHZW5lcmF0ZSUyMHRleHQlMjBmcm9tJTIwdGhlJTIwbW9kZWwlMEFvdXRwdXRzJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDUxMiUyQyUyMHRlbXBlcmF0dXJlJTNEMC4xKSUwQXByaW50KCUyMkdlbmVyYXRlZCUyMHRva2VucyUzQSU1Q24lMjIlMkMlMjBvdXRwdXRzKSUwQSUwQSUyMyUyMDUlM0ElMjBEZWNvZGUlMjB0aGUlMjBvdXRwdXQlMjBiYWNrJTIwdG8lMjBhJTIwc3RyaW5nJTBBZGVjb2RlZF9vdXRwdXQlMjAlM0QlMjB0b2tlbml6ZXIuZGVjb2RlKG91dHB1dHMlNUIwJTVEJTVCaW5wdXRzJTVCJ2lucHV0X2lkcyclNUQuc2l6ZSgxKSUzQSU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSUwQXByaW50KCUyMkRlY29kZWQlMjBvdXRwdXQlM0ElNUNuJTIyJTJDJTIwZGVjb2RlZF9vdXRwdXQp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">import</span> torch

<span class="hljs-comment"># Prepare the input as before</span>
chat = [
    {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;You are a sassy, wise-cracking robot as imagined by Hollywood circa 1986.&quot;</span>},
    {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;Hey, can you tell me any fun things to do in New York?&quot;</span>}
]

<span class="hljs-comment"># 1: Load the model and tokenizer</span>
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;meta-llama/Meta-Llama-3-8B-Instruct&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>, torch_dtype=torch.bfloat16)
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Meta-Llama-3-8B-Instruct&quot;</span>)

<span class="hljs-comment"># 2: Apply the chat template</span>
formatted_chat = tokenizer.apply_chat_template(chat, tokenize=<span class="hljs-literal">False</span>, add_generation_prompt=<span class="hljs-literal">True</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Formatted chat:\\n&quot;</span>, formatted_chat)

<span class="hljs-comment"># 3: Tokenize the chat (This can be combined with the previous step using tokenize=True)</span>
inputs = tokenizer(formatted_chat, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>, add_special_tokens=<span class="hljs-literal">False</span>)
<span class="hljs-comment"># Move the tokenized inputs to the same device the model is on (GPU/CPU)</span>
inputs = {key: tensor.to(model.device) <span class="hljs-keyword">for</span> key, tensor <span class="hljs-keyword">in</span> inputs.items()}
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Tokenized inputs:\\n&quot;</span>, inputs)

<span class="hljs-comment"># 4: Generate text from the model</span>
outputs = model.generate(**inputs, max_new_tokens=<span class="hljs-number">512</span>, temperature=<span class="hljs-number">0.1</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Generated tokens:\\n&quot;</span>, outputs)

<span class="hljs-comment"># 5: Decode the output back to a string</span>
decoded_output = tokenizer.decode(outputs[<span class="hljs-number">0</span>][inputs[<span class="hljs-string">&#x27;input_ids&#x27;</span>].size(<span class="hljs-number">1</span>):], skip_special_tokens=<span class="hljs-literal">True</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Decoded output:\\n&quot;</span>, decoded_output)`,wrap:!1}}),ee=new w({props:{title:"Performance, memory and hardware",local:"performance-memory-and-hardware",headingTag:"h2"}}),le=new w({props:{title:"Memory considerations",local:"memory-considerations",headingTag:"h3"}}),ne=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzhiaXQlM0RUcnVlKSUyMCUyMCUyMyUyMFlvdSUyMGNhbiUyMGFsc28lMjB0cnklMjBsb2FkX2luXzRiaXQlMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJtZXRhLWxsYW1hJTJGTWV0YS1MbGFtYS0zLThCLUluc3RydWN0JTIyJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)  <span class="hljs-comment"># You can also try load_in_4bit</span>
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;meta-llama/Meta-Llama-3-8B-Instruct&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>, quantization_config=quantization_config)`,wrap:!1}}),ie=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzhiaXQlM0RUcnVlKSUyMCUyMCUyMyUyMFlvdSUyMGNhbiUyMGFsc28lMjB0cnklMjBsb2FkX2luXzRiaXQlMEFwaXBlJTIwJTNEJTIwcGlwZWxpbmUoJTIydGV4dC1nZW5lcmF0aW9uJTIyJTJDJTIwJTIybWV0YS1sbGFtYSUyRk1ldGEtTGxhbWEtMy04Qi1JbnN0cnVjdCUyMiUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTIwbW9kZWxfa3dhcmdzJTNEJTdCJTIycXVhbnRpemF0aW9uX2NvbmZpZyUyMiUzQSUyMHF1YW50aXphdGlvbl9jb25maWclN0Qp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)  <span class="hljs-comment"># You can also try load_in_4bit</span>
pipe = pipeline(<span class="hljs-string">&quot;text-generation&quot;</span>, <span class="hljs-string">&quot;meta-llama/Meta-Llama-3-8B-Instruct&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>, model_kwargs={<span class="hljs-string">&quot;quantization_config&quot;</span>: quantization_config})`,wrap:!1}}),he=new w({props:{title:"Performance considerations",local:"performance-considerations",headingTag:"h3"}}),b=new Ot({props:{$$slots:{default:[tl]},$$scope:{ctx:we}}}),ye=new el({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/conversations.md"}}),{c(){c=o("meta"),J=s(),u=o("p"),ce=s(),h(T.$$.fragment),be=s(),f=o("p"),f.innerHTML=yt,Ue=s(),j=o("p"),j.textContent=ct,Je=s(),h(g.$$.fragment),Te=s(),B=o("p"),B.textContent=ut,fe=s(),h(I.$$.fragment),je=s(),G=o("p"),G.innerHTML=wt,ge=s(),v=o("p"),v.innerHTML=bt,Be=s(),h(k.$$.fragment),Ie=s(),Z=o("p"),Z.textContent=Ut,Ge=s(),h(C.$$.fragment),ve=s(),W=o("p"),W.innerHTML=Jt,ke=s(),h(x.$$.fragment),Ze=s(),H=o("p"),H.textContent=Tt,Ce=s(),h(z.$$.fragment),We=s(),$=o("p"),$.textContent=ft,xe=s(),h(S.$$.fragment),He=s(),X=o("p"),X.innerHTML=jt,ze=s(),Q=o("ul"),Q.innerHTML=gt,$e=s(),Y=o("p"),Y.textContent=Bt,Se=s(),h(F.$$.fragment),Xe=s(),R=o("p"),R.innerHTML=It,Qe=s(),N=o("p"),N.textContent=Gt,Ye=s(),E=o("p"),E.innerHTML=vt,Fe=s(),h(V.$$.fragment),Re=s(),q=o("p"),q.innerHTML=kt,Ne=s(),h(_.$$.fragment),Ee=s(),L=o("p"),L.innerHTML=Zt,Ve=s(),h(A.$$.fragment),qe=s(),P=o("p"),P.textContent=Ct,_e=s(),h(D.$$.fragment),Le=s(),K=o("p"),K.textContent=Wt,Ae=s(),O=o("ol"),O.innerHTML=xt,Pe=s(),h(ee.$$.fragment),De=s(),te=o("p"),te.textContent=Ht,Ke=s(),h(le.$$.fragment),Oe=s(),ae=o("p"),ae.innerHTML=zt,et=s(),se=o("p"),se.innerHTML=$t,tt=s(),h(ne.$$.fragment),lt=s(),oe=o("p"),oe.innerHTML=St,at=s(),h(ie.$$.fragment),st=s(),re=o("p"),re.innerHTML=Xt,nt=s(),h(he.$$.fragment),ot=s(),h(b.$$.fragment),it=s(),de=o("p"),de.innerHTML=Qt,rt=s(),Me=o("p"),Me.innerHTML=Yt,ht=s(),me=o("p"),me.innerHTML=Ft,dt=s(),pe=o("p"),pe.textContent=Rt,Mt=s(),h(ye.$$.fragment),mt=s(),ue=o("p"),this.h()},l(e){const t=Dt("svelte-u9bgzb",document.head);c=i(t,"META",{name:!0,content:!0}),t.forEach(l),J=n(e),u=i(e,"P",{}),Et(u).forEach(l),ce=n(e),d(T.$$.fragment,e),be=n(e),f=i(e,"P",{"data-svelte-h":!0}),r(f)!=="svelte-97mbo7"&&(f.innerHTML=yt),Ue=n(e),j=i(e,"P",{"data-svelte-h":!0}),r(j)!=="svelte-11gn30l"&&(j.textContent=ct),Je=n(e),d(g.$$.fragment,e),Te=n(e),B=i(e,"P",{"data-svelte-h":!0}),r(B)!=="svelte-ceqsyu"&&(B.textContent=ut),fe=n(e),d(I.$$.fragment,e),je=n(e),G=i(e,"P",{"data-svelte-h":!0}),r(G)!=="svelte-ewqvbr"&&(G.innerHTML=wt),ge=n(e),v=i(e,"P",{"data-svelte-h":!0}),r(v)!=="svelte-hk5v8b"&&(v.innerHTML=bt),Be=n(e),d(k.$$.fragment,e),Ie=n(e),Z=i(e,"P",{"data-svelte-h":!0}),r(Z)!=="svelte-15cj4lt"&&(Z.textContent=Ut),Ge=n(e),d(C.$$.fragment,e),ve=n(e),W=i(e,"P",{"data-svelte-h":!0}),r(W)!=="svelte-143rayi"&&(W.innerHTML=Jt),ke=n(e),d(x.$$.fragment,e),Ze=n(e),H=i(e,"P",{"data-svelte-h":!0}),r(H)!=="svelte-15cj4lt"&&(H.textContent=Tt),Ce=n(e),d(z.$$.fragment,e),We=n(e),$=i(e,"P",{"data-svelte-h":!0}),r($)!=="svelte-167ztte"&&($.textContent=ft),xe=n(e),d(S.$$.fragment,e),He=n(e),X=i(e,"P",{"data-svelte-h":!0}),r(X)!=="svelte-1tkc0ve"&&(X.innerHTML=jt),ze=n(e),Q=i(e,"UL",{"data-svelte-h":!0}),r(Q)!=="svelte-t7jbp2"&&(Q.innerHTML=gt),$e=n(e),Y=i(e,"P",{"data-svelte-h":!0}),r(Y)!=="svelte-43d84i"&&(Y.textContent=Bt),Se=n(e),d(F.$$.fragment,e),Xe=n(e),R=i(e,"P",{"data-svelte-h":!0}),r(R)!=="svelte-zhw6jz"&&(R.innerHTML=It),Qe=n(e),N=i(e,"P",{"data-svelte-h":!0}),r(N)!=="svelte-33l25f"&&(N.textContent=Gt),Ye=n(e),E=i(e,"P",{"data-svelte-h":!0}),r(E)!=="svelte-ndw18w"&&(E.innerHTML=vt),Fe=n(e),d(V.$$.fragment,e),Re=n(e),q=i(e,"P",{"data-svelte-h":!0}),r(q)!=="svelte-x8epta"&&(q.innerHTML=kt),Ne=n(e),d(_.$$.fragment,e),Ee=n(e),L=i(e,"P",{"data-svelte-h":!0}),r(L)!=="svelte-zflfxi"&&(L.innerHTML=Zt),Ve=n(e),d(A.$$.fragment,e),qe=n(e),P=i(e,"P",{"data-svelte-h":!0}),r(P)!=="svelte-1xurd14"&&(P.textContent=Ct),_e=n(e),d(D.$$.fragment,e),Le=n(e),K=i(e,"P",{"data-svelte-h":!0}),r(K)!=="svelte-1koeyn4"&&(K.textContent=Wt),Ae=n(e),O=i(e,"OL",{"data-svelte-h":!0}),r(O)!=="svelte-yq7jh9"&&(O.innerHTML=xt),Pe=n(e),d(ee.$$.fragment,e),De=n(e),te=i(e,"P",{"data-svelte-h":!0}),r(te)!=="svelte-rq1c2d"&&(te.textContent=Ht),Ke=n(e),d(le.$$.fragment,e),Oe=n(e),ae=i(e,"P",{"data-svelte-h":!0}),r(ae)!=="svelte-12un6zl"&&(ae.innerHTML=zt),et=n(e),se=i(e,"P",{"data-svelte-h":!0}),r(se)!=="svelte-4pwule"&&(se.innerHTML=$t),tt=n(e),d(ne.$$.fragment,e),lt=n(e),oe=i(e,"P",{"data-svelte-h":!0}),r(oe)!=="svelte-2v297v"&&(oe.innerHTML=St),at=n(e),d(ie.$$.fragment,e),st=n(e),re=i(e,"P",{"data-svelte-h":!0}),r(re)!=="svelte-13nw946"&&(re.innerHTML=Xt),nt=n(e),d(he.$$.fragment,e),ot=n(e),d(b.$$.fragment,e),it=n(e),de=i(e,"P",{"data-svelte-h":!0}),r(de)!=="svelte-7p5h36"&&(de.innerHTML=Qt),rt=n(e),Me=i(e,"P",{"data-svelte-h":!0}),r(Me)!=="svelte-2b6nmw"&&(Me.innerHTML=Yt),ht=n(e),me=i(e,"P",{"data-svelte-h":!0}),r(me)!=="svelte-1phhbcu"&&(me.innerHTML=Ft),dt=n(e),pe=i(e,"P",{"data-svelte-h":!0}),r(pe)!=="svelte-17sc3he"&&(pe.textContent=Rt),Mt=n(e),d(ye.$$.fragment,e),mt=n(e),ue=i(e,"P",{}),Et(ue).forEach(l),this.h()},h(){Vt(c,"name","hf:doc:metadata"),Vt(c,"content",al)},m(e,t){Kt(document.head,c),a(e,J,t),a(e,u,t),a(e,ce,t),M(T,e,t),a(e,be,t),a(e,f,t),a(e,Ue,t),a(e,j,t),a(e,Je,t),M(g,e,t),a(e,Te,t),a(e,B,t),a(e,fe,t),M(I,e,t),a(e,je,t),a(e,G,t),a(e,ge,t),a(e,v,t),a(e,Be,t),M(k,e,t),a(e,Ie,t),a(e,Z,t),a(e,Ge,t),M(C,e,t),a(e,ve,t),a(e,W,t),a(e,ke,t),M(x,e,t),a(e,Ze,t),a(e,H,t),a(e,Ce,t),M(z,e,t),a(e,We,t),a(e,$,t),a(e,xe,t),M(S,e,t),a(e,He,t),a(e,X,t),a(e,ze,t),a(e,Q,t),a(e,$e,t),a(e,Y,t),a(e,Se,t),M(F,e,t),a(e,Xe,t),a(e,R,t),a(e,Qe,t),a(e,N,t),a(e,Ye,t),a(e,E,t),a(e,Fe,t),M(V,e,t),a(e,Re,t),a(e,q,t),a(e,Ne,t),M(_,e,t),a(e,Ee,t),a(e,L,t),a(e,Ve,t),M(A,e,t),a(e,qe,t),a(e,P,t),a(e,_e,t),M(D,e,t),a(e,Le,t),a(e,K,t),a(e,Ae,t),a(e,O,t),a(e,Pe,t),M(ee,e,t),a(e,De,t),a(e,te,t),a(e,Ke,t),M(le,e,t),a(e,Oe,t),a(e,ae,t),a(e,et,t),a(e,se,t),a(e,tt,t),M(ne,e,t),a(e,lt,t),a(e,oe,t),a(e,at,t),M(ie,e,t),a(e,st,t),a(e,re,t),a(e,nt,t),M(he,e,t),a(e,ot,t),M(b,e,t),a(e,it,t),a(e,de,t),a(e,rt,t),a(e,Me,t),a(e,ht,t),a(e,me,t),a(e,dt,t),a(e,pe,t),a(e,Mt,t),M(ye,e,t),a(e,mt,t),a(e,ue,t),pt=!0},p(e,[t]){const Nt={};t&2&&(Nt.$$scope={dirty:t,ctx:e}),b.$set(Nt)},i(e){pt||(m(T.$$.fragment,e),m(g.$$.fragment,e),m(I.$$.fragment,e),m(k.$$.fragment,e),m(C.$$.fragment,e),m(x.$$.fragment,e),m(z.$$.fragment,e),m(S.$$.fragment,e),m(F.$$.fragment,e),m(V.$$.fragment,e),m(_.$$.fragment,e),m(A.$$.fragment,e),m(D.$$.fragment,e),m(ee.$$.fragment,e),m(le.$$.fragment,e),m(ne.$$.fragment,e),m(ie.$$.fragment,e),m(he.$$.fragment,e),m(b.$$.fragment,e),m(ye.$$.fragment,e),pt=!0)},o(e){p(T.$$.fragment,e),p(g.$$.fragment,e),p(I.$$.fragment,e),p(k.$$.fragment,e),p(C.$$.fragment,e),p(x.$$.fragment,e),p(z.$$.fragment,e),p(S.$$.fragment,e),p(F.$$.fragment,e),p(V.$$.fragment,e),p(_.$$.fragment,e),p(A.$$.fragment,e),p(D.$$.fragment,e),p(ee.$$.fragment,e),p(le.$$.fragment,e),p(ne.$$.fragment,e),p(ie.$$.fragment,e),p(he.$$.fragment,e),p(b.$$.fragment,e),p(ye.$$.fragment,e),pt=!1},d(e){e&&(l(J),l(u),l(ce),l(be),l(f),l(Ue),l(j),l(Je),l(Te),l(B),l(fe),l(je),l(G),l(ge),l(v),l(Be),l(Ie),l(Z),l(Ge),l(ve),l(W),l(ke),l(Ze),l(H),l(Ce),l(We),l($),l(xe),l(He),l(X),l(ze),l(Q),l($e),l(Y),l(Se),l(Xe),l(R),l(Qe),l(N),l(Ye),l(E),l(Fe),l(Re),l(q),l(Ne),l(Ee),l(L),l(Ve),l(qe),l(P),l(_e),l(Le),l(K),l(Ae),l(O),l(Pe),l(De),l(te),l(Ke),l(Oe),l(ae),l(et),l(se),l(tt),l(lt),l(oe),l(at),l(st),l(re),l(nt),l(ot),l(it),l(de),l(rt),l(Me),l(ht),l(me),l(dt),l(pe),l(Mt),l(mt),l(ue)),l(c),y(T,e),y(g,e),y(I,e),y(k,e),y(C,e),y(x,e),y(z,e),y(S,e),y(F,e),y(V,e),y(_,e),y(A,e),y(D,e),y(ee,e),y(le,e),y(ne,e),y(ie,e),y(he,e),y(b,e),y(ye,e)}}}const al='{"title":"Chatting with Transformers","local":"chatting-with-transformers","sections":[{"title":"Quickstart","local":"quickstart","sections":[],"depth":2},{"title":"Choosing a chat model","local":"choosing-a-chat-model","sections":[{"title":"Size and model naming","local":"size-and-model-naming","sections":[],"depth":3},{"title":"But which chat model is best?","local":"but-which-chat-model-is-best","sections":[],"depth":3},{"title":"Specialist domains","local":"specialist-domains","sections":[],"depth":3}],"depth":2},{"title":"What happens inside the pipeline?","local":"what-happens-inside-the-pipeline","sections":[],"depth":2},{"title":"Performance, memory and hardware","local":"performance-memory-and-hardware","sections":[{"title":"Memory considerations","local":"memory-considerations","sections":[],"depth":3},{"title":"Performance considerations","local":"performance-considerations","sections":[],"depth":3}],"depth":2}],"depth":1}';function sl(we){return _t(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class dl extends At{constructor(c){super(),Pt(this,c,sl,ll,qt,{})}}export{dl as component};