google-cloud/pr_113/en/_app/immutable/nodes/12.ea0212e8.js

import{s as Ya,o as Xa,n as rl}from"../chunks/scheduler.b108d059.js";import{S as Ra,i as Ha,g as o,s as t,r,A as Ea,h as i,f as e,c as n,j as ul,u as m,x as c,k as xa,y as X,a,v as h,d as M,t as u,w as y}from"../chunks/index.008de539.js";import{T as cl}from"../chunks/Tip.aeb15ab7.js";import{C as b}from"../chunks/CodeBlock.3968c746.js";import{H as Z,E as Qa}from"../chunks/EditOnGithub.d1c48e3d.js";function Fa(I){let p,g='Installing the <code>gke-gcloud-auth-plugin</code> does not need to be installed via <code>gcloud</code> specifically, to read more about the alternative installation methods, please visit <a href="https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-access-for-kubectl#install_plugin" rel="nofollow">GKE Documentation - Install kubectl and configure cluster access</a>.';return{c(){p=o("p"),p.innerHTML=g},l(j){p=i(j,"P",{"data-svelte-h":!0}),c(p)!=="svelte-15icaac"&&(p.innerHTML=g)},m(j,J){a(j,p,J)},p:rl,d(j){j&&e(p)}}}function qa(I){let p,g='Important to check before creating the GKE Autopilot Cluster the <a href="https://cloud.google.com/kubernetes-engine/docs/how-to/performance-pods" rel="nofollow">GKE Documentation - Optimize Autopilot Pod performance by choosing a machine series</a>, since not all the versions support GPU accelerators e.g. <code>nvidia-l4</code> is not supported in the GKE cluster versions 1.28.3 or lower.';return{c(){p=o("p"),p.innerHTML=g},l(j){p=i(j,"P",{"data-svelte-h":!0}),c(p)!=="svelte-1cdm5nv"&&(p.innerHTML=g)},m(j,J){a(j,p,J)},p:rl,d(j){j&&e(p)}}}function Aa(I){let p,g="To select the specific version in your location of the GKE Cluster, you can run the following command:",j,J,U,f,G='For more information please visit <a href="https://cloud.google.com/kubernetes-engine/versioning#specifying_cluster_version" rel="nofollow">GKE Documentation - Specifying cluster version</a>.',$;return J=new b({props:{code:"Z2Nsb3VkJTIwY29udGFpbmVyJTIwZ2V0LXNlcnZlci1jb25maWclMjAlNUMlMEElMjAlMjAlMjAlMjAtLWZsYXR0ZW4lM0QlMjJjaGFubmVscyUyMiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tZmlsdGVyJTNEJTIyY2hhbm5lbHMuY2hhbm5lbCUzRFNUQUJMRSUyMiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tZm9ybWF0JTNEJTIyeWFtbChjaGFubmVscy5jaGFubmVsJTJDY2hhbm5lbHMuZGVmYXVsdFZlcnNpb24pJTIyJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1sb2NhdGlvbiUzRCUyNExPQ0FUSU9O",highlighted:`gcloud container get-server-config \\
    --flatten=<span class="hljs-string">&quot;channels&quot;</span> \\
    --filter=<span class="hljs-string">&quot;channels.channel=STABLE&quot;</span> \\
    --format=<span class="hljs-string">&quot;yaml(channels.channel,channels.defaultVersion)&quot;</span> \\
    --location=<span class="hljs-variable">$LOCATION</span>`,wrap:!1}}),{c(){p=o("p"),p.textContent=g,j=t(),r(J.$$.fragment),U=t(),f=o("p"),f.innerHTML=G},l(d){p=i(d,"P",{"data-svelte-h":!0}),c(p)!=="svelte-a3yff9"&&(p.textContent=g),j=n(d),m(J.$$.fragment,d),U=n(d),f=i(d,"P",{"data-svelte-h":!0}),c(f)!=="svelte-1bsy0hk"&&(f.innerHTML=G)},m(d,w){a(d,p,w),a(d,j,w),h(J,d,w),a(d,U,w),a(d,f,w),$=!0},p:rl,i(d){$||(M(J.$$.fragment,d),$=!0)},o(d){u(J.$$.fragment,d),$=!1},d(d){d&&(e(p),e(j),e(U),e(f)),y(J,d)}}}function La(I){let p,g='To explore all the models that can be served via TGI, you can explore <a href="https://huggingface.co/models?other=text-generation-inference" rel="nofollow">the models tagged with <code>text-generation-inference</code> in the Hub</a>.';return{c(){p=o("p"),p.innerHTML=g},l(j){p=i(j,"P",{"data-svelte-h":!0}),c(p)!=="svelte-zgh49r"&&(p.innerHTML=g)},m(j,J){a(j,p,J)},p:rl,d(j){j&&e(p)}}}function za(I){let p,g="The Kubernetes deployment may take a few minutes to be ready, so you can check the status of the deployment with the following command:",j,J,U,f,G="Alternatively, you can just wait for the deployment to be ready with the following command:",$,d,w;return J=new b({props:{code:"a3ViZWN0bCUyMGdldCUyMHBvZHM=",highlighted:"kubectl get pods",wrap:!1}}),d=new b({props:{code:"a3ViZWN0bCUyMHdhaXQlMjAtLWZvciUzRGNvbmRpdGlvbiUzREF2YWlsYWJsZSUyMC0tdGltZW91dCUzRDcwMHMlMjBkZXBsb3ltZW50JTJGdGdpLWRlcGxveW1lbnQ=",highlighted:'kubectl <span class="hljs-built_in">wait</span> --<span class="hljs-keyword">for</span>=condition=Available --<span class="hljs-built_in">timeout</span>=700s deployment/tgi-deployment',wrap:!1}}),{c(){p=o("p"),p.textContent=g,j=t(),r(J.$$.fragment),U=t(),f=o("p"),f.textContent=G,$=t(),r(d.$$.fragment)},l(T){p=i(T,"P",{"data-svelte-h":!0}),c(p)!=="svelte-qgh43p"&&(p.textContent=g),j=n(T),m(J.$$.fragment,T),U=n(T),f=i(T,"P",{"data-svelte-h":!0}),c(f)!=="svelte-16v3quh"&&(f.textContent=G),$=n(T),m(d.$$.fragment,T)},m(T,C){a(T,p,C),a(T,j,C),h(J,T,C),a(T,U,C),a(T,f,C),a(T,$,C),h(d,T,C),w=!0},p:rl,i(T){w||(M(J.$$.fragment,T),M(d.$$.fragment,T),w=!0)},o(T){u(J.$$.fragment,T),u(d.$$.fragment,T),w=!1},d(T){T&&(e(p),e(j),e(U),e(f),e($)),y(J,T),y(d,T)}}}function _a(I){let p,g="In the examples below <code>localhost</code> will be used, but if you did deploy TGI with the ingress, feel free to use the ingress IP as mentioned above (without specifying the port).";return{c(){p=o("p"),p.innerHTML=g},l(j){p=i(j,"P",{"data-svelte-h":!0}),c(p)!=="svelte-1cpu0bq"&&(p.innerHTML=g)},m(j,J){a(j,p,J)},p:rl,d(j){j&&e(p)}}}function Sa(I){let p,g='📍 Find the complete example on GitHub <a href="https://github.com/huggingface/Google-Cloud-Containers/tree/merge/examples/gke/tgi-llama-405b-deployment" rel="nofollow">here</a>!';return{c(){p=o("p"),p.innerHTML=g},l(j){p=i(j,"P",{"data-svelte-h":!0}),c(p)!=="svelte-1qr2r57"&&(p.innerHTML=g)},m(j,J){a(j,p,J)},p:rl,d(j){j&&e(p)}}}function Da(I){let p,g,j,J,U,f,G,$='<a href="https://huggingface.co/blog/llama31" rel="nofollow">Llama 3.1</a> is one of the latest LLMs from the Llama family released by Meta (latest is Llama 3.2 as of October 2024); three sizes: 8B for efficient deployment and development on consumer-size GPU, 70B for large-scale AI native applications, and 405B for synthetic data, LLM as a Judge or distillation; among other use cases; whilst the 405B variant being one of the biggest open LLMs. Text Generation Inference (TGI) is a toolkit developed by Hugging Face for deploying and serving LLMs, with high performance text generation. And, Google Kubernetes Engine (GKE) is a fully-managed Kubernetes service in Google Cloud that can be used to deploy and operate containerized applications at scale using Google infrastructure.',d,w,T='This example showcases how to deploy <a href="https://hf.co/meta-llama/Llama-3.1-405B-Instruct-FP8" rel="nofollow"><code>meta-llama/Llama-3.1-405B-Instruct-FP8</code></a> on a GKE Cluster on a node with 8 NVIDIA H100s via the Hugging Face purpose-built Deep Learning Container (DLC) for Text Generation Inference (TGI) on Google Cloud.',C,R,yl,H,Le="First, you need to install both <code>gcloud</code> and <code>kubectl</code> in your local machine, which are the command-line tools for Google Cloud and Kubernetes, respectively, to interact with the GCP and the GKE Cluster.",jl,E,ze='<li>To install <code>gcloud</code>, follow the instructions at <a href="https://cloud.google.com/sdk/docs/install" rel="nofollow">Cloud SDK Documentation - Install the gcloud CLI</a>.</li> <li>To install <code>kubectl</code>, follow the instructions at <a href="https://kubernetes.io/docs/tasks/tools/#kubectl" rel="nofollow">Kubernetes Documentation - Install Tools</a>.</li>',dl,Q,_e="Optionally, to ease the usage of the commands within this tutorial, you need to set the following environment variables for GCP:",Jl,F,Tl,q,Se="Then you need to login into your GCP account and set the project ID to the one you want to use for the deployment of the GKE Cluster.",bl,A,gl,L,De="Once you are logged in, you need to enable the necessary service APIs in GCP, such as the Google Kubernetes Engine API, the Google Container Registry API, and the Google Container File System API, which are necessary for the deployment of the GKE Cluster and the Hugging Face DLC for TGI.",fl,z,wl,_,Pe="Additionally, to use <code>kubectl</code> with the GKE Cluster credentials, you also need to install the <code>gke-gcloud-auth-plugin</code>, that can be installed with <code>gcloud</code> as follows:",Ul,S,Il,v,Gl,D,Ke='Finally, note that you most likely will need to request a quota increase in order to be able to access the A3 instance with 8 NVIDIA H100 GPUs, as those need a specific manual approval from Google Cloud. To do so you will need to go to <a href="https://console.cloud.google.com/iam-admin/quotas" rel="nofollow">IAM Admin - Quotas</a> and apply the following filters:',$l,P,Oe='<li><p><code>Service: Compute Engine API</code>: as GKE relies on Compute Engine for the resource allocation.</p></li> <li><p><code>Dimensions (e.g. location): region: $LOCATION</code>: replace the <code>$LOCATION</code> value with the location specified above, but note that not all the regions may have NVIDIA H100 GPUs available so check <a href="https://cloud.google.com/compute/docs/regions-zones#available" rel="nofollow">Compute Engine Documentation - Available regions and zones</a>.</p></li> <li><p><code>gpu_family: NVIDIA_H100</code>: is the identified of the NVIDIA H100 GPUs on Google Cloud.</p></li>',Cl,K,sa='And then request a quota increase to 8 NVIDIA H100 GPUs in order to run <a href="https://hf.co/meta-llama/Llama-3.1-405B-Instruct-FP8" rel="nofollow"><code>meta-llama/Llama-3.1-405B-Instruct-FP8</code></a>.',Zl,O,vl,ss,la="Once everything’s set up, you can proceed with the creation of the GKE Cluster and the node pool, which in this case will be a single GPU node, in order to use the GPU accelerator for high performance inference, also following TGI recommendations based on their internal optimizations for GPUs.",Wl,ls,ea="To deploy the GKE Cluster, the “Autopilot” mode will be used as it is the recommended one for most of the workloads, since the underlying infrastructure is managed by Google. Alternatively, you can also use the “Standard” mode.",Bl,W,Vl,es,kl,B,Nl,as,aa='<img src="https://raw.githubusercontent.com/huggingface/Google-Cloud-Containers/merge/examples/gke/tgi-llama-405b-deployment/imgs/gke-cluster.png" alt="GKE Cluster in the GCP Console"/>',xl,ts,ta="Once the GKE Cluster is created, you can get the credentials to access it via <code>kubectl</code> with the following command:",Yl,ns,Xl,ps,Rl,os,na='As <a href="https://hf.co/meta-llama/Llama-3.1-405B-Instruct-FP8" rel="nofollow"><code>meta-llama/Llama-3.1-405B-Instruct-FP8</code></a> is a gated model, you need to set a Kubernetes secret with the Hugging Face Hub token via <code>kubectl</code>.',Hl,is,pa='To generate a custom token for the Hugging Face Hub, you can follow the instructions at <a href="https://huggingface.co/docs/hub/en/security-tokens" rel="nofollow">Hugging Face Hub - User access tokens</a>; and the recommended way of setting it is to install the <code>huggingface_hub</code> Python SDK as follows:',El,cs,Ql,rs,oa="And then login in with the generated token with read-access over the gated/private model:",Fl,ms,ql,hs,ia="Finally, you can create the Kubernetes secret with the generated token for the Hugging Face Hub as follows using the <code>huggingface_hub</code> Python SDK to retrieve the token:",Al,Ms,Ll,us,ca="Or, alternatively, you can directly set the token as follows:",zl,ys,_l,js,ra='<img src="https://raw.githubusercontent.com/huggingface/Google-Cloud-Containers/merge/examples/gke/tgi-llama-405b-deployment/imgs/gke-secrets.png" alt="GKE Secret in the GCP Console"/>',Sl,ds,ma='More information on how to set Kubernetes secrets in a GKE Cluster at <a href="https://cloud.google.com/secret-manager/docs/secret-manager-managed-csi-component" rel="nofollow">Secret Manager Documentation - Use Secret Manager add-on with Google Kubernetes Engine</a>.',Dl,Js,Pl,Ts,ha='Now you can proceed to the Kubernetes deployment of the Hugging Face DLC for TGI, serving the <a href="https://hf.co/meta-llama/Llama-3.1-405B-Instruct-FP8" rel="nofollow"><code>meta-llama/Llama-3.1-405B-Instruct-FP8</code></a> model from the Hugging Face Hub.',Kl,V,Ol,bs,Ma='The Hugging Face DLC for TGI will be deployed via <code>kubectl</code>, from the configuration files in the <a href="https://github.com/huggingface/Google-Cloud-Containers/tree/merge/examples/gke/tgi-llama-405b-deployment/config/" rel="nofollow"><code>config/</code></a> directory:',se,gs,ua='<li><a href="https://github.com/huggingface/Google-Cloud-Containers/tree/merge/examples/gke/tgi-llama-405b-deployment/config/deployment.yaml" rel="nofollow"><code>deployment.yaml</code></a>: contains the deployment details of the pod including the reference to the Hugging Face DLC for TGI setting the <code>MODEL_ID</code> to <a href="https://hf.co/meta-llama/Llama-3.1-405B-Instruct-FP8" rel="nofollow"><code>meta-llama/Llama-3.1-405B-Instruct-FP8</code></a>.</li> <li><a href="https://github.com/huggingface/Google-Cloud-Containers/tree/merge/examples/gke/tgi-llama-405b-deployment/config/service.yaml" rel="nofollow"><code>service.yaml</code></a>: contains the service details of the pod, exposing the port 8080 for the TGI service.</li> <li>(optional) <a href="https://github.com/huggingface/Google-Cloud-Containers/tree/merge/examples/gke/tgi-llama-405b-deployment/config/ingress.yaml" rel="nofollow"><code>ingress.yaml</code></a>: contains the ingress details of the pod, exposing the service to the external world so that it can be accessed via the ingress IP.</li>',le,fs,ee,k,ae,ws,ya='<img src="https://raw.githubusercontent.com/huggingface/Google-Cloud-Containers/merge/examples/gke/tgi-llama-405b-deployment/imgs/gke-deployment.png" alt="GKE Deployment in the GCP Console"/>',te,Us,ja='<img src="https://raw.githubusercontent.com/huggingface/Google-Cloud-Containers/merge/examples/gke/tgi-llama-405b-deployment/imgs/gke-deployment-logs.png" alt="GKE Deployment Logs in the GCP Console"/>',ne,Is,pe,Gs,da="To run the inference over the deployed TGI service, you can either:",oe,N,$s,ml,Ja="Port-forwarding the deployed TGI service to the port 8080, so as to access via <code>localhost</code> with the command:",Fe,Cs,qe,Zs,hl,Ta="Accessing the TGI service via the external IP of the ingress, which is the default scenario here since you have defined the ingress configuration in the <code>config/ingress.yaml</code> file (but it can be skipped in favour of the port-forwarding), that can be retrieved with the following command:",Ae,vs,ie,Ws,ce,Bs,ba="To send a POST request to the TGI service using <code>cURL</code>, you can run the following command:",re,Vs,me,ks,ga="Or send a POST request to the ingress IP instead (without specifying the port as it’s not needed):",he,Ns,Me,xs,fa="Which generates the following output:",ue,Ys,ye,Xs,je,Rs,wa='To run the inference using Python, you can either use the <a href="https://github.com/huggingface/huggingface_hub" rel="nofollow"><code>huggingface_hub</code> Python SDK</a> (recommended) or the <a href="https://github.com/openai/openai-python" rel="nofollow"><code>openai</code> Python SDK</a>.',de,x,Je,Hs,Te,Es,Ua="You can install it via <code>pip</code> as <code>pip install --upgrade --quiet huggingface_hub</code>, and then run the following snippet to mimic the <code>cURL</code> commands above i.e. sending requests to the Messages API:",be,Qs,ge,Fs,Ia="Which generates the following output:",fe,qs,we,As,Ga="Alternatively, you can also format the prompt yourself and send that via the Text Generation API:",Ue,Ls,Ie,zs,$a="Which generates the following output:",Ge,_s,$e,Ss,Ce,Ds,Ca="Additionally, you can also use the Messages API via <code>openai</code>; you can install it via <code>pip</code> as <code>pip install --upgrade openai</code>, and then run:",Ze,Ps,ve,Ks,Za="Which generates the following output:",We,Os,Be,sl,Ve,ll,va="Finally, once you are done using TGI on the GKE Cluster, you can safely delete the GKE Cluster to avoid incurring in unnecessary costs.",ke,el,Ne,al,Wa="Alternatively, you can also downscale the replicas of the deployed pod to 0 in case you want to preserve the cluster, since the default GKE Cluster deployed with GKE Autopilot mode is running just a single <code>e2-small</code> instance.",xe,tl,Ye,Xe,Re,Y,He,nl,Ee,Ml,Qe;return U=new Z({props:{title:"Deploy Llama 3.1 405B with TGI DLC on GKE",local:"deploy-llama-31-405b-with-tgi-dlc-on-gke",headingTag:"h1"}}),R=new Z({props:{title:"Setup / Configuration",local:"setup--configuration",headingTag:"h2"}}),F=new b({props:{code:"ZXhwb3J0JTIwUFJPSkVDVF9JRCUzRHlvdXItcHJvamVjdC1pZCUwQWV4cG9ydCUyMExPQ0FUSU9OJTNEeW91ci1sb2NhdGlvbiUwQWV4cG9ydCUyMENMVVNURVJfTkFNRSUzRHlvdXItY2x1c3Rlci1uYW1l",highlighted:`<span class="hljs-built_in">export</span> PROJECT_ID=your-project-id
<span class="hljs-built_in">export</span> LOCATION=your-location
<span class="hljs-built_in">export</span> CLUSTER_NAME=your-cluster-name`,wrap:!1}}),A=new b({props:{code:"Z2Nsb3VkJTIwYXV0aCUyMGxvZ2luJTBBZ2Nsb3VkJTIwYXV0aCUyMGFwcGxpY2F0aW9uLWRlZmF1bHQlMjBsb2dpbiUyMCUyMCUyMyUyMEZvciUyMGxvY2FsJTIwZGV2ZWxvcG1lbnQlMEFnY2xvdWQlMjBjb25maWclMjBzZXQlMjBwcm9qZWN0JTIwJTI0UFJPSkVDVF9JRA==",highlighted:`gcloud auth login
gcloud auth application-default login  <span class="hljs-comment"># For local development</span>
gcloud config <span class="hljs-built_in">set</span> project <span class="hljs-variable">$PROJECT_ID</span>`,wrap:!1}}),z=new b({props:{code:"Z2Nsb3VkJTIwc2VydmljZXMlMjBlbmFibGUlMjBjb250YWluZXIuZ29vZ2xlYXBpcy5jb20lMEFnY2xvdWQlMjBzZXJ2aWNlcyUyMGVuYWJsZSUyMGNvbnRhaW5lcnJlZ2lzdHJ5Lmdvb2dsZWFwaXMuY29tJTBBZ2Nsb3VkJTIwc2VydmljZXMlMjBlbmFibGUlMjBjb250YWluZXJmaWxlc3lzdGVtLmdvb2dsZWFwaXMuY29t",highlighted:`gcloud services <span class="hljs-built_in">enable</span> container.googleapis.com
gcloud services <span class="hljs-built_in">enable</span> containerregistry.googleapis.com
gcloud services <span class="hljs-built_in">enable</span> containerfilesystem.googleapis.com`,wrap:!1}}),S=new b({props:{code:"Z2Nsb3VkJTIwY29tcG9uZW50cyUyMGluc3RhbGwlMjBna2UtZ2Nsb3VkLWF1dGgtcGx1Z2lu",highlighted:"gcloud components install gke-gcloud-auth-plugin",wrap:!1}}),v=new cl({props:{$$slots:{default:[Fa]},$$scope:{ctx:I}}}),O=new Z({props:{title:"Create GKE Cluster",local:"create-gke-cluster",headingTag:"h2"}}),W=new cl({props:{$$slots:{default:[qa]},$$scope:{ctx:I}}}),es=new b({props:{code:"Z2Nsb3VkJTIwY29udGFpbmVyJTIwY2x1c3RlcnMlMjBjcmVhdGUtYXV0byUyMCUyNENMVVNURVJfTkFNRSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tcHJvamVjdCUzRCUyNFBST0pFQ1RfSUQlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWxvY2F0aW9uJTNEJTI0TE9DQVRJT04lMjAlNUMlMEElMjAlMjAlMjAlMjAtLXJlbGVhc2UtY2hhbm5lbCUzRHN0YWJsZSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tY2x1c3Rlci12ZXJzaW9uJTNEMS4yOSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbm8tYXV0b3Byb3Zpc2lvbmluZy1lbmFibGUtaW5zZWN1cmUta3ViZWxldC1yZWFkb25seS1wb3J0",highlighted:`gcloud container clusters create-auto <span class="hljs-variable">$CLUSTER_NAME</span> \\
    --project=<span class="hljs-variable">$PROJECT_ID</span> \\
    --location=<span class="hljs-variable">$LOCATION</span> \\
    --release-channel=stable \\
    --cluster-version=1.29 \\
    --no-autoprovisioning-enable-insecure-kubelet-readonly-port`,wrap:!1}}),B=new cl({props:{$$slots:{default:[Aa]},$$scope:{ctx:I}}}),ns=new b({props:{code:"Z2Nsb3VkJTIwY29udGFpbmVyJTIwY2x1c3RlcnMlMjBnZXQtY3JlZGVudGlhbHMlMjAlMjRDTFVTVEVSX05BTUUlMjAtLWxvY2F0aW9uJTNEJTI0TE9DQVRJT04=",highlighted:'gcloud container clusters get-credentials <span class="hljs-variable">$CLUSTER_NAME</span> --location=<span class="hljs-variable">$LOCATION</span>',wrap:!1}}),ps=new Z({props:{title:"Get Hugging Face token and set secrets in GKE",local:"get-hugging-face-token-and-set-secrets-in-gke",headingTag:"h2"}}),cs=new b({props:{code:"cGlwJTIwaW5zdGFsbCUyMC0tdXBncmFkZSUyMC0tcXVpZXQlMjBodWdnaW5nZmFjZV9odWI=",highlighted:"pip install --upgrade --quiet huggingface_hub",wrap:!1}}),ms=new b({props:{code:"aHVnZ2luZ2ZhY2UtY2xpJTIwbG9naW4=",highlighted:"huggingface-cli login",wrap:!1}}),Ms=new b({props:{code:"a3ViZWN0bCUyMGNyZWF0ZSUyMHNlY3JldCUyMGdlbmVyaWMlMjBoZi1zZWNyZXQlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWZyb20tbGl0ZXJhbCUzRGhmX3Rva2VuJTNEJTI0KHB5dGhvbiUyMC1jJTIwJTIyZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMGdldF90b2tlbiUzQiUyMHByaW50KGdldF90b2tlbigpKSUyMiklMjAlNUMlMEElMjAlMjAlMjAlMjAtLWRyeS1ydW4lM0RjbGllbnQlMjAtbyUyMHlhbWwlMjAlN0MlMjBrdWJlY3RsJTIwYXBwbHklMjAtZiUyMC0=",highlighted:`kubectl create secret generic hf-secret \\
    --from-literal=hf_token=$(python -c <span class="hljs-string">&quot;from huggingface_hub import get_token; print(get_token())&quot;</span>) \\
    --dry-run=client -o yaml | kubectl apply -f -`,wrap:!1}}),ys=new b({props:{code:"a3ViZWN0bCUyMGNyZWF0ZSUyMHNlY3JldCUyMGdlbmVyaWMlMjBoZi1zZWNyZXQlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWZyb20tbGl0ZXJhbCUzRGhmX3Rva2VuJTNEaGZfKioqJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1kcnktcnVuJTNEY2xpZW50JTIwLW8lMjB5YW1sJTIwJTdDJTIwa3ViZWN0bCUyMGFwcGx5JTIwLWYlMjAt",highlighted:`kubectl create secret generic hf-secret \\
    --from-literal=hf_token=hf_*** \\
    --dry-run=client -o yaml | kubectl apply -f -`,wrap:!1}}),Js=new Z({props:{title:"Deploy TGI",local:"deploy-tgi",headingTag:"h2"}}),V=new cl({props:{$$slots:{default:[La]},$$scope:{ctx:I}}}),fs=new b({props:{code:"a3ViZWN0bCUyMGFwcGx5JTIwLWYlMjBjb25maWclMkY=",highlighted:"kubectl apply -f config/",wrap:!1}}),k=new cl({props:{$$slots:{default:[za]},$$scope:{ctx:I}}}),Is=new Z({props:{title:"Inference with TGI",local:"inference-with-tgi",headingTag:"h2"}}),Cs=new b({props:{code:"a3ViZWN0bCUyMHBvcnQtZm9yd2FyZCUyMHNlcnZpY2UlMkZ0Z2ktc2VydmljZSUyMDgwODAlM0E4MDgw",highlighted:"kubectl port-forward service/tgi-service 8080:8080",wrap:!1}}),vs=new b({props:{code:"a3ViZWN0bCUyMGdldCUyMGluZ3Jlc3MlMjB0Z2ktaW5ncmVzcyUyMC1vJTIwanNvbnBhdGglM0QnJTdCLnN0YXR1cy5sb2FkQmFsYW5jZXIuaW5ncmVzcyU1QjAlNUQuaXAlN0Qn",highlighted:'kubectl get ingress tgi-ingress -o jsonpath=<span class="hljs-string">&#x27;{.status.loadBalancer.ingress[0].ip}&#x27;</span>',wrap:!1}}),Ws=new Z({props:{title:"Via cURL",local:"via-curl",headingTag:"h3"}}),Vs=new b({props:{code:"Y3VybCUyMGh0dHAlM0ElMkYlMkZsb2NhbGhvc3QlM0E4MDgwJTJGdjElMkZjaGF0JTJGY29tcGxldGlvbnMlMjAlNUMlMEElMjAlMjAlMjAlMjAtWCUyMFBPU1QlMjAlNUMlMEElMjAlMjAlMjAlMjAtZCUyMCclN0IlMjJtZXNzYWdlcyUyMiUzQSU1QiU3QiUyMnJvbGUlMjIlM0ElMjJzeXN0ZW0lMjIlMkMlMjJjb250ZW50JTIyJTNBJTIwJTIyWW91JTIwYXJlJTIwYSUyMGhlbHBmdWwlMjBhc3Npc3RhbnQuJTIyJTdEJTJDJTdCJTIycm9sZSUyMiUzQSUyMnVzZXIlMjIlMkMlMjJjb250ZW50JTIyJTNBJTIyV2hhdCclNUMnJ3MlMjBEZWVwJTIwTGVhcm5pbmclM0YlMjIlN0QlNUQlMkMlMjJ0ZW1wZXJhdHVyZSUyMiUzQTAuNyUyQyUyMnRvcF9wJTIyJTNBMC45NSUyQyUyMm1heF90b2tlbnMlMjIlM0ExMjglN0QlN0QnJTIwJTVDJTBBJTIwJTIwJTIwJTIwLUglMjAnQ29udGVudC1UeXBlJTNBJTIwYXBwbGljYXRpb24lMkZqc29uJw==",highlighted:`curl http://localhost:8080/v1/chat/completions \\
    -X POST \\
    -d <span class="hljs-string">&#x27;{&quot;messages&quot;:[{&quot;role&quot;:&quot;system&quot;,&quot;content&quot;: &quot;You are a helpful assistant.&quot;},{&quot;role&quot;:&quot;user&quot;,&quot;content&quot;:&quot;What&#x27;</span>\\<span class="hljs-string">&#x27;&#x27;</span>s Deep Learning?<span class="hljs-string">&quot;}],&quot;</span>temperature<span class="hljs-string">&quot;:0.7,&quot;</span>top_p<span class="hljs-string">&quot;:0.95,&quot;</span>max_tokens<span class="hljs-string">&quot;:128}}&#x27; \\
    -H &#x27;Content-Type: application/json&#x27;</span>`,wrap:!1}}),Ns=new b({props:{code:"Y3VybCUyMGh0dHAlM0ElMkYlMkYlMjQoa3ViZWN0bCUyMGdldCUyMGluZ3Jlc3MlMjB0Z2ktaW5ncmVzcyUyMC1vJTIwanNvbnBhdGglM0QnJTdCLnN0YXR1cy5sb2FkQmFsYW5jZXIuaW5ncmVzcyU1QjAlNUQuaXAlN0QnKSUyRnYxJTJGY2hhdCUyRmNvbXBsZXRpb25zJTIwJTVDJTBBJTIwJTIwJTIwJTIwLVglMjBQT1NUJTIwJTVDJTBBJTIwJTIwJTIwJTIwLWQlMjAnJTdCJTIybWVzc2FnZXMlMjIlM0ElNUIlN0IlMjJyb2xlJTIyJTNBJTIyc3lzdGVtJTIyJTJDJTIyY29udGVudCUyMiUzQSUyMllvdSUyMGFyZSUyMGElMjBoZWxwZnVsJTIwYXNzaXN0YW50LiUyMiU3RCUyQyU3QiUyMnJvbGUlMjIlM0ElMjJ1c2VyJTIyJTJDJTIyY29udGVudCUyMiUzQSUyMldoYXQnJTVDJydzJTIwRGVlcCUyMExlYXJuaW5nJTNGJTIyJTdEJTVEJTJDJTIydGVtcGVyYXR1cmUlMjIlM0EwLjclMkMlMjJ0b3BfcCUyMiUzQTAuOTUlMkMlMjJtYXhfdG9rZW5zJTIyJTNBMTI4JTdEJTdEJyUyMCU1QyUwQSUyMCUyMCUyMCUyMC1IJTIwJ0NvbnRlbnQtVHlwZSUzQSUyMGFwcGxpY2F0aW9uJTJGanNvbic=",highlighted:`curl http://$(kubectl get ingress tgi-ingress -o jsonpath=<span class="hljs-string">&#x27;{.status.loadBalancer.ingress[0].ip}&#x27;</span>)/v1/chat/completions \\
    -X POST \\
    -d <span class="hljs-string">&#x27;{&quot;messages&quot;:[{&quot;role&quot;:&quot;system&quot;,&quot;content&quot;:&quot;You are a helpful assistant.&quot;},{&quot;role&quot;:&quot;user&quot;,&quot;content&quot;:&quot;What&#x27;</span>\\<span class="hljs-string">&#x27;&#x27;</span>s Deep Learning?<span class="hljs-string">&quot;}],&quot;</span>temperature<span class="hljs-string">&quot;:0.7,&quot;</span>top_p<span class="hljs-string">&quot;:0.95,&quot;</span>max_tokens<span class="hljs-string">&quot;:128}}&#x27; \\
    -H &#x27;Content-Type: application/json&#x27;</span>`,wrap:!1}}),Ys=new b({props:{code:"JTdCJTIyb2JqZWN0JTIyJTNBJTIyY2hhdC5jb21wbGV0aW9uJTIyJTJDJTIyaWQlMjIlM0ElMjIlMjIlMkMlMjJjcmVhdGVkJTIyJTNBMTcyNzc4MjI4NyUyQyUyMm1vZGVsJTIyJTNBJTIybWV0YS1sbGFtYSUyRkxsYW1hLTMuMS00MDVCLUluc3RydWN0LUZQOCUyMiUyQyUyMnN5c3RlbV9maW5nZXJwcmludCUyMiUzQSUyMjIuMi4wLW5hdGl2ZSUyMiUyQyUyMmNob2ljZXMlMjIlM0ElNUIlN0IlMjJpbmRleCUyMiUzQTAlMkMlMjJtZXNzYWdlJTIyJTNBJTdCJTIycm9sZSUyMiUzQSUyMmFzc2lzdGFudCUyMiUyQyUyMmNvbnRlbnQlMjIlM0ElMjJEZWVwJTIwbGVhcm5pbmclMjBpcyUyMGElMjBzdWJzZXQlMjBvZiUyMG1hY2hpbmUlMjBsZWFybmluZyUyQyUyMHdoaWNoJTIwaXMlMjBhJTIwZmllbGQlMjBvZiUyMGFydGlmaWNpYWwlMjBpbnRlbGxpZ2VuY2UlMjAoQUkpJTIwdGhhdCUyMGVuYWJsZXMlMjBjb21wdXRlcnMlMjB0byUyMGxlYXJuJTIwZnJvbSUyMGRhdGElMjB3aXRob3V0JTIwYmVpbmclMjBleHBsaWNpdGx5JTIwcHJvZ3JhbW1lZC4lMjBJdCdzJTIwYSUyMHR5cGUlMjBvZiUyMG5ldXJhbCUyMG5ldHdvcmslMjB0aGF0J3MlMjBpbnNwaXJlZCUyMGJ5JTIwdGhlJTIwc3RydWN0dXJlJTIwYW5kJTIwZnVuY3Rpb24lMjBvZiUyMHRoZSUyMGh1bWFuJTIwYnJhaW4uJTVDbiU1Q25JbiUyMHRyYWRpdGlvbmFsJTIwbWFjaGluZSUyMGxlYXJuaW5nJTJDJTIwY29tcHV0ZXJzJTIwYXJlJTIwdHJhaW5lZCUyMG9uJTIwZGF0YSUyMHVzaW5nJTIwYWxnb3JpdGhtcyUyMHRoYXQlMjBhcmUlMjBkZXNpZ25lZCUyMHRvJTIwcmVjb2duaXplJTIwcGF0dGVybnMlMjBhbmQlMjBtYWtlJTIwcHJlZGljdGlvbnMuJTIwSG93ZXZlciUyQyUyMHRoZXNlJTIwYWxnb3JpdGhtcyUyMGFyZSUyMG9mdGVuJTIwbGltaXRlZCUyMGluJTIwdGhlaXIlMjBhYmlsaXR5JTIwdG8lMjBoYW5kbGUlMjBjb21wbGV4JTIwZGF0YSUyQyUyMHN1Y2glMjBhcyUyMGltYWdlcyUyQyUyMHNwZWVjaCUyQyUyMGFuZCUyMHRleHQuJTVDbiU1Q25EZWVwJTIwbGVhcm5pbmclMkMlMjBvbiUyMHRoZSUyMG90aGVyJTIwaGFuZCUyQyUyMHVzZXMlMjBtdWx0aXBsZSUyMGxheWVycyUyMG9mJTIwYXJ0aWZpY2lhbCUyMG5ldXJhbCUyMG5ldHdvcmtzJTIwdG8lMjBhbmFseXplJTIwZGF0YS4lMjBFYWNoJTIwbGF5ZXIlMjBwcm9jZXNzZXMlMjB0aGUlMjBkYXRhJTIwaW4lMjBhJTIwZGlmZmVyZW50JTIwd2F5JTJDJTIwYWxsb3dpbmclMjB0aGUlMjIlN0QlMkMlMjJsb2dwcm9icyUyMiUzQW51bGwlMkMlMjJmaW5pc2hfcmVhc29uJTIyJTNBJTIybGVuZ3RoJTIyJTdEJTVEJTJDJTIydXNhZ2UlMjIlM0ElN0IlMjJwcm9tcHRfdG9rZW5zJTIyJTNBNDYlMkMlMjJjb21wbGV0aW9uX3Rva2VucyUyMiUzQTEyOCUyQyUyMnRvdGFsX3Rva2VucyUyMiUzQTE3NCU3RCU3RA==",highlighted:'{<span class="hljs-comment">&quot;object&quot;</span>:<span class="hljs-comment">&quot;chat.completion&quot;</span>,<span class="hljs-comment">&quot;id&quot;</span>:<span class="hljs-comment">&quot;&quot;</span>,<span class="hljs-comment">&quot;created&quot;</span>:<span class="hljs-number">1727782287</span>,<span class="hljs-comment">&quot;model&quot;</span>:<span class="hljs-comment">&quot;meta-llama/Llama-3.1-405B-Instruct-FP8&quot;</span>,<span class="hljs-comment">&quot;system_fingerprint&quot;</span>:<span class="hljs-comment">&quot;2.2.0-native&quot;</span>,<span class="hljs-comment">&quot;choices&quot;</span>:[{<span class="hljs-comment">&quot;index&quot;</span>:<span class="hljs-number">0</span>,<span class="hljs-comment">&quot;message&quot;</span>:{<span class="hljs-comment">&quot;role&quot;</span>:<span class="hljs-comment">&quot;assistant&quot;</span>,<span class="hljs-comment">&quot;content&quot;</span>:<span class="hljs-comment">&quot;Deep learning is a subset of machine learning, which is a field of artificial intelligence (AI) that enables computers to learn from data without being explicitly programmed. It&#x27;s a type of neural network that&#x27;s inspired by the structure and function of the human brain.\\n\\nIn traditional machine learning, computers are trained on data using algorithms that are designed to recognize patterns and make predictions. However, these algorithms are often limited in their ability to handle complex data, such as images, speech, and text.\\n\\nDeep learning, on the other hand, uses multiple layers of artificial neural networks to analyze data. Each layer processes the data in a different way, allowing the&quot;</span>},<span class="hljs-comment">&quot;logprobs&quot;</span>:null,<span class="hljs-comment">&quot;finish_reason&quot;</span>:<span class="hljs-comment">&quot;length&quot;</span>}],<span class="hljs-comment">&quot;usage&quot;</span>:{<span class="hljs-comment">&quot;prompt_tokens&quot;</span>:<span class="hljs-number">46</span>,<span class="hljs-comment">&quot;completion_tokens&quot;</span>:<span class="hljs-number">128</span>,<span class="hljs-comment">&quot;total_tokens&quot;</span>:<span class="hljs-number">174</span>}}',wrap:!1}}),Xs=new Z({props:{title:"Via Python",local:"via-python",headingTag:"h3"}}),x=new cl({props:{$$slots:{default:[_a]},$$scope:{ctx:I}}}),Hs=new Z({props:{title:"huggingface_hub",local:"huggingfacehub",headingTag:"h4"}}),Qs=new b({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMEluZmVyZW5jZUNsaWVudCUwQSUwQWNsaWVudCUyMCUzRCUyMEluZmVyZW5jZUNsaWVudChiYXNlX3VybCUzRCUyMmh0dHAlM0ElMkYlMkZsb2NhbGhvc3QlM0E4MDgwJTIyJTJDJTIwYXBpX2tleSUzRCUyMi0lMjIpJTBBJTBBY2hhdF9jb21wbGV0aW9uJTIwJTNEJTIwY2xpZW50LmNoYXQuY29tcGxldGlvbnMuY3JlYXRlKCUwQSUyMCUyMCUyMCUyMG1lc3NhZ2VzJTNEJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCJTIycm9sZSUyMiUzQSUyMCUyMnN5c3RlbSUyMiUyQyUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlMjJZb3UlMjBhcmUlMjBhJTIwaGVscGZ1bCUyMGFzc2lzdGFudC4lMjIlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJyb2xlJTIyJTNBJTIwJTIydXNlciUyMiUyQyUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlMjJXaGF0J3MlMjBEZWVwJTIwTGVhcm5pbmclM0YlMjIlN0QlMkMlMEElMjAlMjAlMjAlMjAlNUQlMkMlMEElMjAlMjAlMjAlMjBtYXhfdG9rZW5zJTNEMTI4JTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> InferenceClient

client = InferenceClient(base_url=<span class="hljs-string">&quot;http://localhost:8080&quot;</span>, api_key=<span class="hljs-string">&quot;-&quot;</span>)

chat_completion = client.chat.completions.create(
    messages=[
        {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;You are a helpful assistant.&quot;</span>},
        {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;What&#x27;s Deep Learning?&quot;</span>},
    ],
    max_tokens=<span class="hljs-number">128</span>,
)`,wrap:!1}}),qs=new b({props:{code:"Q2hhdENvbXBsZXRpb25PdXRwdXQoY2hvaWNlcyUzRCU1QkNoYXRDb21wbGV0aW9uT3V0cHV0Q29tcGxldGUoZmluaXNoX3JlYXNvbiUzRCdsZW5ndGgnJTJDJTIwaW5kZXglM0QwJTJDJTIwbWVzc2FnZSUzRENoYXRDb21wbGV0aW9uT3V0cHV0TWVzc2FnZShyb2xlJTNEJ2Fzc2lzdGFudCclMkMlMjBjb250ZW50JTNEJ0RlZXAlMjBsZWFybmluZyUyMGlzJTIwYSUyMHN1YnNldCUyMG9mJTIwbWFjaGluZSUyMGxlYXJuaW5nJTIwdGhhdCUyMGZvY3VzZXMlMjBvbiUyMG5ldXJhbCUyMG5ldHdvcmtzJTIwd2l0aCUyMG1hbnklMjBsYXllcnMlMkMlMjB0eXBpY2FsbHklMjBtb3JlJTIwdGhhbiUyMHR3by4lMjBUaGVzZSUyMG5ldXJhbCUyMG5ldHdvcmtzJTIwYXJlJTIwZGVzaWduZWQlMjB0byUyMG1pbWljJTIwdGhlJTIwc3RydWN0dXJlJTIwYW5kJTIwZnVuY3Rpb24lMjBvZiUyMHRoZSUyMGh1bWFuJTIwYnJhaW4lMkMlMjB3aXRoJTIwZWFjaCUyMGxheWVyJTIwcHJvY2Vzc2luZyUyMGFuZCUyMHRyYW5zZm9ybWluZyUyMGlucHV0cyUyMGluJTIwYSUyMGhpZXJhcmNoaWNhbCUyMG1hbm5lci4lNUNuJTVDbkluJTIwdHJhZGl0aW9uYWwlMjBtYWNoaW5lJTIwbGVhcm5pbmclMkMlMjBtb2RlbHMlMjBhcmUlMjB0cmFpbmVkJTIwdXNpbmclMjBhJTIwc2V0JTIwb2YlMjBwcmVkZWZpbmVkJTIwZmVhdHVyZXMlMkMlMjBzdWNoJTIwYXMlMjBlZGdlcyUyQyUyMHRleHR1cmVzJTJDJTIwb3IlMjBzaGFwZXMuJTIwSW4lMjBjb250cmFzdCUyQyUyMGRlZXAlMjBsZWFybmluZyUyMG1vZGVscyUyMGxlYXJuJTIwdG8lMjBleHRyYWN0JTIwZmVhdHVyZXMlMjBmcm9tJTIwcmF3JTIwZGF0YSUyMGF1dG9tYXRpY2FsbHklMkMlMjB3aXRob3V0JTIwdGhlJTIwbmVlZCUyMGZvciUyMG1hbnVhbCUyMGZlYXR1cmUlMjBlbmdpbmVlcmluZy4lNUNuJTVDbkRlZXAlMjBsZWFybmluZyUyMG1vZGVscyUyMGFyZSUyMHRyYWluZWQlMjB1c2luZyUyMGxhcmdlJTIwYW1vdW50cyUyMG9mJTIwZGF0YSUyMGFuZCUyMGNvbXB1dGF0aW9uYWwlMjBwb3dlciUyQyUyMHdoaWNoJTIwZW5hYmxlcyUyMHRoZW0lMjB0byUyMGxlYXJuJTIwY29tcGxleCUyMHBhdHRlcm5zJTIwYW5kJTIwcmVsYXRpb25zaGlwcyUyMGluJTIwdGhlJTIwZGF0YS4lMjBUaGVzZSUyMG1vZGVscyUyMGNhbiUyMGJlJyUyQyUyMHRvb2xfY2FsbHMlM0ROb25lKSUyQyUyMGxvZ3Byb2JzJTNETm9uZSklNUQlMkMlMjBjcmVhdGVkJTNEMTcyNzc4MjMyMiUyQyUyMGlkJTNEJyclMkMlMjBtb2RlbCUzRCdtZXRhLWxsYW1hJTJGTGxhbWEtMy4xLTQwNUItSW5zdHJ1Y3QtRlA4JyUyQyUyMHN5c3RlbV9maW5nZXJwcmludCUzRCcyLjIuMC1uYXRpdmUnJTJDJTIwdXNhZ2UlM0RDaGF0Q29tcGxldGlvbk91dHB1dFVzYWdlKGNvbXBsZXRpb25fdG9rZW5zJTNEMTI4JTJDJTIwcHJvbXB0X3Rva2VucyUzRDQ2JTJDJTIwdG90YWxfdG9rZW5zJTNEMTc0KSk=",highlighted:'<span class="hljs-constructor">ChatCompletionOutput(<span class="hljs-params">choices</span>=[ChatCompletionOutputComplete(<span class="hljs-params">finish_reason</span>=&#x27;<span class="hljs-params">length</span>&#x27;, <span class="hljs-params">index</span>=0, <span class="hljs-params">message</span>=ChatCompletionOutputMessage(<span class="hljs-params">role</span>=&#x27;<span class="hljs-params">assistant</span>&#x27;, <span class="hljs-params">content</span>=&#x27;Deep <span class="hljs-params">learning</span> <span class="hljs-params">is</span> <span class="hljs-params">a</span> <span class="hljs-params">subset</span> <span class="hljs-params">of</span> <span class="hljs-params">machine</span> <span class="hljs-params">learning</span> <span class="hljs-params">that</span> <span class="hljs-params">focuses</span> <span class="hljs-params">on</span> <span class="hljs-params">neural</span> <span class="hljs-params">networks</span> <span class="hljs-params">with</span> <span class="hljs-params">many</span> <span class="hljs-params">layers</span>, <span class="hljs-params">typically</span> <span class="hljs-params">more</span> <span class="hljs-params">than</span> <span class="hljs-params">two</span>. These <span class="hljs-params">neural</span> <span class="hljs-params">networks</span> <span class="hljs-params">are</span> <span class="hljs-params">designed</span> <span class="hljs-params">to</span> <span class="hljs-params">mimic</span> <span class="hljs-params">the</span> <span class="hljs-params">structure</span> <span class="hljs-params">and</span> <span class="hljs-params">function</span> <span class="hljs-params">of</span> <span class="hljs-params">the</span> <span class="hljs-params">human</span> <span class="hljs-params">brain</span>, <span class="hljs-params">with</span> <span class="hljs-params">each</span> <span class="hljs-params">layer</span> <span class="hljs-params">processing</span> <span class="hljs-params">and</span> <span class="hljs-params">transforming</span> <span class="hljs-params">inputs</span> <span class="hljs-params">in</span> <span class="hljs-params">a</span> <span class="hljs-params">hierarchical</span> <span class="hljs-params">manner</span>.\\<span class="hljs-params">n</span>\\<span class="hljs-params">nIn</span> <span class="hljs-params">traditional</span> <span class="hljs-params">machine</span> <span class="hljs-params">learning</span>, <span class="hljs-params">models</span> <span class="hljs-params">are</span> <span class="hljs-params">trained</span> <span class="hljs-params">using</span> <span class="hljs-params">a</span> <span class="hljs-params">set</span> <span class="hljs-params">of</span> <span class="hljs-params">predefined</span> <span class="hljs-params">features</span>, <span class="hljs-params">such</span> <span class="hljs-params">as</span> <span class="hljs-params">edges</span>, <span class="hljs-params">textures</span>, <span class="hljs-params">or</span> <span class="hljs-params">shapes</span>. In <span class="hljs-params">contrast</span>, <span class="hljs-params">deep</span> <span class="hljs-params">learning</span> <span class="hljs-params">models</span> <span class="hljs-params">learn</span> <span class="hljs-params">to</span> <span class="hljs-params">extract</span> <span class="hljs-params">features</span> <span class="hljs-params">from</span> <span class="hljs-params">raw</span> <span class="hljs-params">data</span> <span class="hljs-params">automatically</span>, <span class="hljs-params">without</span> <span class="hljs-params">the</span> <span class="hljs-params">need</span> <span class="hljs-params">for</span> <span class="hljs-params">manual</span> <span class="hljs-params">feature</span> <span class="hljs-params">engineering</span>.\\<span class="hljs-params">n</span>\\<span class="hljs-params">nDeep</span> <span class="hljs-params">learning</span> <span class="hljs-params">models</span> <span class="hljs-params">are</span> <span class="hljs-params">trained</span> <span class="hljs-params">using</span> <span class="hljs-params">large</span> <span class="hljs-params">amounts</span> <span class="hljs-params">of</span> <span class="hljs-params">data</span> <span class="hljs-params">and</span> <span class="hljs-params">computational</span> <span class="hljs-params">power</span>, <span class="hljs-params">which</span> <span class="hljs-params">enables</span> <span class="hljs-params">them</span> <span class="hljs-params">to</span> <span class="hljs-params">learn</span> <span class="hljs-params">complex</span> <span class="hljs-params">patterns</span> <span class="hljs-params">and</span> <span class="hljs-params">relationships</span> <span class="hljs-params">in</span> <span class="hljs-params">the</span> <span class="hljs-params">data</span>. These <span class="hljs-params">models</span> <span class="hljs-params">can</span> <span class="hljs-params">be</span>&#x27;, <span class="hljs-params">tool_calls</span>=None)</span>, logprobs=None)], created=<span class="hljs-number">1727782322</span>, id=&#x27;&#x27;, model=&#x27;meta-llama/Llama-<span class="hljs-number">3.1</span>-<span class="hljs-number">405</span>B-Instruct-FP8&#x27;, system_fingerprint=&#x27;<span class="hljs-number">2.2</span>.<span class="hljs-number">0</span>-native&#x27;, usage=<span class="hljs-constructor">ChatCompletionOutputUsage(<span class="hljs-params">completion_tokens</span>=128, <span class="hljs-params">prompt_tokens</span>=46, <span class="hljs-params">total_tokens</span>=174)</span>)',wrap:!1}}),Ls=new b({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMEluZmVyZW5jZUNsaWVudCUyQyUyMGdldF90b2tlbiUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBdXRvVG9rZW5pemVyJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIybWV0YS1sbGFtYSUyRkxsYW1hLTMuMS00MDVCLUluc3RydWN0LUZQOCUyMiUyQyUyMHRva2VuJTNEZ2V0X3Rva2VuKCkpJTBBY2xpZW50JTIwJTNEJTIwSW5mZXJlbmNlQ2xpZW50KCUyMmh0dHAlM0ElMkYlMkZsb2NhbGhvc3QlM0E4MDgwJTIyJTJDJTIwYXBpX2tleSUzRCUyMi0lMjIpJTBBJTBBZ2VuZXJhdGlvbiUyMCUzRCUyMGNsaWVudC50ZXh0X2dlbmVyYXRpb24oJTBBJTIwJTIwJTIwJTIwcHJvbXB0JTNEdG9rZW5pemVyLmFwcGx5X2NoYXRfdGVtcGxhdGUoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCJTIycm9sZSUyMiUzQSUyMCUyMnN5c3RlbSUyMiUyQyUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlMjJZb3UlMjBhcmUlMjBhJTIwaGVscGZ1bCUyMGFzc2lzdGFudC4lMjIlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJyb2xlJTIyJTNBJTIwJTIydXNlciUyMiUyQyUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlMjJXaGF0J3MlMjBEZWVwJTIwTGVhcm5pbmclM0YlMjIlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0b2tlbml6ZSUzREZhbHNlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYWRkX2dlbmVyYXRpb25fcHJvbXB0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMCklMkMlMEElMjAlMjAlMjAlMjBtYXhfbmV3X3Rva2VucyUzRDEyOCUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> InferenceClient, get_token
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-3.1-405B-Instruct-FP8&quot;</span>, token=get_token())
client = InferenceClient(<span class="hljs-string">&quot;http://localhost:8080&quot;</span>, api_key=<span class="hljs-string">&quot;-&quot;</span>)

generation = client.text_generation(
    prompt=tokenizer.apply_chat_template(
        [
            {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;You are a helpful assistant.&quot;</span>},
            {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;What&#x27;s Deep Learning?&quot;</span>},
        ],
        tokenize=<span class="hljs-literal">False</span>,
        add_generation_prompt=<span class="hljs-literal">True</span>,
    ),
    max_new_tokens=<span class="hljs-number">128</span>,
)`,wrap:!1}}),_s=new b({props:{code:"J0RlZXAlMjBsZWFybmluZyUyMGlzJTIwYSUyMHN1YnNldCUyMG9mJTIwbWFjaGluZSUyMGxlYXJuaW5nJTIwdGhhdCUyMGludm9sdmVzJTIwdGhlJTIwdXNlJTIwb2YlMjBhcnRpZmljaWFsJTIwbmV1cmFsJTIwbmV0d29ya3MlMjB0byUyMGFuYWx5emUlMjBhbmQlMjBpbnRlcnByZXQlMjBkYXRhLiUyMEluc3BpcmVkJTIwYnklMjB0aGUlMjBzdHJ1Y3R1cmUlMjBhbmQlMjBmdW5jdGlvbiUyMG9mJTIwdGhlJTIwaHVtYW4lMjBicmFpbiUyQyUyMGRlZXAlMjBsZWFybmluZyUyMGFsZ29yaXRobXMlMjBhcmUlMjBkZXNpZ25lZCUyMHRvJTIwbGVhcm4lMjBhbmQlMjBpbXByb3ZlJTIwb24lMjB0aGVpciUyMG93biUyMGJ5JTIwYXV0b21hdGljYWxseSUyMGFkanVzdGluZyUyMHRoZSUyMGNvbm5lY3Rpb25zJTIwYmV0d2VlbiUyMG5vZGVzJTIwb3IlMjAlMjJuZXVyb25zJTIyJTIwaW4lMjB0aGUlMjBuZXR3b3JrLiU1Q24lNUNuSW4lMjB0cmFkaXRpb25hbCUyMG1hY2hpbmUlMjBsZWFybmluZyUyQyUyMGFsZ29yaXRobXMlMjBhcmUlMjB0cmFpbmVkJTIwb24lMjBhJTIwc2V0JTIwb2YlMjBkYXRhJTIwYW5kJTIwdGhlbiUyMHVzZSUyMHRoYXQlMjB0cmFpbmluZyUyMHRvJTIwbWFrZSUyMHByZWRpY3Rpb25zJTIwb3IlMjBkZWNpc2lvbnMlMjBvbiUyMG5ldyUyQyUyMHVuc2VlbiUyMGRhdGEuJTIwSG93ZXZlciUyQyUyMHRoZXNlJTIwYWxnb3JpdGhtcyUyMG9mdGVuJTIwcmVseSUyMG9uJTIwaGFuZC1lbmdpbmVlcmVkJTIwZmVhdHVyZXMlMjBhbmQlMjBydWxlcyUyMHRvJTIwZXh0cmFjdCUyMHJlbGV2YW50JTIwaW5mb3JtYXRpb24lMjBmcm9tJTIwdGhlJTIwZGF0YS4lMjBJbiUyMGNvbnRyYXN0JTJDJTIwZGVlcCUyMGxlYXJuaW5nJTIwYWxnb3JpdGhtcyUyMGNhbiUyMGF1dG9tYXRpY2FsbHklMjBsZWFybiUyMHRvJTIwZXh0cmFjdCUyMHJlbGV2YW50JTIwZmVhdHVyZXMlMjBhbmQlMjBwYXR0ZXJucyUyMGZyb20lMjB0aGUn",highlighted:'<span class="hljs-comment">&#x27;Deep learning is a subset of machine learning that involves the use of artificial neural networks to analyze and interpret data. Inspired by the structure and function of the human brain, deep learning algorithms are designed to learn and improve on their own by automatically adjusting the connections between nodes or &quot;neurons&quot; in the network.\\n\\nIn traditional machine learning, algorithms are trained on a set of data and then use that training to make predictions or decisions on new, unseen data. However, these algorithms often rely on hand-engineered features and rules to extract relevant information from the data. In contrast, deep learning algorithms can automatically learn to extract relevant features and patterns from the&#x27;</span>',wrap:!1}}),Ss=new Z({props:{title:"openai",local:"openai",headingTag:"h4"}}),Ps=new b({props:{code:"ZnJvbSUyMG9wZW5haSUyMGltcG9ydCUyME9wZW5BSSUwQSUwQWNsaWVudCUyMCUzRCUyME9wZW5BSSglMEElMjAlMjAlMjAlMjBiYXNlX3VybCUzRCUyMmh0dHAlM0ElMkYlMkZsb2NhbGhvc3QlM0E4MDgwJTJGdjElMkYlMjIlMkMlMEElMjAlMjAlMjAlMjBhcGlfa2V5JTNEJTIyLSUyMiUyQyUwQSklMEElMEFjaGF0X2NvbXBsZXRpb24lMjAlM0QlMjBjbGllbnQuY2hhdC5jb21wbGV0aW9ucy5jcmVhdGUoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0QlMjJ0Z2klMjIlMkMlMEElMjAlMjAlMjAlMjBtZXNzYWdlcyUzRCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUyMnJvbGUlMjIlM0ElMjAlMjJzeXN0ZW0lMjIlMkMlMjAlMjJjb250ZW50JTIyJTNBJTIwJTIyWW91JTIwYXJlJTIwYSUyMGhlbHBmdWwlMjBhc3Npc3RhbnQuJTIyJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCJTIycm9sZSUyMiUzQSUyMCUyMnVzZXIlMjIlMkMlMjAlMjJjb250ZW50JTIyJTNBJTIwJTIyV2hhdCdzJTIwRGVlcCUyMExlYXJuaW5nJTNGJTIyJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTVEJTJDJTBBJTIwJTIwJTIwJTIwbWF4X3Rva2VucyUzRDEyOCUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> openai <span class="hljs-keyword">import</span> OpenAI

client = OpenAI(
    base_url=<span class="hljs-string">&quot;http://localhost:8080/v1/&quot;</span>,
    api_key=<span class="hljs-string">&quot;-&quot;</span>,
)

chat_completion = client.chat.completions.create(
    model=<span class="hljs-string">&quot;tgi&quot;</span>,
    messages=[
        {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;You are a helpful assistant.&quot;</span>},
        {<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;What&#x27;s Deep Learning?&quot;</span>},
    ],
    max_tokens=<span class="hljs-number">128</span>,
)`,wrap:!1}}),Os=new b({props:{code:"Q2hhdENvbXBsZXRpb24oaWQlM0QnJyUyQyUyMGNob2ljZXMlM0QlNUJDaG9pY2UoZmluaXNoX3JlYXNvbiUzRCdsZW5ndGgnJTJDJTIwaW5kZXglM0QwJTJDJTIwbG9ncHJvYnMlM0ROb25lJTJDJTIwbWVzc2FnZSUzRENoYXRDb21wbGV0aW9uTWVzc2FnZShjb250ZW50JTNEJ0RlZXAlMjBsZWFybmluZyUyMGlzJTIwYSUyMHN1YnNldCUyMG9mJTIwbWFjaGluZSUyMGxlYXJuaW5nJTIwdGhhdCUyMGludm9sdmVzJTIwdGhlJTIwdXNlJTIwb2YlMjBhcnRpZmljaWFsJTIwbmV1cmFsJTIwbmV0d29ya3MlMjB0byUyMGFuYWx5emUlMjBhbmQlMjBpbnRlcnByZXQlMjBkYXRhLiUyMEluc3BpcmVkJTIwYnklMjB0aGUlMjBzdHJ1Y3R1cmUlMjBhbmQlMjBmdW5jdGlvbiUyMG9mJTIwdGhlJTIwaHVtYW4lMjBicmFpbiUyQyUyMGRlZXAlMjBsZWFybmluZyUyMGFsZ29yaXRobXMlMjBhcmUlMjBkZXNpZ25lZCUyMHRvJTIwbGVhcm4lMjBhbmQlMjBpbXByb3ZlJTIwb24lMjB0aGVpciUyMG93biUyMGJ5JTIwYXV0b21hdGljYWxseSUyMGFkanVzdGluZyUyMHRoZSUyMGNvbm5lY3Rpb25zJTIwYmV0d2VlbiUyMG5vZGVzJTIwb3IlMjAlMjJuZXVyb25zJTIyJTIwaW4lMjB0aGUlMjBuZXR3b3JrLiU1Q24lNUNuSW4lMjB0cmFkaXRpb25hbCUyMG1hY2hpbmUlMjBsZWFybmluZyUyQyUyMGFsZ29yaXRobXMlMjBhcmUlMjB0cmFpbmVkJTIwdXNpbmclMjBhJTIwc2V0JTIwb2YlMjBwcmVkZWZpbmVkJTIwcnVsZXMlMjBhbmQlMjBmZWF0dXJlcy4lMjBJbiUyMGNvbnRyYXN0JTJDJTIwZGVlcCUyMGxlYXJuaW5nJTIwYWxnb3JpdGhtcyUyMGxlYXJuJTIwdG8lMjBpZGVudGlmeSUyMHBhdHRlcm5zJTIwYW5kJTIwZmVhdHVyZXMlMjBmcm9tJTIwdGhlJTIwZGF0YSUyMGl0c2VsZiUyQyUyMGVsaW1pbmF0aW5nJTIwdGhlJTIwbmVlZCUyMGZvciUyMG1hbnVhbCUyMGZlYXR1cmUlMjBlbmdpbmVlcmluZy4lMjBUaGlzJTIwYWxsb3dzJTIwZGVlcCUyMGxlYXJuaW5nJTIwbW9kZWxzJTIwdG8lMjBiZSUyMGhpZ2hseSUyMGFjY3VyYXRlJTIwYW5kJTIwZWZmaWNpZW50JTJDJTIwZXNwZWNpYWxseSUyMHdoZW4lMjBkZWFsaW5nJTIwd2l0aCUyMGxhcmdlJTIwYW5kJTIwY29tcGxleCUyMGRhdGFzZXRzLiU1Q24lNUNuS2V5JTIwY2hhcmFjdGVyaXN0aWNzJTIwb2YlMjBkZWVwJyUyQyUyMHJlZnVzYWwlM0ROb25lJTJDJTIwcm9sZSUzRCdhc3Npc3RhbnQnJTJDJTIwZnVuY3Rpb25fY2FsbCUzRE5vbmUlMkMlMjB0b29sX2NhbGxzJTNETm9uZSkpJTVEJTJDJTIwY3JlYXRlZCUzRDE3Mjc3ODI0NzglMkMlMjBtb2RlbCUzRCdtZXRhLWxsYW1hJTJGTGxhbWEtMy4xLTQwNUItSW5zdHJ1Y3QtRlA4JyUyQyUyMG9iamVjdCUzRCdjaGF0LmNvbXBsZXRpb24nJTJDJTIwc2VydmljZV90aWVyJTNETm9uZSUyQyUyMHN5c3RlbV9maW5nZXJwcmludCUzRCcyLjIuMC1uYXRpdmUnJTJDJTIwdXNhZ2UlM0RDb21wbGV0aW9uVXNhZ2UoY29tcGxldGlvbl90b2tlbnMlM0QxMjglMkMlMjBwcm9tcHRfdG9rZW5zJTNENDYlMkMlMjB0b3RhbF90b2tlbnMlM0QxNzQpKQ==",highlighted:'<span class="hljs-constructor">ChatCompletion(<span class="hljs-params">id</span>=&#x27;&#x27;, <span class="hljs-params">choices</span>=[Choice(<span class="hljs-params">finish_reason</span>=&#x27;<span class="hljs-params">length</span>&#x27;, <span class="hljs-params">index</span>=0, <span class="hljs-params">logprobs</span>=None, <span class="hljs-params">message</span>=ChatCompletionMessage(<span class="hljs-params">content</span>=&#x27;Deep <span class="hljs-params">learning</span> <span class="hljs-params">is</span> <span class="hljs-params">a</span> <span class="hljs-params">subset</span> <span class="hljs-params">of</span> <span class="hljs-params">machine</span> <span class="hljs-params">learning</span> <span class="hljs-params">that</span> <span class="hljs-params">involves</span> <span class="hljs-params">the</span> <span class="hljs-params">use</span> <span class="hljs-params">of</span> <span class="hljs-params">artificial</span> <span class="hljs-params">neural</span> <span class="hljs-params">networks</span> <span class="hljs-params">to</span> <span class="hljs-params">analyze</span> <span class="hljs-params">and</span> <span class="hljs-params">interpret</span> <span class="hljs-params">data</span>. Inspired <span class="hljs-params">by</span> <span class="hljs-params">the</span> <span class="hljs-params">structure</span> <span class="hljs-params">and</span> <span class="hljs-params">function</span> <span class="hljs-params">of</span> <span class="hljs-params">the</span> <span class="hljs-params">human</span> <span class="hljs-params">brain</span>, <span class="hljs-params">deep</span> <span class="hljs-params">learning</span> <span class="hljs-params">algorithms</span> <span class="hljs-params">are</span> <span class="hljs-params">designed</span> <span class="hljs-params">to</span> <span class="hljs-params">learn</span> <span class="hljs-params">and</span> <span class="hljs-params">improve</span> <span class="hljs-params">on</span> <span class="hljs-params">their</span> <span class="hljs-params">own</span> <span class="hljs-params">by</span> <span class="hljs-params">automatically</span> <span class="hljs-params">adjusting</span> <span class="hljs-params">the</span> <span class="hljs-params">connections</span> <span class="hljs-params">between</span> <span class="hljs-params">nodes</span> <span class="hljs-params">or</span> <span class="hljs-string">&quot;neurons&quot;</span> <span class="hljs-params">in</span> <span class="hljs-params">the</span> <span class="hljs-params">network</span>.\\<span class="hljs-params">n</span>\\<span class="hljs-params">nIn</span> <span class="hljs-params">traditional</span> <span class="hljs-params">machine</span> <span class="hljs-params">learning</span>, <span class="hljs-params">algorithms</span> <span class="hljs-params">are</span> <span class="hljs-params">trained</span> <span class="hljs-params">using</span> <span class="hljs-params">a</span> <span class="hljs-params">set</span> <span class="hljs-params">of</span> <span class="hljs-params">predefined</span> <span class="hljs-params">rules</span> <span class="hljs-params">and</span> <span class="hljs-params">features</span>. In <span class="hljs-params">contrast</span>, <span class="hljs-params">deep</span> <span class="hljs-params">learning</span> <span class="hljs-params">algorithms</span> <span class="hljs-params">learn</span> <span class="hljs-params">to</span> <span class="hljs-params">identify</span> <span class="hljs-params">patterns</span> <span class="hljs-params">and</span> <span class="hljs-params">features</span> <span class="hljs-params">from</span> <span class="hljs-params">the</span> <span class="hljs-params">data</span> <span class="hljs-params">itself</span>, <span class="hljs-params">eliminating</span> <span class="hljs-params">the</span> <span class="hljs-params">need</span> <span class="hljs-params">for</span> <span class="hljs-params">manual</span> <span class="hljs-params">feature</span> <span class="hljs-params">engineering</span>. This <span class="hljs-params">allows</span> <span class="hljs-params">deep</span> <span class="hljs-params">learning</span> <span class="hljs-params">models</span> <span class="hljs-params">to</span> <span class="hljs-params">be</span> <span class="hljs-params">highly</span> <span class="hljs-params">accurate</span> <span class="hljs-params">and</span> <span class="hljs-params">efficient</span>, <span class="hljs-params">especially</span> <span class="hljs-params">when</span> <span class="hljs-params">dealing</span> <span class="hljs-params">with</span> <span class="hljs-params">large</span> <span class="hljs-params">and</span> <span class="hljs-params">complex</span> <span class="hljs-params">datasets</span>.\\<span class="hljs-params">n</span>\\<span class="hljs-params">nKey</span> <span class="hljs-params">characteristics</span> <span class="hljs-params">of</span> <span class="hljs-params">deep</span>&#x27;, <span class="hljs-params">refusal</span>=None, <span class="hljs-params">role</span>=&#x27;<span class="hljs-params">assistant</span>&#x27;, <span class="hljs-params">function_call</span>=None, <span class="hljs-params">tool_calls</span>=None)</span>)], created=<span class="hljs-number">1727782478</span>, model=&#x27;meta-llama/Llama-<span class="hljs-number">3.1</span>-<span class="hljs-number">405</span>B-Instruct-FP8&#x27;, <span class="hljs-keyword">object</span>=&#x27;chat.completion&#x27;, service_tier=None, system_fingerprint=&#x27;<span class="hljs-number">2.2</span>.<span class="hljs-number">0</span>-native&#x27;, usage=<span class="hljs-constructor">CompletionUsage(<span class="hljs-params">completion_tokens</span>=128, <span class="hljs-params">prompt_tokens</span>=46, <span class="hljs-params">total_tokens</span>=174)</span>)',wrap:!1}}),sl=new Z({props:{title:"Delete GKE Cluster",local:"delete-gke-cluster",headingTag:"h2"}}),el=new b({props:{code:"Z2Nsb3VkJTIwY29udGFpbmVyJTIwY2x1c3RlcnMlMjBkZWxldGUlMjAlMjRDTFVTVEVSX05BTUUlMjAtLWxvY2F0aW9uJTNEJTI0TE9DQVRJT04=",highlighted:'gcloud container clusters delete <span class="hljs-variable">$CLUSTER_NAME</span> --location=<span class="hljs-variable">$LOCATION</span>',wrap:!1}}),tl=new b({props:{code:"a3ViZWN0bCUyMHNjYWxlJTIwLS1yZXBsaWNhcyUzRDAlMjBkZXBsb3ltZW50JTJGdGdpLWRlcGxveW1lbnQ=",highlighted:"kubectl scale --replicas=0 deployment/tgi-deployment",wrap:!1}}),Y=new cl({props:{$$slots:{default:[Sa]},$$scope:{ctx:I}}}),nl=new Qa({props:{source:"https://github.com/huggingface/Google-Cloud-Containers/blob/main/docs/source/examples/gke-tgi-llama-405b-deployment.mdx"}}),{c(){p=o("meta"),g=t(),j=o("p"),J=t(),r(U.$$.fragment),f=t(),G=o("p"),G.innerHTML=$,d=t(),w=o("p"),w.innerHTML=T,C=t(),r(R.$$.fragment),yl=t(),H=o("p"),H.innerHTML=Le,jl=t(),E=o("ul"),E.innerHTML=ze,dl=t(),Q=o("p"),Q.textContent=_e,Jl=t(),r(F.$$.fragment),Tl=t(),q=o("p"),q.textContent=Se,bl=t(),r(A.$$.fragment),gl=t(),L=o("p"),L.textContent=De,fl=t(),r(z.$$.fragment),wl=t(),_=o("p"),_.innerHTML=Pe,Ul=t(),r(S.$$.fragment),Il=t(),r(v.$$.fragment),Gl=t(),D=o("p"),D.innerHTML=Ke,$l=t(),P=o("ul"),P.innerHTML=Oe,Cl=t(),K=o("p"),K.innerHTML=sa,Zl=t(),r(O.$$.fragment),vl=t(),ss=o("p"),ss.textContent=la,Wl=t(),ls=o("p"),ls.textContent=ea,Bl=t(),r(W.$$.fragment),Vl=t(),r(es.$$.fragment),kl=t(),r(B.$$.fragment),Nl=t(),as=o("p"),as.innerHTML=aa,xl=t(),ts=o("p"),ts.innerHTML=ta,Yl=t(),r(ns.$$.fragment),Xl=t(),r(ps.$$.fragment),Rl=t(),os=o("p"),os.innerHTML=na,Hl=t(),is=o("p"),is.innerHTML=pa,El=t(),r(cs.$$.fragment),Ql=t(),rs=o("p"),rs.textContent=oa,Fl=t(),r(ms.$$.fragment),ql=t(),hs=o("p"),hs.innerHTML=ia,Al=t(),r(Ms.$$.fragment),Ll=t(),us=o("p"),us.textContent=ca,zl=t(),r(ys.$$.fragment),_l=t(),js=o("p"),js.innerHTML=ra,Sl=t(),ds=o("p"),ds.innerHTML=ma,Dl=t(),r(Js.$$.fragment),Pl=t(),Ts=o("p"),Ts.innerHTML=ha,Kl=t(),r(V.$$.fragment),Ol=t(),bs=o("p"),bs.innerHTML=Ma,se=t(),gs=o("ul"),gs.innerHTML=ua,le=t(),r(fs.$$.fragment),ee=t(),r(k.$$.fragment),ae=t(),ws=o("p"),ws.innerHTML=ya,te=t(),Us=o("p"),Us.innerHTML=ja,ne=t(),r(Is.$$.fragment),pe=t(),Gs=o("p"),Gs.textContent=da,oe=t(),N=o("ul"),$s=o("li"),ml=o("p"),ml.innerHTML=Ja,Fe=t(),r(Cs.$$.fragment),qe=t(),Zs=o("li"),hl=o("p"),hl.innerHTML=Ta,Ae=t(),r(vs.$$.fragment),ie=t(),r(Ws.$$.fragment),ce=t(),Bs=o("p"),Bs.innerHTML=ba,re=t(),r(Vs.$$.fragment),me=t(),ks=o("p"),ks.textContent=ga,he=t(),r(Ns.$$.fragment),Me=t(),xs=o("p"),xs.textContent=fa,ue=t(),r(Ys.$$.fragment),ye=t(),r(Xs.$$.fragment),je=t(),Rs=o("p"),Rs.innerHTML=wa,de=t(),r(x.$$.fragment),Je=t(),r(Hs.$$.fragment),Te=t(),Es=o("p"),Es.innerHTML=Ua,be=t(),r(Qs.$$.fragment),ge=t(),Fs=o("p"),Fs.textContent=Ia,fe=t(),r(qs.$$.fragment),we=t(),As=o("p"),As.textContent=Ga,Ue=t(),r(Ls.$$.fragment),Ie=t(),zs=o("p"),zs.textContent=$a,Ge=t(),r(_s.$$.fragment),$e=t(),r(Ss.$$.fragment),Ce=t(),Ds=o("p"),Ds.innerHTML=Ca,Ze=t(),r(Ps.$$.fragment),ve=t(),Ks=o("p"),Ks.textContent=Za,We=t(),r(Os.$$.fragment),Be=t(),r(sl.$$.fragment),Ve=t(),ll=o("p"),ll.textContent=va,ke=t(),r(el.$$.fragment),Ne=t(),al=o("p"),al.innerHTML=Wa,xe=t(),r(tl.$$.fragment),Ye=t(),Xe=o("hr"),Re=t(),r(Y.$$.fragment),He=t(),r(nl.$$.fragment),Ee=t(),Ml=o("p"),this.h()},l(s){const l=Ea("svelte-u9bgzb",document.head);p=i(l,"META",{name:!0,content:!0}),l.forEach(e),g=n(s),j=i(s,"P",{}),ul(j).forEach(e),J=n(s),m(U.$$.fragment,s),f=n(s),G=i(s,"P",{"data-svelte-h":!0}),c(G)!=="svelte-62pjph"&&(G.innerHTML=$),d=n(s),w=i(s,"P",{"data-svelte-h":!0}),c(w)!=="svelte-oqlec"&&(w.innerHTML=T),C=n(s),m(R.$$.fragment,s),yl=n(s),H=i(s,"P",{"data-svelte-h":!0}),c(H)!=="svelte-1qnmo38"&&(H.innerHTML=Le),jl=n(s),E=i(s,"UL",{"data-svelte-h":!0}),c(E)!=="svelte-a8ymuw"&&(E.innerHTML=ze),dl=n(s),Q=i(s,"P",{"data-svelte-h":!0}),c(Q)!=="svelte-1i8n6rq"&&(Q.textContent=_e),Jl=n(s),m(F.$$.fragment,s),Tl=n(s),q=i(s,"P",{"data-svelte-h":!0}),c(q)!=="svelte-1g39t6p"&&(q.textContent=Se),bl=n(s),m(A.$$.fragment,s),gl=n(s),L=i(s,"P",{"data-svelte-h":!0}),c(L)!=="svelte-1c0v0sp"&&(L.textContent=De),fl=n(s),m(z.$$.fragment,s),wl=n(s),_=i(s,"P",{"data-svelte-h":!0}),c(_)!=="svelte-wjaltb"&&(_.innerHTML=Pe),Ul=n(s),m(S.$$.fragment,s),Il=n(s),m(v.$$.fragment,s),Gl=n(s),D=i(s,"P",{"data-svelte-h":!0}),c(D)!=="svelte-48mtlq"&&(D.innerHTML=Ke),$l=n(s),P=i(s,"UL",{"data-svelte-h":!0}),c(P)!=="svelte-1l5qqkz"&&(P.innerHTML=Oe),Cl=n(s),K=i(s,"P",{"data-svelte-h":!0}),c(K)!=="svelte-bugwnv"&&(K.innerHTML=sa),Zl=n(s),m(O.$$.fragment,s),vl=n(s),ss=i(s,"P",{"data-svelte-h":!0}),c(ss)!=="svelte-lja57p"&&(ss.textContent=la),Wl=n(s),ls=i(s,"P",{"data-svelte-h":!0}),c(ls)!=="svelte-l7bdz6"&&(ls.textContent=ea),Bl=n(s),m(W.$$.fragment,s),Vl=n(s),m(es.$$.fragment,s),kl=n(s),m(B.$$.fragment,s),Nl=n(s),as=i(s,"P",{"data-svelte-h":!0}),c(as)!=="svelte-1l8tl94"&&(as.innerHTML=aa),xl=n(s),ts=i(s,"P",{"data-svelte-h":!0}),c(ts)!=="svelte-f29e6h"&&(ts.innerHTML=ta),Yl=n(s),m(ns.$$.fragment,s),Xl=n(s),m(ps.$$.fragment,s),Rl=n(s),os=i(s,"P",{"data-svelte-h":!0}),c(os)!=="svelte-g7q3hq"&&(os.innerHTML=na),Hl=n(s),is=i(s,"P",{"data-svelte-h":!0}),c(is)!=="svelte-1n24rkf"&&(is.innerHTML=pa),El=n(s),m(cs.$$.fragment,s),Ql=n(s),rs=i(s,"P",{"data-svelte-h":!0}),c(rs)!=="svelte-4hmhsl"&&(rs.textContent=oa),Fl=n(s),m(ms.$$.fragment,s),ql=n(s),hs=i(s,"P",{"data-svelte-h":!0}),c(hs)!=="svelte-cqv9s0"&&(hs.innerHTML=ia),Al=n(s),m(Ms.$$.fragment,s),Ll=n(s),us=i(s,"P",{"data-svelte-h":!0}),c(us)!=="svelte-o99xdc"&&(us.textContent=ca),zl=n(s),m(ys.$$.fragment,s),_l=n(s),js=i(s,"P",{"data-svelte-h":!0}),c(js)!=="svelte-rkz31z"&&(js.innerHTML=ra),Sl=n(s),ds=i(s,"P",{"data-svelte-h":!0}),c(ds)!=="svelte-rgl2wl"&&(ds.innerHTML=ma),Dl=n(s),m(Js.$$.fragment,s),Pl=n(s),Ts=i(s,"P",{"data-svelte-h":!0}),c(Ts)!=="svelte-j1yl34"&&(Ts.innerHTML=ha),Kl=n(s),m(V.$$.fragment,s),Ol=n(s),bs=i(s,"P",{"data-svelte-h":!0}),c(bs)!=="svelte-1k3dim7"&&(bs.innerHTML=Ma),se=n(s),gs=i(s,"UL",{"data-svelte-h":!0}),c(gs)!=="svelte-1tetsww"&&(gs.innerHTML=ua),le=n(s),m(fs.$$.fragment,s),ee=n(s),m(k.$$.fragment,s),ae=n(s),ws=i(s,"P",{"data-svelte-h":!0}),c(ws)!=="svelte-10xjz8m"&&(ws.innerHTML=ya),te=n(s),Us=i(s,"P",{"data-svelte-h":!0}),c(Us)!=="svelte-hbacpz"&&(Us.innerHTML=ja),ne=n(s),m(Is.$$.fragment,s),pe=n(s),Gs=i(s,"P",{"data-svelte-h":!0}),c(Gs)!=="svelte-1wzt8py"&&(Gs.textContent=da),oe=n(s),N=i(s,"UL",{});var pl=ul(N);$s=i(pl,"LI",{});var ol=ul($s);ml=i(ol,"P",{"data-svelte-h":!0}),c(ml)!=="svelte-k3movz"&&(ml.innerHTML=Ja),Fe=n(ol),m(Cs.$$.fragment,ol),ol.forEach(e),qe=n(pl),Zs=i(pl,"LI",{});var il=ul(Zs);hl=i(il,"P",{"data-svelte-h":!0}),c(hl)!=="svelte-1uskw9x"&&(hl.innerHTML=Ta),Ae=n(il),m(vs.$$.fragment,il),il.forEach(e),pl.forEach(e),ie=n(s),m(Ws.$$.fragment,s),ce=n(s),Bs=i(s,"P",{"data-svelte-h":!0}),c(Bs)!=="svelte-1ks8djk"&&(Bs.innerHTML=ba),re=n(s),m(Vs.$$.fragment,s),me=n(s),ks=i(s,"P",{"data-svelte-h":!0}),c(ks)!=="svelte-ju7jt8"&&(ks.textContent=ga),he=n(s),m(Ns.$$.fragment,s),Me=n(s),xs=i(s,"P",{"data-svelte-h":!0}),c(xs)!=="svelte-3c8ndk"&&(xs.textContent=fa),ue=n(s),m(Ys.$$.fragment,s),ye=n(s),m(Xs.$$.fragment,s),je=n(s),Rs=i(s,"P",{"data-svelte-h":!0}),c(Rs)!=="svelte-1p0ak29"&&(Rs.innerHTML=wa),de=n(s),m(x.$$.fragment,s),Je=n(s),m(Hs.$$.fragment,s),Te=n(s),Es=i(s,"P",{"data-svelte-h":!0}),c(Es)!=="svelte-twqp2d"&&(Es.innerHTML=Ua),be=n(s),m(Qs.$$.fragment,s),ge=n(s),Fs=i(s,"P",{"data-svelte-h":!0}),c(Fs)!=="svelte-3c8ndk"&&(Fs.textContent=Ia),fe=n(s),m(qs.$$.fragment,s),we=n(s),As=i(s,"P",{"data-svelte-h":!0}),c(As)!=="svelte-2ug5mx"&&(As.textContent=Ga),Ue=n(s),m(Ls.$$.fragment,s),Ie=n(s),zs=i(s,"P",{"data-svelte-h":!0}),c(zs)!=="svelte-3c8ndk"&&(zs.textContent=$a),Ge=n(s),m(_s.$$.fragment,s),$e=n(s),m(Ss.$$.fragment,s),Ce=n(s),Ds=i(s,"P",{"data-svelte-h":!0}),c(Ds)!=="svelte-1bza1up"&&(Ds.innerHTML=Ca),Ze=n(s),m(Ps.$$.fragment,s),ve=n(s),Ks=i(s,"P",{"data-svelte-h":!0}),c(Ks)!=="svelte-3c8ndk"&&(Ks.textContent=Za),We=n(s),m(Os.$$.fragment,s),Be=n(s),m(sl.$$.fragment,s),Ve=n(s),ll=i(s,"P",{"data-svelte-h":!0}),c(ll)!=="svelte-1laf0m8"&&(ll.textContent=va),ke=n(s),m(el.$$.fragment,s),Ne=n(s),al=i(s,"P",{"data-svelte-h":!0}),c(al)!=="svelte-qo8r7n"&&(al.innerHTML=Wa),xe=n(s),m(tl.$$.fragment,s),Ye=n(s),Xe=i(s,"HR",{}),Re=n(s),m(Y.$$.fragment,s),He=n(s),m(nl.$$.fragment,s),Ee=n(s),Ml=i(s,"P",{}),ul(Ml).forEach(e),this.h()},h(){xa(p,"name","hf:doc:metadata"),xa(p,"content",Pa)},m(s,l){X(document.head,p),a(s,g,l),a(s,j,l),a(s,J,l),h(U,s,l),a(s,f,l),a(s,G,l),a(s,d,l),a(s,w,l),a(s,C,l),h(R,s,l),a(s,yl,l),a(s,H,l),a(s,jl,l),a(s,E,l),a(s,dl,l),a(s,Q,l),a(s,Jl,l),h(F,s,l),a(s,Tl,l),a(s,q,l),a(s,bl,l),h(A,s,l),a(s,gl,l),a(s,L,l),a(s,fl,l),h(z,s,l),a(s,wl,l),a(s,_,l),a(s,Ul,l),h(S,s,l),a(s,Il,l),h(v,s,l),a(s,Gl,l),a(s,D,l),a(s,$l,l),a(s,P,l),a(s,Cl,l),a(s,K,l),a(s,Zl,l),h(O,s,l),a(s,vl,l),a(s,ss,l),a(s,Wl,l),a(s,ls,l),a(s,Bl,l),h(W,s,l),a(s,Vl,l),h(es,s,l),a(s,kl,l),h(B,s,l),a(s,Nl,l),a(s,as,l),a(s,xl,l),a(s,ts,l),a(s,Yl,l),h(ns,s,l),a(s,Xl,l),h(ps,s,l),a(s,Rl,l),a(s,os,l),a(s,Hl,l),a(s,is,l),a(s,El,l),h(cs,s,l),a(s,Ql,l),a(s,rs,l),a(s,Fl,l),h(ms,s,l),a(s,ql,l),a(s,hs,l),a(s,Al,l),h(Ms,s,l),a(s,Ll,l),a(s,us,l),a(s,zl,l),h(ys,s,l),a(s,_l,l),a(s,js,l),a(s,Sl,l),a(s,ds,l),a(s,Dl,l),h(Js,s,l),a(s,Pl,l),a(s,Ts,l),a(s,Kl,l),h(V,s,l),a(s,Ol,l),a(s,bs,l),a(s,se,l),a(s,gs,l),a(s,le,l),h(fs,s,l),a(s,ee,l),h(k,s,l),a(s,ae,l),a(s,ws,l),a(s,te,l),a(s,Us,l),a(s,ne,l),h(Is,s,l),a(s,pe,l),a(s,Gs,l),a(s,oe,l),a(s,N,l),X(N,$s),X($s,ml),X($s,Fe),h(Cs,$s,null),X(N,qe),X(N,Zs),X(Zs,hl),X(Zs,Ae),h(vs,Zs,null),a(s,ie,l),h(Ws,s,l),a(s,ce,l),a(s,Bs,l),a(s,re,l),h(Vs,s,l),a(s,me,l),a(s,ks,l),a(s,he,l),h(Ns,s,l),a(s,Me,l),a(s,xs,l),a(s,ue,l),h(Ys,s,l),a(s,ye,l),h(Xs,s,l),a(s,je,l),a(s,Rs,l),a(s,de,l),h(x,s,l),a(s,Je,l),h(Hs,s,l),a(s,Te,l),a(s,Es,l),a(s,be,l),h(Qs,s,l),a(s,ge,l),a(s,Fs,l),a(s,fe,l),h(qs,s,l),a(s,we,l),a(s,As,l),a(s,Ue,l),h(Ls,s,l),a(s,Ie,l),a(s,zs,l),a(s,Ge,l),h(_s,s,l),a(s,$e,l),h(Ss,s,l),a(s,Ce,l),a(s,Ds,l),a(s,Ze,l),h(Ps,s,l),a(s,ve,l),a(s,Ks,l),a(s,We,l),h(Os,s,l),a(s,Be,l),h(sl,s,l),a(s,Ve,l),a(s,ll,l),a(s,ke,l),h(el,s,l),a(s,Ne,l),a(s,al,l),a(s,xe,l),h(tl,s,l),a(s,Ye,l),a(s,Xe,l),a(s,Re,l),h(Y,s,l),a(s,He,l),h(nl,s,l),a(s,Ee,l),a(s,Ml,l),Qe=!0},p(s,[l]){const pl={};l&2&&(pl.$$scope={dirty:l,ctx:s}),v.$set(pl);const ol={};l&2&&(ol.$$scope={dirty:l,ctx:s}),W.$set(ol);const il={};l&2&&(il.$$scope={dirty:l,ctx:s}),B.$set(il);const Ba={};l&2&&(Ba.$$scope={dirty:l,ctx:s}),V.$set(Ba);const Va={};l&2&&(Va.$$scope={dirty:l,ctx:s}),k.$set(Va);const ka={};l&2&&(ka.$$scope={dirty:l,ctx:s}),x.$set(ka);const Na={};l&2&&(Na.$$scope={dirty:l,ctx:s}),Y.$set(Na)},i(s){Qe||(M(U.$$.fragment,s),M(R.$$.fragment,s),M(F.$$.fragment,s),M(A.$$.fragment,s),M(z.$$.fragment,s),M(S.$$.fragment,s),M(v.$$.fragment,s),M(O.$$.fragment,s),M(W.$$.fragment,s),M(es.$$.fragment,s),M(B.$$.fragment,s),M(ns.$$.fragment,s),M(ps.$$.fragment,s),M(cs.$$.fragment,s),M(ms.$$.fragment,s),M(Ms.$$.fragment,s),M(ys.$$.fragment,s),M(Js.$$.fragment,s),M(V.$$.fragment,s),M(fs.$$.fragment,s),M(k.$$.fragment,s),M(Is.$$.fragment,s),M(Cs.$$.fragment,s),M(vs.$$.fragment,s),M(Ws.$$.fragment,s),M(Vs.$$.fragment,s),M(Ns.$$.fragment,s),M(Ys.$$.fragment,s),M(Xs.$$.fragment,s),M(x.$$.fragment,s),M(Hs.$$.fragment,s),M(Qs.$$.fragment,s),M(qs.$$.fragment,s),M(Ls.$$.fragment,s),M(_s.$$.fragment,s),M(Ss.$$.fragment,s),M(Ps.$$.fragment,s),M(Os.$$.fragment,s),M(sl.$$.fragment,s),M(el.$$.fragment,s),M(tl.$$.fragment,s),M(Y.$$.fragment,s),M(nl.$$.fragment,s),Qe=!0)},o(s){u(U.$$.fragment,s),u(R.$$.fragment,s),u(F.$$.fragment,s),u(A.$$.fragment,s),u(z.$$.fragment,s),u(S.$$.fragment,s),u(v.$$.fragment,s),u(O.$$.fragment,s),u(W.$$.fragment,s),u(es.$$.fragment,s),u(B.$$.fragment,s),u(ns.$$.fragment,s),u(ps.$$.fragment,s),u(cs.$$.fragment,s),u(ms.$$.fragment,s),u(Ms.$$.fragment,s),u(ys.$$.fragment,s),u(Js.$$.fragment,s),u(V.$$.fragment,s),u(fs.$$.fragment,s),u(k.$$.fragment,s),u(Is.$$.fragment,s),u(Cs.$$.fragment,s),u(vs.$$.fragment,s),u(Ws.$$.fragment,s),u(Vs.$$.fragment,s),u(Ns.$$.fragment,s),u(Ys.$$.fragment,s),u(Xs.$$.fragment,s),u(x.$$.fragment,s),u(Hs.$$.fragment,s),u(Qs.$$.fragment,s),u(qs.$$.fragment,s),u(Ls.$$.fragment,s),u(_s.$$.fragment,s),u(Ss.$$.fragment,s),u(Ps.$$.fragment,s),u(Os.$$.fragment,s),u(sl.$$.fragment,s),u(el.$$.fragment,s),u(tl.$$.fragment,s),u(Y.$$.fragment,s),u(nl.$$.fragment,s),Qe=!1},d(s){s&&(e(g),e(j),e(J),e(f),e(G),e(d),e(w),e(C),e(yl),e(H),e(jl),e(E),e(dl),e(Q),e(Jl),e(Tl),e(q),e(bl),e(gl),e(L),e(fl),e(wl),e(_),e(Ul),e(Il),e(Gl),e(D),e($l),e(P),e(Cl),e(K),e(Zl),e(vl),e(ss),e(Wl),e(ls),e(Bl),e(Vl),e(kl),e(Nl),e(as),e(xl),e(ts),e(Yl),e(Xl),e(Rl),e(os),e(Hl),e(is),e(El),e(Ql),e(rs),e(Fl),e(ql),e(hs),e(Al),e(Ll),e(us),e(zl),e(_l),e(js),e(Sl),e(ds),e(Dl),e(Pl),e(Ts),e(Kl),e(Ol),e(bs),e(se),e(gs),e(le),e(ee),e(ae),e(ws),e(te),e(Us),e(ne),e(pe),e(Gs),e(oe),e(N),e(ie),e(ce),e(Bs),e(re),e(me),e(ks),e(he),e(Me),e(xs),e(ue),e(ye),e(je),e(Rs),e(de),e(Je),e(Te),e(Es),e(be),e(ge),e(Fs),e(fe),e(we),e(As),e(Ue),e(Ie),e(zs),e(Ge),e($e),e(Ce),e(Ds),e(Ze),e(ve),e(Ks),e(We),e(Be),e(Ve),e(ll),e(ke),e(Ne),e(al),e(xe),e(Ye),e(Xe),e(Re),e(He),e(Ee),e(Ml)),e(p),y(U,s),y(R,s),y(F,s),y(A,s),y(z,s),y(S,s),y(v,s),y(O,s),y(W,s),y(es,s),y(B,s),y(ns,s),y(ps,s),y(cs,s),y(ms,s),y(Ms,s),y(ys,s),y(Js,s),y(V,s),y(fs,s),y(k,s),y(Is,s),y(Cs),y(vs),y(Ws,s),y(Vs,s),y(Ns,s),y(Ys,s),y(Xs,s),y(x,s),y(Hs,s),y(Qs,s),y(qs,s),y(Ls,s),y(_s,s),y(Ss,s),y(Ps,s),y(Os,s),y(sl,s),y(el,s),y(tl,s),y(Y,s),y(nl,s)}}}const Pa='{"title":"Deploy Llama 3.1 405B with TGI DLC on GKE","local":"deploy-llama-31-405b-with-tgi-dlc-on-gke","sections":[{"title":"Setup / Configuration","local":"setup--configuration","sections":[],"depth":2},{"title":"Create GKE Cluster","local":"create-gke-cluster","sections":[],"depth":2},{"title":"Get Hugging Face token and set secrets in GKE","local":"get-hugging-face-token-and-set-secrets-in-gke","sections":[],"depth":2},{"title":"Deploy TGI","local":"deploy-tgi","sections":[],"depth":2},{"title":"Inference with TGI","local":"inference-with-tgi","sections":[{"title":"Via cURL","local":"via-curl","sections":[],"depth":3},{"title":"Via Python","local":"via-python","sections":[{"title":"huggingface_hub","local":"huggingfacehub","sections":[],"depth":4},{"title":"openai","local":"openai","sections":[],"depth":4}],"depth":3}],"depth":2},{"title":"Delete GKE Cluster","local":"delete-gke-cluster","sections":[],"depth":2}],"depth":1}';function Ka(I){return Xa(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class tt extends Ra{constructor(p){super(),Ha(this,p,Ka,Da,Ya,{})}}export{tt as component};