Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / hub /main /en /models-inference.html

rtrm

about 1 month ago

download

raw

16 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Serverless Inference API","local":"serverless-inference-api","sections":[{"title":"What technology do you use to power the Serverless Inference API?","local":"what-technology-do-you-use-to-power-the-serverless-inference-api","sections":[],"depth":2},{"title":"How can I turn off the Serverless Inference API for my model?","local":"how-can-i-turn-off-the-serverless-inference-api-for-my-model","sections":[],"depth":2},{"title":"Why don’t I see an inference widget, or why can’t I use the API?","local":"why-dont-i-see-an-inference-widget-or-why-cant-i-use-the-api","sections":[],"depth":2},{"title":"Can I send large volumes of requests? Can I get accelerated APIs?","local":"can-i-send-large-volumes-of-requests-can-i-get-accelerated-apis","sections":[],"depth":2},{"title":"How can I see my usage?","local":"how-can-i-see-my-usage","sections":[],"depth":2},{"title":"Is there programmatic access to the Serverless Inference API?","local":"is-there-programmatic-access-to-the-serverless-inference-api","sections":[],"depth":2}],"depth":1}">
	<link href="/docs/hub/main/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/hub/main/en/_app/immutable/entry/start.d0cd5065.js">
	<link rel="modulepreload" href="/docs/hub/main/en/_app/immutable/chunks/scheduler.d6170356.js">
	<link rel="modulepreload" href="/docs/hub/main/en/_app/immutable/chunks/singletons.d032f1eb.js">
	<link rel="modulepreload" href="/docs/hub/main/en/_app/immutable/chunks/paths.752f1c6b.js">
	<link rel="modulepreload" href="/docs/hub/main/en/_app/immutable/entry/app.b6abe3c1.js">
	<link rel="modulepreload" href="/docs/hub/main/en/_app/immutable/chunks/index.fcd4cc08.js">
	<link rel="modulepreload" href="/docs/hub/main/en/_app/immutable/nodes/0.f045427f.js">
	<link rel="modulepreload" href="/docs/hub/main/en/_app/immutable/nodes/79.bec2ddc2.js">
	<link rel="modulepreload" href="/docs/hub/main/en/_app/immutable/chunks/EditOnGithub.da2b595c.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Serverless Inference API","local":"serverless-inference-api","sections":[{"title":"What technology do you use to power the Serverless Inference API?","local":"what-technology-do-you-use-to-power-the-serverless-inference-api","sections":[],"depth":2},{"title":"How can I turn off the Serverless Inference API for my model?","local":"how-can-i-turn-off-the-serverless-inference-api-for-my-model","sections":[],"depth":2},{"title":"Why don’t I see an inference widget, or why can’t I use the API?","local":"why-dont-i-see-an-inference-widget-or-why-cant-i-use-the-api","sections":[],"depth":2},{"title":"Can I send large volumes of requests? Can I get accelerated APIs?","local":"can-i-send-large-volumes-of-requests-can-i-get-accelerated-apis","sections":[],"depth":2},{"title":"How can I see my usage?","local":"how-can-i-see-my-usage","sections":[],"depth":2},{"title":"Is there programmatic access to the Serverless Inference API?","local":"is-there-programmatic-access-to-the-serverless-inference-api","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="serverless-inference-api" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#serverless-inference-api"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Serverless Inference API</span></h1> <p data-svelte-h="svelte-kxyi5y">Please refer to <a href="https://huggingface.co/docs/api-inference" rel="nofollow">Serverless Inference API Documentation</a> for detailed information.</p> <h2 class="relative group"><a id="what-technology-do-you-use-to-power-the-serverless-inference-api" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#what-technology-do-you-use-to-power-the-serverless-inference-api"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>What technology do you use to power the Serverless Inference API?</span></h2> <p data-svelte-h="svelte-2iniqr">For 🤗 Transformers models, <a href="https://huggingface.co/docs/transformers/main_classes/pipelines" rel="nofollow">Pipelines</a> power the API.</p> <p data-svelte-h="svelte-1lee1s9">On top of <code>Pipelines</code> and depending on the model type, there are several production optimizations like:</p> <ul data-svelte-h="svelte-1fzm9iw"><li>compiling models to optimized intermediary representations (e.g. <a href="https://medium.com/microsoftazure/accelerate-your-nlp-pipelines-using-hugging-face-transformers-and-onnx-runtime-2443578f4333" rel="nofollow">ONNX</a>),</li> <li>maintaining a Least Recently Used cache, ensuring that the most popular models are always loaded,</li> <li>scaling the underlying compute infrastructure on the fly depending on the load constraints.</li></ul> <p data-svelte-h="svelte-hv7bvp">For models from <a href="./models-libraries">other libraries</a>, the API uses <a href="https://www.starlette.io" rel="nofollow">Starlette</a> and runs in <a href="https://github.com/huggingface/api-inference-community/tree/main/docker_images" rel="nofollow">Docker containers</a>. Each library defines the implementation of <a href="https://github.com/huggingface/api-inference-community/tree/main/docker_images/sentence_transformers/app/pipelines" rel="nofollow">different pipelines</a>.</p> <h2 class="relative group"><a id="how-can-i-turn-off-the-serverless-inference-api-for-my-model" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#how-can-i-turn-off-the-serverless-inference-api-for-my-model"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>How can I turn off the Serverless Inference API for my model?</span></h2> <p data-svelte-h="svelte-1o36w5w">Specify <code>inference: false</code> in your model card’s metadata.</p> <h2 class="relative group"><a id="why-dont-i-see-an-inference-widget-or-why-cant-i-use-the-api" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#why-dont-i-see-an-inference-widget-or-why-cant-i-use-the-api"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Why don’t I see an inference widget, or why can’t I use the API?</span></h2> <p data-svelte-h="svelte-kmafgm">For some tasks, there might not be support in the Serverless Inference API, and, hence, there is no widget.
	For all libraries (except 🤗 Transformers), there is a <a href="https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/src/library-to-tasks.ts" rel="nofollow">library-to-tasks.ts file</a> of supported tasks in the API. When a model repository has a task that is not supported by the repository library, the repository has <code>inference: false</code> by default.</p> <h2 class="relative group"><a id="can-i-send-large-volumes-of-requests-can-i-get-accelerated-apis" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#can-i-send-large-volumes-of-requests-can-i-get-accelerated-apis"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Can I send large volumes of requests? Can I get accelerated APIs?</span></h2> <p data-svelte-h="svelte-1g95itu">If you are interested in accelerated inference, higher volumes of requests, or an SLA, please contact us at <code>api-enterprise at huggingface.co</code>.</p> <h2 class="relative group"><a id="how-can-i-see-my-usage" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#how-can-i-see-my-usage"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>How can I see my usage?</span></h2> <p data-svelte-h="svelte-ieyw49">You can check your usage in the <a href="https://ui.endpoints.huggingface.co/endpoints" rel="nofollow">Inference Dashboard</a>. The dashboard shows both your serverless and dedicated endpoints usage.</p> <h2 class="relative group"><a id="is-there-programmatic-access-to-the-serverless-inference-api" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#is-there-programmatic-access-to-the-serverless-inference-api"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Is there programmatic access to the Serverless Inference API?</span></h2> <p data-svelte-h="svelte-5o1vn3">Yes, the <code>huggingface_hub</code> library has a client wrapper documented <a href="https://huggingface.co/docs/huggingface_hub/how-to-inference" rel="nofollow">here</a>.</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/hub-docs/blob/main/docs/hub/models-inference.md" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_1vatp3t = {
	assets: "/docs/hub/main/en",
	base: "/docs/hub/main/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/hub/main/en/_app/immutable/entry/start.d0cd5065.js"),
	import("/docs/hub/main/en/_app/immutable/entry/app.b6abe3c1.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 79],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 16 kB
Xet hash:: c0be005023d873d593899690ac6ad8433fb656b4ee5e4a8b025e72992c1d7880

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.