bitsandbytes/v0.43.0/en/index.html

<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;bitsandbytes&quot;,&quot;local&quot;:&quot;bitsandbytes&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;API documentation&quot;,&quot;local&quot;:&quot;api-documentation&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}">
		<link href="/docs/bitsandbytes/v0.43.0/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
		<link rel="modulepreload" href="/docs/bitsandbytes/v0.43.0/en/_app/immutable/entry/start.eac2c940.js">
		<link rel="modulepreload" href="/docs/bitsandbytes/v0.43.0/en/_app/immutable/chunks/scheduler.0f20f9e5.js">
		<link rel="modulepreload" href="/docs/bitsandbytes/v0.43.0/en/_app/immutable/chunks/singletons.6bcd6fdd.js">
		<link rel="modulepreload" href="/docs/bitsandbytes/v0.43.0/en/_app/immutable/chunks/index.e6666749.js">
		<link rel="modulepreload" href="/docs/bitsandbytes/v0.43.0/en/_app/immutable/chunks/paths.d58d761f.js">
		<link rel="modulepreload" href="/docs/bitsandbytes/v0.43.0/en/_app/immutable/entry/app.24bdf5c5.js">
		<link rel="modulepreload" href="/docs/bitsandbytes/v0.43.0/en/_app/immutable/chunks/index.fbd8b8e3.js">
		<link rel="modulepreload" href="/docs/bitsandbytes/v0.43.0/en/_app/immutable/nodes/0.4860f614.js">
		<link rel="modulepreload" href="/docs/bitsandbytes/v0.43.0/en/_app/immutable/chunks/each.e59479a4.js">
		<link rel="modulepreload" href="/docs/bitsandbytes/v0.43.0/en/_app/immutable/nodes/6.2d6129b3.js">
		<link rel="modulepreload" href="/docs/bitsandbytes/v0.43.0/en/_app/immutable/chunks/Heading.2451f3bb.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;bitsandbytes&quot;,&quot;local&quot;:&quot;bitsandbytes&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;API documentation&quot;,&quot;local&quot;:&quot;api-documentation&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"><!-- HEAD_svelte-u9bgzb_END -->      <p></p>  <h1 class="relative group"><a id="bitsandbytes" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#bitsandbytes"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>bitsandbytes</span></h1> <p data-svelte-h="svelte-qct5cs">The <code>bitsandbytes</code> library is a lightweight Python wrapper around CUDA custom functions, in particular 8-bit optimizers, matrix multiplication (LLM.int8()), and 8 + 4-bit quantization functions.</p> <p data-svelte-h="svelte-mc45wm">The library includes quantization primitives for 8-bit &amp; 4-bit operations, through <code>bitsandbytes.nn.Linear8bitLt</code> and <code>bitsandbytes.nn.Linear4bit</code> and 8bit optimizers through <code>bitsandbytes.optim</code> module.</p> <p data-svelte-h="svelte-xc3m7j">There are ongoing efforts to support further hardware backends, i.e. Intel CPU + GPU, AMD GPU, Apple Silicon. Windows support is on its way as well.</p>  <h2 class="relative group"><a id="api-documentation" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#api-documentation"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>API documentation</span></h2> <ul data-svelte-h="svelte-c9xhro"><li><a href="quantization">Quantization</a></li> <li><a href="integrations">Integrations</a></li> <li><a href="optimizers">Optimizers</a></li></ul>  <h1 class="relative group"><a id="license" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#license"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>License</span></h1> <p data-svelte-h="svelte-1eskf9o">The majority of bitsandbytes is licensed under MIT, however portions of the project are available under separate license terms, as the parts adapted from Pytorch are licensed under the BSD license.</p> <p data-svelte-h="svelte-18bjayv">We thank Fabio Cannizzo for his work on <a href="https://github.com/fabiocannizzo/FastBinarySearch" rel="nofollow">FastBinarySearch</a> which we use for CPU quantization.</p>  <p></p> 
			
			<script>
				{
					__sveltekit_155r5qs = {
						assets: "/docs/bitsandbytes/v0.43.0/en",
						base: "/docs/bitsandbytes/v0.43.0/en",
						env: {}
					};

					const element = document.currentScript.parentElement;

					const data = [null,null];

					Promise.all([
						import("/docs/bitsandbytes/v0.43.0/en/_app/immutable/entry/start.eac2c940.js"),
						import("/docs/bitsandbytes/v0.43.0/en/_app/immutable/entry/app.24bdf5c5.js")
					]).then(([kit, app]) => {
						kit.start(app, element, {
							node_ids: [0, 6],
							data,
							form: null,
							error: null
						});
					});
				}
			</script>