Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / xet /pr_2272 /en /api.html

rtrm

about 2 months ago

download

raw

49.6 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"CAS API Documentation","local":"cas-api-documentation","sections":[{"title":"Authentication","local":"authentication","sections":[],"depth":2},{"title":"Converting Hashes to Strings","local":"converting-hashes-to-strings","sections":[{"title":"Procedure","local":"procedure","sections":[],"depth":3},{"title":"Example","local":"example","sections":[],"depth":3}],"depth":2},{"title":"Endpoints","local":"endpoints","sections":[{"title":"1. Get File Reconstruction","local":"1-get-file-reconstruction","sections":[],"depth":3},{"title":"Example File Reconstruction Response Body","local":"example-file-reconstruction-response-body","sections":[],"depth":3},{"title":"2. Query Chunk Deduplication (Global Deduplication)","local":"2-query-chunk-deduplication-global-deduplication","sections":[{"title":"Example Shard Response Body","local":"example-shard-response-body","sections":[],"depth":4}],"depth":3},{"title":"3. Upload Xorb","local":"3-upload-xorb","sections":[{"title":"Example Xorb Request Body","local":"example-xorb-request-body","sections":[],"depth":4}],"depth":3},{"title":"4. Upload Shard","local":"4-upload-shard","sections":[{"title":"Example Shard Request Body","local":"example-shard-request-body","sections":[],"depth":4}],"depth":3}],"depth":2},{"title":"Error Cases","local":"error-cases","sections":[{"title":"Non-Retryable Errors","local":"non-retryable-errors","sections":[],"depth":3},{"title":"Retryable Errors","local":"retryable-errors","sections":[],"depth":3}],"depth":2}],"depth":1}">
	<link href="/docs/xet/pr_2272/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/xet/pr_2272/en/_app/immutable/entry/start.7209fe0c.js">
	<link rel="modulepreload" href="/docs/xet/pr_2272/en/_app/immutable/chunks/scheduler.de5597d1.js">
	<link rel="modulepreload" href="/docs/xet/pr_2272/en/_app/immutable/chunks/singletons.800e7c38.js">
	<link rel="modulepreload" href="/docs/xet/pr_2272/en/_app/immutable/chunks/paths.d1eab31c.js">
	<link rel="modulepreload" href="/docs/xet/pr_2272/en/_app/immutable/entry/app.54268a84.js">
	<link rel="modulepreload" href="/docs/xet/pr_2272/en/_app/immutable/chunks/preload-helper.1b7d12f9.js">
	<link rel="modulepreload" href="/docs/xet/pr_2272/en/_app/immutable/chunks/index.f8bac2c1.js">
	<link rel="modulepreload" href="/docs/xet/pr_2272/en/_app/immutable/nodes/0.097c0756.js">
	<link rel="modulepreload" href="/docs/xet/pr_2272/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/xet/pr_2272/en/_app/immutable/nodes/2.2cc1d64d.js">
	<link rel="modulepreload" href="/docs/xet/pr_2272/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.4e1a23d9.js">
	<link rel="modulepreload" href="/docs/xet/pr_2272/en/_app/immutable/chunks/CodeBlock.d8c0c481.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"CAS API Documentation","local":"cas-api-documentation","sections":[{"title":"Authentication","local":"authentication","sections":[],"depth":2},{"title":"Converting Hashes to Strings","local":"converting-hashes-to-strings","sections":[{"title":"Procedure","local":"procedure","sections":[],"depth":3},{"title":"Example","local":"example","sections":[],"depth":3}],"depth":2},{"title":"Endpoints","local":"endpoints","sections":[{"title":"1. Get File Reconstruction","local":"1-get-file-reconstruction","sections":[],"depth":3},{"title":"Example File Reconstruction Response Body","local":"example-file-reconstruction-response-body","sections":[],"depth":3},{"title":"2. Query Chunk Deduplication (Global Deduplication)","local":"2-query-chunk-deduplication-global-deduplication","sections":[{"title":"Example Shard Response Body","local":"example-shard-response-body","sections":[],"depth":4}],"depth":3},{"title":"3. Upload Xorb","local":"3-upload-xorb","sections":[{"title":"Example Xorb Request Body","local":"example-xorb-request-body","sections":[],"depth":4}],"depth":3},{"title":"4. Upload Shard","local":"4-upload-shard","sections":[{"title":"Example Shard Request Body","local":"example-shard-request-body","sections":[],"depth":4}],"depth":3}],"depth":2},{"title":"Error Cases","local":"error-cases","sections":[{"title":"Non-Retryable Errors","local":"non-retryable-errors","sections":[],"depth":3},{"title":"Retryable Errors","local":"retryable-errors","sections":[],"depth":3}],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="cas-api-documentation" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#cas-api-documentation"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>CAS API Documentation</span></h1> <p data-svelte-h="svelte-fk2l4i">This document describes the HTTP API endpoints used by the Content Addressable Storage (CAS) client to interact with the remote CAS server.</p> <h2 class="relative group"><a id="authentication" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#authentication"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Authentication</span></h2> <p data-svelte-h="svelte-1gmegwg">To authenticate, authorize, and obtain the API base URL, follow the instructions in <a href="./auth">Authentication</a>.</p> <h2 class="relative group"><a id="converting-hashes-to-strings" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#converting-hashes-to-strings"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Converting Hashes to Strings</span></h2> <p data-svelte-h="svelte-rszaxi">Sometimes hashes are used in API paths as hexadecimal strings (reconstruction, xorb upload, global dedupe API).</p> <p data-svelte-h="svelte-1tqp9u">To convert a 32 hash to a 64 hexadecimal character string to be used as part of an API path there is a specific procedure, MUST NOT directly convert each byte.</p> <h3 class="relative group"><a id="procedure" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#procedure"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Procedure</span></h3> <p data-svelte-h="svelte-19ce41k">For every 8 bytes in the hash (indices 0-7, 8-15, 16-23, 24-31) reverse the order of each byte in those regions then concatenate the regions back in order.</p> <p data-svelte-h="svelte-119lld6">Otherwise stated, consider each 8 byte part of a hash as a little endian 64 bit unsigned integer, then concatenate the hexadecimal representation of the 4 numbers in order (each padded with 0’s to 16 characters).</p> <blockquote class="note" data-svelte-h="svelte-2dsrmt"><p>In all cases that a hash is represented as a string it is converted from a byte array to a string using this procedure.</p></blockquote> <h3 class="relative group"><a id="example" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#example"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Example</span></h3> <p data-svelte-h="svelte-1nwn7m6">Suppose a hash value is:
	<code>[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]</code></p> <p data-svelte-h="svelte-1od7j5y">Then before converting to a string it will first have its bytes reordered to:
	<code>[7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8, 23, 22, 21, 20, 19, 18, 17, 16, 31, 30, 29, 28, 27, 26, 25, 24]</code></p> <p data-svelte-h="svelte-was8mf">So the string value of the provided hash [0..32] is <strong>NOT</strong> <code>000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f</code>.
	It is: <code>07060504030201000f0e0d0c0b0a0908171615141312111f1e1d1c1b1a1918</code>.</p> <h2 class="relative group"><a id="endpoints" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#endpoints"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Endpoints</span></h2> <h3 class="relative group"><a id="1-get-file-reconstruction" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#1-get-file-reconstruction"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>1. Get File Reconstruction</span></h3> <ul><li data-svelte-h="svelte-1l2y1hx"><p><strong>Description</strong>: Retrieves reconstruction information for a specific file, includes byte range support when <code>Range</code> header is set.</p></li> <li data-svelte-h="svelte-5hkjq5"><p><strong>Path</strong>: <code>/v1/reconstructions/{file_id}</code></p></li> <li data-svelte-h="svelte-oss6fi"><p><strong>Method</strong>: <code>GET</code></p></li> <li data-svelte-h="svelte-19yyae1"><p><strong>Parameters</strong>:</p> <ul><li><code>file_id</code>: File hash in hex format (64 lowercase hexadecimal characters).
	See <a href="./hashing#file-hashes">file hashes</a> for computing the file hash and <a href="./api#converting-hashes-to-strings">converting hashes to strings</a>.</li></ul></li> <li data-svelte-h="svelte-16ihd4"><p><strong>Headers</strong>:</p> <ul><li><code>Range</code>: OPTIONAL. Format: <code>bytes={start}-{end}</code> (end is inclusive).</li></ul></li> <li data-svelte-h="svelte-1viu6l2"><p><strong>Minimum Token Scope</strong>: <code>read</code></p></li> <li data-svelte-h="svelte-lyktfm"><p><strong>Body</strong>: None.</p></li> <li><p data-svelte-h="svelte-vlqf27"><strong>Response</strong>: JSON (<code>QueryReconstructionResponse</code>)</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-punctuation">{</span>
	<span class="hljs-attr">"offset_into_first_range"</span><span class="hljs-punctuation">:</span> <span class="hljs-number">0</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"terms"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">[</span>...<span class="hljs-punctuation">]</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"fetch_info"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>...<span class="hljs-punctuation">}</span>
	<span class="hljs-punctuation">}</span><!-- HTML_TAG_END --></pre></div></li> <li data-svelte-h="svelte-176p2hh"><p><strong>Error Responses</strong>: See <a href="./api#error-cases">Error Cases</a></p> <ul><li><code>400 Bad Request</code>: Malformed <code>file_id</code> in the path. Fix the path before retrying.</li> <li><code>401 Unauthorized</code>: Refresh the token to continue making requests, or provide a token in the <code>Authorization</code> header.</li> <li><code>404 Not Found</code>: The file does not exist. Not retryable.</li> <li><code>416 Range Not Satisfiable</code>: The requested byte range start exceeds the end of the file. Not retryable.</li></ul></li></ul> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->GET /v1/reconstructions/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
	-H "Authorization: Bearer <token>"
	OPTIONAL: -H Range: "bytes=0-100000"<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="example-file-reconstruction-response-body" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#example-file-reconstruction-response-body"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Example File Reconstruction Response Body</span></h3> <p data-svelte-h="svelte-bbsjma">See <a href="./download-protocol#queryreconstructionresponse-structure">QueryReconstructionResponse</a> for more details in the download protocol specification.</p> <h3 class="relative group"><a id="2-query-chunk-deduplication-global-deduplication" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#2-query-chunk-deduplication-global-deduplication"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>2. Query Chunk Deduplication (Global Deduplication)</span></h3> <ul data-svelte-h="svelte-jpz7y8"><li><strong>Description</strong>: Checks if a chunk exists in the CAS for deduplication purposes.</li> <li><strong>Path</strong>: <code>/v1/chunks/{prefix}/{hash}</code></li> <li><strong>Method</strong>: <code>GET</code></li> <li><strong>Parameters</strong>:
	<ul><li><code>prefix</code>: The only acceptable prefix for the Global Deduplication API is <code>default-merkledb</code>.</li> <li><code>hash</code>: Chunk hash in hex format (64 lowercase hexadecimal characters).
	See <a href="./hashing#chunk-hashes">Chunk Hashes</a> to compute the chunk hash and <a href="./api#converting-hashes-to-strings">converting hashes to strings</a>.</li></ul></li> <li><strong>Minimum Token Scope</strong>: <code>read</code></li> <li><strong>Body</strong>: None.</li> <li><strong>Response</strong>: Shard format bytes (<code>application/octet-stream</code>), deserialize as a <a href="./shard#global-deduplication">shard</a>.</li> <li><strong>Error Responses</strong>: See <a href="./api#error-cases">Error Cases</a> <ul><li><code>400 Bad Request</code>: Malformed hash in the path. Fix the path before retrying.</li> <li><code>401 Unauthorized</code>: Refresh the token to continue making requests, or provide a token in the <code>Authorization</code> header.</li> <li><code>404 Not Found</code>: Chunk not already tracked by global deduplication. Not retryable.</li></ul></li></ul> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->GET /v1/chunks/default-merkledb/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
	-H "Authorization: Bearer <token>"<!-- HTML_TAG_END --></pre></div> <h4 class="relative group"><a id="example-shard-response-body" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#example-shard-response-body"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Example Shard Response Body</span></h4> <p data-svelte-h="svelte-1h5bmp3">An example shard response body can be found in <a href="https://huggingface.co/datasets/xet-team/xet-spec-reference-files/blob/main/Electric_Vehicle_Population_Data_20250917.csv.shard.dedupe" rel="nofollow">Xet reference files</a>.</p> <h3 class="relative group"><a id="3-upload-xorb" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#3-upload-xorb"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>3. Upload Xorb</span></h3> <ul data-svelte-h="svelte-bz58us"><li><strong>Description</strong>: Uploads a serialized Xorb to the server; uploading real data in serialized format.</li> <li><strong>Path</strong>: <code>/v1/xorbs/{prefix}/{hash}</code></li> <li><strong>Method</strong>: <code>POST</code></li> <li><strong>Parameters</strong>:
	<ul><li><code>prefix</code>: The only acceptable prefix for the Xorb upload API is <code>default</code>.</li> <li><code>hash</code>: Xorb hash in hex format (64 lowercase hexadecimal characters).
	See <a href="./hashing#xorb-hashes">Xorb Hashes</a> to compute the hash, and <a href="./api#converting-hashes-to-strings">converting hashes to strings</a>.</li></ul></li> <li><strong>Minimum Token Scope</strong>: <code>write</code></li> <li><strong>Body</strong>: Serialized Xorb bytes (<code>application/octet-stream</code>).
	See <a href="./xorb">xorb format serialization</a>.</li> <li><strong>Response</strong>: JSON (<code>UploadXorbResponse</code>)</li></ul> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-punctuation">{</span>
	<span class="hljs-attr">"was_inserted"</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span>
	<span class="hljs-punctuation">}</span><!-- HTML_TAG_END --></pre></div> <ul data-svelte-h="svelte-sgku5n"><li><p>Note: <code>was_inserted</code> is <code>false</code> if the Xorb already exists; this is not an error.</p></li> <li><p><strong>Error Responses</strong>: See <a href="./api#error-cases">Error Cases</a></p> <ul><li><code>400 Bad Request</code>: Malformed hash in the path, Xorb hash does not match the body, or body is incorrectly serialized.</li> <li><code>401 Unauthorized</code>: Refresh the token to continue making requests, or provide a token in the <code>Authorization</code> header.</li> <li><code>403 Forbidden</code>: Token provided but does not have a wide enough scope (for example, a <code>read</code> token was provided). Clients MUST retry with a <code>write</code> scope token.</li></ul></li></ul> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->POST /v1/xorbs/default/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
	-H "Authorization: Bearer <token>"<!-- HTML_TAG_END --></pre></div> <h4 class="relative group"><a id="example-xorb-request-body" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#example-xorb-request-body"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Example Xorb Request Body</span></h4> <p data-svelte-h="svelte-lg7avp">An example xorb request body can be found in <a href="https://huggingface.co/datasets/xet-team/xet-spec-reference-files/blob/main/eea25d6ee393ccae385820daed127b96ef0ea034dfb7cf6da3a950ce334b7632.xorb" rel="nofollow">Xet reference files</a>.</p> <h3 class="relative group"><a id="4-upload-shard" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#4-upload-shard"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>4. Upload Shard</span></h3> <ul data-svelte-h="svelte-1ucwhoj"><li><strong>Description</strong>: Uploads a Shard to the CAS.
	Uploads file reconstructions and new xorb listing, serialized into the shard format; marks the files as uploaded.</li> <li><strong>Path</strong>: <code>/v1/shards</code></li> <li><strong>Method</strong>: <code>POST</code></li> <li><strong>Minimum Token Scope</strong>: <code>write</code></li> <li><strong>Body</strong>: Serialized Shard data as bytes (<code>application/octet-stream</code>).
	See <a href="./shard#shard-upload">Shard format guide</a>.</li> <li><strong>Response</strong>: JSON (<code>UploadShardResponse</code>)</li></ul> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-punctuation">{</span>
	<span class="hljs-attr">"result"</span><span class="hljs-punctuation">:</span> <span class="hljs-number">0</span>
	<span class="hljs-punctuation">}</span><!-- HTML_TAG_END --></pre></div> <ul data-svelte-h="svelte-qr4477"><li>Where <code>result</code> is:
	<ul><li><code>0</code>: The Shard already exists.</li> <li><code>1</code>: <code>SyncPerformed</code> — the Shard was registered.</li></ul></li></ul> <p data-svelte-h="svelte-1ogc0n9">The value of <code>result</code> does not carry any meaning, if the upload shard API returns a <code>200 OK</code> status code, the upload was successful and the files listed are considered uploaded.</p> <ul data-svelte-h="svelte-1ihsw4c"><li><strong>Error Responses</strong>: See <a href="./api#error-cases">Error Cases</a> <ul><li><code>400 Bad Request</code>: Shard is incorrectly serialized or Shard contents failed verification.
	<ul><li>Can mean that a referenced Xorb doesn’t exist or the shard is too large</li></ul></li> <li><code>401 Unauthorized</code>: Refresh the token to continue making requests, or provide a token in the <code>Authorization</code> header.</li> <li><code>403 Forbidden</code>: Token provided but does not have a wide enough scope (for example, a <code>read</code> token was provided).</li></ul></li></ul> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->POST /v1/shards
	-H "Authorization: Bearer <token>"<!-- HTML_TAG_END --></pre></div> <h4 class="relative group"><a id="example-shard-request-body" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#example-shard-request-body"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Example Shard Request Body</span></h4> <p data-svelte-h="svelte-2r6zkh">An example shard request body can be found in <a href="https://huggingface.co/datasets/xet-team/xet-spec-reference-files/blob/main/Electric_Vehicle_Population_Data_20250917.csv.shard.verification-no-footer" rel="nofollow">Xet reference files</a>.</p> <h2 class="relative group"><a id="error-cases" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#error-cases"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Error Cases</span></h2> <h3 class="relative group"><a id="non-retryable-errors" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#non-retryable-errors"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Non-Retryable Errors</span></h3> <ul data-svelte-h="svelte-m31doy"><li><strong>400 Bad Request</strong>: Returned when the request parameters are invalid (for example, invalid Xorb/Shard on upload APIs).</li> <li><strong>401 Unauthorized</strong>: Refresh the token to continue making requests, or provide a token in the <code>Authorization</code> header.</li> <li><strong>403 Forbidden</strong>: Token provided but does not have a wide enough scope (for example, a <code>read</code> token was provided for an API requiring <code>write</code> scope).</li> <li><strong>404 Not Found</strong>: Occurs on <code>GET</code> APIs where the resource (Xorb, file) does not exist.</li> <li><strong>416 Range Not Satisfiable</strong>: Reconstruction API only; returned when byte range requests are invalid. Specifically, the requested start range is greater than or equal to the length of the file.</li></ul> <h3 class="relative group"><a id="retryable-errors" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#retryable-errors"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Retryable Errors</span></h3> <ul data-svelte-h="svelte-6qyk9c"><li><strong>Connection Errors</strong>: Often caused by network issues. Retry if intermittent.
	Clients SHOULD ensure no firewall blocks requests and SHOULD NOT use DNS overrides.</li> <li><strong>429 Rate Limiting</strong>: Lower your request rate using a backoff strategy, then wait and retry.
	Assume all APIs are rate limited.</li> <li><strong>500 Internal Server Error</strong>: The server experienced an intermittent issue; clients SHOULD retry their requests.</li> <li><strong>503 Service Unavailable</strong>: Service is temporarily unable to process requests; wait and retry.</li> <li><strong>504 Gateway Timeout</strong>: Service took too long to respond; wait and retry.</li></ul> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/hub-docs/blob/main/docs/xet/api.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_1t7h0ns = {
	assets: "/docs/xet/pr_2272/en",
	base: "/docs/xet/pr_2272/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/xet/pr_2272/en/_app/immutable/entry/start.7209fe0c.js"),
	import("/docs/xet/pr_2272/en/_app/immutable/entry/app.54268a84.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 2],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 49.6 kB
Xet hash:: 3be996c26637402ab6d95c6c6278c852f6a3f3fa58f945e725177472a3fb28e7

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.