Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / huggingface_hub /pr_4113 /de /guides /hf_file_system.html

HuggingFaceDocBuilder

4 months ago

download

raw

27.9 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Interagieren mit dem Hub über die Filesystem API","local":"interagieren-mit-dem-hub-über-die-filesystem-api","sections":[{"title":"Verwendung","local":"verwendung","sections":[],"depth":2},{"title":"Integrationen","local":"integrationen","sections":[],"depth":2},{"title":"Authentifizierung","local":"authentifizierung","sections":[],"depth":2}],"depth":1}">
	<link href="/docs/huggingface_hub/pr_4113/de/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/huggingface_hub/pr_4113/de/_app/immutable/entry/start.78846203.js">
	<link rel="modulepreload" href="/docs/huggingface_hub/pr_4113/de/_app/immutable/chunks/scheduler.de5597d1.js">
	<link rel="modulepreload" href="/docs/huggingface_hub/pr_4113/de/_app/immutable/chunks/singletons.c332abcc.js">
	<link rel="modulepreload" href="/docs/huggingface_hub/pr_4113/de/_app/immutable/chunks/paths.162adda8.js">
	<link rel="modulepreload" href="/docs/huggingface_hub/pr_4113/de/_app/immutable/entry/app.f549f98a.js">
	<link rel="modulepreload" href="/docs/huggingface_hub/pr_4113/de/_app/immutable/chunks/preload-helper.cc57ebc4.js">
	<link rel="modulepreload" href="/docs/huggingface_hub/pr_4113/de/_app/immutable/chunks/index.bf1177c2.js">
	<link rel="modulepreload" href="/docs/huggingface_hub/pr_4113/de/_app/immutable/nodes/0.9ce3ec2a.js">
	<link rel="modulepreload" href="/docs/huggingface_hub/pr_4113/de/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/huggingface_hub/pr_4113/de/_app/immutable/nodes/4.84da6439.js">
	<link rel="modulepreload" href="/docs/huggingface_hub/pr_4113/de/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.c5d5679c.js">
	<link rel="modulepreload" href="/docs/huggingface_hub/pr_4113/de/_app/immutable/chunks/CodeBlock.4f67b398.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Interagieren mit dem Hub über die Filesystem API","local":"interagieren-mit-dem-hub-über-die-filesystem-api","sections":[{"title":"Verwendung","local":"verwendung","sections":[],"depth":2},{"title":"Integrationen","local":"integrationen","sections":[],"depth":2},{"title":"Authentifizierung","local":"authentifizierung","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="interagieren-mit-dem-hub-über-die-filesystem-api" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#interagieren-mit-dem-hub-über-die-filesystem-api"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Interagieren mit dem Hub über die Filesystem API</span></h1> <p data-svelte-h="svelte-18w92na">Zusätzlich zur <code>HfApi</code> bietet die <code>huggingface_hub</code> Bibliothek <code>HfFileSystem</code>, eine pythonische, <a href="https://filesystem-spec.readthedocs.io/en/latest/" rel="nofollow">fsspec-kompatible</a> Dateischnittstelle zum Hugging Face Hub. Das <code>HfFileSystem</code> basiert auf der <code>HfApi</code> und bietet typische Dateisystemoperationen wie <code>cp</code>, <code>mv</code>, <code>ls</code>, <code>du</code>, <code>glob</code>, <code>get_file</code>, und <code>put_file</code>.</p> <h2 class="relative group"><a id="verwendung" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#verwendung"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Verwendung</span></h2> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> hffs

	<span class="hljs-meta">>>> </span><span class="hljs-comment"># Alle Dateien in einem Verzeichnis auflisten</span>
	<span class="hljs-meta">>>> </span>hffs.ls(<span class="hljs-string">"datasets/my-username/my-dataset-repo/data"</span>, detail=<span class="hljs-literal">False</span>)
	[<span class="hljs-string">'datasets/my-username/my-dataset-repo/data/train.csv'</span>, <span class="hljs-string">'datasets/my-username/my-dataset-repo/data/test.csv'</span>]

	<span class="hljs-meta">>>> </span><span class="hljs-comment"># Alle ".csv"-Dateien in einem Repo auflisten</span>
	<span class="hljs-meta">>>> </span>hffs.glob(<span class="hljs-string">"datasets/my-username/my-dataset-repo/**.csv"</span>)
	[<span class="hljs-string">'datasets/my-username/my-dataset-repo/data/train.csv'</span>, <span class="hljs-string">'datasets/my-username/my-dataset-repo/data/test.csv'</span>]

	<span class="hljs-meta">>>> </span><span class="hljs-comment"># Eine entfernte Datei lesen</span>
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">with</span> hffs.<span class="hljs-built_in">open</span>(<span class="hljs-string">"datasets/my-username/my-dataset-repo/data/train.csv"</span>, <span class="hljs-string">"r"</span>) <span class="hljs-keyword">as</span> f:
	<span class="hljs-meta">... </span> train_data = f.readlines()

	<span class="hljs-meta">>>> </span><span class="hljs-comment"># Den Inhalt einer entfernten Datei als Zeichenkette / String lesen</span>
	<span class="hljs-meta">>>> </span>train_data = hffs.read_text(<span class="hljs-string">"datasets/my-username/my-dataset-repo/data/train.csv"</span>, revision=<span class="hljs-string">"dev"</span>)

	<span class="hljs-meta">>>> </span><span class="hljs-comment"># Eine entfernte Datei schreiben</span>
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">with</span> hffs.<span class="hljs-built_in">open</span>(<span class="hljs-string">"datasets/my-username/my-dataset-repo/data/validation.csv"</span>, <span class="hljs-string">"w"</span>) <span class="hljs-keyword">as</span> f:
	<span class="hljs-meta">... </span> f.write(<span class="hljs-string">"text,label"</span>)
	<span class="hljs-meta">... </span> f.write(<span class="hljs-string">"Fantastic movie!,good"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-3z26mg">Das optionale Argument <code>revision</code> kann übergeben werden, um eine Operation von einem spezifischen Commit auszuführen, wie z.B. einem Branch, Tag-Namen oder einem Commit-Hash.</p> <p data-svelte-h="svelte-tbpook">Anders als bei Pythons eingebautem <code>open</code>, ist der Standardmodus von <code>fsspec</code>’s <code>open</code> binär, <code>"rb"</code>. Das bedeutet, dass Sie den Modus explizit auf <code>"r"</code> zum Lesen und <code>"w"</code> zum Schreiben im Textmodus setzen müssen. Das Anhängen an eine Datei (Modi <code>"a"</code> und <code>"ab"</code>) wird noch nicht unterstützt.</p> <h2 class="relative group"><a id="integrationen" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#integrationen"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Integrationen</span></h2> <p data-svelte-h="svelte-eeqd0a">Das <code>HfFileSystem</code> kann mit jeder Bibliothek verwendet werden, die <code>fsspec</code> integriert, vorausgesetzt die URL folgt dem Schema:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->hf://[<span class="hljs-attribute"><repo_type_prefix></span>]<span class="hljs-attribute"><repo_id></span>[@<span class="hljs-attribute"><revision></span>]/<span class="hljs-attribute"><pfad/im/repo></span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-967d81">Der <code>repo_type_prefix</code> ist <code>datasets/</code> für Datensätze, <code>spaces/</code> für Spaces, und Modelle benötigen kein Präfix in der URL.</p> <p data-svelte-h="svelte-wufznp">Einige interessante Integrationen, bei denen <code>HfFileSystem</code> die Interaktion mit dem Hub vereinfacht, sind unten aufgeführt:</p> <ul><li><p data-svelte-h="svelte-5rmyjs">Lesen/Schreiben eines <a href="https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html#reading-writing-remote-files" rel="nofollow">Pandas</a> DataFrame aus/in ein Hub-Repository:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd

	<span class="hljs-meta">>>> </span><span class="hljs-comment"># Eine entfernte CSV-Datei in einen DataFrame lesen</span>
	<span class="hljs-meta">>>> </span>df = pd.read_csv(<span class="hljs-string">"hf://datasets/my-username/my-dataset-repo/train.csv"</span>)

	<span class="hljs-meta">>>> </span><span class="hljs-comment"># Einen DataFrame in eine entfernte CSV-Datei schreiben</span>
	<span class="hljs-meta">>>> </span>df.to_csv(<span class="hljs-string">"hf://datasets/my-username/my-dataset-repo/test.csv"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-11g4kom">Der gleiche Arbeitsablauf kann auch für <a href="https://docs.dask.org/en/stable/how-to/connect-to-remote-data.html" rel="nofollow">Dask</a> und <a href="https://pola-rs.github.io/polars/py-polars/html/reference/io.html" rel="nofollow">Polars</a> verwendet werden.</p></li> <li><p data-svelte-h="svelte-1j784ud">Abfrage von (entfernten) Hub-Dateien mit <a href="https://duckdb.org/docs/guides/python/filesystems" rel="nofollow">DuckDB</a>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfFileSystem
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> duckdb

	<span class="hljs-meta">>>> </span>fs = HfFileSystem()
	<span class="hljs-meta">>>> </span>duckdb.register_filesystem(fs)
	<span class="hljs-meta">>>> </span><span class="hljs-comment"># Eine entfernte Datei abfragen und das Ergebnis als DataFrame zurückbekommen</span>
	<span class="hljs-meta">>>> </span>fs_query_file = <span class="hljs-string">"hf://datasets/my-username/my-dataset-repo/data_dir/data.parquet"</span>
	<span class="hljs-meta">>>> </span>df = duckdb.query(<span class="hljs-string">f"SELECT * FROM '<span class="hljs-subst">{fs_query_file}</span>' LIMIT 10"</span>).df()<!-- HTML_TAG_END --></pre></div></li> <li><p data-svelte-h="svelte-guwc20">Verwendung des Hub als Array-Speicher mit <a href="https://zarr.readthedocs.io/en/stable/tutorial.html#io-with-fsspec" rel="nofollow">Zarr</a>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> zarr

	<span class="hljs-meta">>>> </span>embeddings = np.random.randn(<span class="hljs-number">50000</span>, <span class="hljs-number">1000</span>).astype(<span class="hljs-string">"float32"</span>)

	<span class="hljs-meta">>>> </span><span class="hljs-comment"># Ein Array in ein Repo schreiben</span>
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">with</span> zarr.open_group(<span class="hljs-string">"hf://my-username/my-model-repo/array-store"</span>, mode=<span class="hljs-string">"w"</span>) <span class="hljs-keyword">as</span> root:
	<span class="hljs-meta">... </span> foo = root.create_group(<span class="hljs-string">"embeddings"</span>)
	<span class="hljs-meta">... </span> foobar = foo.zeros(<span class="hljs-string">'experiment_0'</span>, shape=(<span class="hljs-number">50000</span>, <span class="hljs-number">1000</span>), chunks=(<span class="hljs-number">10000</span>, <span class="hljs-number">1000</span>), dtype=<span class="hljs-string">'f4'</span>)
	<span class="hljs-meta">... </span> foobar[:] = embeddings

	<span class="hljs-meta">>>> </span><span class="hljs-comment"># Ein Array aus einem Repo lesen</span>
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">with</span> zarr.open_group(<span class="hljs-string">"hf://my-username/my-model-repo/array-store"</span>, mode=<span class="hljs-string">"r"</span>) <span class="hljs-keyword">as</span> root:
	<span class="hljs-meta">... </span> first_row = root[<span class="hljs-string">"embeddings/experiment_0"</span>][<span class="hljs-number">0</span>]<!-- HTML_TAG_END --></pre></div></li></ul> <h2 class="relative group"><a id="authentifizierung" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#authentifizierung"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Authentifizierung</span></h2> <p data-svelte-h="svelte-aphqiq">In vielen Fällen müssen Sie mit einem Hugging Face-Konto angemeldet sein, um mit dem Hub zu interagieren. Lesen Sie den <a href="../quick-start#login">Login</a>-Abschnitt der Dokumentation, um mehr über Authentifizierungsmethoden auf dem Hub zu erfahren.</p> <p data-svelte-h="svelte-1enyw3z">Es ist auch möglich, sich programmatisch anzumelden, indem Sie Ihr <code>token</code> als Argument an <code>HfFileSystem</code> übergeben:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfFileSystem
	<span class="hljs-meta">>>> </span>hffs = HfFileSystem(token=token)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-16usaea">Wenn Sie sich auf diese Weise anmelden, seien Sie vorsichtig, das Token nicht versehentlich zu veröffentlichen, wenn Sie Ihren Quellcode teilen!</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/huggingface_hub/blob/main/docs/source/de/guides/hf_file_system.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_1iqh95q = {
	assets: "/docs/huggingface_hub/pr_4113/de",
	base: "/docs/huggingface_hub/pr_4113/de",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/huggingface_hub/pr_4113/de/_app/immutable/entry/start.78846203.js"),
	import("/docs/huggingface_hub/pr_4113/de/_app/immutable/entry/app.f549f98a.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 4],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 27.9 kB
Xet hash:: 0054a7a203896e50606a3daa2a82ce4d164dda096e4b8c91aac814fbb759548b

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.