Buckets:
| """Query parquet tables in the bucket with DuckDB, reading only the needed columns over HTTPS range requests | |
| (DuckDB's hf:// scheme does not support buckets yet; https://huggingface.co/buckets/<id>/resolve/<path> does ranges). | |
| Usage (python): from bucket_query import urls, connect; c = connect(); c.execute(f"select ... from read_parquet({urls('processed/x/')})") | |
| Usage (cli): python code/bucket_query.py "<sql with {prefix:processed/x/} placeholders>" | |
| """ | |
| import os, re, sys | |
| os.environ.setdefault("HOME", "/root") # systemd services have no HOME; DuckDB needs it for extensions | |
| from pathlib import Path | |
| import duckdb | |
| from dotenv import load_dotenv | |
| load_dotenv(Path(__file__).resolve().parent.parent / ".env") | |
| from huggingface_hub import HfApi | |
| BUCKET = "Mercity/SkillsStorage" | |
| def urls(prefix, suffix=".parquet"): | |
| files = [f.path for f in HfApi().list_bucket_tree(BUCKET, prefix=prefix.rstrip("/"), recursive=True) | |
| if getattr(f, "size", None) is not None and f.path.endswith(suffix)] | |
| return "[" + ",".join(f"'https://huggingface.co/buckets/{BUCKET}/resolve/{p}'" for p in sorted(files)) + "]" | |
| def connect(mem="250MB"): | |
| c = duckdb.connect(); c.execute(f"INSTALL httpfs; LOAD httpfs; SET threads=2; SET memory_limit='{mem}'") | |
| if os.environ.get("HF_TOKEN"): | |
| c.execute(f"CREATE SECRET hfb (TYPE HTTP, BEARER_TOKEN '{os.environ['HF_TOKEN']}')") | |
| return c | |
| if __name__ == "__main__": | |
| sql = re.sub(r"\{prefix:([^}]+)\}", lambda m: f"read_parquet({urls(m.group(1))})", sys.argv[1]) | |
| for row in connect().execute(sql).fetchall(): print(row) | |
Xet Storage Details
- Size:
- 1.59 kB
- Xet hash:
- 8fa0e12cc9a3b0260c1152d529b336c54ba8522b2f262828d1ae0525d687a343
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.