Buckets:
| """Fetch the prepared bucket bundle without retaining all archive copies locally.""" | |
| import argparse,json,tarfile,tempfile,shutil | |
| from pathlib import Path | |
| from huggingface_hub import HfApi | |
| BUCKET='Disya/LLM-TO-DIT' | |
| def main(): | |
| p=argparse.ArgumentParser();p.add_argument('--dest',required=True);a=p.parse_args();root=Path(a.dest).resolve();root.mkdir(parents=True,exist_ok=True);tmp=root/'.bundle_downloads';tmp.mkdir(exist_ok=True);api=HfApi() | |
| api.download_bucket_files(BUCKET,[('manifest.json',tmp/'manifest.json')],raise_on_missing_files=True);manifest=json.loads((tmp/'manifest.json').read_text());assert manifest['complete'],'Upload is not complete yet' | |
| for item in manifest['archives']: | |
| remote=item['file'];local=tmp/Path(remote).name;api.download_bucket_files(BUCKET,[(remote,local)],raise_on_missing_files=True);assert local.stat().st_size==item['bytes'],remote | |
| with tarfile.open(local,'r:*') as tf: | |
| tf.extractall(root,filter='data') | |
| local.unlink();print('EXTRACTED',remote,flush=True) | |
| shutil.copy2(tmp/'manifest.json',root/'manifest.json');print('READY',root/'qwen_rgb_3060','Training not started.',flush=True) | |
| if __name__=='__main__':main() | |
Xet Storage Details
- Size:
- 1.16 kB
- Xet hash:
- dead0de364a6a6dc25e2840fe0927b03bd93c7c833bad7b1d369a3a11bae6ca6
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.