Buckets:
| """Download all dependencies from HF, apply resume overlay; DO NOT train.""" | |
| import argparse,json,tarfile,shutil | |
| from pathlib import Path | |
| from huggingface_hub import HfApi | |
| BUCKET='Disya/LLM-TO-DIT';PREFIX='resume/step003000' | |
| def main(): | |
| p=argparse.ArgumentParser();p.add_argument('--dest',required=True);a=p.parse_args();root=Path(a.dest).resolve();root.mkdir(parents=True,exist_ok=True);tmp=root/'.bundle_downloads';tmp.mkdir(exist_ok=True);api=HfApi() | |
| def fetch(remote,path,size=None): | |
| path.parent.mkdir(parents=True,exist_ok=True);api.download_bucket_files(BUCKET,[(remote,path)],raise_on_missing_files=True) | |
| if size is not None:assert path.stat().st_size==size,remote | |
| fetch('manifest.json',tmp/'dataset_manifest.json');dataset=json.loads((tmp/'dataset_manifest.json').read_text());assert dataset['complete'] | |
| fetch(PREFIX+'/resume_manifest.json',tmp/'resume_manifest.json');resume=json.loads((tmp/'resume_manifest.json').read_text());assert resume['complete'] and resume['step']==3000 | |
| previous=json.loads((root/'manifest.json').read_text()) if (root/'manifest.json').exists() else {} | |
| have_dataset=previous.get('complete') and previous.get('archives')==dataset['archives'] | |
| if not have_dataset: | |
| stamps=tmp/'extracted';stamps.mkdir(exist_ok=True) | |
| for item in dataset['archives']: | |
| stamp=stamps/(item['file'].replace('/','_')+'.json') | |
| if stamp.exists() and json.loads(stamp.read_text())==item:continue | |
| local=tmp/Path(item['file']).name;fetch(item['file'],local,item['bytes']) | |
| with tarfile.open(local,'r:*') as tf:tf.extractall(root,filter='data') | |
| local.unlink();stamp.write_text(json.dumps(item));print('EXTRACTED',item['file'],flush=True) | |
| shutil.copy2(tmp/'dataset_manifest.json',root/'manifest.json') | |
| overlay=tmp/'trainer_resume3000.tar.gz';fetch(resume['code']['file'],overlay,resume['code']['bytes']) | |
| with tarfile.open(overlay,'r:gz') as tf:tf.extractall(root,filter='data') | |
| overlay.unlink() | |
| checkpoint=root/'qwen_rgb_3060/runs/large_mixed_streaming500_3060/full_latest.pt';download=checkpoint.with_suffix('.download');fetch(resume['checkpoint']['file'],download,resume['checkpoint']['bytes']);download.replace(checkpoint) | |
| shutil.copy2(tmp/'resume_manifest.json',root/'resume_manifest.json');print('READY: checkpoint3000, optimizer, RNG, online means, code, dataset and base model. Training has NOT started.',flush=True) | |
| if __name__=='__main__':main() | |
Xet Storage Details
- Size:
- 2.39 kB
- Xet hash:
- dcdee8142ff9aebad4950dcab2d176af46052c5909e5bb906d49f7fa783e4c91
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.