DeepBeepMeep commited on
Commit
6649478
·
verified ·
1 Parent(s): 5db9f68

Remove migration utility from encoder assets

Browse files
Files changed (1) hide show
  1. Qwen3-VL-8B-Instruct/migrate.py +0 -25
Qwen3-VL-8B-Instruct/migrate.py DELETED
@@ -1,25 +0,0 @@
1
- from huggingface_hub import HfApi, hf_hub_download, CommitOperationAdd, CommitOperationDelete
2
- from pathlib import Path
3
- import hashlib,json,shutil,sys
4
- api=HfApi(token=sys.stdin.readline().strip()); folder='Qwen3-VL-8B-Instruct/'; source='DeepBeepMeep/Qwen_image_2'; dest='DeepBeepMeep/Ideogram4'
5
- stage=Path('D:/ml/wangp/qwen_image_21_preparation/encoder_consolidation')
6
- if sys.argv[1]=='upload':
7
- ops=[]; entries=[]; existing=set(api.list_repo_files(dest))
8
- for file in stage.iterdir():
9
- if file.name=='manifest.json':continue
10
- target=folder+file.name; sha=hashlib.sha256(file.read_bytes()).hexdigest()
11
- if target in existing:
12
- assert hashlib.sha256(Path(hf_hub_download(dest,target)).read_bytes()).hexdigest()==sha
13
- else:ops.append(CommitOperationAdd(path_in_repo=target,path_or_fileobj=str(file)))
14
- entries.append(dict(source=folder+file.name.replace('_legacy.json','.json'),target=target,sha256=sha))
15
- if ops:print('Upload commit:',api.create_commit(dest,operations=ops,commit_message='Consolidate encoder assets with generic legacy export suffixes').oid)
16
- for e in entries:
17
- local=Path(hf_hub_download(dest,e['target'],force_download=True)); assert hashlib.sha256(local.read_bytes()).hexdigest()==e['sha256']
18
- target=Path('D:/ml/wangp/ckpts')/e['target']; shutil.copyfile(local,target)
19
- (stage/'manifest.json').write_text(json.dumps(entries,indent=2))
20
- print('Verified:',len(entries))
21
- else:
22
- entries=json.loads((stage/'manifest.json').read_text())
23
- for e in entries:
24
- assert hashlib.sha256(Path(hf_hub_download(dest,e['target'],force_download=True)).read_bytes()).hexdigest()==e['sha256']
25
- print('Removal commit:',api.create_commit(source,operations=[CommitOperationDelete(path_in_repo=e['source']) for e in entries],commit_message='Remove encoder support folder after verified consolidation into Ideogram4').oid)