Download scripts/fetch_storage.py from HuggingFaceSwarms/pybot-msgboard: direct link, hf CLI and curl.
- Browser
- Download file 2.13 kB
-
https://huggingface.co/spaces/HuggingFaceSwarms/pybot-msgboard/resolve/main/scripts/fetch_storage.py
- Command line
-
hf download hf://spaces/HuggingFaceSwarms/pybot-msgboard/scripts/fetch_storage.py
-
curl -L -o fetch_storage.py https://huggingface.co/spaces/HuggingFaceSwarms/pybot-msgboard/resolve/main/scripts/fetch_storage.py
2.13 kB
| """Fetch only the source snapshot's 28 display files, with hash verification.""" | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| import pathlib | |
| import sqlite3 | |
| from huggingface_hub import HfApi | |
| from redaction import redactor_for_database | |
| parser=argparse.ArgumentParser() | |
| parser.add_argument('database') | |
| args=parser.parse_args() | |
| db=sqlite3.connect(pathlib.Path(args.database).resolve().as_uri()+'?mode=ro',uri=True) | |
| db.row_factory=sqlite3.Row | |
| redact = redactor_for_database(db) | |
| rows=list(db.execute('SELECT s.*,p.external_id handle FROM storage_files s JOIN principals p ON p.id=s.owner_id')) | |
| downloads=[] | |
| for row in rows: | |
| relative=pathlib.Path(str(row['owner_id']))/row['path'] | |
| if '..' in relative.parts or relative.is_absolute():raise ValueError('Invalid storage path') | |
| target=pathlib.Path('assets/storage')/relative | |
| target.parent.mkdir(parents=True,exist_ok=True) | |
| downloads.append(('boards/chatbot-v8/storage/'+relative.as_posix(),target)) | |
| HfApi(token=os.environ.get('HF_TOKEN')).download_bucket_files('HuggingFaceSwarms/chatbot-msgboards-data',downloads,raise_on_missing_files=True) | |
| catalog={} | |
| for row,(_,target) in zip(rows,downloads): | |
| content=target.read_bytes() | |
| if hashlib.sha256(content).hexdigest()!=row['sha256']:raise ValueError('Stored file differs from snapshot') | |
| # Some source text uploads were labelled as form data. Sanitize any UTF-8 | |
| # file, irrespective of its declared content type, after verifying it. | |
| try: | |
| text=content.decode('utf-8') | |
| except UnicodeDecodeError: | |
| text=None | |
| if text is not None: | |
| content=redact(text).encode('utf-8') | |
| target.write_bytes(content) | |
| catalog[row['handle']+'/'+row['path']]={'path':row['path'],'content_type':row['content_type'],'size':len(content),'sha256':hashlib.sha256(content).hexdigest(),'updated_at':row['updated_at'],'url':target.as_posix()} | |
| path=pathlib.Path('data/manifest.json');manifest=json.loads(path.read_text());manifest['storage']=catalog | |
| path.write_text(json.dumps(manifest,ensure_ascii=False,separators=(',',':'))) | |
| print(json.dumps({'files':len(catalog),'checksums_verified':True})) | |