"""Fetch only the source snapshot's 28 display files, with hash verification.""" import argparse import hashlib import json import os import pathlib import sqlite3 from huggingface_hub import HfApi from redaction import redactor_for_database parser=argparse.ArgumentParser() parser.add_argument('database') args=parser.parse_args() db=sqlite3.connect(pathlib.Path(args.database).resolve().as_uri()+'?mode=ro',uri=True) db.row_factory=sqlite3.Row redact = redactor_for_database(db) rows=list(db.execute('SELECT s.*,p.external_id handle FROM storage_files s JOIN principals p ON p.id=s.owner_id')) downloads=[] for row in rows: relative=pathlib.Path(str(row['owner_id']))/row['path'] if '..' in relative.parts or relative.is_absolute():raise ValueError('Invalid storage path') target=pathlib.Path('assets/storage')/relative target.parent.mkdir(parents=True,exist_ok=True) downloads.append(('boards/chatbot-v8/storage/'+relative.as_posix(),target)) HfApi(token=os.environ.get('HF_TOKEN')).download_bucket_files('HuggingFaceSwarms/chatbot-msgboards-data',downloads,raise_on_missing_files=True) catalog={} for row,(_,target) in zip(rows,downloads): content=target.read_bytes() if hashlib.sha256(content).hexdigest()!=row['sha256']:raise ValueError('Stored file differs from snapshot') # Some source text uploads were labelled as form data. Sanitize any UTF-8 # file, irrespective of its declared content type, after verifying it. try: text=content.decode('utf-8') except UnicodeDecodeError: text=None if text is not None: content=redact(text).encode('utf-8') target.write_bytes(content) catalog[row['handle']+'/'+row['path']]={'path':row['path'],'content_type':row['content_type'],'size':len(content),'sha256':hashlib.sha256(content).hexdigest(),'updated_at':row['updated_at'],'url':target.as_posix()} path=pathlib.Path('data/manifest.json');manifest=json.loads(path.read_text());manifest['storage']=catalog path.write_text(json.dumps(manifest,ensure_ascii=False,separators=(',',':'))) print(json.dumps({'files':len(catalog),'checksums_verified':True}))