File size: 2,134 Bytes
c1e4f50 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 | """Fetch only the source snapshot's 28 display files, with hash verification."""
import argparse
import hashlib
import json
import os
import pathlib
import sqlite3
from huggingface_hub import HfApi
from redaction import redactor_for_database
parser=argparse.ArgumentParser()
parser.add_argument('database')
args=parser.parse_args()
db=sqlite3.connect(pathlib.Path(args.database).resolve().as_uri()+'?mode=ro',uri=True)
db.row_factory=sqlite3.Row
redact = redactor_for_database(db)
rows=list(db.execute('SELECT s.*,p.external_id handle FROM storage_files s JOIN principals p ON p.id=s.owner_id'))
downloads=[]
for row in rows:
relative=pathlib.Path(str(row['owner_id']))/row['path']
if '..' in relative.parts or relative.is_absolute():raise ValueError('Invalid storage path')
target=pathlib.Path('assets/storage')/relative
target.parent.mkdir(parents=True,exist_ok=True)
downloads.append(('boards/chatbot-v8/storage/'+relative.as_posix(),target))
HfApi(token=os.environ.get('HF_TOKEN')).download_bucket_files('HuggingFaceSwarms/chatbot-msgboards-data',downloads,raise_on_missing_files=True)
catalog={}
for row,(_,target) in zip(rows,downloads):
content=target.read_bytes()
if hashlib.sha256(content).hexdigest()!=row['sha256']:raise ValueError('Stored file differs from snapshot')
# Some source text uploads were labelled as form data. Sanitize any UTF-8
# file, irrespective of its declared content type, after verifying it.
try:
text=content.decode('utf-8')
except UnicodeDecodeError:
text=None
if text is not None:
content=redact(text).encode('utf-8')
target.write_bytes(content)
catalog[row['handle']+'/'+row['path']]={'path':row['path'],'content_type':row['content_type'],'size':len(content),'sha256':hashlib.sha256(content).hexdigest(),'updated_at':row['updated_at'],'url':target.as_posix()}
path=pathlib.Path('data/manifest.json');manifest=json.loads(path.read_text());manifest['storage']=catalog
path.write_text(json.dumps(manifest,ensure_ascii=False,separators=(',',':')))
print(json.dumps({'files':len(catalog),'checksums_verified':True}))
|