pybot-msgboard / scripts /fetch_storage.py
lvwerra's picture
lvwerra HF Staff
Sanitized PyBot message board archive
c1e4f50
Raw History Blame Contribute Delete
2.13 kB
"""Fetch only the source snapshot's 28 display files, with hash verification."""
import argparse
import hashlib
import json
import os
import pathlib
import sqlite3
from huggingface_hub import HfApi
from redaction import redactor_for_database
parser=argparse.ArgumentParser()
parser.add_argument('database')
args=parser.parse_args()
db=sqlite3.connect(pathlib.Path(args.database).resolve().as_uri()+'?mode=ro',uri=True)
db.row_factory=sqlite3.Row
redact = redactor_for_database(db)
rows=list(db.execute('SELECT s.*,p.external_id handle FROM storage_files s JOIN principals p ON p.id=s.owner_id'))
downloads=[]
for row in rows:
relative=pathlib.Path(str(row['owner_id']))/row['path']
if '..' in relative.parts or relative.is_absolute():raise ValueError('Invalid storage path')
target=pathlib.Path('assets/storage')/relative
target.parent.mkdir(parents=True,exist_ok=True)
downloads.append(('boards/chatbot-v8/storage/'+relative.as_posix(),target))
HfApi(token=os.environ.get('HF_TOKEN')).download_bucket_files('HuggingFaceSwarms/chatbot-msgboards-data',downloads,raise_on_missing_files=True)
catalog={}
for row,(_,target) in zip(rows,downloads):
content=target.read_bytes()
if hashlib.sha256(content).hexdigest()!=row['sha256']:raise ValueError('Stored file differs from snapshot')
# Some source text uploads were labelled as form data. Sanitize any UTF-8
# file, irrespective of its declared content type, after verifying it.
try:
text=content.decode('utf-8')
except UnicodeDecodeError:
text=None
if text is not None:
content=redact(text).encode('utf-8')
target.write_bytes(content)
catalog[row['handle']+'/'+row['path']]={'path':row['path'],'content_type':row['content_type'],'size':len(content),'sha256':hashlib.sha256(content).hexdigest(),'updated_at':row['updated_at'],'url':target.as_posix()}
path=pathlib.Path('data/manifest.json');manifest=json.loads(path.read_text());manifest['storage']=catalog
path.write_text(json.dumps(manifest,ensure_ascii=False,separators=(',',':')))
print(json.dumps({'files':len(catalog),'checksums_verified':True}))