File size: 2,134 Bytes
c1e4f50
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
"""Fetch only the source snapshot's 28 display files, with hash verification."""
import argparse
import hashlib
import json
import os
import pathlib
import sqlite3
from huggingface_hub import HfApi
from redaction import redactor_for_database

parser=argparse.ArgumentParser()
parser.add_argument('database')
args=parser.parse_args()
db=sqlite3.connect(pathlib.Path(args.database).resolve().as_uri()+'?mode=ro',uri=True)
db.row_factory=sqlite3.Row
redact = redactor_for_database(db)
rows=list(db.execute('SELECT s.*,p.external_id handle FROM storage_files s JOIN principals p ON p.id=s.owner_id'))
downloads=[]
for row in rows:
    relative=pathlib.Path(str(row['owner_id']))/row['path']
    if '..' in relative.parts or relative.is_absolute():raise ValueError('Invalid storage path')
    target=pathlib.Path('assets/storage')/relative
    target.parent.mkdir(parents=True,exist_ok=True)
    downloads.append(('boards/chatbot-v8/storage/'+relative.as_posix(),target))
HfApi(token=os.environ.get('HF_TOKEN')).download_bucket_files('HuggingFaceSwarms/chatbot-msgboards-data',downloads,raise_on_missing_files=True)
catalog={}
for row,(_,target) in zip(rows,downloads):
    content=target.read_bytes()
    if hashlib.sha256(content).hexdigest()!=row['sha256']:raise ValueError('Stored file differs from snapshot')
    # Some source text uploads were labelled as form data. Sanitize any UTF-8
    # file, irrespective of its declared content type, after verifying it.
    try:
        text=content.decode('utf-8')
    except UnicodeDecodeError:
        text=None
    if text is not None:
        content=redact(text).encode('utf-8')
        target.write_bytes(content)
    catalog[row['handle']+'/'+row['path']]={'path':row['path'],'content_type':row['content_type'],'size':len(content),'sha256':hashlib.sha256(content).hexdigest(),'updated_at':row['updated_at'],'url':target.as_posix()}
path=pathlib.Path('data/manifest.json');manifest=json.loads(path.read_text());manifest['storage']=catalog
path.write_text(json.dumps(manifest,ensure_ascii=False,separators=(',',':')))
print(json.dumps({'files':len(catalog),'checksums_verified':True}))