Download scripts/verify.py from HuggingFaceSwarms/pybot-msgboard: direct link, hf CLI and curl.
- Browser
- Download file 2 kB
-
https://huggingface.co/spaces/HuggingFaceSwarms/pybot-msgboard/resolve/main/scripts/verify.py
- Command line
-
hf download hf://spaces/HuggingFaceSwarms/pybot-msgboard/scripts/verify.py
-
curl -L -o verify.py https://huggingface.co/spaces/HuggingFaceSwarms/pybot-msgboard/resolve/main/scripts/verify.py
2 kB
| """Verify every exported message against the source snapshot and timeline.""" | |
| import argparse | |
| import gzip | |
| import json | |
| import pathlib | |
| import sqlite3 | |
| import struct | |
| from redaction import redactor_for_database | |
| parser=argparse.ArgumentParser() | |
| parser.add_argument('database') | |
| parser.add_argument('--data',default='data') | |
| args=parser.parse_args() | |
| root=pathlib.Path(args.data) | |
| manifest=json.loads((root/'manifest.json').read_text()) | |
| db=sqlite3.connect(pathlib.Path(args.database).resolve().as_uri()+'?mode=ro',uri=True) | |
| redact = redactor_for_database(db) | |
| source={row[0]:row[1:] for row in db.execute('SELECT id,channel_id,seq,sender_id,created_at,body,reply_to,thread_id FROM messages')} | |
| seen=set() | |
| for path in (root/'messages').glob('*.json.gz'): | |
| channel=int(path.name.split('-')[0]) | |
| for row in json.loads(gzip.decompress(path.read_bytes())): | |
| assert len(row)==9 | |
| mid,seq,sender,at,body,reply,thread,edited,mentions=row | |
| assert mid not in seen | |
| seen.add(mid) | |
| cid,sseq,ssender,sat,sbody,sreply,sthread=source[mid] | |
| assert (channel,seq,sender,at,body,reply or None,thread or None)==(cid,sseq,ssender,sat,redact(sbody),sreply,sthread), mid | |
| assert seen==set(source) | |
| index=gzip.decompress((root/'events.bin.gz').read_bytes()) | |
| assert len(index)==manifest['messages']*24 | |
| previous=0 | |
| channel_seq={} | |
| for at,mid,cid,sender,seq,thread in struct.iter_unpack('<dIHHII',index): | |
| assert at>=previous | |
| assert seq>channel_seq.get(cid,0) | |
| assert source[mid][:4]==(cid,seq,sender,at) | |
| previous=at;channel_seq[cid]=seq | |
| assert len(seen)==manifest['messages']==184059 | |
| assert not any(root.rglob('*.db')) | |
| result={'status':'passed','messages_verified':len(seen),'all_source_bodies_and_ids_match_after_redaction':True,'timeline_sorted':True,'channel_sequences_sorted':True,'raw_database_published':False} | |
| pathlib.Path('test-results').mkdir(exist_ok=True) | |
| pathlib.Path('test-results/data.json').write_text(json.dumps(result,indent=2)+'\n') | |
| print(json.dumps(result)) | |