"""Verify every exported message against the source snapshot and timeline.""" import argparse import gzip import json import pathlib import sqlite3 import struct from redaction import redactor_for_database parser=argparse.ArgumentParser() parser.add_argument('database') parser.add_argument('--data',default='data') args=parser.parse_args() root=pathlib.Path(args.data) manifest=json.loads((root/'manifest.json').read_text()) db=sqlite3.connect(pathlib.Path(args.database).resolve().as_uri()+'?mode=ro',uri=True) redact = redactor_for_database(db) source={row[0]:row[1:] for row in db.execute('SELECT id,channel_id,seq,sender_id,created_at,body,reply_to,thread_id FROM messages')} seen=set() for path in (root/'messages').glob('*.json.gz'): channel=int(path.name.split('-')[0]) for row in json.loads(gzip.decompress(path.read_bytes())): assert len(row)==9 mid,seq,sender,at,body,reply,thread,edited,mentions=row assert mid not in seen seen.add(mid) cid,sseq,ssender,sat,sbody,sreply,sthread=source[mid] assert (channel,seq,sender,at,body,reply or None,thread or None)==(cid,sseq,ssender,sat,redact(sbody),sreply,sthread), mid assert seen==set(source) index=gzip.decompress((root/'events.bin.gz').read_bytes()) assert len(index)==manifest['messages']*24 previous=0 channel_seq={} for at,mid,cid,sender,seq,thread in struct.iter_unpack('=previous assert seq>channel_seq.get(cid,0) assert source[mid][:4]==(cid,seq,sender,at) previous=at;channel_seq[cid]=seq assert len(seen)==manifest['messages']==184059 assert not any(root.rglob('*.db')) result={'status':'passed','messages_verified':len(seen),'all_source_bodies_and_ids_match_after_redaction':True,'timeline_sorted':True,'channel_sequences_sorted':True,'raw_database_published':False} pathlib.Path('test-results').mkdir(exist_ok=True) pathlib.Path('test-results/data.json').write_text(json.dumps(result,indent=2)+'\n') print(json.dumps(result))