File size: 1,996 Bytes
c1e4f50 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 | """Verify every exported message against the source snapshot and timeline."""
import argparse
import gzip
import json
import pathlib
import sqlite3
import struct
from redaction import redactor_for_database
parser=argparse.ArgumentParser()
parser.add_argument('database')
parser.add_argument('--data',default='data')
args=parser.parse_args()
root=pathlib.Path(args.data)
manifest=json.loads((root/'manifest.json').read_text())
db=sqlite3.connect(pathlib.Path(args.database).resolve().as_uri()+'?mode=ro',uri=True)
redact = redactor_for_database(db)
source={row[0]:row[1:] for row in db.execute('SELECT id,channel_id,seq,sender_id,created_at,body,reply_to,thread_id FROM messages')}
seen=set()
for path in (root/'messages').glob('*.json.gz'):
channel=int(path.name.split('-')[0])
for row in json.loads(gzip.decompress(path.read_bytes())):
assert len(row)==9
mid,seq,sender,at,body,reply,thread,edited,mentions=row
assert mid not in seen
seen.add(mid)
cid,sseq,ssender,sat,sbody,sreply,sthread=source[mid]
assert (channel,seq,sender,at,body,reply or None,thread or None)==(cid,sseq,ssender,sat,redact(sbody),sreply,sthread), mid
assert seen==set(source)
index=gzip.decompress((root/'events.bin.gz').read_bytes())
assert len(index)==manifest['messages']*24
previous=0
channel_seq={}
for at,mid,cid,sender,seq,thread in struct.iter_unpack('<dIHHII',index):
assert at>=previous
assert seq>channel_seq.get(cid,0)
assert source[mid][:4]==(cid,seq,sender,at)
previous=at;channel_seq[cid]=seq
assert len(seen)==manifest['messages']==184059
assert not any(root.rglob('*.db'))
result={'status':'passed','messages_verified':len(seen),'all_source_bodies_and_ids_match_after_redaction':True,'timeline_sorted':True,'channel_sequences_sorted':True,'raw_database_published':False}
pathlib.Path('test-results').mkdir(exist_ok=True)
pathlib.Path('test-results/data.json').write_text(json.dumps(result,indent=2)+'\n')
print(json.dumps(result))
|