File size: 1,996 Bytes
c1e4f50
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
"""Verify every exported message against the source snapshot and timeline."""
import argparse
import gzip
import json
import pathlib
import sqlite3
import struct
from redaction import redactor_for_database

parser=argparse.ArgumentParser()
parser.add_argument('database')
parser.add_argument('--data',default='data')
args=parser.parse_args()
root=pathlib.Path(args.data)
manifest=json.loads((root/'manifest.json').read_text())
db=sqlite3.connect(pathlib.Path(args.database).resolve().as_uri()+'?mode=ro',uri=True)
redact = redactor_for_database(db)
source={row[0]:row[1:] for row in db.execute('SELECT id,channel_id,seq,sender_id,created_at,body,reply_to,thread_id FROM messages')}
seen=set()
for path in (root/'messages').glob('*.json.gz'):
    channel=int(path.name.split('-')[0])
    for row in json.loads(gzip.decompress(path.read_bytes())):
        assert len(row)==9
        mid,seq,sender,at,body,reply,thread,edited,mentions=row
        assert mid not in seen
        seen.add(mid)
        cid,sseq,ssender,sat,sbody,sreply,sthread=source[mid]
        assert (channel,seq,sender,at,body,reply or None,thread or None)==(cid,sseq,ssender,sat,redact(sbody),sreply,sthread), mid
assert seen==set(source)
index=gzip.decompress((root/'events.bin.gz').read_bytes())
assert len(index)==manifest['messages']*24
previous=0
channel_seq={}
for at,mid,cid,sender,seq,thread in struct.iter_unpack('<dIHHII',index):
    assert at>=previous
    assert seq>channel_seq.get(cid,0)
    assert source[mid][:4]==(cid,seq,sender,at)
    previous=at;channel_seq[cid]=seq
assert len(seen)==manifest['messages']==184059
assert not any(root.rglob('*.db'))
result={'status':'passed','messages_verified':len(seen),'all_source_bodies_and_ids_match_after_redaction':True,'timeline_sorted':True,'channel_sequences_sorted':True,'raw_database_published':False}
pathlib.Path('test-results').mkdir(exist_ok=True)
pathlib.Path('test-results/data.json').write_text(json.dumps(result,indent=2)+'\n')
print(json.dumps(result))