"""Check published display data for the reviewed infrastructure and verify doc history.""" import argparse import bisect import collections import gzip import ipaddress import json import pathlib import re import sqlite3 from redaction import redactor_for_database parser = argparse.ArgumentParser() parser.add_argument('database') parser.add_argument('--data', default='data') parser.add_argument('--storage', default='assets/storage') parser.add_argument('--report', default='test-results/privacy.json') args = parser.parse_args() root = pathlib.Path(args.data) db = sqlite3.connect(pathlib.Path(args.database).resolve().as_uri() + '?mode=ro', uri=True) manifest = json.loads((root / 'manifest.json').read_text()) redact = redactor_for_database(db) # Independently confirm the reviewed account using the identified source rows, # rather than treating successful discovery as proof that it found the account. reviewed_entries = set() for mid in (15255, 129837): body = db.execute('SELECT body FROM messages WHERE id=?', (mid,)).fetchone()[0] reviewed_entries.update((int(uid), name.casefold()) for uid, name in re.findall(r'\b(?:uid|gid|groups)=(\d+)\(([^)]+)\)', body)) path_accounts = set() for mid in (69263, 106546, 121307, 122663, 159679, 167493): body = db.execute('SELECT body FROM messages WHERE id=?', (mid,)).fetchone()[0] path_accounts.update(part.split('/', 1)[0].casefold() for part in body.split('/' + 'fsx/')[1:]) reviewed_names = {name for _, name in reviewed_entries} & path_accounts reviewed_ids = {uid for uid, name in reviewed_entries if name in reviewed_names} assert reviewed_names and reviewed_names <= redact.private_accounts, 'Reviewed cluster account was not discovered' assert reviewed_ids and reviewed_ids <= redact.private_unix_ids, 'Reviewed Unix ID was not discovered' # Independent detection: match any FSx prefix, and parse dotted addresses from # hostname-shaped strings rather than reusing the exporter's replacement regex. host = re.compile(r'\bip-((?:[0-9]+-){3}[0-9]+)') networks = [ipaddress.ip_network(n) for n in ('10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16')] checked = collections.Counter() changed_doc_channels = set() def inspect(text, location): assert '/' + 'fsx/' not in text, ('cluster path remains', location) assert not any(name in text.casefold() for name in reviewed_names), ('cluster account remains', location) for context in re.findall(r'\b(?:uid|gid|groups)[ \t]*[:=][^\r\n`]*', text, re.IGNORECASE): assert not any(re.search(r'\b' + str(uid) + r'\b', context) for uid in reviewed_ids), ('Unix ID remains', location) for match in host.finditer(text): try: address = ipaddress.ip_address(match[1].replace('-', '.')) except ValueError: continue assert not any(address in network for network in networks), ('internal host remains', location) for path in root.rglob('*'): if not path.is_file() or 'search' in path.relative_to(root).parts: continue content = path.read_bytes() if path.suffix == '.gz': content = gzip.decompress(content) try: text = content.decode('utf-8') except UnicodeDecodeError: checked['binary_indexes'] += 1 continue inspect(text, str(path.relative_to(root))) checked['display_files'] += 1 for path in pathlib.Path(args.storage).rglob('*'): if path.is_file(): inspect(path.read_bytes().decode('utf-8', errors='ignore'), str(path.relative_to(args.storage))) checked['stored_files'] += 1 # Reconstruct every exported line delta, including versions not currently shown # in the browser, and compare it to the independently selected source row. for path in (root / 'docs').glob('*.json.gz'): cid = int(path.name.split('-')[0]) previous = None for version, payload, author, at in json.loads(gzip.decompress(path.read_bytes())): if isinstance(payload, str): body = payload else: prefix, suffix, middle = payload lines = previous.split('\n') body = '\n'.join(lines[:prefix] + middle + (lines[len(lines)-suffix:] if suffix else [])) source = db.execute('SELECT body,author_id,created_at FROM channel_doc_versions WHERE channel_id=? AND version=?', (cid, version)).fetchone() assert source is not None and (body, author, at) == (redact(source[0]), source[1], source[2]), ('doc differs', cid, version) inspect(body, ('doc', cid, version)) if body != source[0]: checked['doc_versions_changed'] += 1 changed_doc_channels.add(cid) previous = body checked['doc_versions'] += 1 assert checked['doc_versions'] == db.execute('SELECT count(*) FROM channel_doc_versions').fetchone()[0] revisions = manifest['revisions'] for mid, version, body, written, replaced in db.execute('SELECT message_id,version,body,written_at,replaced_at FROM message_revisions'): assert [version, redact(body), written, replaced] in revisions[str(mid)], ('revision differs', mid, version) checked['message_revisions'] += 1 # Check removed words at the postings level. A word may legitimately occur in # another message; the changed message must no longer be in its old posting. def words(text): return set(re.findall(r'[^\W_]+', text.lower())) def decode(encoded): import base64 out = []; value = shift = previous = 0 for byte in base64.b64decode(encoded): value |= (byte & 127) << shift if byte & 128: shift += 7 else: previous += value; out.append(previous); value = shift = 0 return out partitions = {} changed = [] for index, (mid, body) in enumerate(db.execute('SELECT id,body FROM messages ORDER BY created_at,id')): cleaned = redact(body) if cleaned == body: continue changed.append(mid) before, after = words(body), words(cleaned) for word, expected in [(w, False) for w in before-after] + [(w, True) for w in after-before]: prefix = word[:2].encode().hex() if prefix not in partitions: path = root / 'search' / (prefix + '.json.gz') partitions[prefix] = json.loads(gzip.decompress(path.read_bytes())) if path.exists() else {} posting = decode(partitions[prefix][word]) if word in partitions[prefix] else [] position = bisect.bisect_left(posting, index) present = position < len(posting) and posting[position] == index assert present == expected, ('stale or missing search entry', mid) checked['search_postings'] += 1 actual = {p.name.removesuffix('.json.gz') for p in (root/'search').glob('*.json.gz')} assert actual == set(manifest['search_prefixes']), 'stale search partition' assert checked['stored_files'] == len(manifest['storage']) == 28 result = {'status':'passed','checks':dict(checked),'changed_message_ids':changed,'changed_doc_channels':sorted(changed_doc_channels),'remaining_reviewed_infrastructure':0,'reviewed_cluster_accounts_checked':len(reviewed_names),'reviewed_unix_identifiers_checked':len(reviewed_ids),'all_doc_versions_match_sanitized_source':True,'all_edit_histories_match_sanitized_source':True,'removed_words_absent_from_changed_message_search_postings':True,'storage_files_checked':28,'redactions':manifest['redactions'],'history_cleanup_required_before_publication':True} pathlib.Path(args.report).parent.mkdir(parents=True, exist_ok=True) pathlib.Path(args.report).write_text(json.dumps(result,indent=2)+'\n') print(json.dumps(result))