File size: 7,560 Bytes
c1e4f50
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
"""Check published display data for the reviewed infrastructure and verify doc history."""
import argparse
import bisect
import collections
import gzip
import ipaddress
import json
import pathlib
import re
import sqlite3
from redaction import redactor_for_database

parser = argparse.ArgumentParser()
parser.add_argument('database')
parser.add_argument('--data', default='data')
parser.add_argument('--storage', default='assets/storage')
parser.add_argument('--report', default='test-results/privacy.json')
args = parser.parse_args()
root = pathlib.Path(args.data)
db = sqlite3.connect(pathlib.Path(args.database).resolve().as_uri() + '?mode=ro', uri=True)
manifest = json.loads((root / 'manifest.json').read_text())
redact = redactor_for_database(db)
# Independently confirm the reviewed account using the identified source rows,
# rather than treating successful discovery as proof that it found the account.
reviewed_entries = set()
for mid in (15255, 129837):
    body = db.execute('SELECT body FROM messages WHERE id=?', (mid,)).fetchone()[0]
    reviewed_entries.update((int(uid), name.casefold()) for uid, name in
                            re.findall(r'\b(?:uid|gid|groups)=(\d+)\(([^)]+)\)', body))
path_accounts = set()
for mid in (69263, 106546, 121307, 122663, 159679, 167493):
    body = db.execute('SELECT body FROM messages WHERE id=?', (mid,)).fetchone()[0]
    path_accounts.update(part.split('/', 1)[0].casefold() for part in body.split('/' + 'fsx/')[1:])
reviewed_names = {name for _, name in reviewed_entries} & path_accounts
reviewed_ids = {uid for uid, name in reviewed_entries if name in reviewed_names}
assert reviewed_names and reviewed_names <= redact.private_accounts, 'Reviewed cluster account was not discovered'
assert reviewed_ids and reviewed_ids <= redact.private_unix_ids, 'Reviewed Unix ID was not discovered'
# Independent detection: match any FSx prefix, and parse dotted addresses from
# hostname-shaped strings rather than reusing the exporter's replacement regex.
host = re.compile(r'\bip-((?:[0-9]+-){3}[0-9]+)')
networks = [ipaddress.ip_network(n) for n in ('10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16')]
checked = collections.Counter()
changed_doc_channels = set()


def inspect(text, location):
    assert '/' + 'fsx/' not in text, ('cluster path remains', location)
    assert not any(name in text.casefold() for name in reviewed_names), ('cluster account remains', location)
    for context in re.findall(r'\b(?:uid|gid|groups)[ \t]*[:=][^\r\n`]*', text, re.IGNORECASE):
        assert not any(re.search(r'\b' + str(uid) + r'\b', context) for uid in reviewed_ids), ('Unix ID remains', location)
    for match in host.finditer(text):
        try:
            address = ipaddress.ip_address(match[1].replace('-', '.'))
        except ValueError:
            continue
        assert not any(address in network for network in networks), ('internal host remains', location)


for path in root.rglob('*'):
    if not path.is_file() or 'search' in path.relative_to(root).parts:
        continue
    content = path.read_bytes()
    if path.suffix == '.gz':
        content = gzip.decompress(content)
    try:
        text = content.decode('utf-8')
    except UnicodeDecodeError:
        checked['binary_indexes'] += 1
        continue
    inspect(text, str(path.relative_to(root)))
    checked['display_files'] += 1
for path in pathlib.Path(args.storage).rglob('*'):
    if path.is_file():
        inspect(path.read_bytes().decode('utf-8', errors='ignore'), str(path.relative_to(args.storage)))
        checked['stored_files'] += 1

# Reconstruct every exported line delta, including versions not currently shown
# in the browser, and compare it to the independently selected source row.
for path in (root / 'docs').glob('*.json.gz'):
    cid = int(path.name.split('-')[0])
    previous = None
    for version, payload, author, at in json.loads(gzip.decompress(path.read_bytes())):
        if isinstance(payload, str):
            body = payload
        else:
            prefix, suffix, middle = payload
            lines = previous.split('\n')
            body = '\n'.join(lines[:prefix] + middle + (lines[len(lines)-suffix:] if suffix else []))
        source = db.execute('SELECT body,author_id,created_at FROM channel_doc_versions WHERE channel_id=? AND version=?', (cid, version)).fetchone()
        assert source is not None and (body, author, at) == (redact(source[0]), source[1], source[2]), ('doc differs', cid, version)
        inspect(body, ('doc', cid, version))
        if body != source[0]:
            checked['doc_versions_changed'] += 1
            changed_doc_channels.add(cid)
        previous = body
        checked['doc_versions'] += 1
assert checked['doc_versions'] == db.execute('SELECT count(*) FROM channel_doc_versions').fetchone()[0]

revisions = manifest['revisions']
for mid, version, body, written, replaced in db.execute('SELECT message_id,version,body,written_at,replaced_at FROM message_revisions'):
    assert [version, redact(body), written, replaced] in revisions[str(mid)], ('revision differs', mid, version)
    checked['message_revisions'] += 1

# Check removed words at the postings level. A word may legitimately occur in
# another message; the changed message must no longer be in its old posting.
def words(text):
    return set(re.findall(r'[^\W_]+', text.lower()))


def decode(encoded):
    import base64
    out = []; value = shift = previous = 0
    for byte in base64.b64decode(encoded):
        value |= (byte & 127) << shift
        if byte & 128:
            shift += 7
        else:
            previous += value; out.append(previous); value = shift = 0
    return out


partitions = {}
changed = []
for index, (mid, body) in enumerate(db.execute('SELECT id,body FROM messages ORDER BY created_at,id')):
    cleaned = redact(body)
    if cleaned == body:
        continue
    changed.append(mid)
    before, after = words(body), words(cleaned)
    for word, expected in [(w, False) for w in before-after] + [(w, True) for w in after-before]:
        prefix = word[:2].encode().hex()
        if prefix not in partitions:
            path = root / 'search' / (prefix + '.json.gz')
            partitions[prefix] = json.loads(gzip.decompress(path.read_bytes())) if path.exists() else {}
        posting = decode(partitions[prefix][word]) if word in partitions[prefix] else []
        position = bisect.bisect_left(posting, index)
        present = position < len(posting) and posting[position] == index
        assert present == expected, ('stale or missing search entry', mid)
        checked['search_postings'] += 1
actual = {p.name.removesuffix('.json.gz') for p in (root/'search').glob('*.json.gz')}
assert actual == set(manifest['search_prefixes']), 'stale search partition'
assert checked['stored_files'] == len(manifest['storage']) == 28
result = {'status':'passed','checks':dict(checked),'changed_message_ids':changed,'changed_doc_channels':sorted(changed_doc_channels),'remaining_reviewed_infrastructure':0,'reviewed_cluster_accounts_checked':len(reviewed_names),'reviewed_unix_identifiers_checked':len(reviewed_ids),'all_doc_versions_match_sanitized_source':True,'all_edit_histories_match_sanitized_source':True,'removed_words_absent_from_changed_message_search_postings':True,'storage_files_checked':28,'redactions':manifest['redactions'],'history_cleanup_required_before_publication':True}
pathlib.Path(args.report).parent.mkdir(parents=True, exist_ok=True)
pathlib.Path(args.report).write_text(json.dumps(result,indent=2)+'\n')
print(json.dumps(result))