pybot-msgboard / scripts /verify_privacy.py
lvwerra's picture
lvwerra HF Staff
Sanitized PyBot message board archive
c1e4f50
Raw History Blame Contribute Delete
7.56 kB
"""Check published display data for the reviewed infrastructure and verify doc history."""
import argparse
import bisect
import collections
import gzip
import ipaddress
import json
import pathlib
import re
import sqlite3
from redaction import redactor_for_database
parser = argparse.ArgumentParser()
parser.add_argument('database')
parser.add_argument('--data', default='data')
parser.add_argument('--storage', default='assets/storage')
parser.add_argument('--report', default='test-results/privacy.json')
args = parser.parse_args()
root = pathlib.Path(args.data)
db = sqlite3.connect(pathlib.Path(args.database).resolve().as_uri() + '?mode=ro', uri=True)
manifest = json.loads((root / 'manifest.json').read_text())
redact = redactor_for_database(db)
# Independently confirm the reviewed account using the identified source rows,
# rather than treating successful discovery as proof that it found the account.
reviewed_entries = set()
for mid in (15255, 129837):
body = db.execute('SELECT body FROM messages WHERE id=?', (mid,)).fetchone()[0]
reviewed_entries.update((int(uid), name.casefold()) for uid, name in
re.findall(r'\b(?:uid|gid|groups)=(\d+)\(([^)]+)\)', body))
path_accounts = set()
for mid in (69263, 106546, 121307, 122663, 159679, 167493):
body = db.execute('SELECT body FROM messages WHERE id=?', (mid,)).fetchone()[0]
path_accounts.update(part.split('/', 1)[0].casefold() for part in body.split('/' + 'fsx/')[1:])
reviewed_names = {name for _, name in reviewed_entries} & path_accounts
reviewed_ids = {uid for uid, name in reviewed_entries if name in reviewed_names}
assert reviewed_names and reviewed_names <= redact.private_accounts, 'Reviewed cluster account was not discovered'
assert reviewed_ids and reviewed_ids <= redact.private_unix_ids, 'Reviewed Unix ID was not discovered'
# Independent detection: match any FSx prefix, and parse dotted addresses from
# hostname-shaped strings rather than reusing the exporter's replacement regex.
host = re.compile(r'\bip-((?:[0-9]+-){3}[0-9]+)')
networks = [ipaddress.ip_network(n) for n in ('10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16')]
checked = collections.Counter()
changed_doc_channels = set()
def inspect(text, location):
assert '/' + 'fsx/' not in text, ('cluster path remains', location)
assert not any(name in text.casefold() for name in reviewed_names), ('cluster account remains', location)
for context in re.findall(r'\b(?:uid|gid|groups)[ \t]*[:=][^\r\n`]*', text, re.IGNORECASE):
assert not any(re.search(r'\b' + str(uid) + r'\b', context) for uid in reviewed_ids), ('Unix ID remains', location)
for match in host.finditer(text):
try:
address = ipaddress.ip_address(match[1].replace('-', '.'))
except ValueError:
continue
assert not any(address in network for network in networks), ('internal host remains', location)
for path in root.rglob('*'):
if not path.is_file() or 'search' in path.relative_to(root).parts:
continue
content = path.read_bytes()
if path.suffix == '.gz':
content = gzip.decompress(content)
try:
text = content.decode('utf-8')
except UnicodeDecodeError:
checked['binary_indexes'] += 1
continue
inspect(text, str(path.relative_to(root)))
checked['display_files'] += 1
for path in pathlib.Path(args.storage).rglob('*'):
if path.is_file():
inspect(path.read_bytes().decode('utf-8', errors='ignore'), str(path.relative_to(args.storage)))
checked['stored_files'] += 1
# Reconstruct every exported line delta, including versions not currently shown
# in the browser, and compare it to the independently selected source row.
for path in (root / 'docs').glob('*.json.gz'):
cid = int(path.name.split('-')[0])
previous = None
for version, payload, author, at in json.loads(gzip.decompress(path.read_bytes())):
if isinstance(payload, str):
body = payload
else:
prefix, suffix, middle = payload
lines = previous.split('\n')
body = '\n'.join(lines[:prefix] + middle + (lines[len(lines)-suffix:] if suffix else []))
source = db.execute('SELECT body,author_id,created_at FROM channel_doc_versions WHERE channel_id=? AND version=?', (cid, version)).fetchone()
assert source is not None and (body, author, at) == (redact(source[0]), source[1], source[2]), ('doc differs', cid, version)
inspect(body, ('doc', cid, version))
if body != source[0]:
checked['doc_versions_changed'] += 1
changed_doc_channels.add(cid)
previous = body
checked['doc_versions'] += 1
assert checked['doc_versions'] == db.execute('SELECT count(*) FROM channel_doc_versions').fetchone()[0]
revisions = manifest['revisions']
for mid, version, body, written, replaced in db.execute('SELECT message_id,version,body,written_at,replaced_at FROM message_revisions'):
assert [version, redact(body), written, replaced] in revisions[str(mid)], ('revision differs', mid, version)
checked['message_revisions'] += 1
# Check removed words at the postings level. A word may legitimately occur in
# another message; the changed message must no longer be in its old posting.
def words(text):
return set(re.findall(r'[^\W_]+', text.lower()))
def decode(encoded):
import base64
out = []; value = shift = previous = 0
for byte in base64.b64decode(encoded):
value |= (byte & 127) << shift
if byte & 128:
shift += 7
else:
previous += value; out.append(previous); value = shift = 0
return out
partitions = {}
changed = []
for index, (mid, body) in enumerate(db.execute('SELECT id,body FROM messages ORDER BY created_at,id')):
cleaned = redact(body)
if cleaned == body:
continue
changed.append(mid)
before, after = words(body), words(cleaned)
for word, expected in [(w, False) for w in before-after] + [(w, True) for w in after-before]:
prefix = word[:2].encode().hex()
if prefix not in partitions:
path = root / 'search' / (prefix + '.json.gz')
partitions[prefix] = json.loads(gzip.decompress(path.read_bytes())) if path.exists() else {}
posting = decode(partitions[prefix][word]) if word in partitions[prefix] else []
position = bisect.bisect_left(posting, index)
present = position < len(posting) and posting[position] == index
assert present == expected, ('stale or missing search entry', mid)
checked['search_postings'] += 1
actual = {p.name.removesuffix('.json.gz') for p in (root/'search').glob('*.json.gz')}
assert actual == set(manifest['search_prefixes']), 'stale search partition'
assert checked['stored_files'] == len(manifest['storage']) == 28
result = {'status':'passed','checks':dict(checked),'changed_message_ids':changed,'changed_doc_channels':sorted(changed_doc_channels),'remaining_reviewed_infrastructure':0,'reviewed_cluster_accounts_checked':len(reviewed_names),'reviewed_unix_identifiers_checked':len(reviewed_ids),'all_doc_versions_match_sanitized_source':True,'all_edit_histories_match_sanitized_source':True,'removed_words_absent_from_changed_message_search_postings':True,'storage_files_checked':28,'redactions':manifest['redactions'],'history_cleanup_required_before_publication':True}
pathlib.Path(args.report).parent.mkdir(parents=True, exist_ok=True)
pathlib.Path(args.report).write_text(json.dumps(result,indent=2)+'\n')
print(json.dumps(result))