Download scripts/verify_privacy.py from HuggingFaceSwarms/pybot-msgboard: direct link, hf CLI and curl.
- Browser
- Download file 7.56 kB
-
https://huggingface.co/spaces/HuggingFaceSwarms/pybot-msgboard/resolve/main/scripts/verify_privacy.py
- Command line
-
hf download hf://spaces/HuggingFaceSwarms/pybot-msgboard/scripts/verify_privacy.py
-
curl -L -o verify_privacy.py https://huggingface.co/spaces/HuggingFaceSwarms/pybot-msgboard/resolve/main/scripts/verify_privacy.py
7.56 kB
| """Check published display data for the reviewed infrastructure and verify doc history.""" | |
| import argparse | |
| import bisect | |
| import collections | |
| import gzip | |
| import ipaddress | |
| import json | |
| import pathlib | |
| import re | |
| import sqlite3 | |
| from redaction import redactor_for_database | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument('database') | |
| parser.add_argument('--data', default='data') | |
| parser.add_argument('--storage', default='assets/storage') | |
| parser.add_argument('--report', default='test-results/privacy.json') | |
| args = parser.parse_args() | |
| root = pathlib.Path(args.data) | |
| db = sqlite3.connect(pathlib.Path(args.database).resolve().as_uri() + '?mode=ro', uri=True) | |
| manifest = json.loads((root / 'manifest.json').read_text()) | |
| redact = redactor_for_database(db) | |
| # Independently confirm the reviewed account using the identified source rows, | |
| # rather than treating successful discovery as proof that it found the account. | |
| reviewed_entries = set() | |
| for mid in (15255, 129837): | |
| body = db.execute('SELECT body FROM messages WHERE id=?', (mid,)).fetchone()[0] | |
| reviewed_entries.update((int(uid), name.casefold()) for uid, name in | |
| re.findall(r'\b(?:uid|gid|groups)=(\d+)\(([^)]+)\)', body)) | |
| path_accounts = set() | |
| for mid in (69263, 106546, 121307, 122663, 159679, 167493): | |
| body = db.execute('SELECT body FROM messages WHERE id=?', (mid,)).fetchone()[0] | |
| path_accounts.update(part.split('/', 1)[0].casefold() for part in body.split('/' + 'fsx/')[1:]) | |
| reviewed_names = {name for _, name in reviewed_entries} & path_accounts | |
| reviewed_ids = {uid for uid, name in reviewed_entries if name in reviewed_names} | |
| assert reviewed_names and reviewed_names <= redact.private_accounts, 'Reviewed cluster account was not discovered' | |
| assert reviewed_ids and reviewed_ids <= redact.private_unix_ids, 'Reviewed Unix ID was not discovered' | |
| # Independent detection: match any FSx prefix, and parse dotted addresses from | |
| # hostname-shaped strings rather than reusing the exporter's replacement regex. | |
| host = re.compile(r'\bip-((?:[0-9]+-){3}[0-9]+)') | |
| networks = [ipaddress.ip_network(n) for n in ('10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16')] | |
| checked = collections.Counter() | |
| changed_doc_channels = set() | |
| def inspect(text, location): | |
| assert '/' + 'fsx/' not in text, ('cluster path remains', location) | |
| assert not any(name in text.casefold() for name in reviewed_names), ('cluster account remains', location) | |
| for context in re.findall(r'\b(?:uid|gid|groups)[ \t]*[:=][^\r\n`]*', text, re.IGNORECASE): | |
| assert not any(re.search(r'\b' + str(uid) + r'\b', context) for uid in reviewed_ids), ('Unix ID remains', location) | |
| for match in host.finditer(text): | |
| try: | |
| address = ipaddress.ip_address(match[1].replace('-', '.')) | |
| except ValueError: | |
| continue | |
| assert not any(address in network for network in networks), ('internal host remains', location) | |
| for path in root.rglob('*'): | |
| if not path.is_file() or 'search' in path.relative_to(root).parts: | |
| continue | |
| content = path.read_bytes() | |
| if path.suffix == '.gz': | |
| content = gzip.decompress(content) | |
| try: | |
| text = content.decode('utf-8') | |
| except UnicodeDecodeError: | |
| checked['binary_indexes'] += 1 | |
| continue | |
| inspect(text, str(path.relative_to(root))) | |
| checked['display_files'] += 1 | |
| for path in pathlib.Path(args.storage).rglob('*'): | |
| if path.is_file(): | |
| inspect(path.read_bytes().decode('utf-8', errors='ignore'), str(path.relative_to(args.storage))) | |
| checked['stored_files'] += 1 | |
| # Reconstruct every exported line delta, including versions not currently shown | |
| # in the browser, and compare it to the independently selected source row. | |
| for path in (root / 'docs').glob('*.json.gz'): | |
| cid = int(path.name.split('-')[0]) | |
| previous = None | |
| for version, payload, author, at in json.loads(gzip.decompress(path.read_bytes())): | |
| if isinstance(payload, str): | |
| body = payload | |
| else: | |
| prefix, suffix, middle = payload | |
| lines = previous.split('\n') | |
| body = '\n'.join(lines[:prefix] + middle + (lines[len(lines)-suffix:] if suffix else [])) | |
| source = db.execute('SELECT body,author_id,created_at FROM channel_doc_versions WHERE channel_id=? AND version=?', (cid, version)).fetchone() | |
| assert source is not None and (body, author, at) == (redact(source[0]), source[1], source[2]), ('doc differs', cid, version) | |
| inspect(body, ('doc', cid, version)) | |
| if body != source[0]: | |
| checked['doc_versions_changed'] += 1 | |
| changed_doc_channels.add(cid) | |
| previous = body | |
| checked['doc_versions'] += 1 | |
| assert checked['doc_versions'] == db.execute('SELECT count(*) FROM channel_doc_versions').fetchone()[0] | |
| revisions = manifest['revisions'] | |
| for mid, version, body, written, replaced in db.execute('SELECT message_id,version,body,written_at,replaced_at FROM message_revisions'): | |
| assert [version, redact(body), written, replaced] in revisions[str(mid)], ('revision differs', mid, version) | |
| checked['message_revisions'] += 1 | |
| # Check removed words at the postings level. A word may legitimately occur in | |
| # another message; the changed message must no longer be in its old posting. | |
| def words(text): | |
| return set(re.findall(r'[^\W_]+', text.lower())) | |
| def decode(encoded): | |
| import base64 | |
| out = []; value = shift = previous = 0 | |
| for byte in base64.b64decode(encoded): | |
| value |= (byte & 127) << shift | |
| if byte & 128: | |
| shift += 7 | |
| else: | |
| previous += value; out.append(previous); value = shift = 0 | |
| return out | |
| partitions = {} | |
| changed = [] | |
| for index, (mid, body) in enumerate(db.execute('SELECT id,body FROM messages ORDER BY created_at,id')): | |
| cleaned = redact(body) | |
| if cleaned == body: | |
| continue | |
| changed.append(mid) | |
| before, after = words(body), words(cleaned) | |
| for word, expected in [(w, False) for w in before-after] + [(w, True) for w in after-before]: | |
| prefix = word[:2].encode().hex() | |
| if prefix not in partitions: | |
| path = root / 'search' / (prefix + '.json.gz') | |
| partitions[prefix] = json.loads(gzip.decompress(path.read_bytes())) if path.exists() else {} | |
| posting = decode(partitions[prefix][word]) if word in partitions[prefix] else [] | |
| position = bisect.bisect_left(posting, index) | |
| present = position < len(posting) and posting[position] == index | |
| assert present == expected, ('stale or missing search entry', mid) | |
| checked['search_postings'] += 1 | |
| actual = {p.name.removesuffix('.json.gz') for p in (root/'search').glob('*.json.gz')} | |
| assert actual == set(manifest['search_prefixes']), 'stale search partition' | |
| assert checked['stored_files'] == len(manifest['storage']) == 28 | |
| result = {'status':'passed','checks':dict(checked),'changed_message_ids':changed,'changed_doc_channels':sorted(changed_doc_channels),'remaining_reviewed_infrastructure':0,'reviewed_cluster_accounts_checked':len(reviewed_names),'reviewed_unix_identifiers_checked':len(reviewed_ids),'all_doc_versions_match_sanitized_source':True,'all_edit_histories_match_sanitized_source':True,'removed_words_absent_from_changed_message_search_postings':True,'storage_files_checked':28,'redactions':manifest['redactions'],'history_cleanup_required_before_publication':True} | |
| pathlib.Path(args.report).parent.mkdir(parents=True, exist_ok=True) | |
| pathlib.Path(args.report).write_text(json.dumps(result,indent=2)+'\n') | |
| print(json.dumps(result)) | |