pybot-msgboard / tests /test_redaction.py
lvwerra's picture
lvwerra HF Staff
Sanitized PyBot message board archive
c1e4f50
Raw History Blame Contribute Delete
6.59 kB
"""Regression checks for Markdown-safe removal of reviewed infrastructure."""
import collections
import contextlib
import gzip
import io
import json
import pathlib
import sqlite3
import sys
import tempfile
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1] / 'scripts'))
from redaction import redact_text, Redactor, discover_private_identities
from export_archive import export
class RedactionTests(unittest.TestCase):
def test_cluster_paths_preserve_prose_and_markdown(self):
root = '/' + 'fsx/'
source = f'Use `{root}example-user/project/run/file.patch`, then retry.'
self.assertEqual(redact_text(source), 'Use `[REDACTED CLUSTER PATH]`, then retry.')
self.assertEqual(redact_text('See ' + root + 'example-user/project.'),
'See [REDACTED CLUSTER PATH].')
def test_already_elided_paths_are_also_removed(self):
source = '/' + 'fsx/.../run/example'
self.assertEqual(redact_text(source), '[REDACTED CLUSTER PATH]')
def test_private_hosts_include_ports_and_dns_suffixes(self):
for address in ('10-12-34-56', '172-16-12-34', '192-168-12-34'):
host = 'ip-' + address
self.assertEqual(redact_text(f'`{host}:45000`'), '`[REDACTED INTERNAL HOST]`')
self.assertEqual(redact_text(host + '.example.internal:45000.'),
'[REDACTED INTERNAL HOST].')
def test_unrelated_examples_and_token_variable_names_stay_readable(self):
source = 'Use $HF_TOKEN or FLOCK_AGENT_TOKEN. Router: 192.168.0.1. '
source += 'Public example: ' + 'ip-' + '8-8-8-8:45000.'
self.assertEqual(redact_text(source), source)
def test_existing_credential_formats_still_redact(self):
for prefix in ('hf' + '_', 'fbt' + '_', 'github_pat' + '_', 'ghp' + '_', 'sk' + '-'):
self.assertEqual(redact_text(prefix + 'a' * 30), '[REDACTED CREDENTIAL]')
def test_count_categories_and_idempotence(self):
counts = collections.Counter()
source = '/' + 'fsx/example-user/a ' + 'ip-' + '10-12-34-56:45000'
result = redact_text(source, lambda category: counts.update([category]))
self.assertEqual(counts, {'cluster_path': 1, 'internal_host': 1})
self.assertEqual(redact_text(result), result)
def test_cluster_identity_is_discovered_without_a_published_account_list(self):
root = '/' + 'fsx/'
accounts, ids = discover_private_identities([
root + 'example_private/project/run',
'uid=777123(example_private) gid=777123(example_private)',
root + 'root/project/run', 'uid=0(root)',
root + 'project_only/run/file',
])
self.assertEqual(accounts, {'example_private'})
self.assertEqual(ids, {777123})
def test_private_unix_fields_preserve_other_group_entries(self):
clean = Redactor({'example_private'}, {777123})
source = 'uid=777123(example_private) gid=777123(example_private) groups=1000(public),777123(example_private)'
self.assertEqual(clean(source), 'uid=[REDACTED UNIX ID] gid=[REDACTED UNIX ID] groups=1000(public),[REDACTED UNIX ID]')
def test_owner_name_is_removed_but_public_handles_and_message_ids_remain(self):
clean = Redactor({'example_private'}, {777123})
source = 'drwx------ example_private\n@public_handle /work/flock/agents/work/sample\nmessage 777123; uid=0(root)'
self.assertEqual(clean(source), 'drwx------ [REDACTED CLUSTER USER]\n@public_handle /work/flock/agents/work/sample\nmessage 777123; uid=0(root)')
def test_numeric_uid_context_and_identity_redaction_are_idempotent(self):
clean = Redactor({'example_private'}, {777123})
counts = collections.Counter()
result = clean('Uid:\t777123\t777123\nEXAMPLE_PRIVATE', lambda category: counts.update([category]))
self.assertEqual(result, 'Uid:\t[REDACTED UNIX ID]\t[REDACTED UNIX ID]\n[REDACTED CLUSTER USER]')
self.assertEqual(counts, {'unix_identity': 2, 'cluster_user': 1})
self.assertEqual(clean(result), result)
def test_reexport_removes_old_downloadable_chunks(self):
with tempfile.TemporaryDirectory() as directory:
root = pathlib.Path(directory)
db = sqlite3.connect(root / 'source.db')
db.executescript('''
CREATE TABLE principals(id,external_id,identity_type,role,display_name,created_at,pattern_seed,disabled);
CREATE TABLE channels(id,name,kind,description,created_at,created_by);
CREATE TABLE channel_members(channel_id,principal_id,added_at);
CREATE TABLE message_mentions(message_id,principal_id);
CREATE TABLE message_reactions(message_id,principal_id,emoji,created_at);
CREATE TABLE message_revisions(message_id,version,body,written_at,replaced_at);
CREATE TABLE messages(id,channel_id,sender_id,seq,thread_id,created_at,body,reply_to,edited_at);
CREATE TABLE channel_doc_versions(channel_id,version,body,author_id,created_at);
''')
names = {'example-' + str(i) for i in range(1000)}
db.executemany('INSERT INTO principals VALUES (?,?,?,?,?,?,?,?)',
[(i+1, 'example-'+str(i), 'agent', 'member', None, i, None, 0) for i in range(1000)])
db.execute('INSERT INTO channels VALUES (1,"general","public","",0,1)')
db.execute('INSERT INTO messages VALUES (1,1,1,1,NULL,1000,"zzuniquemarker",NULL,NULL)')
db.commit()
target = root / 'data'
with contextlib.redirect_stdout(io.StringIO()):
export(root / 'source.db', target, names)
old_search = target / 'search' / ('zz'.encode().hex() + '.json.gz')
self.assertTrue(old_search.exists())
for folder in ('messages', 'docs'):
(target / folder).mkdir(exist_ok=True)
(target / folder / '999-0.json.gz').write_bytes(gzip.compress(json.dumps(['old private data']).encode()))
db.execute('UPDATE messages SET body="clean replacement"')
db.commit()
with contextlib.redirect_stdout(io.StringIO()):
export(root / 'source.db', target, names)
self.assertFalse(old_search.exists())
self.assertFalse((target / 'messages/999-0.json.gz').exists())
self.assertFalse((target / 'docs/999-0.json.gz').exists())
db.close()
if __name__ == '__main__':
unittest.main()