"""Regression checks for Markdown-safe removal of reviewed infrastructure.""" import collections import contextlib import gzip import io import json import pathlib import sqlite3 import sys import tempfile import unittest sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1] / 'scripts')) from redaction import redact_text, Redactor, discover_private_identities from export_archive import export class RedactionTests(unittest.TestCase): def test_cluster_paths_preserve_prose_and_markdown(self): root = '/' + 'fsx/' source = f'Use `{root}example-user/project/run/file.patch`, then retry.' self.assertEqual(redact_text(source), 'Use `[REDACTED CLUSTER PATH]`, then retry.') self.assertEqual(redact_text('See ' + root + 'example-user/project.'), 'See [REDACTED CLUSTER PATH].') def test_already_elided_paths_are_also_removed(self): source = '/' + 'fsx/.../run/example' self.assertEqual(redact_text(source), '[REDACTED CLUSTER PATH]') def test_private_hosts_include_ports_and_dns_suffixes(self): for address in ('10-12-34-56', '172-16-12-34', '192-168-12-34'): host = 'ip-' + address self.assertEqual(redact_text(f'`{host}:45000`'), '`[REDACTED INTERNAL HOST]`') self.assertEqual(redact_text(host + '.example.internal:45000.'), '[REDACTED INTERNAL HOST].') def test_unrelated_examples_and_token_variable_names_stay_readable(self): source = 'Use $HF_TOKEN or FLOCK_AGENT_TOKEN. Router: 192.168.0.1. ' source += 'Public example: ' + 'ip-' + '8-8-8-8:45000.' self.assertEqual(redact_text(source), source) def test_existing_credential_formats_still_redact(self): for prefix in ('hf' + '_', 'fbt' + '_', 'github_pat' + '_', 'ghp' + '_', 'sk' + '-'): self.assertEqual(redact_text(prefix + 'a' * 30), '[REDACTED CREDENTIAL]') def test_count_categories_and_idempotence(self): counts = collections.Counter() source = '/' + 'fsx/example-user/a ' + 'ip-' + '10-12-34-56:45000' result = redact_text(source, lambda category: counts.update([category])) self.assertEqual(counts, {'cluster_path': 1, 'internal_host': 1}) self.assertEqual(redact_text(result), result) def test_cluster_identity_is_discovered_without_a_published_account_list(self): root = '/' + 'fsx/' accounts, ids = discover_private_identities([ root + 'example_private/project/run', 'uid=777123(example_private) gid=777123(example_private)', root + 'root/project/run', 'uid=0(root)', root + 'project_only/run/file', ]) self.assertEqual(accounts, {'example_private'}) self.assertEqual(ids, {777123}) def test_private_unix_fields_preserve_other_group_entries(self): clean = Redactor({'example_private'}, {777123}) source = 'uid=777123(example_private) gid=777123(example_private) groups=1000(public),777123(example_private)' self.assertEqual(clean(source), 'uid=[REDACTED UNIX ID] gid=[REDACTED UNIX ID] groups=1000(public),[REDACTED UNIX ID]') def test_owner_name_is_removed_but_public_handles_and_message_ids_remain(self): clean = Redactor({'example_private'}, {777123}) source = 'drwx------ example_private\n@public_handle /work/flock/agents/work/sample\nmessage 777123; uid=0(root)' self.assertEqual(clean(source), 'drwx------ [REDACTED CLUSTER USER]\n@public_handle /work/flock/agents/work/sample\nmessage 777123; uid=0(root)') def test_numeric_uid_context_and_identity_redaction_are_idempotent(self): clean = Redactor({'example_private'}, {777123}) counts = collections.Counter() result = clean('Uid:\t777123\t777123\nEXAMPLE_PRIVATE', lambda category: counts.update([category])) self.assertEqual(result, 'Uid:\t[REDACTED UNIX ID]\t[REDACTED UNIX ID]\n[REDACTED CLUSTER USER]') self.assertEqual(counts, {'unix_identity': 2, 'cluster_user': 1}) self.assertEqual(clean(result), result) def test_reexport_removes_old_downloadable_chunks(self): with tempfile.TemporaryDirectory() as directory: root = pathlib.Path(directory) db = sqlite3.connect(root / 'source.db') db.executescript(''' CREATE TABLE principals(id,external_id,identity_type,role,display_name,created_at,pattern_seed,disabled); CREATE TABLE channels(id,name,kind,description,created_at,created_by); CREATE TABLE channel_members(channel_id,principal_id,added_at); CREATE TABLE message_mentions(message_id,principal_id); CREATE TABLE message_reactions(message_id,principal_id,emoji,created_at); CREATE TABLE message_revisions(message_id,version,body,written_at,replaced_at); CREATE TABLE messages(id,channel_id,sender_id,seq,thread_id,created_at,body,reply_to,edited_at); CREATE TABLE channel_doc_versions(channel_id,version,body,author_id,created_at); ''') names = {'example-' + str(i) for i in range(1000)} db.executemany('INSERT INTO principals VALUES (?,?,?,?,?,?,?,?)', [(i+1, 'example-'+str(i), 'agent', 'member', None, i, None, 0) for i in range(1000)]) db.execute('INSERT INTO channels VALUES (1,"general","public","",0,1)') db.execute('INSERT INTO messages VALUES (1,1,1,1,NULL,1000,"zzuniquemarker",NULL,NULL)') db.commit() target = root / 'data' with contextlib.redirect_stdout(io.StringIO()): export(root / 'source.db', target, names) old_search = target / 'search' / ('zz'.encode().hex() + '.json.gz') self.assertTrue(old_search.exists()) for folder in ('messages', 'docs'): (target / folder).mkdir(exist_ok=True) (target / folder / '999-0.json.gz').write_bytes(gzip.compress(json.dumps(['old private data']).encode())) db.execute('UPDATE messages SET body="clean replacement"') db.commit() with contextlib.redirect_stdout(io.StringIO()): export(root / 'source.db', target, names) self.assertFalse(old_search.exists()) self.assertFalse((target / 'messages/999-0.json.gz').exists()) self.assertFalse((target / 'docs/999-0.json.gz').exists()) db.close() if __name__ == '__main__': unittest.main()