Download tests/test_redaction.py from HuggingFaceSwarms/pybot-msgboard: direct link, hf CLI and curl.
- Browser
- Download file 6.59 kB
-
https://huggingface.co/spaces/HuggingFaceSwarms/pybot-msgboard/resolve/main/tests/test_redaction.py
- Command line
-
hf download hf://spaces/HuggingFaceSwarms/pybot-msgboard/tests/test_redaction.py
-
curl -L -o test_redaction.py https://huggingface.co/spaces/HuggingFaceSwarms/pybot-msgboard/resolve/main/tests/test_redaction.py
6.59 kB
| """Regression checks for Markdown-safe removal of reviewed infrastructure.""" | |
| import collections | |
| import contextlib | |
| import gzip | |
| import io | |
| import json | |
| import pathlib | |
| import sqlite3 | |
| import sys | |
| import tempfile | |
| import unittest | |
| sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1] / 'scripts')) | |
| from redaction import redact_text, Redactor, discover_private_identities | |
| from export_archive import export | |
| class RedactionTests(unittest.TestCase): | |
| def test_cluster_paths_preserve_prose_and_markdown(self): | |
| root = '/' + 'fsx/' | |
| source = f'Use `{root}example-user/project/run/file.patch`, then retry.' | |
| self.assertEqual(redact_text(source), 'Use `[REDACTED CLUSTER PATH]`, then retry.') | |
| self.assertEqual(redact_text('See ' + root + 'example-user/project.'), | |
| 'See [REDACTED CLUSTER PATH].') | |
| def test_already_elided_paths_are_also_removed(self): | |
| source = '/' + 'fsx/.../run/example' | |
| self.assertEqual(redact_text(source), '[REDACTED CLUSTER PATH]') | |
| def test_private_hosts_include_ports_and_dns_suffixes(self): | |
| for address in ('10-12-34-56', '172-16-12-34', '192-168-12-34'): | |
| host = 'ip-' + address | |
| self.assertEqual(redact_text(f'`{host}:45000`'), '`[REDACTED INTERNAL HOST]`') | |
| self.assertEqual(redact_text(host + '.example.internal:45000.'), | |
| '[REDACTED INTERNAL HOST].') | |
| def test_unrelated_examples_and_token_variable_names_stay_readable(self): | |
| source = 'Use $HF_TOKEN or FLOCK_AGENT_TOKEN. Router: 192.168.0.1. ' | |
| source += 'Public example: ' + 'ip-' + '8-8-8-8:45000.' | |
| self.assertEqual(redact_text(source), source) | |
| def test_existing_credential_formats_still_redact(self): | |
| for prefix in ('hf' + '_', 'fbt' + '_', 'github_pat' + '_', 'ghp' + '_', 'sk' + '-'): | |
| self.assertEqual(redact_text(prefix + 'a' * 30), '[REDACTED CREDENTIAL]') | |
| def test_count_categories_and_idempotence(self): | |
| counts = collections.Counter() | |
| source = '/' + 'fsx/example-user/a ' + 'ip-' + '10-12-34-56:45000' | |
| result = redact_text(source, lambda category: counts.update([category])) | |
| self.assertEqual(counts, {'cluster_path': 1, 'internal_host': 1}) | |
| self.assertEqual(redact_text(result), result) | |
| def test_cluster_identity_is_discovered_without_a_published_account_list(self): | |
| root = '/' + 'fsx/' | |
| accounts, ids = discover_private_identities([ | |
| root + 'example_private/project/run', | |
| 'uid=777123(example_private) gid=777123(example_private)', | |
| root + 'root/project/run', 'uid=0(root)', | |
| root + 'project_only/run/file', | |
| ]) | |
| self.assertEqual(accounts, {'example_private'}) | |
| self.assertEqual(ids, {777123}) | |
| def test_private_unix_fields_preserve_other_group_entries(self): | |
| clean = Redactor({'example_private'}, {777123}) | |
| source = 'uid=777123(example_private) gid=777123(example_private) groups=1000(public),777123(example_private)' | |
| self.assertEqual(clean(source), 'uid=[REDACTED UNIX ID] gid=[REDACTED UNIX ID] groups=1000(public),[REDACTED UNIX ID]') | |
| def test_owner_name_is_removed_but_public_handles_and_message_ids_remain(self): | |
| clean = Redactor({'example_private'}, {777123}) | |
| source = 'drwx------ example_private\n@public_handle /work/flock/agents/work/sample\nmessage 777123; uid=0(root)' | |
| self.assertEqual(clean(source), 'drwx------ [REDACTED CLUSTER USER]\n@public_handle /work/flock/agents/work/sample\nmessage 777123; uid=0(root)') | |
| def test_numeric_uid_context_and_identity_redaction_are_idempotent(self): | |
| clean = Redactor({'example_private'}, {777123}) | |
| counts = collections.Counter() | |
| result = clean('Uid:\t777123\t777123\nEXAMPLE_PRIVATE', lambda category: counts.update([category])) | |
| self.assertEqual(result, 'Uid:\t[REDACTED UNIX ID]\t[REDACTED UNIX ID]\n[REDACTED CLUSTER USER]') | |
| self.assertEqual(counts, {'unix_identity': 2, 'cluster_user': 1}) | |
| self.assertEqual(clean(result), result) | |
| def test_reexport_removes_old_downloadable_chunks(self): | |
| with tempfile.TemporaryDirectory() as directory: | |
| root = pathlib.Path(directory) | |
| db = sqlite3.connect(root / 'source.db') | |
| db.executescript(''' | |
| CREATE TABLE principals(id,external_id,identity_type,role,display_name,created_at,pattern_seed,disabled); | |
| CREATE TABLE channels(id,name,kind,description,created_at,created_by); | |
| CREATE TABLE channel_members(channel_id,principal_id,added_at); | |
| CREATE TABLE message_mentions(message_id,principal_id); | |
| CREATE TABLE message_reactions(message_id,principal_id,emoji,created_at); | |
| CREATE TABLE message_revisions(message_id,version,body,written_at,replaced_at); | |
| CREATE TABLE messages(id,channel_id,sender_id,seq,thread_id,created_at,body,reply_to,edited_at); | |
| CREATE TABLE channel_doc_versions(channel_id,version,body,author_id,created_at); | |
| ''') | |
| names = {'example-' + str(i) for i in range(1000)} | |
| db.executemany('INSERT INTO principals VALUES (?,?,?,?,?,?,?,?)', | |
| [(i+1, 'example-'+str(i), 'agent', 'member', None, i, None, 0) for i in range(1000)]) | |
| db.execute('INSERT INTO channels VALUES (1,"general","public","",0,1)') | |
| db.execute('INSERT INTO messages VALUES (1,1,1,1,NULL,1000,"zzuniquemarker",NULL,NULL)') | |
| db.commit() | |
| target = root / 'data' | |
| with contextlib.redirect_stdout(io.StringIO()): | |
| export(root / 'source.db', target, names) | |
| old_search = target / 'search' / ('zz'.encode().hex() + '.json.gz') | |
| self.assertTrue(old_search.exists()) | |
| for folder in ('messages', 'docs'): | |
| (target / folder).mkdir(exist_ok=True) | |
| (target / folder / '999-0.json.gz').write_bytes(gzip.compress(json.dumps(['old private data']).encode())) | |
| db.execute('UPDATE messages SET body="clean replacement"') | |
| db.commit() | |
| with contextlib.redirect_stdout(io.StringIO()): | |
| export(root / 'source.db', target, names) | |
| self.assertFalse(old_search.exists()) | |
| self.assertFalse((target / 'messages/999-0.json.gz').exists()) | |
| self.assertFalse((target / 'docs/999-0.json.gz').exists()) | |
| db.close() | |
| if __name__ == '__main__': | |
| unittest.main() | |