File size: 6,589 Bytes
c1e4f50
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
"""Regression checks for Markdown-safe removal of reviewed infrastructure."""
import collections
import contextlib
import gzip
import io
import json
import pathlib
import sqlite3
import sys
import tempfile
import unittest

sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1] / 'scripts'))
from redaction import redact_text, Redactor, discover_private_identities
from export_archive import export


class RedactionTests(unittest.TestCase):
    def test_cluster_paths_preserve_prose_and_markdown(self):
        root = '/' + 'fsx/'
        source = f'Use `{root}example-user/project/run/file.patch`, then retry.'
        self.assertEqual(redact_text(source), 'Use `[REDACTED CLUSTER PATH]`, then retry.')
        self.assertEqual(redact_text('See ' + root + 'example-user/project.'),
                         'See [REDACTED CLUSTER PATH].')

    def test_already_elided_paths_are_also_removed(self):
        source = '/' + 'fsx/.../run/example'
        self.assertEqual(redact_text(source), '[REDACTED CLUSTER PATH]')

    def test_private_hosts_include_ports_and_dns_suffixes(self):
        for address in ('10-12-34-56', '172-16-12-34', '192-168-12-34'):
            host = 'ip-' + address
            self.assertEqual(redact_text(f'`{host}:45000`'), '`[REDACTED INTERNAL HOST]`')
            self.assertEqual(redact_text(host + '.example.internal:45000.'),
                             '[REDACTED INTERNAL HOST].')

    def test_unrelated_examples_and_token_variable_names_stay_readable(self):
        source = 'Use $HF_TOKEN or FLOCK_AGENT_TOKEN. Router: 192.168.0.1. '
        source += 'Public example: ' + 'ip-' + '8-8-8-8:45000.'
        self.assertEqual(redact_text(source), source)

    def test_existing_credential_formats_still_redact(self):
        for prefix in ('hf' + '_', 'fbt' + '_', 'github_pat' + '_', 'ghp' + '_', 'sk' + '-'):
            self.assertEqual(redact_text(prefix + 'a' * 30), '[REDACTED CREDENTIAL]')

    def test_count_categories_and_idempotence(self):
        counts = collections.Counter()
        source = '/' + 'fsx/example-user/a ' + 'ip-' + '10-12-34-56:45000'
        result = redact_text(source, lambda category: counts.update([category]))
        self.assertEqual(counts, {'cluster_path': 1, 'internal_host': 1})
        self.assertEqual(redact_text(result), result)

    def test_cluster_identity_is_discovered_without_a_published_account_list(self):
        root = '/' + 'fsx/'
        accounts, ids = discover_private_identities([
            root + 'example_private/project/run',
            'uid=777123(example_private) gid=777123(example_private)',
            root + 'root/project/run', 'uid=0(root)',
            root + 'project_only/run/file',
        ])
        self.assertEqual(accounts, {'example_private'})
        self.assertEqual(ids, {777123})

    def test_private_unix_fields_preserve_other_group_entries(self):
        clean = Redactor({'example_private'}, {777123})
        source = 'uid=777123(example_private) gid=777123(example_private) groups=1000(public),777123(example_private)'
        self.assertEqual(clean(source), 'uid=[REDACTED UNIX ID] gid=[REDACTED UNIX ID] groups=1000(public),[REDACTED UNIX ID]')

    def test_owner_name_is_removed_but_public_handles_and_message_ids_remain(self):
        clean = Redactor({'example_private'}, {777123})
        source = 'drwx------ example_private\n@public_handle /work/flock/agents/work/sample\nmessage 777123; uid=0(root)'
        self.assertEqual(clean(source), 'drwx------ [REDACTED CLUSTER USER]\n@public_handle /work/flock/agents/work/sample\nmessage 777123; uid=0(root)')

    def test_numeric_uid_context_and_identity_redaction_are_idempotent(self):
        clean = Redactor({'example_private'}, {777123})
        counts = collections.Counter()
        result = clean('Uid:\t777123\t777123\nEXAMPLE_PRIVATE', lambda category: counts.update([category]))
        self.assertEqual(result, 'Uid:\t[REDACTED UNIX ID]\t[REDACTED UNIX ID]\n[REDACTED CLUSTER USER]')
        self.assertEqual(counts, {'unix_identity': 2, 'cluster_user': 1})
        self.assertEqual(clean(result), result)

    def test_reexport_removes_old_downloadable_chunks(self):
        with tempfile.TemporaryDirectory() as directory:
            root = pathlib.Path(directory)
            db = sqlite3.connect(root / 'source.db')
            db.executescript('''
                CREATE TABLE principals(id,external_id,identity_type,role,display_name,created_at,pattern_seed,disabled);
                CREATE TABLE channels(id,name,kind,description,created_at,created_by);
                CREATE TABLE channel_members(channel_id,principal_id,added_at);
                CREATE TABLE message_mentions(message_id,principal_id);
                CREATE TABLE message_reactions(message_id,principal_id,emoji,created_at);
                CREATE TABLE message_revisions(message_id,version,body,written_at,replaced_at);
                CREATE TABLE messages(id,channel_id,sender_id,seq,thread_id,created_at,body,reply_to,edited_at);
                CREATE TABLE channel_doc_versions(channel_id,version,body,author_id,created_at);
            ''')
            names = {'example-' + str(i) for i in range(1000)}
            db.executemany('INSERT INTO principals VALUES (?,?,?,?,?,?,?,?)',
                           [(i+1, 'example-'+str(i), 'agent', 'member', None, i, None, 0) for i in range(1000)])
            db.execute('INSERT INTO channels VALUES (1,"general","public","",0,1)')
            db.execute('INSERT INTO messages VALUES (1,1,1,1,NULL,1000,"zzuniquemarker",NULL,NULL)')
            db.commit()
            target = root / 'data'
            with contextlib.redirect_stdout(io.StringIO()):
                export(root / 'source.db', target, names)
            old_search = target / 'search' / ('zz'.encode().hex() + '.json.gz')
            self.assertTrue(old_search.exists())
            for folder in ('messages', 'docs'):
                (target / folder).mkdir(exist_ok=True)
                (target / folder / '999-0.json.gz').write_bytes(gzip.compress(json.dumps(['old private data']).encode()))
            db.execute('UPDATE messages SET body="clean replacement"')
            db.commit()
            with contextlib.redirect_stdout(io.StringIO()):
                export(root / 'source.db', target, names)
            self.assertFalse(old_search.exists())
            self.assertFalse((target / 'messages/999-0.json.gz').exists())
            self.assertFalse((target / 'docs/999-0.json.gz').exists())
            db.close()


if __name__ == '__main__':
    unittest.main()