File size: 4,934 Bytes
c1e4f50
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
"""Remove credential-shaped values and private infrastructure from display text."""
import re

TOKEN = re.compile(r'\b(?:hf_[A-Za-z0-9]{20,}|fbt_[A-Za-z0-9_-]{20,}|github_pat_[A-Za-z0-9_]{20,}|gh[pousr]_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9_-]{24,})\b')
CLUSTER_PATH = re.compile(r"/fsx/[^\s`'\"<>\[\](){},;|]+")
INTERNAL_HOST = re.compile(
    r'\bip-(?:10(?:-\d{1,3}){3}|172-(?:1[6-9]|2\d|3[01])(?:-\d{1,3}){2}'
    r'|192-168(?:-\d{1,3}){2})(?:\.[A-Za-z0-9-]+)*(?::\d{1,5})?\b'
)
RULES = (
    ('credential', TOKEN, '[REDACTED CREDENTIAL]'),
    ('cluster_path', CLUSTER_PATH, '[REDACTED CLUSTER PATH]'),
    ('internal_host', INTERNAL_HOST, '[REDACTED INTERNAL HOST]'),
)
ACCOUNT = r'[A-Za-z_][A-Za-z0-9_.-]*'
UNIX_ENTRY = re.compile(rf'(?P<id>\d+)(?:\((?P<name>{ACCOUNT})\))?')
ENTRY = rf'\d+(?:\({ACCOUNT}\))?'
UNIX_FIELDS = re.compile(
    rf'\b(uid|gid|groups)([ \t]*[:=][ \t]*)({ENTRY}(?:(?:,[ \t]*|[ \t]+){ENTRY})*)',
    re.IGNORECASE,
)
FSX_ACCOUNT = re.compile(rf'/fsx/({ACCOUNT})/')
SYSTEM_ACCOUNTS = {'root', 'nobody', 'node', 'user', 'ubuntu', 'runner'}


def redact_text(text, on_redaction=None):
    """Keep surrounding prose/Markdown intact; optionally count each replacement."""
    text = text or ''
    for name, pattern, replacement in RULES:
        def replace(match):
            if on_redaction is not None:
                on_redaction(name)
            # A plain path at a sentence boundary must not swallow punctuation.
            value = match.group()
            suffix = value[len(value.rstrip('.:!?')):] if name == 'cluster_path' else ''
            return replacement + suffix
        text = pattern.sub(replace, text)
    return text


def discover_private_identities(texts):
    """Identify cluster accounts from FSx paths plus Unix identity output.

    Only counts are exported. Identifying values remain in the private source,
    so the public redaction code does not itself disclose the account or UID.
    """
    path_accounts = set()
    identities = set()
    for text in texts:
        text = text or ''
        if '/fsx/' in text:
            path_accounts.update(m[1].casefold() for m in FSX_ACCOUNT.finditer(text))
        folded = text.casefold()
        if not any(label in folded for label in ('uid', 'gid', 'groups')):
            continue
        for field in UNIX_FIELDS.finditer(text):
            for entry in UNIX_ENTRY.finditer(field[3]):
                if entry['name']:
                    identities.add((int(entry['id']), entry['name'].casefold()))
    accounts = (path_accounts & {name for _, name in identities}) - SYSTEM_ACCOUNTS
    identifiers = {uid for uid, name in identities if name in accounts}
    return accounts, identifiers


class Redactor:
    def __init__(self, private_accounts=(), private_unix_ids=()):
        self.private_accounts = frozenset(name.casefold() for name in private_accounts)
        self.private_unix_ids = frozenset(map(int, private_unix_ids))
        self.id_literals = tuple(map(str, self.private_unix_ids))
        self.account_pattern = re.compile(
            '|'.join(re.escape(name) for name in sorted(self.private_accounts, key=len, reverse=True)),
            re.IGNORECASE,
        ) if self.private_accounts else None

    def __call__(self, text, on_redaction=None):
        text = redact_text(text, on_redaction)
        if not self.private_accounts and not self.private_unix_ids:
            return text
        # Large doc histories rarely contain an identity. Avoid expensive
        # regular-expression passes when no identifying value is present.
        if (not any(uid in text for uid in self.id_literals)
                and not any(name in text.casefold() for name in self.private_accounts)):
            return text

        def clean_entry(match):
            if (int(match['id']) in self.private_unix_ids
                    or (match['name'] or '').casefold() in self.private_accounts):
                if on_redaction is not None:
                    on_redaction('unix_identity')
                return '[REDACTED UNIX ID]'
            return match.group()

        text = UNIX_FIELDS.sub(
            lambda match: match[1] + match[2] + UNIX_ENTRY.sub(clean_entry, match[3]), text,
        )
        if self.account_pattern is not None:
            def clean_account(match):
                if on_redaction is not None:
                    on_redaction('cluster_user')
                return '[REDACTED CLUSTER USER]'
            text = self.account_pattern.sub(clean_account, text)
        return text


def redactor_for_database(db):
    queries = (
        'SELECT body FROM messages',
        'SELECT body FROM message_revisions',
        'SELECT body FROM channel_doc_versions',
        'SELECT description FROM channels',
        'SELECT display_name FROM principals',
    )
    texts = (row[0] for query in queries for row in db.execute(query))
    return Redactor(*discover_private_identities(texts))