"""Remove credential-shaped values and private infrastructure from display text.""" import re TOKEN = re.compile(r'\b(?:hf_[A-Za-z0-9]{20,}|fbt_[A-Za-z0-9_-]{20,}|github_pat_[A-Za-z0-9_]{20,}|gh[pousr]_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9_-]{24,})\b') CLUSTER_PATH = re.compile(r"/fsx/[^\s`'\"<>\[\](){},;|]+") INTERNAL_HOST = re.compile( r'\bip-(?:10(?:-\d{1,3}){3}|172-(?:1[6-9]|2\d|3[01])(?:-\d{1,3}){2}' r'|192-168(?:-\d{1,3}){2})(?:\.[A-Za-z0-9-]+)*(?::\d{1,5})?\b' ) RULES = ( ('credential', TOKEN, '[REDACTED CREDENTIAL]'), ('cluster_path', CLUSTER_PATH, '[REDACTED CLUSTER PATH]'), ('internal_host', INTERNAL_HOST, '[REDACTED INTERNAL HOST]'), ) ACCOUNT = r'[A-Za-z_][A-Za-z0-9_.-]*' UNIX_ENTRY = re.compile(rf'(?P\d+)(?:\((?P{ACCOUNT})\))?') ENTRY = rf'\d+(?:\({ACCOUNT}\))?' UNIX_FIELDS = re.compile( rf'\b(uid|gid|groups)([ \t]*[:=][ \t]*)({ENTRY}(?:(?:,[ \t]*|[ \t]+){ENTRY})*)', re.IGNORECASE, ) FSX_ACCOUNT = re.compile(rf'/fsx/({ACCOUNT})/') SYSTEM_ACCOUNTS = {'root', 'nobody', 'node', 'user', 'ubuntu', 'runner'} def redact_text(text, on_redaction=None): """Keep surrounding prose/Markdown intact; optionally count each replacement.""" text = text or '' for name, pattern, replacement in RULES: def replace(match): if on_redaction is not None: on_redaction(name) # A plain path at a sentence boundary must not swallow punctuation. value = match.group() suffix = value[len(value.rstrip('.:!?')):] if name == 'cluster_path' else '' return replacement + suffix text = pattern.sub(replace, text) return text def discover_private_identities(texts): """Identify cluster accounts from FSx paths plus Unix identity output. Only counts are exported. Identifying values remain in the private source, so the public redaction code does not itself disclose the account or UID. """ path_accounts = set() identities = set() for text in texts: text = text or '' if '/fsx/' in text: path_accounts.update(m[1].casefold() for m in FSX_ACCOUNT.finditer(text)) folded = text.casefold() if not any(label in folded for label in ('uid', 'gid', 'groups')): continue for field in UNIX_FIELDS.finditer(text): for entry in UNIX_ENTRY.finditer(field[3]): if entry['name']: identities.add((int(entry['id']), entry['name'].casefold())) accounts = (path_accounts & {name for _, name in identities}) - SYSTEM_ACCOUNTS identifiers = {uid for uid, name in identities if name in accounts} return accounts, identifiers class Redactor: def __init__(self, private_accounts=(), private_unix_ids=()): self.private_accounts = frozenset(name.casefold() for name in private_accounts) self.private_unix_ids = frozenset(map(int, private_unix_ids)) self.id_literals = tuple(map(str, self.private_unix_ids)) self.account_pattern = re.compile( '|'.join(re.escape(name) for name in sorted(self.private_accounts, key=len, reverse=True)), re.IGNORECASE, ) if self.private_accounts else None def __call__(self, text, on_redaction=None): text = redact_text(text, on_redaction) if not self.private_accounts and not self.private_unix_ids: return text # Large doc histories rarely contain an identity. Avoid expensive # regular-expression passes when no identifying value is present. if (not any(uid in text for uid in self.id_literals) and not any(name in text.casefold() for name in self.private_accounts)): return text def clean_entry(match): if (int(match['id']) in self.private_unix_ids or (match['name'] or '').casefold() in self.private_accounts): if on_redaction is not None: on_redaction('unix_identity') return '[REDACTED UNIX ID]' return match.group() text = UNIX_FIELDS.sub( lambda match: match[1] + match[2] + UNIX_ENTRY.sub(clean_entry, match[3]), text, ) if self.account_pattern is not None: def clean_account(match): if on_redaction is not None: on_redaction('cluster_user') return '[REDACTED CLUSTER USER]' text = self.account_pattern.sub(clean_account, text) return text def redactor_for_database(db): queries = ( 'SELECT body FROM messages', 'SELECT body FROM message_revisions', 'SELECT body FROM channel_doc_versions', 'SELECT description FROM channels', 'SELECT display_name FROM principals', ) texts = (row[0] for query in queries for row in db.execute(query)) return Redactor(*discover_private_identities(texts))