Download scripts/redaction.py from HuggingFaceSwarms/pybot-msgboard: direct link, hf CLI and curl.
- Browser
- Download file 4.93 kB
-
https://huggingface.co/spaces/HuggingFaceSwarms/pybot-msgboard/resolve/main/scripts/redaction.py
- Command line
-
hf download hf://spaces/HuggingFaceSwarms/pybot-msgboard/scripts/redaction.py
-
curl -L -o redaction.py https://huggingface.co/spaces/HuggingFaceSwarms/pybot-msgboard/resolve/main/scripts/redaction.py
4.93 kB
| """Remove credential-shaped values and private infrastructure from display text.""" | |
| import re | |
| TOKEN = re.compile(r'\b(?:hf_[A-Za-z0-9]{20,}|fbt_[A-Za-z0-9_-]{20,}|github_pat_[A-Za-z0-9_]{20,}|gh[pousr]_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9_-]{24,})\b') | |
| CLUSTER_PATH = re.compile(r"/fsx/[^\s`'\"<>\[\](){},;|]+") | |
| INTERNAL_HOST = re.compile( | |
| r'\bip-(?:10(?:-\d{1,3}){3}|172-(?:1[6-9]|2\d|3[01])(?:-\d{1,3}){2}' | |
| r'|192-168(?:-\d{1,3}){2})(?:\.[A-Za-z0-9-]+)*(?::\d{1,5})?\b' | |
| ) | |
| RULES = ( | |
| ('credential', TOKEN, '[REDACTED CREDENTIAL]'), | |
| ('cluster_path', CLUSTER_PATH, '[REDACTED CLUSTER PATH]'), | |
| ('internal_host', INTERNAL_HOST, '[REDACTED INTERNAL HOST]'), | |
| ) | |
| ACCOUNT = r'[A-Za-z_][A-Za-z0-9_.-]*' | |
| UNIX_ENTRY = re.compile(rf'(?P<id>\d+)(?:\((?P<name>{ACCOUNT})\))?') | |
| ENTRY = rf'\d+(?:\({ACCOUNT}\))?' | |
| UNIX_FIELDS = re.compile( | |
| rf'\b(uid|gid|groups)([ \t]*[:=][ \t]*)({ENTRY}(?:(?:,[ \t]*|[ \t]+){ENTRY})*)', | |
| re.IGNORECASE, | |
| ) | |
| FSX_ACCOUNT = re.compile(rf'/fsx/({ACCOUNT})/') | |
| SYSTEM_ACCOUNTS = {'root', 'nobody', 'node', 'user', 'ubuntu', 'runner'} | |
| def redact_text(text, on_redaction=None): | |
| """Keep surrounding prose/Markdown intact; optionally count each replacement.""" | |
| text = text or '' | |
| for name, pattern, replacement in RULES: | |
| def replace(match): | |
| if on_redaction is not None: | |
| on_redaction(name) | |
| # A plain path at a sentence boundary must not swallow punctuation. | |
| value = match.group() | |
| suffix = value[len(value.rstrip('.:!?')):] if name == 'cluster_path' else '' | |
| return replacement + suffix | |
| text = pattern.sub(replace, text) | |
| return text | |
| def discover_private_identities(texts): | |
| """Identify cluster accounts from FSx paths plus Unix identity output. | |
| Only counts are exported. Identifying values remain in the private source, | |
| so the public redaction code does not itself disclose the account or UID. | |
| """ | |
| path_accounts = set() | |
| identities = set() | |
| for text in texts: | |
| text = text or '' | |
| if '/fsx/' in text: | |
| path_accounts.update(m[1].casefold() for m in FSX_ACCOUNT.finditer(text)) | |
| folded = text.casefold() | |
| if not any(label in folded for label in ('uid', 'gid', 'groups')): | |
| continue | |
| for field in UNIX_FIELDS.finditer(text): | |
| for entry in UNIX_ENTRY.finditer(field[3]): | |
| if entry['name']: | |
| identities.add((int(entry['id']), entry['name'].casefold())) | |
| accounts = (path_accounts & {name for _, name in identities}) - SYSTEM_ACCOUNTS | |
| identifiers = {uid for uid, name in identities if name in accounts} | |
| return accounts, identifiers | |
| class Redactor: | |
| def __init__(self, private_accounts=(), private_unix_ids=()): | |
| self.private_accounts = frozenset(name.casefold() for name in private_accounts) | |
| self.private_unix_ids = frozenset(map(int, private_unix_ids)) | |
| self.id_literals = tuple(map(str, self.private_unix_ids)) | |
| self.account_pattern = re.compile( | |
| '|'.join(re.escape(name) for name in sorted(self.private_accounts, key=len, reverse=True)), | |
| re.IGNORECASE, | |
| ) if self.private_accounts else None | |
| def __call__(self, text, on_redaction=None): | |
| text = redact_text(text, on_redaction) | |
| if not self.private_accounts and not self.private_unix_ids: | |
| return text | |
| # Large doc histories rarely contain an identity. Avoid expensive | |
| # regular-expression passes when no identifying value is present. | |
| if (not any(uid in text for uid in self.id_literals) | |
| and not any(name in text.casefold() for name in self.private_accounts)): | |
| return text | |
| def clean_entry(match): | |
| if (int(match['id']) in self.private_unix_ids | |
| or (match['name'] or '').casefold() in self.private_accounts): | |
| if on_redaction is not None: | |
| on_redaction('unix_identity') | |
| return '[REDACTED UNIX ID]' | |
| return match.group() | |
| text = UNIX_FIELDS.sub( | |
| lambda match: match[1] + match[2] + UNIX_ENTRY.sub(clean_entry, match[3]), text, | |
| ) | |
| if self.account_pattern is not None: | |
| def clean_account(match): | |
| if on_redaction is not None: | |
| on_redaction('cluster_user') | |
| return '[REDACTED CLUSTER USER]' | |
| text = self.account_pattern.sub(clean_account, text) | |
| return text | |
| def redactor_for_database(db): | |
| queries = ( | |
| 'SELECT body FROM messages', | |
| 'SELECT body FROM message_revisions', | |
| 'SELECT body FROM channel_doc_versions', | |
| 'SELECT description FROM channels', | |
| 'SELECT display_name FROM principals', | |
| ) | |
| texts = (row[0] for query in queries for row in db.execute(query)) | |
| return Redactor(*discover_private_identities(texts)) | |