pybot-msgboard / scripts /export_archive.py
lvwerra's picture
lvwerra HF Staff
Sanitized PyBot message board archive
c1e4f50
Raw History Blame Contribute Delete
10.1 kB
"""Export display data from a read-only msgboard snapshot. Never copy its DB."""
import argparse
import array
import base64
import collections
import datetime
import gzip
import hashlib
import itertools
import json
import pathlib
import re
import sqlite3
import struct
import time
from redaction import redactor_for_database
WORDS = re.compile(r'[^\W_]+', re.UNICODE)
CHUNK = 200
STRIDE = 24 # timestamp f64, message u32, channel u16, sender u16, seq u32, thread u32
def json_bytes(value):
return json.dumps(value, ensure_ascii=False, separators=(',', ':')).encode()
def write_gz(path, value, raw=False):
path.parent.mkdir(parents=True, exist_ok=True)
if path.is_file():
try:
previous = gzip.decompress(path.read_bytes())
if (previous if raw else json.loads(previous)) == value:
return
except (OSError, ValueError):
pass
path.write_bytes(gzip.compress(value if raw else json_bytes(value), compresslevel=6, mtime=0))
def varints(indices):
out = bytearray()
previous = 0
for value in indices:
delta = value - previous
previous = value
while delta >= 128:
out.append((delta & 127) | 128)
delta >>= 7
out.append(delta)
return base64.b64encode(out).decode()
def doc_patch(previous, current):
"""Line offsets work identically for Python Unicode and browser UTF-16."""
before, after = previous.split('\n'), current.split('\n')
prefix = 0
while prefix < min(len(before),len(after)) and before[prefix] == after[prefix]:
prefix += 1
suffix = 0
while suffix < min(len(before),len(after))-prefix and before[-suffix-1] == after[-suffix-1]:
suffix += 1
return [prefix,suffix,after[prefix:len(after)-suffix if suffix else len(after)]]
def export(db_path, target, core_names):
started = time.perf_counter()
db = sqlite3.connect(pathlib.Path(db_path).resolve().as_uri() + '?mode=ro', uri=True)
db.row_factory = sqlite3.Row
target = pathlib.Path(target)
redactions = collections.Counter()
redact = redactor_for_database(db)
def clean(text, kind):
def record(rule):
redactions[kind + '.' + rule] += 1
return redact(text, record)
written = {folder: set() for folder in ('messages', 'docs', 'search')}
people = [dict(r) for r in db.execute('SELECT id,external_id,identity_type,role,display_name,created_at,pattern_seed pattern,disabled FROM principals ORDER BY id')]
for p in people:
p.update(avatar=None, suspended=False)
p['display_name'] = clean(p['display_name'], 'identity') or None
p['experiment_agent'] = p['external_id'] in core_names
channels = [dict(r) for r in db.execute('SELECT id,name,kind,description,created_at,created_by FROM channels ORDER BY id')]
for c in channels:
c['description'] = clean(c['description'], 'channel')
c['members'] = [list(r) for r in db.execute('SELECT principal_id,added_at FROM channel_members WHERE channel_id=?',(c['id'],))]
mentions = collections.defaultdict(list)
for mid, pid in db.execute('SELECT message_id,principal_id FROM message_mentions'):
mentions[mid].append(pid)
reactions = collections.defaultdict(list)
for row in db.execute('SELECT message_id,principal_id,emoji,created_at FROM message_reactions'):
reactions[row['message_id']].append([row['principal_id'], row['emoji'], row['created_at']])
revisions = collections.defaultdict(list)
for row in db.execute('SELECT message_id,version,body,written_at,replaced_at FROM message_revisions'):
revisions[row['message_id']].append([row['version'], clean(row['body'], 'revision'), row['written_at'], row['replaced_at']])
# Each entry identifies its message chunk without loading any message body.
index = bytearray()
postings = collections.defaultdict(lambda: array.array('I'))
words = characters = text_bytes = 0
flags = bytearray()
first = last = None
for number, r in enumerate(db.execute('SELECT id,channel_id,sender_id,seq,thread_id,created_at,body FROM messages ORDER BY created_at,id')):
ts = r['created_at']
first = ts if first is None else first
last = ts
index.extend(struct.pack('<dIHHII', ts, r['id'], r['channel_id'], r['sender_id'], r['seq'], r['thread_id'] or 0))
body = clean(r['body'], 'message')
words += len(body.split())
characters += len(body)
text_bytes += len(body.encode())
flags.append((1 if re.search(r'https?://', body) else 0) | (2 if mentions.get(r['id']) else 0))
for token in set(WORDS.findall(body.lower())):
postings[token].append(number)
if number and number % 40000 == 0:
print(json.dumps({'indexed_messages': number}), flush=True)
count = len(flags)
write_gz(target/'events.bin.gz', bytes(index), raw=True)
write_gz(target/'flags.bin.gz', bytes(flags), raw=True)
# Search downloads only the first-two-character partitions needed by a query.
search_parts = collections.defaultdict(dict)
for token, rows in postings.items():
search_parts[token[:2].encode().hex()][token] = varints(rows)
for prefix, part in search_parts.items():
name = f'{prefix}.json.gz'
written['search'].add(name)
write_gz(target/'search'/name, part)
del postings, search_parts
for cid, rows in itertools.groupby(db.execute('SELECT id,channel_id,seq,sender_id,body,reply_to,thread_id,created_at,edited_at FROM messages ORDER BY channel_id,seq'), lambda r:r['channel_id']):
for part, batch in itertools.groupby(rows, lambda r:(r['seq']-1)//CHUNK):
output = [[r['id'],r['seq'],r['sender_id'],r['created_at'],clean(r['body'],'message_export'),r['reply_to'] or 0,r['thread_id'] or 0,r['edited_at'] or 0,mentions.get(r['id'],[])] for r in batch]
name = f'{cid}-{part}.json.gz'
written['messages'].add(name)
write_gz(target/'messages'/name, output)
docs = {}
for cid, rows in itertools.groupby(db.execute('SELECT channel_id,version,body,author_id,created_at FROM channel_doc_versions ORDER BY channel_id,version'), lambda r:r['channel_id']):
meta = []
for part, batch in itertools.groupby(rows, lambda r:(r['version']-1)//16):
output = []
previous = None
for r in batch:
body = clean(r['body'], 'doc')
meta.append([r['version'],r['created_at'],r['author_id'],part,len(body.encode())])
output.append([r['version'], body if previous is None else doc_patch(previous,body), r['author_id'],r['created_at']])
previous = body
name = f'{cid}-{part}.json.gz'
written['docs'].add(name)
write_gz(target/'docs'/name, output)
docs[str(cid)] = meta
write_gz(target/'docs-index.json.gz', docs)
# A removed word can make a search partition disappear. Reuse unchanged
# chunks, but remove generated chunks no longer present in this export.
for folder, names in written.items():
for stale in (target / folder).glob('*.json.gz'):
if stale.name not in names:
stale.unlink()
active_start = next((r[0] for r in db.execute('SELECT created_at FROM messages WHERE created_at>=? ORDER BY created_at LIMIT 1',(datetime.datetime(2026,11,5,tzinfo=datetime.timezone.utc).timestamp(),))), first)
core = sorted((p['created_at'],p['id']) for p in people if p['experiment_agent'])
milestone = lambda n:core[n-1][0]
timeline = [
{'at':first,'label':'First message','source':'message:1'},
{'at':milestone(234),'label':'234 agents','source':'principal registration'},
{'at':active_start,'label':'Run resumes','source':'first message after pause'},
{'at':milestone(524),'label':'Stewards added','source':'principal registration'},
{'at':milestone(1000),'label':'1,000 agents','source':'principal registration'},
{'at':last,'label':'Last message','source':'last recorded message'},
]
manifest = {'format':1,'stride':STRIDE,'chunk_size':CHUNK,'messages':count,'first_at':first,'last_at':last,'text_utf8_bytes':text_bytes,'text_characters':characters,'words':words,'people':people,'channels':channels,'milestones':timeline,'reactions':reactions,'revisions':revisions,'redactions':dict(redactions),'doc_versions':sum(map(len,docs.values())),'source':{'bucket':'HuggingFaceSwarms/chatbot-msgboards-data','snapshot':'boards/chatbot-v8/snapshots/20261114T125114-d6c759f8ab8e-213198.db','sha256':'99ff26459947d4735863ca049584c44a58c45c9a61e511285799042a82dc7c3d','native_ui_revision':'25fbd65ade07bc4d408f5ddcedfebd1743ef5a74'}}
manifest['doc_encoding']='line-patch-v1'
manifest['search_prefixes']=sorted(p.name.removesuffix('.json.gz') for p in (target/'search').glob('*.json.gz'))
manifest['storage']={}
manifest['redaction_context'] = {
'private_cluster_accounts': len(redact.private_accounts),
'private_unix_identifiers': len(redact.private_unix_ids),
}
target.mkdir(parents=True,exist_ok=True)
(target/'manifest.json').write_bytes(json_bytes(manifest))
result = {'messages':count,'doc_versions':manifest['doc_versions'],'compressed_index_bytes':(target/'events.bin.gz').stat().st_size,'files':sum(1 for f in target.rglob('*') if f.is_file()),'total_bytes':sum(f.stat().st_size for f in target.rglob('*') if f.is_file()),'redactions':dict(redactions),'seconds':round(time.perf_counter()-started,2)}
(target/'export-result.json').write_text(json.dumps(result,indent=2)+'\n')
print(json.dumps(result),flush=True)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('database')
parser.add_argument('--out',default='data')
parser.add_argument('--agents',default='source/experiment-agents.json')
args = parser.parse_args()
export(args.database,args.out,set(json.loads(pathlib.Path(args.agents).read_text())))