"""Export display data from a read-only msgboard snapshot. Never copy its DB.""" import argparse import array import base64 import collections import datetime import gzip import hashlib import itertools import json import pathlib import re import sqlite3 import struct import time from redaction import redactor_for_database WORDS = re.compile(r'[^\W_]+', re.UNICODE) CHUNK = 200 STRIDE = 24 # timestamp f64, message u32, channel u16, sender u16, seq u32, thread u32 def json_bytes(value): return json.dumps(value, ensure_ascii=False, separators=(',', ':')).encode() def write_gz(path, value, raw=False): path.parent.mkdir(parents=True, exist_ok=True) if path.is_file(): try: previous = gzip.decompress(path.read_bytes()) if (previous if raw else json.loads(previous)) == value: return except (OSError, ValueError): pass path.write_bytes(gzip.compress(value if raw else json_bytes(value), compresslevel=6, mtime=0)) def varints(indices): out = bytearray() previous = 0 for value in indices: delta = value - previous previous = value while delta >= 128: out.append((delta & 127) | 128) delta >>= 7 out.append(delta) return base64.b64encode(out).decode() def doc_patch(previous, current): """Line offsets work identically for Python Unicode and browser UTF-16.""" before, after = previous.split('\n'), current.split('\n') prefix = 0 while prefix < min(len(before),len(after)) and before[prefix] == after[prefix]: prefix += 1 suffix = 0 while suffix < min(len(before),len(after))-prefix and before[-suffix-1] == after[-suffix-1]: suffix += 1 return [prefix,suffix,after[prefix:len(after)-suffix if suffix else len(after)]] def export(db_path, target, core_names): started = time.perf_counter() db = sqlite3.connect(pathlib.Path(db_path).resolve().as_uri() + '?mode=ro', uri=True) db.row_factory = sqlite3.Row target = pathlib.Path(target) redactions = collections.Counter() redact = redactor_for_database(db) def clean(text, kind): def record(rule): redactions[kind + '.' + rule] += 1 return redact(text, record) written = {folder: set() for folder in ('messages', 'docs', 'search')} people = [dict(r) for r in db.execute('SELECT id,external_id,identity_type,role,display_name,created_at,pattern_seed pattern,disabled FROM principals ORDER BY id')] for p in people: p.update(avatar=None, suspended=False) p['display_name'] = clean(p['display_name'], 'identity') or None p['experiment_agent'] = p['external_id'] in core_names channels = [dict(r) for r in db.execute('SELECT id,name,kind,description,created_at,created_by FROM channels ORDER BY id')] for c in channels: c['description'] = clean(c['description'], 'channel') c['members'] = [list(r) for r in db.execute('SELECT principal_id,added_at FROM channel_members WHERE channel_id=?',(c['id'],))] mentions = collections.defaultdict(list) for mid, pid in db.execute('SELECT message_id,principal_id FROM message_mentions'): mentions[mid].append(pid) reactions = collections.defaultdict(list) for row in db.execute('SELECT message_id,principal_id,emoji,created_at FROM message_reactions'): reactions[row['message_id']].append([row['principal_id'], row['emoji'], row['created_at']]) revisions = collections.defaultdict(list) for row in db.execute('SELECT message_id,version,body,written_at,replaced_at FROM message_revisions'): revisions[row['message_id']].append([row['version'], clean(row['body'], 'revision'), row['written_at'], row['replaced_at']]) # Each entry identifies its message chunk without loading any message body. index = bytearray() postings = collections.defaultdict(lambda: array.array('I')) words = characters = text_bytes = 0 flags = bytearray() first = last = None for number, r in enumerate(db.execute('SELECT id,channel_id,sender_id,seq,thread_id,created_at,body FROM messages ORDER BY created_at,id')): ts = r['created_at'] first = ts if first is None else first last = ts index.extend(struct.pack('=? ORDER BY created_at LIMIT 1',(datetime.datetime(2026,11,5,tzinfo=datetime.timezone.utc).timestamp(),))), first) core = sorted((p['created_at'],p['id']) for p in people if p['experiment_agent']) milestone = lambda n:core[n-1][0] timeline = [ {'at':first,'label':'First message','source':'message:1'}, {'at':milestone(234),'label':'234 agents','source':'principal registration'}, {'at':active_start,'label':'Run resumes','source':'first message after pause'}, {'at':milestone(524),'label':'Stewards added','source':'principal registration'}, {'at':milestone(1000),'label':'1,000 agents','source':'principal registration'}, {'at':last,'label':'Last message','source':'last recorded message'}, ] manifest = {'format':1,'stride':STRIDE,'chunk_size':CHUNK,'messages':count,'first_at':first,'last_at':last,'text_utf8_bytes':text_bytes,'text_characters':characters,'words':words,'people':people,'channels':channels,'milestones':timeline,'reactions':reactions,'revisions':revisions,'redactions':dict(redactions),'doc_versions':sum(map(len,docs.values())),'source':{'bucket':'HuggingFaceSwarms/chatbot-msgboards-data','snapshot':'boards/chatbot-v8/snapshots/20261114T125114-d6c759f8ab8e-213198.db','sha256':'99ff26459947d4735863ca049584c44a58c45c9a61e511285799042a82dc7c3d','native_ui_revision':'25fbd65ade07bc4d408f5ddcedfebd1743ef5a74'}} manifest['doc_encoding']='line-patch-v1' manifest['search_prefixes']=sorted(p.name.removesuffix('.json.gz') for p in (target/'search').glob('*.json.gz')) manifest['storage']={} manifest['redaction_context'] = { 'private_cluster_accounts': len(redact.private_accounts), 'private_unix_identifiers': len(redact.private_unix_ids), } target.mkdir(parents=True,exist_ok=True) (target/'manifest.json').write_bytes(json_bytes(manifest)) result = {'messages':count,'doc_versions':manifest['doc_versions'],'compressed_index_bytes':(target/'events.bin.gz').stat().st_size,'files':sum(1 for f in target.rglob('*') if f.is_file()),'total_bytes':sum(f.stat().st_size for f in target.rglob('*') if f.is_file()),'redactions':dict(redactions),'seconds':round(time.perf_counter()-started,2)} (target/'export-result.json').write_text(json.dumps(result,indent=2)+'\n') print(json.dumps(result),flush=True) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('database') parser.add_argument('--out',default='data') parser.add_argument('--agents',default='source/experiment-agents.json') args = parser.parse_args() export(args.database,args.out,set(json.loads(pathlib.Path(args.agents).read_text())))