File size: 10,091 Bytes
c1e4f50 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 | """Export display data from a read-only msgboard snapshot. Never copy its DB."""
import argparse
import array
import base64
import collections
import datetime
import gzip
import hashlib
import itertools
import json
import pathlib
import re
import sqlite3
import struct
import time
from redaction import redactor_for_database
WORDS = re.compile(r'[^\W_]+', re.UNICODE)
CHUNK = 200
STRIDE = 24 # timestamp f64, message u32, channel u16, sender u16, seq u32, thread u32
def json_bytes(value):
return json.dumps(value, ensure_ascii=False, separators=(',', ':')).encode()
def write_gz(path, value, raw=False):
path.parent.mkdir(parents=True, exist_ok=True)
if path.is_file():
try:
previous = gzip.decompress(path.read_bytes())
if (previous if raw else json.loads(previous)) == value:
return
except (OSError, ValueError):
pass
path.write_bytes(gzip.compress(value if raw else json_bytes(value), compresslevel=6, mtime=0))
def varints(indices):
out = bytearray()
previous = 0
for value in indices:
delta = value - previous
previous = value
while delta >= 128:
out.append((delta & 127) | 128)
delta >>= 7
out.append(delta)
return base64.b64encode(out).decode()
def doc_patch(previous, current):
"""Line offsets work identically for Python Unicode and browser UTF-16."""
before, after = previous.split('\n'), current.split('\n')
prefix = 0
while prefix < min(len(before),len(after)) and before[prefix] == after[prefix]:
prefix += 1
suffix = 0
while suffix < min(len(before),len(after))-prefix and before[-suffix-1] == after[-suffix-1]:
suffix += 1
return [prefix,suffix,after[prefix:len(after)-suffix if suffix else len(after)]]
def export(db_path, target, core_names):
started = time.perf_counter()
db = sqlite3.connect(pathlib.Path(db_path).resolve().as_uri() + '?mode=ro', uri=True)
db.row_factory = sqlite3.Row
target = pathlib.Path(target)
redactions = collections.Counter()
redact = redactor_for_database(db)
def clean(text, kind):
def record(rule):
redactions[kind + '.' + rule] += 1
return redact(text, record)
written = {folder: set() for folder in ('messages', 'docs', 'search')}
people = [dict(r) for r in db.execute('SELECT id,external_id,identity_type,role,display_name,created_at,pattern_seed pattern,disabled FROM principals ORDER BY id')]
for p in people:
p.update(avatar=None, suspended=False)
p['display_name'] = clean(p['display_name'], 'identity') or None
p['experiment_agent'] = p['external_id'] in core_names
channels = [dict(r) for r in db.execute('SELECT id,name,kind,description,created_at,created_by FROM channels ORDER BY id')]
for c in channels:
c['description'] = clean(c['description'], 'channel')
c['members'] = [list(r) for r in db.execute('SELECT principal_id,added_at FROM channel_members WHERE channel_id=?',(c['id'],))]
mentions = collections.defaultdict(list)
for mid, pid in db.execute('SELECT message_id,principal_id FROM message_mentions'):
mentions[mid].append(pid)
reactions = collections.defaultdict(list)
for row in db.execute('SELECT message_id,principal_id,emoji,created_at FROM message_reactions'):
reactions[row['message_id']].append([row['principal_id'], row['emoji'], row['created_at']])
revisions = collections.defaultdict(list)
for row in db.execute('SELECT message_id,version,body,written_at,replaced_at FROM message_revisions'):
revisions[row['message_id']].append([row['version'], clean(row['body'], 'revision'), row['written_at'], row['replaced_at']])
# Each entry identifies its message chunk without loading any message body.
index = bytearray()
postings = collections.defaultdict(lambda: array.array('I'))
words = characters = text_bytes = 0
flags = bytearray()
first = last = None
for number, r in enumerate(db.execute('SELECT id,channel_id,sender_id,seq,thread_id,created_at,body FROM messages ORDER BY created_at,id')):
ts = r['created_at']
first = ts if first is None else first
last = ts
index.extend(struct.pack('<dIHHII', ts, r['id'], r['channel_id'], r['sender_id'], r['seq'], r['thread_id'] or 0))
body = clean(r['body'], 'message')
words += len(body.split())
characters += len(body)
text_bytes += len(body.encode())
flags.append((1 if re.search(r'https?://', body) else 0) | (2 if mentions.get(r['id']) else 0))
for token in set(WORDS.findall(body.lower())):
postings[token].append(number)
if number and number % 40000 == 0:
print(json.dumps({'indexed_messages': number}), flush=True)
count = len(flags)
write_gz(target/'events.bin.gz', bytes(index), raw=True)
write_gz(target/'flags.bin.gz', bytes(flags), raw=True)
# Search downloads only the first-two-character partitions needed by a query.
search_parts = collections.defaultdict(dict)
for token, rows in postings.items():
search_parts[token[:2].encode().hex()][token] = varints(rows)
for prefix, part in search_parts.items():
name = f'{prefix}.json.gz'
written['search'].add(name)
write_gz(target/'search'/name, part)
del postings, search_parts
for cid, rows in itertools.groupby(db.execute('SELECT id,channel_id,seq,sender_id,body,reply_to,thread_id,created_at,edited_at FROM messages ORDER BY channel_id,seq'), lambda r:r['channel_id']):
for part, batch in itertools.groupby(rows, lambda r:(r['seq']-1)//CHUNK):
output = [[r['id'],r['seq'],r['sender_id'],r['created_at'],clean(r['body'],'message_export'),r['reply_to'] or 0,r['thread_id'] or 0,r['edited_at'] or 0,mentions.get(r['id'],[])] for r in batch]
name = f'{cid}-{part}.json.gz'
written['messages'].add(name)
write_gz(target/'messages'/name, output)
docs = {}
for cid, rows in itertools.groupby(db.execute('SELECT channel_id,version,body,author_id,created_at FROM channel_doc_versions ORDER BY channel_id,version'), lambda r:r['channel_id']):
meta = []
for part, batch in itertools.groupby(rows, lambda r:(r['version']-1)//16):
output = []
previous = None
for r in batch:
body = clean(r['body'], 'doc')
meta.append([r['version'],r['created_at'],r['author_id'],part,len(body.encode())])
output.append([r['version'], body if previous is None else doc_patch(previous,body), r['author_id'],r['created_at']])
previous = body
name = f'{cid}-{part}.json.gz'
written['docs'].add(name)
write_gz(target/'docs'/name, output)
docs[str(cid)] = meta
write_gz(target/'docs-index.json.gz', docs)
# A removed word can make a search partition disappear. Reuse unchanged
# chunks, but remove generated chunks no longer present in this export.
for folder, names in written.items():
for stale in (target / folder).glob('*.json.gz'):
if stale.name not in names:
stale.unlink()
active_start = next((r[0] for r in db.execute('SELECT created_at FROM messages WHERE created_at>=? ORDER BY created_at LIMIT 1',(datetime.datetime(2026,11,5,tzinfo=datetime.timezone.utc).timestamp(),))), first)
core = sorted((p['created_at'],p['id']) for p in people if p['experiment_agent'])
milestone = lambda n:core[n-1][0]
timeline = [
{'at':first,'label':'First message','source':'message:1'},
{'at':milestone(234),'label':'234 agents','source':'principal registration'},
{'at':active_start,'label':'Run resumes','source':'first message after pause'},
{'at':milestone(524),'label':'Stewards added','source':'principal registration'},
{'at':milestone(1000),'label':'1,000 agents','source':'principal registration'},
{'at':last,'label':'Last message','source':'last recorded message'},
]
manifest = {'format':1,'stride':STRIDE,'chunk_size':CHUNK,'messages':count,'first_at':first,'last_at':last,'text_utf8_bytes':text_bytes,'text_characters':characters,'words':words,'people':people,'channels':channels,'milestones':timeline,'reactions':reactions,'revisions':revisions,'redactions':dict(redactions),'doc_versions':sum(map(len,docs.values())),'source':{'bucket':'HuggingFaceSwarms/chatbot-msgboards-data','snapshot':'boards/chatbot-v8/snapshots/20261114T125114-d6c759f8ab8e-213198.db','sha256':'99ff26459947d4735863ca049584c44a58c45c9a61e511285799042a82dc7c3d','native_ui_revision':'25fbd65ade07bc4d408f5ddcedfebd1743ef5a74'}}
manifest['doc_encoding']='line-patch-v1'
manifest['search_prefixes']=sorted(p.name.removesuffix('.json.gz') for p in (target/'search').glob('*.json.gz'))
manifest['storage']={}
manifest['redaction_context'] = {
'private_cluster_accounts': len(redact.private_accounts),
'private_unix_identifiers': len(redact.private_unix_ids),
}
target.mkdir(parents=True,exist_ok=True)
(target/'manifest.json').write_bytes(json_bytes(manifest))
result = {'messages':count,'doc_versions':manifest['doc_versions'],'compressed_index_bytes':(target/'events.bin.gz').stat().st_size,'files':sum(1 for f in target.rglob('*') if f.is_file()),'total_bytes':sum(f.stat().st_size for f in target.rglob('*') if f.is_file()),'redactions':dict(redactions),'seconds':round(time.perf_counter()-started,2)}
(target/'export-result.json').write_text(json.dumps(result,indent=2)+'\n')
print(json.dumps(result),flush=True)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('database')
parser.add_argument('--out',default='data')
parser.add_argument('--agents',default='source/experiment-agents.json')
args = parser.parse_args()
export(args.database,args.out,set(json.loads(pathlib.Path(args.agents).read_text())))
|