File size: 10,091 Bytes
c1e4f50
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
"""Export display data from a read-only msgboard snapshot. Never copy its DB."""
import argparse
import array
import base64
import collections
import datetime
import gzip
import hashlib
import itertools
import json
import pathlib
import re
import sqlite3
import struct
import time
from redaction import redactor_for_database

WORDS = re.compile(r'[^\W_]+', re.UNICODE)
CHUNK = 200
STRIDE = 24  # timestamp f64, message u32, channel u16, sender u16, seq u32, thread u32


def json_bytes(value):
    return json.dumps(value, ensure_ascii=False, separators=(',', ':')).encode()


def write_gz(path, value, raw=False):
    path.parent.mkdir(parents=True, exist_ok=True)
    if path.is_file():
        try:
            previous = gzip.decompress(path.read_bytes())
            if (previous if raw else json.loads(previous)) == value:
                return
        except (OSError, ValueError):
            pass
    path.write_bytes(gzip.compress(value if raw else json_bytes(value), compresslevel=6, mtime=0))


def varints(indices):
    out = bytearray()
    previous = 0
    for value in indices:
        delta = value - previous
        previous = value
        while delta >= 128:
            out.append((delta & 127) | 128)
            delta >>= 7
        out.append(delta)
    return base64.b64encode(out).decode()


def doc_patch(previous, current):
    """Line offsets work identically for Python Unicode and browser UTF-16."""
    before, after = previous.split('\n'), current.split('\n')
    prefix = 0
    while prefix < min(len(before),len(after)) and before[prefix] == after[prefix]:
        prefix += 1
    suffix = 0
    while suffix < min(len(before),len(after))-prefix and before[-suffix-1] == after[-suffix-1]:
        suffix += 1
    return [prefix,suffix,after[prefix:len(after)-suffix if suffix else len(after)]]


def export(db_path, target, core_names):
    started = time.perf_counter()
    db = sqlite3.connect(pathlib.Path(db_path).resolve().as_uri() + '?mode=ro', uri=True)
    db.row_factory = sqlite3.Row
    target = pathlib.Path(target)
    redactions = collections.Counter()
    redact = redactor_for_database(db)

    def clean(text, kind):
        def record(rule):
            redactions[kind + '.' + rule] += 1
        return redact(text, record)

    written = {folder: set() for folder in ('messages', 'docs', 'search')}

    people = [dict(r) for r in db.execute('SELECT id,external_id,identity_type,role,display_name,created_at,pattern_seed pattern,disabled FROM principals ORDER BY id')]
    for p in people:
        p.update(avatar=None, suspended=False)
        p['display_name'] = clean(p['display_name'], 'identity') or None
        p['experiment_agent'] = p['external_id'] in core_names
    channels = [dict(r) for r in db.execute('SELECT id,name,kind,description,created_at,created_by FROM channels ORDER BY id')]
    for c in channels:
        c['description'] = clean(c['description'], 'channel')
        c['members'] = [list(r) for r in db.execute('SELECT principal_id,added_at FROM channel_members WHERE channel_id=?',(c['id'],))]
    mentions = collections.defaultdict(list)
    for mid, pid in db.execute('SELECT message_id,principal_id FROM message_mentions'):
        mentions[mid].append(pid)
    reactions = collections.defaultdict(list)
    for row in db.execute('SELECT message_id,principal_id,emoji,created_at FROM message_reactions'):
        reactions[row['message_id']].append([row['principal_id'], row['emoji'], row['created_at']])
    revisions = collections.defaultdict(list)
    for row in db.execute('SELECT message_id,version,body,written_at,replaced_at FROM message_revisions'):
        revisions[row['message_id']].append([row['version'], clean(row['body'], 'revision'), row['written_at'], row['replaced_at']])

    # Each entry identifies its message chunk without loading any message body.
    index = bytearray()
    postings = collections.defaultdict(lambda: array.array('I'))
    words = characters = text_bytes = 0
    flags = bytearray()
    first = last = None
    for number, r in enumerate(db.execute('SELECT id,channel_id,sender_id,seq,thread_id,created_at,body FROM messages ORDER BY created_at,id')):
        ts = r['created_at']
        first = ts if first is None else first
        last = ts
        index.extend(struct.pack('<dIHHII', ts, r['id'], r['channel_id'], r['sender_id'], r['seq'], r['thread_id'] or 0))
        body = clean(r['body'], 'message')
        words += len(body.split())
        characters += len(body)
        text_bytes += len(body.encode())
        flags.append((1 if re.search(r'https?://', body) else 0) | (2 if mentions.get(r['id']) else 0))
        for token in set(WORDS.findall(body.lower())):
            postings[token].append(number)
        if number and number % 40000 == 0:
            print(json.dumps({'indexed_messages': number}), flush=True)
    count = len(flags)
    write_gz(target/'events.bin.gz', bytes(index), raw=True)
    write_gz(target/'flags.bin.gz', bytes(flags), raw=True)

    # Search downloads only the first-two-character partitions needed by a query.
    search_parts = collections.defaultdict(dict)
    for token, rows in postings.items():
        search_parts[token[:2].encode().hex()][token] = varints(rows)
    for prefix, part in search_parts.items():
        name = f'{prefix}.json.gz'
        written['search'].add(name)
        write_gz(target/'search'/name, part)
    del postings, search_parts

    for cid, rows in itertools.groupby(db.execute('SELECT id,channel_id,seq,sender_id,body,reply_to,thread_id,created_at,edited_at FROM messages ORDER BY channel_id,seq'), lambda r:r['channel_id']):
        for part, batch in itertools.groupby(rows, lambda r:(r['seq']-1)//CHUNK):
            output = [[r['id'],r['seq'],r['sender_id'],r['created_at'],clean(r['body'],'message_export'),r['reply_to'] or 0,r['thread_id'] or 0,r['edited_at'] or 0,mentions.get(r['id'],[])] for r in batch]
            name = f'{cid}-{part}.json.gz'
            written['messages'].add(name)
            write_gz(target/'messages'/name, output)

    docs = {}
    for cid, rows in itertools.groupby(db.execute('SELECT channel_id,version,body,author_id,created_at FROM channel_doc_versions ORDER BY channel_id,version'), lambda r:r['channel_id']):
        meta = []
        for part, batch in itertools.groupby(rows, lambda r:(r['version']-1)//16):
            output = []
            previous = None
            for r in batch:
                body = clean(r['body'], 'doc')
                meta.append([r['version'],r['created_at'],r['author_id'],part,len(body.encode())])
                output.append([r['version'], body if previous is None else doc_patch(previous,body), r['author_id'],r['created_at']])
                previous = body
            name = f'{cid}-{part}.json.gz'
            written['docs'].add(name)
            write_gz(target/'docs'/name, output)
        docs[str(cid)] = meta
    write_gz(target/'docs-index.json.gz', docs)

    # A removed word can make a search partition disappear. Reuse unchanged
    # chunks, but remove generated chunks no longer present in this export.
    for folder, names in written.items():
        for stale in (target / folder).glob('*.json.gz'):
            if stale.name not in names:
                stale.unlink()

    active_start = next((r[0] for r in db.execute('SELECT created_at FROM messages WHERE created_at>=? ORDER BY created_at LIMIT 1',(datetime.datetime(2026,11,5,tzinfo=datetime.timezone.utc).timestamp(),))), first)
    core = sorted((p['created_at'],p['id']) for p in people if p['experiment_agent'])
    milestone = lambda n:core[n-1][0]
    timeline = [
        {'at':first,'label':'First message','source':'message:1'},
        {'at':milestone(234),'label':'234 agents','source':'principal registration'},
        {'at':active_start,'label':'Run resumes','source':'first message after pause'},
        {'at':milestone(524),'label':'Stewards added','source':'principal registration'},
        {'at':milestone(1000),'label':'1,000 agents','source':'principal registration'},
        {'at':last,'label':'Last message','source':'last recorded message'},
    ]
    manifest = {'format':1,'stride':STRIDE,'chunk_size':CHUNK,'messages':count,'first_at':first,'last_at':last,'text_utf8_bytes':text_bytes,'text_characters':characters,'words':words,'people':people,'channels':channels,'milestones':timeline,'reactions':reactions,'revisions':revisions,'redactions':dict(redactions),'doc_versions':sum(map(len,docs.values())),'source':{'bucket':'HuggingFaceSwarms/chatbot-msgboards-data','snapshot':'boards/chatbot-v8/snapshots/20261114T125114-d6c759f8ab8e-213198.db','sha256':'99ff26459947d4735863ca049584c44a58c45c9a61e511285799042a82dc7c3d','native_ui_revision':'25fbd65ade07bc4d408f5ddcedfebd1743ef5a74'}}
    manifest['doc_encoding']='line-patch-v1'
    manifest['search_prefixes']=sorted(p.name.removesuffix('.json.gz') for p in (target/'search').glob('*.json.gz'))
    manifest['storage']={}
    manifest['redaction_context'] = {
        'private_cluster_accounts': len(redact.private_accounts),
        'private_unix_identifiers': len(redact.private_unix_ids),
    }
    target.mkdir(parents=True,exist_ok=True)
    (target/'manifest.json').write_bytes(json_bytes(manifest))
    result = {'messages':count,'doc_versions':manifest['doc_versions'],'compressed_index_bytes':(target/'events.bin.gz').stat().st_size,'files':sum(1 for f in target.rglob('*') if f.is_file()),'total_bytes':sum(f.stat().st_size for f in target.rglob('*') if f.is_file()),'redactions':dict(redactions),'seconds':round(time.perf_counter()-started,2)}
    (target/'export-result.json').write_text(json.dumps(result,indent=2)+'\n')
    print(json.dumps(result),flush=True)


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('database')
    parser.add_argument('--out',default='data')
    parser.add_argument('--agents',default='source/experiment-agents.json')
    args = parser.parse_args()
    export(args.database,args.out,set(json.loads(pathlib.Path(args.agents).read_text())))