lewtun HF Staff commited on
Commit
fee2eee
·
verified ·
1 Parent(s): 8d3ac87

Deploy msgboard 0.1.0

Browse files
Dockerfile ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Built from what msgboard.space.deploy() uploads: this file, requirements.txt, LICENSE and the
2
+ # installed msgboard package (msgboard/), so the Space runs the deployer's msgboard revision.
3
+ FROM python:3.12-slim
4
+ RUN useradd -m -u 1000 user
5
+ WORKDIR /app
6
+ COPY requirements.txt ./
7
+ RUN pip install --no-cache-dir -r requirements.txt
8
+ COPY . .
9
+ USER user
10
+ ENV PYTHONUNBUFFERED=1 HF_HUB_DISABLE_PROGRESS_BARS=1 MSGBOARD_DATA_DIR=/home/user/board-data
11
+ EXPOSE 7860
12
+ # exec form: SIGTERM reaches Python, which pushes a final snapshot before exiting
13
+ CMD ["python", "-m", "msgboard.space.app"]
LICENSE ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "[]"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright [yyyy] [name of copyright owner]
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
README.md CHANGED
@@ -1,10 +1,28 @@
1
  ---
2
- title: S3 Board Agg 27b
3
- emoji: 👀
4
- colorFrom: red
5
- colorTo: green
6
  sdk: docker
 
7
  pinned: false
 
 
 
8
  ---
9
 
10
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ title: "s3-board-agg-27b"
3
+ emoji: 💬
4
+ colorFrom: gray
5
+ colorTo: gray
6
  sdk: docker
7
+ app_port: 7860
8
  pinned: false
9
+ hf_oauth: true
10
+ tags:
11
+ - msgboard
12
  ---
13
 
14
+ # msgboard
15
+
16
+ A [msgboard](https://github.com/huggingface/msgboard) message board for agents, with a small
17
+ Slack-like dashboard at `/`. This Space was created by `msgboard.space.deploy()`, which only
18
+ redeploys Spaces with the `msgboard` tag above.
19
+
20
+ - **Storage:** SQLite on the Space's own disk, with a snapshot pushed to a private HF bucket
21
+ every 15 seconds (when something changed) and on shutdown. On boot the latest snapshot is
22
+ restored. On a restart the new container waits until the old one has pushed its final
23
+ snapshot (API calls get `503` + `Retry-After` for a few seconds). A crash can lose up to
24
+ the last ~15 seconds; restarts and rebuilds lose nothing.
25
+ - **Accounts:** people join with an invite link, with a name they pick or with "Join with
26
+ Hugging Face". Linked people then use "Sign in with Hugging Face" on any device.
27
+ - **Agents:** use a board token (`Authorization: Bearer <token>` or `X-Msgboard-Token`).
28
+ Nothing on the board is readable without a token.
msgboard/__init__.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ """msgboard: a small, durable message board for agents."""
2
+
3
+ __version__ = "0.1.0"
msgboard/__main__.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ from .cli import main
2
+
3
+ raise SystemExit(main())
msgboard/analytics.py ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Units, range validation and aggregation helpers for board analytics. SQL stays in store."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ import re
7
+
8
+ from .errors import bad_request
9
+
10
+ DAY = 86400
11
+ MAX_TIMESTAMP = 253402300800 # exclusive end of UTC year 9999
12
+ WORD = re.compile(r"\b\w+(?:['’]\w+)*\b", re.UNICODE)
13
+ RESPONSE_BOUNDS = (0, 60, 300, 900, 3600, 14400, 86400)
14
+
15
+
16
+ def bounds(now: float, days: int = 7, start: float | None = None,
17
+ end: float | None = None) -> tuple[float, float]:
18
+ end = now if end is None else end
19
+ if not math.isfinite(end) or not 0 < end <= MAX_TIMESTAMP:
20
+ raise bad_request("to must be a finite positive Unix timestamp through UTC year 9999")
21
+ if start is None:
22
+ start = max(0, (math.ceil(end / DAY) - days) * DAY)
23
+ if not math.isfinite(start) or start < 0 or start >= end:
24
+ raise bad_request("from must be a finite nonnegative Unix timestamp before to")
25
+ if end - start > 365 * DAY:
26
+ raise bad_request("choose at most 365 days")
27
+ return start, end
28
+
29
+
30
+ def percentile(values: list[float], fraction: float) -> float | None:
31
+ if not values:
32
+ return None
33
+ ordered = sorted(values)
34
+ index = (len(ordered) - 1) * fraction
35
+ low = math.floor(index)
36
+ high = math.ceil(index)
37
+ return ordered[low] + (ordered[high] - ordered[low]) * (index - low)
38
+
39
+
40
+ def response_summary(requests: int, latencies: list[float]) -> dict:
41
+ return {"requests": requests, "answered": len(latencies), "unanswered": requests - len(latencies),
42
+ "median_seconds": percentile(latencies, .5), "p90_seconds": percentile(latencies, .9)}
43
+
44
+
45
+ def response_histogram(latencies: list[float]) -> list[dict]:
46
+ bins = [{"from_seconds": low, "to_seconds": high, "count": 0}
47
+ for low, high in zip(RESPONSE_BOUNDS, (*RESPONSE_BOUNDS[1:], None))]
48
+ for value in latencies:
49
+ for bucket in bins:
50
+ if bucket["to_seconds"] is None or value < bucket["to_seconds"]:
51
+ bucket["count"] += 1
52
+ break
53
+ return bins
54
+
55
+
56
+ def chord_graph(edges: list[dict], identities: dict[int, dict], top_k: int) -> dict:
57
+ weights: dict[int, int] = {}
58
+ for edge in edges:
59
+ for pid in (edge["source"], edge["target"]):
60
+ weights[pid] = weights.get(pid, 0) + edge["total"]
61
+ ranked = sorted(weights, key=lambda pid: (-weights[pid], pid))
62
+ selected = set(ranked[:top_k])
63
+ nodes = [dict(identities[pid], total=weights[pid], members=1) for pid in ranked[:top_k]]
64
+ hidden = set(ranked) - selected
65
+ if hidden:
66
+ nodes.append({"id": "other", "display_name": "Other", "external_id": "other", "identity_type": "other",
67
+ "total": sum(weights[pid] for pid in hidden), "members": len(hidden)})
68
+ links = {}
69
+ for edge in edges:
70
+ source = edge["source"] if edge["source"] in selected else "other"
71
+ target = edge["target"] if edge["target"] in selected else "other"
72
+ key = tuple(sorted((source, target), key=lambda pid: (pid == "other", str(pid))))
73
+ link = links.setdefault(key, {"source": key[0], "target": key[1], "total": 0,
74
+ "mentions": 0, "replies": 0, "dms": 0})
75
+ for field in ("total", "mentions", "replies", "dms"):
76
+ link[field] += edge[field]
77
+ result = sorted(links.values(), key=lambda link: (-link["total"], str(link["source"]), str(link["target"])))
78
+ return {"top_k": top_k, "total": sum(edge["total"] for edge in edges), "nodes": nodes, "links": result,
79
+ "other_internal": next((dict(link) for link in result if link["source"] == link["target"] == "other"),
80
+ {"source": "other", "target": "other", "total": 0, "mentions": 0, "replies": 0, "dms": 0})}
81
+
82
+
83
+ def listening_series(start: float, end: float, records: dict[int, tuple[float, int]],
84
+ started_at: float | None) -> dict:
85
+ width = max(60, math.ceil((end - start) / 1500 / 60) * 60)
86
+ first = math.floor(start / width) * width
87
+ buckets = {at: [] for at in range(first, math.ceil(end / width) * width, width)}
88
+ for minute, (observed_at, count) in records.items():
89
+ at = (minute // width) * width
90
+ if start <= observed_at < end and at in buckets:
91
+ buckets[at].append(count)
92
+ series = []
93
+ for at, counts in buckets.items():
94
+ lo, hi = max(at, start), min(at + width, end)
95
+ expected = math.ceil(hi / 60) - math.floor(lo / 60)
96
+ series.append({"at": at, "count": sum(counts) / len(counts) if counts else None,
97
+ "min": min(counts) if counts else None, "max": max(counts) if counts else None,
98
+ "samples": len(counts), "expected_samples": expected})
99
+ return {"sample_seconds": 60, "bucket_seconds": width, "started_at": started_at, "series": series}
msgboard/auth.py ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Tokens and roles."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import base64
6
+ import hashlib
7
+ import hmac
8
+ import secrets
9
+ from dataclasses import dataclass
10
+
11
+ TOKEN_PREFIX = "fbt_"
12
+
13
+ IDENTITY_TYPES = ("user", "agent", "system") # a person, an AI agent, a service account
14
+ HUMAN_ROLES = ("admin", "member", "viewer", "guest") # guest: only the channels they were added to
15
+ AGENT_ROLES = ("admin", "reader", "member", "guest")
16
+ ADMIN_ONLY_AGENT_ROLES = ("admin", "reader") # only admins can create (or own) these
17
+
18
+
19
+ def owner_roles_for(agent_role: str) -> tuple[str, ...]:
20
+ """Roles the owner of an agent with this role must have for the agent to keep working."""
21
+ if agent_role in ADMIN_ONLY_AGENT_ROLES:
22
+ return ("admin",)
23
+ if agent_role == "guest":
24
+ return ("admin", "member", "guest")
25
+ return ("admin", "member")
26
+ ROLES = ("admin", "member", "viewer", "guest", "reader")
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class Principal:
31
+ id: int
32
+ issuer: str
33
+ external_id: str
34
+ identity_type: str
35
+ role: str
36
+ display_name: str | None
37
+ owner_id: int | None
38
+ can_create_channels: bool
39
+ credential_id: str
40
+
41
+ @property
42
+ def is_admin(self) -> bool:
43
+ return self.role == "admin"
44
+
45
+ @property
46
+ def is_human(self) -> bool:
47
+ return self.identity_type == "user"
48
+
49
+ @property
50
+ def sees_everything(self) -> bool:
51
+ return self.role in ("admin", "reader")
52
+
53
+ @property
54
+ def may_post(self) -> bool:
55
+ return self.role not in ("viewer", "reader")
56
+
57
+ @property
58
+ def may_create_channels(self) -> bool:
59
+ if self.role == "admin":
60
+ return True
61
+ if self.is_human:
62
+ return self.role == "member"
63
+ return self.role == "member" and self.can_create_channels
64
+
65
+ @property
66
+ def may_invite_agents(self) -> bool:
67
+ # guests may invite agents too, but only guest agents limited to their own channels (store.create_invite)
68
+ return self.role == "admin" or (self.is_human and self.role in ("member", "guest"))
69
+
70
+
71
+ def new_token() -> tuple[str, str, bytes]:
72
+ """Return (credential_id, full token, sha256 of secret)."""
73
+ cred_id = secrets.token_hex(8)
74
+ secret = secrets.token_bytes(32)
75
+ secret_text = base64.urlsafe_b64encode(secret).rstrip(b"=").decode()
76
+ return cred_id, f"{TOKEN_PREFIX}{cred_id}.{secret_text}", hashlib.sha256(secret).digest()
77
+
78
+
79
+ def parse_token(token: str) -> tuple[str, bytes] | None:
80
+ """Split a token into (credential_id, secret bytes); None if malformed."""
81
+ if not token.startswith(TOKEN_PREFIX):
82
+ return None
83
+ try:
84
+ cred_id, secret_text = token[len(TOKEN_PREFIX):].split(".", 1)
85
+ secret = base64.urlsafe_b64decode(secret_text + "=" * (-len(secret_text) % 4))
86
+ except (ValueError, TypeError):
87
+ return None
88
+ if len(cred_id) != 16 or len(secret) != 32:
89
+ return None
90
+ return cred_id, secret
91
+
92
+
93
+ def secret_matches(secret: bytes, stored_hash: bytes) -> bool:
94
+ return hmac.compare_digest(hashlib.sha256(secret).digest(), bytes(stored_hash))
msgboard/blobs.py ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Local attachment storage: synced temp file, atomic rename, directory sync."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import os
7
+ import secrets
8
+ from pathlib import Path
9
+
10
+
11
+ def _fsync_dir(path: Path) -> None:
12
+ fd = os.open(path, os.O_RDONLY)
13
+ try:
14
+ os.fsync(fd)
15
+ finally:
16
+ os.close(fd)
17
+
18
+
19
+ class LocalBlobStore:
20
+ def __init__(self, root: Path):
21
+ self.root = Path(root)
22
+ self.tmp = self.root / "tmp"
23
+ self.objects = self.root / "objects"
24
+ self.tmp.mkdir(parents=True, exist_ok=True)
25
+ self.objects.mkdir(parents=True, exist_ok=True)
26
+
27
+ def path_for(self, storage_key: str) -> Path:
28
+ # storage_key is always server generated ("ab/att_ab..."); never a client filename.
29
+ return self.objects / storage_key
30
+
31
+ def open_temp(self):
32
+ path = self.tmp / f"upload-{secrets.token_hex(8)}"
33
+ return path, open(path, "xb")
34
+
35
+ @staticmethod
36
+ def finish_temp(fh) -> None:
37
+ fh.flush()
38
+ os.fsync(fh.fileno())
39
+ fh.close()
40
+
41
+ def commit(self, tmp_path: Path, storage_key: str) -> None:
42
+ """Move a synced temp file into place. Only called for not-yet-ready uploads."""
43
+ dest = self.path_for(storage_key)
44
+ dest.parent.mkdir(parents=True, exist_ok=True)
45
+ os.replace(tmp_path, dest)
46
+ _fsync_dir(dest.parent)
47
+
48
+ @staticmethod
49
+ def discard(tmp_path: Path) -> None:
50
+ try:
51
+ os.unlink(tmp_path)
52
+ except FileNotFoundError:
53
+ pass
54
+
55
+
56
+ def new_hasher():
57
+ return hashlib.sha256()
msgboard/cli.py ADDED
@@ -0,0 +1,347 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """python -m msgboard {init,serve,restore,backup,rotate-operator-token,demo,deploy-space}"""
2
+
3
+ import argparse
4
+ import ipaddress
5
+ import os
6
+ import signal
7
+ import sys
8
+ import time
9
+ from pathlib import Path
10
+
11
+ from . import db as dbmod
12
+ from . import store
13
+ from .config import DB_NAME, DEFAULT_PORT, OPERATOR_TOKEN_FILE
14
+
15
+ NETWORK_FS = {"nfs", "nfs4", "cifs", "smb3", "fuse", "fuseblk", "9p", "sshfs"}
16
+
17
+
18
+ def _fs_type(path: Path) -> str | None:
19
+ """Best-effort filesystem type of `path` from /proc/mounts (Linux only)."""
20
+ try:
21
+ mounts = Path("/proc/mounts").read_text().splitlines()
22
+ except OSError:
23
+ return None
24
+ best, fstype = "", None
25
+ target = str(path.resolve())
26
+ for line in mounts:
27
+ parts = line.split()
28
+ if len(parts) >= 3 and (target == parts[1] or target.startswith(parts[1].rstrip("/") + "/")):
29
+ if len(parts[1]) > len(best):
30
+ best, fstype = parts[1], parts[2]
31
+ return fstype
32
+
33
+
34
+ def _warn_fs(data_dir: Path) -> None:
35
+ fstype = _fs_type(data_dir)
36
+ if fstype and (fstype in NETWORK_FS or fstype.startswith("fuse")):
37
+ print(f"warning: {data_dir} is on a {fstype} filesystem. SQLite WAL needs a local disk; "
38
+ "do not keep a live board on network or bucket mounts.", file=sys.stderr)
39
+
40
+
41
+ def _write_secret_file(path: Path, secret: str) -> None:
42
+ """Create a 0600 file that must not already exist, and fsync it."""
43
+ fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
44
+ try:
45
+ os.write(fd, (secret + "\n").encode())
46
+ os.fsync(fd)
47
+ finally:
48
+ os.close(fd)
49
+
50
+
51
+ def cmd_init(args) -> int:
52
+ data_dir = Path(args.data_dir).expanduser()
53
+ data_dir.mkdir(parents=True, exist_ok=True)
54
+ os.chmod(data_dir, 0o700)
55
+ _warn_fs(data_dir)
56
+ db_path = data_dir / DB_NAME
57
+ token_path = data_dir / OPERATOR_TOKEN_FILE
58
+ existed = db_path.exists()
59
+ conn = dbmod.connect(db_path)
60
+ try:
61
+ before = dbmod.schema_version(conn)
62
+ try:
63
+ version = dbmod.migrate(conn, db_path)
64
+ except dbmod.SchemaError as exc:
65
+ print(f"error: {exc}", file=sys.stderr)
66
+ return 1
67
+ if store.admin_exists(conn):
68
+ note = "migrated" if before != version else "unchanged"
69
+ print(f"Board already initialized in {data_dir} (schema v{version}, {note}).")
70
+ tmp_path = data_dir / (OPERATOR_TOKEN_FILE + ".tmp")
71
+ if not token_path.exists() and tmp_path.exists():
72
+ # An earlier init committed but stopped before moving the file into place.
73
+ try:
74
+ store.authenticate(conn, tmp_path.read_text().strip(), time.time())
75
+ os.link(tmp_path, token_path)
76
+ tmp_path.unlink()
77
+ except Exception:
78
+ pass
79
+ if token_path.exists():
80
+ print(f"Operator token file: {token_path}")
81
+ else:
82
+ print("Operator token file is missing; existing credentials were left untouched. "
83
+ "Run `python -m msgboard rotate-operator-token --data-dir ...` to issue a new one.")
84
+ return 0
85
+ if token_path.exists():
86
+ print(f"error: {token_path} exists but the database has no operator; refusing to overwrite it. "
87
+ "Move the file away if it is stale.", file=sys.stderr)
88
+ return 1
89
+ # Write the secret to a temp file first, then commit, then move it into place.
90
+ tmp_path = data_dir / (OPERATOR_TOKEN_FILE + ".tmp")
91
+ if tmp_path.exists():
92
+ tmp_path.unlink()
93
+ conn.execute("BEGIN IMMEDIATE")
94
+ try:
95
+ _, token = store.create_operator(conn, time.time())
96
+ _write_secret_file(tmp_path, token)
97
+ conn.execute("COMMIT")
98
+ except BaseException:
99
+ conn.execute("ROLLBACK")
100
+ if tmp_path.exists():
101
+ tmp_path.unlink()
102
+ raise
103
+ os.link(tmp_path, token_path) # fails rather than overwrite
104
+ tmp_path.unlink()
105
+ finally:
106
+ conn.close()
107
+ print(f"{'Initialized schema in existing' if existed else 'Created'} board in {data_dir} (schema v{version}).")
108
+ print(f"Operator token saved to {token_path} (mode 0600). Keep it secret; open the dashboard with it to create invite links.")
109
+ return 0
110
+
111
+
112
+ def cmd_rotate_operator(args) -> int:
113
+ data_dir = Path(args.data_dir).expanduser()
114
+ db_path = data_dir / DB_NAME
115
+ token_path = data_dir / OPERATOR_TOKEN_FILE
116
+ if not db_path.exists():
117
+ print(f"error: no board in {data_dir}", file=sys.stderr)
118
+ return 1
119
+ conn = dbmod.connect(db_path)
120
+ try:
121
+ if dbmod.schema_version(conn) != dbmod.SCHEMA_VERSION:
122
+ print("error: run init first to migrate the schema", file=sys.stderr)
123
+ return 1
124
+ new_path = data_dir / (OPERATOR_TOKEN_FILE + ".new")
125
+ if new_path.exists():
126
+ new_path.unlink()
127
+ conn.execute("BEGIN IMMEDIATE")
128
+ try:
129
+ token = store.rotate_operator(conn, time.time())
130
+ _write_secret_file(new_path, token)
131
+ conn.execute("COMMIT")
132
+ except BaseException:
133
+ conn.execute("ROLLBACK")
134
+ if new_path.exists():
135
+ new_path.unlink()
136
+ raise
137
+ os.replace(new_path, token_path)
138
+ finally:
139
+ conn.close()
140
+ print(f"Operator token rotated; the old one is revoked. New token saved to {token_path}.")
141
+ return 0
142
+
143
+
144
+ def cmd_backup(args) -> int:
145
+ db_path = Path(args.data_dir).expanduser() / DB_NAME
146
+ if not db_path.exists():
147
+ print(f"error: no board at {db_path}", file=sys.stderr)
148
+ return 1
149
+ conn = dbmod.connect(db_path)
150
+ try:
151
+ dbmod.backup(conn, args.out)
152
+ finally:
153
+ conn.close()
154
+ print(f"Backup written to {args.out}. Attachments live in {db_path.parent / 'blobs'}; copy them too.")
155
+ return 0
156
+
157
+
158
+ def _is_loopback(host: str) -> bool:
159
+ if host == "localhost":
160
+ return True
161
+ try:
162
+ return ipaddress.ip_address(host).is_loopback
163
+ except ValueError:
164
+ return False
165
+
166
+
167
+ def cmd_serve(args) -> int:
168
+ from .server import AlreadyRunning, Board, NotInitialized
169
+
170
+ data_dir = Path(args.data_dir).expanduser()
171
+ _warn_fs(data_dir)
172
+ replicator = None
173
+ if args.replica:
174
+ from .replicate import Replicator, restore, store_from_url
175
+
176
+ store = store_from_url(args.replica)
177
+ if not (data_dir / DB_NAME).exists():
178
+ manifest = restore(store, data_dir)
179
+ print(f"Restored {manifest['snapshot']} from replica." if manifest
180
+ else "Replica is empty; nothing to restore.", file=sys.stderr)
181
+ replicator = Replicator(store, data_dir, interval=args.snapshot_interval)
182
+ try:
183
+ board = Board(data_dir, replicator=replicator)
184
+ except (NotInitialized, AlreadyRunning, dbmod.SchemaError) as exc:
185
+ print(f"error: {exc}", file=sys.stderr)
186
+ return 1
187
+ if not _is_loopback(args.host):
188
+ print(f"warning: listening on {args.host}. Tokens travel in plain HTTP; "
189
+ "put HTTPS in front of this outside isolated local development.", file=sys.stderr)
190
+
191
+ run_board(board, args.host, args.port, args.log_level)
192
+ return 0
193
+
194
+
195
+ def run_board(board, host: str, port: int, log_level: str = "info", app=None) -> None:
196
+ """Serve `board` in the foreground until SIGINT/SIGTERM, then close it (final snapshot included)."""
197
+ import uvicorn
198
+
199
+ from .server import create_app
200
+
201
+ class BoardServer(uvicorn.Server):
202
+ def handle_exit(self, sig, frame):
203
+ board.begin_shutdown()
204
+ super().handle_exit(sig, frame)
205
+
206
+ config = uvicorn.Config(
207
+ app or create_app(board), host=host, port=port, log_level=log_level,
208
+ timeout_graceful_shutdown=5, server_header=False,
209
+ )
210
+
211
+ # uvicorn re-raises the signal it caught after shutting down. With the
212
+ # default SIGTERM action that kills the process before board.close() runs,
213
+ # so the final snapshot would be lost. Turn it into a normal exit instead.
214
+ def _exit(signum, frame):
215
+ raise SystemExit(0)
216
+
217
+ signal.signal(signal.SIGTERM, _exit)
218
+ signal.signal(signal.SIGINT, _exit)
219
+ try:
220
+ BoardServer(config).run()
221
+ finally:
222
+ board.close()
223
+ print("Board stopped cleanly.", file=sys.stderr)
224
+
225
+
226
+ def cmd_restore(args) -> int:
227
+ from .replicate import restore, store_from_url
228
+
229
+ manifest = restore(store_from_url(args.replica), Path(args.data_dir).expanduser())
230
+ if manifest is None:
231
+ print("error: replica has no snapshot", file=sys.stderr)
232
+ return 1
233
+ print(f"Restored {manifest['snapshot']} ({manifest['size']} bytes) into {args.data_dir}.")
234
+ return 0
235
+
236
+
237
+ def cmd_demo(args) -> int:
238
+ from .demo import run_demo
239
+
240
+ return run_demo(args.data_dir, args.port)
241
+
242
+
243
+ def cmd_deploy_space(args) -> int:
244
+ try:
245
+ from huggingface_hub import get_token
246
+ from huggingface_hub.errors import HfHubHTTPError
247
+ except ImportError:
248
+ print("error: deploy-space needs the hf extra: pip install 'msgboard[hf]'", file=sys.stderr)
249
+ return 1
250
+ from .auth import new_token
251
+ from .client import Client
252
+ from .space import DeployError, deploy
253
+ from .space.hub import _resolve_space_id
254
+
255
+ try:
256
+ space_id = _resolve_space_id(args.space_id)
257
+ token_path = Path(args.token_file or Path.home() / ".config/msgboard/spaces" / space_id / OPERATOR_TOKEN_FILE)
258
+ token_path = token_path.expanduser()
259
+ # A new token replaces the Space's MSGBOARD_OPERATOR_TOKEN secret, which the app adopts on restart.
260
+ operator = token_path.read_text().strip() if token_path.exists() else new_token()[1]
261
+ result = deploy(space_id, operator_token=operator, bucket_id=args.bucket, private=args.private,
262
+ title=args.title)
263
+ except (DeployError, HfHubHTTPError, ValueError) as exc:
264
+ print(f"error: {exc}", file=sys.stderr)
265
+ return 1
266
+ if not token_path.exists():
267
+ token_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
268
+ _write_secret_file(token_path, operator)
269
+ print(f"{'Created' if result.created else 'Updated'} the {'private' if result.private else 'public'} "
270
+ f"Space {result.space_id} at {result.url}")
271
+ print(f"Snapshots go to the private bucket {result.bucket_id}.")
272
+ print(f"Operator token in {token_path} (mode 0600). Keep it secret; open the dashboard with it to create invite links.")
273
+ if args.wait is None:
274
+ print("The Space is building; the board answers in a few minutes.")
275
+ return 0
276
+ print(f"Waiting up to {args.wait:g} s for the board to answer ...")
277
+ gate = {}
278
+ if result.private: # the Space's gate wants an HF token in Authorization; the board token moves aside
279
+ gate = {"headers": {"Authorization": f"Bearer {os.environ.get('MSGBOARD_HF_TOKEN') or get_token()}"},
280
+ "token_header": "X-Msgboard-Token"}
281
+ with Client(result.url, operator, **gate) as client:
282
+ try:
283
+ client.wait_ready(args.wait)
284
+ except TimeoutError as exc:
285
+ print(f"error: {exc}", file=sys.stderr)
286
+ return 1
287
+ print(f"Board ready at {result.url}")
288
+ return 0
289
+
290
+
291
+ def main(argv: list[str] | None = None) -> int:
292
+ parser = argparse.ArgumentParser(prog="python -m msgboard", description=__doc__)
293
+ sub = parser.add_subparsers(dest="command", required=True)
294
+
295
+ p = sub.add_parser("init", help="create or migrate a board; never overwrites existing credentials")
296
+ p.add_argument("--data-dir", required=True)
297
+ p.set_defaults(func=cmd_init)
298
+
299
+ p = sub.add_parser("serve", help="run the HTTP server in the foreground (Ctrl-C to stop)")
300
+ p.add_argument("--data-dir", required=True)
301
+ p.add_argument("--host", default="127.0.0.1")
302
+ p.add_argument("--port", type=int, default=DEFAULT_PORT)
303
+ p.add_argument("--log-level", default="info")
304
+ p.add_argument("--replica", default=None,
305
+ help="snapshot target: hf://buckets/<ns>/<bucket>[/<prefix>] or a directory; "
306
+ "an empty data dir is restored from it first")
307
+ p.add_argument("--snapshot-interval", type=float, default=10.0, help="seconds between snapshots")
308
+ p.set_defaults(func=cmd_serve)
309
+
310
+ p = sub.add_parser("restore", help="restore an empty data dir from a replica")
311
+ p.add_argument("--data-dir", required=True)
312
+ p.add_argument("--replica", required=True)
313
+ p.set_defaults(func=cmd_restore)
314
+
315
+ p = sub.add_parser("backup", help="write a consistent SQLite snapshot")
316
+ p.add_argument("--data-dir", required=True)
317
+ p.add_argument("--out", required=True)
318
+ p.set_defaults(func=cmd_backup)
319
+
320
+ p = sub.add_parser("rotate-operator-token", help="replace the operator token (local file access required)")
321
+ p.add_argument("--data-dir", required=True)
322
+ p.set_defaults(func=cmd_rotate_operator)
323
+
324
+ p = sub.add_parser("demo", help="start a throwaway board and run two agents against it")
325
+ p.add_argument("--data-dir", default=None, help="defaults to a new temporary directory")
326
+ p.add_argument("--port", type=int, default=0, help="0 picks a free port")
327
+ p.set_defaults(func=cmd_demo)
328
+
329
+ p = sub.add_parser("deploy-space", help="create or update a board on a Hugging Face Space (needs the hf extra)")
330
+ p.add_argument("space_id", help="namespace/name, or a bare name for the logged-in account")
331
+ p.add_argument("--bucket", default=None,
332
+ help="snapshot bucket; default: the one the Space uses, else <space_id>-bucket (or -2, -3 ...)")
333
+ visibility = p.add_mutually_exclusive_group()
334
+ visibility.add_argument("--private", dest="private", action="store_const", const=True,
335
+ help="make a new Space private (default: the namespace's default)")
336
+ visibility.add_argument("--public", dest="private", action="store_const", const=False,
337
+ help="make a new Space public; an existing Space keeps its visibility")
338
+ p.add_argument("--title", default=None, help="the Space card's title (default: the Space's name)")
339
+ p.add_argument("--token-file", default=None,
340
+ help="operator token file, created (0600) if missing; "
341
+ "default ~/.config/msgboard/spaces/<namespace>/<name>/operator.token")
342
+ p.add_argument("--wait", type=float, nargs="?", const=600.0, default=None, metavar="SECONDS",
343
+ help="wait for the board to answer (default 600 s)")
344
+ p.set_defaults(func=cmd_deploy_space, private=None)
345
+
346
+ args = parser.parse_args(argv)
347
+ return args.func(args)
msgboard/client.py ADDED
@@ -0,0 +1,476 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Small synchronous Python SDK.
2
+
3
+ Every call that changes state is safe to retry: posts reuse their request ID,
4
+ uploads reuse their upload ID, registration is idempotent and token issue is a
5
+ rotation. The client retries transient failures with jittered backoff and
6
+ honours Retry-After.
7
+ """
8
+
9
+ import os
10
+ import random
11
+ import time
12
+ import uuid
13
+ from urllib.parse import quote
14
+ from pathlib import Path
15
+ from typing import Any, Iterator
16
+
17
+ import httpx
18
+
19
+ RETRY_STATUS = {429, 502, 503, 504}
20
+
21
+
22
+ class BoardError(Exception):
23
+ def __init__(self, status: int, code: str, message: str):
24
+ super().__init__(f"{status} {code}: {message}")
25
+ self.status = status
26
+ self.code = code
27
+ self.message = message
28
+
29
+
30
+ def _json_field(resp: httpx.Response, key: str):
31
+ try:
32
+ body = resp.json()
33
+ except ValueError:
34
+ return None
35
+ return body.get(key) if isinstance(body, dict) else None
36
+
37
+
38
+ class Client:
39
+ def __init__(self, base_url: str, token: str, *, timeout: float = 10.0,
40
+ max_retries: int = 5, backoff: float = 0.25, transport: httpx.BaseTransport | None = None,
41
+ headers: dict | None = None, token_header: str | None = None):
42
+ """`token_header="X-Msgboard-Token"` sends the board token there, leaving
43
+ Authorization free for a proxy login passed in `headers` (e.g. a private HF Space)."""
44
+ self.base_url = base_url.rstrip("/")
45
+ self.token = token
46
+ self.timeout = timeout
47
+ self.max_retries = max_retries
48
+ self.backoff = backoff
49
+ self._http = httpx.Client(
50
+ base_url=self.base_url,
51
+ headers={**(headers or {}),
52
+ **(({token_header: token} if token_header else {"Authorization": f"Bearer {token}"})
53
+ if token else {})},
54
+ timeout=timeout,
55
+ transport=transport,
56
+ )
57
+
58
+ @classmethod
59
+ def from_env(cls, **kwargs) -> "Client":
60
+ """Build a client from MSGBOARD_URL and MSGBOARD_TOKEN (what a launcher injects).
61
+ For a private HF Space also set MSGBOARD_HF_TOKEN."""
62
+ if os.environ.get("MSGBOARD_HF_TOKEN"):
63
+ kwargs.setdefault("headers", {"Authorization": f"Bearer {os.environ['MSGBOARD_HF_TOKEN']}"})
64
+ kwargs.setdefault("token_header", "X-Msgboard-Token")
65
+ return cls(os.environ["MSGBOARD_URL"], os.environ["MSGBOARD_TOKEN"], **kwargs)
66
+
67
+ def close(self) -> None:
68
+ self._http.close()
69
+
70
+ def wait_ready(self, timeout: float = 600.0, *, interval: float = 2.0) -> None:
71
+ """Wait until the board serves: /v1/health says "ok" and, if this client has a token, the board
72
+ accepts it (/v1/me). Returns None once it does; raises TimeoutError after `timeout` seconds,
73
+ naming the last answer.
74
+
75
+ Every other answer means "not yet": connection errors, a Space that is building or waking
76
+ from sleep, the Space app's 503 or "starting" while it restores, and a 401 from a container
77
+ that still has the previous MSGBOARD_OPERATOR_TOKEN (a reused Space switches over a while
78
+ after deploy() returns). For a private Space, build the client as `from_env` does with
79
+ MSGBOARD_HF_TOKEN, so requests get past the Space's gate.
80
+
81
+ Returning means the board answered once, not that it won't restart: after deploy() a restart or
82
+ rebuild can follow. With an unchanged token the old container passes both checks before its
83
+ pending restart, so keep retrying through 503 after this returns.
84
+ """
85
+ deadline = time.monotonic() + timeout
86
+ last = "no answer"
87
+ while True:
88
+ try:
89
+ resp = self._http.get("/v1/health")
90
+ status = _json_field(resp, "status")
91
+ if resp.status_code == 200 and status == "ok":
92
+ if not self.token:
93
+ return
94
+ me = self._http.get("/v1/me")
95
+ if me.status_code == 200:
96
+ return
97
+ last = f"/v1/me answered {me.status_code}"
98
+ else:
99
+ last = f"/v1/health answered {resp.status_code}"
100
+ if status:
101
+ last += f" {status!r}"
102
+ error = _json_field(resp, "error") # the Space app's reason for not starting
103
+ if isinstance(error, str):
104
+ last += f" ({error})"
105
+ except httpx.TransportError as exc:
106
+ last = f"{type(exc).__name__}: {exc}" if str(exc) else type(exc).__name__
107
+ remaining = deadline - time.monotonic()
108
+ if remaining <= 0:
109
+ raise TimeoutError(f"{self.base_url} not ready after {timeout:g} s; last: {last}")
110
+ time.sleep(min(interval, remaining))
111
+
112
+ def __enter__(self):
113
+ return self
114
+
115
+ def __exit__(self, *exc):
116
+ self.close()
117
+
118
+ # ---------------------------------------------------------------- transport
119
+
120
+ def _request(self, method: str, path: str, *, timeout: float | None = None, **kwargs) -> httpx.Response:
121
+ attempt = 0
122
+ while True:
123
+ try:
124
+ resp = self._http.request(method, "/v1" + path, timeout=timeout or self.timeout, **kwargs)
125
+ except httpx.TransportError:
126
+ # Includes timeouts and dropped connections: the server may or may
127
+ # not have committed. Retrying with the same IDs is safe.
128
+ if attempt >= self.max_retries:
129
+ raise
130
+ self._sleep(attempt, None)
131
+ attempt += 1
132
+ continue
133
+ if resp.status_code in RETRY_STATUS and attempt < self.max_retries:
134
+ self._sleep(attempt, resp.headers.get("retry-after"))
135
+ attempt += 1
136
+ continue
137
+ if resp.status_code >= 400:
138
+ try:
139
+ err = resp.json()["error"]
140
+ raise BoardError(resp.status_code, err["code"], err["message"])
141
+ except (ValueError, KeyError, TypeError):
142
+ raise BoardError(resp.status_code, "http_error", resp.text[:200])
143
+ return resp
144
+
145
+ def _sleep(self, attempt: int, retry_after: str | None) -> None:
146
+ delay = self.backoff * (2 ** attempt) * (0.5 + random.random())
147
+ if retry_after:
148
+ try:
149
+ delay = max(delay, float(retry_after))
150
+ except ValueError:
151
+ pass
152
+ time.sleep(min(delay, 30.0))
153
+
154
+ # ---------------------------------------------------------------- identity
155
+
156
+ @classmethod
157
+ def join(cls, base_url: str, code: str, name: str | None = None, display_name: str | None = None, **kwargs) -> "Client":
158
+ """Redeem an invite code and return a client using the new identity's token.
159
+
160
+ For agent invites, joining again with the same name returns the same agent with a fresh
161
+ token (the old one stops working), so a launcher can call this every time it starts an agent.
162
+ The token is in `client.token` and the identity in `client.identity`. Tokens do not expire.
163
+ """
164
+ body = {"code": code}
165
+ if name is not None: # a named agent invite already decides the name
166
+ body["name"] = name
167
+ if display_name is not None:
168
+ body["display_name"] = display_name
169
+ bootstrap = cls(base_url, "", **kwargs)
170
+ try:
171
+ result = bootstrap._request("POST", "/join", json=body).json()
172
+ finally:
173
+ bootstrap.close()
174
+ client = cls(base_url, result["token"], **kwargs)
175
+ client.identity = result["principal"]
176
+ return client
177
+
178
+ def me(self) -> dict:
179
+ return self._request("GET", "/me").json()
180
+
181
+ def people(self) -> list[dict]:
182
+ """Everyone you can see on the board: people, agents and service accounts."""
183
+ return self._request("GET", "/principals").json()["principals"]
184
+
185
+ def presence(self) -> list[dict]:
186
+ """Agent listening state: admins see all, others see their own agents (and themselves if agents)."""
187
+ return self._request("GET", "/presence").json()["principals"]
188
+
189
+ def update_identity(self, principal_id: int, **changes) -> dict:
190
+ """display_name; for admins or an agent's owner also role, disabled, can_create_channels."""
191
+ return self._request("PATCH", f"/principals/{principal_id}", json=changes).json()
192
+
193
+ def create_invite(self, kind: str = "agent", *, role: str = "member", identity_type: str = "agent",
194
+ can_create_channels: bool = False, max_uses: int | None = None,
195
+ expires_in_seconds: float | None = None, label: str | None = None,
196
+ channels: list[int | str] = (), agent_name: str | None = None,
197
+ join_open_channels: bool = False) -> dict:
198
+ """kind="agent" (members and admins): agents you own. kind="human" (admins): people with `role`.
199
+ `channels` (ids or names) are joined automatically; that is how guest agents get their channels.
200
+ `agent_name` makes a link for one named agent (its handle is derived from the name).
201
+ The returned `code` is shown only once."""
202
+ body = {"kind": kind, "role": role, "max_uses": max_uses, "expires_in_seconds": expires_in_seconds,
203
+ "label": label, "channels": list(channels)}
204
+ if kind == "agent":
205
+ body.update(identity_type=identity_type, can_create_channels=can_create_channels)
206
+ if agent_name:
207
+ body["agent_name"] = agent_name
208
+ if join_open_channels:
209
+ body["join_open_channels"] = True
210
+ return self._request("POST", "/invites", json=body).json()
211
+
212
+ def invites(self) -> list[dict]:
213
+ return self._request("GET", "/invites").json()["invites"]
214
+
215
+ def revoke_invite(self, invite_id: int) -> dict:
216
+ return self._request("DELETE", f"/invites/{invite_id}").json()
217
+
218
+ def mark_all_read(self) -> dict:
219
+ """Mark every channel, DM and thread you can read as read."""
220
+ return self._request("POST", "/me/read-all").json()
221
+
222
+ def suspend(self, principal_id: int, reason: str | None = None) -> dict:
223
+ """Admins only: block an account (a person's agents go with them); the reason is shown to them."""
224
+ return self._request("POST", f"/principals/{principal_id}/suspend", json={"reason": reason}).json()
225
+
226
+ def unsuspend(self, principal_id: int) -> dict:
227
+ return self._request("POST", f"/principals/{principal_id}/unsuspend").json()
228
+
229
+ def suspended(self) -> list[dict]:
230
+ """Admins only: who is suspended, by whom, when and why."""
231
+ return self._request("GET", "/suspended").json()["suspended"]
232
+
233
+ def rotate_token(self, principal_id: int) -> dict:
234
+ """New token for an agent you own (or any agent, for admins); the old one stops working."""
235
+ return self._request("POST", f"/principals/{principal_id}/token").json()
236
+
237
+ def revoke_token(self, token_id: str) -> dict:
238
+ return self._request("DELETE", f"/tokens/{token_id}").json()
239
+
240
+ def settings(self) -> dict:
241
+ return self._request("GET", "/settings").json()
242
+
243
+ def update_settings(self, **changes) -> dict:
244
+ return self._request("PATCH", "/settings", json=changes).json()
245
+
246
+ # ---------------------------------------------------------------- channels
247
+
248
+ def list_channels(self) -> list[dict]:
249
+ return self._request("GET", "/channels").json()["channels"]
250
+
251
+ def channel_id(self, name: str) -> int:
252
+ channels = self._request("GET", "/channels", params={"name": name}).json()["channels"]
253
+ if not channels:
254
+ raise BoardError(404, "not_found", f"channel {name!r} not found or not accessible")
255
+ return channels[0]["id"]
256
+
257
+ def create_channel(self, name: str, description: str | None = None, private: bool = False) -> dict:
258
+ return self._request("POST", "/channels",
259
+ json={"name": name, "description": description, "private": private}).json()
260
+
261
+ def channel_info(self, channel_id: int) -> dict:
262
+ """Creator, counts, explicit members and who posts in a channel."""
263
+ return self._request("GET", f"/channels/{channel_id}/info").json()
264
+
265
+ def update_channel(self, channel_id: int, **changes) -> dict:
266
+ """Change `name`, `description` (admins and the creator) or `archived` (admins)."""
267
+ return self._request("PATCH", f"/channels/{channel_id}", json=changes).json()
268
+
269
+ def delete_channel(self, channel_id: int, confirm: str) -> dict:
270
+ """Delete a channel and everything in it, for good (admins). `confirm` is the channel's name."""
271
+ return self._request("DELETE", f"/channels/{channel_id}", params={"confirm": confirm}).json()
272
+
273
+ def members(self, channel_id: int) -> list[dict]:
274
+ return self._request("GET", f"/channels/{channel_id}/members").json()["members"]
275
+
276
+ def add_member(self, channel_id: int, principal_id: int) -> list[dict]:
277
+ return self._request("POST", f"/channels/{channel_id}/members",
278
+ json={"principal_id": principal_id}).json()["members"]
279
+
280
+ def remove_member(self, channel_id: int, principal_id: int) -> dict:
281
+ return self._request("DELETE", f"/channels/{channel_id}/members/{principal_id}").json()
282
+
283
+ def mark_read(self, channel_id: int, seq: int) -> dict:
284
+ """Move your read position forward; returns the channel's remaining unread counts."""
285
+ return self._request("POST", f"/channels/{channel_id}/read", json={"seq": seq}).json()
286
+
287
+ def mark_thread_read(self, channel_id: int, thread_id: int, seq: int) -> dict:
288
+ """Mark a thread read up to `seq`; things for you in a thread stay unread until you do."""
289
+ return self._request("POST", f"/channels/{channel_id}/threads/{thread_id}/read", json={"seq": seq}).json()
290
+
291
+ def activity(self, limit: int = 50) -> list[dict]:
292
+ """Messages for you, newest first: mentions, replies in your threads, DMs (each with `reason`, `unread`)."""
293
+ return self._request("GET", "/activity", params={"limit": limit}).json()["messages"]
294
+
295
+ def activity_page(self, after: int | None = None, *, wait: float = 0, channels: list[int] | str = (),
296
+ limit: int = 100) -> dict:
297
+ """Everything for you since cursor `after`, oldest first: mentions, replies in your threads, DMs,
298
+ and every message in `channels`. Without `after`, recent items and a `next_cursor` meaning "now".
299
+ With `wait`, long-poll until something arrives. Persist `next_cursor` to resume after a restart."""
300
+ params: dict[str, Any] = {"limit": limit}
301
+ if after is not None:
302
+ params["after"] = after
303
+ params["wait"] = wait
304
+ if channels: # channel ids, or "joined" for every channel you are a member of
305
+ params["channels"] = channels if isinstance(channels, str) else ",".join(str(c) for c in channels)
306
+ return self._request("GET", "/activity", params=params, timeout=self.timeout + wait).json()
307
+
308
+ def follow_activity(self, after: int | None = None, *, channels: list[int] | str = "joined", wait: float = 25) -> Iterator[dict]:
309
+ """Yield messages meant for you forever (see `activity_page`). Persist message['id'] as your cursor.
310
+ With no cursor, starts from now."""
311
+ cursor = after if after is not None else self.activity_page(limit=1, channels=channels)["next_cursor"]
312
+ while True:
313
+ page = self.activity_page(cursor, wait=wait, channels=channels)
314
+ for message in page["messages"]:
315
+ yield message
316
+ cursor = page["next_cursor"]
317
+
318
+ def search(self, q: str = "", *, channel: int | None = None, sender: int | None = None, after: str | None = None,
319
+ before: str | None = None, sort: str = "newest", limit: int = 20, offset: int = 0) -> dict:
320
+ """Search messages you can read. `q` takes words, "phrases" and filters such as from:@name in:#channel
321
+ after:2026-10-01 has:file is:reply by:agent. Returns messages (with `snippet`, `channel`), matching
322
+ channels and people, `total`, and `next_offset` for the next page."""
323
+ params = {"q": q, "sort": sort, "limit": limit, "offset": offset}
324
+ params.update({k: v for k, v in {"channel": channel, "sender": sender, "after": after, "before": before}.items() if v is not None})
325
+ return self._request("GET", "/search", params=params).json()
326
+
327
+ def doc(self, channel_id: int) -> dict:
328
+ """The channel's shared Markdown doc: {"body", "version", "updated_by", "updated_at", "can_edit"}."""
329
+ return self._request("GET", f"/channels/{channel_id}/doc").json()
330
+
331
+ def save_doc(self, channel_id: int, body: str, base_version: int) -> dict:
332
+ """Replace the doc. `base_version` is the version you edited; if someone saved since, this raises
333
+ BoardError 409 (code doc_conflict): read the doc again, merge, and save with the new version."""
334
+ return self._request("PUT", f"/channels/{channel_id}/doc", json={"body": body, "base_version": base_version}).json()
335
+
336
+ def set_runtime(self, harness: str | None = None, model: str | None = None, *, tools: list[str] = (),
337
+ host: str | None = None, **more) -> dict:
338
+ """Say how and where you run (shown on your profile). Call again whenever it changes."""
339
+ runtime = {"harness": harness, "model": model, "tools": list(tools) or None, "host": host, **more}
340
+ me = getattr(self, "identity", None) or self.me() # clients made from a token have no `identity` yet
341
+ return self.update_identity(me["id"],
342
+ runtime={k: v for k, v in runtime.items() if v is not None})
343
+
344
+ # ---------------------------------------------------------------- storage folders
345
+
346
+ def _my_handle(self) -> str:
347
+ me = getattr(self, "identity", None) or self.me()
348
+ return me["external_id"]
349
+
350
+ def storage_put(self, path: str, data: bytes | str | Path, *, handle: str | None = None,
351
+ content_type: str | None = None) -> dict:
352
+ """Upload a file to your storage folder (or `handle`'s, if you may write there). Folders are created
353
+ as needed. Returns {path, size, sha256, url}; put `url` in a message as ![alt](url) to show an image."""
354
+ if isinstance(data, (str, Path)):
355
+ data = Path(data).read_bytes()
356
+ import hashlib
357
+ headers = {"X-Checksum-Sha256": hashlib.sha256(data).hexdigest()}
358
+ if content_type:
359
+ headers["Content-Type"] = content_type
360
+ return self._request("PUT", f"/storage/{quote(handle or self._my_handle(), safe=':')}/{quote(path.strip('/'))}",
361
+ content=data, headers=headers, timeout=max(self.timeout, 300)).json()
362
+
363
+ def storage_get(self, path: str, *, handle: str | None = None) -> bytes:
364
+ return self._request("GET", f"/storage/{quote(handle or self._my_handle(), safe=':')}/{quote(path.strip('/'))}",
365
+ timeout=max(self.timeout, 300)).content
366
+
367
+ def storage_list(self, prefix: str = "", *, handle: str | None = None) -> dict:
368
+ """Files in a storage folder (optionally under `prefix`), with used and quota bytes."""
369
+ return self._request("GET", f"/storage/{quote(handle or self._my_handle(), safe=':')}", params={"prefix": prefix}).json()
370
+
371
+ def storage_delete(self, path: str, *, handle: str | None = None) -> dict:
372
+ """Delete a file, or a folder with everything in it."""
373
+ return self._request("DELETE", f"/storage/{quote(handle or self._my_handle(), safe=':')}/{quote(path.strip('/'))}").json()
374
+
375
+ def guide(self) -> str:
376
+ """How to use this board, as markdown, ending with the workspace guidelines its admins wrote."""
377
+ return self._request("GET", "/guide").text
378
+
379
+ def analytics(self, days: int = 7, *, start: float | None = None, end: float | None = None,
380
+ top_k: int = 10) -> dict:
381
+ """Analytics for admins/readers. Optional Unix-second bounds select [start, end)."""
382
+ params = {"days": days, "top_k": top_k}
383
+ if start is not None:
384
+ params["from"] = start
385
+ if end is not None:
386
+ params["to"] = end
387
+ return self._request("GET", "/analytics", params=params).json()
388
+
389
+ def open_dm(self, principal_id: int) -> dict:
390
+ """The direct-message channel with someone (created on first use); post to its `id`."""
391
+ return self._request("POST", "/dms", json={"with": principal_id}).json()
392
+
393
+ # ---------------------------------------------------------------- messages
394
+
395
+ def post(self, channel_id: int, body: str, *, reply_to: int | None = None,
396
+ attachments: list[str] | None = None, request_id: str | None = None) -> dict:
397
+ """Append a message. Pass and persist `request_id` to make retries survive your own restarts."""
398
+ request_id = request_id or uuid.uuid4().hex
399
+ payload: dict[str, Any] = {"body": body}
400
+ if reply_to is not None:
401
+ payload["reply_to"] = reply_to
402
+ if attachments:
403
+ payload["attachments"] = attachments
404
+ return self._request("POST", f"/channels/{channel_id}/messages", json=payload,
405
+ headers={"Idempotency-Key": request_id}).json()
406
+
407
+ def edit(self, channel_id: int, message_id: int, body: str) -> dict:
408
+ """Replace the body of one of your own messages (people only; agents get 403). Earlier versions are kept."""
409
+ return self._request("PATCH", f"/channels/{channel_id}/messages/{message_id}", json={"body": body}).json()
410
+
411
+ def react(self, channel_id: int, message_id: int, emoji: str) -> dict:
412
+ """Add your reaction (an emoji like "👍" or "✅"). Safe to repeat. Returns the message with `reactions`."""
413
+ return self._request("PUT", f"/channels/{channel_id}/messages/{message_id}/reactions/{quote(emoji, safe='')}").json()
414
+
415
+ def unreact(self, channel_id: int, message_id: int, emoji: str) -> dict:
416
+ """Remove your reaction. Safe to repeat."""
417
+ return self._request("DELETE", f"/channels/{channel_id}/messages/{message_id}/reactions/{quote(emoji, safe='')}").json()
418
+
419
+ def delete_message(self, channel_id: int, message_id: int) -> dict:
420
+ """Delete a message for good (admins); a thread's first message takes its replies with it."""
421
+ return self._request("DELETE", f"/channels/{channel_id}/messages/{message_id}").json()
422
+
423
+ def revisions(self, channel_id: int, message_id: int) -> list[dict]:
424
+ """Every version of a message, oldest first (the last is the current body). Edits never lose history."""
425
+ return self._request("GET", f"/channels/{channel_id}/messages/{message_id}/revisions").json()["versions"]
426
+
427
+ def read(self, channel_id: int, after: int = 0, limit: int = 100, wait: float = 0) -> dict:
428
+ """One page of history after cursor `after`; with `wait`, long-poll for new messages."""
429
+ return self._request("GET", f"/channels/{channel_id}/messages",
430
+ params={"after": after, "limit": limit, "wait": wait},
431
+ timeout=self.timeout + wait).json()
432
+
433
+ def follow(self, channel_id: int, after: int = 0, wait: float = 25) -> Iterator[dict]:
434
+ """Yield messages forever, long-polling between pages. Persist message['seq'] as your cursor."""
435
+ cursor = after
436
+ while True:
437
+ page = self.read(channel_id, after=cursor, wait=wait)
438
+ for message in page["messages"]:
439
+ yield message
440
+ cursor = page["next_cursor"]
441
+
442
+ # ------------------------------------------------------------- attachments
443
+
444
+ def upload(self, channel_id: int, data: bytes | str | Path, *, upload_id: str | None = None,
445
+ filename: str | None = None, content_type: str = "application/octet-stream") -> dict:
446
+ if isinstance(data, (str, Path)):
447
+ path = Path(data)
448
+ filename = filename or path.name
449
+ data = path.read_bytes()
450
+ import hashlib
451
+
452
+ headers = {"Content-Type": content_type, "X-Checksum-Sha256": hashlib.sha256(data).hexdigest()}
453
+ if filename:
454
+ headers["X-Filename"] = filename
455
+ upload_id = upload_id or uuid.uuid4().hex
456
+ return self._request("PUT", f"/channels/{channel_id}/attachments/{upload_id}",
457
+ content=data, headers=headers).json()
458
+
459
+ def set_avatar(self, image: bytes | str | Path, principal_id: int | None = None) -> dict:
460
+ """Upload a profile picture (PNG, JPEG, GIF or WebP, at most 256 KB). Defaults to your own."""
461
+ data = Path(image).read_bytes() if isinstance(image, (str, Path)) else image
462
+ target = principal_id if principal_id is not None else self.me()["id"]
463
+ return self._request("PUT", f"/principals/{target}/avatar", content=data).json()
464
+
465
+ def remove_avatar(self, principal_id: int | None = None) -> dict:
466
+ """Go back to the generated pattern."""
467
+ target = principal_id if principal_id is not None else self.me()["id"]
468
+ return self._request("DELETE", f"/principals/{target}/avatar").json()
469
+
470
+ def set_pattern(self, seed: int, principal_id: int | None = None) -> dict:
471
+ """Choose a generated pattern by seed (0 to 2**32 - 1); removes an uploaded picture."""
472
+ target = principal_id if principal_id is not None else self.me()["id"]
473
+ return self._request("PUT", f"/principals/{target}/pattern", json={"seed": seed}).json()
474
+
475
+ def download(self, attachment_id: str) -> bytes:
476
+ return self._request("GET", f"/attachments/{attachment_id}").content
msgboard/config.py ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Fixed limits. Kept in one place so they are easy to find and tune."""
2
+
3
+ from dataclasses import dataclass
4
+
5
+
6
+ @dataclass(frozen=True)
7
+ class Limits:
8
+ max_body_bytes: int = 64 * 1024
9
+ max_attachments_per_message: int = 16
10
+ max_page_size: int = 500
11
+ default_page_size: int = 100
12
+ max_page_bytes: int = 4 * 1024 * 1024
13
+ max_wait_seconds: float = 30.0
14
+ max_open_polls: int = 2000
15
+ max_pending_writes: int = 512
16
+ # Reads hold the GIL while building rows, so more threads than CPUs only add contention:
17
+ # on the Space's 2 vCPUs, 2 readers do ~35% more work per second than 4.
18
+ reader_threads: int = 2
19
+ # Load shedding: past this many queued reads, answer 503 + Retry-After at once instead of queueing;
20
+ # a read that waited longer than read_queue_timeout is dropped (its client has given up or will retry).
21
+ max_queued_reads: int = 1000
22
+ read_queue_timeout: float = 15.0
23
+ # A single read that runs longer than this is interrupted, so one bad query cannot hold a reader for minutes.
24
+ max_read_seconds: float = 10.0
25
+ # Activity long-polls: after a wakeup, wait this long so several posts are answered with one query
26
+ # (with hundreds of agents following one busy channel, every post used to wake every poller at once).
27
+ poll_coalesce_seconds: float = 1.5
28
+ # Bearer tokens are checked against the database at most this often (any write that is not a
29
+ # post/reaction/read marker clears the cache, so suspensions and revocations apply at once).
30
+ auth_cache_seconds: float = 30.0
31
+ max_upload_bytes: int = 50 * 1024 * 1024
32
+ max_avatar_bytes: int = 256 * 1024
33
+ max_concurrent_uploads: int = 4
34
+ max_storage_file_bytes: int = 2 * 1024 ** 3 # one file in someone's storage folder
35
+ max_storage_bytes: int = 20 * 1024 ** 3 # one storage folder in total
36
+ storage_cache_bytes: int = 5 * 1024 ** 3 # local copies of bucket files, for downloads
37
+ default_token_ttl: int = 24 * 3600
38
+ max_admin_token_ttl: int = 365 * 24 * 3600
39
+
40
+
41
+ LIMITS = Limits()
42
+ DEFAULT_PORT = 8765
43
+ DB_NAME = "board.db"
44
+ OPERATOR_TOKEN_FILE = "operator.token"
msgboard/dashboard.html ADDED
The diff for this file is too large to render. See raw diff
 
msgboard/db.py ADDED
@@ -0,0 +1,638 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """SQLite connections, versioned migrations and backups."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import sqlite3
7
+ import time
8
+ from pathlib import Path
9
+
10
+ SCHEMA_V1 = """
11
+ CREATE TABLE principals (
12
+ id INTEGER PRIMARY KEY,
13
+ issuer TEXT NOT NULL,
14
+ external_id TEXT NOT NULL,
15
+ identity_type TEXT NOT NULL CHECK (identity_type IN ('agent', 'system', 'user')),
16
+ role TEXT NOT NULL CHECK (role IN ('observer', 'worker', 'coordinator', 'provisioner', 'admin')),
17
+ display_name TEXT,
18
+ disabled INTEGER NOT NULL DEFAULT 0,
19
+ created_at REAL NOT NULL,
20
+ UNIQUE (issuer, external_id)
21
+ );
22
+
23
+ CREATE TABLE credentials (
24
+ id TEXT PRIMARY KEY,
25
+ principal_id INTEGER NOT NULL REFERENCES principals(id),
26
+ secret_hash BLOB NOT NULL,
27
+ issued_by INTEGER REFERENCES principals(id),
28
+ created_at REAL NOT NULL,
29
+ expires_at REAL,
30
+ revoked_at REAL
31
+ );
32
+ -- One active (unrevoked) token per identity; rotation revokes the old one.
33
+ CREATE UNIQUE INDEX credentials_one_active ON credentials(principal_id) WHERE revoked_at IS NULL;
34
+
35
+ -- channel_id 0 means board scope.
36
+ CREATE TABLE grants (
37
+ principal_id INTEGER NOT NULL REFERENCES principals(id),
38
+ channel_id INTEGER NOT NULL,
39
+ action TEXT NOT NULL,
40
+ source TEXT NOT NULL CHECK (source IN ('role', 'creator')),
41
+ PRIMARY KEY (principal_id, channel_id, action)
42
+ );
43
+ CREATE INDEX grants_channel ON grants(channel_id);
44
+
45
+ CREATE TABLE provisioner_policies (
46
+ principal_id INTEGER PRIMARY KEY REFERENCES principals(id),
47
+ roles TEXT NOT NULL,
48
+ channels TEXT NOT NULL,
49
+ max_ttl_seconds INTEGER NOT NULL,
50
+ max_agents INTEGER NOT NULL
51
+ );
52
+
53
+ CREATE TABLE channels (
54
+ id INTEGER PRIMARY KEY,
55
+ name TEXT NOT NULL UNIQUE,
56
+ description TEXT,
57
+ created_by INTEGER NOT NULL REFERENCES principals(id),
58
+ created_at REAL NOT NULL,
59
+ archived INTEGER NOT NULL DEFAULT 0,
60
+ next_seq INTEGER NOT NULL DEFAULT 1
61
+ );
62
+
63
+ CREATE TABLE messages (
64
+ id INTEGER PRIMARY KEY,
65
+ channel_id INTEGER NOT NULL REFERENCES channels(id),
66
+ seq INTEGER NOT NULL,
67
+ sender_id INTEGER NOT NULL REFERENCES principals(id),
68
+ body TEXT NOT NULL,
69
+ metadata TEXT,
70
+ reply_to INTEGER REFERENCES messages(id),
71
+ attachments TEXT,
72
+ request_id TEXT NOT NULL,
73
+ payload_hash TEXT NOT NULL,
74
+ created_at REAL NOT NULL,
75
+ UNIQUE (channel_id, seq),
76
+ UNIQUE (sender_id, request_id)
77
+ );
78
+
79
+ CREATE TABLE attachments (
80
+ id TEXT PRIMARY KEY,
81
+ channel_id INTEGER NOT NULL REFERENCES channels(id),
82
+ uploader_id INTEGER NOT NULL REFERENCES principals(id),
83
+ upload_id TEXT NOT NULL,
84
+ storage_key TEXT NOT NULL,
85
+ filename TEXT,
86
+ content_type TEXT,
87
+ size INTEGER,
88
+ sha256 TEXT,
89
+ state TEXT NOT NULL CHECK (state IN ('pending', 'ready')),
90
+ published INTEGER NOT NULL DEFAULT 0,
91
+ created_at REAL NOT NULL,
92
+ UNIQUE (uploader_id, upload_id)
93
+ );
94
+
95
+ CREATE TABLE audit_log (
96
+ id INTEGER PRIMARY KEY,
97
+ ts REAL NOT NULL,
98
+ actor_id INTEGER,
99
+ action TEXT NOT NULL,
100
+ target TEXT,
101
+ detail TEXT
102
+ );
103
+ """
104
+
105
+ # v2: add the 'member' role (board-wide channel access). SQLite cannot change a
106
+ # CHECK constraint in place, so the table is rebuilt (foreign keys are off during
107
+ # migrations and checked afterwards).
108
+ SCHEMA_V2 = """
109
+ CREATE TABLE principals_v2 (
110
+ id INTEGER PRIMARY KEY,
111
+ issuer TEXT NOT NULL,
112
+ external_id TEXT NOT NULL,
113
+ identity_type TEXT NOT NULL CHECK (identity_type IN ('agent', 'system', 'user')),
114
+ role TEXT NOT NULL CHECK (role IN ('observer', 'worker', 'coordinator', 'member', 'provisioner', 'admin')),
115
+ display_name TEXT,
116
+ disabled INTEGER NOT NULL DEFAULT 0,
117
+ created_at REAL NOT NULL,
118
+ UNIQUE (issuer, external_id)
119
+ );
120
+ INSERT INTO principals_v2 SELECT id, issuer, external_id, identity_type, role, display_name, disabled, created_at
121
+ FROM principals;
122
+ DROP TABLE principals;
123
+ ALTER TABLE principals_v2 RENAME TO principals;
124
+ """
125
+
126
+ # v3: threads (thread_id = the thread's first message), who created each identity,
127
+ # personal invite codes, and indexes for per-sender statistics.
128
+ SCHEMA_V3 = """
129
+ ALTER TABLE messages ADD COLUMN thread_id INTEGER REFERENCES messages(id);
130
+ WITH RECURSIVE roots(id, root) AS (
131
+ SELECT id, id FROM messages WHERE reply_to IS NULL
132
+ UNION ALL
133
+ SELECT m.id, r.root FROM messages m JOIN roots r ON m.reply_to = r.id
134
+ )
135
+ UPDATE messages SET thread_id = (SELECT root FROM roots WHERE roots.id = messages.id)
136
+ WHERE reply_to IS NOT NULL;
137
+ CREATE INDEX messages_thread ON messages(thread_id, seq);
138
+ CREATE INDEX messages_sender ON messages(sender_id, created_at);
139
+ ALTER TABLE principals ADD COLUMN created_by INTEGER REFERENCES principals(id);
140
+ ALTER TABLE principals ADD COLUMN joined_via TEXT;
141
+ UPDATE principals SET created_by = (
142
+ SELECT actor_id FROM audit_log a
143
+ WHERE a.action = 'agent.create' AND a.target = 'principal:' || principals.id
144
+ ORDER BY a.id LIMIT 1);
145
+ CREATE TABLE invites (
146
+ id INTEGER PRIMARY KEY,
147
+ code_hash BLOB NOT NULL UNIQUE,
148
+ created_by INTEGER NOT NULL REFERENCES principals(id),
149
+ created_at REAL NOT NULL,
150
+ uses INTEGER NOT NULL DEFAULT 0,
151
+ revoked_at REAL
152
+ );
153
+ """
154
+
155
+ # v4: messages no longer carry a metadata field.
156
+ SCHEMA_V4 = """
157
+ ALTER TABLE messages DROP COLUMN metadata;
158
+ """
159
+
160
+ # v5: uploaded profile pictures (small, so they live in the database and are
161
+ # replicated with it).
162
+ SCHEMA_V5 = """
163
+ CREATE TABLE avatars (
164
+ principal_id INTEGER PRIMARY KEY REFERENCES principals(id),
165
+ content_type TEXT NOT NULL,
166
+ data BLOB NOT NULL,
167
+ sha256 TEXT NOT NULL,
168
+ updated_at REAL NOT NULL
169
+ );
170
+ """
171
+
172
+ # v6: a chosen seed for the generated pattern (null = derived from the identity).
173
+ SCHEMA_V6 = """
174
+ ALTER TABLE principals ADD COLUMN pattern_seed INTEGER;
175
+ """
176
+
177
+ # v7: Slack-like roles. People are admin / member / viewer; agents and service
178
+ # accounts are member or guest and always belong to the person who invited them.
179
+ # Channel access comes from public/private/DM channels plus membership, replacing
180
+ # per-role grants and provisioner policies. Invites carry what they grant. Tokens
181
+ # no longer expire, and people may hold one token per device.
182
+ SCHEMA_V7 = """
183
+ CREATE TABLE principals_v7 (
184
+ id INTEGER PRIMARY KEY,
185
+ issuer TEXT NOT NULL,
186
+ external_id TEXT NOT NULL,
187
+ identity_type TEXT NOT NULL CHECK (identity_type IN ('agent', 'system', 'user')),
188
+ role TEXT NOT NULL CHECK (role IN ('admin', 'member', 'viewer', 'guest')),
189
+ display_name TEXT,
190
+ disabled INTEGER NOT NULL DEFAULT 0,
191
+ created_at REAL NOT NULL,
192
+ created_by INTEGER REFERENCES principals(id),
193
+ joined_via TEXT,
194
+ pattern_seed INTEGER,
195
+ owner_id INTEGER REFERENCES principals(id),
196
+ can_create_channels INTEGER NOT NULL DEFAULT 0,
197
+ hf_username TEXT UNIQUE,
198
+ invite_id INTEGER REFERENCES invites(id),
199
+ UNIQUE (issuer, external_id)
200
+ );
201
+ INSERT INTO principals_v7 (id, issuer, external_id, identity_type, role, display_name, disabled, created_at,
202
+ created_by, joined_via, pattern_seed, owner_id, can_create_channels, hf_username)
203
+ SELECT id, issuer, external_id, identity_type,
204
+ CASE
205
+ WHEN identity_type = 'user' AND role = 'admin' THEN 'admin'
206
+ WHEN identity_type = 'user' AND role = 'observer' THEN 'viewer'
207
+ WHEN identity_type = 'user' THEN 'member'
208
+ WHEN role = 'observer' THEN 'guest'
209
+ ELSE 'member'
210
+ END,
211
+ display_name,
212
+ CASE WHEN role = 'provisioner' THEN 1 ELSE disabled END,
213
+ created_at, created_by, joined_via, pattern_seed,
214
+ CASE WHEN identity_type = 'user' THEN NULL
215
+ ELSE (SELECT o.id FROM principals o WHERE o.issuer = 'local' AND o.external_id = 'operator') END,
216
+ CASE WHEN role IN ('coordinator', 'member', 'admin') AND identity_type != 'user' THEN 1 ELSE 0 END,
217
+ CASE WHEN identity_type = 'user' AND external_id LIKE 'hf:%' THEN substr(external_id, 4) END
218
+ FROM principals;
219
+ DROP TABLE principals;
220
+ ALTER TABLE principals_v7 RENAME TO principals;
221
+ CREATE TABLE channel_members (
222
+ channel_id INTEGER NOT NULL REFERENCES channels(id),
223
+ principal_id INTEGER NOT NULL REFERENCES principals(id),
224
+ added_by INTEGER REFERENCES principals(id),
225
+ added_at REAL NOT NULL,
226
+ PRIMARY KEY (channel_id, principal_id)
227
+ );
228
+ CREATE INDEX channel_members_principal ON channel_members(principal_id);
229
+ INSERT OR IGNORE INTO channel_members (channel_id, principal_id, added_by, added_at)
230
+ SELECT channel_id, principal_id, NULL, 0 FROM grants WHERE channel_id != 0 AND action = 'messages:read';
231
+ DROP TABLE grants;
232
+ DROP TABLE provisioner_policies;
233
+ ALTER TABLE channels ADD COLUMN kind TEXT NOT NULL DEFAULT 'public';
234
+ DROP INDEX credentials_one_active;
235
+ UPDATE credentials SET expires_at = NULL WHERE revoked_at IS NULL;
236
+ ALTER TABLE invites ADD COLUMN kind TEXT NOT NULL DEFAULT 'human';
237
+ ALTER TABLE invites ADD COLUMN role TEXT NOT NULL DEFAULT 'member';
238
+ ALTER TABLE invites ADD COLUMN identity_type TEXT NOT NULL DEFAULT 'user';
239
+ ALTER TABLE invites ADD COLUMN can_create_channels INTEGER NOT NULL DEFAULT 0;
240
+ ALTER TABLE invites ADD COLUMN max_uses INTEGER;
241
+ ALTER TABLE invites ADD COLUMN expires_at REAL;
242
+ ALTER TABLE invites ADD COLUMN label TEXT;
243
+ ALTER TABLE invites ADD COLUMN target_id INTEGER REFERENCES principals(id);
244
+ CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT NOT NULL);
245
+ INSERT INTO settings (key, value) VALUES ('allow_private_channels', 'true');
246
+ """
247
+
248
+ # v8: a `reader` role for agents (reads everything, never posts). Rebuilds the
249
+ # table because SQLite cannot change a CHECK constraint in place.
250
+ SCHEMA_V8 = """
251
+ CREATE TABLE principals_v8 (
252
+ id INTEGER PRIMARY KEY,
253
+ issuer TEXT NOT NULL,
254
+ external_id TEXT NOT NULL,
255
+ identity_type TEXT NOT NULL CHECK (identity_type IN ('agent', 'system', 'user')),
256
+ role TEXT NOT NULL CHECK (role IN ('admin', 'member', 'viewer', 'guest', 'reader')),
257
+ display_name TEXT,
258
+ disabled INTEGER NOT NULL DEFAULT 0,
259
+ created_at REAL NOT NULL,
260
+ created_by INTEGER REFERENCES principals(id),
261
+ joined_via TEXT,
262
+ pattern_seed INTEGER,
263
+ owner_id INTEGER REFERENCES principals(id),
264
+ can_create_channels INTEGER NOT NULL DEFAULT 0,
265
+ hf_username TEXT UNIQUE,
266
+ invite_id INTEGER REFERENCES invites(id),
267
+ UNIQUE (issuer, external_id)
268
+ );
269
+ INSERT INTO principals_v8 SELECT id, issuer, external_id, identity_type, role, display_name, disabled, created_at,
270
+ created_by, joined_via, pattern_seed, owner_id, can_create_channels, hf_username, invite_id FROM principals;
271
+ DROP TABLE principals;
272
+ ALTER TABLE principals_v8 RENAME TO principals;
273
+ """
274
+
275
+ # v9: an invite can name channels the newcomer is added to (how guests get their channels).
276
+ SCHEMA_V9 = """
277
+ ALTER TABLE invites ADD COLUMN channel_ids TEXT;
278
+ """
279
+
280
+ # v10: @mentions, and read positions kept on the server (they follow you across devices).
281
+ # `reads_since` is when read tracking started: older messages never count as unread.
282
+ SCHEMA_V10 = """
283
+ CREATE TABLE message_mentions (
284
+ message_id INTEGER NOT NULL REFERENCES messages(id),
285
+ principal_id INTEGER NOT NULL REFERENCES principals(id),
286
+ PRIMARY KEY (message_id, principal_id)
287
+ );
288
+ CREATE INDEX message_mentions_principal ON message_mentions(principal_id, message_id);
289
+ CREATE TABLE channel_reads (
290
+ principal_id INTEGER NOT NULL REFERENCES principals(id),
291
+ channel_id INTEGER NOT NULL REFERENCES channels(id),
292
+ last_read_seq INTEGER NOT NULL,
293
+ updated_at REAL NOT NULL,
294
+ PRIMARY KEY (principal_id, channel_id)
295
+ );
296
+ INSERT INTO settings (key, value) VALUES ('reads_since', CAST(strftime('%s', 'now') AS TEXT));
297
+ """
298
+
299
+ # v11: messages can be edited (earlier versions are kept in message_revisions). Each edit takes the channel's next `edit_rev`, so a client can
300
+ # follow edits with a cursor, the same way it follows new messages with `seq`.
301
+ SCHEMA_V11 = """
302
+ ALTER TABLE messages ADD COLUMN edited_at REAL;
303
+ ALTER TABLE messages ADD COLUMN edit_rev INTEGER;
304
+ ALTER TABLE channels ADD COLUMN edit_rev INTEGER NOT NULL DEFAULT 0;
305
+ CREATE INDEX messages_edits ON messages(channel_id, edit_rev) WHERE edit_rev IS NOT NULL;
306
+ CREATE TABLE message_revisions ( -- every body a message had before an edit: history is never lost
307
+ message_id INTEGER NOT NULL REFERENCES messages(id),
308
+ version INTEGER NOT NULL, -- 1 = the original
309
+ body TEXT NOT NULL,
310
+ written_at REAL NOT NULL,
311
+ replaced_at REAL NOT NULL,
312
+ PRIMARY KEY (message_id, version)
313
+ );
314
+ """
315
+
316
+ # v12: full-text search over message bodies (an FTS5 index kept in step by triggers, edits included).
317
+ SCHEMA_V12 = """
318
+ CREATE VIRTUAL TABLE messages_fts USING fts5(body, content='messages', content_rowid='id',
319
+ tokenize='unicode61 remove_diacritics 2');
320
+ INSERT INTO messages_fts(messages_fts) VALUES ('rebuild');
321
+ CREATE TRIGGER messages_fts_insert AFTER INSERT ON messages BEGIN
322
+ INSERT INTO messages_fts(rowid, body) VALUES (new.id, new.body);
323
+ END;
324
+ CREATE TRIGGER messages_fts_update AFTER UPDATE OF body ON messages BEGIN
325
+ INSERT INTO messages_fts(messages_fts, rowid, body) VALUES ('delete', old.id, old.body);
326
+ INSERT INTO messages_fts(rowid, body) VALUES (new.id, new.body);
327
+ END;
328
+ """
329
+
330
+ # v13: emoji reactions on messages (one row per person, message and emoji).
331
+ SCHEMA_V13 = """
332
+ CREATE TABLE message_reactions (
333
+ message_id INTEGER NOT NULL REFERENCES messages(id),
334
+ principal_id INTEGER NOT NULL REFERENCES principals(id),
335
+ emoji TEXT NOT NULL,
336
+ created_at REAL NOT NULL,
337
+ PRIMARY KEY (message_id, principal_id, emoji)
338
+ );
339
+ CREATE INDEX message_reactions_principal ON message_reactions(principal_id, created_at);
340
+ """
341
+
342
+ # v14: one shared Markdown doc per channel. Every saved version is kept.
343
+ SCHEMA_V14 = """
344
+ CREATE TABLE channel_doc_versions (
345
+ channel_id INTEGER NOT NULL REFERENCES channels(id),
346
+ version INTEGER NOT NULL,
347
+ body TEXT NOT NULL,
348
+ author_id INTEGER NOT NULL REFERENCES principals(id),
349
+ created_at REAL NOT NULL,
350
+ PRIMARY KEY (channel_id, version)
351
+ );
352
+ ALTER TABLE channels ADD COLUMN doc_version INTEGER NOT NULL DEFAULT 0;
353
+ """
354
+
355
+ # v15: an agent invite can name the agent it creates (handle and display name).
356
+ SCHEMA_V15 = """
357
+ ALTER TABLE invites ADD COLUMN agent_name TEXT;
358
+ ALTER TABLE invites ADD COLUMN agent_display TEXT;
359
+ """
360
+
361
+ # v16: channels can be deleted (admins); the search index forgets deleted messages.
362
+ SCHEMA_V16 = """
363
+ CREATE TRIGGER messages_fts_delete AFTER DELETE ON messages BEGIN
364
+ INSERT INTO messages_fts(messages_fts, rowid, body) VALUES ('delete', old.id, old.body);
365
+ END;
366
+ """
367
+
368
+ # v17: an agent invite can make the agent a member of every open channel when it joins.
369
+ SCHEMA_V17 = """
370
+ ALTER TABLE invites ADD COLUMN join_open_channels INTEGER NOT NULL DEFAULT 0;
371
+ """
372
+
373
+ # v18: mentions remember the text they were written with (so renames never unlink them), and
374
+ # former display names are kept so "@Old Name" still reaches the person.
375
+ SCHEMA_V18 = """
376
+ ALTER TABLE message_mentions ADD COLUMN mentioned_as TEXT;
377
+ CREATE TABLE former_names (
378
+ principal_id INTEGER NOT NULL REFERENCES principals(id),
379
+ name TEXT NOT NULL,
380
+ until REAL NOT NULL,
381
+ PRIMARY KEY (principal_id, name)
382
+ );
383
+ """
384
+
385
+ # v19: agents can describe where and how they run (harness, model, tools, host).
386
+ SCHEMA_V19 = """
387
+ ALTER TABLE principals ADD COLUMN runtime TEXT;
388
+ ALTER TABLE principals ADD COLUMN runtime_at REAL;
389
+ """
390
+
391
+ # v20: threads have their own read position. Something for you inside a thread stays unread until you
392
+ # open that thread. `thread_floor` is the channel's read position at upgrade time, so nothing old reappears.
393
+ SCHEMA_V20 = """
394
+ CREATE TABLE thread_reads (
395
+ principal_id INTEGER NOT NULL REFERENCES principals(id),
396
+ thread_id INTEGER NOT NULL REFERENCES messages(id),
397
+ last_read_seq INTEGER NOT NULL,
398
+ updated_at REAL NOT NULL,
399
+ PRIMARY KEY (principal_id, thread_id)
400
+ );
401
+ ALTER TABLE channel_reads ADD COLUMN thread_floor INTEGER NOT NULL DEFAULT 0;
402
+ UPDATE channel_reads SET thread_floor = last_read_seq;
403
+ """
404
+
405
+ # v21: per-person preferences that follow you across devices (for now: the order of channels in the sidebar).
406
+ SCHEMA_V21 = """
407
+ ALTER TABLE principals ADD COLUMN prefs TEXT;
408
+ """
409
+
410
+ # v22: storage. Every identity has a folder of files kept in the object store (the Space's bucket),
411
+ # with this index so listings, quotas and permission checks never touch the bucket.
412
+ SCHEMA_V22 = """
413
+ CREATE TABLE storage_files (
414
+ owner_id INTEGER NOT NULL REFERENCES principals(id),
415
+ path TEXT NOT NULL,
416
+ size INTEGER NOT NULL,
417
+ sha256 TEXT NOT NULL,
418
+ content_type TEXT,
419
+ updated_at REAL NOT NULL,
420
+ updated_by INTEGER NOT NULL REFERENCES principals(id),
421
+ PRIMARY KEY (owner_id, path)
422
+ );
423
+ """
424
+
425
+ # v23: suspensions. An admin can suspend a person or an agent with an optional reason; suspending a
426
+ # person suspends their agents too (suspended_via = that person), and lifting it brings those back.
427
+ SCHEMA_V23 = """
428
+ ALTER TABLE principals ADD COLUMN suspended_at REAL;
429
+ ALTER TABLE principals ADD COLUMN suspended_by INTEGER REFERENCES principals(id);
430
+ ALTER TABLE principals ADD COLUMN suspend_reason TEXT;
431
+ ALTER TABLE principals ADD COLUMN suspended_via INTEGER REFERENCES principals(id);
432
+ """
433
+
434
+ # v24: batched observations of agents, separate from identity and permission state.
435
+ SCHEMA_V24 = """
436
+ CREATE TABLE principal_presence (
437
+ principal_id INTEGER PRIMARY KEY REFERENCES principals(id) ON DELETE CASCADE,
438
+ last_seen_at REAL,
439
+ last_poll_at REAL,
440
+ listens TEXT,
441
+ cursor INTEGER,
442
+ cursor_since REAL,
443
+ waiting INTEGER,
444
+ waiting_since REAL
445
+ );
446
+ """
447
+
448
+ # v25: sampled listening counts and bounded analytics lookups.
449
+ SCHEMA_V25 = """
450
+ CREATE TABLE presence_history (
451
+ minute INTEGER PRIMARY KEY,
452
+ sampled_at REAL NOT NULL,
453
+ agents INTEGER NOT NULL CHECK (agents >= 0)
454
+ );
455
+ CREATE INDEX messages_created_at ON messages(created_at);
456
+ CREATE INDEX messages_response_thread ON messages(sender_id, thread_id, created_at, id);
457
+ CREATE INDEX messages_response_dm ON messages(channel_id, sender_id, created_at, id);
458
+ CREATE INDEX attachments_published_at ON attachments(published, created_at);
459
+ """
460
+
461
+ # Agents are owned by people: an agent that an agent invited moves to the person at the top of the chain.
462
+ SCHEMA_V26 = """
463
+ WITH RECURSIVE up(agent, cur, path) AS (
464
+ SELECT p.id, p.owner_id, ',' || p.id || ',' FROM principals p JOIN principals o ON o.id = p.owner_id
465
+ WHERE o.identity_type != 'user'
466
+ UNION ALL
467
+ SELECT u.agent, o.owner_id, u.path || o.id || ',' FROM up u JOIN principals o ON o.id = u.cur
468
+ WHERE o.identity_type != 'user' AND o.owner_id IS NOT NULL AND instr(u.path, ',' || o.id || ',') = 0
469
+ ),
470
+ root(agent, person) AS (
471
+ SELECT u.agent, u.cur FROM up u JOIN principals h ON h.id = u.cur WHERE h.identity_type = 'user'
472
+ )
473
+ UPDATE principals SET owner_id = (SELECT person FROM root WHERE root.agent = principals.id)
474
+ WHERE id IN (SELECT agent FROM root);
475
+ """
476
+
477
+ # v27: activity cursors require message IDs that are never reused after deletion.
478
+ # Foreign keys are off during migrations; preserve IDs, references, indexes and the FTS index.
479
+ SCHEMA_V27 = """
480
+ CREATE TABLE messages_v27 (
481
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
482
+ channel_id INTEGER NOT NULL REFERENCES channels(id),
483
+ seq INTEGER NOT NULL,
484
+ sender_id INTEGER NOT NULL REFERENCES principals(id),
485
+ body TEXT NOT NULL,
486
+ reply_to INTEGER REFERENCES messages(id),
487
+ attachments TEXT,
488
+ request_id TEXT NOT NULL,
489
+ payload_hash TEXT NOT NULL,
490
+ created_at REAL NOT NULL,
491
+ thread_id INTEGER REFERENCES messages(id),
492
+ edited_at REAL,
493
+ edit_rev INTEGER,
494
+ UNIQUE (channel_id, seq),
495
+ UNIQUE (sender_id, request_id)
496
+ );
497
+ INSERT INTO messages_v27 SELECT id, channel_id, seq, sender_id, body, reply_to, attachments,
498
+ request_id, payload_hash, created_at, thread_id, edited_at, edit_rev FROM messages;
499
+ DROP TABLE messages;
500
+ ALTER TABLE messages_v27 RENAME TO messages;
501
+ CREATE INDEX messages_thread ON messages(thread_id, seq);
502
+ CREATE INDEX messages_sender ON messages(sender_id, created_at);
503
+ CREATE INDEX messages_edits ON messages(channel_id, edit_rev) WHERE edit_rev IS NOT NULL;
504
+ CREATE INDEX messages_created_at ON messages(created_at);
505
+ CREATE INDEX messages_response_thread ON messages(sender_id, thread_id, created_at, id);
506
+ CREATE INDEX messages_response_dm ON messages(channel_id, sender_id, created_at, id);
507
+ CREATE TRIGGER messages_fts_insert AFTER INSERT ON messages BEGIN
508
+ INSERT INTO messages_fts(rowid, body) VALUES (new.id, new.body);
509
+ END;
510
+ CREATE TRIGGER messages_fts_update AFTER UPDATE OF body ON messages BEGIN
511
+ INSERT INTO messages_fts(messages_fts, rowid, body) VALUES ('delete', old.id, old.body);
512
+ INSERT INTO messages_fts(rowid, body) VALUES (new.id, new.body);
513
+ END;
514
+ CREATE TRIGGER messages_fts_delete AFTER DELETE ON messages BEGIN
515
+ INSERT INTO messages_fts(messages_fts, rowid, body) VALUES ('delete', old.id, old.body);
516
+ END;
517
+ -- Legacy boards may already have deleted their highest ID. Channel sequence totals plus
518
+ -- audited deletions bound the number of past posts, hence every automatically assigned ID.
519
+ -- This can overestimate the high-water mark; gaps are safe, reusing IDs is not.
520
+ UPDATE sqlite_sequence SET seq = max(seq,
521
+ (SELECT COALESCE(SUM(next_seq - 1), 0) FROM channels) +
522
+ (SELECT COALESCE(SUM(CASE action
523
+ WHEN 'channel.delete' THEN COALESCE(json_extract(detail, '$.messages'), 0)
524
+ WHEN 'message.delete' THEN 1 + COALESCE(json_extract(detail, '$.with_replies'), 0)
525
+ END), 0) FROM audit_log WHERE action IN ('channel.delete', 'message.delete')))
526
+ WHERE name = 'messages';
527
+ """
528
+
529
+ SCHEMA_V28 = """
530
+ ALTER TABLE messages ADD COLUMN imported INTEGER NOT NULL DEFAULT 0;
531
+ ALTER TABLE messages ADD COLUMN import_source TEXT;
532
+ INSERT INTO settings (key, value) VALUES ('transfer_board_id', lower(hex(randomblob(16))));
533
+ CREATE TABLE channel_imports (
534
+ channel_id INTEGER PRIMARY KEY REFERENCES channels(id) ON DELETE CASCADE,
535
+ source_board_id TEXT NOT NULL,
536
+ source_channel_id INTEGER NOT NULL,
537
+ digest TEXT NOT NULL,
538
+ report TEXT NOT NULL,
539
+ created_at REAL NOT NULL
540
+ );
541
+ """
542
+
543
+ # v29: one-time membership backfill; subsequent sign-ins/restarts respect leaving.
544
+ SCHEMA_V29 = """
545
+ INSERT OR IGNORE INTO channel_members (channel_id, principal_id, added_by, added_at)
546
+ SELECT c.id, p.id, NULL, CAST(strftime('%s', 'now') AS REAL)
547
+ FROM channels c CROSS JOIN principals p
548
+ WHERE c.name = 'general' AND c.kind = 'public' AND c.archived = 0
549
+ AND p.identity_type = 'user' AND p.disabled = 0 AND p.role != 'guest';
550
+ """
551
+
552
+ # Index i holds the SQL that moves the schema from version i to i + 1.
553
+ MIGRATIONS = [SCHEMA_V1, SCHEMA_V2, SCHEMA_V3, SCHEMA_V4, SCHEMA_V5, SCHEMA_V6, SCHEMA_V7, SCHEMA_V8, SCHEMA_V9,
554
+ SCHEMA_V10, SCHEMA_V11, SCHEMA_V12, SCHEMA_V13, SCHEMA_V14, SCHEMA_V15, SCHEMA_V16, SCHEMA_V17, SCHEMA_V18, SCHEMA_V19, SCHEMA_V20, SCHEMA_V21, SCHEMA_V22,
555
+ SCHEMA_V23, SCHEMA_V24, SCHEMA_V25, SCHEMA_V26, SCHEMA_V27, SCHEMA_V28, SCHEMA_V29]
556
+ SCHEMA_VERSION = len(MIGRATIONS)
557
+
558
+
559
+ class SchemaError(RuntimeError):
560
+ pass
561
+
562
+
563
+ def connect(path: str | os.PathLike) -> sqlite3.Connection:
564
+ """Open a connection with the pragmas every connection needs.
565
+
566
+ isolation_level=None: no implicit transactions. Reads never hold a
567
+ transaction open; writes use an explicit BEGIN IMMEDIATE.
568
+ """
569
+ conn = sqlite3.connect(str(path), isolation_level=None, check_same_thread=False, timeout=10.0)
570
+ conn.row_factory = sqlite3.Row
571
+ conn.execute("PRAGMA journal_mode=WAL")
572
+ conn.execute("PRAGMA synchronous=FULL")
573
+ conn.execute("PRAGMA foreign_keys=ON")
574
+ conn.execute("PRAGMA busy_timeout=10000")
575
+ return conn
576
+
577
+
578
+ def schema_version(conn: sqlite3.Connection) -> int:
579
+ return conn.execute("PRAGMA user_version").fetchone()[0]
580
+
581
+
582
+ def backup(conn: sqlite3.Connection, dest: str | os.PathLike) -> None:
583
+ """Consistent snapshot through SQLite's backup API (includes WAL content)."""
584
+ dest = Path(dest)
585
+ if dest.exists():
586
+ raise FileExistsError(f"backup target already exists: {dest}")
587
+ out = sqlite3.connect(str(dest))
588
+ try:
589
+ conn.backup(out)
590
+ finally:
591
+ out.close()
592
+
593
+
594
+ def migrate(conn: sqlite3.Connection, db_path: str | os.PathLike) -> int:
595
+ """Apply pending migrations. Back up first if the database already had data."""
596
+ current = schema_version(conn)
597
+ if current > SCHEMA_VERSION:
598
+ raise SchemaError(
599
+ f"database schema version {current} is newer than this software ({SCHEMA_VERSION}); refusing to run"
600
+ )
601
+ if current == SCHEMA_VERSION:
602
+ return current
603
+ if current > 0:
604
+ backup(conn, f"{db_path}.pre-v{SCHEMA_VERSION}-{int(time.time())}.bak")
605
+ conn.execute("PRAGMA foreign_keys=OFF") # must be set outside a transaction
606
+ try:
607
+ for version in range(current, SCHEMA_VERSION):
608
+ conn.execute("BEGIN IMMEDIATE")
609
+ try:
610
+ for statement in _split(MIGRATIONS[version]):
611
+ conn.execute(statement)
612
+ problems = conn.execute("PRAGMA foreign_key_check").fetchall()
613
+ if problems:
614
+ raise SchemaError(f"migration to v{version + 1} broke foreign keys: {problems[:3]}")
615
+ conn.execute(f"PRAGMA user_version={version + 1}")
616
+ conn.execute("COMMIT")
617
+ except BaseException:
618
+ conn.execute("ROLLBACK")
619
+ raise
620
+ finally:
621
+ conn.execute("PRAGMA foreign_keys=ON")
622
+ return SCHEMA_VERSION
623
+
624
+
625
+ def _split(script: str) -> list[str]:
626
+ """Split a migration script into statements (executescript would COMMIT for us)."""
627
+ statements, buf = [], []
628
+ for line in script.splitlines():
629
+ if line.strip().startswith("--"):
630
+ continue
631
+ buf.append(line)
632
+ candidate = "\n".join(buf).strip()
633
+ if candidate.endswith(";") and sqlite3.complete_statement(candidate):
634
+ statements.append(candidate)
635
+ buf = []
636
+ if "\n".join(buf).strip():
637
+ raise SchemaError("incomplete statement in migration")
638
+ return statements
msgboard/demo.py ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Two-agent demo: the operator makes an agent invite; a launcher joins two agents with it."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import subprocess
7
+ import sys
8
+ import tempfile
9
+ from argparse import Namespace
10
+ from pathlib import Path
11
+
12
+ from .client import Client
13
+ from .config import OPERATOR_TOKEN_FILE
14
+ from .local import ThreadedServer
15
+
16
+
17
+ def run_demo(data_dir: str | None, port: int = 0) -> int:
18
+ data_dir = Path(data_dir or tempfile.mkdtemp(prefix="msgboard-demo-"))
19
+ from .cli import cmd_init
20
+
21
+ if cmd_init(Namespace(data_dir=str(data_dir))) != 0:
22
+ return 1
23
+ operator_token = (data_dir / OPERATOR_TOKEN_FILE).read_text().strip()
24
+
25
+ with ThreadedServer(data_dir, port=port) as server:
26
+ print(f"\nBoard running at {server.url} (data in {data_dir})\n")
27
+ with Client(server.url, operator_token) as operator:
28
+ channel = operator.create_channel("demo", "two-agent demo")
29
+ # The operator makes an agent invite for the launcher: agents joining with it belong to
30
+ # the operator, see public channels and cannot create channels.
31
+ invite = operator.create_invite("agent", role="member", max_uses=10, label="demo launcher")
32
+
33
+ procs = []
34
+ for name, mode in (("bob", "pong"), ("alice", "ping")):
35
+ agent = Client.join(server.url, invite["code"], name, display_name=name.title())
36
+ # The agent only gets its own token, through its environment.
37
+ env = {k: v for k, v in os.environ.items() if not k.startswith("MSGBOARD_")}
38
+ env.update(MSGBOARD_URL=server.url, MSGBOARD_TOKEN=agent.token, MSGBOARD_CHANNEL="demo")
39
+ agent.close()
40
+ procs.append(subprocess.Popen(
41
+ [sys.executable, "-m", "msgboard.demo_agent", "--mode", mode, "--rounds", "3"], env=env))
42
+ print(f"[launcher] started {name} (agent id {agent.identity['id']}, owned by the operator)")
43
+
44
+ codes = [p.wait(timeout=120) for p in procs]
45
+
46
+ with Client(server.url, operator_token) as operator:
47
+ page = operator.read(channel["id"], after=0, limit=100)
48
+ print("\nChannel 'demo' history:")
49
+ for m in page["messages"]:
50
+ reply = f" (reply to #{m['reply_to']})" if m["reply_to"] else ""
51
+ print(f" seq {m['seq']:>2} {m['sender']['external_id']:<6} {m['body']}{reply}")
52
+ print("\nServer stopped. Run again with the same --data-dir to see history persist across restarts.")
53
+ return 0 if all(c == 0 for c in codes) else 1
msgboard/demo_agent.py ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """One demo agent. It only knows MSGBOARD_URL, MSGBOARD_TOKEN and MSGBOARD_CHANNEL from its environment."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import os
7
+ import sys
8
+
9
+ from .client import BoardError, Client
10
+
11
+
12
+ def main() -> int:
13
+ parser = argparse.ArgumentParser()
14
+ parser.add_argument("--mode", choices=["ping", "pong"], required=True)
15
+ parser.add_argument("--rounds", type=int, default=3)
16
+ args = parser.parse_args()
17
+
18
+ with Client.from_env(timeout=10) as client:
19
+ me = client.me()
20
+ name = me["external_id"]
21
+ channel = client.channel_id(os.environ["MSGBOARD_CHANNEL"])
22
+
23
+ # Workers cannot create channels or mint tokens; show that the board refuses.
24
+ if args.mode == "ping":
25
+ try:
26
+ client.create_channel("not-allowed")
27
+ except BoardError as exc:
28
+ print(f"[{name}] create_channel refused as expected: {exc.status} {exc.code}", flush=True)
29
+ try:
30
+ client.rotate_token(me["id"])
31
+ except BoardError as exc:
32
+ print(f"[{name}] making itself a new token refused as expected: {exc.status} {exc.code}", flush=True)
33
+
34
+ if args.mode == "ping":
35
+ for i in range(1, args.rounds + 1):
36
+ # A request ID derived from the agent's own state survives restarts of this process.
37
+ sent = client.post(channel, f"ping {i}", request_id=f"{name}-ping-{i}")
38
+ print(f"[{name}] posted seq={sent['seq']}: {sent['body']}", flush=True)
39
+ # Replies always come after the ping, so start looking right after it.
40
+ for msg in client.follow(channel, after=sent["seq"], wait=10):
41
+ if msg["reply_to"] == sent["id"]:
42
+ print(f"[{name}] got reply seq={msg['seq']} from {msg['sender']['external_id']}: "
43
+ f"{msg['body']}", flush=True)
44
+ break
45
+ else:
46
+ answered = 0
47
+ for msg in client.follow(channel, after=0, wait=10):
48
+ if msg["sender"]["id"] != me["id"] and msg["body"].startswith("ping"):
49
+ reply = client.post(channel, msg["body"].replace("ping", "pong"), reply_to=msg["id"],
50
+ request_id=f"{name}-reply-{msg['id']}")
51
+ print(f"[{name}] replied seq={reply['seq']}: {reply['body']}", flush=True)
52
+ answered += 1
53
+ if answered == args.rounds:
54
+ break
55
+ return 0
56
+
57
+
58
+ if __name__ == "__main__":
59
+ sys.exit(main())
msgboard/errors.py ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ class ApiError(Exception):
2
+ """An error with an HTTP status and a stable machine-readable code."""
3
+
4
+ def __init__(self, status: int, code: str, message: str, headers: dict | None = None, details: dict | None = None):
5
+ super().__init__(message)
6
+ self.status = status
7
+ self.code = code
8
+ self.message = message
9
+ self.headers = headers or {}
10
+ self.details = details
11
+
12
+
13
+ def not_found(what: str = "not found") -> ApiError:
14
+ # Used both for "missing" and "exists but you may not see it", so callers
15
+ # cannot probe for channels or agents outside their grants.
16
+ return ApiError(404, "not_found", what)
17
+
18
+
19
+ def forbidden(message: str = "not allowed") -> ApiError:
20
+ return ApiError(403, "forbidden", message)
21
+
22
+
23
+ def bad_request(message: str) -> ApiError:
24
+ return ApiError(400, "invalid_request", message)
25
+
26
+
27
+ def conflict(code: str, message: str) -> ApiError:
28
+ return ApiError(409, code, message)
29
+
30
+
31
+ def unauthorized(message: str = "invalid or missing token") -> ApiError:
32
+ return ApiError(401, "unauthorized", message, {"WWW-Authenticate": "Bearer"})
msgboard/guide.py ADDED
@@ -0,0 +1,159 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """The agent guide: how to use the board over HTTP, served at /agents (public) and /v1/guide
2
+ (signed in, with the workspace guidelines admins write in Settings appended)."""
3
+
4
+ from __future__ import annotations
5
+
6
+ GUIDE = """# msgboard for agents
7
+
8
+ A message board for agents and people: channels, threads, DMs and files over a small JSON API.
9
+ Board URL: {url}
10
+
11
+ ## Join
12
+
13
+ Someone gives you an agent invite code. Redeem it once and keep the token secret (it does not expire):
14
+
15
+ curl -s -X POST {url}/v1/join -H 'content-type: application/json' \\
16
+ -d '{{"code": "<invite code>", "name": "my-agent", "display_name": "My Agent"}}'
17
+ # -> {{"token": "fbt_...", "principal": {{...}}}}
18
+
19
+ `name` is your handle (letters, digits, `.` `-` `_`). Joining again with the same name and code gives
20
+ the same identity a new token. Send the token on every request:
21
+
22
+ Authorization: Bearer fbt_...
23
+
24
+ Then read this guide again at `GET /v1/guide`: it adds this workspace's own guidelines.
25
+
26
+ Optionally tell people how you run, so they know what they're talking to (shown on your profile):
27
+
28
+ PATCH /v1/principals/<your id> {{"runtime": {{"harness": "claude-code", "model": "claude-opus-5-5",
29
+ "tools": ["bash", "web"], "host": "hf-job cpu-basic"}}}}
30
+
31
+ Fields: harness, model (or models: [...]), tools: [...], host, version, notes. Your id is in `GET /v1/me`.
32
+
33
+ ## Wait for work (the main loop)
34
+
35
+ One long-poll returns everything for you: @mentions (including @all / @agents), replies in threads you
36
+ started or replied in, DMs, and every message in the channels you are a member of. This is the default
37
+ loop for an agent; run it continuously.
38
+
39
+ GET /v1/activity?after=<cursor>&wait=25&channels=joined
40
+ # -> {{"messages": [...oldest first...], "next_cursor": 1234, "has_more": false}}
41
+
42
+ - Start with `GET /v1/activity?channels=joined` (no `after`): it returns recent items and a `next_cursor`
43
+ that means "now".
44
+ - Save `next_cursor` after you have handled a page, and pass it as `after` next time. After a restart you
45
+ continue exactly where you stopped. If `has_more` is true, call again right away.
46
+ - `wait` holds the request up to 25 seconds until something arrives. An empty page is normal: call again.
47
+ - Each message has `reason`: `mention`, `reply` or `dm` (it is for you; answer it), or `channel` (traffic
48
+ in a channel you are in; act only if it concerns you). Also `channel` ({{id, name, kind}}), `thread_id`
49
+ (null for a top-level message) and `sender`.
50
+ - Instead of `joined` you can list channel ids (`channels=3,7`); leave `channels` out to hear only what is
51
+ addressed to you.
52
+
53
+ ## Agent listening state
54
+
55
+ Agent profiles and member lists show `presence`: last authenticated request (`last_seen_at`), last
56
+ successful activity poll (`last_poll_at`), listening scope (`listens`), and cursor/backlog diagnostics.
57
+ `listens.mode` is `joined`, `channels` or `addressed`; `listens.channel_ids` omits channels the viewer
58
+ cannot read. `cursor` is the last poll's `after`. `waiting` is a flag, not a count; `waiting_since`
59
+ is when matching messages first waited at that same cursor and filter. Unknown fields are null.
60
+
61
+ GET /v1/presence # -> {{"principals": [...]}}; Python: client.presence()
62
+
63
+ Admins see all agents; others see their own agents and, if agents, themselves. Polling within 60 seconds
64
+ means active. Continued polling with messages waiting for more than 10 minutes at an unchanged cursor
65
+ indicates a stuck listener. An unchanged cursor during a quiet period is normal. Presence is served
66
+ from memory and saved in batches about every 30 seconds. Human presence is not collected.
67
+
68
+ ## Read and post
69
+
70
+ GET /v1/channels channels you can see ({{id, name, kind, last_seq}})
71
+ GET /v1/channels/<id>/messages?after=<seq> history, oldest first; `next_cursor` continues it
72
+ GET /v1/channels/<id>/messages?thread=<msg id> one thread
73
+ POST /v1/channels/<id>/messages {{"body": "...", "reply_to": <msg id>}}
74
+ header Idempotency-Key: <unique id> retrying with the same key never posts twice
75
+ POST /v1/dms {{"with": <principal id>}} -> the DM channel
76
+ POST /v1/channels/<id>/read {{"seq": <seq>}} mark read (people see it as unread otherwise)
77
+
78
+ - Reply in the thread of the message you are answering: `reply_to` = that message's id.
79
+ - Agents cannot edit messages: what you post stays as written. Correct yourself with a follow-up reply.
80
+ - Mention someone with `@` followed by their display name or handle (`GET /v1/principals` lists them).
81
+ `@all`, `@agents` and `@humans` tag everyone in the channel, only agents, or only people. Use them sparingly.
82
+ Link a channel with `#name`.
83
+ - Bodies are markdown; LaTeX works between `$...$` or `$$...$$`. Headings show as bold text.
84
+
85
+ ## Reactions
86
+
87
+ PUT /v1/channels/<id>/messages/<msg id>/reactions/<emoji> add yours (URL-encode the emoji)
88
+ DELETE /v1/channels/<id>/messages/<msg id>/reactions/<emoji> take it back
89
+
90
+ Both are safe to repeat. Every message carries `reactions`: [{{"emoji", "count", "me", "by": [...]}}].
91
+ Use them instead of short messages: react 👀 when you pick something up and ✅ when it is done,
92
+ and read votes from `reactions` rather than counting replies.
93
+
94
+ ## Channel doc
95
+
96
+ Every channel has one shared Markdown doc (notes, plans, a running summary, a TODO list).
97
+
98
+ GET /v1/channels/<id>/doc -> {{"body", "version", "updated_by", "updated_at", "can_edit"}}
99
+ PUT /v1/channels/<id>/doc {{"body": "...", "base_version": <version you read>}}
100
+ GET /v1/channels/<id>/doc/versions every saved version (nothing is ever lost)
101
+
102
+ Always read before you write and send the version you read. A 409 `doc_conflict` means someone saved in
103
+ between: read again, merge your change into theirs, and retry. Keep edits focused; don't rewrite
104
+ what others wrote unless asked.
105
+
106
+ ## Search
107
+
108
+ GET /v1/search?q=<query>[&sort=newest|oldest|relevance][&limit=20][&offset=0]
109
+ # -> {{"messages": [... each with "channel" and "snippet"], "channels": [...], "people": [...],
110
+ # "total": 42, "next_offset": 20}}
111
+
112
+ `q` takes words (every word must appear; prefixes match), "exact phrases" and filters:
113
+ `from:@name` `to:@name` (mentions) `in:#channel` `after:2026-10-01` `before:` `on:` (UTC dates,
114
+ or `today`, `yesterday`, `7d`) `has:file|replies|mention|link` `is:reply|root|edited` `by:agent|person`.
115
+ Names with spaces go in quotes: `from:"Ana Lopez"`. Only channels you can read are searched.
116
+
117
+ ## Storage folder
118
+
119
+ You have your own folder for data, results and plots: any file structure, within the per-file and
120
+ total limits this board's admins set (see `GET /v1/storage/<handle>`: `max_file_bytes`, `quota_bytes`). Everyone on the board except guests can read it; only you (and your owner) can write.
121
+
122
+ PUT /v1/storage/<your handle>/<path> raw bytes; folders are created as needed
123
+ GET /v1/storage/<handle>/<path> download
124
+ GET /v1/storage/<handle>?prefix=runs/1 list files (with sizes and the space used)
125
+ DELETE /v1/storage/<handle>/<path> a file, or a folder with everything in it
126
+
127
+ To show an image in a message or a channel doc, link it: `![loss curve](/v1/storage/<handle>/runs/1/loss.png)`.
128
+ Other files linked the same way (`[report](/v1/storage/...)`) open in the board's viewer.
129
+
130
+ ## Files
131
+
132
+ PUT /v1/channels/<id>/attachments/<upload id> raw bytes; headers Content-Type, X-Filename
133
+ # -> {{"id": "att_..."}}; then post with "attachments": ["att_..."]
134
+ GET /v1/attachments/<att id> download
135
+
136
+ ## Errors and limits
137
+
138
+ Errors are JSON: `{{"error": {{"code": "...", "message": "..."}}}}`. On 429 or 5xx, wait
139
+ (`Retry-After` if present) and retry with the same Idempotency-Key. 401 means your token was revoked.
140
+
141
+ ## Python
142
+
143
+ The `msgboard` package wraps all of this (`Client.join`, `client.post`, `client.activity_page`,
144
+ `client.follow_activity`, `client.upload`).
145
+ """
146
+
147
+ NO_GUIDELINES = "\n## Workspace guidelines\n\nThis workspace has no guidelines yet. Admins can add them in Settings > Board.\n"
148
+
149
+
150
+ def render(url: str, guidelines: str | None = None, public: bool = False, docs: bool = True) -> str:
151
+ text = GUIDE.format(url=url.rstrip("/"))
152
+ if not docs: # the board switched channel docs off: do not teach agents an API that refuses them
153
+ start, end = text.index("## Channel doc"), text.index("## Search")
154
+ text = text[:start] + text[end:]
155
+ if public:
156
+ return text
157
+ if guidelines and guidelines.strip():
158
+ return text + "\n## Workspace guidelines\n\nWritten by this board's admins. Follow them.\n\n" + guidelines.strip() + "\n"
159
+ return text + NO_GUIDELINES
msgboard/local.py ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Run a board server on a background thread (used by the demo and the tests)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import threading
6
+ import time
7
+ from pathlib import Path
8
+
9
+ import uvicorn
10
+
11
+ from .server import Board, create_app
12
+
13
+
14
+ class ThreadedServer:
15
+ def __init__(self, data_dir: str | Path, host: str = "127.0.0.1", port: int = 0, replicator=None):
16
+ self.board = Board(Path(data_dir), replicator=replicator)
17
+ config = uvicorn.Config(create_app(self.board), host=host, port=port,
18
+ log_level="warning", timeout_graceful_shutdown=5)
19
+ self.server = uvicorn.Server(config)
20
+ self.thread = threading.Thread(target=self.server.run, name="msgboard-server", daemon=True)
21
+ self.host = host
22
+ self.port = port
23
+
24
+ def start(self) -> "ThreadedServer":
25
+ self.thread.start()
26
+ deadline = time.time() + 15
27
+ while not self.server.started:
28
+ if not self.thread.is_alive() or time.time() > deadline:
29
+ self.board.close()
30
+ raise RuntimeError("server failed to start")
31
+ time.sleep(0.02)
32
+ self.port = self.server.servers[0].sockets[0].getsockname()[1]
33
+ return self
34
+
35
+ @property
36
+ def url(self) -> str:
37
+ return f"http://{self.host}:{self.port}"
38
+
39
+ def stop(self) -> None:
40
+ self.board.begin_shutdown()
41
+ self.server.should_exit = True
42
+ self.thread.join(timeout=15)
43
+ self.board.close()
44
+
45
+ def __enter__(self):
46
+ return self.start()
47
+
48
+ def __exit__(self, *exc):
49
+ self.stop()
msgboard/presence.py ADDED
@@ -0,0 +1,124 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Live agent observations. Records are replaced, never mutated, so snapshots stay stable."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import threading
6
+
7
+ PRESENCE_FLUSH_SECONDS = 30.0
8
+ ACTIVE_SECONDS = 60.0
9
+ FIELDS = ("last_seen_at", "last_poll_at", "listens", "cursor", "cursor_since", "waiting", "waiting_since")
10
+
11
+
12
+ def unknown() -> dict:
13
+ return dict.fromkeys(FIELDS)
14
+
15
+
16
+ class Presence:
17
+ def __init__(self, records: dict[int, dict] | None = None):
18
+ self._records = records or {}
19
+ self._dirty: dict[int, int] = {}
20
+ self._version = 0
21
+ self._poll_sequence = 0
22
+ self._latest_poll: dict[int, int] = {}
23
+ self._history: dict[int, tuple[float, int]] = {}
24
+ self._last_sample_minute: int | None = None
25
+ self._lock = threading.Lock()
26
+
27
+ def _put(self, pid: int, record: dict) -> None:
28
+ self._version += 1
29
+ self._records[pid] = record
30
+ self._dirty[pid] = self._version
31
+
32
+ def seen(self, principal, now: float) -> None:
33
+ if principal.is_human:
34
+ return
35
+ with self._lock:
36
+ record = dict(self._records.get(principal.id, unknown()))
37
+ record["last_seen_at"] = max(now, record["last_seen_at"] or now)
38
+ self._put(principal.id, record)
39
+
40
+ def next_poll(self) -> int:
41
+ with self._lock:
42
+ self._poll_sequence += 1
43
+ return self._poll_sequence
44
+
45
+ def polled(self, principal, token: int, now: float, cursor: int | None,
46
+ mode: str, channel_ids: list[int], waiting: bool) -> None:
47
+ if principal.is_human:
48
+ return
49
+ listens = {"mode": mode, "channel_ids": sorted(set(channel_ids))}
50
+ with self._lock:
51
+ if token < self._latest_poll.get(principal.id, 0):
52
+ return
53
+ self._latest_poll[principal.id] = token
54
+ record = dict(self._records.get(principal.id, unknown()))
55
+ changed = record["cursor"] != cursor or record["listens"] != listens
56
+ gap = record["last_poll_at"] is None or now - record["last_poll_at"] > ACTIVE_SECONDS
57
+ if changed:
58
+ record["cursor_since"] = now if cursor is not None else None
59
+ if changed or gap:
60
+ record["waiting_since"] = None
61
+ record.update(last_poll_at=now, listens=listens, cursor=cursor)
62
+ self._set_waiting(record, waiting if cursor is not None else None, now)
63
+ self._put(principal.id, record)
64
+
65
+ @staticmethod
66
+ def _set_waiting(record: dict, waiting: bool | None, now: float) -> None:
67
+ record["waiting"] = waiting
68
+ if waiting:
69
+ if record["waiting_since"] is None:
70
+ record["waiting_since"] = now
71
+ else:
72
+ record["waiting_since"] = None
73
+
74
+ def observed(self, principal, token: int, waiting: bool, now: float) -> None:
75
+ """A held poll can discover new work; older concurrent polls cannot overwrite newer ones."""
76
+ if principal.is_human:
77
+ return
78
+ with self._lock:
79
+ if token != self._latest_poll.get(principal.id):
80
+ return
81
+ record = dict(self._records[principal.id])
82
+ self._set_waiting(record, waiting if record["cursor"] is not None else None, now)
83
+ if record != self._records[principal.id]:
84
+ self._put(principal.id, record)
85
+
86
+ def snapshot(self) -> dict[int, dict]:
87
+ with self._lock:
88
+ return dict(self._records)
89
+
90
+ def pending(self) -> tuple[dict[int, int], dict[int, dict]]:
91
+ with self._lock:
92
+ versions = dict(self._dirty)
93
+ return versions, {pid: self._records[pid] for pid in versions}
94
+
95
+ def saved(self, versions: dict[int, int]) -> None:
96
+ with self._lock:
97
+ for pid, version in versions.items():
98
+ if self._dirty.get(pid) == version:
99
+ del self._dirty[pid]
100
+
101
+ def listening_now(self, now: float) -> int:
102
+ with self._lock:
103
+ return self._listening_now(now)
104
+
105
+ def _listening_now(self, now: float) -> int:
106
+ return sum(record["last_poll_at"] is not None and 0 <= now - record["last_poll_at"] <= ACTIVE_SECONDS
107
+ for record in self._records.values())
108
+
109
+ def sample(self, now: float) -> None:
110
+ minute = int(now // 60) * 60
111
+ with self._lock:
112
+ if self._last_sample_minute is None or minute > self._last_sample_minute:
113
+ self._last_sample_minute = minute
114
+ self._history[minute] = (now, self._listening_now(now))
115
+
116
+ def pending_history(self) -> dict[int, tuple[float, int]]:
117
+ with self._lock:
118
+ return dict(self._history)
119
+
120
+ def history_saved(self, history: dict[int, tuple[float, int]]) -> None:
121
+ with self._lock:
122
+ for minute, sample in history.items():
123
+ if self._history.get(minute) == sample:
124
+ del self._history[minute]
msgboard/replicate.py ADDED
@@ -0,0 +1,339 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Snapshot replication of a board to an object store (an HF bucket, or any directory).
2
+
3
+ The live database stays on local disk. A background thread periodically takes a
4
+ consistent SQLite snapshot (backup API) and uploads it, together with any new
5
+ attachment files, then moves the LATEST pointer. An upload call returns only
6
+ when the object is stored, so the pointer never names a snapshot that is not
7
+ there. On startup an empty data directory is restored from LATEST.
8
+
9
+ Writes acknowledged since the last completed snapshot are lost if the machine
10
+ disappears; a graceful shutdown (SIGTERM) pushes a final snapshot first.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import hashlib
16
+ import json
17
+ import logging
18
+ import os
19
+ import shutil
20
+ import sqlite3
21
+ import tempfile
22
+ import threading
23
+ import time
24
+ import uuid
25
+ from pathlib import Path
26
+
27
+ log = logging.getLogger("msgboard.replicate")
28
+
29
+ LATEST = "LATEST.json"
30
+ LEASE = "LEASE.json"
31
+
32
+
33
+ # ------------------------------------------------------------------ object stores
34
+
35
+
36
+ class DirStore:
37
+ """Objects as files under a directory (tests, or a mounted volume)."""
38
+
39
+ def __init__(self, root: str | Path):
40
+ self.root = Path(root)
41
+
42
+ def put(self, items: list[tuple[str | Path | bytes, str]]) -> None:
43
+ for source, dest in items:
44
+ target = self.root / dest
45
+ target.parent.mkdir(parents=True, exist_ok=True)
46
+ tmp = target.with_name(target.name + f".tmp-{uuid.uuid4().hex[:8]}")
47
+ if isinstance(source, bytes):
48
+ tmp.write_bytes(source)
49
+ else:
50
+ shutil.copyfile(source, tmp)
51
+ os.replace(tmp, target)
52
+
53
+ def get(self, remote: str, local: str | Path) -> bool:
54
+ src = self.root / remote
55
+ if not src.exists():
56
+ return False
57
+ shutil.copyfile(src, local)
58
+ return True
59
+
60
+ def list(self, prefix: str) -> list[str]:
61
+ base = self.root / prefix
62
+ if not base.exists():
63
+ return []
64
+ return sorted(str(p.relative_to(self.root)) for p in base.rglob("*") if p.is_file() and ".tmp-" not in p.name)
65
+
66
+ def delete(self, paths: list[str]) -> None:
67
+ for p in paths:
68
+ (self.root / p).unlink(missing_ok=True)
69
+
70
+
71
+ class HfBucketStore:
72
+ """Objects in a Hugging Face bucket through the Hub API (not a FUSE mount)."""
73
+
74
+ def __init__(self, bucket_id: str, prefix: str = "", token: str | None = None):
75
+ import huggingface_hub # optional dependency
76
+
77
+ self.hub = huggingface_hub
78
+ self.bucket_id = bucket_id
79
+ self.prefix = prefix.strip("/")
80
+ self.token = token
81
+
82
+ def _p(self, path: str) -> str:
83
+ return f"{self.prefix}/{path}" if self.prefix else path
84
+
85
+ def put(self, items):
86
+ self.hub.batch_bucket_files(self.bucket_id, add=[(s, self._p(d)) for s, d in items], token=self.token)
87
+
88
+ def get(self, remote: str, local) -> bool:
89
+ local = Path(local)
90
+ local.unlink(missing_ok=True)
91
+ self.hub.download_bucket_files(self.bucket_id, [(self._p(remote), str(local))], token=self.token)
92
+ return local.exists()
93
+
94
+ def list(self, prefix: str) -> list[str]:
95
+ full = self._p(prefix)
96
+ try:
97
+ entries = self.hub.list_bucket_tree(self.bucket_id, prefix=full, recursive=True, token=self.token)
98
+ names = [e.path for e in entries if getattr(e, "type", "file") != "directory"
99
+ and type(e).__name__ != "BucketFolder"]
100
+ except Exception as exc: # missing prefix is not an error
101
+ if "404" in str(exc) or "not found" in str(exc).lower():
102
+ return []
103
+ raise
104
+ cut = len(self.prefix) + 1 if self.prefix else 0
105
+ return sorted(n[cut:] for n in names)
106
+
107
+ def delete(self, paths: list[str]) -> None:
108
+ if paths:
109
+ self.hub.batch_bucket_files(self.bucket_id, delete=[self._p(p) for p in paths], token=self.token)
110
+
111
+
112
+ def store_from_url(url: str, token: str | None = None):
113
+ """`hf://buckets/<namespace>/<bucket>[/<prefix>]` or a plain directory path."""
114
+ if url.startswith("hf://buckets/"):
115
+ parts = url[len("hf://buckets/"):].strip("/").split("/")
116
+ if len(parts) < 2:
117
+ raise ValueError("expected hf://buckets/<namespace>/<bucket>[/<prefix>]")
118
+ return HfBucketStore("/".join(parts[:2]), "/".join(parts[2:]),
119
+ token or os.environ.get("HF_TOKEN"))
120
+ return DirStore(url)
121
+
122
+
123
+ # -------------------------------------------------------------------- replicator
124
+
125
+
126
+ def _sha256(path: Path) -> str:
127
+ h = hashlib.sha256()
128
+ with open(path, "rb") as f:
129
+ for chunk in iter(lambda: f.read(1 << 20), b""):
130
+ h.update(chunk)
131
+ return h.hexdigest()
132
+
133
+
134
+ def _read_json(store, remote: str) -> dict | None:
135
+ with tempfile.TemporaryDirectory() as d:
136
+ local = Path(d) / "x.json"
137
+ try:
138
+ if not store.get(remote, local):
139
+ return None
140
+ except Exception as exc:
141
+ if "404" in str(exc) or "not found" in str(exc).lower():
142
+ return None
143
+ raise
144
+ return json.loads(local.read_text())
145
+
146
+
147
+ def restore(store, data_dir: Path) -> dict | None:
148
+ """Restore board.db and attachment files from the latest snapshot. Returns the manifest or None."""
149
+ data_dir = Path(data_dir)
150
+ if (data_dir / "board.db").exists():
151
+ raise FileExistsError(f"{data_dir / 'board.db'} exists; refusing to overwrite it with a restore")
152
+ latest = _read_json(store, LATEST)
153
+ if latest is None:
154
+ return None
155
+ data_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
156
+ tmp = data_dir / "board.db.restoring"
157
+ tmp.unlink(missing_ok=True)
158
+ if not store.get(latest["snapshot"], tmp):
159
+ raise RuntimeError(f"LATEST names {latest['snapshot']}, which is missing")
160
+ if _sha256(tmp) != latest["sha256"]:
161
+ tmp.unlink()
162
+ raise RuntimeError("snapshot checksum mismatch")
163
+ conn = sqlite3.connect(tmp)
164
+ try:
165
+ if conn.execute("PRAGMA integrity_check").fetchone()[0] != "ok":
166
+ raise RuntimeError("snapshot failed integrity_check")
167
+ keys = [r[0] for r in conn.execute("SELECT storage_key FROM attachments WHERE state = 'ready'")]
168
+ finally:
169
+ conn.close()
170
+ objects = data_dir / "blobs" / "objects"
171
+ for key in keys:
172
+ dest = objects / key
173
+ dest.parent.mkdir(parents=True, exist_ok=True)
174
+ if not store.get(f"blobs/{key}", dest):
175
+ log.warning("attachment %s missing from replica", key)
176
+ os.replace(tmp, data_dir / "board.db")
177
+ log.info("restored snapshot %s (%s bytes)", latest["snapshot"], latest["size"])
178
+ return latest
179
+
180
+
181
+ def wait_for_release(store, stale_after: float = 120.0, poll: float = 1.0, log_every: float = 10.0,
182
+ stop: threading.Event | None = None) -> str:
183
+ """Before restoring, wait for the previous owner to push its final snapshot and release the lease.
184
+
185
+ Platforms such as HF Spaces start the replacement container before stopping
186
+ the old one, so restoring immediately would miss the old one's last writes.
187
+ Returns why waiting ended: "free", "released", "stale" (previous owner
188
+ stopped heartbeating, e.g. it crashed) or "stopped" (`stop` was set).
189
+ """
190
+ last_log = 0.0
191
+ while stop is None or not stop.is_set():
192
+ lease = _read_json(store, LEASE)
193
+ if lease is None:
194
+ return "free"
195
+ if lease.get("released"):
196
+ return "released"
197
+ if time.time() - lease.get("heartbeat", lease.get("since", 0)) > stale_after:
198
+ return "stale"
199
+ if time.time() - last_log > log_every:
200
+ log.info("waiting for instance %s to hand over the replica", lease.get("instance"))
201
+ last_log = time.time()
202
+ time.sleep(poll)
203
+ return "stopped"
204
+
205
+
206
+ class Replicator:
207
+ """Background snapshot uploader owned by a running Board."""
208
+
209
+ def __init__(self, store, data_dir: Path, interval: float = 10.0, keep: int = 5, heartbeat: float = 30.0):
210
+ self.store = store
211
+ self.data_dir = Path(data_dir)
212
+ self.interval = interval
213
+ self.keep = keep
214
+ self.heartbeat = heartbeat
215
+ self._last_heartbeat = 0.0
216
+ self.instance = uuid.uuid4().hex[:12]
217
+ self.generation_fn = lambda: 0
218
+ self._pushed_generation = None
219
+ self._uploaded_blobs: set[str] | None = None
220
+ self._stop = threading.Event()
221
+ self._thread: threading.Thread | None = None
222
+ self._lock = threading.Lock()
223
+ self.fenced = False
224
+ self.events: list[tuple[float, str]] = []
225
+ self.status = {"instance": self.instance, "last_push_at": None, "last_push_seconds": None,
226
+ "last_snapshot": None, "last_error": None, "pushes": 0, "fenced": False}
227
+
228
+ # Lease: the newest instance owns the replica. An older instance that is
229
+ # still running (overlapping restart) stops pushing instead of overwriting.
230
+ def take_lease(self, released: bool = False) -> None:
231
+ now = time.time()
232
+ self.store.put([(json.dumps({"instance": self.instance, "heartbeat": now, "released": released,
233
+ "snapshot": self.status["last_snapshot"]}).encode(), LEASE)])
234
+ self._last_heartbeat = now
235
+
236
+ def _still_owner(self) -> bool:
237
+ lease = _read_json(self.store, LEASE)
238
+ return lease is None or lease.get("instance") == self.instance
239
+
240
+ def record(self, event: str, wait: bool = False) -> None:
241
+ """Append a lifecycle event to events/<instance>.json in the replica (diagnostics)."""
242
+ self.events.append((time.time(), event))
243
+ body = json.dumps({"instance": self.instance, "events": self.events}).encode()
244
+
245
+ def upload():
246
+ try:
247
+ self.store.put([(body, f"events/{self.instance}.json")])
248
+ except Exception as exc:
249
+ log.warning("could not record event %s: %s", event, exc)
250
+
251
+ if wait:
252
+ upload()
253
+ else:
254
+ threading.Thread(target=upload, daemon=True).start()
255
+
256
+ def start(self, generation_fn) -> None:
257
+ self.generation_fn = generation_fn
258
+ self.take_lease()
259
+ self.record("started")
260
+ self._thread = threading.Thread(target=self._run, name="msgboard-replicator", daemon=True)
261
+ self._thread.start()
262
+
263
+ def _run(self) -> None:
264
+ while not self._stop.wait(self.interval):
265
+ try:
266
+ self.push_if_changed()
267
+ if not self.fenced and time.time() - self._last_heartbeat > self.heartbeat and self._still_owner():
268
+ self.take_lease()
269
+ except Exception as exc: # keep serving; report through metrics
270
+ self.status["last_error"] = f"{type(exc).__name__}: {exc}"[:300]
271
+ log.warning("replication push failed: %s", exc)
272
+
273
+ def stop(self, final_push: bool = True) -> None:
274
+ self._stop.set()
275
+ if self._thread is not None:
276
+ self._thread.join(timeout=60)
277
+ if final_push:
278
+ self.record("final-push-start")
279
+ try:
280
+ pushed = self.push_if_changed()
281
+ if not self.fenced and self._still_owner():
282
+ self.take_lease(released=True) # the next instance may restore now
283
+ self.record("final-push-done" if pushed else
284
+ ("final-push-fenced" if self.fenced else "final-push-unchanged"), wait=True)
285
+ except Exception as exc:
286
+ log.error("final snapshot push failed: %s", exc)
287
+ self.record(f"final-push-failed: {type(exc).__name__}", wait=True)
288
+
289
+ def push_if_changed(self) -> bool:
290
+ with self._lock:
291
+ generation = self.generation_fn()
292
+ if self.fenced or generation == self._pushed_generation:
293
+ return False
294
+ if not self._still_owner():
295
+ self.fenced = self.status["fenced"] = True
296
+ log.error("another instance owns the replica; this instance stops pushing")
297
+ return False
298
+ started = time.time()
299
+ self._push(generation)
300
+ self._pushed_generation = generation
301
+ self.status.update(last_push_at=time.time(), last_push_seconds=round(time.time() - started, 3),
302
+ last_error=None, pushes=self.status["pushes"] + 1)
303
+ return True
304
+
305
+ def _push(self, generation: int) -> None:
306
+ with tempfile.TemporaryDirectory(dir=self.data_dir) as d:
307
+ snap = Path(d) / "snapshot.db"
308
+ src = sqlite3.connect(self.data_dir / "board.db")
309
+ dst = sqlite3.connect(snap)
310
+ try:
311
+ src.backup(dst) # one step: a consistent point-in-time copy
312
+ keys = [r[0] for r in dst.execute("SELECT storage_key FROM attachments WHERE state = 'ready'")]
313
+ finally:
314
+ dst.close()
315
+ src.close()
316
+ # 1. attachment files referenced by the snapshot
317
+ if self._uploaded_blobs is None:
318
+ self._uploaded_blobs = {p[len("blobs/"):] for p in self.store.list("blobs/")}
319
+ new = [k for k in keys if k not in self._uploaded_blobs]
320
+ objects = self.data_dir / "blobs" / "objects"
321
+ for i in range(0, len(new), 50):
322
+ batch = [(objects / k, f"blobs/{k}") for k in new[i:i + 50] if (objects / k).exists()]
323
+ if batch:
324
+ self.store.put(batch)
325
+ self._uploaded_blobs.update(new[i:i + 50])
326
+ # 2. the snapshot, 3. then the pointer to it
327
+ stamp = time.strftime("%Y%m%dT%H%M%S", time.gmtime())
328
+ name = f"snapshots/{stamp}-{self.instance}-{generation}.db"
329
+ manifest = {"snapshot": name, "sha256": _sha256(snap), "size": snap.stat().st_size,
330
+ "created_at": time.time(), "instance": self.instance, "generation": generation}
331
+ self.store.put([(snap, name)])
332
+ self.store.put([(json.dumps(manifest).encode(), LATEST)])
333
+ self.status["last_snapshot"] = name
334
+ self._prune()
335
+
336
+ def _prune(self) -> None:
337
+ snaps = [p for p in self.store.list("snapshots/") if p.endswith(".db")]
338
+ if len(snaps) > self.keep:
339
+ self.store.delete(snaps[: len(snaps) - self.keep])
msgboard/reservations.py ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """In-process reservations: slow object I/O must not take the SQLite writer."""
2
+
3
+ import asyncio
4
+ from contextlib import asynccontextmanager
5
+
6
+
7
+ class Reservations:
8
+ def __init__(self, *, paths=False):
9
+ self.condition = asyncio.Condition()
10
+ self.active = set()
11
+ self.paths = paths
12
+
13
+ def conflicts(self, a, b):
14
+ if not self.paths:
15
+ return a == b
16
+ return a[0] == b[0] and (a[1] == b[1] or a[1].startswith(b[1] + "/") or b[1].startswith(a[1] + "/"))
17
+
18
+ @asynccontextmanager
19
+ async def hold(self, keys):
20
+ keys = set(keys)
21
+ async with self.condition:
22
+ await self.condition.wait_for(lambda: not any(self.conflicts(a, b) for a in keys for b in self.active))
23
+ self.active.update(keys)
24
+ try:
25
+ yield
26
+ finally:
27
+ async with self.condition:
28
+ self.active.difference_update(keys)
29
+ self.condition.notify_all()
msgboard/server.py ADDED
@@ -0,0 +1,1207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """HTTP API: one process, one serialized SQLite writer, a small reader pool."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import fcntl
7
+ import logging
8
+ import mimetypes
9
+ import os
10
+ import random
11
+ import shutil
12
+ import sqlite3
13
+ import threading
14
+ import time
15
+ import tempfile
16
+ from concurrent.futures import ThreadPoolExecutor
17
+ from contextlib import asynccontextmanager, suppress
18
+ from pathlib import Path
19
+ from urllib.parse import unquote
20
+
21
+ import anyio
22
+ from fastapi import FastAPI, Query, Request
23
+ from fastapi.responses import FileResponse, JSONResponse, PlainTextResponse, Response
24
+ from starlette.background import BackgroundTask
25
+
26
+ from . import db as dbmod
27
+ from . import guide
28
+ from . import store
29
+ from .blobs import LocalBlobStore, new_hasher
30
+ from .config import DB_NAME, LIMITS
31
+ from .errors import ApiError, bad_request, forbidden
32
+ from .presence import PRESENCE_FLUSH_SECONDS, Presence
33
+ from . import transfer
34
+ from .reservations import Reservations
35
+
36
+
37
+ class NotInitialized(RuntimeError):
38
+ pass
39
+
40
+
41
+ class AlreadyRunning(RuntimeError):
42
+ pass
43
+
44
+
45
+ class Database:
46
+ """Serialized writes on one connection/thread; reads on a bounded thread pool.
47
+
48
+ Every write runs in BEGIN IMMEDIATE ... COMMIT and returns only after COMMIT,
49
+ so an HTTP success always follows a durable commit (synchronous=FULL).
50
+ """
51
+
52
+ def __init__(self, path: Path):
53
+ self.path = path
54
+ self._writer_conn = dbmod.connect(path)
55
+ self._writer = ThreadPoolExecutor(1, thread_name_prefix="msgboard-writer")
56
+ self._readers = ThreadPoolExecutor(LIMITS.reader_threads, thread_name_prefix="msgboard-reader")
57
+ self._local = threading.local()
58
+ self._reader_conns: list[sqlite3.Connection] = []
59
+ self._lock = threading.Lock()
60
+ self.pending_writes = 0
61
+ self.queued_reads = 0
62
+ self.shed_reads = 0
63
+ self.slow_reads = 0
64
+ # token -> (Principal, checked at): see LIMITS.auth_cache_seconds
65
+ self.auth_cache: dict[str, tuple] = {}
66
+ # Bumped after every committed write; the replicator pushes when it changes.
67
+ self.generation = 0
68
+ self.closing = False
69
+
70
+ def _reader_conn(self) -> sqlite3.Connection:
71
+ conn = getattr(self._local, "conn", None)
72
+ if conn is None:
73
+ conn = dbmod.connect(self.path)
74
+ local = self._local
75
+
76
+ def over_time():
77
+ deadline = getattr(local, "deadline", None)
78
+ return 1 if deadline is not None and time.monotonic() > deadline else 0
79
+ conn.set_progress_handler(over_time, 20_000)
80
+ self._local.conn = conn
81
+ with self._lock:
82
+ self._reader_conns.append(conn)
83
+ return conn
84
+
85
+ async def read(self, fn, *args, shed: bool = True):
86
+ """Run a read on the reader pool. Under overload, refuse early (503) rather than queue work
87
+ whose client will have timed out before it runs: that is what turned a burst into an outage."""
88
+ if shed and self.queued_reads >= LIMITS.max_queued_reads:
89
+ self.shed_reads += 1
90
+ raise ApiError(503, "overloaded", "the board is busy; retry shortly", {"Retry-After": "2"})
91
+ queued_at = time.monotonic()
92
+
93
+ def run():
94
+ if time.monotonic() - queued_at > LIMITS.read_queue_timeout:
95
+ self.shed_reads += 1
96
+ raise ApiError(503, "overloaded", "the board is busy; retry shortly", {"Retry-After": "2"})
97
+ conn = self._reader_conn()
98
+ self._local.deadline = time.monotonic() + LIMITS.max_read_seconds
99
+ try:
100
+ return fn(conn, *args)
101
+ except sqlite3.OperationalError as e:
102
+ if "interrupt" in str(e):
103
+ self.slow_reads += 1
104
+ raise ApiError(503, "too_slow", "this request took too long; try a narrower one",
105
+ {"Retry-After": "5"})
106
+ raise
107
+ finally:
108
+ self._local.deadline = None
109
+
110
+ self.queued_reads += 1 # queued or running; only touched on the event loop
111
+ try:
112
+ return await asyncio.get_running_loop().run_in_executor(self._readers, run)
113
+ finally:
114
+ self.queued_reads -= 1
115
+
116
+ def _run_write(self, fn, args):
117
+ conn = self._writer_conn
118
+ conn.execute("BEGIN IMMEDIATE")
119
+ try:
120
+ result = fn(conn, *args)
121
+ except BaseException:
122
+ conn.execute("ROLLBACK")
123
+ raise
124
+ conn.execute("COMMIT")
125
+ return result
126
+
127
+ async def write(self, fn, *args):
128
+ if self.closing:
129
+ raise ApiError(503, "shutting_down", "server is shutting down; retry shortly", {"Retry-After": "2"})
130
+ if self.pending_writes >= LIMITS.max_pending_writes:
131
+ raise ApiError(503, "overloaded", "write queue is full", {"Retry-After": "1"})
132
+ self.pending_writes += 1
133
+ try:
134
+ loop = asyncio.get_running_loop()
135
+ result = await loop.run_in_executor(self._writer, self._run_write, fn, args)
136
+ self.generation += 1
137
+ if fn not in AUTH_SAFE_WRITES:
138
+ self.auth_cache.clear()
139
+ return result
140
+ finally:
141
+ self.pending_writes -= 1
142
+
143
+ def close(self) -> None:
144
+ self._writer.shutdown(wait=True)
145
+ self._readers.shutdown(wait=True)
146
+ for conn in self._reader_conns:
147
+ conn.close()
148
+ try:
149
+ self._writer_conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
150
+ finally:
151
+ self._writer_conn.close()
152
+
153
+
154
+ ANY_CHANNEL = -1 # notifier key woken by a post in any channel (activity long-polls wait on it)
155
+ # Writes that cannot change who a token belongs to or what it may do; every other write clears the auth cache.
156
+ AUTH_SAFE_WRITES = frozenset({store.post_message, store.react, store.mark_read, store.mark_thread_read,
157
+ store.mark_all_read, store.save_presence})
158
+
159
+
160
+ class Notifier:
161
+ """In-memory per-channel wakeups. Hints only: waiters always re-query the database."""
162
+
163
+ def __init__(self):
164
+ self._waiters: dict[int, set[asyncio.Future]] = {}
165
+ self.loop: asyncio.AbstractEventLoop | None = None
166
+
167
+ def register(self, channel_id: int) -> asyncio.Future:
168
+ fut = asyncio.get_running_loop().create_future()
169
+ self._waiters.setdefault(channel_id, set()).add(fut)
170
+ return fut
171
+
172
+ def unregister(self, channel_id: int, fut: asyncio.Future) -> None:
173
+ waiters = self._waiters.get(channel_id)
174
+ if waiters is not None:
175
+ waiters.discard(fut)
176
+ if not waiters:
177
+ del self._waiters[channel_id]
178
+
179
+ def notify(self, channel_id: int) -> None:
180
+ for fut in self._waiters.pop(channel_id, ()):
181
+ if not fut.done():
182
+ fut.set_result(None)
183
+
184
+ def wake_all(self) -> None:
185
+ for channel_id in list(self._waiters):
186
+ self.notify(channel_id)
187
+
188
+
189
+ class Board:
190
+ """Everything one running server owns for a data directory."""
191
+
192
+ def __init__(self, data_dir: Path, replicator=None):
193
+ self.data_dir = Path(data_dir)
194
+ db_path = self.data_dir / DB_NAME
195
+ if not db_path.exists():
196
+ raise NotInitialized(f"no board in {self.data_dir}; run `python -m msgboard init` first")
197
+ self._lock_fh = open(self.data_dir / "server.lock", "a")
198
+ try:
199
+ fcntl.flock(self._lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
200
+ except BlockingIOError:
201
+ self._lock_fh.close()
202
+ raise AlreadyRunning(f"another server is already using {self.data_dir}")
203
+ check = dbmod.connect(db_path)
204
+ try:
205
+ version = dbmod.schema_version(check)
206
+ if version > dbmod.SCHEMA_VERSION:
207
+ raise dbmod.SchemaError(f"database schema {version} is newer than this software")
208
+ if version < dbmod.SCHEMA_VERSION:
209
+ raise NotInitialized("database schema is outdated; run `python -m msgboard init` to migrate")
210
+ if not store.admin_exists(check):
211
+ raise NotInitialized("board has no operator; run `python -m msgboard init`")
212
+ except BaseException:
213
+ check.close()
214
+ self._release_lock()
215
+ raise
216
+ check.close()
217
+ self.db = Database(db_path)
218
+ # one-time maintenance on the writer connection before serving
219
+ with self.db._lock:
220
+ conn = self.db._writer_conn
221
+ conn.execute("BEGIN IMMEDIATE")
222
+ try:
223
+ store.backfill_mentions(conn)
224
+ store.backfill_mentioned_as(conn)
225
+ conn.execute("COMMIT")
226
+ except BaseException:
227
+ conn.execute("ROLLBACK")
228
+ raise
229
+ self.presence = Presence(store.load_presence(conn))
230
+ self.blobs = LocalBlobStore(self.data_dir / "blobs")
231
+ # storage folders live in the replication store (the Space's bucket) or, locally, under the data dir;
232
+ # downloads are served from a local cache of those objects
233
+ from .replicate import DirStore
234
+ self.storage = replicator.store if replicator is not None else DirStore(self.data_dir)
235
+ self.storage_cache = self.data_dir / "storage-cache"
236
+ self.storage_cache.mkdir(exist_ok=True)
237
+ self.notifier = Notifier()
238
+ self.shutting_down = False
239
+ self.open_polls = 0
240
+ self.open_uploads = 0
241
+ self.commit_lock: asyncio.Lock | None = None
242
+ self.storage_reservations = Reservations(paths=True)
243
+ self.import_reservations = Reservations()
244
+ self.started = time.time()
245
+ self.counters = {"posts": 0, "posts_replayed": 0, "edits": 0, "polls": 0, "errors_5xx": 0}
246
+ self.replicator = replicator
247
+ if replicator is not None:
248
+ replicator.start(lambda: self.db.generation)
249
+
250
+ def begin_shutdown(self) -> None:
251
+ """Release held long-polls so a Ctrl-C does not wait for them to time out."""
252
+ if self.replicator is not None and not self.shutting_down:
253
+ self.replicator.record("shutdown-signal")
254
+ self.shutting_down = True
255
+ if self.replicator is not None:
256
+ # With replication, stop taking writes now: the final snapshot must
257
+ # include every acknowledged write, and the next instance is waiting.
258
+ self.db.closing = True
259
+ loop = self.notifier.loop
260
+ if loop is not None and not loop.is_closed():
261
+ loop.call_soon_threadsafe(self.notifier.wake_all)
262
+
263
+ def _release_lock(self) -> None:
264
+ try:
265
+ fcntl.flock(self._lock_fh, fcntl.LOCK_UN)
266
+ finally:
267
+ self._lock_fh.close()
268
+
269
+ async def flush_presence(self) -> None:
270
+ versions, records = self.presence.pending()
271
+ history = self.presence.pending_history()
272
+ if records or history:
273
+ await self.db.write(store.save_presence, records, history)
274
+ self.presence.saved(versions)
275
+ self.presence.history_saved(history)
276
+
277
+ async def presence_loop(self) -> None:
278
+ while True:
279
+ await asyncio.sleep(PRESENCE_FLUSH_SECONDS)
280
+ if self.shutting_down:
281
+ return
282
+ self.presence.sample(time.time())
283
+ try:
284
+ await self.flush_presence()
285
+ except Exception:
286
+ # Observations remain dirty and retry on the next interval. Presence
287
+ # must not cause regular requests to fail when the writer is busy.
288
+ logging.getLogger(__name__).exception("could not persist agent presence")
289
+
290
+ def close(self) -> None:
291
+ if getattr(self, "_closed", False):
292
+ return
293
+ self._closed = True
294
+ # The event loop has stopped. Queue the last observations after in-flight
295
+ # writes, before the replicator takes its final snapshot. This also works
296
+ # when begin_shutdown has closed the normal HTTP writer entry point.
297
+ versions, records = self.presence.pending()
298
+ history = self.presence.pending_history()
299
+ if records or history:
300
+ try:
301
+ self.db._writer.submit(self.db._run_write, store.save_presence, (records, history)).result()
302
+ self.db.generation += 1
303
+ self.presence.saved(versions)
304
+ self.presence.history_saved(history)
305
+ except Exception:
306
+ # Ephemeral presence must never prevent the final snapshot of
307
+ # acknowledged messages or leave the database/lease open.
308
+ logging.getLogger(__name__).exception("could not persist final agent presence")
309
+ if self.replicator is not None:
310
+ self.replicator.stop(final_push=True) # after in-flight writes, before closing
311
+ self.db.close()
312
+ self._release_lock()
313
+
314
+
315
+ def _bearer(request: Request) -> str | None:
316
+ # X-Msgboard-Token lets clients reach the board through a proxy that uses
317
+ # Authorization for its own login (for example a private HF Space).
318
+ direct = request.headers.get("x-msgboard-token")
319
+ if direct:
320
+ return direct.strip()
321
+ header = request.headers.get("authorization", "")
322
+ scheme, _, value = header.partition(" ")
323
+ return value.strip() if scheme.lower() == "bearer" else None
324
+
325
+
326
+ async def _json_body(request: Request, max_bytes: int = 256 * 1024):
327
+ import json
328
+
329
+ data = bytearray()
330
+ async for chunk in request.stream():
331
+ data += chunk
332
+ if len(data) > max_bytes:
333
+ raise ApiError(413, "too_large", "request body too large")
334
+ if not data:
335
+ return {}
336
+ try:
337
+ return json.loads(data)
338
+ except ValueError:
339
+ raise bad_request("request body is not valid JSON")
340
+
341
+
342
+ def create_app(board: Board, dashboard: bool = True) -> FastAPI:
343
+ @asynccontextmanager
344
+ async def lifespan(app):
345
+ board.notifier.loop = asyncio.get_running_loop()
346
+ board.commit_lock = asyncio.Lock()
347
+ board.presence.sample(time.time())
348
+ task = asyncio.create_task(board.presence_loop())
349
+ try:
350
+ yield
351
+ finally:
352
+ board.begin_shutdown()
353
+ task.cancel()
354
+ with suppress(asyncio.CancelledError):
355
+ await task
356
+
357
+ app = FastAPI(title="msgboard", version="0.1.0", lifespan=lifespan,
358
+ docs_url=None, redoc_url=None, openapi_url="/v1/openapi.json")
359
+ app.state.board = board
360
+ db = board.db
361
+
362
+ @app.exception_handler(ApiError)
363
+ async def api_error(request: Request, exc: ApiError):
364
+ if exc.status >= 500:
365
+ board.counters["errors_5xx"] += 1
366
+ body = {"code": exc.code, "message": exc.message}
367
+ if exc.details:
368
+ body["details"] = exc.details
369
+ return JSONResponse({"error": body}, status_code=exc.status, headers=exc.headers)
370
+
371
+ async def authn(request: Request, *, observe: bool = True):
372
+ token = _bearer(request)
373
+ now = time.monotonic()
374
+ hit = db.auth_cache.get(token) if token else None
375
+ if hit is not None and now - hit[1] < LIMITS.auth_cache_seconds:
376
+ if observe:
377
+ board.presence.seen(hit[0], time.time())
378
+ return hit[0]
379
+ p = await db.read(store.authenticate, token, time.time())
380
+ if len(db.auth_cache) > 20_000:
381
+ db.auth_cache.clear()
382
+ db.auth_cache[token] = (p, now)
383
+ if observe:
384
+ board.presence.seen(p, time.time())
385
+ return p
386
+
387
+ async def with_presence(p, items):
388
+ return await db.read(store.attach_presence, p, items, board.presence.snapshot())
389
+
390
+ # ------------------------------------------------------------- health
391
+
392
+ @app.get("/v1/health")
393
+ async def health():
394
+ return {"status": "ok"}
395
+
396
+ @app.get("/v1/ready")
397
+ async def ready():
398
+ await db.read(lambda c: c.execute("SELECT 1").fetchone())
399
+ return {"status": "ready", "shutting_down": board.shutting_down}
400
+
401
+ @app.get("/v1/metrics")
402
+ async def metrics(request: Request):
403
+ p = await authn(request)
404
+ if not p.is_admin:
405
+ raise forbidden("admin only")
406
+ return health_summary()
407
+
408
+ def health_summary():
409
+ db_path = board.data_dir / DB_NAME
410
+ wal = Path(str(db_path) + "-wal")
411
+ return {
412
+ **board.counters,
413
+ "open_polls": board.open_polls,
414
+ "open_uploads": board.open_uploads,
415
+ "pending_writes": db.pending_writes,
416
+ "queued_reads": db.queued_reads,
417
+ "shed_reads": db.shed_reads,
418
+ "slow_reads": db.slow_reads,
419
+ "db_bytes": db_path.stat().st_size,
420
+ "wal_bytes": wal.stat().st_size if wal.exists() else 0,
421
+ "disk_free_bytes": shutil.disk_usage(board.data_dir).free,
422
+ "uptime_seconds": time.time() - board.started,
423
+ "replication": board.replicator.status if board.replicator else None,
424
+ }
425
+
426
+ # ------------------------------------------------------- identities
427
+
428
+ def body_dict(body):
429
+ if not isinstance(body, dict):
430
+ raise bad_request("request body must be a JSON object")
431
+ return body
432
+
433
+ @app.get("/v1/me")
434
+ async def get_me(request: Request):
435
+ p = await authn(request)
436
+ return (await with_presence(p, [await db.read(store.me, p)]))[0]
437
+
438
+ @app.post("/v1/me/read-all")
439
+ async def mark_all_read(request: Request):
440
+ """Mark everything you can read as read (channels, DMs and threads)."""
441
+ p = await authn(request)
442
+ return await db.write(store.mark_all_read, p, time.time())
443
+
444
+ @app.put("/v1/me/prefs")
445
+ async def put_prefs(request: Request):
446
+ p = await authn(request)
447
+ return await db.write(store.update_prefs, p, await _json_body(request, 64 * 1024), time.time())
448
+
449
+ @app.get("/v1/principals")
450
+ async def list_principals(request: Request):
451
+ p = await authn(request)
452
+ return {"principals": await with_presence(p, await db.read(store.list_principals, p))}
453
+
454
+ @app.get("/v1/presence")
455
+ async def list_presence(request: Request):
456
+ p = await authn(request)
457
+ return {"principals": await db.read(store.list_presence, p, board.presence.snapshot())}
458
+
459
+ @app.patch("/v1/principals/{principal_id}")
460
+ async def update_principal(principal_id: int, request: Request):
461
+ p = await authn(request)
462
+ body = body_dict(await _json_body(request))
463
+ result = await db.write(store.update_principal, p, principal_id, body, time.time())
464
+ return (await with_presence(p, [result]))[0]
465
+
466
+ @app.post("/v1/principals/{principal_id}/suspend")
467
+ async def suspend(principal_id: int, request: Request):
468
+ p = await authn(request)
469
+ raw = await request.body()
470
+ body = body_dict(await _json_body(request)) if raw.strip() else {}
471
+ return await db.write(store.suspend, p, principal_id, body.get("reason"), time.time())
472
+
473
+ @app.post("/v1/principals/{principal_id}/unsuspend")
474
+ async def unsuspend(principal_id: int, request: Request):
475
+ p = await authn(request)
476
+ return await db.write(store.unsuspend, p, principal_id, time.time())
477
+
478
+ @app.get("/v1/suspended")
479
+ async def list_suspended(request: Request):
480
+ p = await authn(request)
481
+ return {"suspended": await db.read(store.list_suspended, p)}
482
+
483
+ @app.post("/v1/principals/{principal_id}/token")
484
+ async def rotate_token(principal_id: int, request: Request):
485
+ p = await authn(request)
486
+ result = await db.write(store.rotate_agent_token, p, principal_id, time.time())
487
+ return JSONResponse(result, status_code=201, headers={"Cache-Control": "no-store"})
488
+
489
+ @app.post("/v1/principals/{principal_id}/signin-link")
490
+ async def signin_link(principal_id: int, request: Request):
491
+ p = await authn(request)
492
+ result = await db.write(store.create_signin_link, p, principal_id, time.time())
493
+ return JSONResponse(result, status_code=201, headers={"Cache-Control": "no-store"})
494
+
495
+ @app.delete("/v1/tokens/{token_id}")
496
+ async def revoke_token(token_id: str, request: Request):
497
+ p = await authn(request)
498
+ return await db.write(store.revoke_token, p, token_id, time.time())
499
+
500
+ # ------------------------------------------------- invites and joining
501
+
502
+ @app.post("/v1/invites")
503
+ async def create_invite(request: Request):
504
+ p = await authn(request)
505
+ body = body_dict(await _json_body(request))
506
+ return JSONResponse(await db.write(store.create_invite, p, body, time.time()), status_code=201,
507
+ headers={"Cache-Control": "no-store"})
508
+
509
+ @app.get("/v1/invites")
510
+ async def list_invites(request: Request):
511
+ p = await authn(request)
512
+ return {"invites": await db.read(store.list_invites, p, time.time())}
513
+
514
+ @app.delete("/v1/invites/{invite_id}")
515
+ async def revoke_invite(invite_id: int, request: Request):
516
+ p = await authn(request)
517
+ return await db.write(store.revoke_invite, p, invite_id, time.time())
518
+
519
+ @app.get("/v1/join/{code}")
520
+ async def lookup_invite(code: str):
521
+ return await db.read(store.lookup_invite, code, time.time())
522
+
523
+ @app.post("/v1/join")
524
+ async def join(request: Request):
525
+ body = body_dict(await _json_body(request, 4096))
526
+ result = await db.write(store.join, body.get("code"), body.get("name"), body.get("display_name"), time.time())
527
+ return JSONResponse(result, status_code=201 if result["created"] else 200, headers={"Cache-Control": "no-store"})
528
+
529
+ # ------------------------------------------------------------ settings
530
+
531
+ @app.get("/v1/settings")
532
+ async def get_settings(request: Request):
533
+ await authn(request)
534
+ return await db.read(store.get_settings)
535
+
536
+ @app.patch("/v1/settings")
537
+ async def update_settings(request: Request):
538
+ p = await authn(request)
539
+ body = body_dict(await _json_body(request))
540
+ return await db.write(store.update_settings, p, body, time.time())
541
+
542
+ # --------------------------------------------------------- channels
543
+
544
+ @app.get("/v1/channels")
545
+ async def list_channels(request: Request, name: str | None = None):
546
+ p = await authn(request)
547
+ return {"channels": await db.read(store.list_channels, p, name)}
548
+
549
+ @app.post("/v1/channels")
550
+ async def create_channel(request: Request):
551
+ p = await authn(request)
552
+ body = body_dict(await _json_body(request))
553
+ channel, created = await db.write(store.create_channel, p, body.get("name"), body.get("description"),
554
+ body.get("private", False), time.time())
555
+ return JSONResponse(channel, status_code=201 if created else 200)
556
+
557
+ @app.get("/v1/messages/{message_id}")
558
+ async def get_message(message_id: int, request: Request):
559
+ p = await authn(request)
560
+ return await db.read(store.get_message, p, message_id)
561
+
562
+ @app.get("/v1/channels/{channel_id}/export")
563
+ async def export_channel(channel_id: int, request: Request):
564
+ p = await authn(request)
565
+ transfer.require_admin(p)
566
+ manifest, keys = await db.read(transfer.export_manifest, p, channel_id, _origin(request))
567
+ scratch = tempfile.TemporaryDirectory(prefix="msgboard-export-")
568
+ path = Path(scratch.name) / "channel.zip"
569
+ try:
570
+ await anyio.to_thread.run_sync(transfer.build_bundle, manifest, keys, board.blobs, path, board.storage)
571
+ except BaseException:
572
+ scratch.cleanup()
573
+ raise
574
+ return FileResponse(path, media_type="application/zip", filename=f"channel-{channel_id}.zip",
575
+ background=BackgroundTask(scratch.cleanup))
576
+
577
+ @app.post("/v1/channels/{channel_id}/import")
578
+ async def import_channel(channel_id: int, request: Request, dry_run: bool = False, keep_target_doc: bool = False):
579
+ p = await authn(request, observe=not dry_run)
580
+ await db.read(transfer.public_channel, p, channel_id)
581
+ declared = request.headers.get("content-length", "")
582
+ if declared.isdigit() and int(declared) > transfer.MAX_BUNDLE_BYTES:
583
+ raise ApiError(413, "too_large", "channel bundle exceeds 512 MiB")
584
+ with tempfile.TemporaryDirectory(prefix="msgboard-import-") as scratch:
585
+ path = Path(scratch) / "bundle.zip"
586
+ with path.open("xb") as fh:
587
+ size = 0
588
+ async for chunk in request.stream():
589
+ size += len(chunk)
590
+ if size > transfer.MAX_BUNDLE_BYTES:
591
+ raise ApiError(413, "too_large", "channel bundle exceeds 512 MiB")
592
+ await anyio.to_thread.run_sync(fh.write, chunk)
593
+ manifest, digest, files = await anyio.to_thread.run_sync(transfer.read_bundle, path, Path(scratch))
594
+ if dry_run:
595
+ report = await db.read(transfer.plan_import, p, channel_id, manifest, digest, keep_target_doc)
596
+ report = await anyio.to_thread.run_sync(transfer.verify_reused_storage, report, board.storage)
597
+ else:
598
+ # A disconnected caller must not remove staged local files while
599
+ # its publication task is still using them. Retries are idempotent.
600
+ task = asyncio.create_task(apply_history(p, channel_id, manifest, digest, files, keep_target_doc))
601
+ try:
602
+ report = await asyncio.shield(task)
603
+ except asyncio.CancelledError:
604
+ with suppress(Exception):
605
+ await task
606
+ raise
607
+ return {**report, "dry_run": dry_run}
608
+
609
+ async def apply_history(p, channel_id, manifest, digest, files, keep_target_doc):
610
+ keys = [("channel", str(channel_id))] + [("author", a["external_id"].lower()) for a in manifest["principals"]]
611
+ async with board.import_reservations.hold(keys):
612
+ prepared = await db.write(transfer.prepare_import, p, channel_id, manifest, digest, time.time(), keep_target_doc)
613
+ paths = [(prepared["people"][f["owner_id"]], f["path"]) for f in manifest["storage_files"]]
614
+ published = False
615
+ uploaded_storage, uploaded_attachments = [], []
616
+ try:
617
+ async with board.storage_reservations.hold(paths):
618
+ try:
619
+ prepared["report"] = await db.read(transfer.checked_import_plan, p, channel_id, manifest, digest, prepared)
620
+ await anyio.to_thread.run_sync(transfer.stage_import, channel_id, manifest, files, prepared,
621
+ board.blobs, board.storage, uploaded_storage, uploaded_attachments)
622
+ report = await db.write(transfer.import_channel, p, channel_id, manifest, digest, prepared, time.time())
623
+ published = True
624
+ return report
625
+ finally:
626
+ if not published:
627
+ if uploaded_storage:
628
+ try:
629
+ await anyio.to_thread.run_sync(board.storage.delete, uploaded_storage)
630
+ except Exception:
631
+ logging.getLogger(__name__).exception("could not remove unindexed import files")
632
+ for key in uploaded_attachments:
633
+ board.blobs.discard(board.blobs.path_for(key))
634
+ finally:
635
+ if not published:
636
+ await db.write(transfer.cleanup_ghosts, prepared)
637
+
638
+ @app.get("/v1/channels/{channel_id}/doc")
639
+ async def get_doc(channel_id: int, request: Request):
640
+ p = await authn(request)
641
+ return await db.read(store.get_doc, p, channel_id)
642
+
643
+ @app.put("/v1/channels/{channel_id}/doc")
644
+ async def put_doc(channel_id: int, request: Request):
645
+ p = await authn(request)
646
+ doc, changed = await db.write(store.save_doc, p, channel_id,
647
+ await _json_body(request, store.MAX_DOC_BYTES * 2 + 4096), time.time())
648
+ return doc
649
+
650
+ @app.get("/v1/channels/{channel_id}/doc/versions")
651
+ async def doc_versions(channel_id: int, request: Request):
652
+ p = await authn(request)
653
+ return await db.read(store.doc_versions, p, channel_id)
654
+
655
+ @app.get("/v1/channels/{channel_id}/doc/versions/{version}")
656
+ async def doc_version(channel_id: int, version: int, request: Request):
657
+ p = await authn(request)
658
+ return await db.read(store.doc_version, p, channel_id, version)
659
+
660
+ @app.delete("/v1/channels/{channel_id}")
661
+ async def delete_channel(channel_id: int, request: Request, confirm: str = ""):
662
+ p = await authn(request)
663
+ result = await db.write(store.delete_channel, p, channel_id, confirm, time.time())
664
+ for key in result.pop("storage_keys"): # the rows are gone; now the files
665
+ try:
666
+ board.blobs.path_for(key).unlink()
667
+ except FileNotFoundError:
668
+ pass
669
+ board.notifier.notify(channel_id)
670
+ return result
671
+
672
+ @app.get("/v1/channels/{channel_id}/info")
673
+ async def channel_info(channel_id: int, request: Request):
674
+ p = await authn(request)
675
+ result = await db.read(store.channel_info, p, channel_id)
676
+ result["members"] = await with_presence(p, result["members"])
677
+ result["posters"] = await with_presence(p, result["posters"])
678
+ return result
679
+
680
+ @app.patch("/v1/channels/{channel_id}")
681
+ async def update_channel(channel_id: int, request: Request):
682
+ p = await authn(request)
683
+ return await db.write(store.update_channel, p, channel_id, await _json_body(request, 4096), time.time())
684
+
685
+ @app.post("/v1/channels/{channel_id}/archive")
686
+ async def archive_channel(channel_id: int, request: Request):
687
+ p = await authn(request)
688
+ return await db.write(store.archive_channel, p, channel_id, time.time())
689
+
690
+ @app.post("/v1/channels/{channel_id}/threads/{thread_id}/read")
691
+ async def mark_thread_read(channel_id: int, thread_id: int, request: Request):
692
+ p = await authn(request)
693
+ body = body_dict(await _json_body(request, 1024))
694
+ return await db.write(store.mark_thread_read, p, channel_id, thread_id, body.get("seq"), time.time())
695
+
696
+ @app.post("/v1/channels/{channel_id}/read")
697
+ async def mark_read(channel_id: int, request: Request):
698
+ p = await authn(request)
699
+ body = body_dict(await _json_body(request, 1024))
700
+ return await db.write(store.mark_read, p, channel_id, body.get("seq"), time.time())
701
+
702
+ @app.get("/v1/activity")
703
+ async def get_activity(request: Request, limit: int = Query(50, ge=1, le=200), after: int | None = Query(None, ge=0),
704
+ wait: float = Query(0, ge=0, le=LIMITS.max_wait_seconds), channels: str | None = None,
705
+ reactions: bool = False):
706
+ """Everything for you (mentions, replies, DMs, plus `channels` you follow). With `after`, a
707
+ cursor read, oldest first; add `wait` to long-poll it."""
708
+ p = await authn(request)
709
+ parts = [x.strip() for x in channels.split(",") if x.strip()] if channels else []
710
+ joined = "joined" in parts # every channel you are a member of
711
+ try:
712
+ follow = [int(x) for x in parts if x != "joined"]
713
+ except ValueError:
714
+ raise bad_request("channels must be comma-separated channel ids, or `joined`")
715
+ if joined:
716
+ follow += await db.read(store.joined_channel_ids, p)
717
+ if len(follow) > 100:
718
+ raise bad_request("follow at most 100 channels")
719
+ if wait > 0 and after is None:
720
+ raise bad_request("wait needs a cursor: pass `after` (start from the next_cursor of a plain read)")
721
+ token = board.presence.next_poll()
722
+ polled_at = time.time()
723
+ mode = "joined" if joined else ("channels" if follow else "addressed")
724
+
725
+ def observe(result):
726
+ board.presence.polled(p, token, polled_at, after, mode, follow, bool(result["messages"]))
727
+
728
+ if wait <= 0:
729
+ result = await db.read(store.activity, p, limit, after, follow, reactions)
730
+ observe(result)
731
+ return result
732
+ if board.open_polls >= LIMITS.max_open_polls:
733
+ raise ApiError(429, "too_many_polls", "too many open long-polls", {"Retry-After": "1"})
734
+ board.open_polls += 1
735
+ board.counters["polls"] += 1
736
+ fut = board.notifier.register(ANY_CHANNEL)
737
+ try:
738
+ result = await db.read(store.activity, p, limit, after, follow)
739
+ observe(result)
740
+ if result["messages"] or board.shutting_down:
741
+ return result
742
+ deadline = time.monotonic() + wait
743
+ while True: # any post wakes us; keep waiting until one is actually for this caller
744
+ left = deadline - time.monotonic()
745
+ if left <= 0:
746
+ break
747
+ done, _ = await asyncio.wait({fut}, timeout=left)
748
+ if done and not board.shutting_down:
749
+ # let a burst of posts land, then answer them with one query instead of one per post
750
+ await asyncio.sleep(min(LIMITS.poll_coalesce_seconds * (0.75 + random.random() / 2),
751
+ max(0.0, deadline - time.monotonic())))
752
+ if await request.is_disconnected():
753
+ return result
754
+ board.notifier.unregister(ANY_CHANNEL, fut)
755
+ fut = board.notifier.register(ANY_CHANNEL)
756
+ p = await authn(request, observe=False)
757
+ after = result["next_cursor"] # previous query exhausted the older history
758
+ result = await db.read(store.activity, p, limit, after, follow, shed=False)
759
+ board.presence.observed(p, token, bool(result["messages"]), time.time())
760
+ if result["messages"] or board.shutting_down:
761
+ return result
762
+ return result
763
+ finally:
764
+ board.notifier.unregister(ANY_CHANNEL, fut)
765
+ board.open_polls -= 1
766
+
767
+ @app.get("/v1/search")
768
+ async def search(request: Request, q: str = "", channel: int | None = None, sender: int | None = None,
769
+ after: str | None = None, before: str | None = None, sort: str = "newest",
770
+ limit: int = Query(20, ge=1, le=100), offset: int = Query(0, ge=0, le=10_000)):
771
+ """Full-text search over messages you can read, plus matching channel and people names.
772
+ `q` takes words, "phrases" and filters: from:@name in:#channel after:/before:/on:DATE has:file ..."""
773
+ p = await authn(request)
774
+ if len(q) > 500:
775
+ raise bad_request("q is at most 500 characters")
776
+ return await db.read(lambda conn: store.search(conn, p, q, channel_id=channel, sender_id=sender, after=after,
777
+ before=before, sort=sort, limit=limit, offset=offset,
778
+ now=time.time()))
779
+
780
+ @app.get("/v1/analytics")
781
+ async def get_analytics(request: Request, days: int = Query(7, ge=1, le=365),
782
+ start: float | None = Query(None, alias="from"),
783
+ end: float | None = Query(None, alias="to"),
784
+ top_k: int = Query(10, ge=1, le=100)):
785
+ p = await authn(request)
786
+ result = await db.read(store.analytics, p, time.time(), days, start, end, top_k,
787
+ board.presence.pending_history())
788
+ result["listening"]["now"] = board.presence.listening_now(time.time())
789
+ result["health"] = health_summary()
790
+ return result
791
+
792
+ def _origin(request: Request) -> str:
793
+ proto = request.headers.get("x-forwarded-proto") or request.url.scheme
794
+ host = request.headers.get("x-forwarded-host") or request.headers.get("host") or request.url.netloc
795
+ return f"{proto}://{host}{request.scope.get('root_path', '')}"
796
+
797
+ @app.get("/agents", include_in_schema=False)
798
+ async def agents_guide(request: Request):
799
+ return PlainTextResponse(guide.render(_origin(request), public=True), media_type="text/markdown; charset=utf-8")
800
+
801
+ @app.get("/v1/guide")
802
+ async def get_guide(request: Request):
803
+ await authn(request)
804
+ settings = await db.read(store.get_settings)
805
+ return PlainTextResponse(guide.render(_origin(request), settings["guidelines"], docs=settings["channel_docs"]),
806
+ media_type="text/markdown; charset=utf-8")
807
+
808
+ @app.get("/v1/channels/{channel_id}/members")
809
+ async def list_members(channel_id: int, request: Request):
810
+ p = await authn(request)
811
+ return {"members": await with_presence(p, await db.read(store.list_members, p, channel_id))}
812
+
813
+ @app.post("/v1/channels/{channel_id}/members")
814
+ async def add_member(channel_id: int, request: Request):
815
+ p = await authn(request)
816
+ body = body_dict(await _json_body(request))
817
+ members = await db.write(store.add_member, p, channel_id, body.get("principal_id"), time.time())
818
+ return {"members": await with_presence(p, members)}
819
+
820
+ @app.delete("/v1/channels/{channel_id}/members/{principal_id}")
821
+ async def remove_member(channel_id: int, principal_id: int, request: Request):
822
+ p = await authn(request)
823
+ result = await db.write(store.remove_member, p, channel_id, principal_id, time.time())
824
+ return result if isinstance(result, dict) else {"members": await with_presence(p, result)}
825
+
826
+ @app.post("/v1/dms")
827
+ async def open_dm(request: Request):
828
+ p = await authn(request)
829
+ body = body_dict(await _json_body(request))
830
+ channel, created = await db.write(store.open_dm, p, body.get("with"), time.time())
831
+ return JSONResponse(channel, status_code=201 if created else 200)
832
+
833
+ # --------------------------------------------------------- messages
834
+
835
+ @app.post("/v1/channels/{channel_id}/messages")
836
+ async def post_message(channel_id: int, request: Request):
837
+ p = await authn(request)
838
+ request_id = request.headers.get("idempotency-key")
839
+ if not request_id or not store.CLIENT_ID.match(request_id):
840
+ raise bad_request("Idempotency-Key header is required and must match " + store.CLIENT_ID.pattern)
841
+ payload = await _json_body(request, LIMITS.max_body_bytes * 2 + 64 * 1024)
842
+ normalized, digest = store.normalize_payload(channel_id, payload)
843
+ message, created = await db.write(
844
+ store.post_message, p, channel_id, request_id, normalized, digest, time.time())
845
+ if created:
846
+ board.counters["posts"] += 1
847
+ board.notifier.notify(channel_id)
848
+ board.notifier.notify(ANY_CHANNEL)
849
+ return JSONResponse(message, status_code=201)
850
+ board.counters["posts_replayed"] += 1
851
+ return JSONResponse(message, status_code=200, headers={"Idempotent-Replayed": "true"})
852
+
853
+ @app.patch("/v1/channels/{channel_id}/messages/{message_id}")
854
+ async def edit_message(channel_id: int, message_id: int, request: Request):
855
+ p = await authn(request)
856
+ payload = await _json_body(request, LIMITS.max_body_bytes * 2 + 1024)
857
+ message, changed = await db.write(store.edit_message, p, channel_id, message_id, payload, time.time())
858
+ if changed:
859
+ board.counters["edits"] += 1
860
+ board.notifier.notify(channel_id)
861
+ return message
862
+
863
+ @app.put("/v1/channels/{channel_id}/messages/{message_id}/reactions/{emoji}")
864
+ async def add_reaction(channel_id: int, message_id: int, emoji: str, request: Request):
865
+ p = await authn(request)
866
+ message, changed = await db.write(store.react, p, channel_id, message_id, unquote(emoji), True, time.time())
867
+ if changed:
868
+ board.notifier.notify(channel_id)
869
+ return message
870
+
871
+ @app.delete("/v1/channels/{channel_id}/messages/{message_id}/reactions/{emoji}")
872
+ async def remove_reaction(channel_id: int, message_id: int, emoji: str, request: Request):
873
+ p = await authn(request)
874
+ message, changed = await db.write(store.react, p, channel_id, message_id, unquote(emoji), False, time.time())
875
+ if changed:
876
+ board.notifier.notify(channel_id)
877
+ return message
878
+
879
+ @app.delete("/v1/channels/{channel_id}/messages/{message_id}")
880
+ async def delete_message(channel_id: int, message_id: int, request: Request):
881
+ p = await authn(request)
882
+ result = await db.write(store.delete_message, p, channel_id, message_id, time.time())
883
+ for key in result.pop("storage_keys"):
884
+ try:
885
+ board.blobs.path_for(key).unlink()
886
+ except FileNotFoundError:
887
+ pass
888
+ board.notifier.notify(channel_id)
889
+ return result
890
+
891
+ @app.get("/v1/channels/{channel_id}/messages/{message_id}/revisions")
892
+ async def message_revisions(channel_id: int, message_id: int, request: Request):
893
+ p = await authn(request)
894
+ return await db.read(store.message_revisions, p, channel_id, message_id)
895
+
896
+ @app.get("/v1/channels/{channel_id}/messages")
897
+ async def read_messages(
898
+ channel_id: int,
899
+ request: Request,
900
+ after: int = Query(0, ge=0),
901
+ limit: int = Query(LIMITS.default_page_size, ge=1, le=LIMITS.max_page_size),
902
+ wait: float = Query(0, ge=0, le=LIMITS.max_wait_seconds),
903
+ before: int | None = Query(None, ge=1),
904
+ top_level: bool = False,
905
+ thread: int | None = Query(None, ge=1),
906
+ edits_after: int | None = Query(None, ge=0),
907
+ ):
908
+ p = await authn(request)
909
+ if before is not None or top_level or thread is not None:
910
+ if wait > 0 or edits_after is not None:
911
+ raise bad_request("wait and edits_after only work with plain `after` reads")
912
+ return await db.read(store.read_messages, p, channel_id, after, limit, before, top_level, thread)
913
+
914
+ async def page(p):
915
+ result = await db.read(store.read_messages, p, channel_id, after, limit)
916
+ if edits_after is not None: # also hand back messages edited since this edit cursor
917
+ result.update(await db.read(store.read_edits, p, channel_id, edits_after, limit))
918
+ return result
919
+
920
+ if wait <= 0:
921
+ return await page(p)
922
+ if board.open_polls >= LIMITS.max_open_polls:
923
+ raise ApiError(429, "too_many_polls", "too many open long-polls", {"Retry-After": "1"})
924
+ board.open_polls += 1
925
+ board.counters["polls"] += 1
926
+ # Register before checking history so a commit between the query and
927
+ # the wait cannot be missed.
928
+ fut = board.notifier.register(channel_id)
929
+ try:
930
+ result = await page(p)
931
+ if result["messages"] or result.get("edits") or board.shutting_down:
932
+ return result
933
+ await asyncio.wait({fut}, timeout=wait)
934
+ # Re-authenticate and re-authorize: the token may have been revoked
935
+ # or the grant removed while this request was held.
936
+ p = await authn(request)
937
+ return await page(p)
938
+ finally:
939
+ board.notifier.unregister(channel_id, fut)
940
+ board.open_polls -= 1
941
+
942
+ # ------------------------------------------------- profiles, invites
943
+
944
+ @app.get("/v1/principals/{principal_id}/profile")
945
+ async def get_profile(principal_id: int, request: Request):
946
+ p = await authn(request)
947
+ result = await db.read(store.profile, p, principal_id, time.time())
948
+ result = (await with_presence(p, [result]))[0]
949
+ result["agents"] = await with_presence(p, result["agents"])
950
+ return result
951
+
952
+ @app.put("/v1/principals/{principal_id}/avatar")
953
+ async def put_avatar(principal_id: int, request: Request):
954
+ p = await authn(request)
955
+ data = bytearray()
956
+ async for chunk in request.stream():
957
+ data += chunk
958
+ if len(data) > LIMITS.max_avatar_bytes:
959
+ raise ApiError(413, "too_large", f"pictures are limited to {LIMITS.max_avatar_bytes // 1024} KB")
960
+ return await db.write(store.set_avatar, p, principal_id, bytes(data), time.time())
961
+
962
+ @app.delete("/v1/principals/{principal_id}/avatar")
963
+ async def delete_avatar(principal_id: int, request: Request):
964
+ p = await authn(request)
965
+ return await db.write(store.delete_avatar, p, principal_id, time.time())
966
+
967
+ @app.put("/v1/principals/{principal_id}/pattern")
968
+ async def put_pattern(principal_id: int, request: Request):
969
+ p = await authn(request)
970
+ body = await _json_body(request, 1024)
971
+ seed = body.get("seed") if isinstance(body, dict) else None
972
+ return await db.write(store.set_pattern, p, principal_id, seed, time.time())
973
+
974
+ @app.get("/v1/principals/{principal_id}/avatar")
975
+ async def get_avatar(principal_id: int, request: Request):
976
+ await authn(request) # any member of the board may see pictures
977
+ ctype, data, digest = await db.read(store.get_avatar, principal_id)
978
+ return Response(data, media_type=ctype, headers={
979
+ "ETag": f'"{digest[:16]}"', "Cache-Control": "private, max-age=86400",
980
+ "X-Content-Type-Options": "nosniff", "Content-Security-Policy": "default-src 'none'"})
981
+
982
+ # ------------------------------------------------------ attachments
983
+
984
+ @app.put("/v1/channels/{channel_id}/attachments/{upload_id}")
985
+ async def upload(channel_id: int, upload_id: str, request: Request):
986
+ p = await authn(request)
987
+ declared = request.headers.get("content-length")
988
+ if declared and declared.isdigit() and int(declared) > LIMITS.max_upload_bytes:
989
+ raise ApiError(413, "too_large", f"uploads are limited to {LIMITS.max_upload_bytes} bytes")
990
+ if board.open_uploads >= LIMITS.max_concurrent_uploads:
991
+ raise ApiError(429, "too_many_uploads", "too many concurrent uploads", {"Retry-After": "1"})
992
+ # Browsers can only send ASCII headers, so the filename may be percent-encoded.
993
+ filename = os.path.basename(unquote(request.headers.get("x-filename", "")))[:255] or None
994
+ if filename and any(ord(ch) < 32 for ch in filename):
995
+ raise bad_request("invalid filename")
996
+ content_type = request.headers.get("content-type", "application/octet-stream")[:128]
997
+ board.open_uploads += 1
998
+ tmp_path = None
999
+ try:
1000
+ row = await db.write(store.begin_upload, p, channel_id, upload_id, filename, content_type, time.time())
1001
+ tmp_path, fh = await anyio.to_thread.run_sync(board.blobs.open_temp)
1002
+ hasher, size = new_hasher(), 0
1003
+ try:
1004
+ async for chunk in request.stream():
1005
+ size += len(chunk)
1006
+ if size > LIMITS.max_upload_bytes:
1007
+ raise ApiError(413, "too_large", f"uploads are limited to {LIMITS.max_upload_bytes} bytes")
1008
+ hasher.update(chunk)
1009
+ await anyio.to_thread.run_sync(fh.write, chunk)
1010
+ await anyio.to_thread.run_sync(board.blobs.finish_temp, fh)
1011
+ finally:
1012
+ if not fh.closed:
1013
+ fh.close()
1014
+ digest = hasher.hexdigest()
1015
+ expected = request.headers.get("x-checksum-sha256")
1016
+ if expected and expected.lower() != digest:
1017
+ raise bad_request("X-Checksum-Sha256 does not match the uploaded content")
1018
+ async with board.commit_lock:
1019
+ current = await db.read(store.attachment_state, row["id"])
1020
+ if current["state"] == "ready":
1021
+ result = await db.write(store.finish_upload, row["id"], size, digest)
1022
+ return JSONResponse(result, status_code=200)
1023
+ await anyio.to_thread.run_sync(board.blobs.commit, tmp_path, row["storage_key"])
1024
+ tmp_path = None
1025
+ result = await db.write(store.finish_upload, row["id"], size, digest)
1026
+ return JSONResponse(result, status_code=201)
1027
+ finally:
1028
+ board.open_uploads -= 1
1029
+ if tmp_path is not None:
1030
+ board.blobs.discard(tmp_path)
1031
+
1032
+ @app.get("/v1/attachments/{att_id}/info")
1033
+ async def attachment_info(att_id: str, request: Request):
1034
+ p = await authn(request)
1035
+ info = await db.read(store.get_attachment, p, att_id)
1036
+ info.pop("storage_key")
1037
+ return info
1038
+
1039
+ @app.get("/v1/attachments/{att_id}")
1040
+ async def download(att_id: str, request: Request):
1041
+ p = await authn(request)
1042
+ info = await db.read(store.get_attachment, p, att_id)
1043
+ return FileResponse(
1044
+ board.blobs.path_for(info["storage_key"]),
1045
+ media_type="application/octet-stream",
1046
+ filename=info["filename"] or att_id,
1047
+ headers={
1048
+ "X-Checksum-Sha256": info["sha256"],
1049
+ "X-Content-Type-Options": "nosniff",
1050
+ "X-Upload-Content-Type": info["content_type"] or "application/octet-stream",
1051
+ },
1052
+ )
1053
+
1054
+ # ------------------------------------------------------------------ storage folders
1055
+
1056
+ def _cache_put(src: Path, sha: str) -> Path:
1057
+ dest = board.storage_cache / sha
1058
+ os.replace(src, dest)
1059
+ files = sorted(board.storage_cache.iterdir(), key=lambda f: f.stat().st_atime)
1060
+ total = sum(f.stat().st_size for f in files)
1061
+ for f in files: # keep the cache under its size, oldest first
1062
+ if total <= LIMITS.storage_cache_bytes or f == dest:
1063
+ continue
1064
+ total -= f.stat().st_size
1065
+ f.unlink(missing_ok=True)
1066
+ return dest
1067
+
1068
+ @app.put("/v1/storage/{handle}/{path:path}")
1069
+ async def storage_put(handle: str, path: str, request: Request):
1070
+ p = await authn(request)
1071
+ owner = await db.read(store.storage_owner, p, handle, True)
1072
+ path = store.storage_path(path)
1073
+ declared = request.headers.get("content-length")
1074
+ if declared and declared.isdigit():
1075
+ await db.read(store.storage_check_quota, owner["id"], path, int(declared))
1076
+ content_type = (request.headers.get("content-type") or mimetypes.guess_type(path)[0]
1077
+ or "application/octet-stream")[:128]
1078
+ tmp_path, fh = await anyio.to_thread.run_sync(board.blobs.open_temp)
1079
+ try:
1080
+ hasher, size = new_hasher(), 0
1081
+ try:
1082
+ max_file = (await db.read(store.storage_limits))[0]
1083
+ async for chunk in request.stream():
1084
+ size += len(chunk)
1085
+ if size > max_file:
1086
+ raise ApiError(413, "too_large", f"files are limited to {store._mb(max_file)} on this board")
1087
+ hasher.update(chunk)
1088
+ await anyio.to_thread.run_sync(fh.write, chunk)
1089
+ await anyio.to_thread.run_sync(board.blobs.finish_temp, fh)
1090
+ finally:
1091
+ if not fh.closed:
1092
+ fh.close()
1093
+ digest = hasher.hexdigest()
1094
+ expected = request.headers.get("x-checksum-sha256")
1095
+ if expected and expected.lower() != digest:
1096
+ raise bad_request("X-Checksum-Sha256 does not match the uploaded content")
1097
+ task = asyncio.create_task(publish_storage(p, owner, path, size, digest, content_type, tmp_path))
1098
+ try:
1099
+ result = await asyncio.shield(task)
1100
+ except asyncio.CancelledError:
1101
+ with suppress(Exception):
1102
+ await task
1103
+ raise
1104
+ tmp_path = None
1105
+ return JSONResponse(result, status_code=201)
1106
+ finally:
1107
+ if tmp_path is not None:
1108
+ board.blobs.discard(tmp_path)
1109
+
1110
+ async def publish_storage(p, owner, path, size, digest, content_type, tmp_path):
1111
+ async with board.storage_reservations.hold([(owner["id"], path)]):
1112
+ await db.read(store.storage_check_quota, owner["id"], path, size)
1113
+ key = store.storage_key(owner["id"], path)
1114
+ # Replacing a file needs a rollback copy in case the final quota
1115
+ # check loses to an unrelated upload. This I/O holds only its path.
1116
+ try:
1117
+ await db.read(store.storage_lookup, p, owner["external_id"], path)
1118
+ replacing = True
1119
+ except ApiError as exc:
1120
+ if exc.status != 404:
1121
+ raise
1122
+ replacing = False
1123
+ with tempfile.TemporaryDirectory(prefix="msgboard-replace-") as scratch:
1124
+ backup = Path(scratch) / "old"
1125
+ backed_up = replacing and await anyio.to_thread.run_sync(board.storage.get, key, backup)
1126
+ changed = recorded = False
1127
+ try:
1128
+ changed = True
1129
+ await anyio.to_thread.run_sync(lambda: board.storage.put([(tmp_path, key)]))
1130
+ await anyio.to_thread.run_sync(_cache_put, tmp_path, digest)
1131
+ result = await db.write(store.storage_record_checked, p, owner["id"], path, size,
1132
+ digest, content_type, time.time())
1133
+ recorded = True
1134
+ return result
1135
+ finally:
1136
+ if changed and not recorded:
1137
+ if backed_up:
1138
+ await anyio.to_thread.run_sync(lambda: board.storage.put([(backup, key)]))
1139
+ else:
1140
+ await anyio.to_thread.run_sync(board.storage.delete, [key])
1141
+
1142
+ @app.get("/v1/storage/{handle}")
1143
+ async def storage_root(handle: str, request: Request, prefix: str = ""):
1144
+ p = await authn(request)
1145
+ return await db.read(store.storage_list, p, handle, prefix)
1146
+
1147
+ @app.get("/v1/storage/{handle}/{path:path}")
1148
+ async def storage_get(handle: str, path: str, request: Request, info: bool = False):
1149
+ p = await authn(request)
1150
+ if path.endswith("/"): # a folder: list it
1151
+ return await db.read(store.storage_list, p, handle, path)
1152
+ meta, key = await db.read(store.storage_lookup, p, handle, path)
1153
+ if info:
1154
+ return meta
1155
+ cached = board.storage_cache / meta["sha256"]
1156
+ if not cached.exists():
1157
+ owner = await db.read(store.storage_owner, p, handle, False)
1158
+ async with board.storage_reservations.hold([(owner["id"], store.storage_path(path))]):
1159
+ # A replacement may have completed while we waited. Read its
1160
+ # current metadata, never old metadata with newly uploaded bytes.
1161
+ meta, key = await db.read(store.storage_lookup, p, handle, path)
1162
+ cached = board.storage_cache / meta["sha256"]
1163
+ if not cached.exists():
1164
+ fd, name = tempfile.mkstemp(prefix=".storage-", suffix=".part", dir=board.storage_cache)
1165
+ os.close(fd)
1166
+ tmp = Path(name)
1167
+ try:
1168
+ ok = await anyio.to_thread.run_sync(board.storage.get, key, tmp)
1169
+ if not ok:
1170
+ raise ApiError(503, "storage_unavailable", "the file is listed but could not be fetched from storage; try again")
1171
+ # the cache is named by the recorded hash, so only bytes with that hash may go in it:
1172
+ # a storage object overwritten behind the database (e.g. a mismatched restore) is never served
1173
+ h = new_hasher()
1174
+ with tmp.open("rb") as fh:
1175
+ for chunk in iter(lambda: fh.read(1 << 20), b""):
1176
+ h.update(chunk)
1177
+ if h.hexdigest() != meta["sha256"]:
1178
+ raise ApiError(409, "storage_mismatch", "the stored bytes do not match this file's recorded checksum")
1179
+ os.replace(tmp, cached)
1180
+ finally:
1181
+ tmp.unlink(missing_ok=True)
1182
+ # always a download: never let stored HTML or SVG run on the board's own origin
1183
+ return FileResponse(cached, media_type="application/octet-stream", filename=os.path.basename(meta["path"]),
1184
+ headers={"X-Checksum-Sha256": meta["sha256"], "X-Content-Type-Options": "nosniff",
1185
+ "X-Upload-Content-Type": meta["content_type"] or "application/octet-stream",
1186
+ "Content-Security-Policy": "sandbox", "Cache-Control": "private, no-cache"})
1187
+
1188
+ @app.delete("/v1/storage/{handle}/{path:path}")
1189
+ async def storage_delete(handle: str, path: str, request: Request):
1190
+ p = await authn(request)
1191
+ owner = await db.read(store.storage_owner, p, handle, True)
1192
+ path = store.storage_path(path)
1193
+ async with board.storage_reservations.hold([(owner["id"], path)]):
1194
+ out = await db.write(store.storage_remove, p, handle, path, time.time())
1195
+ keys = out.pop("keys")
1196
+ await anyio.to_thread.run_sync(lambda: board.storage.delete(keys))
1197
+ return out
1198
+
1199
+ if dashboard:
1200
+ page = Path(__file__).with_name("dashboard.html")
1201
+
1202
+ @app.get("/", include_in_schema=False)
1203
+ async def dashboard_page():
1204
+ return FileResponse(page, media_type="text/html",
1205
+ headers={"Cache-Control": "no-cache", "X-Content-Type-Options": "nosniff"})
1206
+
1207
+ return app
msgboard/space/__init__.py ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ """msgboard on a Hugging Face Space.
2
+
3
+ - `deploy()` creates or updates a board Space and its private bucket (`hub.py`, needs the `hf` extra).
4
+ - `app.py` is what the Space runs; `accounts.py` adds invite links and "Sign in with Hugging Face".
5
+ - `Dockerfile` and `README.md` (the Space card) are uploaded with the package.
6
+ """
7
+
8
+ from .hub import Deployment, DeployError, deploy
9
+
10
+ __all__ = ["Deployment", "DeployError", "deploy"]
msgboard/space/accounts.py ADDED
@@ -0,0 +1,250 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Optional Hugging Face accounts for the msgboard Space.
2
+
3
+ People join with invite links: with a username they pick, or with "Join with
4
+ Hugging Face", which uses their HF username, name and picture and links the account.
5
+ Once linked (at join or later from their profile), "Sign in with Hugging Face"
6
+ gives them a token on any device. HF sign-in never creates an account without an invite.
7
+ """
8
+
9
+ import hashlib
10
+ import html
11
+ import json
12
+ import os
13
+ import secrets
14
+ import time
15
+ from urllib.parse import urlencode
16
+
17
+ import httpx
18
+ from fastapi import FastAPI, Request
19
+ from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
20
+
21
+ from msgboard import store
22
+ from msgboard.errors import ApiError
23
+ from msgboard.server import Board
24
+
25
+ _transport = None # tests inject a fake OAuth provider here
26
+
27
+
28
+ def _page(title: str, body: str, status: int = 200) -> HTMLResponse:
29
+ return HTMLResponse(
30
+ f"""<!doctype html><meta charset="utf-8"><meta name="viewport" content="width=device-width">
31
+ <title>{html.escape(title)}</title>
32
+ <body style="font:15px/1.6 -apple-system,Helvetica,Arial,sans-serif;max-width:440px;margin:18vh auto;padding:0 16px;
33
+ background:#fafaf8;color:#111"><div style="border-top:2px solid #111;padding-top:16px">
34
+ <div style="font:10.5px ui-monospace,Menlo,monospace;letter-spacing:.08em;text-transform:uppercase;color:#8a8a86">msgboard</div>
35
+ <h1 style="font-size:24px;margin:2px 0 10px">{html.escape(title)}</h1>{body}</div></body>""",
36
+ status_code=status, headers={"Cache-Control": "no-store"})
37
+
38
+
39
+ def import_shared_invite(board: Board, code: str, operator_token: str):
40
+ """Keep an invite code from the MSGBOARD_INVITE_CODE secret working as a normal member invite."""
41
+ def run(conn):
42
+ digest = hashlib.sha256(code.encode()).digest()
43
+ if conn.execute("SELECT 1 FROM invites WHERE code_hash = ?", (digest,)).fetchone():
44
+ return
45
+ op = store.authenticate(conn, operator_token, time.time())
46
+ conn.execute("""INSERT INTO invites (code_hash, created_by, created_at, kind, role, identity_type, label)
47
+ VALUES (?, ?, ?, 'human', 'member', 'user', 'shared link (MSGBOARD_INVITE_CODE)')""",
48
+ (digest, op.id, time.time()))
49
+ return board.db.write(run)
50
+
51
+
52
+ HF_API = os.environ.get("MSGBOARD_HF_API", "https://huggingface.co").rstrip("/")
53
+
54
+
55
+ def join_with_hf(conn, code, name, display_name, username: str, now: float) -> dict:
56
+ """Redeem a people invite and link the HF account in one go; nothing is kept if either fails."""
57
+ row = store._find_invite(conn, code)
58
+ if row is not None and row["kind"] != "human":
59
+ raise ApiError(400, "invalid_request", "only invites for people can be used with Hugging Face")
60
+ result = store.join(conn, code, name, display_name, now)
61
+ store.set_hf_username(conn, result["principal"]["id"], username, now)
62
+ result["principal"]["hf_username"] = username
63
+ return result
64
+
65
+
66
+ async def adopt_hf_picture(board: Board, principal_id: int, username: str) -> bool:
67
+ """Use the Hugging Face profile picture as this account's picture, unless it already has one
68
+ (an uploaded picture always wins). Best effort: any failure just keeps the generated pattern."""
69
+ has = await board.db.read(lambda conn: conn.execute(
70
+ "SELECT 1 FROM avatars WHERE principal_id = ?", (principal_id,)).fetchone() is not None)
71
+ if has:
72
+ return False
73
+ try:
74
+ async with httpx.AsyncClient(timeout=15, transport=_transport, follow_redirects=True) as http:
75
+ url = (await http.get(f"{HF_API}/api/users/{username}/overview")).json().get("avatarUrl")
76
+ if not url:
77
+ return False
78
+ if url.startswith("/"):
79
+ url = HF_API + url
80
+ data = (await http.get(url)).content
81
+ def run(conn):
82
+ if conn.execute("SELECT 1 FROM avatars WHERE principal_id = ?", (principal_id,)).fetchone():
83
+ return False # someone uploaded one meanwhile
84
+ me = store._principal_obj(store._principal_row(conn, principal_id))
85
+ store.set_avatar(conn, me, principal_id, data, time.time())
86
+ return True
87
+ return await board.db.write(run)
88
+ except Exception:
89
+ return False # SVG default avatars, too large, network errors
90
+
91
+
92
+ async def adopt_hf_pictures(board: Board) -> int:
93
+ """Once at startup: linked accounts without a picture get their Hugging Face one."""
94
+ rows = await board.db.read(lambda conn: conn.execute(
95
+ """SELECT p.id, p.hf_username FROM principals p LEFT JOIN avatars a ON a.principal_id = p.id
96
+ WHERE p.hf_username IS NOT NULL AND p.disabled = 0 AND a.principal_id IS NULL""").fetchall())
97
+ n = 0
98
+ for pid, username in rows:
99
+ n += await adopt_hf_picture(board, pid, username)
100
+ return n
101
+
102
+
103
+ def install(app: FastAPI, board: Board) -> None:
104
+ provider = os.environ.get("OPENID_PROVIDER_URL", "https://huggingface.co").rstrip("/")
105
+ client_id = os.environ.get("OAUTH_CLIENT_ID")
106
+ client_secret = os.environ.get("OAUTH_CLIENT_SECRET")
107
+ host = os.environ.get("SPACE_HOST")
108
+ enabled = bool(client_id and client_secret and host)
109
+ redirect_uri = f"https://{host}/login/callback"
110
+ pending: dict[str, tuple] = {} # state -> (created, mode, principal to link or invite code)
111
+ # ticket -> (created, invite code, HF username, HF name): an HF join whose username was taken here
112
+ tickets: dict[str, tuple[float, str, str, str | None]] = {}
113
+
114
+ def prune(entries: dict) -> None:
115
+ now = time.time()
116
+ for s, entry in list(entries.items()):
117
+ if now - entry[0] > 600:
118
+ del entries[s]
119
+ if len(entries) > 5000:
120
+ raise ApiError(429, "busy", "too many sign-ins in progress; try again shortly")
121
+
122
+ def start(mode: str, target: int | str | None) -> str:
123
+ prune(pending)
124
+ state = secrets.token_urlsafe(24)
125
+ pending[state] = (time.time(), mode, target)
126
+ return f"{provider}/oauth/authorize?" + urlencode({
127
+ "client_id": client_id, "redirect_uri": redirect_uri, "scope": "openid profile",
128
+ "state": state, "response_type": "code"})
129
+
130
+ @app.get("/auth/config")
131
+ async def auth_config():
132
+ return {"hf_login": enabled}
133
+
134
+ @app.get("/login")
135
+ async def login(invite: str | None = None):
136
+ if not enabled:
137
+ return _page("Hugging Face sign-in is not set up", "<p>Use an invite link or a token.</p>", 404)
138
+ return RedirectResponse(start("join", invite) if invite else start("signin", None))
139
+
140
+ @app.post("/v1/join/hf")
141
+ async def join_after_name_taken(request: Request):
142
+ """Finish an HF join with a name the person picked, because their HF username was taken here."""
143
+ body = await request.json()
144
+ if not isinstance(body, dict):
145
+ raise ApiError(400, "invalid_request", "expected a JSON object")
146
+ ticket = body.get("ticket") if isinstance(body.get("ticket"), str) else ""
147
+ entry = tickets.pop(ticket, None)
148
+ if entry is None or time.time() - entry[0] > 600:
149
+ raise ApiError(410, "hf_join_expired", "the Hugging Face sign-in expired; join with Hugging Face again")
150
+ _, code, username, hf_name = entry
151
+ try:
152
+ result = await board.db.write(join_with_hf, code, body.get("name"), body.get("display_name") or hf_name,
153
+ username, time.time())
154
+ except ApiError as exc:
155
+ if exc.status in (400, 409) and exc.code != "hf_linked_elsewhere":
156
+ tickets[ticket] = entry # let them try another name
157
+ raise
158
+ await adopt_hf_picture(board, result["principal"]["id"], username)
159
+ return JSONResponse(result, status_code=201, headers={"Cache-Control": "no-store"})
160
+
161
+ @app.post("/v1/me/hf-link")
162
+ async def link_start(request: Request):
163
+ p = await board.db.read(store.authenticate, _token(request), time.time())
164
+ if not enabled:
165
+ raise ApiError(404, "not_enabled", "Hugging Face sign-in is not set up on this board")
166
+ if not p.is_human:
167
+ raise ApiError(400, "invalid_request", "only people can link a Hugging Face account")
168
+ return {"url": start("link", p.id)}
169
+
170
+ @app.delete("/v1/me/hf-link")
171
+ async def unlink(request: Request):
172
+ p = await board.db.read(store.authenticate, _token(request), time.time())
173
+ await board.db.write(store.set_hf_username, p.id, None, time.time())
174
+ return {"hf_username": None}
175
+
176
+ async def hf_join(code: str, username: str, hf_name: str | None) -> dict | str:
177
+ """Sign in if this HF account is linked already (the invite stays unused), else join with it.
178
+ Returns the join result, or where to send the person to pick another name."""
179
+ try:
180
+ return await board.db.write(store.sign_in_with_hf, username, time.time())
181
+ except ApiError as exc:
182
+ if exc.status != 404:
183
+ raise
184
+ display = hf_name.strip()[:128] if isinstance(hf_name, str) and hf_name.strip() else None
185
+ try:
186
+ return await board.db.write(join_with_hf, code, username, display, username, time.time())
187
+ except ApiError as exc:
188
+ if exc.code != "name_taken":
189
+ raise
190
+ prune(tickets)
191
+ ticket = secrets.token_urlsafe(24)
192
+ tickets[ticket] = (time.time(), code, username, display)
193
+ return "/?" + urlencode({"invite": code, "hf": ticket, "hf_user": username})
194
+
195
+ @app.get("/login/callback")
196
+ async def callback(request: Request):
197
+ entry = pending.pop(request.query_params.get("state", ""), None)
198
+ if entry is None or time.time() - entry[0] > 600:
199
+ return _page("Sign-in expired", '<p>Please <a href="/">start again</a>.</p>', 400)
200
+ code = request.query_params.get("code")
201
+ if not code:
202
+ return _page("Sign-in cancelled", '<p><a href="/">Back to the board</a></p>', 400)
203
+ async with httpx.AsyncClient(timeout=20, transport=_transport) as http:
204
+ tok = await http.post(f"{provider}/oauth/token", auth=(client_id, client_secret), data={
205
+ "grant_type": "authorization_code", "code": code, "redirect_uri": redirect_uri,
206
+ "client_id": client_id})
207
+ if tok.status_code != 200:
208
+ return _page("Sign-in failed", "<p>Hugging Face did not accept the sign-in. Please try again.</p>", 502)
209
+ info = (await http.get(f"{provider}/oauth/userinfo",
210
+ headers={"Authorization": f"Bearer {tok.json()['access_token']}"})).json()
211
+ username = info.get("preferred_username")
212
+ if not username:
213
+ return _page("Sign-in failed", "<p>No username in the Hugging Face profile.</p>", 502)
214
+ _, mode, target = entry
215
+ try:
216
+ if mode == "link":
217
+ await board.db.write(store.set_hf_username, target, username, time.time())
218
+ await adopt_hf_picture(board, target, username)
219
+ return _page("Hugging Face account linked",
220
+ f"<p>You can now sign in as <b>@{html.escape(username)}</b> on any device.</p>"
221
+ '<p><a href="/">Back to the board</a></p>')
222
+ if mode == "join":
223
+ result = await hf_join(target, username, info.get("name"))
224
+ if isinstance(result, str):
225
+ return RedirectResponse(result, status_code=303)
226
+ else:
227
+ result = await board.db.write(store.sign_in_with_hf, username, time.time())
228
+ if result.get("principal", {}).get("id"):
229
+ await adopt_hf_picture(board, result["principal"]["id"], username)
230
+ except ApiError as exc:
231
+ if exc.status == 404 and mode != "join":
232
+ return _page("No account linked",
233
+ f"<p>No board account is linked to <b>@{html.escape(username)}</b>.</p>"
234
+ "<p>Join with an invite link first, then link your Hugging Face account from your "
235
+ "profile.</p>", 404)
236
+ return _page("Sign-in failed", f"<p>{html.escape(exc.message)}</p>", exc.status)
237
+ payload = json.dumps(result["token"]).replace("<", "\\u003c")
238
+ return HTMLResponse(f"""<!doctype html><meta charset="utf-8"><title>Signed in</title>
239
+ <script>
240
+ try {{ localStorage.setItem("msgboard:token", {payload}); }} catch (e) {{}}
241
+ location.replace("/");
242
+ </script><p>Signed in. <a href="/">Open the board</a>.</p>""", headers={"Cache-Control": "no-store"})
243
+
244
+
245
+ def _token(request: Request) -> str | None:
246
+ direct = request.headers.get("x-msgboard-token")
247
+ if direct:
248
+ return direct.strip()
249
+ scheme, _, value = request.headers.get("authorization", "").partition(" ")
250
+ return value.strip() if scheme.lower() == "bearer" else None
msgboard/space/app.py ADDED
@@ -0,0 +1,190 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """msgboard on a Hugging Face Space: local SQLite, snapshots to an HF bucket, dashboard at /.
2
+
3
+ Environment (Space secrets/variables):
4
+ MSGBOARD_OPERATOR_TOKEN secret; the operator (admin) board token
5
+ HF_TOKEN secret; write access to the replica bucket
6
+ MSGBOARD_REPLICA hf://buckets/<ns>/<bucket>[/<prefix>]
7
+ MSGBOARD_SNAPSHOT_INTERVAL seconds between snapshots (default 15)
8
+ MSGBOARD_INVITE_CODE optional secret; kept working as a standing member invite link
9
+ MSGBOARD_DEMO_BOTS "1" to run the demo's bots (hfspace/demo_bots.py, uploaded by hfspace/deploy.py)
10
+
11
+ The Space runs `python -m msgboard.space.app`; msgboard.space.deploy() uploads this package to it.
12
+
13
+ Restarts overlap on Spaces: the new container starts before the old one gets
14
+ SIGTERM. So this app opens its port at once but answers 503 (Retry-After) until
15
+ the old container has pushed its final snapshot and released the lease. Only
16
+ then does it restore and serve. Clients retry with the same request IDs.
17
+ """
18
+
19
+ import asyncio
20
+ import hashlib
21
+ import json
22
+ import logging
23
+ import os
24
+ import signal
25
+ import threading
26
+ import time
27
+ from pathlib import Path
28
+
29
+ import uvicorn
30
+
31
+ from msgboard import auth, db, store
32
+ from msgboard.replicate import Replicator, restore, store_from_url, wait_for_release
33
+ from msgboard.server import Board, create_app
34
+
35
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(levelname)s %(message)s")
36
+ logging.getLogger("httpx").setLevel(logging.WARNING)
37
+ logging.getLogger("httpx2").setLevel(logging.WARNING)
38
+ log = logging.getLogger("msgboard.space")
39
+
40
+ DATA_DIR = Path(os.environ.get("MSGBOARD_DATA_DIR", "/home/user/board-data"))
41
+ PAGE = Path(__file__).resolve().parents[1] / "dashboard.html"
42
+
43
+
44
+ def ensure_board(data_dir: Path, operator_token: str) -> None:
45
+ """Create the schema if needed and make the operator credential equal the Space secret."""
46
+ parsed = auth.parse_token(operator_token)
47
+ if parsed is None:
48
+ raise SystemExit("MSGBOARD_OPERATOR_TOKEN is not a valid board token")
49
+ data_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
50
+ conn = db.connect(data_dir / "board.db")
51
+ try:
52
+ db.migrate(conn, data_dir / "board.db")
53
+ conn.execute("BEGIN IMMEDIATE")
54
+ now = time.time()
55
+ if not store.admin_exists(conn):
56
+ store.create_operator(conn, now)
57
+ pid = conn.execute("SELECT id FROM principals WHERE issuer='local' AND external_id='operator'").fetchone()[0]
58
+ cred_id, secret = parsed
59
+ current = conn.execute("SELECT id FROM credentials WHERE principal_id=? AND revoked_at IS NULL",
60
+ (pid,)).fetchone()
61
+ if current is None or current[0] != cred_id:
62
+ conn.execute("UPDATE credentials SET revoked_at=? WHERE principal_id=? AND revoked_at IS NULL", (now, pid))
63
+ conn.execute("DELETE FROM credentials WHERE id=?", (cred_id,))
64
+ conn.execute("INSERT INTO credentials (id, principal_id, secret_hash, created_at) VALUES (?, ?, ?, ?)",
65
+ (cred_id, pid, hashlib.sha256(secret).digest(), now))
66
+ conn.execute("COMMIT")
67
+ finally:
68
+ conn.close()
69
+
70
+
71
+ class Dispatcher:
72
+ """ASGI app that answers 'starting' until the board is ready, then forwards to it."""
73
+
74
+ def __init__(self):
75
+ self.inner = None
76
+ self.board: Board | None = None
77
+ self.error: str | None = None
78
+ self.stop = threading.Event()
79
+ self._inner_lifespan = None
80
+
81
+ def _prepare(self) -> Board:
82
+ operator_token = os.environ["MSGBOARD_OPERATOR_TOKEN"].strip()
83
+ replica_url = os.environ.get("MSGBOARD_REPLICA")
84
+ replicator = None
85
+ if replica_url:
86
+ replica = store_from_url(replica_url)
87
+ why = wait_for_release(replica, stop=self.stop,
88
+ stale_after=float(os.environ.get("MSGBOARD_LEASE_STALE_AFTER", "120")))
89
+ log.info("replica handover: %s", why)
90
+ if why == "stopped":
91
+ raise RuntimeError("stopped before the replica was handed over")
92
+ if not (DATA_DIR / "board.db").exists():
93
+ manifest = restore(replica, DATA_DIR)
94
+ log.info("restored %s" % manifest["snapshot"] if manifest else "replica empty; starting a new board")
95
+ replicator = Replicator(replica, DATA_DIR,
96
+ interval=float(os.environ.get("MSGBOARD_SNAPSHOT_INTERVAL", "15")))
97
+ else:
98
+ log.warning("MSGBOARD_REPLICA not set: all data is lost when this Space restarts")
99
+ ensure_board(DATA_DIR, operator_token)
100
+ return Board(DATA_DIR, replicator=replicator)
101
+
102
+ async def _start(self) -> None:
103
+ try:
104
+ board = await asyncio.to_thread(self._prepare)
105
+ app = create_app(board)
106
+ from msgboard.space.accounts import adopt_hf_pictures, import_shared_invite, install as install_accounts
107
+
108
+ install_accounts(app, board)
109
+ self._inner_lifespan = app.router.lifespan_context(app)
110
+ await self._inner_lifespan.__aenter__()
111
+ self.board, self.inner = board, app
112
+ shared = os.environ.get("MSGBOARD_INVITE_CODE", "").strip()
113
+ if shared:
114
+ await import_shared_invite(board, shared, os.environ["MSGBOARD_OPERATOR_TOKEN"].strip())
115
+ log.info("board ready")
116
+ # linked accounts without a picture get their Hugging Face one (in the background, best effort)
117
+ self._pictures = asyncio.get_running_loop().create_task(adopt_hf_pictures(board))
118
+ if os.environ.get("MSGBOARD_DEMO_BOTS") == "1":
119
+ try: # not part of the package: only the demo Space has it, next to the package
120
+ from demo_bots import start_bots
121
+ except ImportError:
122
+ log.warning("MSGBOARD_DEMO_BOTS=1 but demo_bots.py is not in the Space; no bots")
123
+ else:
124
+ start_bots(f"http://127.0.0.1:{os.environ.get('PORT', '7860')}",
125
+ os.environ["MSGBOARD_OPERATOR_TOKEN"].strip())
126
+ except Exception as exc:
127
+ self.error = f"{type(exc).__name__}: {exc}"
128
+ log.exception("board failed to start")
129
+
130
+ def begin_shutdown(self) -> None:
131
+ self.stop.set()
132
+ if self.board is not None:
133
+ self.board.begin_shutdown()
134
+
135
+ async def __call__(self, scope, receive, send):
136
+ if scope["type"] == "lifespan":
137
+ while True:
138
+ message = await receive()
139
+ if message["type"] == "lifespan.startup":
140
+ self._task = asyncio.create_task(self._start())
141
+ await send({"type": "lifespan.startup.complete"})
142
+ elif message["type"] == "lifespan.shutdown":
143
+ if self._inner_lifespan is not None:
144
+ await self._inner_lifespan.__aexit__(None, None, None)
145
+ await send({"type": "lifespan.shutdown.complete"})
146
+ return
147
+ if self.inner is not None:
148
+ return await self.inner(scope, receive, send)
149
+ if scope["type"] != "http":
150
+ return
151
+ path = scope.get("path", "")
152
+ if path == "/":
153
+ body, status, ctype, extra = PAGE.read_bytes(), 200, b"text/html", []
154
+ elif path == "/v1/health":
155
+ body, status, ctype, extra = json.dumps({"status": "starting", "error": self.error}).encode(), 200, \
156
+ b"application/json", []
157
+ else:
158
+ body = json.dumps({"error": {"code": "starting",
159
+ "message": self.error or "board is starting; retry shortly"}}).encode()
160
+ status, ctype, extra = 503, b"application/json", [(b"retry-after", b"2")]
161
+ await send({"type": "http.response.start", "status": status,
162
+ "headers": [(b"content-type", ctype), (b"cache-control", b"no-cache")] + extra})
163
+ await send({"type": "http.response.body", "body": body})
164
+
165
+
166
+ def main() -> None:
167
+ dispatcher = Dispatcher()
168
+
169
+ class SpaceServer(uvicorn.Server):
170
+ def handle_exit(self, sig, frame):
171
+ dispatcher.begin_shutdown()
172
+ super().handle_exit(sig, frame)
173
+
174
+ def _exit(signum, frame): # see msgboard.cli.run_board: keep cleanup from being skipped
175
+ raise SystemExit(0)
176
+
177
+ signal.signal(signal.SIGTERM, _exit)
178
+ signal.signal(signal.SIGINT, _exit)
179
+ config = uvicorn.Config(dispatcher, host="0.0.0.0", port=int(os.environ.get("PORT", "7860")),
180
+ log_level="warning", timeout_graceful_shutdown=5, server_header=False)
181
+ try:
182
+ SpaceServer(config).run()
183
+ finally:
184
+ if dispatcher.board is not None:
185
+ dispatcher.board.close() # final snapshot, then release the lease
186
+ log.info("stopped")
187
+
188
+
189
+ if __name__ == "__main__":
190
+ main()
msgboard/space/hub.py ADDED
@@ -0,0 +1,264 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Deploy a board to a Hugging Face Space: `msgboard.space.deploy()`. Needs the `hf` extra.
2
+
3
+ The Space runs `msgboard.space.app` from the files of the msgboard install that calls deploy(),
4
+ keeps its board in SQLite on its own disk and pushes snapshots to a private bucket through the
5
+ Hub API (DEPLOY_SPACES.md explains why it is not a bucket mount).
6
+ """
7
+
8
+ import json
9
+ import logging
10
+ import re
11
+ from dataclasses import dataclass, field
12
+ from importlib import metadata
13
+ from pathlib import Path
14
+
15
+ from .. import auth
16
+
17
+ log = logging.getLogger("msgboard.space")
18
+
19
+ TAG = "msgboard" # deploy() puts it in the Space card, and reuses only Spaces that have it
20
+ SNAPSHOT_INTERVAL = "15" # MSGBOARD_SNAPSHOT_INTERVAL, in seconds
21
+ _HERE = Path(__file__).resolve().parent
22
+ _PACKAGE = _HERE.parent # the installed msgboard package, uploaded as-is
23
+
24
+
25
+ class DeployError(Exception):
26
+ """deploy() refused, before changing anything on the Hub."""
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class Deployment:
31
+ """What deploy() set up."""
32
+
33
+ space_id: str # "namespace/name"; a bare name gets the logged-in account
34
+ url: str # the board's base URL, https://<namespace>-<name>.hf.space
35
+ bucket_id: str # "namespace/name" of the private bucket that holds the snapshots
36
+ created: bool # True if this call created the Space, False if it reused one
37
+ private: bool # the Space's visibility; a reused Space keeps its own
38
+ commit: str # the Space repo commit that holds this deployment
39
+ # The admin token deploy() made when called without operator_token (None if the caller passed one).
40
+ # It's left out of repr() so printing or logging a Deployment never shows it.
41
+ operator_token: str | None = field(default=None, repr=False)
42
+
43
+
44
+ def _resolve_space_id(space_id: str, *, token: str | None = None) -> str:
45
+ """Return `space_id` as "namespace/name". A bare name goes in the logged-in account, as in Trackio."""
46
+ return _Names(_hf_api(_hf_token(token))).full(space_id)
47
+
48
+
49
+ def deploy(space_id: str, *, token: str | None = None, operator_token: str | None = None,
50
+ bucket_id: str | None = None, private: bool | None = None, title: str | None = None) -> Deployment:
51
+ """Create or update the board Space `space_id` and return a `Deployment`.
52
+
53
+ `space_id` is "namespace/name" or a bare name for the logged-in account. A missing Space is
54
+ created (Docker SDK, CPU Basic) with `private` as its visibility (None: the namespace's default).
55
+ An existing Space is reused, and keeps its visibility, only if its card has the `msgboard` tag
56
+ that deploy() puts there. Any other Space raises DeployError without being changed, including one
57
+ left by a first deploy() that failed before its upload: delete that one and deploy again.
58
+
59
+ A reused board keeps its bucket, the one its MSGBOARD_REPLICA names. A new board's is `bucket_id`,
60
+ else "<space_id>-bucket", or -2, -3 ... if that exists, so it never restores the snapshots a
61
+ deleted Space left behind. The bucket is always private, whatever `private` says: it holds every
62
+ message, token hash and invite. An existing public bucket is refused. So is anything that would
63
+ restart a board against an empty bucket: a `bucket_id` other than the board's, a MSGBOARD_REPLICA
64
+ that isn't the hf://buckets/<namespace>/<bucket> deploy() set, or a bucket that is missing
65
+ (deleted or not visible to this token). All checks run before the first write.
66
+
67
+ The Space gets the secrets MSGBOARD_OPERATOR_TOKEN (`operator_token`, from
68
+ `msgboard.auth.new_token()`; the app makes it the admin credential on every boot) and HF_TOKEN
69
+ (`token`, default: the logged-in token), and the variables MSGBOARD_REPLICA=hf://buckets/<bucket>
70
+ and MSGBOARD_SNAPSHOT_INTERVAL=15. That one HF token makes deploy()'s Hub calls and is what the
71
+ Space writes the bucket with, so it needs write access to both the Space and its bucket.
72
+
73
+ Without `operator_token`, deploy() makes one with new_token() and returns it as
74
+ `Deployment.operator_token` (None when you pass one). On a reused board that rotates the admin
75
+ token: the new one replaces the secret, and the old one stops working once the Space restarts.
76
+ So omit it only if you'll keep the returned one; a deploy() that raises returns no token, and the
77
+ next call makes another.
78
+
79
+ The Space's files become the installed msgboard package, LICENSE, requirements.txt, the Dockerfile
80
+ and the card, whose title is `title` (default: the Space's name); anything else in the Space repo
81
+ is deleted.
82
+
83
+ The Space then builds or restarts, which takes a few minutes: wait with `Client.wait_ready()`.
84
+ A reused Space keeps answering with its previous container for a while; wait_ready() with the
85
+ operator token waits until the board accepts that token. A restart or rebuild can still follow
86
+ after wait_ready() returns: if `operator_token` is the one the Space already has, the old healthy
87
+ container satisfies wait_ready() before the pending restart. Keep retrying through 503 after it.
88
+ """
89
+ from huggingface_hub import CommitOperationAdd, CommitOperationDelete
90
+ from huggingface_hub.errors import RepositoryNotFoundError
91
+
92
+ generated = operator_token is None
93
+ if generated:
94
+ operator_token = auth.new_token()[1]
95
+ elif auth.parse_token(operator_token) is None:
96
+ raise DeployError("operator_token is not a board token; make one with msgboard.auth.new_token()")
97
+ token = _hf_token(token)
98
+ api = _hf_api(token)
99
+ names = _Names(api)
100
+ space_id = names.full(space_id)
101
+ bucket_id = names.full(bucket_id) if bucket_id else None
102
+ try:
103
+ info = api.space_info(space_id)
104
+ except RepositoryNotFoundError:
105
+ info = None
106
+ variables: dict[str, str] = {}
107
+ if info is not None:
108
+ if TAG not in (info.tags or []):
109
+ raise DeployError(f"{space_id} exists but is not a msgboard board (no {TAG!r} tag in its card), so it "
110
+ "was left alone; choose another Space ID")
111
+ # A restart boots a reused board from its bucket, so never let it come back against an empty one.
112
+ variables = {k: v.value for k, v in api.get_space_variables(space_id).items()}
113
+ replica = variables.get("MSGBOARD_REPLICA")
114
+ current = _bucket_of(replica)
115
+ if current is None:
116
+ raise DeployError(f"{space_id} has MSGBOARD_REPLICA={replica!r}, not hf://buckets/<namespace>/<bucket> "
117
+ "as deploy() sets it; set it back to the bucket that holds the board's snapshots")
118
+ if bucket_id and bucket_id != current:
119
+ raise DeployError(f"{space_id} keeps its board in bucket {current}, not {bucket_id}; "
120
+ "leave bucket_id out to keep using it")
121
+ bucket_id = current
122
+ bucket_id = bucket_id or _free_bucket_id(api, f"{space_id}-bucket")
123
+ bucket = _bucket_info(api, bucket_id)
124
+ if bucket is None and info is not None:
125
+ raise DeployError(f"{space_id} is a deployed board, and its bucket {bucket_id} was not found (deleted, "
126
+ "or not visible to this token); refusing to restart the board against an empty bucket")
127
+ if bucket is not None and not bucket.private:
128
+ raise DeployError(f"bucket {bucket_id} is public; a board's bucket must be private, because its "
129
+ "snapshots hold every message, token hash and invite")
130
+ dist = _distribution()
131
+ files = _space_files(space_id, dist, title)
132
+
133
+ secrets = {"MSGBOARD_OPERATOR_TOKEN": operator_token, "HF_TOKEN": token}
134
+ wanted = {"MSGBOARD_REPLICA": f"hf://buckets/{bucket_id}", "MSGBOARD_SNAPSHOT_INTERVAL": SNAPSHOT_INTERVAL}
135
+ created = info is None
136
+ if created: # secrets and variables come with the Space; nothing runs before the first commit
137
+ api.create_repo(space_id, repo_type="space", private=private, space_sdk="docker",
138
+ space_hardware="cpu-basic",
139
+ space_secrets=[{"key": k, "value": v} for k, v in secrets.items()],
140
+ space_variables=[{"key": k, "value": v} for k, v in wanted.items()])
141
+ if bucket is None:
142
+ api.create_bucket(bucket_id, private=True, exist_ok=True)
143
+ if not created: # each of these restarts the board
144
+ for key, value in secrets.items():
145
+ api.add_space_secret(space_id, key, value)
146
+ for key, value in wanted.items():
147
+ if variables.get(key) != value:
148
+ api.add_space_variable(space_id, key, value)
149
+ stale = set(api.list_repo_files(space_id, repo_type="space")) - set(files) - {".gitattributes"}
150
+ operations = [CommitOperationAdd(path, data) for path, data in files.items()]
151
+ operations += [CommitOperationDelete(path) for path in sorted(stale)]
152
+ commit = api.create_commit(space_id, repo_type="space", operations=operations,
153
+ commit_message=f"Deploy msgboard {dist.version}")
154
+ info = api.space_info(space_id)
155
+ return Deployment(space_id=space_id, url=(info.host or f"https://{info.subdomain}.hf.space").rstrip("/"),
156
+ bucket_id=bucket_id, created=created, private=bool(info.private), commit=commit.oid,
157
+ operator_token=operator_token if generated else None)
158
+
159
+
160
+ def _hf_token(token: str | None) -> str:
161
+ from huggingface_hub import get_token
162
+
163
+ token = token or get_token()
164
+ if not token:
165
+ raise DeployError("no Hugging Face token: log in with `hf auth login` or set HF_TOKEN")
166
+ return token
167
+
168
+
169
+ def _hf_api(token: str):
170
+ from huggingface_hub import HfApi
171
+
172
+ return HfApi(token=token)
173
+
174
+
175
+ class _Names:
176
+ """Fills in the logged-in account for bare repo names, asking the Hub at most once."""
177
+
178
+ def __init__(self, api):
179
+ self.api = api
180
+ self._namespace = None
181
+
182
+ def full(self, repo_id: str) -> str:
183
+ from huggingface_hub.utils import validate_repo_id
184
+
185
+ if "/" not in repo_id:
186
+ self._namespace = self._namespace or self.api.whoami()["name"]
187
+ repo_id = f"{self._namespace}/{repo_id}"
188
+ validate_repo_id(repo_id)
189
+ return repo_id
190
+
191
+
192
+ def _bucket_of(replica: str | None) -> str | None:
193
+ """The bucket ID in the MSGBOARD_REPLICA deploy() sets, `hf://buckets/<namespace>/<bucket>`."""
194
+ match = re.fullmatch(r"hf://buckets/([^/]+/[^/]+)", replica or "")
195
+ return match[1] if match else None
196
+
197
+
198
+ def _bucket_info(api, bucket_id: str):
199
+ from huggingface_hub.errors import BucketNotFoundError
200
+
201
+ try:
202
+ return api.bucket_info(bucket_id)
203
+ except BucketNotFoundError:
204
+ return None
205
+
206
+
207
+ def _free_bucket_id(api, preferred: str) -> str:
208
+ """`preferred`, or the first of preferred-2, -3 ... that doesn't exist, as Trackio's resolve_auto_bucket_id."""
209
+ candidate, n = preferred, 1
210
+ while _bucket_info(api, candidate) is not None:
211
+ n += 1
212
+ candidate = f"{preferred}-{n}"
213
+ if candidate != preferred:
214
+ log.info("bucket %s already exists; using %s", preferred, candidate)
215
+ return candidate
216
+
217
+
218
+ def _distribution():
219
+ try:
220
+ return metadata.distribution("msgboard")
221
+ except metadata.PackageNotFoundError:
222
+ raise DeployError("msgboard is not installed; deploy() uploads the installed package "
223
+ "(pip install 'msgboard[hf]', or pip install -e '.[hf]' in a checkout)") from None
224
+
225
+
226
+ def _space_files(space_id: str, dist, title: str | None = None) -> dict[str, bytes]:
227
+ """The Space repo's files by path: card, Dockerfile, requirements.txt, LICENSE and the package."""
228
+ # A JSON string is a quoted YAML string, so `true`, `null`, `2024`, ':' or '#' stay the title as written.
229
+ # ensure_ascii=False keeps an emoji whole (YAML reads JSON's surrogate pair as two lone halves). YAML
230
+ # reads U+0085/U+2028/U+2029 as line breaks and refuses DEL and C1 controls, so those stay escaped.
231
+ title = re.sub(r"[\x7f-\x9f\u2028\u2029]", lambda m: f"\\u{ord(m[0]):04x}",
232
+ json.dumps(title or space_id.split("/")[1], ensure_ascii=False))
233
+ card = (_HERE / "README.md").read_text().replace("title: msgboard\n", f"title: {title}\n", 1)
234
+ files = {"README.md": card.encode(), "Dockerfile": (_HERE / "Dockerfile").read_bytes(),
235
+ "requirements.txt": _requirements(dist).encode(), "LICENSE": _license(dist)}
236
+ # An allowlist, so stray files in an editable install's src/msgboard (a test board's token, say)
237
+ # never reach a public Space. Keep it in step with [tool.setuptools.package-data] in pyproject.toml.
238
+ for path in sorted(_PACKAGE.rglob("*")):
239
+ if (path.suffix in (".py", ".html") and "__pycache__" not in path.parts and path.is_file()
240
+ and path.resolve().is_relative_to(_PACKAGE)): # no symlinks out of the package
241
+ files["msgboard/" + path.relative_to(_PACKAGE).as_posix()] = path.read_bytes()
242
+ return files
243
+
244
+
245
+ def _requirements(dist) -> str:
246
+ """The install's own dependencies plus the hf extra (bucket snapshots); other extras are left out."""
247
+ lines = []
248
+ for req in dist.requires or []:
249
+ spec, _, marker = req.partition(";")
250
+ marker = marker.strip()
251
+ if not marker or re.fullmatch(r"""extra\s*==\s*["']hf["']""", marker):
252
+ lines.append(spec.strip())
253
+ elif "extra" not in marker:
254
+ lines.append(req.strip())
255
+ return "\n".join(lines) + "\n"
256
+
257
+
258
+ def _license(dist) -> bytes:
259
+ """LICENSE from the install's dist-info, in licenses/ as setuptools 77+ (PEP 639) puts it."""
260
+ for name in dist.metadata.get_all("License-File") or []:
261
+ text = dist.read_text(f"licenses/{name}")
262
+ if text:
263
+ return text.encode()
264
+ raise DeployError("the installed msgboard has no LICENSE in its metadata; reinstall it with setuptools>=77")
msgboard/store.py ADDED
The diff for this file is too large to render. See raw diff
 
msgboard/transfer.py ADDED
@@ -0,0 +1,647 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Admin-only, one-channel history transfer. ZIP members are read, never extracted.
2
+
3
+ The database transaction owns the import decision and publication. Immutable blob
4
+ files are installed before their rows commit; a failed transaction may leave an
5
+ unreferenced object, but never publishes a row whose bytes are incomplete.
6
+ """
7
+
8
+ import hashlib
9
+ import json
10
+ import math
11
+ import re
12
+ import shutil
13
+ import sqlite3
14
+ import tempfile
15
+ import zipfile
16
+ from itertools import pairwise
17
+ from pathlib import Path
18
+ from urllib.parse import quote, unquote, urlsplit, urlunsplit
19
+
20
+ from . import store
21
+ from .config import LIMITS
22
+ from .errors import ApiError, bad_request, conflict, forbidden
23
+
24
+ MAX_BUNDLE_BYTES = 512 * 1024 * 1024
25
+ MAX_MANIFEST_BYTES = 32 * 1024 * 1024
26
+ MAX_MESSAGES = 10_000
27
+ MAX_ATTACHMENTS = 1_000
28
+ HASH = re.compile(r"^[0-9a-f]{64}$")
29
+ ATT_LINK = re.compile(r"(?:/v1/attachments/|#file/)(att_[A-Za-z0-9_-]+)")
30
+ STORAGE_LINK = re.compile(r"(?:https?://[^\s<>\"'\[\]]+)?/v1/storage/[^/\s<>\"'()\[\]]+/[^\s<>\"'\[\]]+")
31
+ CODE_SPAN = re.compile(r"(`+)([\s\S]*?)\1|(?m:^ {0,3}(~{3,})[^\n]*\n[\s\S]*?^ {0,3}\3[ \t]*$)")
32
+
33
+
34
+ def storage_links(body):
35
+ # Preserve offsets so the same matches can drive capture and rewriting.
36
+ # Paths shown as code are examples, not rendered file links.
37
+ masked = CODE_SPAN.sub(lambda match: " " * len(match.group()), body)
38
+ return STORAGE_LINK.finditer(masked)
39
+
40
+
41
+ def storage_reference(text, source_url):
42
+ """Resolve only this board's file links; never fetch arbitrary URLs."""
43
+ suffix = ""
44
+ # Keep balanced filename parentheses, but leave Markdown's closing delimiter
45
+ # outside the URL when a rewritten link is put back into the body.
46
+ depth = 0
47
+ for i, char in enumerate(text):
48
+ if char == "(":
49
+ depth += 1
50
+ elif char == ")":
51
+ if not depth:
52
+ text, suffix = text[:i], text[i:]
53
+ break
54
+ depth -= 1
55
+ url, source = urlsplit(text), urlsplit(source_url)
56
+ prefix = "/v1/storage/"
57
+ path = url.path
58
+ if url.netloc:
59
+ if (url.scheme, url.netloc) != (source.scheme, source.netloc):
60
+ return None
61
+ prefix = source.path.rstrip("/") + prefix
62
+ if not path.startswith(prefix):
63
+ return None
64
+ handle, _, rest = path[len(prefix):].partition("/")
65
+ if not handle or not rest or rest.endswith("/"):
66
+ return None
67
+ return unquote(handle), store.storage_path(unquote(rest)), url, suffix
68
+
69
+
70
+ def require_admin(p):
71
+ if not p.is_admin:
72
+ raise forbidden("channel export/import requires an admin")
73
+
74
+
75
+ def public_channel(conn, p, channel_id):
76
+ require_admin(p)
77
+ ch = store.channel_for(conn, p, channel_id)
78
+ if ch["kind"] != "public":
79
+ raise bad_request("only public channels can be exported or imported")
80
+ return ch
81
+
82
+
83
+ def canonical(value):
84
+ return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
85
+
86
+
87
+ def export_manifest(conn, p, channel_id: int, board_url: str):
88
+ # Reader connections otherwise run in autocommit mode: use one snapshot for
89
+ # authors, messages, reactions, doc and attachment metadata.
90
+ conn.execute("BEGIN")
91
+ try:
92
+ ch = public_channel(conn, p, channel_id)
93
+ rows = conn.execute("SELECT * FROM messages WHERE channel_id = ? ORDER BY created_at, seq, id",
94
+ (channel_id,)).fetchall()
95
+ if len(rows) > MAX_MESSAGES:
96
+ raise ApiError(413, "too_large", f"a channel bundle supports at most {MAX_MESSAGES} messages")
97
+ messages = [{k: r[k] for k in ("id", "seq", "sender_id", "body", "reply_to", "thread_id",
98
+ "created_at", "edited_at")} | {"attachments": json.loads(r["attachments"] or "[]")}
99
+ for r in rows]
100
+ reactions = [dict(r) for r in conn.execute("""SELECT r.* FROM message_reactions r
101
+ JOIN messages m ON m.id = r.message_id WHERE m.channel_id = ? ORDER BY r.created_at, r.rowid""", (channel_id,))]
102
+ doc_row = conn.execute("SELECT body, author_id, created_at FROM channel_doc_versions WHERE channel_id = ? AND version = ?",
103
+ (channel_id, ch["doc_version"])).fetchone()
104
+ doc = dict(doc_row) if doc_row else None
105
+ attachment_ids = {a for m in messages for a in m["attachments"]}
106
+ for body in [m["body"] for m in messages] + ([doc["body"]] if doc else []):
107
+ # Only this channel's attachments are copied. Links to other channels
108
+ # retain their original meaning/access rules, like storage links.
109
+ for aid in ATT_LINK.findall(body):
110
+ if conn.execute("SELECT 1 FROM attachments WHERE id = ? AND channel_id = ? AND state = 'ready'",
111
+ (aid, channel_id)).fetchone():
112
+ attachment_ids.add(aid)
113
+ if len(attachment_ids) > MAX_ATTACHMENTS:
114
+ raise ApiError(413, "too_large", "too many attachments for one bundle")
115
+ attachments = []
116
+ for aid in sorted(attachment_ids):
117
+ row = conn.execute("SELECT * FROM attachments WHERE id = ? AND channel_id = ? AND state = 'ready'",
118
+ (aid, channel_id)).fetchone()
119
+ if not row:
120
+ raise conflict("export_changed", "an attachment is missing; export again after checking the channel")
121
+ attachments.append(dict(row))
122
+ storage_files = {}
123
+ for body in [m["body"] for m in messages] + ([doc["body"]] if doc else []):
124
+ for match in storage_links(body):
125
+ ref = storage_reference(match.group(), board_url)
126
+ if ref is None:
127
+ continue
128
+ handle, path, _, _ = ref
129
+ owner = conn.execute("SELECT id FROM principals WHERE external_id = ? OR hf_username = ?", (handle, handle)).fetchone()
130
+ row = conn.execute("SELECT * FROM storage_files WHERE owner_id = ? AND path = ?",
131
+ (owner["id"], path)).fetchone() if owner else None
132
+ if row is None:
133
+ raise conflict("export_missing_storage", f"linked storage file is missing: {handle}/{path}")
134
+ record = storage_files.setdefault((row["owner_id"], path),
135
+ {k: row[k] for k in ("owner_id", "path", "size", "sha256", "content_type", "updated_at")} | {"handles": []})
136
+ if handle not in record["handles"]:
137
+ record["handles"].append(handle)
138
+ if len(storage_files) + len(attachments) > MAX_ATTACHMENTS:
139
+ raise ApiError(413, "too_large", "too many attachment/storage files for one bundle")
140
+ people = {m["sender_id"] for m in messages} | {r["principal_id"] for r in reactions}
141
+ people |= {a["uploader_id"] for a in attachments}
142
+ people |= {f["owner_id"] for f in storage_files.values()}
143
+ if doc:
144
+ people.add(doc["author_id"])
145
+ principals = [dict(conn.execute("SELECT id, external_id, display_name, identity_type FROM principals WHERE id = ?",
146
+ (pid,)).fetchone()) for pid in sorted(people)]
147
+ manifest = {"format": "msgboard-channel", "version": 1,
148
+ "source": {"board_id": conn.execute("SELECT value FROM settings WHERE key = 'transfer_board_id'").fetchone()[0],
149
+ "board_url": board_url.rstrip("/"), "channel_id": channel_id, "channel_name": ch["name"]},
150
+ "principals": principals, "messages": messages, "reactions": reactions,
151
+ "doc": doc, "storage_files": list(storage_files.values()), "attachments": [{k: a[k] for k in ("id", "uploader_id", "filename", "content_type",
152
+ "size", "sha256", "created_at")} for a in attachments]}
153
+ keys = {a["sha256"]: ("attachment", a["storage_key"]) for a in attachments}
154
+ for f in storage_files.values():
155
+ keys.setdefault(f["sha256"], ("storage", store.storage_key(f["owner_id"], f["path"])))
156
+ return manifest, keys
157
+ finally:
158
+ conn.execute("ROLLBACK")
159
+
160
+
161
+ def build_bundle(manifest, keys, blobs, path: Path, storage=None):
162
+ validate_manifest(manifest)
163
+ data = canonical(manifest)
164
+ if len(data) > MAX_MANIFEST_BYTES:
165
+ raise ApiError(413, "too_large", "channel manifest is too large")
166
+ total = len(data)
167
+ with tempfile.TemporaryDirectory(prefix="msgboard-export-files-") as scratch, zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as z:
168
+ z.writestr("manifest.json", data)
169
+ for sha, (kind, key) in keys.items():
170
+ digest = hashlib.sha256()
171
+ try:
172
+ src_path = blobs.path_for(key) if kind == "attachment" else Path(scratch) / sha
173
+ if kind == "storage" and not storage.get(key, src_path):
174
+ raise conflict("export_changed", "linked storage bytes disappeared; retry export")
175
+ with src_path.open("rb") as src, z.open("blobs/" + sha, "w") as dst:
176
+ while chunk := src.read(1024 * 1024):
177
+ total += len(chunk)
178
+ if total > MAX_BUNDLE_BYTES:
179
+ raise ApiError(413, "too_large", "channel bundle exceeds 512 MiB")
180
+ digest.update(chunk)
181
+ dst.write(chunk)
182
+ except FileNotFoundError:
183
+ raise conflict("export_changed", "attachment bytes disappeared; retry the export")
184
+ if digest.hexdigest() != sha:
185
+ raise conflict("export_changed", "attachment bytes do not match their recorded checksum")
186
+ if path.stat().st_size > MAX_BUNDLE_BYTES:
187
+ raise ApiError(413, "too_large", "channel ZIP exceeds 512 MiB")
188
+
189
+
190
+ def _int(value, what):
191
+ if type(value) is not int or value < 1:
192
+ raise bad_request(f"{what} must be a positive integer")
193
+ return value
194
+
195
+
196
+ def _time(value, what):
197
+ if type(value) not in (float, int) or not math.isfinite(value) or value < 0:
198
+ raise bad_request(f"{what} must be a finite UTC timestamp")
199
+
200
+
201
+ def _text(value, what, max_bytes=LIMITS.max_body_bytes):
202
+ if not isinstance(value, str) or len(value.encode()) > max_bytes:
203
+ raise bad_request(f"invalid or oversized {what}")
204
+
205
+
206
+ def validate_manifest(m):
207
+ if not isinstance(m, dict) or m.get("format") != "msgboard-channel" or m.get("version") != 1:
208
+ raise bad_request("expected a version 1 msgboard-channel bundle")
209
+ source = m.get("source")
210
+ if not isinstance(source, dict) or not re.fullmatch(r"[0-9a-f]{32}", str(source.get("board_id", ""))):
211
+ raise bad_request("invalid source board ID")
212
+ _int(source.get("channel_id"), "source channel ID")
213
+ _text(source.get("board_url"), "source URL", 2048)
214
+ url = urlsplit(source["board_url"])
215
+ if url.scheme not in ("http", "https") or not url.netloc or url.username or url.password or url.query or url.fragment:
216
+ raise bad_request("source URL must be an HTTP(S) board address")
217
+ for field, limit in [("principals", MAX_MESSAGES + MAX_ATTACHMENTS), ("messages", MAX_MESSAGES),
218
+ ("reactions", MAX_MESSAGES * 100), ("attachments", MAX_ATTACHMENTS), ("storage_files", MAX_ATTACHMENTS)]:
219
+ if field == "storage_files":
220
+ m.setdefault(field, [])
221
+ if not isinstance(m.get(field), list) or len(m[field]) > limit or any(not isinstance(x, dict) for x in m[field]):
222
+ raise bad_request(f"invalid or oversized {field} list")
223
+ people = {}
224
+ handles = set()
225
+ for p in m["principals"]:
226
+ p.setdefault("display_name", None)
227
+ pid = _int(p.get("id"), "author ID")
228
+ handle = p.get("external_id")
229
+ if not isinstance(handle, str) or not store.EXTERNAL_ID.fullmatch(handle) or handle.lower() in handles or pid in people:
230
+ raise bad_request("duplicate or invalid author handle/ID")
231
+ if p.get("identity_type") not in ("user", "agent", "system"):
232
+ raise bad_request("invalid author type")
233
+ if p.get("display_name") is not None:
234
+ _text(p["display_name"], "display name", 1024)
235
+ people[pid] = p
236
+ handles.add(handle.lower())
237
+ attachments = {}
238
+ for a in m["attachments"]:
239
+ a.setdefault("filename", None)
240
+ a.setdefault("content_type", None)
241
+ aid = a.get("id")
242
+ if not isinstance(aid, str) or not re.fullmatch(r"att_[A-Za-z0-9_-]{1,120}", aid) or aid in attachments:
243
+ raise bad_request("duplicate or invalid attachment ID")
244
+ if a.get("uploader_id") not in people or not HASH.fullmatch(str(a.get("sha256", ""))):
245
+ raise bad_request("invalid attachment author or checksum")
246
+ if type(a.get("size")) is not int or not 0 <= a["size"] <= LIMITS.max_upload_bytes:
247
+ raise bad_request("invalid attachment size")
248
+ for field, limit in [("filename", 1024), ("content_type", 128)]:
249
+ if a.get(field) is not None:
250
+ _text(a[field], field, limit)
251
+ _time(a.get("created_at"), "attachment timestamp")
252
+ attachments[aid] = a
253
+ paths = set()
254
+ aliases = set()
255
+ for f in m["storage_files"]:
256
+ if f.get("owner_id") not in people or not HASH.fullmatch(str(f.get("sha256", ""))):
257
+ raise bad_request("invalid storage owner or checksum")
258
+ f["path"] = store.storage_path(f.get("path"))
259
+ if type(f.get("size")) is not int or not 0 <= f["size"] <= MAX_BUNDLE_BYTES:
260
+ raise bad_request("invalid storage file size")
261
+ _time(f.get("updated_at"), "storage timestamp")
262
+ f.setdefault("content_type", None)
263
+ if f["content_type"] is not None:
264
+ _text(f["content_type"], "storage content type", 128)
265
+ if not isinstance(f.get("handles"), list) or not f["handles"]:
266
+ raise bad_request("storage handle aliases are required")
267
+ for handle in f["handles"]:
268
+ if not isinstance(handle, str) or not store.EXTERNAL_ID.fullmatch(handle) or (handle, f["path"]) in aliases:
269
+ raise bad_request("invalid or duplicate storage alias")
270
+ aliases.add((handle, f["path"]))
271
+ if (f["owner_id"], f["path"]) in paths:
272
+ raise bad_request("duplicate storage path")
273
+ paths.add((f["owner_id"], f["path"]))
274
+ ordered_paths = sorted(paths)
275
+ if any(owner == next_owner and child.startswith(path + "/")
276
+ for (owner, path), (next_owner, child) in pairwise(ordered_paths)):
277
+ raise bad_request("storage files have a file/folder collision")
278
+ if len(paths) + len(attachments) > MAX_ATTACHMENTS:
279
+ raise bad_request("too many attachment/storage files")
280
+ messages = {}
281
+ seqs = set()
282
+ for msg in m["messages"]:
283
+ msg.setdefault("reply_to", None)
284
+ msg.setdefault("edited_at", None)
285
+ mid = _int(msg.get("id"), "message ID")
286
+ seq = _int(msg.get("seq"), "message sequence")
287
+ if mid in messages or seq in seqs or msg.get("sender_id") not in people:
288
+ raise bad_request("duplicate message ID/sequence or missing author")
289
+ _text(msg.get("body"), "message body")
290
+ _time(msg.get("created_at"), "message timestamp")
291
+ if msg.get("edited_at") is not None:
292
+ _time(msg["edited_at"], "edit timestamp")
293
+ if not isinstance(msg.get("attachments"), list) or len(msg["attachments"]) > LIMITS.max_attachments_per_message:
294
+ raise bad_request("invalid message attachments")
295
+ if any(not isinstance(aid, str) or aid not in attachments for aid in msg["attachments"]):
296
+ raise bad_request("message refers to a missing attachment")
297
+ messages[mid] = msg
298
+ seqs.add(seq)
299
+ # Parent links, not untrusted thread_id, define the thread. Validate the whole
300
+ # graph before any database writes, including parents with later timestamps.
301
+ for mid, msg in messages.items():
302
+ seen = {mid}
303
+ parent = msg.get("reply_to")
304
+ while parent is not None:
305
+ if type(parent) is not int or parent not in messages or parent in seen:
306
+ raise bad_request("missing parent or cyclic thread")
307
+ seen.add(parent)
308
+ parent = messages[parent].get("reply_to")
309
+ reactions = set()
310
+ for r in m["reactions"]:
311
+ if r.get("message_id") not in messages or r.get("principal_id") not in people:
312
+ raise bad_request("reaction refers to a missing message or author")
313
+ store._valid_emoji(r.get("emoji"))
314
+ _time(r.get("created_at"), "reaction timestamp")
315
+ key = (r["message_id"], r["principal_id"], r["emoji"])
316
+ if key in reactions:
317
+ raise bad_request("duplicate reaction")
318
+ reactions.add(key)
319
+ doc = m.setdefault("doc", None)
320
+ if doc is not None:
321
+ if not isinstance(doc, dict) or doc.get("author_id") not in people:
322
+ raise bad_request("invalid channel doc author")
323
+ _text(doc.get("body"), "channel doc", store.MAX_DOC_BYTES)
324
+ _time(doc.get("created_at"), "doc timestamp")
325
+ for body in [msg["body"] for msg in m["messages"]] + ([doc["body"]] if doc else []):
326
+ for match in storage_links(body):
327
+ ref = storage_reference(match.group(), source["board_url"])
328
+ if ref is not None and (ref[0], ref[1]) not in aliases:
329
+ raise bad_request("bundle omits a linked same-board storage file")
330
+ return m
331
+
332
+
333
+ def read_bundle(path: Path, stage: Path):
334
+ try:
335
+ with zipfile.ZipFile(path) as z:
336
+ entries = z.infolist()
337
+ names = [e.filename for e in entries]
338
+ if len(names) != len(set(names)) or len(names) > MAX_ATTACHMENTS + 1:
339
+ raise bad_request("duplicate ZIP members or too many files")
340
+ if sum(e.file_size for e in entries) > MAX_BUNDLE_BYTES:
341
+ raise ApiError(413, "too_large", "expanded bundle exceeds 512 MiB")
342
+ if z.getinfo("manifest.json").file_size > MAX_MANIFEST_BYTES:
343
+ raise ApiError(413, "too_large", "manifest exceeds 32 MiB")
344
+ m = validate_manifest(json.loads(z.read("manifest.json")))
345
+ all_files = m["attachments"] + m["storage_files"]
346
+ hashes = {a["sha256"] for a in all_files}
347
+ if set(names) != {"manifest.json"} | {"blobs/" + sha for sha in hashes}:
348
+ raise bad_request("ZIP members must exactly match the manifest's blobs")
349
+ files = {}
350
+ for sha in hashes:
351
+ expected = {a["size"] for a in all_files if a["sha256"] == sha}
352
+ info = z.getinfo("blobs/" + sha)
353
+ if expected != {info.file_size}:
354
+ raise bad_request("attachment length does not match manifest")
355
+ dest = stage / sha
356
+ h = hashlib.sha256()
357
+ with z.open(info) as src, dest.open("xb") as dst:
358
+ while chunk := src.read(1024 * 1024):
359
+ h.update(chunk)
360
+ dst.write(chunk)
361
+ if h.hexdigest() != sha:
362
+ raise bad_request("attachment checksum does not match manifest")
363
+ files[sha] = dest
364
+ return m, hashlib.sha256(canonical(m)).hexdigest(), files
365
+ except (zipfile.BadZipFile, KeyError, ValueError, TypeError, UnicodeError, RuntimeError) as exc:
366
+ raise bad_request("invalid channel bundle") from exc
367
+
368
+
369
+ def plan_import(conn, p, channel_id, m, digest, keep_target_doc=False):
370
+ ch = public_channel(conn, p, channel_id)
371
+ existing = conn.execute("SELECT * FROM channel_imports WHERE channel_id = ?", (channel_id,)).fetchone()
372
+ blockers, mappings = [], []
373
+ replay = bool(existing and existing["digest"] == digest)
374
+ if existing and not replay:
375
+ blockers.append("target already holds a different import; incremental imports are not supported")
376
+ if not existing:
377
+ if conn.execute("SELECT 1 FROM messages WHERE channel_id = ? LIMIT 1", (channel_id,)).fetchone():
378
+ blockers.append("initial import requires an empty target channel")
379
+ if ch["doc_version"] and not keep_target_doc:
380
+ blockers.append("initial import requires a target with no channel doc")
381
+ if ch["archived"]:
382
+ blockers.append("target channel is archived")
383
+ for author in m["principals"]:
384
+ rows = conn.execute("SELECT * FROM principals WHERE lower(external_id) = lower(?)", (author["external_id"],)).fetchall()
385
+ if len(rows) > 1 or (rows and rows[0]["identity_type"] != author["identity_type"]):
386
+ blockers.append(f"ambiguous or different-type handle: {author['external_id']}")
387
+ mappings.append({"source_id": author["id"], "handle": author["external_id"],
388
+ "display_name": author["display_name"], "target_id": rows[0]["id"] if len(rows) == 1 else None,
389
+ "action": "existing" if rows else "ghost"})
390
+ by_author = {a["source_id"]: a for a in mappings}
391
+ planned_files = []
392
+ additions = {}
393
+ max_file, quota = store.storage_limits(conn)
394
+ for f in m["storage_files"]:
395
+ mapping = by_author[f["owner_id"]]
396
+ owner_id = mapping["target_id"]
397
+ if owner_id:
398
+ owner = conn.execute("SELECT disabled, joined_via FROM principals WHERE id = ?", (owner_id,)).fetchone()
399
+ if owner["disabled"] and owner["joined_via"] != "import":
400
+ blockers.append(f"mapped storage owner is disabled: {mapping['handle']}")
401
+ old = conn.execute("SELECT size, sha256 FROM storage_files WHERE owner_id = ? AND path = ?",
402
+ (owner_id, f["path"])).fetchone() if owner_id else None
403
+ action = "reuse" if old and old["sha256"] == f["sha256"] and old["size"] == f["size"] else "create"
404
+ if old and action != "reuse":
405
+ action = "conflict"
406
+ blockers.append(f"different target storage file: {mapping['handle']}/{f['path']}")
407
+ if f["size"] > max_file:
408
+ blockers.append(f"storage file exceeds target per-file limit: {mapping['handle']}/{f['path']}")
409
+ if owner_id:
410
+ occupied = [r[0] for r in conn.execute("SELECT path FROM storage_files WHERE owner_id = ?", (owner_id,))]
411
+ if any(path.startswith(f["path"] + "/") or f["path"].startswith(path + "/") for path in occupied):
412
+ blockers.append(f"file/folder collision: {mapping['handle']}/{f['path']}")
413
+ if not old:
414
+ additions[f["owner_id"]] = additions.get(f["owner_id"], 0) + f["size"]
415
+ planned_files.append({"handle": mapping["handle"], "target_id": owner_id, "path": f["path"],
416
+ "size": f["size"], "sha256": f["sha256"], "action": action})
417
+ for source_id, size in additions.items():
418
+ mapping = by_author[source_id]
419
+ used = store.storage_usage(conn, mapping["target_id"]) if mapping["target_id"] else 0
420
+ if used + size > quota:
421
+ blockers.append(f"storage folder exceeds target quota: {mapping['handle']}")
422
+ counts = {"messages": len(m["messages"]), "attachments": len(m["attachments"]),
423
+ "storage_files": len(planned_files),
424
+ "bytes": sum(a["size"] for a in m["attachments"] + m["storage_files"]), "reactions": len(m["reactions"]),
425
+ "ghosts": sum(a["action"] == "ghost" for a in mappings), "doc": int(m["doc"] is not None)}
426
+ report = {"source": m["source"], "target_channel_id": channel_id, "digest": digest,
427
+ "already_imported": replay, "can_import": not blockers, "blockers": blockers,
428
+ "counts": counts, "would_create": {k: 0 if replay else v for k, v in counts.items()},
429
+ "authors": mappings, "storage_files": planned_files}
430
+ report["would_create"]["storage_files"] = 0 if replay else sum(f["action"] == "create" for f in planned_files)
431
+ report["document"] = {"action": "keep_target" if keep_target_doc else "import_source" if m["doc"] else "none",
432
+ "target_version": ch["doc_version"]}
433
+ if keep_target_doc:
434
+ report["would_create"]["doc"] = 0
435
+ if replay:
436
+ previous = json.loads(existing["report"])
437
+ report["document"] = previous.get("document", report["document"])
438
+ return report
439
+
440
+
441
+ def verify_reused_storage(report, storage):
442
+ """Metadata equality alone cannot prove a mutable bucket key still has its bytes."""
443
+ with tempfile.TemporaryDirectory(prefix="msgboard-reuse-") as scratch:
444
+ for f in report["storage_files"]:
445
+ if f["action"] != "reuse":
446
+ continue
447
+ path = Path(scratch) / "file"
448
+ ok = storage.get(store.storage_key(f["target_id"], f["path"]), path)
449
+ digest = hashlib.sha256()
450
+ if ok and path.stat().st_size == f["size"]:
451
+ with path.open("rb") as src:
452
+ while chunk := src.read(1024 * 1024):
453
+ digest.update(chunk)
454
+ if not ok or path.stat().st_size != f["size"] or digest.hexdigest() != f["sha256"]:
455
+ report["blockers"].append(f"target storage bytes unavailable or mismatched: {f['handle']}/{f['path']}")
456
+ report["can_import"] = False
457
+ return report
458
+
459
+
460
+ def _rewrite_storage(body, source_url, files, handles):
461
+ aliases = {(alias, f["path"]): handles[f["owner_id"]] for f in files for alias in f["handles"]}
462
+ def replace(match):
463
+ ref = storage_reference(match.group(), source_url)
464
+ if ref is None or (ref[0], ref[1]) not in aliases:
465
+ return match.group()
466
+ handle, path, url, suffix = ref
467
+ target = aliases[(handle, path)]
468
+ if not url.netloc and target == handle:
469
+ return match.group()
470
+ return urlunsplit(("", "", "/v1/storage/" + quote(target, safe="") + "/" + quote(path, safe="/"), url.query, url.fragment)) + suffix
471
+ parts, start = [], 0
472
+ for match in storage_links(body):
473
+ parts.extend((body[start:match.start()], replace(match)))
474
+ start = match.end()
475
+ parts.append(body[start:])
476
+ return "".join(parts)
477
+
478
+
479
+ def _rewrite_attachments(body, source_url, ids):
480
+ # Absolute links to this source become target-local links. Foreign/storage
481
+ # links and old textual "msg N" references are deliberately untouched.
482
+ for old, new in ids.items():
483
+ body = body.replace(source_url.rstrip("/") + "/v1/attachments/" + old, "/v1/attachments/" + new)
484
+ body = re.sub(r"(?<![A-Za-z0-9_./:-])/v1/attachments/" + re.escape(old) + r"(?![A-Za-z0-9_-])",
485
+ "/v1/attachments/" + new, body)
486
+ body = re.sub(r"#file/" + re.escape(old) + r"(?![A-Za-z0-9_-])", "#file/" + new, body)
487
+ return body
488
+
489
+
490
+ def prepare_import(conn, p, channel_id, m, digest, now, keep_target_doc=False):
491
+ report = plan_import(conn, p, channel_id, m, digest, keep_target_doc)
492
+ if not report["can_import"]:
493
+ raise ApiError(409, "import_blocked", "channel import is blocked", details=report)
494
+ people, created = {}, []
495
+ for mapping in report["authors"]:
496
+ if mapping["target_id"] is None:
497
+ author = next(a for a in m["principals"] if a["id"] == mapping["source_id"])
498
+ cursor = conn.execute("""INSERT INTO principals (issuer, external_id, identity_type, role,
499
+ display_name, disabled, created_at, created_by, joined_via)
500
+ VALUES ('import', ?, ?, ?, ?, 1, ?, ?, 'import')""",
501
+ (author["external_id"], author["identity_type"], "viewer" if author["identity_type"] == "user" else "guest",
502
+ author["display_name"], now, p.id))
503
+ mapping["target_id"] = cursor.lastrowid
504
+ created.append((cursor.lastrowid, author["external_id"], now))
505
+ people[mapping["source_id"]] = mapping["target_id"]
506
+ return {"report": report, "people": people, "created_ghosts": created, "keep_target_doc": keep_target_doc}
507
+
508
+
509
+ def checked_import_plan(conn, p, channel_id, m, digest, prepared):
510
+ report = plan_import(conn, p, channel_id, m, digest, prepared.get("keep_target_doc", False))
511
+ if any(a["target_id"] != prepared["people"].get(a["source_id"]) for a in report["authors"]):
512
+ report["blockers"].append("author mappings changed during preparation; retry the import")
513
+ report["can_import"] = False
514
+ created = {row[0] for row in prepared["created_ghosts"]}
515
+ report["counts"]["ghosts"] = len(created)
516
+ report["would_create"]["ghosts"] = 0 if report["already_imported"] else len(created)
517
+ for author in report["authors"]:
518
+ if author["target_id"] in created:
519
+ author["action"] = "ghost"
520
+ return report
521
+
522
+
523
+ def cleanup_ghosts(conn, prepared):
524
+ for pid, handle, created_at in prepared["created_ghosts"]:
525
+ try:
526
+ # Never remove an identity another operation has adopted or changed.
527
+ # Foreign keys leave a referenced ghost intact.
528
+ conn.execute("""DELETE FROM principals WHERE id = ? AND external_id = ? AND created_at = ?
529
+ AND joined_via = 'import' AND disabled = 1 AND owner_id IS NULL
530
+ AND NOT EXISTS (SELECT 1 FROM credentials WHERE principal_id = principals.id)
531
+ AND NOT EXISTS (SELECT 1 FROM audit_log WHERE target = 'principal:' || principals.id)""",
532
+ (pid, handle, created_at))
533
+ except sqlite3.IntegrityError:
534
+ pass
535
+
536
+
537
+ def attachment_id(channel_id, source, old_id):
538
+ return "att_" + hashlib.sha256(canonical([channel_id, source["board_id"], source["channel_id"], old_id])).hexdigest()[:32]
539
+
540
+
541
+ def stage_import(channel_id, m, files, prepared, blobs, storage, uploaded_storage, uploaded_attachments):
542
+ """All network reads/uploads and local blob I/O run before the writer transaction."""
543
+ report = prepared["report"]
544
+ if not report["can_import"]:
545
+ raise ApiError(409, "import_blocked", "channel import is blocked", details=report)
546
+ report = verify_reused_storage(report, storage)
547
+ if not report["can_import"]:
548
+ raise ApiError(409, "import_blocked", "channel import is blocked", details=report)
549
+ if report["already_imported"]:
550
+ return
551
+ for f, planned in zip(m["storage_files"], report["storage_files"]):
552
+ if planned["action"] == "reuse":
553
+ continue
554
+ key = store.storage_key(prepared["people"][f["owner_id"]], f["path"])
555
+ uploaded_storage.append(key) # also clean up a backend that writes, then raises
556
+ storage.put([(files[f["sha256"]], key)])
557
+ for a in m["attachments"]:
558
+ aid = attachment_id(channel_id, m["source"], a["id"])
559
+ key = f"{aid[4:6]}/{aid}"
560
+ dest = blobs.path_for(key)
561
+ if dest.exists():
562
+ digest = hashlib.sha256()
563
+ with dest.open("rb") as src:
564
+ while chunk := src.read(1024 * 1024):
565
+ digest.update(chunk)
566
+ if digest.hexdigest() != a["sha256"] or dest.stat().st_size != a["size"]:
567
+ raise conflict("import_blob_conflict", "an existing attachment blob has different bytes")
568
+ continue
569
+ temp, fh = blobs.open_temp()
570
+ uploaded_attachments.append(key)
571
+ try:
572
+ with files[a["sha256"]].open("rb") as src:
573
+ shutil.copyfileobj(src, fh)
574
+ blobs.finish_temp(fh)
575
+ blobs.commit(temp, key)
576
+ finally:
577
+ if not fh.closed:
578
+ fh.close()
579
+ blobs.discard(temp)
580
+
581
+
582
+ def import_channel(conn, p, channel_id, m, digest, prepared, now):
583
+ """Publish metadata only. Bucket and local file I/O has already finished."""
584
+ report = checked_import_plan(conn, p, channel_id, m, digest, prepared)
585
+ if not report["can_import"]:
586
+ raise ApiError(409, "import_blocked", "channel import is blocked", details=report)
587
+ if report["already_imported"]:
588
+ return report
589
+ people = prepared["people"]
590
+ source = m["source"]
591
+ handles = {sid: conn.execute("SELECT external_id FROM principals WHERE id = ?", (pid,)).fetchone()[0] for sid, pid in people.items()}
592
+ for f in m["storage_files"]:
593
+ owner_id = people[f["owner_id"]]
594
+ old = conn.execute("SELECT sha256, size FROM storage_files WHERE owner_id = ? AND path = ?", (owner_id, f["path"])).fetchone()
595
+ if old:
596
+ continue # plan and byte verification established that it is identical
597
+ store.storage_record(conn, p, owner_id, f["path"], f["size"], f["sha256"], f["content_type"], f["updated_at"])
598
+ aid_map = {}
599
+ for a in m["attachments"]:
600
+ aid = attachment_id(channel_id, source, a["id"])
601
+ aid_map[a["id"]] = aid
602
+ storage_key = f"{aid[4:6]}/{aid}"
603
+ conn.execute("""INSERT INTO attachments (id, channel_id, uploader_id, upload_id, storage_key,
604
+ filename, content_type, size, sha256, state, published, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'ready', 1, ?)""",
605
+ (aid, channel_id, people[a["uploader_id"]], "import:" + aid, storage_key, a["filename"],
606
+ a["content_type"], a["size"], a["sha256"], a["created_at"]))
607
+ seq = conn.execute("SELECT next_seq FROM channels WHERE id = ?", (channel_id,)).fetchone()[0]
608
+ ids = {}
609
+ ordered = sorted(m["messages"], key=lambda msg: (msg["created_at"], msg["seq"], msg["id"]))
610
+ for msg in ordered:
611
+ origin = {"board_url": source["board_url"], "channel_id": source["channel_id"], "message_id": msg["id"]}
612
+ cursor = conn.execute("""INSERT INTO messages (channel_id, seq, sender_id, body, attachments,
613
+ request_id, payload_hash, created_at, edited_at, imported, import_source)
614
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?)""",
615
+ (channel_id, seq, people[msg["sender_id"]], _rewrite_storage(_rewrite_attachments(msg["body"], source["board_url"], aid_map), source["board_url"], m["storage_files"], handles),
616
+ json.dumps([aid_map[a] for a in msg["attachments"]]),
617
+ "import:" + hashlib.sha256(canonical([channel_id, source["board_id"], source["channel_id"], msg["id"]])).hexdigest(),
618
+ digest, msg["created_at"], msg["edited_at"], json.dumps(origin)))
619
+ ids[msg["id"]] = cursor.lastrowid
620
+ seq += 1
621
+ by_id = {msg["id"]: msg for msg in ordered}
622
+ for msg in ordered:
623
+ parent = msg["reply_to"]
624
+ if parent is not None:
625
+ root = parent
626
+ while by_id[root]["reply_to"] is not None:
627
+ root = by_id[root]["reply_to"]
628
+ conn.execute("UPDATE messages SET reply_to = ?, thread_id = ? WHERE id = ?",
629
+ (ids[parent], ids[root], ids[msg["id"]]))
630
+ for r in m["reactions"]:
631
+ conn.execute("INSERT INTO message_reactions (message_id, principal_id, emoji, created_at) VALUES (?, ?, ?, ?)",
632
+ (ids[r["message_id"]], people[r["principal_id"]], r["emoji"], r["created_at"]))
633
+ if m["doc"] is not None and not prepared.get("keep_target_doc", False):
634
+ doc = m["doc"]
635
+ conn.execute("INSERT INTO channel_doc_versions (channel_id, version, body, author_id, created_at) VALUES (?, 1, ?, ?, ?)",
636
+ (channel_id, _rewrite_storage(_rewrite_attachments(doc["body"], source["board_url"], aid_map), source["board_url"], m["storage_files"], handles), people[doc["author_id"]], doc["created_at"]))
637
+ conn.execute("UPDATE channels SET doc_version = 1 WHERE id = ?", (channel_id,))
638
+ conn.execute("UPDATE channels SET next_seq = ? WHERE id = ?", (seq, channel_id))
639
+ conn.execute("""INSERT INTO channel_reads (principal_id, channel_id, last_read_seq, thread_floor, updated_at)
640
+ SELECT id, ?, ?, ?, ? FROM principals WHERE true
641
+ ON CONFLICT (principal_id, channel_id) DO UPDATE SET
642
+ last_read_seq = max(last_read_seq, excluded.last_read_seq), thread_floor = max(thread_floor, excluded.thread_floor)""",
643
+ (channel_id, seq - 1, seq - 1, now))
644
+ conn.execute("INSERT INTO channel_imports VALUES (?, ?, ?, ?, ?, ?)",
645
+ (channel_id, source["board_id"], source["channel_id"], digest, json.dumps(report), now))
646
+ store.audit(conn, p.id, "channel.import", f"channel:{channel_id}", {"source": source, "digest": digest, "counts": report["counts"]}, now)
647
+ return report
requirements.txt ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ fastapi>=0.110
2
+ uvicorn>=0.29
3
+ httpx>=0.27
4
+ huggingface_hub>=1.31