Deploy ContextEcho donation relay

donate/relay_server.py

@@ -160,6 +160,51 @@ def _read_jsonl(path: Path, limit: int = 200) -> list[dict]:
 
 _SUBMISSION_ID_RE = re.compile(r"submission-[A-Za-z0-9_-]{4,64}")
 
+# Free-tier Spaces have no persistent disk: the local state dir is wiped on
+# every rebuild/restart. The device->donations index is therefore mirrored
+# to the private staging dataset (like the seen-hashes backfill) and lazily
+# restored when the local copy is missing.
+DEVICE_DONATIONS_REPO_PATH = "maintainer/device_donations.jsonl"
+_DEVICE_BACKFILL = {"attempted": False}
+
+
+def _staging_write_token() -> str | None:
+    return os.environ.get("HF_STAGING_TOKEN") or os.environ.get("CONTEXTECHO_STAGING_TOKEN")
+
+
+def _backfill_device_donations() -> None:
+    if _DEVICE_BACKFILL["attempted"]:
+        return
+    _DEVICE_BACKFILL["attempted"] = True
+    if DEVICE_DONATIONS.exists() and DEVICE_DONATIONS.stat().st_size > 0:
+        return
+    token = _staging_write_token()
+    if not token:
+        return
+    try:
+        data = _read_hf_file(STAGING_REPO, DEVICE_DONATIONS_REPO_PATH, token)
+        STATE_DIR.mkdir(parents=True, exist_ok=True)
+        DEVICE_DONATIONS.write_bytes(data)
+    except Exception:
+        pass
+
+
+def _persist_device_donations() -> None:
+    token = _staging_write_token()
+    if not token or not DEVICE_DONATIONS.exists():
+        return
+    try:
+        api = HfApi(token=token)
+        api.upload_file(
+            path_or_fileobj=str(DEVICE_DONATIONS),
+            path_in_repo=DEVICE_DONATIONS_REPO_PATH,
+            repo_id=STAGING_REPO,
+            repo_type="dataset",
+            commit_message="Update device donations index",
+        )
+    except Exception:
+        pass
+
 
 def _record_device_donation(device_id: str, submission_id: str, turns: int = 0,
                             submitted_utc: str = "", source: str = "submission") -> bool:
@@ -168,6 +213,7 @@ def _record_device_donation(device_id: str, submission_id: str, turns: int = 0,
         return False
     if not _SUBMISSION_ID_RE.fullmatch(str(submission_id or "")):
         return False
+    _backfill_device_donations()
     existing = {
         (row.get("device_id"), row.get("submission_id"))
         for row in _read_jsonl(DEVICE_DONATIONS, limit=100_000)
@@ -189,6 +235,7 @@ def _record_device_donation(device_id: str, submission_id: str, turns: int = 0,
 
 def _device_donations(device_id: str) -> list[dict]:
     """All recorded donations for one device, newest first, deduped by submission."""
+    _backfill_device_donations()
     rows = [
         row for row in _read_jsonl(DEVICE_DONATIONS, limit=100_000)
         if row.get("device_id") == device_id
@@ -1556,6 +1603,8 @@ def claim_donations(payload: Annotated[dict, Body()]) -> dict:
             source="claim",
         ):
             claimed += 1
+    if claimed:
+        _persist_device_donations()
     return {"ok": True, "claimed": claimed}
 
 
@@ -1638,13 +1687,14 @@ async def donate(
             )
             raise
         _record_seen_hash(artifact_hash, submission_id, manifest)
-        _record_device_donation(
+        if _record_device_donation(
             str(manifest.get("donor_device_id") or ""),
             submission_id,
             turns=_count_value(manifest.get("turns")),
             submitted_utc=str(manifest.get("submitted_utc") or ""),
             source="submission",
-        )
+        ):
+            _persist_device_donations()
         _append_submission_event(
             "submitted",
             submission_id=submission_id,

donate/web.py

@@ -335,10 +335,36 @@ def relay_url() -> str:
 RELAY_STATUS_CHUNK = 200
 
 
+# The relay runs on a free-tier Space that sleeps when idle and takes
+# 45-60s to wake; a request racing the cold start times out and the wizard
+# silently degrades to local receipts. Wake it once per process first.
+_RELAY_AWAKE = {"ok": False}
+
+
+def _ensure_relay_awake(timeout_each: int = 45, attempts: int = 3) -> bool:
+    if _RELAY_AWAKE["ok"]:
+        return True
+    url = relay_url()
+    if not url:
+        return False
+    for _ in range(attempts):
+        try:
+            req = Request(f"{url}/health", headers={"user-agent": "contextecho-donate"})
+            with urlopen(req, timeout=timeout_each) as resp:
+                if getattr(resp, "status", 200) == 200:
+                    _RELAY_AWAKE["ok"] = True
+                    return True
+        except Exception:
+            time.sleep(2)
+    return False
+
+
 def relay_donation_status(sessions: list[dict]) -> list[dict]:
     url = relay_url()
     if not url or not sessions:
         return []
+    if not _ensure_relay_awake():
+        return []
     statuses: list[dict] = []
     for start in range(0, len(sessions), RELAY_STATUS_CHUNK):
         chunk = sessions[start : start + RELAY_STATUS_CHUNK]
@@ -361,7 +387,7 @@ def relay_donation_status(sessions: list[dict]) -> list[dict]:
             method="POST",
         )
         try:
-            with urlopen(req, timeout=30) as resp:
+            with urlopen(req, timeout=90) as resp:
                 result = json.loads(resp.read().decode("utf-8"))
         except Exception:
             return []
@@ -374,10 +400,11 @@ def relay_donation_status(sessions: list[dict]) -> list[dict]:
     return statuses
 
 
-def _relay_post_json(path: str, payload: dict, timeout: int = 15) -> dict:
+def _relay_post_json(path: str, payload: dict, timeout: int = 60) -> dict:
     url = relay_url()
     if not url:
         raise ValueError("Relay URL is not configured.")
+    _ensure_relay_awake()
     data = json.dumps(payload).encode("utf-8")
     req = Request(
         f"{url}{path}",
@@ -442,6 +469,65 @@ def my_donations_summary() -> dict:
     }
 
 
+def merge_archived_donations(sessions: list[dict]) -> list[dict]:
+    """Append donated sessions whose local source file no longer exists.
+
+    The picker lists what is on disk; donations outlive the local logs, so
+    donations with no matching discovered row are synthesized from the
+    device-linked history and shown (non-actionable) in the same table.
+    """
+    try:
+        donations = my_donations_summary().get("donations") or []
+    except Exception:
+        return sessions
+    if not donations:
+        return sessions
+    present_ids = {normalize_submission_id(row.get("relay_submission_id")) for row in sessions}
+    present_ids.discard("")
+    present_path_keys = {source_path_key(row["path"]) for row in sessions if row.get("path")}
+    receipt_path_keys: dict[str, str] = {}
+    for item in load_donation_registry().get("submissions", []):
+        sid = normalize_submission_id(item.get("submission_id") or item.get("submission"))
+        if sid:
+            receipt_path_keys.setdefault(sid, str(item.get("source_path_key") or ""))
+    out = list(sessions)
+    for d in donations:
+        sid = normalize_submission_id(d.get("submission_id"))
+        if not sid or sid in present_ids:
+            continue
+        # If the receipt maps to a file that is still discoverable, the
+        # discovered row already represents this donation (covers the
+        # relay-unreachable case where relay_submission_id is empty).
+        if receipt_path_keys.get(sid) and receipt_path_keys[sid] in present_path_keys:
+            continue
+        turns = int(d.get("turns") or 0)
+        out.append({
+            "agent": "Donated session",
+            "project": "archive",
+            "session_label": "local log deleted",
+            "path": f"archived://{sid}",
+            "turns": turns,
+            "records": 0,
+            "compactions": 0,
+            "last_active": str(d.get("submitted_utc") or "")[:10],
+            "donated": True,
+            "donated_before": True,
+            "donated_turns": turns,
+            "new_turns": 0,
+            "update_ready": False,
+            "relay_submission_id": sid if is_support_submission_id(sid) else "",
+            "relay_public_session_id": "",
+            "relay_received": True,
+            "relay_checked": True,
+            "local_credit_name": "",
+            "local_contributor_email": "",
+            "local_institute": "",
+            "local_public_anonymous": True,
+            "archived_donation": True,
+        })
+    return out
+
+
 def relay_metadata_update(payload: dict) -> dict:
     url = relay_url()
     if not url:
@@ -1765,7 +1851,6 @@ INDEX_HTML = r"""<!doctype html>
           </div>
         </div>
         <div id="datasetComposition" class="composition-panel" aria-label="Public dataset composition"></div>
-        <div id="myDonations" class="composition-panel" aria-label="My donations from this machine" style="display:none"></div>
         <button id="discoverBtn" class="discover-main">Discover Sessions</button>
         <div id="discoverStatus" class="muted" style="margin-top:16px; text-align:center">Scanning starts automatically. Use Discover Sessions to rerun the scan.</div>
         <div id="discoverProgress" class="progress"><div></div></div>
@@ -2470,35 +2555,6 @@ async function loadProjectStats(){
     renderProjectStats();
   }
 }
-async function loadMyDonations(){
-  const target = $('myDonations');
-  if(!target) return;
-  try {
-    const r = await fetch('/api/my_donations', {cache:'no-store'});
-    if(!r.ok) return;
-    const data = await r.json();
-    const donations = data.donations || [];
-    if(!donations.length){ target.style.display = 'none'; return; }
-    const source = data.relay_checked
-      ? 'Synced with the donation relay by this machine’s anonymous device hash.'
-      : 'From local receipts; relay sync unavailable right now.';
-    target.innerHTML = `
-      <div class="composition-head">
-        <div class="composition-title">My Donations · ${donations.length}</div>
-        <div class="composition-subtitle">${escapeHtml(fmtStat(data.total_turns || 0))} turns donated from this machine. ${escapeHtml(source)} Donations stay in the dataset even after local session logs are deleted.</div>
-      </div>
-      <div class="composition-list" style="max-height:220px; overflow-y:auto">
-        ${donations.map(d => `
-          <div class="composition-row">
-            <div class="composition-label"><code>${escapeHtml(d.submission_id)}</code><small>${escapeHtml((d.submitted_utc || '').slice(0, 10))}</small></div>
-            <div class="composition-value">${escapeHtml(fmtStat(d.turns))} turns</div>
-          </div>
-        `).join('')}
-      </div>
-    `;
-    target.style.display = '';
-  } catch(e) { /* panel is best-effort; never block the flow */ }
-}
 function renderRedactResult(data){
   const stats = data.stats || {};
   const autoStats = {};
@@ -3499,6 +3555,7 @@ function renderSessions(){
   const readyCount = (readyCounts.best || 0) + (readyCounts.good || 0);
   const improveCount = sessions.reduce((count, s) => count + (sessionNeedsMoreTurns(s) ? 1 : 0), 0);
   const donatedTotal = sessions.reduce((count, s) => count + (sessionIsBlockedDonation(s) ? 1 : 0), 0);
+  const discoveredCount = sessions.reduce((count, s) => count + (s.archived_donation ? 0 : 1), 0);
   const agentCounts = agentFamilyCounts();
   const sessionSummaryTitle = `Claude: ${agentCounts.claude}\nCodex: ${agentCounts.codex}\nOther: ${agentCounts.other}`;
   const readySummaryTitle = `Best: ${readyCounts.best || 0}\nExcellent: ${readyCounts.good || 0}`;
@@ -3512,7 +3569,7 @@ function renderSessions(){
   $('sessionCount').setAttribute('aria-label', sessionSummaryTitle);
   $('sessionCount').classList.toggle('active', sessionStatusFilter === 'all');
   $('sessionCount').setAttribute('aria-pressed', sessionStatusFilter === 'all' ? 'true' : 'false');
-  $('sessionCount').innerHTML = `<strong>${sessions.length}</strong><span>found</span>`;
+  $('sessionCount').innerHTML = `<strong>${discoveredCount}</strong><span>found</span>`;
   $('fitSummary').innerHTML = sessions.length
     ? `<button type="button" class="fit-chip donated${sessionStatusFilter === 'donated' ? ' active' : ''}" data-session-filter="donated" data-tooltip="${escapeHtml(donatedSummaryTitle)}" aria-label="${escapeHtml(donatedSummaryTitle)}" aria-pressed="${sessionStatusFilter === 'donated' ? 'true' : 'false'}">Donated ${donatedDisplay}</button><button type="button" class="fit-chip ready${sessionStatusFilter === 'ready' ? ' active' : ''}" data-session-filter="ready" data-tooltip="${escapeHtml(readySummaryTitle)}" aria-label="${escapeHtml(readySummaryTitle)}" aria-pressed="${sessionStatusFilter === 'ready' ? 'true' : 'false'}">Ready ${readyCount}</button><button type="button" class="fit-chip improve${sessionStatusFilter === 'improve' ? ' active' : ''}" data-session-filter="improve" data-tooltip="Not ready yet: needs more turns or a context compaction" aria-label="Not ready yet: needs more turns or a context compaction" aria-pressed="${sessionStatusFilter === 'improve' ? 'true' : 'false'}">Keep chatting ${improveCount}</button>`
     : '';
@@ -3688,11 +3745,14 @@ async function discoverSessions(){
     donatedLifetime = (final && final.donated_lifetime) || 0;
     page = 0;
     discoverTiming = `Completed in ${fmtElapsed(Date.now() - progressTimers.discoverProgress.start)}`;
-    status('discoverStatus', sessions.length === 0
+    const discovered = sessions.filter(s => !s.archived_donation).length;
+    const archived = sessions.length - discovered;
+    const archivedNote = archived ? ` Includes ${archived} donated session${archived === 1 ? '' : 's'} whose local logs were deleted (kept in the dataset).` : '';
+    status('discoverStatus', discovered === 0 && !archived
       ? noSessionsMessage()
       : (!relayStatusChecked()
-        ? `Found ${sessions.length} sessions. Donation status could not be checked with the relay, so previously donated sessions may not be marked here. You can still pick a session to donate.`
-        : (allSessionsDonated() ? allSessionsDonatedMessage() : `Found ${sessions.length} sessions. Click a row to select.`)));
+        ? `Found ${discovered} sessions. Donation status could not be checked with the relay, so previously donated sessions may not be marked here. You can still pick a session to donate.`
+        : (allSessionsDonated() ? allSessionsDonatedMessage() : `Found ${discovered} sessions.${archivedNote} Click a row to select.`)));
     renderSessions();
   } catch(e) { status('discoverStatus','ERROR: '+friendlyRequestError(e, 'discovery scan')); }
   finally {
@@ -3996,7 +4056,6 @@ $('submitBtn').onclick = async () => {
   }
 };
 loadProjectStats();
-loadMyDonations();
 discoverSessions();
 </script>
 </body>
@@ -4087,7 +4146,7 @@ class Handler(BaseHTTPRequestHandler):
             max_per_agent = None if raw_max == "all" else int(raw_max)
             sessions = discover_mod.discover(max_per_agent=max_per_agent, progress=False)
             self._json({
-                "sessions": annotate_donated(sessions),
+                "sessions": merge_archived_donations(annotate_donated(sessions)),
                 "donated_lifetime": donated_lifetime_count(),
             })
             return
@@ -4109,7 +4168,7 @@ class Handler(BaseHTTPRequestHandler):
                 for event in discover_mod.discover_iter(max_per_agent=max_per_agent):
                     if event.get("event") == "done":
                         event = dict(event)
-                        event["sessions"] = annotate_donated(list(event.get("sessions") or []))
+                        event["sessions"] = merge_archived_donations(annotate_donated(list(event.get("sessions") or [])))
                         event["donated_lifetime"] = donated_lifetime_count()
                     self._write_body((json.dumps(event) + "\n").encode(), stream=True)
             except ClientDisconnected: