sync-external-storage.mjs

#!/usr/bin/env node
import crypto from "node:crypto";
import fs from "node:fs";
import path from "node:path";

const home = process.env.OPENCLAW_HOME || process.env.HOME || "/home/user";
const stateDir = path.join(home, ".openclaw");
const supabaseUrl = process.env.SUPABASE_URL?.trim();
const supabaseKey = process.env.SUPABASE_KEY?.trim();
const tableName = process.env.OPENCLAW_SUPABASE_TABLE?.trim() || "openclaw_state";
const intervalMs = Number(process.env.OPENCLAW_SYNC_INTERVAL_MS || 300000);
const maxFileBytes = Number(process.env.OPENCLAW_SYNC_MAX_FILE_BYTES || 5 * 1024 * 1024);
const includeGlobs = (process.env.OPENCLAW_SYNC_INCLUDE_EXTENSIONS || ".json,.jsonl,.md,.txt")
  .split(",")
  .map((value) => value.trim().toLowerCase())
  .filter(Boolean);

if (!supabaseUrl || !supabaseKey) {
  console.error("[openclaw-sync] SUPABASE_URL or SUPABASE_KEY is missing");
  process.exit(1);
}

const seenHashes = new Map();

function shouldSyncFile(filePath, stats) {
  if (!stats.isFile()) return false;
  if (stats.size > maxFileBytes) return false;
  const normalized = filePath.toLowerCase();
  return includeGlobs.some((ext) => normalized.endsWith(ext));
}

function listFilesRecursive(rootDir) {
  if (!fs.existsSync(rootDir)) return [];
  const results = [];
  const stack = [rootDir];

  while (stack.length > 0) {
    const currentDir = stack.pop();
    const entries = fs.readdirSync(currentDir, { withFileTypes: true });
    for (const entry of entries) {
      const fullPath = path.join(currentDir, entry.name);
      if (entry.isDirectory()) {
        stack.push(fullPath);
        continue;
      }
      const stats = fs.statSync(fullPath);
      if (shouldSyncFile(fullPath, stats)) {
        results.push({ fullPath, stats });
      }
    }
  }

  return results.sort((a, b) => a.fullPath.localeCompare(b.fullPath));
}

function sha256(content) {
  return crypto.createHash("sha256").update(content).digest("hex");
}

function classifyPath(relativePath) {
  if (relativePath === "openclaw.json") return "config";
  if (relativePath.includes("sessions")) return "session";
  if (relativePath.includes("memory")) return "memory";
  if (relativePath.includes("logs")) return "log";
  return "state";
}

async function upsertRows(rows) {
  const endpoint = `${supabaseUrl.replace(/\/$/, "")}/rest/v1/${tableName}?on_conflict=path`;
  const response = await fetch(endpoint, {
    method: "POST",
    headers: {
      apikey: supabaseKey,
      Authorization: `Bearer ${supabaseKey}`,
      "Content-Type": "application/json",
      Prefer: "resolution=merge-duplicates",
    },
    body: JSON.stringify(rows),
  });

  if (!response.ok) {
    const body = await response.text();
    throw new Error(`Supabase upsert failed (${response.status}): ${body}`);
  }
}

async function syncOnce() {
  const files = listFilesRecursive(stateDir);
  const changedRows = [];

  for (const { fullPath, stats } of files) {
    const relativePath = path.relative(stateDir, fullPath).replaceAll("\\", "/");
    const content = fs.readFileSync(fullPath, "utf-8");
    const hash = sha256(content);
    if (seenHashes.get(relativePath) === hash) continue;

    seenHashes.set(relativePath, hash);
    changedRows.push({
      path: relativePath,
      kind: classifyPath(relativePath),
      content,
      sha256: hash,
      size_bytes: stats.size,
      updated_at: new Date(stats.mtimeMs).toISOString(),
      synced_at: new Date().toISOString(),
    });
  }

  if (changedRows.length === 0) {
    console.log("[openclaw-sync] no changes");
    return;
  }

  for (let index = 0; index < changedRows.length; index += 25) {
    const batch = changedRows.slice(index, index + 25);
    await upsertRows(batch);
  }

  console.log(`[openclaw-sync] synced ${changedRows.length} file(s) from ${stateDir}`);
}

async function main() {
  console.log(
    `[openclaw-sync] provider=supabase table=${tableName} interval_ms=${intervalMs} state_dir=${stateDir}`,
  );

  while (true) {
    try {
      await syncOnce();
    } catch (error) {
      console.error(`[openclaw-sync] ${error instanceof Error ? error.message : String(error)}`);
    }
    await new Promise((resolve) => setTimeout(resolve, intervalMs));
  }
}

main().catch((error) => {
  console.error(`[openclaw-sync] fatal ${error instanceof Error ? error.stack : String(error)}`);
  process.exit(1);
});