/** * git-sync.js * Syncs the ./data directory with a private GitHub repository. * * On startup: clones the repo into ./data (or pulls if already cloned) * After writes: debounced git add + commit + push (batches rapid writes) * * Environment variables (set as HF Space Secrets): * GITHUB_TOKEN – GitHub Personal Access Token (with repo scope) * GITHUB_USERNAME – GitHub username * GITHUB_REPO – e.g. "username/usagi-data" * GIT_USER_NAME – (optional) commit author name, default "UsagiBot" * GIT_USER_EMAIL – (optional) commit author email, default "bot@usagi.local" */ const { execSync, exec } = require("child_process"); const path = require("path"); const GITHUB_TOKEN = process.env.GITHUB_TOKEN || ""; const GITHUB_USERNAME = process.env.GITHUB_USERNAME || process.env.GIT_USER_NAME || ""; const GITHUB_REPO = process.env.GITHUB_REPO || ""; const GIT_USER_NAME = process.env.GIT_USER_NAME || "UsagiBot"; const GIT_USER_EMAIL = process.env.GIT_USER_EMAIL || "bot@usagi.local"; // Debounce: wait 15 seconds after last write before pushing const PUSH_DEBOUNCE_MS = parseInt(process.env.PUSH_DEBOUNCE_MS) || 15000; const DATA_DIR = path.join(__dirname, "data"); let _pushTimer = null; let _pushing = false; let _pendingPush = false; function isConfigured() { return !!(GITHUB_TOKEN && GITHUB_USERNAME && GITHUB_REPO); } function getRepoUrl() { return `https://${GITHUB_USERNAME}:${GITHUB_TOKEN}@github.com/${GITHUB_REPO}.git`; } function git(cmd, cwd = DATA_DIR) { return execSync(`git ${cmd}`, { cwd, encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"], timeout: 60000, }).trim(); } function gitAsync(cmd, cwd = DATA_DIR) { return new Promise((resolve, reject) => { exec(`git ${cmd}`, { cwd, encoding: "utf-8", timeout: 60000 }, (err, stdout, stderr) => { if (err) reject(err); else resolve((stdout || "").trim()); }); }); } /** * Clone the data repo on startup, or pull latest if already cloned. * If the repo is empty / doesn't exist yet, initialize it. */ function initSync() { if (!isConfigured()) { console.log("⚠️ Git sync disabled (GITHUB_TOKEN or GITHUB_REPO not set)"); return false; } const fs = require("fs"); try { // Check if data dir is already a git repo const isGitRepo = fs.existsSync(path.join(DATA_DIR, ".git")) || fs.existsSync(path.join(DATA_DIR, ".git", "HEAD")); if (isGitRepo) { console.log("📥 Pulling latest data from GitHub..."); try { git("pull --rebase --autostash origin main"); console.log("✅ Data pulled successfully"); } catch (e) { // If pull fails (e.g. diverged), force reset to remote console.log("⚠️ Pull failed, resetting to remote..."); try { git("fetch origin"); git("reset --hard origin/main"); } catch (e2) { console.log("⚠️ Reset also failed, continuing with local data"); } } } else { // Clone the repo console.log(`📥 Cloning data repo: ${GITHUB_REPO}...`); // Remove data dir if it exists (has local files but no git) const hadLocalData = fs.existsSync(DATA_DIR); const localFiles = {}; if (hadLocalData) { // Backup local files before clone const files = fs.readdirSync(DATA_DIR); for (const f of files) { const fp = path.join(DATA_DIR, f); if (fs.statSync(fp).isFile()) { localFiles[f] = fs.readFileSync(fp, "utf-8"); } } fs.rmSync(DATA_DIR, { recursive: true, force: true }); } try { // Clone into data dir execSync(`git clone ${getRepoUrl()} data`, { cwd: __dirname, encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"], timeout: 60000, }); console.log("✅ Data repo cloned successfully"); } catch (cloneErr) { // If repo is empty or doesn't exist, init fresh console.log("⚠️ Clone failed, initializing fresh repo..."); fs.mkdirSync(DATA_DIR, { recursive: true }); git("init", DATA_DIR); git(`remote add origin ${getRepoUrl()}`, DATA_DIR); git("checkout -b main", DATA_DIR); } // Restore local files that weren't in the cloned repo if (hadLocalData) { for (const [filename, content] of Object.entries(localFiles)) { const fp = path.join(DATA_DIR, filename); if (!fs.existsSync(fp)) { fs.writeFileSync(fp, content, "utf-8"); console.log(` 📄 Restored local file: ${filename}`); } } } } // Configure git user try { git(`config user.name "${GIT_USER_NAME}"`); git(`config user.email "${GIT_USER_EMAIL}"`); } catch (e) {} return true; } catch (e) { console.error("❌ Git sync init failed:", e.message); return false; } } /** * Schedule a debounced push. Call this after every database write. * Multiple rapid writes will be batched into a single commit+push. */ function scheduleSync() { if (!isConfigured()) return; if (_pushTimer) clearTimeout(_pushTimer); _pushTimer = setTimeout(() => { _pushTimer = null; doSync(); }, PUSH_DEBOUNCE_MS); } /** * Actually commit and push all changes. */ async function doSync() { if (_pushing) { _pendingPush = true; return; } _pushing = true; try { // Check for changes const status = await gitAsync("status --porcelain"); if (!status) { _pushing = false; return; // nothing to commit } await gitAsync("add -A"); const timestamp = new Date().toISOString().replace("T", " ").slice(0, 19); const changedFiles = status .split("\n") .map((l) => l.trim().split(/\s+/).pop()) .join(", "); await gitAsync(`commit -m "sync: ${changedFiles} [${timestamp}]"`); // Push with retry let retries = 3; while (retries > 0) { try { await gitAsync("push origin main"); console.log(`📤 Data synced to GitHub (${changedFiles})`); break; } catch (pushErr) { retries--; if (retries > 0) { // Pull and retry try { await gitAsync("pull --rebase origin main"); } catch (e) {} } else { console.error("❌ Push failed after retries:", pushErr.message); } } } } catch (e) { // Don't crash on sync errors if (!e.message?.includes("nothing to commit")) { console.error("❌ Sync error:", e.message); } } finally { _pushing = false; // If another write happened while we were pushing, push again if (_pendingPush) { _pendingPush = false; setTimeout(doSync, 3000); } } } /** * Force an immediate sync (e.g. on shutdown). */ async function forceSync() { if (!isConfigured()) return; if (_pushTimer) { clearTimeout(_pushTimer); _pushTimer = null; } await doSync(); } module.exports = { initSync, scheduleSync, forceSync, isConfigured };