8900 commited on
Commit
8a58dbf
Β·
verified Β·
1 Parent(s): 15be3eb

Update hf-sync-manager.mjs

Browse files
Files changed (1) hide show
  1. hf-sync-manager.mjs +119 -122
hf-sync-manager.mjs CHANGED
@@ -13,50 +13,50 @@
13
  // Excluded: openclaw.json, sessions, qmd, canvas, *.bak
14
  // ============================================================
15
 
16
- import fs from β€œnode:fs”;
17
- import path from β€œnode:path”;
18
- import { execSync } from β€œnode:child_process”;
19
 
20
  // –– config β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”
21
 
22
- var HOME = process.env.OPENCLAW_HOME || process.env.HOME || β€œ/home/user”;
23
- var OPENCLAW_DIR = path.join(HOME, β€œ.openclaw”);
24
- var WORKSPACE = path.join(OPENCLAW_DIR, β€œworkspace”);
25
- var HF_TOKEN = (process.env.HF_TOKEN || β€œβ€).trim();
26
- var DATASET_ID = (process.env.HF_DATASET_ID || β€œβ€).trim();
27
- var REPO_DIR = β€œ/tmp/oc-dataset”;
28
  var INTERVAL = 30 * 60 * 1000;
29
- var RUNNING_FLAG = β€œ/tmp/.hf-sync-running”;
30
 
31
  var EXCLUDE_NAMES = [
32
- β€œopenclaw.json”,
33
- β€œopenclaw.json.bak”,
34
- β€œsessions”,
35
- β€œqmd”,
36
- β€œcanvas”,
37
- β€œ.git”
38
  ];
39
- var EXCLUDE_EXT = [”.bak”, β€œ.tmp”, β€œ.log”];
40
 
41
  // –– helpers ———————————————–
42
 
43
- function log(msg) { console.log(”[hf-sync] β€œ + msg); }
44
 
45
  function sleep(ms) {
46
  return new Promise(function(r) { setTimeout(r, ms); });
47
  }
48
 
49
  function repoUrl() {
50
- return β€œhttps://user:” + HF_TOKEN +
51
- β€œ@huggingface.co/datasets/” + DATASET_ID;
52
  }
53
 
54
  function git(args) {
55
- return execSync(β€œgit β€œ + args, {
56
  cwd: REPO_DIR,
57
- stdio: β€œpipe”,
58
  timeout: 120000,
59
- env: Object.assign({}, process.env, { GIT_TERMINAL_PROMPT: β€œ0” })
60
  }).toString().trim();
61
  }
62
 
@@ -74,49 +74,49 @@ return false;
74
  // –– git ––––––––––––––––––––––––––
75
 
76
  function ensureRepo() {
77
- if (fs.existsSync(path.join(REPO_DIR, β€œ.git”))) return;
78
- execSync(β€œrm -rf β€œ + REPO_DIR, { stdio: β€œpipe” });
79
  try {
80
- execSync(β€œgit clone –depth 1 β€œ + repoUrl() + β€œ β€œ + REPO_DIR, {
81
- stdio: β€œpipe”, timeout: 120000,
82
- env: Object.assign({}, process.env, { GIT_TERMINAL_PROMPT: β€œ0” })
83
  });
84
- log(β€œRepo cloned”);
85
  } catch (e) {
86
- log(β€œClone failed, init empty: β€œ + e.message);
87
  fs.mkdirSync(REPO_DIR, { recursive: true });
88
- execSync(β€œgit init β€œ + REPO_DIR, { stdio: β€œpipe” });
89
- git(β€œremote add origin β€œ + repoUrl());
90
  }
91
  try {
92
- git(β€œconfig user.email openclaw@hf.space”);
93
- git(β€œconfig user.name OpenClaw-Sync”);
94
- git(β€œconfig pull.rebase false”);
95
  } catch (e) { /* non-fatal */ }
96
  }
97
 
98
  function pull() {
99
- try { git(β€œfetch –quiet origin”); } catch (e) { /* ignore */ }
100
- try { git(β€œpull –quiet –no-rebase origin main”); }
101
  catch (e) {
102
- try { git(β€œpull –quiet –no-rebase origin master”); }
103
  catch (e2) { /* empty repo */ }
104
  }
105
  }
106
 
107
  function push() {
108
  try {
109
- git(β€œadd -A”);
110
- var changed = git(β€œdiff –cached –name-only”);
111
  if (!changed) return 0;
112
- var n = changed.split(”\n”).filter(Boolean).length;
113
- var ts = new Date().toISOString().replace(β€œT”, β€œ β€œ).substring(0, 19);
114
- git(β€œcommit -m "sync β€œ + ts + β€œ"”);
115
- try { git(β€œpush –quiet origin HEAD:main”); }
116
- catch (e) { git(β€œpush –quiet origin HEAD:master”); }
117
  return n;
118
  } catch (e) {
119
- log(β€œPush failed: β€œ + e.message);
120
  return 0;
121
  }
122
  }
@@ -139,7 +139,7 @@ fs.copyFileSync(src, dst);
139
  count++;
140
  }
141
  } catch (e) {
142
- log(β€œCopy error [” + name + β€œ]: β€œ + e.message);
143
  }
144
  });
145
  return count;
@@ -161,7 +161,7 @@ fs.copyFileSync(src, dst);
161
  count++;
162
  }
163
  } catch (e) {
164
- log(β€œRestore error [” + name + β€œ]: β€œ + e.message);
165
  }
166
  });
167
  return count;
@@ -171,59 +171,59 @@ return count;
171
 
172
  function seedWorkspaceIfNeeded() {
173
  fs.mkdirSync(WORKSPACE, { recursive: true });
174
- fs.mkdirSync(path.join(WORKSPACE, β€œmemory”), { recursive: true });
175
 
176
- var soul = path.join(WORKSPACE, β€œSOUL.md”);
177
  if (!fs.existsSync(soul)) {
178
  fs.writeFileSync(soul, [
179
- β€œ# Soul”,
180
- β€œβ€,
181
- β€œYou are a helpful, warm, concise AI assistant.”,
182
- β€œβ€,
183
- β€œ## Language”,
184
- β€œβ€,
185
- β€œDefault language: Simplified Chinese.”,
186
- β€œAlways reply in Chinese unless the user writes in another language first.”,
187
- β€œβ€,
188
- β€œ## Tone”,
189
- β€œβ€,
190
- β€œ- Natural and friendly, not overly formal”,
191
- β€œ- Concise and to the point”,
192
- β€œ- Ask one clarifying question at a time when needed”
193
- ].join(”\n”) + β€œ\n”, β€œutf-8”);
194
- log(β€œSeeded workspace/SOUL.md”);
195
- }
196
-
197
- var agents = path.join(WORKSPACE, β€œAGENTS.md”);
198
  if (!fs.existsSync(agents)) {
199
  fs.writeFileSync(agents, [
200
- β€œ# Agent Instructions”,
201
- β€œβ€,
202
- β€œ## Boot sequence”,
203
- β€œβ€,
204
- β€œ1. Read SOUL.md - language and persona”,
205
- β€œ2. Read USER.md if present - user profile”,
206
- β€œ3. Read MEMORY.md - long-term facts and rules”,
207
- β€œ4. Read today and yesterday memory/YYYY-MM-DD.md if present”,
208
- β€œβ€,
209
- β€œ## Memory rules”,
210
- β€œβ€,
211
- β€œ- Write important facts to MEMORY.md when asked”,
212
- β€œ- Log daily context to memory/YYYY-MM-DD.md”,
213
- β€œ- Do not ask for information already provided”
214
- ].join(”\n”) + β€œ\n”, β€œutf-8”);
215
- log(β€œSeeded workspace/AGENTS.md”);
216
- }
217
-
218
- var mem = path.join(WORKSPACE, β€œMEMORY.md”);
219
  if (!fs.existsSync(mem)) {
220
  fs.writeFileSync(mem, [
221
- β€œ# Long-term Memory”,
222
- β€œβ€,
223
- β€œ<!-- OpenClaw writes important facts here. -->”,
224
- β€œ<!-- Loaded at the start of every session. -->”
225
- ].join(”\n”) + β€œ\n”, β€œutf-8”);
226
- log(β€œSeeded workspace/MEMORY.md”);
227
  }
228
  }
229
 
@@ -231,31 +231,31 @@ log(β€œSeeded workspace/MEMORY.md”);
231
 
232
  function bootRestore() {
233
  if (!HF_TOKEN || !DATASET_ID) {
234
- log(β€œNo HF_TOKEN/HF_DATASET_ID - skipping Dataset restore”);
235
  seedWorkspaceIfNeeded();
236
  return;
237
  }
238
 
239
- log(β€œBoot restore from β€œ + DATASET_ID + β€œβ€¦β€);
240
  try {
241
  ensureRepo();
242
  pull();
243
  } catch (e) {
244
- log(β€œRepo init failed (” + e.message + β€œ) - starting fresh”);
245
  seedWorkspaceIfNeeded();
246
  return;
247
  }
248
 
249
- var repoOC = path.join(REPO_DIR, β€œopenclaw”);
250
  if (fs.existsSync(repoOC)) {
251
  var n = mirrorFromRepo(repoOC, OPENCLAW_DIR);
252
- log(β€œBoot restore done: β€œ + n + β€œ file(s) restored (openclaw.json excluded)”);
253
  } else {
254
- log(β€œNo data in Dataset yet - starting fresh”);
255
  }
256
 
257
  seedWorkspaceIfNeeded();
258
- log(β€œBoot restore complete”);
259
  }
260
 
261
  // –– sync cycle ––––––––––––––––––––––
@@ -263,14 +263,13 @@ log(β€œBoot restore complete”);
263
  function runSync() {
264
  if (!HF_TOKEN || !DATASET_ID) return;
265
 
266
- var ts = new Date().toISOString().replace(β€œT”, οΏ½οΏ½ β€œ).substring(0, 19);
267
- log(β€œSync at β€œ + ts + β€œβ€¦β€);
268
 
269
  try {
270
  ensureRepo();
271
  pull();
272
 
273
- ```
274
  var repoOC = path.join(REPO_DIR, "openclaw");
275
  if (fs.existsSync(repoOC)) {
276
  execSync("rm -rf " + repoOC, { stdio: "pipe" });
@@ -298,55 +297,53 @@ if (pushed > 0) {
298
  } else {
299
  log("Sync done: no changes");
300
  }
301
- ```
302
-
303
  } catch (e) {
304
- log(β€œSync failed (retry in 30 min): β€œ + e.message);
305
  }
306
  }
307
 
308
  // –– main –––––––––––––––––––––––––
309
 
310
  (async function() {
311
- log(β€œStarting…”);
312
- log(β€œOPENCLAW_DIR : β€œ + OPENCLAW_DIR);
313
- log(β€œDATASET : β€œ + (DATASET_ID || β€œNOT SET”));
314
- log(β€œRUNNING_FLAG : β€œ + RUNNING_FLAG);
315
 
316
  fs.mkdirSync(OPENCLAW_DIR, { recursive: true });
317
  fs.mkdirSync(WORKSPACE, { recursive: true });
318
 
319
  try {
320
- execSync(β€œgit config –global user.email openclaw@hf.space”, { stdio: β€œpipe” });
321
- execSync(β€œgit config –global user.name OpenClaw-Sync”, { stdio: β€œpipe” });
322
- execSync(β€œgit config –global http.postBuffer 52428800”, { stdio: β€œpipe” });
323
- execSync(β€œgit config –global pull.rebase false”, { stdio: β€œpipe” });
324
  } catch (e) { /* non-fatal */ }
325
 
326
  var isGatewayRestart = fs.existsSync(RUNNING_FLAG);
327
 
328
  if (isGatewayRestart) {
329
- log(β€œGateway restart - skipping boot restore (data already on disk)”);
330
  } else {
331
- log(β€œContainer start - running full boot restore”);
332
  bootRestore();
333
  }
334
 
335
  // Write flag. NEVER delete it on exit.
336
  // Only container restart clears /tmp.
337
- fs.writeFileSync(RUNNING_FLAG, String(process.pid), β€œutf-8”);
338
- log(β€œRunning flag written”);
339
 
340
  if (!HF_TOKEN || !DATASET_ID) {
341
- log(β€œSync disabled: set HF_TOKEN and HF_DATASET_ID in Secrets”);
342
  return;
343
  }
344
 
345
- log(β€œSync loop starting (first run in 10 min)…”);
346
  await sleep(10 * 60 * 1000);
347
 
348
  while (true) {
349
  runSync();
350
  await sleep(INTERVAL);
351
  }
352
- })().catch(function(e) { console.error(”[hf-sync] Fatal: β€œ + e.message); });
 
13
  // Excluded: openclaw.json, sessions, qmd, canvas, *.bak
14
  // ============================================================
15
 
16
+ import fs from "node:fs";
17
+ import path from "node:path";
18
+ import { execSync } from "node:child_process";
19
 
20
  // –– config β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”
21
 
22
+ var HOME = process.env.OPENCLAW_HOME || process.env.HOME || "/home/user";
23
+ var OPENCLAW_DIR = path.join(HOME, ".openclaw");
24
+ var WORKSPACE = path.join(OPENCLAW_DIR, "workspace");
25
+ var HF_TOKEN = (process.env.HF_TOKEN || "").trim();
26
+ var DATASET_ID = (process.env.HF_DATASET_ID || "").trim();
27
+ var REPO_DIR = "/tmp/oc-dataset";
28
  var INTERVAL = 30 * 60 * 1000;
29
+ var RUNNING_FLAG = "/tmp/.hf-sync-running";
30
 
31
  var EXCLUDE_NAMES = [
32
+ "openclaw.json",
33
+ "openclaw.json.bak",
34
+ "sessions",
35
+ "qmd",
36
+ "canvas",
37
+ ".git"
38
  ];
39
+ var EXCLUDE_EXT = [".bak", ".tmp", ".log"];
40
 
41
  // –– helpers ———————————————–
42
 
43
+ function log(msg) { console.log("[hf-sync] " + msg); }
44
 
45
  function sleep(ms) {
46
  return new Promise(function(r) { setTimeout(r, ms); });
47
  }
48
 
49
  function repoUrl() {
50
+ return "https://user:" + HF_TOKEN +
51
+ "@huggingface.co/datasets/" + DATASET_ID;
52
  }
53
 
54
  function git(args) {
55
+ return execSync("git " + args, {
56
  cwd: REPO_DIR,
57
+ stdio: "pipe",
58
  timeout: 120000,
59
+ env: Object.assign({}, process.env, { GIT_TERMINAL_PROMPT: "0" })
60
  }).toString().trim();
61
  }
62
 
 
74
  // –– git ––––––––––––––––––––––––––
75
 
76
  function ensureRepo() {
77
+ if (fs.existsSync(path.join(REPO_DIR, ".git"))) return;
78
+ execSync("rm -rf " + REPO_DIR, { stdio: "pipe" });
79
  try {
80
+ execSync("git clone --depth 1 " + repoUrl() + " " + REPO_DIR, {
81
+ stdio: "pipe", timeout: 120000,
82
+ env: Object.assign({}, process.env, { GIT_TERMINAL_PROMPT: "0" })
83
  });
84
+ log("Repo cloned");
85
  } catch (e) {
86
+ log("Clone failed, init empty: " + e.message);
87
  fs.mkdirSync(REPO_DIR, { recursive: true });
88
+ execSync("git init " + REPO_DIR, { stdio: "pipe" });
89
+ git("remote add origin " + repoUrl());
90
  }
91
  try {
92
+ git("config user.email openclaw@hf.space");
93
+ git("config user.name OpenClaw-Sync");
94
+ git("config pull.rebase false");
95
  } catch (e) { /* non-fatal */ }
96
  }
97
 
98
  function pull() {
99
+ try { git("fetch --quiet origin"); } catch (e) { /* ignore */ }
100
+ try { git("pull --quiet --no-rebase origin main"); }
101
  catch (e) {
102
+ try { git("pull --quiet --no-rebase origin master"); }
103
  catch (e2) { /* empty repo */ }
104
  }
105
  }
106
 
107
  function push() {
108
  try {
109
+ git("add -A");
110
+ var changed = git("diff --cached --name-only");
111
  if (!changed) return 0;
112
+ var n = changed.split("\n").filter(Boolean).length;
113
+ var ts = new Date().toISOString().replace("T", " ").substring(0, 19);
114
+ git('commit -m "sync ' + ts + '"');
115
+ try { git("push --quiet origin HEAD:main"); }
116
+ catch (e) { git("push --quiet origin HEAD:master"); }
117
  return n;
118
  } catch (e) {
119
+ log("Push failed: " + e.message);
120
  return 0;
121
  }
122
  }
 
139
  count++;
140
  }
141
  } catch (e) {
142
+ log("Copy error [" + name + "]: " + e.message);
143
  }
144
  });
145
  return count;
 
161
  count++;
162
  }
163
  } catch (e) {
164
+ log("Restore error [" + name + "]: " + e.message);
165
  }
166
  });
167
  return count;
 
171
 
172
  function seedWorkspaceIfNeeded() {
173
  fs.mkdirSync(WORKSPACE, { recursive: true });
174
+ fs.mkdirSync(path.join(WORKSPACE, "memory"), { recursive: true });
175
 
176
+ var soul = path.join(WORKSPACE, "SOUL.md");
177
  if (!fs.existsSync(soul)) {
178
  fs.writeFileSync(soul, [
179
+ "# Soul",
180
+ "",
181
+ "You are a helpful, warm, concise AI assistant.",
182
+ "",
183
+ "## Language",
184
+ "",
185
+ "Default language: Simplified Chinese.",
186
+ "Always reply in Chinese unless the user writes in another language first.",
187
+ "",
188
+ "## Tone",
189
+ "",
190
+ "- Natural and friendly, not overly formal",
191
+ "- Concise and to the point",
192
+ "- Ask one clarifying question at a time when needed"
193
+ ].join("\n") + "\n", "utf-8");
194
+ log("Seeded workspace/SOUL.md");
195
+ }
196
+
197
+ var agents = path.join(WORKSPACE, "AGENTS.md");
198
  if (!fs.existsSync(agents)) {
199
  fs.writeFileSync(agents, [
200
+ "# Agent Instructions",
201
+ "",
202
+ "## Boot sequence",
203
+ "",
204
+ "1. Read SOUL.md - language and persona",
205
+ "2. Read USER.md if present - user profile",
206
+ "3. Read MEMORY.md - long-term facts and rules",
207
+ "4. Read today and yesterday memory/YYYY-MM-DD.md if present",
208
+ "",
209
+ "## Memory rules",
210
+ "",
211
+ "- Write important facts to MEMORY.md when asked",
212
+ "- Log daily context to memory/YYYY-MM-DD.md",
213
+ "- Do not ask for information already provided"
214
+ ].join("\n") + "\n", "utf-8");
215
+ log("Seeded workspace/AGENTS.md");
216
+ }
217
+
218
+ var mem = path.join(WORKSPACE, "MEMORY.md");
219
  if (!fs.existsSync(mem)) {
220
  fs.writeFileSync(mem, [
221
+ "# Long-term Memory",
222
+ "",
223
+ "<!-- OpenClaw writes important facts here. -->",
224
+ "<!-- Loaded at the start of every session. -->"
225
+ ].join("\n") + "\n", "utf-8");
226
+ log("Seeded workspace/MEMORY.md");
227
  }
228
  }
229
 
 
231
 
232
  function bootRestore() {
233
  if (!HF_TOKEN || !DATASET_ID) {
234
+ log("No HF_TOKEN/HF_DATASET_ID - skipping Dataset restore");
235
  seedWorkspaceIfNeeded();
236
  return;
237
  }
238
 
239
+ log("Boot restore from " + DATASET_ID + "…");
240
  try {
241
  ensureRepo();
242
  pull();
243
  } catch (e) {
244
+ log("Repo init failed (" + e.message + ") - starting fresh");
245
  seedWorkspaceIfNeeded();
246
  return;
247
  }
248
 
249
+ var repoOC = path.join(REPO_DIR, "openclaw");
250
  if (fs.existsSync(repoOC)) {
251
  var n = mirrorFromRepo(repoOC, OPENCLAW_DIR);
252
+ log("Boot restore done: " + n + " file(s) restored (openclaw.json excluded)");
253
  } else {
254
+ log("No data in Dataset yet - starting fresh");
255
  }
256
 
257
  seedWorkspaceIfNeeded();
258
+ log("Boot restore complete");
259
  }
260
 
261
  // –– sync cycle ––––––––––––––––––––––
 
263
  function runSync() {
264
  if (!HF_TOKEN || !DATASET_ID) return;
265
 
266
+ var ts = new Date().toISOString().replace("T", " ").substring(0, 19);
267
+ log("Sync at " + ts + "…");
268
 
269
  try {
270
  ensureRepo();
271
  pull();
272
 
 
273
  var repoOC = path.join(REPO_DIR, "openclaw");
274
  if (fs.existsSync(repoOC)) {
275
  execSync("rm -rf " + repoOC, { stdio: "pipe" });
 
297
  } else {
298
  log("Sync done: no changes");
299
  }
 
 
300
  } catch (e) {
301
+ log("Sync failed (retry in 30 min): " + e.message);
302
  }
303
  }
304
 
305
  // –– main –––––––––––––––––––––––––
306
 
307
  (async function() {
308
+ log("Starting…");
309
+ log("OPENCLAW_DIR : " + OPENCLAW_DIR);
310
+ log("DATASET : " + (DATASET_ID || "NOT SET"));
311
+ log("RUNNING_FLAG : " + RUNNING_FLAG);
312
 
313
  fs.mkdirSync(OPENCLAW_DIR, { recursive: true });
314
  fs.mkdirSync(WORKSPACE, { recursive: true });
315
 
316
  try {
317
+ execSync("git config --global user.email openclaw@hf.space", { stdio: "pipe" });
318
+ execSync("git config --global user.name OpenClaw-Sync", { stdio: "pipe" });
319
+ execSync("git config --global http.postBuffer 52428800", { stdio: "pipe" });
320
+ execSync("git config --global pull.rebase false", { stdio: "pipe" });
321
  } catch (e) { /* non-fatal */ }
322
 
323
  var isGatewayRestart = fs.existsSync(RUNNING_FLAG);
324
 
325
  if (isGatewayRestart) {
326
+ log("Gateway restart - skipping boot restore (data already on disk)");
327
  } else {
328
+ log("Container start - running full boot restore");
329
  bootRestore();
330
  }
331
 
332
  // Write flag. NEVER delete it on exit.
333
  // Only container restart clears /tmp.
334
+ fs.writeFileSync(RUNNING_FLAG, String(process.pid), "utf-8");
335
+ log("Running flag written");
336
 
337
  if (!HF_TOKEN || !DATASET_ID) {
338
+ log("Sync disabled: set HF_TOKEN and HF_DATASET_ID in Secrets");
339
  return;
340
  }
341
 
342
+ log("Sync loop starting (first run in 10 min)…");
343
  await sleep(10 * 60 * 1000);
344
 
345
  while (true) {
346
  runSync();
347
  await sleep(INTERVAL);
348
  }
349
+ })().catch(function(e) { console.error("[hf-sync] Fatal: " + e.message); });