shubhjn commited on
Commit
a3467e0
·
1 Parent(s): 924843f
lib/docker/manager.ts CHANGED
@@ -176,6 +176,8 @@ async function performProvisioning(config: WorkspaceConfig): Promise<WorkspaceOp
176
 
177
  if (!fs.existsSync(workspacePath)) {
178
  fs.mkdirSync(workspacePath, { recursive: true });
 
 
179
  log(`Allocated isolated filesystem segment: ${config.id.slice(0, 8)}`);
180
  }
181
 
@@ -306,6 +308,49 @@ export async function startWorkspaceContainer(config: WorkspaceConfig): Promise<
306
  return await pending;
307
  }
308
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
309
  /**
310
  * 🟢 ENGINE WATCHDOG: Background health monitor for native IDE processes.
311
  */
 
176
 
177
  if (!fs.existsSync(workspacePath)) {
178
  fs.mkdirSync(workspacePath, { recursive: true });
179
+ // Store full ID for reliable reconnection after server restarts
180
+ fs.writeFileSync(path.join(workspacePath, '.codeverse-id'), config.id);
181
  log(`Allocated isolated filesystem segment: ${config.id.slice(0, 8)}`);
182
  }
183
 
 
308
  return await pending;
309
  }
310
 
311
+ /**
312
+ * 🛠️ SELF-HEALING: Scans for running code-server instances to repopulate the proxy map.
313
+ * This allows the IDE to survive server restarts or cold boots by probing active ports.
314
+ */
315
+ export async function reconnectRunningWorkspaces() {
316
+ const { execSync } = require('child_process');
317
+ const workspaceRoot = process.env.WORKSPACE_ROOT || '/home/node/w';
318
+
319
+ console.log(`[BOOT] Probing filesystem segment: ${workspaceRoot} for existing sessions...`);
320
+ try {
321
+ // Find all code-server processes
322
+ const output = execSync("ps aux | grep code-server | grep -v grep").toString();
323
+ const lines = output.split('\n');
324
+
325
+ for (const line of lines) {
326
+ // Looking for: ... --bind-addr 127.0.0.1:8548 ... /home/node/w/44c7597c
327
+ const bindMatch = line.match(/--bind-addr 127\.0\.0\.1:(\d+)/);
328
+ const pathMatch = line.match(new RegExp(`${workspaceRoot}/([a-zA-Z0-9]+)`));
329
+
330
+ if (bindMatch && pathMatch) {
331
+ const port = parseInt(bindMatch[1]);
332
+ const shortId = pathMatch[1];
333
+ const fullPath = path.join(workspaceRoot, shortId);
334
+ const idFile = path.join(fullPath, '.codeverse-id');
335
+
336
+ if (fs.existsSync(idFile)) {
337
+ const fullId = fs.readFileSync(idFile, 'utf-8').trim();
338
+ if (!nativeProcesses.has(fullId)) {
339
+ console.log(`[RECONNECT] Identified active IDE ${fullId.slice(0,8)}... on port ${port}. Restoring proxy link.`);
340
+ nativeProcesses.set(fullId, {
341
+ pid: 0, // Placeholder for PID
342
+ port,
343
+ process: { kill: () => { try { execSync(`fuser -k ${port}/tcp`); } catch {} } } as any
344
+ });
345
+ }
346
+ }
347
+ }
348
+ }
349
+ } catch (e) {
350
+ // No processes found or ps failed
351
+ }
352
+ }
353
+
354
  /**
355
  * 🟢 ENGINE WATCHDOG: Background health monitor for native IDE processes.
356
  */
lib/env-config.ts CHANGED
@@ -6,7 +6,7 @@ export const ENV_CONFIG = {
6
  // 1. Storage & Persistence
7
  HF_TOKEN: process.env.HF_TOKEN || process.env.hfToken || process.env.HF_SPACE || process.env.HuggingFaceToken,
8
  HF_DATASET_ID: process.env.HF_DATASET_ID || process.env.hfDataset || process.env.HF_DATASET,
9
- WORKSPACE_ROOT: process.env.WORKSPACE_ROOT || '/home/node/app/workspaces',
10
 
11
  // 2. Build Acceleration
12
  CACHIX_CACHE_NAME: process.env.CACHIX_CACHE_NAME || 'code-nix',
@@ -22,6 +22,8 @@ export const ENV_CONFIG = {
22
  AUTH_SECRET: process.env.AUTH_SECRET || process.env.authSecret,
23
  TURSO_URL: process.env.TURSO_URL || process.env.turso_url || process.env.database_url || process.env.TURSO_DATABASE_URL || process.env.DB_URL || process.env.turso_database_url,
24
  TURSO_AUTH_TOKEN: process.env.TURSO_AUTH_TOKEN || process.env.turso_auth_token || process.env.DB_TOKEN,
 
 
25
  };
26
 
27
  /**
 
6
  // 1. Storage & Persistence
7
  HF_TOKEN: process.env.HF_TOKEN || process.env.hfToken || process.env.HF_SPACE || process.env.HuggingFaceToken,
8
  HF_DATASET_ID: process.env.HF_DATASET_ID || process.env.hfDataset || process.env.HF_DATASET,
9
+ WORKSPACE_ROOT: process.env.WORKSPACE_ROOT || '/home/node/w',
10
 
11
  // 2. Build Acceleration
12
  CACHIX_CACHE_NAME: process.env.CACHIX_CACHE_NAME || 'code-nix',
 
22
  AUTH_SECRET: process.env.AUTH_SECRET || process.env.authSecret,
23
  TURSO_URL: process.env.TURSO_URL || process.env.turso_url || process.env.database_url || process.env.TURSO_DATABASE_URL || process.env.DB_URL || process.env.turso_database_url,
24
  TURSO_AUTH_TOKEN: process.env.TURSO_AUTH_TOKEN || process.env.turso_auth_token || process.env.DB_TOKEN,
25
+ TMPDIR: '/tmp',
26
+ HF_HOME: '/tmp/.cache/huggingface',
27
  };
28
 
29
  /**
lib/fs/isolation.ts CHANGED
@@ -1,23 +1,17 @@
1
  import path from "path";
2
  import fs from "fs/promises";
3
- import { existsSync, mkdirSync, writeFileSync, unlinkSync } from "fs";
4
 
5
  import { ENV_CONFIG } from "@/lib/env-config";
6
 
7
  function resolveWorkspaceBase(): string {
8
  // Standardize for production-grade isolation
9
- const workspaceRoot = ENV_CONFIG.WORKSPACE_ROOT || path.join(/*turbopackIgnore: true*/ '/home/nodejs/app/workspaces');
10
 
11
  try {
12
  if (!existsSync(workspaceRoot)) {
13
  mkdirSync(workspaceRoot, { recursive: true });
14
  }
15
-
16
- // Critical Test: Check if we can actually write to this path
17
- const testFile = path.join(/*turbopackIgnore: true*/ workspaceRoot, `.write_test_${Math.random().toString(36).substring(7)}`);
18
- writeFileSync(testFile, "test");
19
- unlinkSync(testFile);
20
-
21
  return workspaceRoot;
22
  } catch (e) {
23
  console.error(`[SYSTEM] Critical Storage Error for ${workspaceRoot}:`, e);
 
1
  import path from "path";
2
  import fs from "fs/promises";
3
+ import { existsSync, mkdirSync } from "fs";
4
 
5
  import { ENV_CONFIG } from "@/lib/env-config";
6
 
7
  function resolveWorkspaceBase(): string {
8
  // Standardize for production-grade isolation
9
+ const workspaceRoot = ENV_CONFIG.WORKSPACE_ROOT;
10
 
11
  try {
12
  if (!existsSync(workspaceRoot)) {
13
  mkdirSync(workspaceRoot, { recursive: true });
14
  }
 
 
 
 
 
 
15
  return workspaceRoot;
16
  } catch (e) {
17
  console.error(`[SYSTEM] Critical Storage Error for ${workspaceRoot}:`, e);
lib/hf/storage.ts CHANGED
@@ -57,12 +57,18 @@ export class HFStorage {
57
  const tmpDir = '/tmp/hf-sync';
58
  if (!fs.existsSync(tmpDir)) fs.mkdirSync(tmpDir, { recursive: true });
59
 
60
- // 🟢 DELTA SYNCING: Use 'download' for raw folder syncing instead of tarballs
61
- // This allows HF CLI to perform block-level diffing internally.
62
- const cmd = `(command -v huggingface-cli >/dev/null && huggingface-cli download ${this.HF_DATASET_ID} --local-dir ${process.env.HOME || '/home/node'} --local-dir-use-symlinks False --token ${this.HF_TOKEN} --include "*" --exclude "node_modules/*" --exclude ".nix/*" --exclude ".direnv/*" --exclude ".cache/*") || (hf download ${this.HF_DATASET_ID} --local-dir ${process.env.HOME || '/home/node'} --token ${this.HF_TOKEN})`;
63
 
64
- onLog?.(`[HF:STORAGE] Restoring differential profile from '${this.HF_DATASET_ID}'...`);
65
- await this.execAsync(cmd, onLog);
 
 
 
 
 
 
66
  onLog?.(`[HF:STORAGE] Profile restoration complete.`);
67
  } catch (e: unknown) {
68
  const errorMessage = e instanceof Error ? e.message : String(e);
@@ -76,16 +82,20 @@ export class HFStorage {
76
  static async syncToDataset(onLog?: (msg: string) => void): Promise<void> {
77
  if (!this.HF_TOKEN || !this.HF_DATASET_ID) return;
78
 
79
- onLog?.(`[HF:STORAGE] Saving persistent profile to '${this.HF_DATASET_ID}'...`);
80
  try {
81
- // 🟢 DELTA SYNCING: Use 'upload-folder' for granular updates
82
- // This skips files that haven't changed, making uploads nearly instant for small edits.
83
- // We MUST exclude .cache/ and other high-volume folders to prevent HF Dataset 'CommitOperation' errors and 'Large Folder' warnings.
84
- const cmd = `(command -v huggingface-cli >/dev/null && huggingface-cli upload ${this.HF_DATASET_ID} ${process.env.HOME || '/home/node'} . --token ${this.HF_TOKEN} --message "CodeVerse Sync: ${new Date().toISOString()}" --exclude "node_modules/*" --exclude ".nix/*" --exclude ".direnv/*" --exclude ".cache/*" --exclude ".npm/*" --exclude ".local/share/code-server/*" --exclude ".vscode-server/extensions/*") || (hf upload ${this.HF_DATASET_ID} ${process.env.HOME || '/home/node'} . --token ${this.HF_TOKEN})`;
85
-
86
- onLog?.(`[HF:STORAGE] Performing differential backup to '${this.HF_DATASET_ID}'...`);
87
- await this.execAsync(cmd, onLog);
88
- onLog?.(`[HF:STORAGE] Profile backup successful.`);
 
 
 
 
 
89
  } catch (e: unknown) {
90
  const errorMessage = e instanceof Error ? e.message : String(e);
91
  onLog?.(`[ERROR] Profile synchronization failed: ${errorMessage}`);
 
57
  const tmpDir = '/tmp/hf-sync';
58
  if (!fs.existsSync(tmpDir)) fs.mkdirSync(tmpDir, { recursive: true });
59
 
60
+ // 🟢 DELTA SYNCING: Only sync the specific workspace and IDE state directories
61
+ const home = process.env.HOME || '/home/node';
62
+ const persistDirs = ['w', '.vscode-server', '.config/code-server'];
63
 
64
+ for (const dir of persistDirs) {
65
+ const localPath = path.join(home, dir);
66
+ if (!fs.existsSync(localPath)) fs.mkdirSync(localPath, { recursive: true });
67
+
68
+ const cmd = `(command -v huggingface-cli >/dev/null && huggingface-cli download ${this.HF_DATASET_ID} --local-dir ${localPath} --include "${dir}/*" --token ${this.HF_TOKEN}) || (hf download ${this.HF_DATASET_ID} --local-dir ${localPath} --include "${dir}/*" --token ${this.HF_TOKEN})`;
69
+ onLog?.(`[HF:STORAGE] Restoring ${dir} from differential profile...`);
70
+ await this.execAsync(cmd, onLog).catch(() => {}); // Continue if one dir fails
71
+ }
72
  onLog?.(`[HF:STORAGE] Profile restoration complete.`);
73
  } catch (e: unknown) {
74
  const errorMessage = e instanceof Error ? e.message : String(e);
 
82
  static async syncToDataset(onLog?: (msg: string) => void): Promise<void> {
83
  if (!this.HF_TOKEN || !this.HF_DATASET_ID) return;
84
 
 
85
  try {
86
+ onLog?.(`[HF:STORAGE] Saving persistent profile to '${this.HF_DATASET_ID}'...`);
87
+ const home = process.env.HOME || '/home/node';
88
+ const persistDirs = ['w', '.vscode-server', '.config/code-server'];
89
+
90
+ for (const dir of persistDirs) {
91
+ const localPath = path.join(home, dir);
92
+ if (!fs.existsSync(localPath)) continue;
93
+
94
+ const cmd = `(command -v huggingface-cli >/dev/null && huggingface-cli upload ${this.HF_DATASET_ID} ${localPath} ${dir} --token ${this.HF_TOKEN} --message "CodeVerse Sync [${dir}]: ${new Date().toISOString()}" --exclude "node_modules/*" --exclude ".nix/*" --exclude ".direnv/*" --exclude ".cache/*") || (hf upload ${this.HF_DATASET_ID} ${localPath} ${dir} --token ${this.HF_TOKEN})`;
95
+ onLog?.(`[HF:STORAGE] Performing differential backup of ${dir}...`);
96
+ await this.execAsync(cmd, onLog).catch(err => onLog?.(`[WARN] Sync failed for ${dir}: ${err.message}`));
97
+ }
98
+ onLog?.(`[HF:STORAGE] Profile backup successful.`);
99
  } catch (e: unknown) {
100
  const errorMessage = e instanceof Error ? e.message : String(e);
101
  onLog?.(`[ERROR] Profile synchronization failed: ${errorMessage}`);
server.ts CHANGED
@@ -12,11 +12,18 @@ import * as pty from "node-pty";
12
  import os from "os";
13
  import { Duplex } from "stream";
14
  import { startAutoSleepCron } from "./lib/jobs/auto-sleep";
15
- import { getNativeWorkspacePort, getAndroidPort, isNativeWorkspaceRunning, prewarmWorkspace } from "./lib/docker/manager";
16
  import { initDb } from "./lib/db/schema";
17
- import { validateEnvironment } from "./lib/env-config";
18
  import httpProxy from "http-proxy";
19
 
 
 
 
 
 
 
 
20
  const dev = process.env.NODE_ENV !== "production";
21
  const app = next({ dev });
22
  const handle = app.getRequestHandler();
@@ -142,7 +149,9 @@ app.prepare()
142
  .catch(err => console.error("[BOOT] Warmup failed:", err));
143
  })
144
  .catch(err => console.error("[BOOT] Database init failed:", err));
145
-
 
 
146
  startAutoSleepCron();
147
 
148
  const server = createServer((req: IncomingMessage, res: ServerResponse) => {
 
12
  import os from "os";
13
  import { Duplex } from "stream";
14
  import { startAutoSleepCron } from "./lib/jobs/auto-sleep";
15
+ import { getNativeWorkspacePort, getAndroidPort, isNativeWorkspaceRunning, prewarmWorkspace, reconnectRunningWorkspaces } from "./lib/docker/manager";
16
  import { initDb } from "./lib/db/schema";
17
+ import { ENV_CONFIG, validateEnvironment } from "./lib/env-config";
18
  import httpProxy from "http-proxy";
19
 
20
+ /**
21
+ * PRODUCTION HARDENING (April 2026): Force writable temp paths for HF Spaces.
22
+ */
23
+ process.env.TMPDIR = '/tmp';
24
+ process.env.HF_HOME = '/tmp/.cache/huggingface';
25
+ if (!process.env.HOME) process.env.HOME = '/home/node';
26
+
27
  const dev = process.env.NODE_ENV !== "production";
28
  const app = next({ dev });
29
  const handle = app.getRequestHandler();
 
149
  .catch(err => console.error("[BOOT] Warmup failed:", err));
150
  })
151
  .catch(err => console.error("[BOOT] Database init failed:", err));
152
+
153
+ // 🛠️ Self-Healing: Reconnect to orphans from previous instance
154
+ reconnectRunningWorkspaces();
155
  startAutoSleepCron();
156
 
157
  const server = createServer((req: IncomingMessage, res: ServerResponse) => {