fix
Browse files- lib/docker/manager.ts +45 -0
- lib/env-config.ts +3 -1
- lib/fs/isolation.ts +2 -8
- lib/hf/storage.ts +24 -14
- server.ts +12 -3
lib/docker/manager.ts
CHANGED
|
@@ -176,6 +176,8 @@ async function performProvisioning(config: WorkspaceConfig): Promise<WorkspaceOp
|
|
| 176 |
|
| 177 |
if (!fs.existsSync(workspacePath)) {
|
| 178 |
fs.mkdirSync(workspacePath, { recursive: true });
|
|
|
|
|
|
|
| 179 |
log(`Allocated isolated filesystem segment: ${config.id.slice(0, 8)}`);
|
| 180 |
}
|
| 181 |
|
|
@@ -306,6 +308,49 @@ export async function startWorkspaceContainer(config: WorkspaceConfig): Promise<
|
|
| 306 |
return await pending;
|
| 307 |
}
|
| 308 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 309 |
/**
|
| 310 |
* 🟢 ENGINE WATCHDOG: Background health monitor for native IDE processes.
|
| 311 |
*/
|
|
|
|
| 176 |
|
| 177 |
if (!fs.existsSync(workspacePath)) {
|
| 178 |
fs.mkdirSync(workspacePath, { recursive: true });
|
| 179 |
+
// Store full ID for reliable reconnection after server restarts
|
| 180 |
+
fs.writeFileSync(path.join(workspacePath, '.codeverse-id'), config.id);
|
| 181 |
log(`Allocated isolated filesystem segment: ${config.id.slice(0, 8)}`);
|
| 182 |
}
|
| 183 |
|
|
|
|
| 308 |
return await pending;
|
| 309 |
}
|
| 310 |
|
| 311 |
+
/**
|
| 312 |
+
* 🛠️ SELF-HEALING: Scans for running code-server instances to repopulate the proxy map.
|
| 313 |
+
* This allows the IDE to survive server restarts or cold boots by probing active ports.
|
| 314 |
+
*/
|
| 315 |
+
export async function reconnectRunningWorkspaces() {
|
| 316 |
+
const { execSync } = require('child_process');
|
| 317 |
+
const workspaceRoot = process.env.WORKSPACE_ROOT || '/home/node/w';
|
| 318 |
+
|
| 319 |
+
console.log(`[BOOT] Probing filesystem segment: ${workspaceRoot} for existing sessions...`);
|
| 320 |
+
try {
|
| 321 |
+
// Find all code-server processes
|
| 322 |
+
const output = execSync("ps aux | grep code-server | grep -v grep").toString();
|
| 323 |
+
const lines = output.split('\n');
|
| 324 |
+
|
| 325 |
+
for (const line of lines) {
|
| 326 |
+
// Looking for: ... --bind-addr 127.0.0.1:8548 ... /home/node/w/44c7597c
|
| 327 |
+
const bindMatch = line.match(/--bind-addr 127\.0\.0\.1:(\d+)/);
|
| 328 |
+
const pathMatch = line.match(new RegExp(`${workspaceRoot}/([a-zA-Z0-9]+)`));
|
| 329 |
+
|
| 330 |
+
if (bindMatch && pathMatch) {
|
| 331 |
+
const port = parseInt(bindMatch[1]);
|
| 332 |
+
const shortId = pathMatch[1];
|
| 333 |
+
const fullPath = path.join(workspaceRoot, shortId);
|
| 334 |
+
const idFile = path.join(fullPath, '.codeverse-id');
|
| 335 |
+
|
| 336 |
+
if (fs.existsSync(idFile)) {
|
| 337 |
+
const fullId = fs.readFileSync(idFile, 'utf-8').trim();
|
| 338 |
+
if (!nativeProcesses.has(fullId)) {
|
| 339 |
+
console.log(`[RECONNECT] Identified active IDE ${fullId.slice(0,8)}... on port ${port}. Restoring proxy link.`);
|
| 340 |
+
nativeProcesses.set(fullId, {
|
| 341 |
+
pid: 0, // Placeholder for PID
|
| 342 |
+
port,
|
| 343 |
+
process: { kill: () => { try { execSync(`fuser -k ${port}/tcp`); } catch {} } } as any
|
| 344 |
+
});
|
| 345 |
+
}
|
| 346 |
+
}
|
| 347 |
+
}
|
| 348 |
+
}
|
| 349 |
+
} catch (e) {
|
| 350 |
+
// No processes found or ps failed
|
| 351 |
+
}
|
| 352 |
+
}
|
| 353 |
+
|
| 354 |
/**
|
| 355 |
* 🟢 ENGINE WATCHDOG: Background health monitor for native IDE processes.
|
| 356 |
*/
|
lib/env-config.ts
CHANGED
|
@@ -6,7 +6,7 @@ export const ENV_CONFIG = {
|
|
| 6 |
// 1. Storage & Persistence
|
| 7 |
HF_TOKEN: process.env.HF_TOKEN || process.env.hfToken || process.env.HF_SPACE || process.env.HuggingFaceToken,
|
| 8 |
HF_DATASET_ID: process.env.HF_DATASET_ID || process.env.hfDataset || process.env.HF_DATASET,
|
| 9 |
-
WORKSPACE_ROOT: process.env.WORKSPACE_ROOT || '/home/node/
|
| 10 |
|
| 11 |
// 2. Build Acceleration
|
| 12 |
CACHIX_CACHE_NAME: process.env.CACHIX_CACHE_NAME || 'code-nix',
|
|
@@ -22,6 +22,8 @@ export const ENV_CONFIG = {
|
|
| 22 |
AUTH_SECRET: process.env.AUTH_SECRET || process.env.authSecret,
|
| 23 |
TURSO_URL: process.env.TURSO_URL || process.env.turso_url || process.env.database_url || process.env.TURSO_DATABASE_URL || process.env.DB_URL || process.env.turso_database_url,
|
| 24 |
TURSO_AUTH_TOKEN: process.env.TURSO_AUTH_TOKEN || process.env.turso_auth_token || process.env.DB_TOKEN,
|
|
|
|
|
|
|
| 25 |
};
|
| 26 |
|
| 27 |
/**
|
|
|
|
| 6 |
// 1. Storage & Persistence
|
| 7 |
HF_TOKEN: process.env.HF_TOKEN || process.env.hfToken || process.env.HF_SPACE || process.env.HuggingFaceToken,
|
| 8 |
HF_DATASET_ID: process.env.HF_DATASET_ID || process.env.hfDataset || process.env.HF_DATASET,
|
| 9 |
+
WORKSPACE_ROOT: process.env.WORKSPACE_ROOT || '/home/node/w',
|
| 10 |
|
| 11 |
// 2. Build Acceleration
|
| 12 |
CACHIX_CACHE_NAME: process.env.CACHIX_CACHE_NAME || 'code-nix',
|
|
|
|
| 22 |
AUTH_SECRET: process.env.AUTH_SECRET || process.env.authSecret,
|
| 23 |
TURSO_URL: process.env.TURSO_URL || process.env.turso_url || process.env.database_url || process.env.TURSO_DATABASE_URL || process.env.DB_URL || process.env.turso_database_url,
|
| 24 |
TURSO_AUTH_TOKEN: process.env.TURSO_AUTH_TOKEN || process.env.turso_auth_token || process.env.DB_TOKEN,
|
| 25 |
+
TMPDIR: '/tmp',
|
| 26 |
+
HF_HOME: '/tmp/.cache/huggingface',
|
| 27 |
};
|
| 28 |
|
| 29 |
/**
|
lib/fs/isolation.ts
CHANGED
|
@@ -1,23 +1,17 @@
|
|
| 1 |
import path from "path";
|
| 2 |
import fs from "fs/promises";
|
| 3 |
-
import { existsSync, mkdirSync
|
| 4 |
|
| 5 |
import { ENV_CONFIG } from "@/lib/env-config";
|
| 6 |
|
| 7 |
function resolveWorkspaceBase(): string {
|
| 8 |
// Standardize for production-grade isolation
|
| 9 |
-
const workspaceRoot = ENV_CONFIG.WORKSPACE_ROOT
|
| 10 |
|
| 11 |
try {
|
| 12 |
if (!existsSync(workspaceRoot)) {
|
| 13 |
mkdirSync(workspaceRoot, { recursive: true });
|
| 14 |
}
|
| 15 |
-
|
| 16 |
-
// Critical Test: Check if we can actually write to this path
|
| 17 |
-
const testFile = path.join(/*turbopackIgnore: true*/ workspaceRoot, `.write_test_${Math.random().toString(36).substring(7)}`);
|
| 18 |
-
writeFileSync(testFile, "test");
|
| 19 |
-
unlinkSync(testFile);
|
| 20 |
-
|
| 21 |
return workspaceRoot;
|
| 22 |
} catch (e) {
|
| 23 |
console.error(`[SYSTEM] Critical Storage Error for ${workspaceRoot}:`, e);
|
|
|
|
| 1 |
import path from "path";
|
| 2 |
import fs from "fs/promises";
|
| 3 |
+
import { existsSync, mkdirSync } from "fs";
|
| 4 |
|
| 5 |
import { ENV_CONFIG } from "@/lib/env-config";
|
| 6 |
|
| 7 |
function resolveWorkspaceBase(): string {
|
| 8 |
// Standardize for production-grade isolation
|
| 9 |
+
const workspaceRoot = ENV_CONFIG.WORKSPACE_ROOT;
|
| 10 |
|
| 11 |
try {
|
| 12 |
if (!existsSync(workspaceRoot)) {
|
| 13 |
mkdirSync(workspaceRoot, { recursive: true });
|
| 14 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
return workspaceRoot;
|
| 16 |
} catch (e) {
|
| 17 |
console.error(`[SYSTEM] Critical Storage Error for ${workspaceRoot}:`, e);
|
lib/hf/storage.ts
CHANGED
|
@@ -57,12 +57,18 @@ export class HFStorage {
|
|
| 57 |
const tmpDir = '/tmp/hf-sync';
|
| 58 |
if (!fs.existsSync(tmpDir)) fs.mkdirSync(tmpDir, { recursive: true });
|
| 59 |
|
| 60 |
-
// 🟢 DELTA SYNCING:
|
| 61 |
-
|
| 62 |
-
const
|
| 63 |
|
| 64 |
-
|
| 65 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
onLog?.(`[HF:STORAGE] Profile restoration complete.`);
|
| 67 |
} catch (e: unknown) {
|
| 68 |
const errorMessage = e instanceof Error ? e.message : String(e);
|
|
@@ -76,16 +82,20 @@ export class HFStorage {
|
|
| 76 |
static async syncToDataset(onLog?: (msg: string) => void): Promise<void> {
|
| 77 |
if (!this.HF_TOKEN || !this.HF_DATASET_ID) return;
|
| 78 |
|
| 79 |
-
onLog?.(`[HF:STORAGE] Saving persistent profile to '${this.HF_DATASET_ID}'...`);
|
| 80 |
try {
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
} catch (e: unknown) {
|
| 90 |
const errorMessage = e instanceof Error ? e.message : String(e);
|
| 91 |
onLog?.(`[ERROR] Profile synchronization failed: ${errorMessage}`);
|
|
|
|
| 57 |
const tmpDir = '/tmp/hf-sync';
|
| 58 |
if (!fs.existsSync(tmpDir)) fs.mkdirSync(tmpDir, { recursive: true });
|
| 59 |
|
| 60 |
+
// 🟢 DELTA SYNCING: Only sync the specific workspace and IDE state directories
|
| 61 |
+
const home = process.env.HOME || '/home/node';
|
| 62 |
+
const persistDirs = ['w', '.vscode-server', '.config/code-server'];
|
| 63 |
|
| 64 |
+
for (const dir of persistDirs) {
|
| 65 |
+
const localPath = path.join(home, dir);
|
| 66 |
+
if (!fs.existsSync(localPath)) fs.mkdirSync(localPath, { recursive: true });
|
| 67 |
+
|
| 68 |
+
const cmd = `(command -v huggingface-cli >/dev/null && huggingface-cli download ${this.HF_DATASET_ID} --local-dir ${localPath} --include "${dir}/*" --token ${this.HF_TOKEN}) || (hf download ${this.HF_DATASET_ID} --local-dir ${localPath} --include "${dir}/*" --token ${this.HF_TOKEN})`;
|
| 69 |
+
onLog?.(`[HF:STORAGE] Restoring ${dir} from differential profile...`);
|
| 70 |
+
await this.execAsync(cmd, onLog).catch(() => {}); // Continue if one dir fails
|
| 71 |
+
}
|
| 72 |
onLog?.(`[HF:STORAGE] Profile restoration complete.`);
|
| 73 |
} catch (e: unknown) {
|
| 74 |
const errorMessage = e instanceof Error ? e.message : String(e);
|
|
|
|
| 82 |
static async syncToDataset(onLog?: (msg: string) => void): Promise<void> {
|
| 83 |
if (!this.HF_TOKEN || !this.HF_DATASET_ID) return;
|
| 84 |
|
|
|
|
| 85 |
try {
|
| 86 |
+
onLog?.(`[HF:STORAGE] Saving persistent profile to '${this.HF_DATASET_ID}'...`);
|
| 87 |
+
const home = process.env.HOME || '/home/node';
|
| 88 |
+
const persistDirs = ['w', '.vscode-server', '.config/code-server'];
|
| 89 |
+
|
| 90 |
+
for (const dir of persistDirs) {
|
| 91 |
+
const localPath = path.join(home, dir);
|
| 92 |
+
if (!fs.existsSync(localPath)) continue;
|
| 93 |
+
|
| 94 |
+
const cmd = `(command -v huggingface-cli >/dev/null && huggingface-cli upload ${this.HF_DATASET_ID} ${localPath} ${dir} --token ${this.HF_TOKEN} --message "CodeVerse Sync [${dir}]: ${new Date().toISOString()}" --exclude "node_modules/*" --exclude ".nix/*" --exclude ".direnv/*" --exclude ".cache/*") || (hf upload ${this.HF_DATASET_ID} ${localPath} ${dir} --token ${this.HF_TOKEN})`;
|
| 95 |
+
onLog?.(`[HF:STORAGE] Performing differential backup of ${dir}...`);
|
| 96 |
+
await this.execAsync(cmd, onLog).catch(err => onLog?.(`[WARN] Sync failed for ${dir}: ${err.message}`));
|
| 97 |
+
}
|
| 98 |
+
onLog?.(`[HF:STORAGE] Profile backup successful.`);
|
| 99 |
} catch (e: unknown) {
|
| 100 |
const errorMessage = e instanceof Error ? e.message : String(e);
|
| 101 |
onLog?.(`[ERROR] Profile synchronization failed: ${errorMessage}`);
|
server.ts
CHANGED
|
@@ -12,11 +12,18 @@ import * as pty from "node-pty";
|
|
| 12 |
import os from "os";
|
| 13 |
import { Duplex } from "stream";
|
| 14 |
import { startAutoSleepCron } from "./lib/jobs/auto-sleep";
|
| 15 |
-
import { getNativeWorkspacePort, getAndroidPort, isNativeWorkspaceRunning, prewarmWorkspace } from "./lib/docker/manager";
|
| 16 |
import { initDb } from "./lib/db/schema";
|
| 17 |
-
import { validateEnvironment } from "./lib/env-config";
|
| 18 |
import httpProxy from "http-proxy";
|
| 19 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
const dev = process.env.NODE_ENV !== "production";
|
| 21 |
const app = next({ dev });
|
| 22 |
const handle = app.getRequestHandler();
|
|
@@ -142,7 +149,9 @@ app.prepare()
|
|
| 142 |
.catch(err => console.error("[BOOT] Warmup failed:", err));
|
| 143 |
})
|
| 144 |
.catch(err => console.error("[BOOT] Database init failed:", err));
|
| 145 |
-
|
|
|
|
|
|
|
| 146 |
startAutoSleepCron();
|
| 147 |
|
| 148 |
const server = createServer((req: IncomingMessage, res: ServerResponse) => {
|
|
|
|
| 12 |
import os from "os";
|
| 13 |
import { Duplex } from "stream";
|
| 14 |
import { startAutoSleepCron } from "./lib/jobs/auto-sleep";
|
| 15 |
+
import { getNativeWorkspacePort, getAndroidPort, isNativeWorkspaceRunning, prewarmWorkspace, reconnectRunningWorkspaces } from "./lib/docker/manager";
|
| 16 |
import { initDb } from "./lib/db/schema";
|
| 17 |
+
import { ENV_CONFIG, validateEnvironment } from "./lib/env-config";
|
| 18 |
import httpProxy from "http-proxy";
|
| 19 |
|
| 20 |
+
/**
|
| 21 |
+
* PRODUCTION HARDENING (April 2026): Force writable temp paths for HF Spaces.
|
| 22 |
+
*/
|
| 23 |
+
process.env.TMPDIR = '/tmp';
|
| 24 |
+
process.env.HF_HOME = '/tmp/.cache/huggingface';
|
| 25 |
+
if (!process.env.HOME) process.env.HOME = '/home/node';
|
| 26 |
+
|
| 27 |
const dev = process.env.NODE_ENV !== "production";
|
| 28 |
const app = next({ dev });
|
| 29 |
const handle = app.getRequestHandler();
|
|
|
|
| 149 |
.catch(err => console.error("[BOOT] Warmup failed:", err));
|
| 150 |
})
|
| 151 |
.catch(err => console.error("[BOOT] Database init failed:", err));
|
| 152 |
+
|
| 153 |
+
// 🛠️ Self-Healing: Reconnect to orphans from previous instance
|
| 154 |
+
reconnectRunningWorkspaces();
|
| 155 |
startAutoSleepCron();
|
| 156 |
|
| 157 |
const server = createServer((req: IncomingMessage, res: ServerResponse) => {
|