File size: 16,365 Bytes
cb3d548
95f0734
cb3d548
 
 
 
e10fb47
cb3d548
 
 
 
3f09a34
 
 
cb3d548
e10fb47
 
 
 
 
 
 
cb3d548
 
 
e10fb47
 
 
9f0db96
9763168
e10fb47
ee83e90
cb3d548
 
a3dd95d
e10fb47
 
a3dd95d
 
e10fb47
 
 
 
 
cb3d548
27a25ce
3f09a34
27a25ce
cb3d548
e10fb47
 
 
 
 
 
 
 
 
 
 
3f09a34
 
e10fb47
 
 
cb3d548
 
27a25ce
9763168
27a25ce
cb3d548
e10fb47
 
 
 
cb3d548
e10fb47
cb3d548
 
a3dd95d
e10fb47
 
a3dd95d
 
cb3d548
e10fb47
 
 
 
 
 
 
cb3d548
 
 
e10fb47
 
 
 
 
 
 
 
cb3d548
 
3f09a34
95f0734
3f09a34
95f0734
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3f09a34
27a25ce
 
 
95f0734
27a25ce
95f0734
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
cb3d548
 
9763168
95f0734
27a25ce
3f09a34
 
 
 
 
 
 
 
 
 
 
e10fb47
3f09a34
 
 
9763168
459ab58
9763168
 
 
 
 
95f0734
9763168
 
 
 
 
 
 
 
 
95f0734
459ab58
 
 
9763168
459ab58
 
9763168
 
 
 
 
 
95f0734
9763168
 
 
 
95f0734
 
9763168
 
 
 
 
 
 
3f09a34
 
95f0734
3f09a34
 
95f0734
3f09a34
 
 
95f0734
9763168
cb3d548
e10fb47
 
 
 
9763168
e10fb47
95f0734
3f09a34
 
95f0734
e10fb47
 
 
9763168
e10fb47
 
 
9763168
 
 
 
 
 
 
 
 
 
cb3d548
 
9763168
9f0db96
e10fb47
 
 
cb3d548
95f0734
 
3f09a34
 
e10fb47
9763168
9f0db96
3f09a34
 
 
 
9763168
cb3d548
95f0734
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3f09a34
9763168
95f0734
 
 
 
 
 
 
 
3f09a34
9763168
3f09a34
95f0734
3f09a34
 
95f0734
3f09a34
 
95f0734
3f09a34
 
 
 
9763168
95f0734
9763168
9f0db96
 
3f09a34
95f0734
 
 
9763168
3f09a34
 
 
9f0db96
3f09a34
 
 
95f0734
3f09a34
 
95f0734
 
9f0db96
 
9763168
3f09a34
 
 
 
 
 
 
 
 
e10fb47
 
 
cb3d548
 
27a25ce
3f09a34
27a25ce
cb3d548
e10fb47
9763168
e10fb47
 
 
 
 
 
 
 
 
9763168
e10fb47
 
95f0734
 
9763168
 
 
e10fb47
 
 
 
 
cb3d548
 
 
e10fb47
 
 
cb3d548
 
ee83e90
9763168
ee83e90
cb3d548
e10fb47
 
cb3d548
9763168
cb3d548
e10fb47
 
cb3d548
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
import * as http from 'node:http';
import { exec, spawn } from 'node:child_process';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { fileURLToPath } from 'node:url';
import axios from 'axios';
import { Resend } from 'resend';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

// ============================================================
// CONFIG
// ============================================================
interface Config {
  RES1_REPO: string;
  GITHUB_PAT: string;
  VERSIONS_DIR: string;
  HEALTH_PORT: number;
  ADMIN_EMAIL: string;
  RESEND_API_KEY: string;
  FROM_EMAIL: string;
}

const CONFIG: Config = {
  RES1_REPO: 'https://github.com/NepsenX/Oracus-AI.git',
  GITHUB_PAT: process.env.GITHUB_PAT || '',
  VERSIONS_DIR: '/app/versions',
  HEALTH_PORT: parseInt(process.env.HEALTH_PORT || '7860', 10),
  ADMIN_EMAIL: 'oracusai.nepsenx@gmail.com',
  RESEND_API_KEY: process.env.RESEND_API_KEY || '',
  FROM_EMAIL: 'onboarding@resend.dev'
};

if (!CONFIG.GITHUB_PAT) {
  console.error('❌ GITHUB_PAT not set!');
  process.exit(1);
}

if (!CONFIG.RESEND_API_KEY) {
  console.warn('⚠️ RESEND_API_KEY not set. Email notifications disabled.');
}

const resend = CONFIG.RESEND_API_KEY ? new Resend(CONFIG.RESEND_API_KEY) : null;

// ============================================================
// EMAIL
// ============================================================
async function sendEmail(subject: string, text: string): Promise<void> {
  if (!resend) {
    console.log(`πŸ“§ [SKIP] Email would be sent: "${subject}"`);
    return;
  }
  try {
    const { data, error } = await resend.emails.send({
      from: CONFIG.FROM_EMAIL,
      to: [CONFIG.ADMIN_EMAIL],
      subject: `[Oracus Runner] ${subject}`,
      text: text
    });
    if (error) console.error('❌ Resend error:', error);
    else console.log(`βœ… Email sent! ID: ${data?.id}`);
  } catch (err: any) {
    console.error(`⚠️ Email failed (ignored): ${err.message}`);
  }
}

// ============================================================
// GIT UTILITIES
// ============================================================
function runGitCommand(cmd: string, cwd: string): Promise<string> {
  return new Promise((resolve, reject) => {
    exec(cmd, { cwd }, (error, stdout, stderr) => {
      if (error) reject(stderr || error.message);
      else resolve(stdout.trim());
    });
  });
}

function getRepoUrlWithPat(repoUrl: string): string {
  if (repoUrl.includes('@')) return repoUrl;
  return repoUrl.replace('https://', `https://${CONFIG.GITHUB_PAT}@`);
}

async function getCurrentCommitSha(repoUrl: string): Promise<string | null> {
  try {
    const urlWithPat = getRepoUrlWithPat(repoUrl);
    const output = await runGitCommand(`git ls-remote ${urlWithPat} main`, process.cwd());
    return output.split(/\s/)[0];
  } catch {
    return null;
  }
}

async function cloneOrPull(repoUrl: string, targetDir: string): Promise<void> {
  const urlWithPat = getRepoUrlWithPat(repoUrl);
  if (!fs.existsSync(targetDir)) {
    console.log(`πŸ“₯ Cloning into ${targetDir}...`);
    await runGitCommand(`git clone ${urlWithPat} ${targetDir}`, process.cwd());
  } else {
    console.log(`πŸ”„ Pulling latest...`);
    await runGitCommand(`git pull`, targetDir);
  }
}

// ============================================================
// 🐳 DOCKER UTILITIES FOR RES1
// ============================================================
function execDocker(cmd: string): Promise<string> {
  return new Promise((resolve, reject) => {
    exec(cmd, (error, stdout, stderr) => {
      if (error) reject(stderr || error.message);
      else resolve(stdout.trim());
    });
  });
}

async function buildAndRunRes1(
  versionDir: string,
  containerName: string,
  portMapping: string // e.g., "7000:7000" or "2000:2000"
): Promise<number> {
  const imageTag = `oracus-${Date.now()}`;
  console.log(`🐳 Building Docker image ${imageTag} from ${versionDir}...`);
  await execDocker(`docker build -t ${imageTag} ${versionDir}`);

  console.log(`🐳 Running container ${containerName} with port ${portMapping}...`);
  await execDocker(
    `docker run -d --name ${containerName} -p ${portMapping} ${imageTag}`
  );

  // Find the actual host port mapped
  const inspect = await execDocker(
    `docker inspect ${containerName} --format='{{range $p, $conf := .NetworkSettings.Ports}}{{(index $conf 0).HostPort}} {{end}}'`
  );
  const ports = inspect.trim().split(/\s+/);
  if (ports.length === 0) throw new Error('No ports exposed');
  const hostPort = parseInt(ports[0], 10);
  console.log(`βœ… Container running on host port ${hostPort}`);
  return hostPort;
}

async function stopAndRemoveContainer(containerName: string): Promise<void> {
  try {
    await execDocker(`docker stop ${containerName}`);
    await execDocker(`docker rm ${containerName}`);
    console.log(`🧹 Removed container ${containerName}`);
  } catch (e) {
    console.warn(`⚠️ Could not remove container ${containerName}:`, e);
  }
}

// ============================================================
// πŸ€– DEPLOY AGENT (USING DOCKER)
// ============================================================
async function deployAgentWithDocker(
  versionDir: string,
  containerName: string,
  port: number
): Promise<number> {
  // Res1's Dockerfile exposes ports; we map the same port internally
  // but let Docker assign a random host port (-P) or we can map explicitly.
  // Here we use explicit mapping: hostPort:containerPort
  // Since Res1 listens on the port we give (via PORT env), we map that.
  // But Res1's Dockerfile may have multiple EXPOSE; we'll use the first one.
  // Simpler: just use -P (random host port) and inspect.
  // But we need health check on that port.
  // We'll build with PORT env set in the container.
  // Actually, we can pass -e PORT=7000 during docker run.
  // But Res1's code reads PORT env. So we set it.
  const imageTag = `oracus-${Date.now()}`;
  console.log(`🐳 Building Docker image ${imageTag} from ${versionDir}...`);
  await execDocker(`docker build -t ${imageTag} ${versionDir}`);

  console.log(`🐳 Running container ${containerName} with PORT=${port}...`);
  // Run with -P to let Docker assign random host ports, but we also set PORT env.
  await execDocker(
    `docker run -d --name ${containerName} -P -e PORT=${port} ${imageTag}`
  );

  // Inspect to get mapped host port for the container's exposed port
  const inspect = await execDocker(
    `docker inspect ${containerName} --format='{{range $p, $conf := .NetworkSettings.Ports}}{{(index $conf 0).HostPort}} {{end}}'`
  );
  const ports = inspect.trim().split(/\s+/);
  if (ports.length === 0) throw new Error('No ports exposed by container');
  const hostPort = parseInt(ports[0], 10);
  console.log(`βœ… Container running on host port ${hostPort}`);

  // Health check on hostPort
  console.log(`⏳ Waiting for health check on port ${hostPort}...`);
  let healthy = false;
  for (let attempt = 0; attempt < 20; attempt++) {
    try {
      const res = await axios.get(`http://localhost:${hostPort}/health`, { timeout: 2000 });
      if (res.status === 200) {
        healthy = true;
        break;
      }
    } catch (e) {
      // ignore
    }
    await new Promise(r => setTimeout(r, 1000));
  }
  if (!healthy) {
    throw new Error(`Health check failed on port ${hostPort}`);
  }
  console.log(`βœ… Health check passed on port ${hostPort}`);
  return hostPort;
}

// ============================================================
// MEMORY COPY (between version directories)
// ============================================================
function copyMemoryFolder(srcPath: string, dstPath: string): void {
  const srcMemory = path.join(srcPath, 'memory');
  const dstMemory = path.join(dstPath, 'memory');
  if (fs.existsSync(srcMemory)) {
    if (fs.existsSync(dstMemory)) fs.rmSync(dstMemory, { recursive: true, force: true });
    fs.cpSync(srcMemory, dstMemory, { recursive: true });
    console.log(`πŸ“‚ Memory copied from ${srcPath} to ${dstPath}`);
  } else {
    console.log(`⚠️ No memory folder found in ${srcPath}`);
  }
}

// ============================================================
// FAILED VERSIONS TRACKING (MAX 100)
// ============================================================
const FAILED_VERSIONS_FILE = '/app/failed-versions.json';
const MAX_FAILED_VERSIONS = 100;

function loadFailedVersions(): string[] {
  try {
    if (fs.existsSync(FAILED_VERSIONS_FILE)) {
      const data = fs.readFileSync(FAILED_VERSIONS_FILE, 'utf-8');
      return JSON.parse(data);
    }
  } catch (e) {
    console.warn('⚠️ Failed to load failed-versions.json');
  }
  return [];
}

function saveFailedVersion(sha: string): void {
  try {
    let failed = loadFailedVersions().filter(v => v !== sha);
    failed.push(sha);
    if (failed.length > MAX_FAILED_VERSIONS) {
      failed = failed.slice(-MAX_FAILED_VERSIONS);
    }
    fs.writeFileSync(FAILED_VERSIONS_FILE, JSON.stringify(failed, null, 2));
    console.log(`πŸ’Ύ Failed version ${sha} saved. Total failed: ${failed.length}`);
  } catch (e) {
    console.error('❌ Failed to save failed version:', e);
  }
}

function isVersionFailed(sha: string): boolean {
  return loadFailedVersions().includes(sha);
}

function clearFailedVersion(sha: string): void {
  try {
    const failed = loadFailedVersions().filter(v => v !== sha);
    fs.writeFileSync(FAILED_VERSIONS_FILE, JSON.stringify(failed, null, 2));
    console.log(`πŸ—‘οΈ Failed version ${sha} removed from tracking list.`);
  } catch (e) {
    console.error('❌ Failed to clear failed version:', e);
  }
}

// ============================================================
// STATE
// ============================================================
let prodContainerName: string | null = null;
let prodVersionPath: string | null = null;
let prodCommitSha: string | null = null;
let prodHostPort: number | null = null;
let isUpdating: boolean = false;

// ============================================================
// 🧠 MAIN UPDATE LOGIC (Docker-based)
// ============================================================
async function checkAndUpdate(): Promise<void> {
  if (isUpdating) {
    console.log('⏳ Update already in progress...');
    return;
  }

  isUpdating = true;
  let testContainerName: string | null = null;
  let testVersionPath: string | null = null;
  let testCommitSha: string | null = null;
  let testHostPort: number | null = null;

  try {
    console.log('πŸ” Checking for updates...');

    const latestSha = await getCurrentCommitSha(CONFIG.RES1_REPO);
    if (!latestSha) throw new Error('Could not fetch latest SHA');

    console.log(`πŸ“Œ Latest SHA: ${latestSha.substring(0, 7)}`);
    console.log(`πŸ“Œ Current Prod SHA: ${prodCommitSha ? prodCommitSha.substring(0, 7) : 'none'}`);

    if (latestSha === prodCommitSha) {
      console.log('βœ… No new code. Current version is already running.');
      if (prodCommitSha && isVersionFailed(prodCommitSha)) {
        clearFailedVersion(prodCommitSha);
      }
      isUpdating = false;
      return;
    }

    if (isVersionFailed(latestSha)) {
      console.log(`⏳ Version ${latestSha.substring(0, 7)} is marked as FAILED. Waiting for a newer version.`);
      isUpdating = false;
      return;
    }

    console.log(`πŸ†• New version: ${latestSha.substring(0, 7)}. Deploying test container on port 2000...`);

    testCommitSha = latestSha;
    testVersionPath = path.join(CONFIG.VERSIONS_DIR, `v2_${Date.now()}`);
    await cloneOrPull(CONFIG.RES1_REPO, testVersionPath);

    // Copy memory from production if exists
    if (prodVersionPath) {
      copyMemoryFolder(prodVersionPath, testVersionPath);
    } else {
      console.log('⚠️ No previous version. Starting fresh without memory copy.');
    }

    // Deploy test container using Res1's Dockerfile, mapping internal 2000 to host
    // But we use -P, so we don't need to specify host port; Docker assigns random.
    // However, Res1's Dockerfile must have EXPOSE 2000. It does.
    // We'll run with PORT=2000 env.
    testContainerName = `oracus-test-${Date.now()}`;
    testHostPort = await deployAgentWithDocker(testVersionPath, testContainerName, 2000);
    console.log(`βœ… Test container running on host port ${testHostPort} (internal 2000)`);

    // Promote to production (port 7000)
    console.log(`πŸš€ Promoting to production (internal port 7000)...`);

    // Stop and remove test container
    if (testContainerName) {
      await stopAndRemoveContainer(testContainerName);
      testContainerName = null;
    }

    // Build and run production container with internal port 7000
    const prodContainerNameNew = `oracus-prod-${Date.now()}`;
    const prodHostPortNew = await deployAgentWithDocker(testVersionPath, prodContainerNameNew, 7000);
    console.log(`βœ… Production container running on host port ${prodHostPortNew} (internal 7000)`);

    // Stop old production container
    if (prodContainerName) {
      await stopAndRemoveContainer(prodContainerName);
    }

    // Update state
    prodContainerName = prodContainerNameNew;
    prodVersionPath = testVersionPath;
    prodCommitSha = testCommitSha;
    prodHostPort = prodHostPortNew;
    testVersionPath = null;
    testCommitSha = null;
    testHostPort = null;

    if (prodCommitSha && isVersionFailed(prodCommitSha)) {
      clearFailedVersion(prodCommitSha);
    }

    console.log(`βœ… Successfully deployed V2 (${latestSha.substring(0, 7)}) on host port ${prodHostPort}`);

  } catch (error: any) {
    console.error('❌ Deployment failed:', error);

    if (testContainerName) {
      await stopAndRemoveContainer(testContainerName);
      testContainerName = null;
    }

    if (testCommitSha) {
      saveFailedVersion(testCommitSha);
    }

    await sendEmail(
      'Deployment FAILED',
      `Version: ${testCommitSha ? testCommitSha.substring(0, 7) : 'unknown'}\nError: ${error.message}\nPort ${prodHostPort || '?'} is unchanged.`
    );

    if (prodContainerName) {
      console.log(`πŸ”„ Keeping old version running on host port ${prodHostPort}`);
    } else {
      console.log('⚠️ No previous version available.');
    }

    if (testVersionPath && fs.existsSync(testVersionPath)) {
      try {
        fs.rmSync(testVersionPath, { recursive: true, force: true });
        console.log(`🧹 Cleaned up test directory: ${testVersionPath}`);
      } catch (e) {
        // ignore
      }
    }
  } finally {
    isUpdating = false;
  }
}

// ============================================================
// HTTP SERVER (Health check for UptimeRobot)
// ============================================================
const server = http.createServer(async (req, res) => {
  const url = new URL(req.url || '/', `http://${req.headers.host}`);

  if (url.pathname === '/') {
    res.writeHead(200, { 'Content-Type': 'text/plain' });
    res.end('Oracus Runner is Alive!');
    setImmediate(() => checkAndUpdate());
  } else if (url.pathname === '/force-update') {
    res.writeHead(200);
    res.end('Force update triggered.');
    setImmediate(() => checkAndUpdate());
  } else if (url.pathname === '/status') {
    const failed = loadFailedVersions();
    res.writeHead(200, { 'Content-Type': 'application/json' });
    res.end(JSON.stringify({
      prodRunning: prodContainerName !== null,
      prodHostPort: prodHostPort,
      prodVersion: prodCommitSha ? prodCommitSha.substring(0, 7) : 'none',
      isUpdating,
      failedVersions: failed.map(s => s.substring(0, 7))
    }, null, 2));
  } else {
    res.writeHead(404);
    res.end('Not Found');
  }
});

server.listen(CONFIG.HEALTH_PORT, '0.0.0.0', () => {
  console.log(`βœ… Health server running on port ${CONFIG.HEALTH_PORT}`);
  console.log('⏳ Initial setup...');
  checkAndUpdate();
});

// ============================================================
// CRASH HANDLING
// ============================================================
process.on('uncaughtException', (err) => {
  console.error('πŸ’₯ Uncaught Exception:', err);
  sendEmail('CRASH - Uncaught Exception', err.stack || err.message);
});

process.on('unhandledRejection', (reason) => {
  console.error('πŸ’₯ Unhandled Rejection:', reason);
  sendEmail('CRASH - Unhandled Rejection', String(reason));
});