Spaces:
Sleeping
Sleeping
| import * as http from 'node:http'; | |
| import { exec, spawn, ChildProcess } from 'node:child_process'; | |
| import * as fs from 'node:fs'; | |
| import * as path from 'node:path'; | |
| import { fileURLToPath } from 'node:url'; | |
| import axios from 'axios'; | |
| import { Resend } from 'resend'; | |
| const __filename = fileURLToPath(import.meta.url); | |
| const __dirname = path.dirname(__filename); | |
| interface Config { | |
| RES1_REPO: string; | |
| GITHUB_PAT: string; | |
| VERSIONS_DIR: string; | |
| HEALTH_PORT: number; | |
| PROD_PORT: number; | |
| TEST_PORT: number; | |
| ADMIN_EMAIL: string; | |
| RESEND_API_KEY: string; | |
| FROM_EMAIL: string; | |
| } | |
| const CONFIG: Config = { | |
| RES1_REPO: 'https://github.com/NepsenX/Oracus-AI.git', | |
| GITHUB_PAT: process.env.GITHUB_PAT || '', | |
| VERSIONS_DIR: '/app/versions', | |
| HEALTH_PORT: 7860, | |
| PROD_PORT: 7000, | |
| TEST_PORT: 2000, | |
| ADMIN_EMAIL: 'oracusai.nepsenx@gmail.com', | |
| RESEND_API_KEY: process.env.RESEND_API_KEY || '', | |
| FROM_EMAIL: 'onboarding@resend.dev' | |
| }; | |
| if (!CONFIG.GITHUB_PAT) { | |
| console.error('β GITHUB_PAT not set!'); | |
| process.exit(1); | |
| } | |
| if (!CONFIG.RESEND_API_KEY) { | |
| console.warn('β οΈ RESEND_API_KEY not set. Email notifications disabled.'); | |
| } | |
| const resend = CONFIG.RESEND_API_KEY ? new Resend(CONFIG.RESEND_API_KEY) : null; | |
| // ============================================================ | |
| // EMAIL FUNCTION (only on failure/crash) | |
| // ============================================================ | |
| async function sendEmail(subject: string, text: string): Promise<void> { | |
| if (!resend) { | |
| console.log(`π§ [SKIP] Email would be sent: "${subject}"`); | |
| return; | |
| } | |
| try { | |
| const { data, error } = await resend.emails.send({ | |
| from: CONFIG.FROM_EMAIL, | |
| to: [CONFIG.ADMIN_EMAIL], | |
| subject: `[Oracus Runner] ${subject}`, | |
| text: text | |
| }); | |
| if (error) { | |
| console.error('β Resend error:', error); | |
| } else { | |
| console.log(`β Email sent! ID: ${data?.id}`); | |
| } | |
| } catch (err: any) { | |
| console.error(`β οΈ Email failed (ignored): ${err.message}`); | |
| } | |
| } | |
| // ============================================================ | |
| // GIT UTILITIES | |
| // ============================================================ | |
| function runGitCommand(cmd: string, cwd: string): Promise<string> { | |
| return new Promise((resolve, reject) => { | |
| exec(cmd, { cwd }, (error, stdout, stderr) => { | |
| if (error) reject(stderr || error.message); | |
| else resolve(stdout.trim()); | |
| }); | |
| }); | |
| } | |
| function getRepoUrlWithPat(repoUrl: string): string { | |
| if (repoUrl.includes('@')) return repoUrl; | |
| return repoUrl.replace('https://', `https://${CONFIG.GITHUB_PAT}@`); | |
| } | |
| async function getCurrentCommitSha(repoUrl: string): Promise<string | null> { | |
| try { | |
| const urlWithPat = getRepoUrlWithPat(repoUrl); | |
| const output = await runGitCommand(`git ls-remote ${urlWithPat} main`, process.cwd()); | |
| return output.split(/\s/)[0]; | |
| } catch { | |
| return null; | |
| } | |
| } | |
| async function cloneOrPull(repoUrl: string, targetDir: string): Promise<void> { | |
| const urlWithPat = getRepoUrlWithPat(repoUrl); | |
| if (!fs.existsSync(targetDir)) { | |
| console.log(`π₯ Cloning into ${targetDir}...`); | |
| await runGitCommand(`git clone ${urlWithPat} ${targetDir}`, process.cwd()); | |
| } else { | |
| console.log(`π Pulling latest...`); | |
| await runGitCommand(`git pull`, targetDir); | |
| } | |
| } | |
| function copyMemoryFolder(srcPath: string, dstPath: string): void { | |
| const srcMemory = path.join(srcPath, 'memory'); | |
| const dstMemory = path.join(dstPath, 'memory'); | |
| if (fs.existsSync(srcMemory)) { | |
| if (fs.existsSync(dstMemory)) fs.rmSync(dstMemory, { recursive: true, force: true }); | |
| fs.cpSync(srcMemory, dstMemory, { recursive: true }); | |
| console.log(`π Memory copied from ${srcPath} to ${dstPath}`); | |
| } else { | |
| console.log(`β οΈ No memory folder found in ${srcPath}`); | |
| } | |
| } | |
| // ============================================================ | |
| // AGENT DEPLOYMENT | |
| // ============================================================ | |
| function deployAgent(versionDir: string, port: number): Promise<ChildProcess> { | |
| return new Promise((resolve, reject) => { | |
| console.log(`π Starting agent on port ${port} from ${versionDir}...`); | |
| exec(`npm install && npm run build`, { cwd: versionDir }, (err) => { | |
| if (err) return reject(`Build failed: ${err.message}`); | |
| const proc = spawn('npm', ['start'], { | |
| cwd: versionDir, | |
| env: { ...process.env, PORT: String(port) }, | |
| stdio: 'pipe' | |
| }); | |
| proc.stdout.on('data', (data) => console.log(`[AGENT:${port}] ${data}`)); | |
| proc.stderr.on('data', (data) => console.error(`[AGENT ERR:${port}] ${data}`)); | |
| setTimeout(async () => { | |
| try { | |
| const res = await axios.get(`http://localhost:${port}/health`, { timeout: 3000 }); | |
| if (res.status === 200) { | |
| console.log(`β Health check passed on port ${port}!`); | |
| resolve(proc); | |
| } else reject(`Health check failed on port ${port}`); | |
| } catch (err: any) { | |
| reject(`Health check error on port ${port}: ${err.message}`); | |
| } | |
| }, 8000); | |
| }); | |
| }); | |
| } | |
| // ============================================================ | |
| // STATE | |
| // ============================================================ | |
| let prodProcess: ChildProcess | null = null; | |
| let prodVersionPath: string | null = null; | |
| let prodCommitSha: string | null = null; | |
| let isUpdating: boolean = false; | |
| // ============================================================ | |
| // FAILED VERSIONS TRACKING (LIMITED TO LAST 100) | |
| // ============================================================ | |
| const FAILED_VERSIONS_FILE = '/app/failed-versions.json'; | |
| const MAX_FAILED_VERSIONS = 100; | |
| function loadFailedVersions(): string[] { | |
| try { | |
| if (fs.existsSync(FAILED_VERSIONS_FILE)) { | |
| const data = fs.readFileSync(FAILED_VERSIONS_FILE, 'utf-8'); | |
| const parsed = JSON.parse(data); | |
| if (Array.isArray(parsed)) return parsed; | |
| } | |
| } catch (e) { | |
| console.warn('β οΈ Failed to load failed-versions.json'); | |
| } | |
| return []; | |
| } | |
| function saveFailedVersion(sha: string): void { | |
| try { | |
| let failed = loadFailedVersions(); | |
| // Remove duplicate if exists | |
| failed = failed.filter(v => v !== sha); | |
| // Add new at the end | |
| failed.push(sha); | |
| // Keep only last MAX_FAILED_VERSIONS | |
| if (failed.length > MAX_FAILED_VERSIONS) { | |
| failed = failed.slice(-MAX_FAILED_VERSIONS); | |
| } | |
| fs.writeFileSync(FAILED_VERSIONS_FILE, JSON.stringify(failed, null, 2)); | |
| console.log(`πΎ Failed version ${sha} saved. Total failed: ${failed.length}`); | |
| } catch (e) { | |
| console.error('β Failed to save failed version:', e); | |
| } | |
| } | |
| function isVersionFailed(sha: string): boolean { | |
| const failed = loadFailedVersions(); | |
| return failed.includes(sha); | |
| } | |
| function clearFailedVersion(sha: string): void { | |
| try { | |
| const failed = loadFailedVersions(); | |
| const updated = failed.filter(v => v !== sha); | |
| fs.writeFileSync(FAILED_VERSIONS_FILE, JSON.stringify(updated, null, 2)); | |
| console.log(`ποΈ Failed version ${sha} removed from tracking list.`); | |
| } catch (e) { | |
| console.error('β Failed to clear failed version:', e); | |
| } | |
| } | |
| // ============================================================ | |
| // MAIN UPDATE LOGIC (Your Exact Flow) | |
| // ============================================================ | |
| async function checkAndUpdate(): Promise<void> { | |
| if (isUpdating) { | |
| console.log('β³ Update already in progress...'); | |
| return; | |
| } | |
| isUpdating = true; | |
| let testProcess: ChildProcess | null = null; | |
| let testVersionPath: string | null = null; | |
| let testCommitSha: string | null = null; | |
| try { | |
| console.log('π Checking for updates...'); | |
| const latestSha = await getCurrentCommitSha(CONFIG.RES1_REPO); | |
| if (!latestSha) throw new Error('Could not fetch latest SHA'); | |
| console.log(`π Latest SHA: ${latestSha.substring(0, 7)}`); | |
| console.log(`π Current Prod SHA: ${prodCommitSha ? prodCommitSha.substring(0, 7) : 'none'}`); | |
| // CHECK 1: New code available? | |
| if (latestSha === prodCommitSha) { | |
| console.log('β No new code. Current version is already running.'); | |
| if (prodCommitSha && isVersionFailed(prodCommitSha)) { | |
| console.log(`π Version ${prodCommitSha.substring(0, 7)} was previously failed but now running. Clearing from failed list.`); | |
| clearFailedVersion(prodCommitSha); | |
| } | |
| isUpdating = false; | |
| return; | |
| } | |
| // CHECK 2: Is this new version previously failed? | |
| if (isVersionFailed(latestSha)) { | |
| console.log(`β³ Version ${latestSha.substring(0, 7)} is marked as FAILED in history.`); | |
| console.log('π Waiting for a NEW version after this failed one.'); | |
| isUpdating = false; | |
| return; | |
| } | |
| // NEW VERSION DETECTED β Deploy on port 2000 | |
| console.log(`π New version: ${latestSha.substring(0, 7)}. Deploying on port ${CONFIG.TEST_PORT}...`); | |
| testCommitSha = latestSha; | |
| testVersionPath = path.join(CONFIG.VERSIONS_DIR, `v2_${Date.now()}`); | |
| await cloneOrPull(CONFIG.RES1_REPO, testVersionPath); | |
| // TEST ON PORT 2000 | |
| testProcess = await deployAgent(testVersionPath, CONFIG.TEST_PORT); | |
| console.log(`β V2 passed health check on port ${CONFIG.TEST_PORT}!`); | |
| // COPY MEMORY FROM 7000 β 2000 | |
| if (prodVersionPath) { | |
| copyMemoryFolder(prodVersionPath, testVersionPath); | |
| } else { | |
| console.log('β οΈ No previous version. Starting fresh without memory copy.'); | |
| } | |
| // STOP TEST (2000) | |
| if (testProcess) { | |
| console.log('π Stopping test agent on port 2000...'); | |
| testProcess.kill('SIGTERM'); | |
| await new Promise(resolve => setTimeout(resolve, 2000)); | |
| if (!testProcess.killed) testProcess.kill('SIGKILL'); | |
| testProcess = null; | |
| } | |
| // PROMOTE TO 7000 | |
| console.log(`π Promoting to production port ${CONFIG.PROD_PORT}...`); | |
| const newProdProc = await deployAgent(testVersionPath, CONFIG.PROD_PORT); | |
| // Stop old production | |
| if (prodProcess) { | |
| console.log('π Stopping old production on port 7000...'); | |
| prodProcess.kill('SIGTERM'); | |
| setTimeout(() => { | |
| if (!prodProcess?.killed) prodProcess?.kill('SIGKILL'); | |
| }, 3000); | |
| } | |
| // Update state | |
| prodProcess = newProdProc; | |
| prodVersionPath = testVersionPath; | |
| prodCommitSha = testCommitSha; | |
| testVersionPath = null; | |
| testCommitSha = null; | |
| // Clear failed flag if this version was previously failed | |
| if (prodCommitSha && isVersionFailed(prodCommitSha)) { | |
| clearFailedVersion(prodCommitSha); | |
| } | |
| console.log(`β Successfully deployed V2 (${latestSha.substring(0, 7)}) on port ${CONFIG.PROD_PORT}`); | |
| } catch (error: any) { | |
| console.error('β Deployment failed:', error); | |
| // Stop test process (port 2000) if running | |
| if (testProcess) { | |
| console.log('π Stopping failed test agent on port 2000...'); | |
| testProcess.kill('SIGTERM'); | |
| setTimeout(() => { | |
| if (!testProcess?.killed) testProcess?.kill('SIGKILL'); | |
| }, 2000); | |
| testProcess = null; | |
| } | |
| // Save the failed version (with size limit) | |
| if (testCommitSha) { | |
| console.log(`πΎ Saving failed version ${testCommitSha.substring(0, 7)} to tracking list.`); | |
| saveFailedVersion(testCommitSha); | |
| } | |
| // Send email notification | |
| await sendEmail('Deployment FAILED', `Version: ${testCommitSha ? testCommitSha.substring(0, 7) : 'unknown'}\nError: ${error.message}\nPort 7000 is unchanged.`); | |
| // Keep old production running | |
| if (prodProcess) { | |
| console.log('π Keeping old version running on port 7000.'); | |
| } else { | |
| console.log('β οΈ No previous version available. Port 7000 is empty.'); | |
| } | |
| // Clean up failed test version directory | |
| if (testVersionPath && fs.existsSync(testVersionPath)) { | |
| try { | |
| fs.rmSync(testVersionPath, { recursive: true, force: true }); | |
| console.log(`π§Ή Cleaned up failed test directory: ${testVersionPath}`); | |
| } catch (e) { | |
| // ignore | |
| } | |
| } | |
| } finally { | |
| isUpdating = false; | |
| } | |
| } | |
| // ============================================================ | |
| // HTTP SERVER (Port 7860 - UptimeRobot) | |
| // ============================================================ | |
| const server = http.createServer(async (req, res) => { | |
| const url = new URL(req.url || '/', `http://${req.headers.host}`); | |
| if (url.pathname === '/') { | |
| res.writeHead(200, { 'Content-Type': 'text/plain' }); | |
| res.end('Oracus Runner is Alive!'); | |
| setImmediate(() => checkAndUpdate()); | |
| } else if (url.pathname === '/force-update') { | |
| res.writeHead(200); | |
| res.end('Force update triggered.'); | |
| setImmediate(() => checkAndUpdate()); | |
| } else if (url.pathname === '/status') { | |
| const failed = loadFailedVersions(); | |
| res.writeHead(200, { 'Content-Type': 'application/json' }); | |
| res.end(JSON.stringify({ | |
| prodRunning: prodProcess !== null, | |
| prodVersion: prodCommitSha ? prodCommitSha.substring(0, 7) : 'none', | |
| prodVersionPath: prodVersionPath ? path.basename(prodVersionPath) : 'none', | |
| isUpdating, | |
| failedVersions: failed.map(s => s.substring(0, 7)) | |
| }, null, 2)); | |
| } else { | |
| res.writeHead(404); | |
| res.end('Not Found'); | |
| } | |
| }); | |
| server.listen(CONFIG.HEALTH_PORT, '0.0.0.0', () => { | |
| console.log(`β Health server running on port ${CONFIG.HEALTH_PORT}`); | |
| console.log('β³ Initial setup...'); | |
| checkAndUpdate(); | |
| }); | |
| // ============================================================ | |
| // CRASH HANDLING | |
| // ============================================================ | |
| process.on('uncaughtException', (err) => { | |
| console.error('π₯ Uncaught Exception:', err); | |
| sendEmail('CRASH - Uncaught Exception', err.stack || err.message); | |
| }); | |
| process.on('unhandledRejection', (reason) => { | |
| console.error('π₯ Unhandled Rejection:', reason); | |
| sendEmail('CRASH - Unhandled Rejection', String(reason)); | |
| }); |