Buckets:
| ; | |
| var __importDefault = (this && this.__importDefault) || function (mod) { | |
| return (mod && mod.__esModule) ? mod : { "default": mod }; | |
| }; | |
| Object.defineProperty(exports, "__esModule", { value: true }); | |
| exports.default = startJob; | |
| const prisma_1 = __importDefault(require("../prisma")); | |
| const child_process_1 = require("child_process"); | |
| const path_1 = __importDefault(require("path")); | |
| const fs_1 = __importDefault(require("fs")); | |
| const paths_1 = require("../paths"); | |
| const pythonPath_1 = require("../pythonPath"); | |
| const isWindows = process.platform === 'win32'; | |
| const appendJobLog = (logPath, message) => { | |
| fs_1.default.appendFile(logPath, message, error => { | |
| if (error) | |
| console.error('Error writing to job log:', error); | |
| }); | |
| }; | |
| // Windows only. Launched as `node -e <this>` so the job ends up outside the | |
| // worker's process tree: `taskkill /T` (the dev script's `concurrently -k`, | |
| // or any shutdown that kills the tree) walks parent/child links and would take | |
| // a direct child down with the UI. This relay exits immediately, orphaning the | |
| // job, and `detached` keeps the job alive once its parent is gone. Its own | |
| // stdout/stderr are the job log, so the job inherits them as fds 1 and 2. | |
| // Python failing to launch at all (broken venv, missing interpreter) happens | |
| // inside the relay, so the relay -- not the worker -- is what sees that error. | |
| // It reports it two ways: on stderr, which is the job log, and through the pid | |
| // file, so the worker can put the real reason in the database. | |
| const RELAY_ERROR_PREFIX = 'error:'; | |
| const WINDOWS_RELAY_SCRIPT = ` | |
| const { spawn } = require('child_process'); | |
| const fs = require('fs'); | |
| const [pidFile, command, ...args] = process.argv.slice(1); | |
| const child = spawn(command, args, { | |
| detached: true, | |
| windowsHide: true, | |
| stdio: ['ignore', 1, 2], | |
| }); | |
| child.once('error', error => { | |
| process.stderr.write('Error launching job process: ' + error.message + '\\n'); | |
| try { | |
| fs.writeFileSync(pidFile, '${RELAY_ERROR_PREFIX}' + error.message); | |
| } catch (e) { | |
| process.stderr.write('Could not write job pid file: ' + e.message + '\\n'); | |
| } | |
| process.exit(1); | |
| }); | |
| if (child.pid) { | |
| fs.writeFileSync(pidFile, String(child.pid)); | |
| child.unref(); | |
| } | |
| `; | |
| const RELAY_PID_TIMEOUT_MS = 30000; | |
| // The relay exits as soon as it has launched the job, leaving the real pid in | |
| // pidPath. Without this we would only ever know the (already dead) relay's pid. | |
| const readRelayPid = (relay, pidPath) => { | |
| return new Promise(resolve => { | |
| let settled = false; | |
| const finish = (value) => { | |
| if (settled) | |
| return; | |
| settled = true; | |
| clearTimeout(timer); | |
| resolve(value); | |
| }; | |
| const timer = setTimeout(() => finish({ pid: null, error: 'Timed out waiting for the job process to start' }), RELAY_PID_TIMEOUT_MS); | |
| relay.once('exit', () => { | |
| let contents; | |
| try { | |
| contents = fs_1.default.readFileSync(pidPath, 'utf8').trim(); | |
| } | |
| catch { | |
| finish({ pid: null, error: 'Job process did not report a pid' }); | |
| return; | |
| } | |
| if (contents.startsWith(RELAY_ERROR_PREFIX)) { | |
| finish({ pid: null, error: contents.slice(RELAY_ERROR_PREFIX.length) }); | |
| return; | |
| } | |
| const pid = Number(contents); | |
| finish(Number.isInteger(pid) && pid > 0 | |
| ? { pid } | |
| : { pid: null, error: 'Job process did not report a usable pid' }); | |
| }); | |
| relay.once('error', error => finish({ pid: null, error: error.message })); | |
| }); | |
| }; | |
| const isProcessAlive = (pid) => { | |
| try { | |
| process.kill(pid, 0); | |
| return true; | |
| } | |
| catch (e) { | |
| // EPERM means it exists but belongs to someone else, which still counts. | |
| return e?.code === 'EPERM'; | |
| } | |
| }; | |
| // We cannot read an exit code off a process that is not our child, so pull the | |
| // last thing it said instead -- for a job that dies on startup (bad venv, | |
| // missing CUDA libs) that traceback line is the whole diagnosis. | |
| const LOG_TAIL_BYTES = 4096; | |
| const LOG_TAIL_MAX_CHARS = 300; | |
| const readLogTail = (logPath) => { | |
| let fd = null; | |
| try { | |
| const size = fs_1.default.statSync(logPath).size; | |
| const length = Math.min(size, LOG_TAIL_BYTES); | |
| if (length === 0) | |
| return null; | |
| const buffer = Buffer.alloc(length); | |
| fd = fs_1.default.openSync(logPath, 'r'); | |
| fs_1.default.readSync(fd, buffer, 0, length, size - length); | |
| const lines = buffer.toString('utf8').split(/\r?\n/).filter(line => line.trim() !== ''); | |
| const lastLine = lines[lines.length - 1]; | |
| if (!lastLine) | |
| return null; | |
| return lastLine.length > LOG_TAIL_MAX_CHARS ? `${lastLine.slice(-LOG_TAIL_MAX_CHARS)}` : lastLine; | |
| } | |
| catch { | |
| return null; | |
| } | |
| finally { | |
| if (fd !== null) { | |
| try { | |
| fs_1.default.closeSync(fd); | |
| } | |
| catch { | |
| // nothing useful to do if the log handle will not close | |
| } | |
| } | |
| } | |
| }; | |
| // The job is not our child anymore, so there is no 'exit' event to listen for. | |
| // Poll instead, so a job that dies without updating its own row (OOM kill, | |
| // hard crash) still gets marked as an error rather than sitting on 'running'. | |
| const JOB_POLL_INTERVAL_MS = 2000; | |
| const watchDetachedJob = (pid, jobID, logPath) => { | |
| const timer = setInterval(() => { | |
| if (isProcessAlive(pid)) | |
| return; | |
| clearInterval(timer); | |
| // A stopped or completed job writes its own ending (status row + final | |
| // log lines) via its KeyboardInterrupt/done handlers -- stay out of the | |
| // way. Only a job that vanished while still marked 'running' died without | |
| // getting to say anything; record that. There is no exit code to read off | |
| // a process that is not our child, so report the last thing it logged. | |
| const tail = readLogTail(logPath); | |
| const message = tail | |
| ? `Job process exited unexpectedly. Last log line: ${tail}` | |
| : 'Job process exited unexpectedly.'; | |
| void prisma_1.default.job | |
| .updateMany({ | |
| where: { id: jobID, status: 'running' }, | |
| data: { status: 'error', info: message, pid: null }, | |
| }) | |
| .then(result => { | |
| if (result.count > 0) | |
| appendJobLog(logPath, `\n${message}\n`); | |
| }) | |
| .catch(updateError => { | |
| console.error('Error updating job after process disappeared:', updateError); | |
| }); | |
| }, JOB_POLL_INTERVAL_MS); | |
| // Never hold the worker open on account of this poll. | |
| if (timer.unref) | |
| timer.unref(); | |
| }; | |
| const startAndWatchJob = (job) => { | |
| // starts and watches the job asynchronously | |
| return new Promise(async (resolve, reject) => { | |
| const jobID = job.id; | |
| // setup the training | |
| const trainingRoot = await (0, paths_1.getTrainingFolder)(); | |
| const trainingFolder = path_1.default.join(trainingRoot, job.name); | |
| if (!fs_1.default.existsSync(trainingFolder)) { | |
| fs_1.default.mkdirSync(trainingFolder, { recursive: true }); | |
| } | |
| // make the config file | |
| const configPath = path_1.default.join(trainingFolder, '.job_config.json'); | |
| //log to path | |
| const logPath = path_1.default.join(trainingFolder, 'log.txt'); | |
| try { | |
| // if the log path exists, move it to a folder called logs and rename it {num}_log.txt, looking for the highest num | |
| // if the log path does not exist, create it | |
| if (fs_1.default.existsSync(logPath)) { | |
| const logsFolder = path_1.default.join(trainingFolder, 'logs'); | |
| if (!fs_1.default.existsSync(logsFolder)) { | |
| fs_1.default.mkdirSync(logsFolder, { recursive: true }); | |
| } | |
| let num = 0; | |
| while (fs_1.default.existsSync(path_1.default.join(logsFolder, `${num}_log.txt`))) { | |
| num++; | |
| } | |
| fs_1.default.renameSync(logPath, path_1.default.join(logsFolder, `${num}_log.txt`)); | |
| } | |
| } | |
| catch (e) { | |
| console.error('Error moving log file:', e); | |
| } | |
| // update the config dataset path | |
| const jobConfig = JSON.parse(job.job_config); | |
| jobConfig.config.process[0].sqlite_db_path = path_1.default.join(paths_1.TOOLKIT_ROOT, 'aitk_db.db'); | |
| if (job.job_type === 'inference') { | |
| // the engine publishes engine.json (endpoint + token) and writes its | |
| // outputs under the job folder, which the UI proxy/file routes know | |
| jobConfig.config.process[0].engine = { | |
| ...(jobConfig.config.process[0].engine || {}), | |
| job_folder: trainingFolder, | |
| }; | |
| } | |
| // write the config file | |
| fs_1.default.writeFileSync(configPath, JSON.stringify(jobConfig, null, 2)); | |
| const pythonPath = (0, pythonPath_1.resolveDetachedPythonPath)(); | |
| const runFilePath = path_1.default.join(paths_1.TOOLKIT_ROOT, 'run.py'); | |
| if (!fs_1.default.existsSync(runFilePath)) { | |
| console.error(`run.py not found at path: ${runFilePath}`); | |
| await prisma_1.default.job.update({ | |
| where: { id: jobID }, | |
| data: { | |
| status: 'error', | |
| info: `Error launching job: run.py not found`, | |
| }, | |
| }); | |
| return; | |
| } | |
| const additionalEnv = { | |
| AITK_JOB_ID: jobID, | |
| CUDA_DEVICE_ORDER: 'PCI_BUS_ID', | |
| CUDA_VISIBLE_DEVICES: `${job.gpu_ids}`, | |
| IS_AI_TOOLKIT_UI: '1', | |
| PYTHONUNBUFFERED: '1', // write Python output immediately so it is not lost on a crash | |
| }; | |
| // HF_TOKEN | |
| const hfToken = await (0, paths_1.getHFToken)(); | |
| if (hfToken && hfToken.trim() !== '') { | |
| additionalEnv.HF_TOKEN = hfToken; | |
| } | |
| // MODELS_PATH - one set in the env always takes precedence (it passes | |
| // through via process.env); only fall back to the setting if it is not set | |
| if (!process.env.MODELS_PATH || process.env.MODELS_PATH.trim() === '') { | |
| const modelsPath = await (0, paths_1.getModelsPath)(); | |
| if (modelsPath && modelsPath.trim() !== '') { | |
| additionalEnv.MODELS_PATH = modelsPath; | |
| } | |
| } | |
| const args = [runFilePath, configPath]; | |
| // Where the Windows relay reports the job's real pid back to us. | |
| const relayPidPath = path_1.default.join(trainingFolder, '.job_pid'); | |
| let logFd = null; | |
| try { | |
| // Capture errors that occur before run.py can initialize file logging. | |
| logFd = fs_1.default.openSync(logPath, 'a'); | |
| let subprocess; | |
| if (isWindows) { | |
| // Launch through the relay (see WINDOWS_RELAY_SCRIPT) so the job is not | |
| // a descendant of this worker and survives the UI being shut down or | |
| // tree-killed. The relay spawns the job `detached`, which is what keeps | |
| // it alive once the relay exits; that in turn means DETACHED_PROCESS, | |
| // so pythonPath is pythonw.exe to avoid Windows handing the job a | |
| // console window of its own. | |
| try { | |
| fs_1.default.unlinkSync(relayPidPath); | |
| } | |
| catch { | |
| // no stale pid file to clear | |
| } | |
| subprocess = (0, child_process_1.spawn)(process.execPath, ['-e', WINDOWS_RELAY_SCRIPT, relayPidPath, pythonPath, ...args], { | |
| env: { | |
| ...process.env, | |
| ...additionalEnv, | |
| }, | |
| cwd: paths_1.TOOLKIT_ROOT, | |
| windowsHide: true, | |
| stdio: ['ignore', logFd, logFd], // don't tie stdio to parent; log fd passed as stdout and stderr | |
| }); | |
| } | |
| else { | |
| // For non-Windows platforms, fully detach and ignore stdio so it survives daemon-like | |
| subprocess = (0, child_process_1.spawn)(pythonPath, args, { | |
| detached: true, | |
| stdio: ['ignore', logFd, logFd], // don't tie stdio to parent; log fd passed as stdout and stderr | |
| env: { | |
| ...process.env, | |
| ...additionalEnv, | |
| }, | |
| cwd: paths_1.TOOLKIT_ROOT, | |
| }); | |
| } | |
| // Handle failures where the child process could not be started. | |
| subprocess.once('error', error => { | |
| const message = `Error launching job process: ${error.message}`; | |
| console.error(message); | |
| appendJobLog(logPath, `${message}\n`); | |
| void prisma_1.default.job | |
| .update({ | |
| where: { id: jobID }, | |
| data: { status: 'error', info: message, pid: null }, | |
| }) | |
| .catch(updateError => { | |
| console.error('Error updating job after process launch failure:', updateError); | |
| }); | |
| }); | |
| let pid; | |
| if (isWindows) { | |
| // The relay is gone within a few hundred ms; the pid it leaves behind is | |
| // the job's. Poll that pid for liveness since we get no 'exit' event. | |
| const relayResult = await readRelayPid(subprocess, relayPidPath); | |
| if (relayResult.pid == null) { | |
| throw new Error(relayResult.error ?? 'Job process did not report a pid'); | |
| } | |
| pid = relayResult.pid; | |
| watchDetachedJob(pid, jobID, logPath); | |
| } | |
| else { | |
| pid = subprocess.pid ?? null; | |
| // Record abnormal termination and repair jobs Python could not update itself. | |
| subprocess.once('exit', (code, signal) => { | |
| if (code === 0) | |
| return; | |
| const result = signal ? `signal ${signal}` : `exit code ${code}`; | |
| const message = `Job process terminated with ${result}.`; | |
| appendJobLog(logPath, `\n${message}\n`); | |
| void prisma_1.default.job | |
| .updateMany({ | |
| where: { id: jobID, status: 'running' }, | |
| data: { status: 'error', info: message, pid: null }, | |
| }) | |
| .catch(updateError => { | |
| console.error('Error updating job after abnormal process exit:', updateError); | |
| }); | |
| }); | |
| } | |
| // Save the PID to the database and a file for future management (stop/inspect) | |
| if (pid != null) { | |
| await prisma_1.default.job.update({ | |
| where: { id: jobID }, | |
| data: { pid }, | |
| }); | |
| } | |
| try { | |
| fs_1.default.writeFileSync(path_1.default.join(trainingFolder, 'pid.txt'), String(pid ?? ''), { flag: 'w' }); | |
| } | |
| catch (e) { | |
| console.error('Error writing pid file:', e); | |
| } | |
| // Important: let the child run independently of this Node process. | |
| if (subprocess.unref) { | |
| subprocess.unref(); | |
| } | |
| // The child remains independent; these listeners only record failures | |
| // while the worker is alive. | |
| } | |
| catch (error) { | |
| // Handle any exceptions during process launch | |
| console.error('Error launching process:', error); | |
| appendJobLog(logPath, `Error launching job process: ${error?.message || 'Unknown error'}\n`); | |
| await prisma_1.default.job.update({ | |
| where: { id: jobID }, | |
| data: { | |
| status: 'error', | |
| info: `Error launching job: ${error?.message || 'Unknown error'}`, | |
| }, | |
| }); | |
| return; | |
| } | |
| finally { | |
| if (logFd !== null) { | |
| fs_1.default.closeSync(logFd); | |
| } | |
| } | |
| // Resolve the promise immediately after starting the process | |
| resolve(); | |
| }); | |
| }; | |
| async function startJob(jobID) { | |
| const job = await prisma_1.default.job.findUnique({ | |
| where: { id: jobID }, | |
| }); | |
| if (!job) { | |
| console.error(`Job with ID ${jobID} not found`); | |
| return; | |
| } | |
| // update job status to 'running', this will run sync so we don't start multiple jobs. | |
| await prisma_1.default.job.update({ | |
| where: { id: jobID }, | |
| data: { | |
| status: 'running', | |
| stop: false, | |
| return_to_queue: false, | |
| info: 'Starting job...', | |
| }, | |
| }); | |
| // start and watch the job asynchronously so the cron can continue | |
| startAndWatchJob(job); | |
| } | |
Xet Storage Details
- Size:
- 17.3 kB
- Xet hash:
- de28c464c6bb60296b4b7ce404604e513d5ae23eb606bef35f4dd57e8cc8a7f4
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.