Quantumbraid's picture
download
raw
16.7 kB
import { spawn, execFile, ChildProcess } from 'child_process';
import { promisify } from 'util';
import os from 'os';
import si from 'systeminformation';
import { loadMacstats } from '@/server/macstats';
import { createLoadSampler, readCpuTemperature, readMemory } from '@/server/cpuStats';
import { CpuInfo, GpuInfo, GPUApiResponse, MonitorHistoryPoint, MonitorInit, MonitorSample } from '@/types';
import { historyPointFromSample, MONITOR_HISTORY_LENGTH, MONITOR_TICK_MS } from '@/utils/monitorSample';
const execFileAsync = promisify(execFile);
/**
* Always-on system monitor. Samples CPU + GPU every MONITOR_TICK_MS, keeps a
* 2-minute rolling history (load + memory only), and pushes every full sample
* to subscribers (the SSE route at /api/monitor).
*
* GPU stats come from a single resident `nvidia-smi ... -lms` child process —
* NVML stays initialized between samples, which is what made the old
* spawn-per-request /api/gpu route slow. If loop mode never produces output
* (or nvidia-smi keeps hanging), we fall back to a one-shot spawn per tick,
* which matches the old route's behavior exactly.
*/
const NV_QUERY_ARGS = [
'--query-gpu=index,name,driver_version,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used,power.draw,power.limit,clocks.current.graphics,clocks.current.memory,fan.speed',
'--format=csv,noheader,nounits',
];
const NV_ENV = { ...process.env, CUDA_DEVICE_ORDER: 'PCI_BUS_ID' };
// No stdout line for this long means the loop child is buffering or hung.
const NV_WATCHDOG_MS = 15_000;
// A line-less gap this long after the last line closes out a batch (all
// lines of one iteration arrive together; iterations are MONITOR_TICK_MS
// apart, so this can never bleed into the next batch).
const NV_BATCH_FLUSH_MS = 100;
// Temperature refresh is decoupled from the tick (see refreshCpuTemp)
const CPU_TEMP_REFRESH_MS = 5000;
function parseGpuLine(line: string): GpuInfo | null {
const [
index,
name,
driverVersion,
temperature,
gpuUtil,
memoryUtil,
memoryTotal,
memoryFree,
memoryUsed,
powerDraw,
powerLimit,
clockGraphics,
clockMemory,
fanSpeed,
] = line.split(', ').map(item => item.trim());
if (isNaN(parseInt(index))) return null;
return {
index: parseInt(index),
name,
driverVersion,
temperature: parseInt(temperature),
utilization: {
gpu: parseInt(gpuUtil),
memory: parseInt(memoryUtil),
},
memory: {
total: parseInt(memoryTotal),
free: parseInt(memoryFree),
used: parseInt(memoryUsed),
},
power: {
draw: parseFloat(powerDraw),
limit: parseFloat(powerLimit),
},
clocks: {
graphics: parseInt(clockGraphics),
memory: parseInt(clockMemory),
},
fan: {
speed: parseInt(fanSpeed) || 0, // Some GPUs might not report fan speed, default to 0
},
};
}
// Serialized once per tick and shared, so N subscribers don't stringify N times
type Subscriber = (sample: MonitorSample, serialized: string) => void;
class SystemMonitor {
private isMac = os.platform() === 'darwin';
private history: MonitorHistoryPoint[] = [];
private subscribers = new Set<Subscriber>();
private latestCpu: CpuInfo | null = null;
private latestGpu: GPUApiResponse = { hasNvidiaSmi: false, isMac: this.isMac, gpus: [] };
private macGpuName = 'Apple GPU';
private nvChild: ChildProcess | null = null;
private nvBatch: GpuInfo[] = [];
private nvStdoutBuffer = '';
private nvFlushTimer: NodeJS.Timeout | null = null;
private nvUnavailable = false;
private nvReaped = false;
private nvEverGotLine = false;
private nvOneShotMode = false;
private nvOneShotInFlight = false;
private lastNvLineAt = 0;
private tickInFlight = false;
private lastCpuTemp = 0;
private cpuTempInFlight = false;
private lastCpuTempAt = 0;
private cpuStatic: { name: string; cores: number } | null = null;
// Differences os.cpus() times between ticks; no child process involved.
private sampleLoad = createLoadSampler();
start(): void {
if (this.isMac) {
this.initMacGpuName();
} else {
this.startNvLoop();
}
// Never leave a resident nvidia-smi behind.
process.once('exit', () => {
try {
this.nvChild?.kill('SIGKILL');
} catch {
// already gone
}
});
// Fixed cadence; a tick that overruns the interval (slow temperature
// read, hung nvidia-smi one-shot) just skips beats instead of stacking.
this.tick();
setInterval(() => this.tick(), MONITOR_TICK_MS);
}
subscribe(fn: Subscriber): () => void {
this.subscribers.add(fn);
return () => this.subscribers.delete(fn);
}
getInit(): MonitorInit {
return {
t: Date.now(),
cpu: this.latestCpu,
gpu: this.latestGpu,
history: [...this.history],
};
}
// -------------------------------------------------------------------------
// Tick loop
// -------------------------------------------------------------------------
private async tick(): Promise<void> {
if (this.tickInFlight) return;
this.tickInFlight = true;
try {
await this.doTick();
} finally {
this.tickInFlight = false;
}
}
private async doTick(): Promise<void> {
const t = Date.now();
try {
this.latestCpu = await this.sampleCpu();
} catch (error) {
console.error('Monitor: CPU sample failed:', error);
}
try {
if (this.isMac) {
this.latestGpu = this.sampleMacGpu();
} else if (this.nvOneShotMode) {
await this.sampleNvOneShot();
} else {
this.nvWatchdog();
}
} catch (error) {
console.error('Monitor: GPU sample failed:', error);
}
const sample: MonitorSample = { t, cpu: this.latestCpu, gpu: this.latestGpu };
this.history.push(historyPointFromSample(sample));
if (this.history.length > MONITOR_HISTORY_LENGTH) {
this.history.splice(0, this.history.length - MONITOR_HISTORY_LENGTH);
}
const serialized = JSON.stringify(sample);
for (const subscriber of this.subscribers) {
try {
subscriber(sample, serialized);
} catch (error) {
console.error('Monitor: subscriber failed:', error);
}
}
}
// -------------------------------------------------------------------------
// CPU (mirrors /api/cpu exactly)
// -------------------------------------------------------------------------
private async sampleCpu(): Promise<CpuInfo> {
// si.cpu() shells out to lscpu/dmidecode on every call — at tick cadence
// that alone was eating a measurable slice of a core. Name and core count
// never change, so resolve them exactly once.
if (!this.cpuStatic) {
const cpuInfoRaw = await si.cpu();
this.cpuStatic = { name: `${cpuInfoRaw.manufacturer} ${cpuInfoRaw.brand}`, cores: cpuInfoRaw.cores };
}
const cpuStatic = this.cpuStatic;
if (this.isMac) {
try {
const ms = loadMacstats();
if (!ms) throw new Error('macstats unavailable');
const ramData = ms.getRAMUsageSync();
const cpuData = ms.getCpuDataSync();
return {
name: cpuStatic.name,
cores: cpuStatic.cores,
temperature: cpuData.temperature || 0,
totalMemory: ramData.total / (1024 * 1024),
availableMemory: ramData.free / (1024 * 1024),
freeMemory: ramData.free / (1024 * 1024),
currentLoad: this.sampleLoad(),
};
} catch {
// Fallback to the generic path if macstats fails
}
}
// The temperature read can take >1s on some machines and would make the
// tick skip beats, so it refreshes concurrently and we use the cached
// value (at most a tick or two old).
this.refreshCpuTemp();
// Nothing below spawns a process. Every child process forks this server
// first, and forking a multi-GB Node heap freezes the event loop for
// hundreds of ms — at tick cadence that stalled every request in the UI.
const memoryData = await readMemory();
return {
name: cpuStatic.name,
cores: cpuStatic.cores,
temperature: this.lastCpuTemp,
totalMemory: memoryData.total / (1024 * 1024),
availableMemory: memoryData.available / (1024 * 1024),
freeMemory: memoryData.free / (1024 * 1024),
currentLoad: this.sampleLoad(),
};
}
private refreshCpuTemp(): void {
// On Linux this is a sysfs read; elsewhere it may still shell out (via
// systeminformation), and e.g. `sensors` can take ~1s of slow SMBus
// polling — refreshed back-to-back it becomes a permanently resident
// child. Every few seconds is plenty for a temperature.
if (this.cpuTempInFlight || Date.now() - this.lastCpuTempAt < CPU_TEMP_REFRESH_MS) return;
this.lastCpuTempAt = Date.now();
this.cpuTempInFlight = true;
readCpuTemperature()
.then(t => {
this.lastCpuTemp = t ?? 0;
})
.catch(() => {
// keep the previous value
})
.finally(() => {
this.cpuTempInFlight = false;
});
}
// -------------------------------------------------------------------------
// Mac GPU (mirrors /api/gpu's mac path)
// -------------------------------------------------------------------------
private initMacGpuName(): void {
execFileAsync('sh', ['-c', 'system_profiler SPDisplaysDataType 2>/dev/null | grep -E "Chipset Model|Total Number of Cores"'], {
timeout: 5000,
})
.then(({ stdout }) => {
const nameMatch = stdout.match(/Chipset Model:\s*(.+)/);
const coresMatch = stdout.match(/Total Number of Cores:\s*(\d+)/);
if (nameMatch) {
this.macGpuName = nameMatch[1].trim();
if (coresMatch) {
this.macGpuName += ` GPU (${coresMatch[1]} cores)`;
}
}
})
.catch(() => {
// fallback to generic name
});
}
private sampleMacGpu(): GPUApiResponse {
let temperature = 0;
let gpuLoad = 0;
let fanSpeed = 0;
let powerDraw = 0;
let memUsed = 0;
let memTotal = os.totalmem() / (1024 * 1024);
const ms = loadMacstats();
if (ms) {
try {
const gpuData = ms.getGpuDataSync();
temperature = gpuData.temperature || 0;
gpuLoad = gpuData.usage || 0;
} catch {
// ignore
}
try {
const fanData = ms.getFanDataSync();
const fanKeys = Object.keys(fanData);
if (fanKeys.length > 0) {
fanSpeed = fanData[fanKeys[0]].rpm || 0;
}
} catch {
// ignore
}
try {
const powerData = ms.getPowerDataSync();
powerDraw = powerData.gpu || 0;
} catch {
// ignore
}
try {
const ramData = ms.getRAMUsageSync();
memUsed = ramData.used / (1024 * 1024);
memTotal = ramData.total / (1024 * 1024);
} catch {
// ignore
}
}
return {
hasNvidiaSmi: false,
isMac: true,
gpus: [
{
index: 0,
name: this.macGpuName,
driverVersion: 'macOS',
temperature: Math.round(temperature),
utilization: {
gpu: gpuLoad,
memory: memTotal > 0 ? Math.round((memUsed / memTotal) * 100) : 0,
},
memory: {
total: Math.round(memTotal),
free: Math.round(memTotal - memUsed),
used: Math.round(memUsed),
},
power: { draw: powerDraw, limit: 0 },
clocks: { graphics: 0, memory: 0 },
fan: { speed: fanSpeed },
},
],
};
}
// -------------------------------------------------------------------------
// NVIDIA loop-mode child
// -------------------------------------------------------------------------
private startNvLoop(): void {
if (this.nvUnavailable || this.nvOneShotMode || this.nvChild) return;
// A hard-killed server (SIGKILL never runs the exit hook) can orphan the
// resident loop child. Our exact query string only ever appears in
// children we spawned, so reap any stray once at boot — after it
// completes, to not race the kill against our own fresh child.
if (!this.nvReaped && process.platform !== 'win32') {
this.nvReaped = true;
// No loop flag in the pattern so strays from any past cadence match
execFile('pkill', ['-9', '-f', `nvidia-smi ${NV_QUERY_ARGS.join(' ')}`], () => this.startNvLoop());
return;
}
this.nvReaped = true;
let child: ChildProcess;
try {
child = spawn('nvidia-smi', [...NV_QUERY_ARGS, '-lms', String(MONITOR_TICK_MS)], {
env: NV_ENV,
stdio: ['ignore', 'pipe', 'pipe'],
});
} catch {
this.markNvUnavailable();
return;
}
this.nvChild = child;
this.nvStdoutBuffer = '';
this.nvBatch = [];
this.lastNvLineAt = Date.now();
child.stdout!.on('data', (chunk: Buffer) => this.onNvData(chunk.toString()));
child.stderr!.on('data', () => {
// nvidia-smi warnings are not actionable here
});
child.on('error', (err: NodeJS.ErrnoException) => {
if (err.code === 'ENOENT') {
this.markNvUnavailable();
}
});
child.on('exit', () => {
if (this.nvChild !== child) return;
this.nvChild = null;
if (this.nvFlushTimer) {
clearTimeout(this.nvFlushTimer);
this.nvFlushTimer = null;
}
if (!this.nvUnavailable && !this.nvOneShotMode) {
setTimeout(() => this.startNvLoop(), 5000);
}
});
}
private onNvData(text: string): void {
this.nvStdoutBuffer += text;
const lines = this.nvStdoutBuffer.split('\n');
this.nvStdoutBuffer = lines.pop() || '';
for (const line of lines) {
if (!line.trim()) continue;
const gpu = parseGpuLine(line);
if (!gpu) continue;
this.nvEverGotLine = true;
this.lastNvLineAt = Date.now();
this.nvBatch.push(gpu);
}
if (this.nvFlushTimer) clearTimeout(this.nvFlushTimer);
this.nvFlushTimer = setTimeout(() => this.flushNvBatch(), NV_BATCH_FLUSH_MS);
}
private flushNvBatch(): void {
this.nvFlushTimer = null;
if (this.nvBatch.length === 0) return;
this.latestGpu = {
hasNvidiaSmi: true,
isMac: false,
gpus: this.nvBatch.sort((a, b) => a.index - b.index),
};
this.nvBatch = [];
}
private nvWatchdog(): void {
if (this.nvUnavailable || !this.nvChild) return;
if (Date.now() - this.lastNvLineAt <= NV_WATCHDOG_MS) return;
console.warn('Monitor: no output from nvidia-smi loop, restarting it');
// Loop mode that never produced a single line isn't going to start —
// switch to a one-shot spawn per tick instead of kill/respawn forever.
if (!this.nvEverGotLine) {
this.nvOneShotMode = true;
}
this.nvChild.kill('SIGKILL');
}
private async sampleNvOneShot(): Promise<void> {
if (this.nvUnavailable || this.nvOneShotInFlight) return;
this.nvOneShotInFlight = true;
try {
const { stdout } = await execFileAsync('nvidia-smi', NV_QUERY_ARGS, { env: NV_ENV });
const gpus = stdout
.trim()
.split('\n')
.map(parseGpuLine)
.filter((gpu): gpu is GpuInfo => gpu !== null)
.sort((a, b) => a.index - b.index);
this.latestGpu = { hasNvidiaSmi: true, isMac: false, gpus };
} catch (err) {
if ((err as NodeJS.ErrnoException).code === 'ENOENT') {
this.markNvUnavailable();
} else {
console.error('Monitor: one-shot nvidia-smi failed:', err);
}
} finally {
this.nvOneShotInFlight = false;
}
}
private markNvUnavailable(): void {
this.nvUnavailable = true;
this.latestGpu = {
hasNvidiaSmi: false,
isMac: false,
gpus: [],
error: 'nvidia-smi not found or not accessible',
};
}
}
/**
* Idempotent starter. Guarded on globalThis so dev-mode module reloads never
* stack a second sampler (and a second resident nvidia-smi) in the same
* process.
*/
export function startMonitor(): SystemMonitor {
const g = globalThis as unknown as { __aiToolkitSystemMonitor?: SystemMonitor };
if (!g.__aiToolkitSystemMonitor) {
g.__aiToolkitSystemMonitor = new SystemMonitor();
// Escape hatch: with AI_TOOLKIT_DISABLE_MONITOR=1 the monitor object
// exists (the SSE route stays functional) but never samples anything.
if (process.env.AI_TOOLKIT_DISABLE_MONITOR !== '1') {
g.__aiToolkitSystemMonitor.start();
}
}
return g.__aiToolkitSystemMonitor;
}

Xet Storage Details

Size:
16.7 kB
·
Xet hash:
79877a8022fc3b7c78e2d5bc15daa6da36953695f7ed3dc5153e2eaa1be4512c

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.