/** * src/services/zipService.ts * * All ZIP I/O is handled through Node.js streams to keep RAM usage flat * regardless of archive size — critical on Hugging Face Spaces where * memory is limited (typically 16 GB shared across all processes). * * Key design decisions: * - Extract via `unzipper.Parse({ forceStream: true })` — never buffers * the whole archive. * - Each extracted file is written to disk first, then read; files * exceeding MAX_FILE_SIZE_BYTES are skipped entirely. * - Archives are created via `archiver` which also streams. * - `createTempDir` uses `fs.mkdtemp` for atomic, collision-free paths. * - `cleanup()` must always be called in a `finally` block — see examples * in llmOrchestrator.ts. */ import fs from 'fs'; import path from 'path'; import archiver from 'archiver'; import unzipper from 'unzipper'; import { logger } from '../utils/logger'; import { ENV } from '../config/env'; // ─── Types ──────────────────────────────────────────────────────────────────── export interface ExtractedFile { /** Path relative to the ZIP root, e.g. "src/components/Header.jsx" */ relativePath: string; /** Absolute path on disk inside the temp extraction directory */ absolutePath: string; /** UTF-8 file contents */ content: string; /** Lowercase file extension including dot, e.g. ".jsx" */ extension: string; } // ─── Constants ──────────────────────────────────────────────────────────────── /** Extensions we care about for transpilation. Everything else is drained. */ const RELEVANT_EXTENSIONS = new Set([ '.js', '.jsx', '.ts', '.tsx', '.css', '.scss', '.sass', '.json', '.html', '.svg', '.env', ]); /** Path segments that indicate the file should be skipped entirely. */ const SKIP_SEGMENTS = [ 'node_modules/', '/.git/', '/dist/', '/build/', '/.next/', '/coverage/', '/out/', ]; // ─── Temp Directory Management ──────────────────────────────────────────────── /** * Creates a unique temporary directory. * Uses `fs.mkdtemp` which is atomic and guaranteed collision-free. * * @param prefix - Short label for easier log tracing */ export async function createTempDir(prefix = 'titan-'): Promise { const base = path.join(ENV.TEMP_DIR, prefix); const dir = await fs.promises.mkdtemp(base); logger.debug(`ZipService: created temp dir ${dir}`); return dir; } /** * Recursively removes a directory. Safe to call even if the path doesn't exist. * * ALWAYS call this inside `finally {}` — never rely on GC for disk cleanup. */ export async function cleanup(dirPath: string): Promise { try { await fs.promises.rm(dirPath, { recursive: true, force: true }); logger.info(`ZipService: cleaned up ${dirPath}`); } catch (err) { // Non-fatal: log and continue — a dirty /tmp is better than a crash logger.warn(`ZipService: cleanup failed for "${dirPath}"`, { err: String(err) }); } } // ─── ZIP Extraction ─────────────────────────────────────────────────────────── /** * Extracts a ZIP archive into `destDir` using a streaming parser. * * Memory model: * - Each entry is piped to a WriteStream on disk, never fully buffered. * - After writing, files are stat-checked against MAX_FILE_SIZE_BYTES. * - Only RELEVANT_EXTENSIONS are kept; others are autodrained. * * @param zipPath - Absolute path to the .zip file * @param destDir - Absolute path to the destination directory (must exist or be created) * @returns - Array of text files with their content */ export async function extractZip(zipPath: string, destDir: string): Promise { logger.info(`ZipService: extracting "${path.basename(zipPath)}" → "${destDir}"`); await fs.promises.mkdir(destDir, { recursive: true }); const extracted: ExtractedFile[] = []; await new Promise((resolve, reject) => { const readStream = fs.createReadStream(zipPath); readStream .pipe(unzipper.Parse({ forceStream: true })) .on('entry', (entry: unzipper.Entry) => { const entryPath: string = entry.path; const entryType: string = entry.type; // 'Directory' | 'File' // ── Skip directories ───────────────────────────────────────────── if (entryType === 'Directory') { entry.autodrain(); return; } // ── Skip hidden files and ignored directories ───────────────────── const basename = path.basename(entryPath); const normalised = entryPath.replace(/\\/g, '/'); if ( basename.startsWith('.') || SKIP_SEGMENTS.some(seg => normalised.includes(seg)) ) { entry.autodrain(); return; } // ── Skip irrelevant extensions ──────────────────────────────────── const ext = path.extname(basename).toLowerCase(); if (!RELEVANT_EXTENSIONS.has(ext)) { entry.autodrain(); return; } // ── Stream → disk ───────────────────────────────────────────────── const absolutePath = path.join(destDir, entryPath); const dir = path.dirname(absolutePath); // Create parent dirs synchronously is not possible inside a stream // callback — we queue the work instead. (async () => { try { await fs.promises.mkdir(dir, { recursive: true }); const writeStream = fs.createWriteStream(absolutePath); await new Promise((res, rej) => { entry.pipe(writeStream); writeStream.on('finish', res); writeStream.on('error', rej); entry.on('error', rej); }); // ── Size guard ─────────────────────────────────────────────── const { size } = await fs.promises.stat(absolutePath); if (size > ENV.MAX_FILE_SIZE_BYTES) { logger.warn( `ZipService: skipping "${entryPath}" — ${size} bytes exceeds limit` ); await fs.promises.unlink(absolutePath).catch(() => undefined); return; } const content = await fs.promises.readFile(absolutePath, 'utf-8'); extracted.push({ relativePath: entryPath, absolutePath, content, extension: ext }); } catch (fileErr) { logger.warn(`ZipService: failed to extract "${entryPath}"`, { err: String(fileErr) }); entry.autodrain(); } })(); }) .on('finish', resolve) .on('error', reject); readStream.on('error', reject); }); logger.info(`ZipService: extraction complete — ${extracted.length} usable file(s)`); return extracted; } // ─── ZIP Creation ───────────────────────────────────────────────────────────── /** * Creates a ZIP archive of an entire directory using streaming. * Compression level 9 (maximum). Does NOT load the full archive into RAM. * * @param sourceDir - Root directory whose contents will be zipped * @param outputPath - Destination .zip path */ export async function createZip(sourceDir: string, outputPath: string): Promise { return new Promise((resolve, reject) => { const output = fs.createWriteStream(outputPath); const arc = archiver('zip', { zlib: { level: 9 } }); output.on('close', () => { logger.info( `ZipService: archive created — ${outputPath} (${arc.pointer()} bytes)` ); resolve(); }); arc.on('warning', (warn) => { logger.warn('ZipService: archiver warning', { warn: String(warn) }); }); arc.on('error', reject); arc.pipe(output); // `false` as second arg means files go at the root of the ZIP, not inside a sub-folder arc.directory(sourceDir, false); arc.finalize(); }); } // ─── Flutter Project Packager ───────────────────────────────────────────────── export interface DartFileEntry { /** Path relative to the Flutter `lib/` directory, e.g. "screens/home_screen.dart" */ relativePath: string; content: string; } /** * Writes generated Dart files + pubspec.yaml into a temp directory, * zips the whole thing, and returns both paths for upload and cleanup. * * Layout inside the ZIP: * pubspec.yaml * lib/ * screens/home_screen.dart * models/user_model.dart * ... * * Caller is responsible for calling `cleanup(tempDir)` in a finally block. */ export async function packageFlutterCode( dartFiles: DartFileEntry[], pubspecContent: string ): Promise<{ zipPath: string; tempDir: string }> { const tempDir = await createTempDir('flutter-out-'); const libDir = path.join(tempDir, 'lib'); await fs.promises.mkdir(libDir, { recursive: true }); // Write pubspec.yaml at the project root await fs.promises.writeFile( path.join(tempDir, 'pubspec.yaml'), pubspecContent, 'utf-8' ); // Write each Dart file under lib/ for (const file of dartFiles) { const filePath = path.join(libDir, file.relativePath); await fs.promises.mkdir(path.dirname(filePath), { recursive: true }); await fs.promises.writeFile(filePath, file.content, 'utf-8'); } // Create the ZIP const zipPath = path.join(ENV.TEMP_DIR, `flutter-${Date.now()}.zip`); await createZip(tempDir, zipPath); logger.info( `ZipService: packaged ${dartFiles.length} Dart file(s) → ${path.basename(zipPath)}` ); return { zipPath, tempDir }; }