Buckets:
| /* eslint-disable */ | |
| import { NextRequest, NextResponse } from 'next/server'; | |
| import fs from 'fs'; | |
| import fsp from 'fs/promises'; | |
| import path from 'path'; | |
| import archiver from 'archiver'; | |
| import { getDatasetsRoot, getTrainingFolder } from '@/server/settings'; | |
| export const runtime = 'nodejs'; // ensure Node APIs are available | |
| export const dynamic = 'force-dynamic'; // long-running, non-cached | |
| type PostBody = { | |
| zipTarget: 'samples' | 'dataset' | 'dataset_captions'; | |
| jobName?: string; // required for 'samples' | |
| datasetName?: string; // required for dataset targets | |
| }; | |
| const CAPTION_EXTS = ['.txt', '.json', '.caption']; | |
| async function resolveSafe(p: string) { | |
| // resolve symlinks + normalize | |
| return await fsp.realpath(p); | |
| } | |
| /** | |
| * Recursively collects files to zip, skipping hidden entries (.thumbs etc). | |
| * When captionsOnly is set, only caption files (.txt/.json/.caption) are kept. | |
| */ | |
| async function collectFiles(dir: string, captionsOnly: boolean): Promise<string[]> { | |
| const entries = await fsp.readdir(dir, { withFileTypes: true }); | |
| let out: string[] = []; | |
| for (const entry of entries) { | |
| if (entry.name.startsWith('.')) continue; | |
| const itemPath = path.join(dir, entry.name); | |
| if (entry.isDirectory()) { | |
| out = out.concat(await collectFiles(itemPath, captionsOnly)); | |
| } else if (entry.isFile()) { | |
| if (!captionsOnly || CAPTION_EXTS.includes(path.extname(entry.name).toLowerCase())) { | |
| out.push(itemPath); | |
| } | |
| } | |
| } | |
| return out; | |
| } | |
| async function zipDataset(datasetName: string, captionsOnly: boolean) { | |
| const datasetsRoot = await resolveSafe(await getDatasetsRoot()); | |
| // basename strips any path segments so the name can't escape the datasets root | |
| const safeName = path.basename(datasetName); | |
| let folderPath: string; | |
| try { | |
| folderPath = await resolveSafe(path.join(datasetsRoot, safeName)); | |
| } catch { | |
| return NextResponse.json({ error: `Dataset '${datasetName}' not found` }, { status: 404 }); | |
| } | |
| const files = await collectFiles(folderPath, captionsOnly); | |
| if (files.length === 0) { | |
| return NextResponse.json({ error: captionsOnly ? 'No captions found' : 'Dataset is empty' }, { status: 404 }); | |
| } | |
| // Hidden dir at the datasets root: skipped by the dataset list/image walkers, | |
| // but still under datasetsRoot so /api/files/[...filePath] is allowed to serve it. | |
| const zipDir = path.join(datasetsRoot, '.zips'); | |
| await fsp.mkdir(zipDir, { recursive: true }); | |
| const outputPath = path.join(zipDir, `${safeName}_${captionsOnly ? 'captions' : 'dataset'}.zip`); | |
| await fsp.rm(outputPath, { force: true }); | |
| await new Promise<void>((resolve, reject) => { | |
| const output = fs.createWriteStream(outputPath); | |
| const archive = archiver('zip', { zlib: { level: 9 } }); | |
| output.on('close', () => resolve()); | |
| output.on('error', reject); | |
| archive.on('error', reject); | |
| archive.pipe(output); | |
| for (const file of files) { | |
| archive.file(file, { name: path.join(safeName, path.relative(folderPath, file)) }); | |
| } | |
| archive.finalize().catch(reject); | |
| }); | |
| return NextResponse.json({ | |
| ok: true, | |
| zipPath: outputPath, | |
| fileName: path.basename(outputPath), | |
| }); | |
| } | |
| export async function POST(request: NextRequest) { | |
| try { | |
| const body = (await request.json()) as PostBody; | |
| if (body?.zipTarget === 'dataset' || body?.zipTarget === 'dataset_captions') { | |
| if (!body.datasetName) { | |
| return NextResponse.json({ error: 'datasetName is required' }, { status: 400 }); | |
| } | |
| return await zipDataset(body.datasetName, body.zipTarget === 'dataset_captions'); | |
| } | |
| if (!body || !body.jobName) { | |
| return NextResponse.json({ error: 'jobName is required' }, { status: 400 }); | |
| } | |
| const trainingRoot = await resolveSafe(await getTrainingFolder()); | |
| const folderPath = await resolveSafe(path.join(trainingRoot, body.jobName, 'samples')); | |
| const outputPath = path.resolve(trainingRoot, body.jobName, 'samples.zip'); | |
| // Must be a directory | |
| let stat: fs.Stats; | |
| try { | |
| stat = await fsp.stat(folderPath); | |
| } catch { | |
| return new NextResponse('Folder not found', { status: 404 }); | |
| } | |
| if (!stat.isDirectory()) { | |
| return new NextResponse('Not a directory', { status: 400 }); | |
| } | |
| // delete current one if it exists (force:true => no error if missing) | |
| await fsp.rm(outputPath, { force: true }); | |
| // Create write stream & archive | |
| await new Promise<void>((resolve, reject) => { | |
| const output = fs.createWriteStream(outputPath); | |
| const archive = archiver('zip', { zlib: { level: 9 } }); | |
| output.on('close', () => resolve()); | |
| output.on('error', reject); | |
| archive.on('error', reject); | |
| archive.pipe(output); | |
| // Add the directory contents (place them under the folder's base name in the zip) | |
| const rootName = path.basename(folderPath); | |
| archive.directory(folderPath, rootName); | |
| archive.finalize().catch(reject); | |
| }); | |
| // Return the absolute path so your existing /api/files/[...filePath] can serve it | |
| // Example download URL (client-side): `/api/files/${encodeURIComponent(resolvedOutPath)}` | |
| return NextResponse.json({ | |
| ok: true, | |
| zipPath: outputPath, | |
| fileName: path.basename(outputPath), | |
| }); | |
| } catch (err) { | |
| console.error('Zip error:', err); | |
| return new NextResponse('Internal Server Error', { status: 500 }); | |
| } | |
| } | |
Xet Storage Details
- Size:
- 5.43 kB
- Xet hash:
- fd875fbb8c5af1e19b8f13399530180d5a982d22ea7b907a9b9f9a6e264208b1
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.