Quantumbraid's picture
download
raw
5.43 kB
/* eslint-disable */
import { NextRequest, NextResponse } from 'next/server';
import fs from 'fs';
import fsp from 'fs/promises';
import path from 'path';
import archiver from 'archiver';
import { getDatasetsRoot, getTrainingFolder } from '@/server/settings';
export const runtime = 'nodejs'; // ensure Node APIs are available
export const dynamic = 'force-dynamic'; // long-running, non-cached
type PostBody = {
zipTarget: 'samples' | 'dataset' | 'dataset_captions';
jobName?: string; // required for 'samples'
datasetName?: string; // required for dataset targets
};
const CAPTION_EXTS = ['.txt', '.json', '.caption'];
async function resolveSafe(p: string) {
// resolve symlinks + normalize
return await fsp.realpath(p);
}
/**
* Recursively collects files to zip, skipping hidden entries (.thumbs etc).
* When captionsOnly is set, only caption files (.txt/.json/.caption) are kept.
*/
async function collectFiles(dir: string, captionsOnly: boolean): Promise<string[]> {
const entries = await fsp.readdir(dir, { withFileTypes: true });
let out: string[] = [];
for (const entry of entries) {
if (entry.name.startsWith('.')) continue;
const itemPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
out = out.concat(await collectFiles(itemPath, captionsOnly));
} else if (entry.isFile()) {
if (!captionsOnly || CAPTION_EXTS.includes(path.extname(entry.name).toLowerCase())) {
out.push(itemPath);
}
}
}
return out;
}
async function zipDataset(datasetName: string, captionsOnly: boolean) {
const datasetsRoot = await resolveSafe(await getDatasetsRoot());
// basename strips any path segments so the name can't escape the datasets root
const safeName = path.basename(datasetName);
let folderPath: string;
try {
folderPath = await resolveSafe(path.join(datasetsRoot, safeName));
} catch {
return NextResponse.json({ error: `Dataset '${datasetName}' not found` }, { status: 404 });
}
const files = await collectFiles(folderPath, captionsOnly);
if (files.length === 0) {
return NextResponse.json({ error: captionsOnly ? 'No captions found' : 'Dataset is empty' }, { status: 404 });
}
// Hidden dir at the datasets root: skipped by the dataset list/image walkers,
// but still under datasetsRoot so /api/files/[...filePath] is allowed to serve it.
const zipDir = path.join(datasetsRoot, '.zips');
await fsp.mkdir(zipDir, { recursive: true });
const outputPath = path.join(zipDir, `${safeName}_${captionsOnly ? 'captions' : 'dataset'}.zip`);
await fsp.rm(outputPath, { force: true });
await new Promise<void>((resolve, reject) => {
const output = fs.createWriteStream(outputPath);
const archive = archiver('zip', { zlib: { level: 9 } });
output.on('close', () => resolve());
output.on('error', reject);
archive.on('error', reject);
archive.pipe(output);
for (const file of files) {
archive.file(file, { name: path.join(safeName, path.relative(folderPath, file)) });
}
archive.finalize().catch(reject);
});
return NextResponse.json({
ok: true,
zipPath: outputPath,
fileName: path.basename(outputPath),
});
}
export async function POST(request: NextRequest) {
try {
const body = (await request.json()) as PostBody;
if (body?.zipTarget === 'dataset' || body?.zipTarget === 'dataset_captions') {
if (!body.datasetName) {
return NextResponse.json({ error: 'datasetName is required' }, { status: 400 });
}
return await zipDataset(body.datasetName, body.zipTarget === 'dataset_captions');
}
if (!body || !body.jobName) {
return NextResponse.json({ error: 'jobName is required' }, { status: 400 });
}
const trainingRoot = await resolveSafe(await getTrainingFolder());
const folderPath = await resolveSafe(path.join(trainingRoot, body.jobName, 'samples'));
const outputPath = path.resolve(trainingRoot, body.jobName, 'samples.zip');
// Must be a directory
let stat: fs.Stats;
try {
stat = await fsp.stat(folderPath);
} catch {
return new NextResponse('Folder not found', { status: 404 });
}
if (!stat.isDirectory()) {
return new NextResponse('Not a directory', { status: 400 });
}
// delete current one if it exists (force:true => no error if missing)
await fsp.rm(outputPath, { force: true });
// Create write stream & archive
await new Promise<void>((resolve, reject) => {
const output = fs.createWriteStream(outputPath);
const archive = archiver('zip', { zlib: { level: 9 } });
output.on('close', () => resolve());
output.on('error', reject);
archive.on('error', reject);
archive.pipe(output);
// Add the directory contents (place them under the folder's base name in the zip)
const rootName = path.basename(folderPath);
archive.directory(folderPath, rootName);
archive.finalize().catch(reject);
});
// Return the absolute path so your existing /api/files/[...filePath] can serve it
// Example download URL (client-side): `/api/files/${encodeURIComponent(resolvedOutPath)}`
return NextResponse.json({
ok: true,
zipPath: outputPath,
fileName: path.basename(outputPath),
});
} catch (err) {
console.error('Zip error:', err);
return new NextResponse('Internal Server Error', { status: 500 });
}
}

Xet Storage Details

Size:
5.43 kB
·
Xet hash:
fd875fbb8c5af1e19b8f13399530180d5a982d22ea7b907a9b9f9a6e264208b1

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.