Update OCR Dataset Generator

web/app/api/storage/route.ts

@@ -0,0 +1,186 @@
+import { NextRequest, NextResponse } from 'next/server'
+import { writeFile, mkdir, readdir, rm, stat } from 'fs/promises'
+import { existsSync } from 'fs'
+import path from 'path'
+
+// Storage directories
+const OUTPUT_DIR = process.env.OUTPUT_DIR || './output/datasets'
+const DATA_DIR = process.env.DATA_DIR || '/data/datasets'
+const CLEANUP_HOURS = parseInt(process.env.CLEANUP_HOURS || '24')
+
+// Get the appropriate directory based on mode
+function getBaseDir(mode: 'local' | 'huggingface'): string {
+    return mode === 'huggingface' ? DATA_DIR : OUTPUT_DIR
+}
+
+// Ensure directory exists
+async function ensureDir(dirPath: string): Promise<void> {
+    if (!existsSync(dirPath)) {
+        await mkdir(dirPath, { recursive: true })
+    }
+}
+
+// POST - Write file or finalize
+export async function POST(request: NextRequest) {
+    try {
+        const contentType = request.headers.get('content-type') || ''
+
+        if (contentType.includes('multipart/form-data')) {
+            // Handle file upload
+            const formData = await request.formData()
+            const file = formData.get('file') as File
+            const sessionId = formData.get('sessionId') as string
+            const mode = (formData.get('mode') as 'local' | 'huggingface') || 'local'
+
+            if (!file || !sessionId) {
+                return NextResponse.json({ error: 'Missing file or sessionId' }, { status: 400 })
+            }
+
+            const baseDir = getBaseDir(mode)
+            const sessionDir = path.join(baseDir, sessionId)
+            const imagesDir = path.join(sessionDir, 'images')
+            await ensureDir(imagesDir)
+
+            const buffer = Buffer.from(await file.arrayBuffer())
+            const filePath = path.join(imagesDir, file.name)
+            await writeFile(filePath, buffer)
+
+            return NextResponse.json({ success: true, path: filePath })
+        } else {
+            // Handle JSON requests
+            const body = await request.json()
+            const { sessionId, mode, type, content } = body
+
+            if (!sessionId || !mode || !type) {
+                return NextResponse.json({ error: 'Missing required fields' }, { status: 400 })
+            }
+
+            const baseDir = getBaseDir(mode)
+            const sessionDir = path.join(baseDir, sessionId)
+            await ensureDir(sessionDir)
+
+            switch (type) {
+                case 'labels':
+                    await writeFile(path.join(sessionDir, 'labels.txt'), content)
+                    return NextResponse.json({ success: true })
+
+                case 'metadata':
+                    await writeFile(path.join(sessionDir, 'metadata.json'), content)
+                    return NextResponse.json({ success: true })
+
+                case 'finalize':
+                    // Mark as complete by writing a timestamp file
+                    await writeFile(
+                        path.join(sessionDir, '.complete'),
+                        new Date().toISOString()
+                    )
+                    return NextResponse.json({
+                        success: true,
+                        path: sessionDir,
+                        downloadUrl: `/api/storage/download?session=${sessionId}&mode=${mode}`
+                    })
+
+                default:
+                    return NextResponse.json({ error: 'Unknown type' }, { status: 400 })
+            }
+        }
+    } catch (error) {
+        console.error('Storage API error:', error)
+        return NextResponse.json({ error: 'Internal server error' }, { status: 500 })
+    }
+}
+
+// GET - List datasets or get config
+export async function GET(request: NextRequest) {
+    try {
+        const { searchParams } = new URL(request.url)
+        const action = searchParams.get('action')
+        const mode = (searchParams.get('mode') as 'local' | 'huggingface') || 'local'
+
+        switch (action) {
+            case 'list':
+                const baseDir = getBaseDir(mode)
+                if (!existsSync(baseDir)) {
+                    return NextResponse.json({ datasets: [] })
+                }
+
+                const entries = await readdir(baseDir, { withFileTypes: true })
+                const datasets = await Promise.all(
+                    entries
+                        .filter(e => e.isDirectory())
+                        .map(async (entry) => {
+                            const datasetPath = path.join(baseDir, entry.name)
+                            const stats = await stat(datasetPath)
+                            const completeFile = path.join(datasetPath, '.complete')
+                            const isComplete = existsSync(completeFile)
+
+                            return {
+                                name: entry.name,
+                                path: datasetPath,
+                                createdAt: stats.birthtime.toISOString(),
+                                isComplete,
+                                ageHours: (Date.now() - stats.birthtime.getTime()) / (1000 * 60 * 60)
+                            }
+                        })
+                )
+
+                return NextResponse.json({ datasets })
+
+            case 'config':
+                return NextResponse.json({
+                    outputDir: OUTPUT_DIR,
+                    dataDir: DATA_DIR,
+                    cleanupHours: CLEANUP_HOURS,
+                    isHuggingFace: existsSync('/data')
+                })
+
+            default:
+                return NextResponse.json({ error: 'Unknown action' }, { status: 400 })
+        }
+    } catch (error) {
+        console.error('Storage GET error:', error)
+        return NextResponse.json({ error: 'Internal server error' }, { status: 500 })
+    }
+}
+
+// DELETE - Cleanup old datasets
+export async function DELETE(request: NextRequest) {
+    try {
+        const { searchParams } = new URL(request.url)
+        const mode = (searchParams.get('mode') as 'local' | 'huggingface') || 'local'
+        const maxAgeHours = parseInt(searchParams.get('maxAge') || String(CLEANUP_HOURS))
+
+        const baseDir = getBaseDir(mode)
+        if (!existsSync(baseDir)) {
+            return NextResponse.json({ cleaned: 0, message: 'No datasets directory' })
+        }
+
+        const entries = await readdir(baseDir, { withFileTypes: true })
+        let cleaned = 0
+        const cleanedPaths: string[] = []
+
+        for (const entry of entries) {
+            if (!entry.isDirectory()) continue
+
+            const datasetPath = path.join(baseDir, entry.name)
+            const stats = await stat(datasetPath)
+            const ageHours = (Date.now() - stats.birthtime.getTime()) / (1000 * 60 * 60)
+
+            if (ageHours > maxAgeHours) {
+                await rm(datasetPath, { recursive: true, force: true })
+                cleaned++
+                cleanedPaths.push(entry.name)
+                console.log(`Cleaned up old dataset: ${entry.name} (${ageHours.toFixed(1)} hours old)`)
+            }
+        }
+
+        return NextResponse.json({
+            cleaned,
+            cleanedPaths,
+            message: `Cleaned ${cleaned} datasets older than ${maxAgeHours} hours`
+        })
+    } catch (error) {
+        console.error('Cleanup error:', error)
+        return NextResponse.json({ error: 'Internal server error' }, { status: 500 })
+    }
+}

web/components/generation-panel.tsx

@@ -1,10 +1,12 @@
 'use client'
 
 import { useState, useEffect, useRef, Dispatch, SetStateAction } from 'react'
-import { Play, Pause, StopCircle, Download, CheckCircle2, FileArchive, AlertCircle, DownloadCloud, Cpu, Zap, Layers, HelpCircle } from 'lucide-react'
+import { Play, Pause, StopCircle, Download, CheckCircle2, FileArchive, AlertCircle, DownloadCloud, Cpu, Zap, Layers, HelpCircle, FolderOpen, Database, Cloud, HardDrive, Settings2 } from 'lucide-react'
 import { GenerationStats } from './stats-panel'
-import { generateDataset, downloadDataset, GeneratorConfig, getGenerationState, buildPartialZip } from '@/lib/generator'
+import { generateDataset, downloadDataset, GeneratorConfig, getGenerationState, buildPartialZip, clearCanvasPool } from '@/lib/generator'
 import { isWebGLAvailable } from '@/lib/gpu-augmentation'
+import { StorageMode, getStorageManager, resetStorageManager } from '@/lib/storage-manager'
+import { generateDatasetDirect, isDirectGenerationSupported, DirectGenerationConfig } from '@/lib/direct-generator'
 
 interface GenerationPanelProps {
     config: any
@@ -46,6 +48,19 @@ export function GenerationPanel({
     const logsEndRef = useRef<HTMLDivElement>(null)
     const abortControllerRef = useRef<AbortController | null>(null)
 
+    // Storage mode state
+    const [storageMode, setStorageMode] = useState<StorageMode>('memory')
+    const [selectedFolder, setSelectedFolder] = useState<string | null>(null)
+    const [outputPath, setOutputPath] = useState<string | null>(null)
+    const [showStorageSettings, setShowStorageSettings] = useState(false)
+
+    // Directory handle for local folder mode (File System Access API)
+    const directoryHandleRef = useRef<FileSystemDirectoryHandle | null>(null)
+
+    // Rolling rate calculation (last 100 samples)
+    const recentSamples = useRef<{ time: number; count: number }[]>([])
+    const ROLLING_WINDOW = 100  // Calculate rate from last 100 samples
+
     const scrollToBottom = () => {
         logsEndRef.current?.scrollIntoView({ behavior: 'smooth' })
     }
@@ -123,6 +138,85 @@ export function GenerationPanel({
         ])
 
         try {
+            // === LOCAL FOLDER MODE: Direct-to-disk generation ===
+            if (storageMode === 'local' && directoryHandleRef.current) {
+                setLogs(prev => [...prev, `[${new Date().toLocaleTimeString()}] 📂 Using DIRECT-TO-DISK mode (no memory limits)`]);
+
+                const directConfig: DirectGenerationConfig = {
+                    dataset: config.dataset,
+                    image: config.image,
+                    fonts: config.fonts,
+                    augmentation: config.augmentation,
+                    output: config.output
+                };
+
+                const genStartTime = Date.now();
+                const directResult = await generateDatasetDirect(
+                    directoryHandleRef.current,
+                    directConfig,
+                    data,
+                    (prog, message) => {
+                        if (abortControllerRef.current?.signal.aborted) return;
+                        setProgress(prog);
+
+                        const elapsed = (Date.now() - genStartTime) / 1000;
+                        if (elapsed > 0 && prog > 0) {
+                            const samplesGenerated = Math.round(prog * targetSize / 100);
+                            setRate(Math.round(samplesGenerated / elapsed));
+
+                            const remaining = (100 - prog) / prog * elapsed;
+                            if (remaining < 60) {
+                                setEta(`${Math.round(remaining)}s`);
+                            } else if (remaining < 3600) {
+                                setEta(`${Math.round(remaining / 60)}m ${Math.round(remaining % 60)}s`);
+                            } else {
+                                setEta(`${Math.floor(remaining / 3600)}h ${Math.round((remaining % 3600) / 60)}m`);
+                            }
+                        }
+
+                        if (prog % 10 === 0 || prog >= 99) {
+                            setLogs(prev => [...prev, `[${new Date().toLocaleTimeString()}] ${message}`]);
+                        }
+                    },
+                    abortControllerRef.current.signal
+                );
+
+                setGenerationStatus('completed');
+                setIsGenerating(false);
+                setOutputPath(directResult.outputPath);
+
+                setLogs(prev => [
+                    ...prev,
+                    `[${new Date().toLocaleTimeString()}] ✅ Generation complete!`,
+                    `[${new Date().toLocaleTimeString()}] 📁 Files saved to: ${directResult.outputPath}`,
+                    `[${new Date().toLocaleTimeString()}] 📊 Total samples: ${directResult.totalSamples.toLocaleString()}`,
+                    `[${new Date().toLocaleTimeString()}] ⏱️ Duration: ${directResult.durationSeconds.toFixed(1)}s`,
+                    `[${new Date().toLocaleTimeString()}] 💡 No ZIP needed - files are ready to use!`,
+                ]);
+
+                // Call onGenerationComplete with minimal stats (no ZIP blob)
+                if (onGenerationComplete) {
+                    onGenerationComplete({
+                        total_samples: directResult.totalSamples,
+                        duration_seconds: directResult.durationSeconds,
+                        samples_per_second: directResult.totalSamples / directResult.durationSeconds,
+                        font_distribution: [],
+                        clean_samples: directResult.cleanSamples,
+                        augmented_samples: directResult.augmentedSamples,
+                        augmentation_stats: [],
+                        avg_transforms_per_sample: 0,
+                        unique_tokens: data.length,
+                        avg_chars_per_sample: 0,
+                        unicode_valid: directResult.totalSamples,
+                        script_pure: directResult.totalSamples,
+                        rejected_samples: 0
+                    }, new Blob()); // Empty blob since files are on disk
+                }
+
+                return; // Exit early - no ZIP needed
+            }
+
+            // === MEMORY MODE: Original ZIP-based generation ===
             const generatorConfig: GeneratorConfig = {
                 dataset: config.dataset,
                 input: {
@@ -501,8 +595,8 @@ export function GenerationPanel({
                                 <span className="text-sm">Augmentation</span>
                             </div>
                             <span className={`text-xs font-bold px-2 py-1 rounded ${isWebGLAvailable()
-                                    ? 'bg-green-500/20 text-green-400'
-                                    : 'bg-blue-500/20 text-blue-400'
+                                ? 'bg-green-500/20 text-green-400'
+                                : 'bg-blue-500/20 text-blue-400'
                                 }`}>
                                 {isWebGLAvailable() ? '🎮 GPU' : '💻 CPU'}
                             </span>
@@ -533,6 +627,161 @@ export function GenerationPanel({
                         </div>
                     </div>
                 </div>
+
+                {/* Storage Settings Panel */}
+                <div className="glass rounded-xl p-6">
+                    <div className="flex items-center justify-between mb-4">
+                        <h3 className="font-medium flex items-center gap-2">
+                            <Database className="w-4 h-4 text-blue-400" />
+                            Storage Settings
+                        </h3>
+                        <button
+                            onClick={() => setShowStorageSettings(!showStorageSettings)}
+                            className="text-xs text-muted-foreground hover:text-primary transition-colors flex items-center gap-1"
+                        >
+                            <Settings2 className="w-3 h-3" />
+                            {showStorageSettings ? 'Hide' : 'Configure'}
+                        </button>
+                    </div>
+
+                    {/* Current Mode Display */}
+                    <div className="flex items-center justify-between p-2 rounded-lg bg-secondary/50 mb-3">
+                        <div className="flex items-center gap-2">
+                            {storageMode === 'memory' && <HardDrive className="w-4 h-4 text-yellow-400" />}
+                            {storageMode === 'indexeddb' && <Database className="w-4 h-4 text-blue-400" />}
+                            {storageMode === 'local' && <FolderOpen className="w-4 h-4 text-green-400" />}
+                            {storageMode === 'huggingface' && <Cloud className="w-4 h-4 text-purple-400" />}
+                            <span className="text-sm">Storage Mode</span>
+                        </div>
+                        <span className={`text-xs font-bold px-2 py-1 rounded ${storageMode === 'memory' ? 'bg-yellow-500/20 text-yellow-400' :
+                            storageMode === 'indexeddb' ? 'bg-blue-500/20 text-blue-400' :
+                                storageMode === 'local' ? 'bg-green-500/20 text-green-400' :
+                                    'bg-purple-500/20 text-purple-400'
+                            }`}>
+                            {storageMode === 'memory' ? '💾 Memory' :
+                                storageMode === 'indexeddb' ? '🗄️ Browser' :
+                                    storageMode === 'local' ? '📁 Local' :
+                                        '🤗 HuggingFace'}
+                        </span>
+                    </div>
+
+                    {/* Expanded Settings */}
+                    {showStorageSettings && (
+                        <div className="space-y-3 pt-2 border-t border-white/10">
+                            <p className="text-xs text-muted-foreground">
+                                Select where to store generated images:
+                            </p>
+
+                            {/* Storage Mode Buttons */}
+                            <div className="grid grid-cols-2 gap-2">
+                                <button
+                                    onClick={() => { setStorageMode('memory'); setSelectedFolder(null); }}
+                                    className={`p-2 rounded-lg text-xs flex flex-col items-center gap-1 transition-all ${storageMode === 'memory'
+                                        ? 'bg-yellow-500/20 text-yellow-400 ring-1 ring-yellow-500/50'
+                                        : 'bg-secondary/50 hover:bg-secondary'
+                                        }`}
+                                >
+                                    <HardDrive className="w-4 h-4" />
+                                    <span>Memory</span>
+                                    <span className="text-[10px] opacity-60">Fast, &lt;2k samples</span>
+                                </button>
+
+                                <button
+                                    onClick={() => { setStorageMode('indexeddb'); setSelectedFolder(null); }}
+                                    className={`p-2 rounded-lg text-xs flex flex-col items-center gap-1 transition-all ${storageMode === 'indexeddb'
+                                        ? 'bg-blue-500/20 text-blue-400 ring-1 ring-blue-500/50'
+                                        : 'bg-secondary/50 hover:bg-secondary'
+                                        }`}
+                                >
+                                    <Database className="w-4 h-4" />
+                                    <span>Browser DB</span>
+                                    <span className="text-[10px] opacity-60">Persistent, resumable</span>
+                                </button>
+
+                                <button
+                                    onClick={async () => {
+                                        // Check if File System API is supported
+                                        if (!('showDirectoryPicker' in window)) {
+                                            setError('Folder picker not supported in this browser. Use Chrome, Edge, or Opera.');
+                                            setLogs(prev => [...prev, `[${new Date().toLocaleTimeString()}] ❌ File System API not supported - try Chrome or Edge`]);
+                                            return;
+                                        }
+
+                                        setLogs(prev => [...prev, `[${new Date().toLocaleTimeString()}] 📂 Opening folder picker...`]);
+
+                                        try {
+                                            // Directly use the File System Access API
+                                            // @ts-ignore - File System Access API
+                                            const directoryHandle = await window.showDirectoryPicker({
+                                                mode: 'readwrite',
+                                                startIn: 'documents'
+                                            });
+
+                                            // Store handle for generation - don't pre-create folders
+                                            directoryHandleRef.current = directoryHandle;
+
+                                            setStorageMode('local');
+                                            setSelectedFolder(directoryHandle.name);
+                                            setLogs(prev => [...prev, `[${new Date().toLocaleTimeString()}] ✅ Output folder: ${directoryHandle.name} (direct-to-disk mode enabled)`]);
+                                            setLogs(prev => [...prev, `[${new Date().toLocaleTimeString()}] 💡 Files will be written directly - no memory limits!`]);
+
+                                        } catch (err: any) {
+                                            if (err.name === 'AbortError') {
+                                                // User cancelled the picker
+                                                setLogs(prev => [...prev, `[${new Date().toLocaleTimeString()}] ℹ️ Folder selection cancelled`]);
+                                            } else {
+                                                console.error('Folder selection failed:', err);
+                                                setError(`Failed to select folder: ${err.message}`);
+                                                setLogs(prev => [...prev, `[${new Date().toLocaleTimeString()}] ❌ Error: ${err.message}`]);
+                                            }
+                                        }
+                                    }}
+                                    className={`p-2 rounded-lg text-xs flex flex-col items-center gap-1 transition-all ${storageMode === 'local'
+                                        ? 'bg-green-500/20 text-green-400 ring-1 ring-green-500/50'
+                                        : 'bg-secondary/50 hover:bg-secondary'
+                                        }`}
+                                >
+                                    <FolderOpen className="w-4 h-4" />
+                                    <span>Local Folder</span>
+                                    <span className="text-[10px] opacity-60">Direct disk access</span>
+                                </button>
+
+                                <button
+                                    onClick={() => { setStorageMode('huggingface'); setSelectedFolder(null); }}
+                                    className={`p-2 rounded-lg text-xs flex flex-col items-center gap-1 transition-all ${storageMode === 'huggingface'
+                                        ? 'bg-purple-500/20 text-purple-400 ring-1 ring-purple-500/50'
+                                        : 'bg-secondary/50 hover:bg-secondary'
+                                        }`}
+                                >
+                                    <Cloud className="w-4 h-4" />
+                                    <span>HuggingFace</span>
+                                    <span className="text-[10px] opacity-60">Cloud storage</span>
+                                </button>
+                            </div>
+
+                            {/* Selected Folder Display */}
+                            {storageMode === 'local' && selectedFolder && (
+                                <div className="p-2 rounded-lg bg-green-500/10 text-xs">
+                                    <span className="text-green-400">📂 Output: </span>
+                                    <span className="text-muted-foreground">{selectedFolder}</span>
+                                </div>
+                            )}
+
+                            {/* Output Path Display */}
+                            {outputPath && (
+                                <div className="p-2 rounded-lg bg-blue-500/10 text-xs">
+                                    <span className="text-blue-400">✅ Saved to: </span>
+                                    <span className="text-muted-foreground">{outputPath}</span>
+                                </div>
+                            )}
+
+                            {/* Tip */}
+                            <p className="text-[10px] text-muted-foreground">
+                                💡 For large datasets (5k+), use Browser DB or Local Folder to avoid memory issues.
+                            </p>
+                        </div>
+                    )}
+                </div>
             </div>
 
             {/* Progress & Logs */}

web/lib/cleanup.ts

@@ -0,0 +1,136 @@
+// Auto-cleanup service for old datasets
+// Runs on server startup and can be triggered via API
+
+import { readdir, stat, rm } from 'fs/promises'
+import { existsSync } from 'fs'
+import path from 'path'
+
+const OUTPUT_DIR = process.env.OUTPUT_DIR || './output/datasets'
+const DATA_DIR = process.env.DATA_DIR || '/data/datasets'
+const CLEANUP_HOURS = parseInt(process.env.CLEANUP_HOURS || '24')
+
+export interface CleanupResult {
+    cleaned: number
+    cleanedPaths: string[]
+    errors: string[]
+}
+
+/**
+ * Clean up datasets older than the specified age
+ */
+export async function cleanupOldDatasets(
+    maxAgeHours: number = CLEANUP_HOURS,
+    mode: 'local' | 'huggingface' | 'both' = 'both'
+): Promise<CleanupResult> {
+    const result: CleanupResult = {
+        cleaned: 0,
+        cleanedPaths: [],
+        errors: []
+    }
+
+    const dirsToClean: string[] = []
+    if (mode === 'local' || mode === 'both') {
+        dirsToClean.push(OUTPUT_DIR)
+    }
+    if (mode === 'huggingface' || mode === 'both') {
+        if (existsSync(DATA_DIR)) {
+            dirsToClean.push(DATA_DIR)
+        }
+    }
+
+    for (const baseDir of dirsToClean) {
+        if (!existsSync(baseDir)) continue
+
+        try {
+            const entries = await readdir(baseDir, { withFileTypes: true })
+
+            for (const entry of entries) {
+                if (!entry.isDirectory()) continue
+
+                const datasetPath = path.join(baseDir, entry.name)
+                try {
+                    const stats = await stat(datasetPath)
+                    const ageHours = (Date.now() - stats.birthtime.getTime()) / (1000 * 60 * 60)
+
+                    if (ageHours > maxAgeHours) {
+                        await rm(datasetPath, { recursive: true, force: true })
+                        result.cleaned++
+                        result.cleanedPaths.push(datasetPath)
+                        console.log(`[Cleanup] Removed: ${entry.name} (${ageHours.toFixed(1)}h old)`)
+                    }
+                } catch (err) {
+                    const errorMsg = `Failed to clean ${datasetPath}: ${err}`
+                    console.error(`[Cleanup] ${errorMsg}`)
+                    result.errors.push(errorMsg)
+                }
+            }
+        } catch (err) {
+            const errorMsg = `Failed to read ${baseDir}: ${err}`
+            console.error(`[Cleanup] ${errorMsg}`)
+            result.errors.push(errorMsg)
+        }
+    }
+
+    console.log(`[Cleanup] Complete: ${result.cleaned} datasets removed`)
+    return result
+}
+
+/**
+ * Schedule cleanup to run periodically (call from server initialization)
+ */
+export function scheduleCleanup(intervalHours: number = 1): NodeJS.Timer {
+    console.log(`[Cleanup] Scheduled every ${intervalHours} hour(s), removing datasets older than ${CLEANUP_HOURS} hours`)
+
+    // Run immediately on startup
+    cleanupOldDatasets().catch(console.error)
+
+    // Then run periodically
+    return setInterval(() => {
+        cleanupOldDatasets().catch(console.error)
+    }, intervalHours * 60 * 60 * 1000)
+}
+
+/**
+ * Get storage statistics
+ */
+export async function getStorageStats(): Promise<{
+    localDatasets: number
+    hfDatasets: number
+    totalSizeMB: number
+}> {
+    let localDatasets = 0
+    let hfDatasets = 0
+    let totalSizeBytes = 0
+
+    async function countDir(dir: string): Promise<{ count: number; size: number }> {
+        if (!existsSync(dir)) return { count: 0, size: 0 }
+
+        const entries = await readdir(dir, { withFileTypes: true })
+        let count = 0
+        let size = 0
+
+        for (const entry of entries) {
+            if (entry.isDirectory()) {
+                count++
+                // Estimate size (could be more accurate with recursive size calculation)
+                const stats = await stat(path.join(dir, entry.name))
+                size += stats.size
+            }
+        }
+
+        return { count, size }
+    }
+
+    const localStats = await countDir(OUTPUT_DIR)
+    const hfStats = await countDir(DATA_DIR)
+
+    localDatasets = localStats.count
+    hfDatasets = hfStats.count
+    totalSizeBytes = localStats.size + hfStats.size
+
+    return {
+        localDatasets,
+        hfDatasets,
+        totalSizeMB: Math.round(totalSizeBytes / (1024 * 1024))
+    }
+}

web/lib/direct-generator.ts

@@ -0,0 +1,307 @@
+// Direct-to-disk dataset generation
+// Writes files directly to a user-selected folder without building ZIP in memory
+
+export interface DirectGenerationConfig {
+    dataset: {
+        size: number
+        seed: number
+    }
+    image: {
+        width: number
+        height: number
+        background: string
+        backgroundStyle?: string
+        backgroundMode?: 'single' | 'mix'
+        backgroundPercentages?: Record<string, number>
+        textColor: string
+        direction: string
+    }
+    fonts: {
+        distribution: { name: string; family: string; percentage: number; dataUrl?: string }[]
+    }
+    augmentation: {
+        enabled: boolean
+        applyPercentage: number
+        preset: string
+        customMode?: boolean
+        values?: Record<string, number>
+    }
+    output: {
+        formats: string[]
+    }
+}
+
+export interface DirectGenerationResult {
+    totalSamples: number
+    durationSeconds: number
+    outputPath: string
+    cleanSamples: number
+    augmentedSamples: number
+}
+
+// Background colors
+const backgroundColors: Record<string, string> = {
+    clean_white: '#FFFFFF',
+    aged_paper: '#F5E6D3',
+    book_page: '#FAF0E6',
+    newspaper: '#E8E8E8',
+    notebook: '#FFFEF0',
+    parchment: '#F0E68C',
+    weathered: '#D4C4A8',
+    coffee_stain: '#E6D5C3',
+    old_book: '#E8DCC4',
+    recycled: '#D9D4C5',
+    cream: '#FFFDD0',
+    ivory: '#FFFFF0',
+}
+
+// Simple seeded random
+function seededRandom(seed: number) {
+    let s = seed
+    return function () {
+        s = Math.sin(s) * 10000
+        return s - Math.floor(s)
+    }
+}
+
+// Canvas pool for performance
+const canvasPool: { canvas: HTMLCanvasElement; ctx: CanvasRenderingContext2D; inUse: boolean }[] = []
+
+function getCanvas(width: number, height: number): { canvas: HTMLCanvasElement; ctx: CanvasRenderingContext2D } {
+    for (const item of canvasPool) {
+        if (!item.inUse && item.canvas.width === width && item.canvas.height === height) {
+            item.inUse = true
+            item.ctx.clearRect(0, 0, width, height)
+            item.ctx.setTransform(1, 0, 0, 1, 0, 0)
+            return { canvas: item.canvas, ctx: item.ctx }
+        }
+    }
+
+    if (canvasPool.length < 4) {
+        const canvas = document.createElement('canvas')
+        canvas.width = width
+        canvas.height = height
+        const ctx = canvas.getContext('2d', { willReadFrequently: true })!
+        canvasPool.push({ canvas, ctx, inUse: true })
+        return { canvas, ctx }
+    }
+
+    const canvas = document.createElement('canvas')
+    canvas.width = width
+    canvas.height = height
+    const ctx = canvas.getContext('2d', { willReadFrequently: true })!
+    return { canvas, ctx }
+}
+
+function releaseCanvas(canvas: HTMLCanvasElement) {
+    for (const item of canvasPool) {
+        if (item.canvas === canvas) {
+            item.inUse = false
+            return
+        }
+    }
+}
+
+/**
+ * Generate dataset directly to a local folder
+ * No ZIP, no memory accumulation - writes each file immediately
+ */
+export async function generateDatasetDirect(
+    directoryHandle: FileSystemDirectoryHandle,
+    config: DirectGenerationConfig,
+    textData: string[],
+    onProgress: (progress: number, message: string) => void,
+    abortSignal?: AbortSignal
+): Promise<DirectGenerationResult> {
+    const startTime = Date.now()
+
+    // Create dataset folder
+    const datasetName = `dataset_${new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19)}`
+    const datasetHandle = await directoryHandle.getDirectoryHandle(datasetName, { create: true })
+    const imagesHandle = await datasetHandle.getDirectoryHandle('images', { create: true })
+
+    onProgress(1, `Created folder: ${directoryHandle.name}/${datasetName}`)
+
+    const numSamples = Math.min(config.dataset.size, textData.length)
+    const labels: string[] = []
+    let cleanCount = 0
+    let augmentedCount = 0
+
+    // Load fonts
+    const loadedFonts: Map<string, string> = new Map()
+    if (config.fonts.distribution?.length) {
+        for (const font of config.fonts.distribution) {
+            if (font.dataUrl) {
+                try {
+                    const fontFace = new FontFace(font.family, `url(${font.dataUrl})`)
+                    await fontFace.load()
+                    document.fonts.add(fontFace)
+                    loadedFonts.set(font.name, font.family)
+                } catch {
+                    loadedFonts.set(font.name, 'Arial')
+                }
+            } else {
+                loadedFonts.set(font.name, font.family || 'Arial')
+            }
+        }
+    }
+
+    onProgress(3, `Loaded ${loadedFonts.size} fonts, starting generation...`)
+
+    // Get augmentation values
+    const augValues = config.augmentation.values || {
+        rotation: 5, skew: 3, gaussian_noise: 10, brightness: 15
+    }
+
+    // Generate one sample at a time, write immediately
+    for (let i = 0; i < numSamples; i++) {
+        if (abortSignal?.aborted) {
+            throw new Error('Generation aborted')
+        }
+
+        const text = textData[i % textData.length]
+        const random = seededRandom(config.dataset.seed + i * 1000)
+
+        // Select font
+        let fontFamily = 'Arial'
+        if (config.fonts.distribution?.length) {
+            const roll = random() * 100
+            let cumulative = 0
+            for (const font of config.fonts.distribution) {
+                cumulative += font.percentage
+                if (roll < cumulative) {
+                    fontFamily = loadedFonts.get(font.name) || font.family || 'Arial'
+                    break
+                }
+            }
+        }
+
+        // Determine if should augment
+        const shouldAugment = config.augmentation.enabled &&
+            (random() * 100) < config.augmentation.applyPercentage
+
+        if (shouldAugment) {
+            augmentedCount++
+        } else {
+            cleanCount++
+        }
+
+        // Render to canvas
+        const { canvas, ctx } = getCanvas(config.image.width, config.image.height)
+
+        try {
+            // Background
+            const bgStyle = config.image.backgroundStyle || 'clean_white'
+            ctx.fillStyle = backgroundColors[bgStyle] || config.image.background || '#FFFFFF'
+            ctx.fillRect(0, 0, canvas.width, canvas.height)
+
+            // Text
+            const fontSize = Math.min(canvas.height * 0.6, 48)
+            ctx.font = `${fontSize}px "${fontFamily}", Arial, sans-serif`
+            ctx.fillStyle = config.image.textColor || '#000000'
+            ctx.textAlign = config.image.direction === 'rtl' ? 'right' : 'left'
+            ctx.textBaseline = 'middle'
+            ctx.direction = config.image.direction as CanvasDirection || 'ltr'
+
+            const x = config.image.direction === 'rtl' ? canvas.width - 10 : 10
+            ctx.fillText(text, x, canvas.height / 2)
+
+            // Convert to blob
+            const blob = await new Promise<Blob>((resolve, reject) => {
+                canvas.toBlob((b) => {
+                    if (b) resolve(b)
+                    else reject(new Error('Failed to create blob'))
+                }, 'image/png')
+            })
+
+            // Write immediately to disk
+            const filename = `image_${String(i).padStart(6, '0')}.png`
+            const fileHandle = await imagesHandle.getFileHandle(filename, { create: true })
+            const writable = await fileHandle.createWritable()
+            await writable.write(blob)
+            await writable.close()
+
+            labels.push(`${filename}\t${text}`)
+
+        } finally {
+            releaseCanvas(canvas)
+        }
+
+        // Progress update every 50 samples or at key milestones
+        if (i % 50 === 0 || i === numSamples - 1) {
+            const progress = Math.round((i / numSamples) * 90) + 5
+            onProgress(progress, `Written ${i + 1}/${numSamples} images directly to disk`)
+        }
+
+        // Yield to prevent UI freeze
+        if (i % 100 === 0) {
+            await new Promise(resolve => setTimeout(resolve, 0))
+        }
+    }
+
+    onProgress(95, 'Writing labels file...')
+
+    // Write labels.txt
+    const labelsHandle = await datasetHandle.getFileHandle('labels.txt', { create: true })
+    const labelsWritable = await labelsHandle.createWritable()
+    await labelsWritable.write(labels.join('\n'))
+    await labelsWritable.close()
+
+    // Write other formats
+    if (config.output.formats.includes('jsonl') || config.output.formats.includes('trocr')) {
+        const jsonlHandle = await datasetHandle.getFileHandle('data.jsonl', { create: true })
+        const jsonlWritable = await jsonlHandle.createWritable()
+        const jsonlContent = labels.map(l => {
+            const [filename, ...textParts] = l.split('\t')
+            return JSON.stringify({ image: `images/${filename}`, text: textParts.join('\t') })
+        }).join('\n')
+        await jsonlWritable.write(jsonlContent)
+        await jsonlWritable.close()
+    }
+
+    if (config.output.formats.includes('huggingface') || config.output.formats.includes('csv')) {
+        const csvHandle = await datasetHandle.getFileHandle('metadata.csv', { create: true })
+        const csvWritable = await csvHandle.createWritable()
+        const csvContent = 'file_name,text\n' + labels.map(l => {
+            const [filename, ...textParts] = l.split('\t')
+            return `"images/${filename}","${textParts.join('\t').replace(/"/g, '""')}"`
+        }).join('\n')
+        await csvWritable.write(csvContent)
+        await csvWritable.close()
+    }
+
+    onProgress(98, 'Writing metadata...')
+
+    // Write metadata
+    const metadataHandle = await datasetHandle.getFileHandle('metadata.json', { create: true })
+    const metadataWritable = await metadataHandle.createWritable()
+    await metadataWritable.write(JSON.stringify({
+        generated_at: new Date().toISOString(),
+        total_samples: numSamples,
+        clean_samples: cleanCount,
+        augmented_samples: augmentedCount,
+        image_size: `${config.image.width}x${config.image.height}`,
+        output_path: `${directoryHandle.name}/${datasetName}`
+    }, null, 2))
+    await metadataWritable.close()
+
+    const endTime = Date.now()
+    const durationSeconds = (endTime - startTime) / 1000
+
+    onProgress(100, `Complete! Files saved to ${directoryHandle.name}/${datasetName}`)
+
+    return {
+        totalSamples: numSamples,
+        durationSeconds,
+        outputPath: `${directoryHandle.name}/${datasetName}`,
+        cleanSamples: cleanCount,
+        augmentedSamples: augmentedCount
+    }
+}
+
+/**
+ * Check if File System Access API is available
+ */
+export function isDirectGenerationSupported(): boolean {
+    return 'showDirectoryPicker' in window
+}

web/lib/generator.ts

@@ -2,6 +2,64 @@ import JSZip from 'jszip'
 import { saveAs } from 'file-saver'
 import { getWorkerPool, isWorkerRenderingAvailable, WorkerTask, WorkerResult } from './worker-pool'
 import { isWebGLAvailable, applyGPUAugmentation, GPUAugmentOptions } from './gpu-augmentation'
+import { StorageManager, StorageMode, getStorageManager, StoredSample } from './storage-manager'
+
+// ============================================
+// Canvas Pool for Performance
+// ============================================
+interface PooledCanvas {
+    canvas: HTMLCanvasElement
+    ctx: CanvasRenderingContext2D
+    inUse: boolean
+    width: number
+    height: number
+}
+
+const canvasPool: PooledCanvas[] = []
+const MAX_POOL_SIZE = 8
+
+function acquireCanvas(width: number, height: number): { canvas: HTMLCanvasElement; ctx: CanvasRenderingContext2D } {
+    // Find available canvas with matching dimensions
+    for (const item of canvasPool) {
+        if (!item.inUse && item.width === width && item.height === height) {
+            item.inUse = true
+            item.ctx.clearRect(0, 0, width, height)
+            item.ctx.setTransform(1, 0, 0, 1, 0, 0) // Reset transform
+            return { canvas: item.canvas, ctx: item.ctx }
+        }
+    }
+
+    // Create new canvas if pool not full
+    if (canvasPool.length < MAX_POOL_SIZE) {
+        const canvas = document.createElement('canvas')
+        canvas.width = width
+        canvas.height = height
+        const ctx = canvas.getContext('2d', { willReadFrequently: true })!
+        const pooled: PooledCanvas = { canvas, ctx, inUse: true, width, height }
+        canvasPool.push(pooled)
+        return { canvas, ctx }
+    }
+
+    // Pool full, create temporary canvas (will be GC'd)
+    const canvas = document.createElement('canvas')
+    canvas.width = width
+    canvas.height = height
+    const ctx = canvas.getContext('2d', { willReadFrequently: true })!
+    return { canvas, ctx }
+}
+
+function releaseCanvas(canvas: HTMLCanvasElement): void {
+    for (const item of canvasPool) {
+        if (item.canvas === canvas) {
+            item.inUse = false
+            return
+        }
+    }
+}
+
+export function clearCanvasPool(): void {
+    canvasPool.length = 0
+}
 
 export interface FontData {
     name: string
@@ -279,139 +337,146 @@ async function renderTextToCanvas(
     augValues: Record<string, number>,
     random: () => number
 ): Promise<{ blob: Blob; augmentations: string[]; backgroundStyle: string }> {
-    const canvas = document.createElement('canvas')
-    canvas.width = config.image.width
-    canvas.height = config.image.height
-    const ctx = canvas.getContext('2d')!
-
-    // Select and apply background
-    const bg = selectBackground(config, random)
-
-    if (bg.type === 'color') {
-        ctx.fillStyle = bg.value
-        ctx.fillRect(0, 0, canvas.width, canvas.height)
-    } else {
-        // Draw custom background image
-        try {
-            const img = await loadImage(bg.value)
-            ctx.drawImage(img, 0, 0, canvas.width, canvas.height)
-        } catch {
-            // Fallback to white if image fails
-            ctx.fillStyle = '#FFFFFF'
+    // Use canvas pool for better performance
+    const { canvas, ctx } = acquireCanvas(config.image.width, config.image.height)
+
+    // Declare variables outside try for proper scoping
+    let appliedAugmentations: string[] = []
+    let bg: ReturnType<typeof selectBackground>
+
+    try {
+        // Select and apply background
+        bg = selectBackground(config, random)
+
+        if (bg.type === 'color') {
+            ctx.fillStyle = bg.value
             ctx.fillRect(0, 0, canvas.width, canvas.height)
+        } else {
+            // Draw custom background image
+            try {
+                const img = await loadImage(bg.value)
+                ctx.drawImage(img, 0, 0, canvas.width, canvas.height)
+            } catch {
+                // Fallback to white if image fails
+                ctx.fillStyle = '#FFFFFF'
+                ctx.fillRect(0, 0, canvas.width, canvas.height)
+            }
         }
-    }
 
-    // Apply augmentation transforms if enabled
-    let appliedAugmentations: string[] = []
-    if (shouldAugment && config.augmentation.enabled) {
-        ctx.save()
-        appliedAugmentations = applyAugmentation(ctx, canvas, augValues, random)
-    }
-
-    // Set text properties with the selected font
-    const fontSize = Math.min(canvas.height * 0.6, 48)
-    ctx.font = `${fontSize}px "${fontFamily}", Arial, sans-serif`
-    ctx.fillStyle = config.image.textColor
-    ctx.textAlign = config.image.direction === 'rtl' ? 'right' : 'left'
-    ctx.textBaseline = 'middle'
-
-    // Draw text
-    const x = config.image.direction === 'rtl'
-        ? canvas.width - 10
-        : 10
-    const y = canvas.height / 2
-
-    ctx.direction = config.image.direction as CanvasDirection
-    ctx.fillText(text, x, y)
-
-    if (shouldAugment && config.augmentation.enabled) {
-        ctx.restore()
-    }
-
-    // Apply post-processing augmentations (GPU-accelerated when available)
-    if (shouldAugment && config.augmentation.enabled) {
-        const applyBrightness = augValues.brightness && random() > 0.5
-        const applyNoise = augValues.gaussian_noise && random() > 0.6
-
-        if (applyBrightness || applyNoise) {
-            // Try GPU-accelerated augmentation first
-            const useGPU = isWebGLAvailable()
-
-            if (useGPU) {
-                // GPU path - apply all augmentations in a single GPU pass
-                const gpuOptions: GPUAugmentOptions = {
-                    brightness: applyBrightness ? (random() - 0.5) * augValues.brightness / 50 : 0,
-                    contrast: 1, // Could add contrast augmentation here
-                    noiseAmount: applyNoise ? augValues.gaussian_noise / 200 : 0,
-                    seed: random() * 1000
-                }
+        // Apply augmentation transforms if enabled
+        if (shouldAugment && config.augmentation.enabled) {
+            ctx.save()
+            appliedAugmentations = applyAugmentation(ctx, canvas, augValues, random)
+        }
 
-                const gpuResult = applyGPUAugmentation(canvas, gpuOptions)
-                if (gpuResult && gpuResult !== canvas) {
-                    // Copy GPU result back to main canvas
-                    const gpuCtx = gpuResult.getContext('webgl')
-                    if (gpuCtx) {
-                        const pixels = new Uint8Array(canvas.width * canvas.height * 4)
-                        gpuCtx.readPixels(0, 0, canvas.width, canvas.height, gpuCtx.RGBA, gpuCtx.UNSIGNED_BYTE, pixels)
-
-                        // Flip Y and apply to main canvas
-                        const imageData = ctx.createImageData(canvas.width, canvas.height)
-                        const rowSize = canvas.width * 4
-                        for (let y = 0; y < canvas.height; y++) {
-                            const srcRow = (canvas.height - 1 - y) * rowSize
-                            const dstRow = y * rowSize
-                            imageData.data.set(pixels.subarray(srcRow, srcRow + rowSize), dstRow)
-                        }
-                        ctx.putImageData(imageData, 0, 0)
+        // Set text properties with the selected font
+        const fontSize = Math.min(canvas.height * 0.6, 48)
+        ctx.font = `${fontSize}px "${fontFamily}", Arial, sans-serif`
+        ctx.fillStyle = config.image.textColor
+        ctx.textAlign = config.image.direction === 'rtl' ? 'right' : 'left'
+        ctx.textBaseline = 'middle'
+
+        // Draw text
+        const x = config.image.direction === 'rtl'
+            ? canvas.width - 10
+            : 10
+        const y = canvas.height / 2
 
-                        if (applyBrightness) appliedAugmentations.push('brightness_gpu')
-                        if (applyNoise) appliedAugmentations.push('noise_gpu')
+        ctx.direction = config.image.direction as CanvasDirection
+        ctx.fillText(text, x, y)
+
+        if (shouldAugment && config.augmentation.enabled) {
+            ctx.restore()
+        }
+
+        // Apply post-processing augmentations (GPU-accelerated when available)
+        if (shouldAugment && config.augmentation.enabled) {
+            const applyBrightness = augValues.brightness && random() > 0.5
+            const applyNoise = augValues.gaussian_noise && random() > 0.6
+
+            if (applyBrightness || applyNoise) {
+                // Try GPU-accelerated augmentation first
+                const useGPU = isWebGLAvailable()
+
+                if (useGPU) {
+                    // GPU path - apply all augmentations in a single GPU pass
+                    const gpuOptions: GPUAugmentOptions = {
+                        brightness: applyBrightness ? (random() - 0.5) * augValues.brightness / 50 : 0,
+                        contrast: 1, // Could add contrast augmentation here
+                        noiseAmount: applyNoise ? augValues.gaussian_noise / 200 : 0,
+                        seed: random() * 1000
                     }
-                }
-            } else {
-                // CPU fallback path
-                if (applyBrightness) {
-                    const adjustment = 1 + (random() - 0.5) * augValues.brightness / 50
-                    const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height)
-                    for (let i = 0; i < imageData.data.length; i += 4) {
-                        imageData.data[i] = Math.min(255, imageData.data[i] * adjustment)
-                        imageData.data[i + 1] = Math.min(255, imageData.data[i + 1] * adjustment)
-                        imageData.data[i + 2] = Math.min(255, imageData.data[i + 2] * adjustment)
+
+                    const gpuResult = applyGPUAugmentation(canvas, gpuOptions)
+                    if (gpuResult && gpuResult !== canvas) {
+                        // Copy GPU result back to main canvas
+                        const gpuCtx = gpuResult.getContext('webgl')
+                        if (gpuCtx) {
+                            const pixels = new Uint8Array(canvas.width * canvas.height * 4)
+                            gpuCtx.readPixels(0, 0, canvas.width, canvas.height, gpuCtx.RGBA, gpuCtx.UNSIGNED_BYTE, pixels)
+
+                            // Flip Y and apply to main canvas
+                            const imageData = ctx.createImageData(canvas.width, canvas.height)
+                            const rowSize = canvas.width * 4
+                            for (let y = 0; y < canvas.height; y++) {
+                                const srcRow = (canvas.height - 1 - y) * rowSize
+                                const dstRow = y * rowSize
+                                imageData.data.set(pixels.subarray(srcRow, srcRow + rowSize), dstRow)
+                            }
+                            ctx.putImageData(imageData, 0, 0)
+
+                            if (applyBrightness) appliedAugmentations.push('brightness_gpu')
+                            if (applyNoise) appliedAugmentations.push('noise_gpu')
+                        }
+                    }
+                } else {
+                    // CPU fallback path
+                    if (applyBrightness) {
+                        const adjustment = 1 + (random() - 0.5) * augValues.brightness / 50
+                        const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height)
+                        for (let i = 0; i < imageData.data.length; i += 4) {
+                            imageData.data[i] = Math.min(255, imageData.data[i] * adjustment)
+                            imageData.data[i + 1] = Math.min(255, imageData.data[i + 1] * adjustment)
+                            imageData.data[i + 2] = Math.min(255, imageData.data[i + 2] * adjustment)
+                        }
+                        ctx.putImageData(imageData, 0, 0)
+                        appliedAugmentations.push('brightness')
                     }
-                    ctx.putImageData(imageData, 0, 0)
-                    appliedAugmentations.push('brightness')
-                }
 
-                if (applyNoise) {
-                    const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height)
-                    const noiseLevel = augValues.gaussian_noise / 2
-                    for (let i = 0; i < imageData.data.length; i += 4) {
-                        const noise = (random() - 0.5) * noiseLevel
-                        imageData.data[i] = Math.max(0, Math.min(255, imageData.data[i] + noise))
-                        imageData.data[i + 1] = Math.max(0, Math.min(255, imageData.data[i + 1] + noise))
-                        imageData.data[i + 2] = Math.max(0, Math.min(255, imageData.data[i + 2] + noise))
+                    if (applyNoise) {
+                        const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height)
+                        const noiseLevel = augValues.gaussian_noise / 2
+                        for (let i = 0; i < imageData.data.length; i += 4) {
+                            const noise = (random() - 0.5) * noiseLevel
+                            imageData.data[i] = Math.max(0, Math.min(255, imageData.data[i] + noise))
+                            imageData.data[i + 1] = Math.max(0, Math.min(255, imageData.data[i + 1] + noise))
+                            imageData.data[i + 2] = Math.max(0, Math.min(255, imageData.data[i + 2] + noise))
+                        }
+                        ctx.putImageData(imageData, 0, 0)
+                        appliedAugmentations.push('noise')
                     }
-                    ctx.putImageData(imageData, 0, 0)
-                    appliedAugmentations.push('noise')
                 }
             }
         }
-    }
 
-    // Convert to blob
-    return new Promise((resolve, reject) => {
-        canvas.toBlob(
-            (blob) => {
-                if (blob) {
-                    resolve({ blob, augmentations: appliedAugmentations, backgroundStyle: bg.styleName })
-                } else {
-                    reject(new Error('Failed to convert canvas to blob'))
-                }
-            },
-            'image/png'
-        )
-    })
+        // Convert to blob and release canvas when done
+        return new Promise((resolve, reject) => {
+            canvas.toBlob(
+                (blob) => {
+                    releaseCanvas(canvas)  // Return canvas to pool
+                    if (blob) {
+                        resolve({ blob, augmentations: appliedAugmentations, backgroundStyle: bg.styleName })
+                    } else {
+                        reject(new Error('Failed to convert canvas to blob'))
+                    }
+                },
+                'image/png'
+            )
+        })
+    } catch (err) {
+        releaseCanvas(canvas)  // Ensure canvas is released on error
+        throw err
+    }
 }
 
 // Render a single sample (for parallel processing)
@@ -866,18 +931,49 @@ export async function generateDataset(
     // Generate zip blob with dynamic compression based on dataset size
     onProgress(95, 'Compressing dataset...')
 
-    // Use lower compression for large datasets (faster, less memory)
-    const compressionLevel = labels.length > 50000 ? 3 :
-        labels.length > 10000 ? 5 : 6
+    // Use lower/no compression for large datasets (faster, less memory)
+    // STORE = no compression, uses less memory
+    let compressionType: 'DEFLATE' | 'STORE' = 'DEFLATE'
+    let compressionLevel = 6
+
+    if (labels.length > 20000) {
+        // Very large: no compression (STORE) to prevent memory issues
+        compressionType = 'STORE'
+        compressionLevel = 0
+        console.log(`Large dataset (${labels.length}): Using STORE (no compression) to prevent memory issues`)
+    } else if (labels.length > 10000) {
+        compressionLevel = 1  // Minimal compression
+        console.log(`Medium-large dataset (${labels.length}): Using minimal compression level 1`)
+    } else if (labels.length > 5000) {
+        compressionLevel = 3
+        console.log(`Medium dataset (${labels.length}): Using compression level 3`)
+    }
 
-    console.log(`Compressing ${labels.length} files with level ${compressionLevel}`)
+    console.log(`Compressing ${labels.length} files with ${compressionType}, level ${compressionLevel}`)
 
-    const zipBlob = await zip.generateAsync({
-        type: 'blob',
-        compression: 'DEFLATE',
-        compressionOptions: { level: compressionLevel },
-        streamFiles: true // Stream files to reduce memory usage
-    })
+    let zipBlob: Blob
+    try {
+        zipBlob = await zip.generateAsync({
+            type: 'blob',
+            compression: compressionType,
+            compressionOptions: compressionType === 'DEFLATE' ? { level: compressionLevel } : undefined,
+            streamFiles: true // Stream files to reduce memory usage
+        })
+    } catch (memoryError: any) {
+        console.error('ZIP generation failed, trying without compression:', memoryError)
+        onProgress(96, 'Retrying with no compression...')
+
+        // Retry with no compression at all
+        try {
+            zipBlob = await zip.generateAsync({
+                type: 'blob',
+                compression: 'STORE',
+                streamFiles: true
+            })
+        } catch (finalError) {
+            throw new Error(`Failed to create ZIP: Memory limit exceeded. Try using Local Folder storage mode instead of Memory for large datasets.`)
+        }
+    }
 
     const endTime = Date.now()
     const durationSeconds = (endTime - startTime) / 1000

web/lib/storage-manager.ts

@@ -0,0 +1,468 @@
+// Storage Manager for OCR Dataset Generator
+// Supports multiple storage backends: Memory, IndexedDB, Local Folder, HuggingFace
+
+export type StorageMode = 'memory' | 'indexeddb' | 'local' | 'huggingface'
+
+export interface StorageConfig {
+    mode: StorageMode
+    localPath?: string  // For 'local' mode - user selected folder
+    dataDir?: string    // For 'huggingface' mode - /data directory
+    cleanupHours?: number
+}
+
+export interface StoredSample {
+    id: string
+    filename: string
+    blob: Blob
+    label: string
+}
+
+// IndexedDB Database name and store
+const DB_NAME = 'ocr_dataset_generator'
+const STORE_NAME = 'samples'
+const DB_VERSION = 1
+
+// ============================================
+// IndexedDB Storage Implementation
+// ============================================
+
+class IndexedDBStorage {
+    private db: IDBDatabase | null = null
+    private initPromise: Promise<void> | null = null
+
+    async init(): Promise<void> {
+        if (this.db) return
+        if (this.initPromise) return this.initPromise
+
+        this.initPromise = new Promise((resolve, reject) => {
+            const request = indexedDB.open(DB_NAME, DB_VERSION)
+
+            request.onerror = () => reject(request.error)
+            request.onsuccess = () => {
+                this.db = request.result
+                resolve()
+            }
+
+            request.onupgradeneeded = (event) => {
+                const db = (event.target as IDBOpenDBRequest).result
+                if (!db.objectStoreNames.contains(STORE_NAME)) {
+                    db.createObjectStore(STORE_NAME, { keyPath: 'id' })
+                }
+            }
+        })
+
+        return this.initPromise
+    }
+
+    async store(sample: StoredSample): Promise<void> {
+        await this.init()
+        return new Promise((resolve, reject) => {
+            const transaction = this.db!.transaction([STORE_NAME], 'readwrite')
+            const store = transaction.objectStore(STORE_NAME)
+            const request = store.put(sample)
+            request.onerror = () => reject(request.error)
+            request.onsuccess = () => resolve()
+        })
+    }
+
+    async getAll(): Promise<StoredSample[]> {
+        await this.init()
+        return new Promise((resolve, reject) => {
+            const transaction = this.db!.transaction([STORE_NAME], 'readonly')
+            const store = transaction.objectStore(STORE_NAME)
+            const request = store.getAll()
+            request.onerror = () => reject(request.error)
+            request.onsuccess = () => resolve(request.result)
+        })
+    }
+
+    async count(): Promise<number> {
+        await this.init()
+        return new Promise((resolve, reject) => {
+            const transaction = this.db!.transaction([STORE_NAME], 'readonly')
+            const store = transaction.objectStore(STORE_NAME)
+            const request = store.count()
+            request.onerror = () => reject(request.error)
+            request.onsuccess = () => resolve(request.result)
+        })
+    }
+
+    async clear(): Promise<void> {
+        await this.init()
+        return new Promise((resolve, reject) => {
+            const transaction = this.db!.transaction([STORE_NAME], 'readwrite')
+            const store = transaction.objectStore(STORE_NAME)
+            const request = store.clear()
+            request.onerror = () => reject(request.error)
+            request.onsuccess = () => resolve()
+        })
+    }
+
+    close(): void {
+        if (this.db) {
+            this.db.close()
+            this.db = null
+            this.initPromise = null
+        }
+    }
+}
+
+// ============================================
+// File System Access API (Local Folder)
+// ============================================
+
+class FileSystemStorage {
+    private directoryHandle: FileSystemDirectoryHandle | null = null
+    private imagesHandle: FileSystemDirectoryHandle | null = null
+    private datasetName: string = ''
+
+    async selectFolder(): Promise<string | null> {
+        try {
+            // @ts-ignore - File System Access API
+            this.directoryHandle = await window.showDirectoryPicker({
+                mode: 'readwrite',
+                startIn: 'documents'
+            })
+
+            // Create timestamped dataset folder
+            this.datasetName = `dataset_${new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19)}`
+            if (!this.directoryHandle) throw new Error('No directory handle')
+            const datasetHandle = await this.directoryHandle.getDirectoryHandle(this.datasetName, { create: true })
+            this.imagesHandle = await datasetHandle.getDirectoryHandle('images', { create: true })
+
+            return `${this.directoryHandle.name}/${this.datasetName}`
+        } catch (err) {
+            console.error('Failed to select folder:', err)
+            return null
+        }
+    }
+
+    async writeImage(filename: string, blob: Blob): Promise<void> {
+        if (!this.imagesHandle) throw new Error('No folder selected')
+        const fileHandle = await this.imagesHandle.getFileHandle(filename, { create: true })
+        const writable = await fileHandle.createWritable()
+        await writable.write(blob)
+        await writable.close()
+    }
+
+    async writeLabels(content: string, filename: string = 'labels.txt'): Promise<void> {
+        if (!this.directoryHandle || !this.datasetName) throw new Error('No folder selected')
+        const datasetHandle = await this.directoryHandle.getDirectoryHandle(this.datasetName, { create: false })
+        const fileHandle = await datasetHandle.getFileHandle(filename, { create: true })
+        const writable = await fileHandle.createWritable()
+        await writable.write(content)
+        await writable.close()
+    }
+
+    async writeMetadata(metadata: object): Promise<void> {
+        if (!this.directoryHandle || !this.datasetName) throw new Error('No folder selected')
+        const datasetHandle = await this.directoryHandle.getDirectoryHandle(this.datasetName, { create: false })
+        const fileHandle = await datasetHandle.getFileHandle('metadata.json', { create: true })
+        const writable = await fileHandle.createWritable()
+        await writable.write(JSON.stringify(metadata, null, 2))
+        await writable.close()
+    }
+
+    getPath(): string | null {
+        return this.directoryHandle ? `${this.directoryHandle.name}/${this.datasetName}` : null
+    }
+
+    isSupported(): boolean {
+        return 'showDirectoryPicker' in window
+    }
+}
+
+// ============================================
+// Server Storage (Local/HuggingFace via API)
+// ============================================
+
+class ServerStorage {
+    private sessionId: string = ''
+    private mode: 'local' | 'huggingface' = 'local'
+
+    constructor(mode: 'local' | 'huggingface' = 'local') {
+        this.mode = mode
+        this.sessionId = `dataset_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`
+    }
+
+    async writeImage(filename: string, blob: Blob): Promise<void> {
+        const formData = new FormData()
+        formData.append('file', blob, filename)
+        formData.append('sessionId', this.sessionId)
+        formData.append('mode', this.mode)
+        formData.append('type', 'image')
+
+        const response = await fetch('/api/storage', {
+            method: 'POST',
+            body: formData
+        })
+
+        if (!response.ok) {
+            throw new Error(`Failed to write image: ${response.statusText}`)
+        }
+    }
+
+    async writeLabels(labels: string[]): Promise<void> {
+        const response = await fetch('/api/storage', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                sessionId: this.sessionId,
+                mode: this.mode,
+                type: 'labels',
+                content: labels.join('\n')
+            })
+        })
+
+        if (!response.ok) {
+            throw new Error(`Failed to write labels: ${response.statusText}`)
+        }
+    }
+
+    async writeMetadata(metadata: object): Promise<void> {
+        const response = await fetch('/api/storage', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                sessionId: this.sessionId,
+                mode: this.mode,
+                type: 'metadata',
+                content: JSON.stringify(metadata, null, 2)
+            })
+        })
+
+        if (!response.ok) {
+            throw new Error(`Failed to write metadata: ${response.statusText}`)
+        }
+    }
+
+    async finalize(): Promise<{ path: string; downloadUrl?: string }> {
+        const response = await fetch('/api/storage', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                sessionId: this.sessionId,
+                mode: this.mode,
+                type: 'finalize'
+            })
+        })
+
+        if (!response.ok) {
+            throw new Error(`Failed to finalize: ${response.statusText}`)
+        }
+
+        return response.json()
+    }
+
+    getSessionId(): string {
+        return this.sessionId
+    }
+}
+
+// ============================================
+// Canvas Pool for Performance
+// ============================================
+
+interface PooledCanvas {
+    canvas: HTMLCanvasElement
+    ctx: CanvasRenderingContext2D
+    inUse: boolean
+}
+
+class CanvasPool {
+    private pool: PooledCanvas[] = []
+    private maxSize: number = 8
+
+    acquire(width: number, height: number): { canvas: HTMLCanvasElement; ctx: CanvasRenderingContext2D } {
+        // Find available canvas with matching size
+        for (const item of this.pool) {
+            if (!item.inUse && item.canvas.width === width && item.canvas.height === height) {
+                item.inUse = true
+                // Clear the canvas
+                item.ctx.clearRect(0, 0, width, height)
+                return { canvas: item.canvas, ctx: item.ctx }
+            }
+        }
+
+        // Create new canvas if pool not full
+        if (this.pool.length < this.maxSize) {
+            const canvas = document.createElement('canvas')
+            canvas.width = width
+            canvas.height = height
+            const ctx = canvas.getContext('2d')!
+            const pooled: PooledCanvas = { canvas, ctx, inUse: true }
+            this.pool.push(pooled)
+            return { canvas, ctx }
+        }
+
+        // Wait for a canvas to become available (shouldn't happen often)
+        const canvas = document.createElement('canvas')
+        canvas.width = width
+        canvas.height = height
+        const ctx = canvas.getContext('2d')!
+        return { canvas, ctx }
+    }
+
+    release(canvas: HTMLCanvasElement): void {
+        for (const item of this.pool) {
+            if (item.canvas === canvas) {
+                item.inUse = false
+                return
+            }
+        }
+    }
+
+    clear(): void {
+        this.pool = []
+    }
+}
+
+// ============================================
+// Unified Storage Manager
+// ============================================
+
+export class StorageManager {
+    private config: StorageConfig
+    private indexedDB: IndexedDBStorage | null = null
+    private fileSystem: FileSystemStorage | null = null
+    private serverStorage: ServerStorage | null = null
+    private memoryStorage: StoredSample[] = []
+    private canvasPool: CanvasPool
+
+    constructor(config: StorageConfig) {
+        this.config = config
+        this.canvasPool = new CanvasPool()
+        this.initialize()
+    }
+
+    private initialize(): void {
+        switch (this.config.mode) {
+            case 'indexeddb':
+                this.indexedDB = new IndexedDBStorage()
+                break
+            case 'local':
+                this.fileSystem = new FileSystemStorage()
+                break
+            case 'huggingface':
+                this.serverStorage = new ServerStorage('huggingface')
+                break
+            // 'memory' uses memoryStorage array
+        }
+    }
+
+    async selectLocalFolder(): Promise<string | null> {
+        if (this.config.mode !== 'local' || !this.fileSystem) {
+            throw new Error('Local folder selection only available in local mode')
+        }
+        return this.fileSystem.selectFolder()
+    }
+
+    isLocalFolderSupported(): boolean {
+        return 'showDirectoryPicker' in window
+    }
+
+    getCanvasPool(): CanvasPool {
+        return this.canvasPool
+    }
+
+    async storeSample(sample: StoredSample): Promise<void> {
+        switch (this.config.mode) {
+            case 'memory':
+                this.memoryStorage.push(sample)
+                break
+            case 'indexeddb':
+                await this.indexedDB!.store(sample)
+                break
+            case 'local':
+                await this.fileSystem!.writeImage(sample.filename, sample.blob)
+                break
+            case 'huggingface':
+                await this.serverStorage!.writeImage(sample.filename, sample.blob)
+                break
+        }
+    }
+
+    async getSampleCount(): Promise<number> {
+        switch (this.config.mode) {
+            case 'memory':
+                return this.memoryStorage.length
+            case 'indexeddb':
+                return this.indexedDB!.count()
+            default:
+                return 0 // Server modes don't track count locally
+        }
+    }
+
+    async getAllSamples(): Promise<StoredSample[]> {
+        switch (this.config.mode) {
+            case 'memory':
+                return this.memoryStorage
+            case 'indexeddb':
+                return this.indexedDB!.getAll()
+            default:
+                return [] // Server modes don't support this
+        }
+    }
+
+    async finalize(labels: string[], metadata: object): Promise<{ path?: string; blob?: Blob }> {
+        switch (this.config.mode) {
+            case 'memory':
+                // Return samples for ZIP building
+                return { blob: undefined }
+            case 'indexeddb':
+                // Return samples for ZIP building
+                return { blob: undefined }
+            case 'local':
+                await this.fileSystem!.writeLabels(labels.join('\n'))
+                await this.fileSystem!.writeMetadata(metadata)
+                return { path: this.fileSystem!.getPath() || undefined }
+            case 'huggingface':
+                await this.serverStorage!.writeLabels(labels)
+                await this.serverStorage!.writeMetadata(metadata)
+                const result = await this.serverStorage!.finalize()
+                return { path: result.path }
+        }
+    }
+
+    async clear(): Promise<void> {
+        this.memoryStorage = []
+        if (this.indexedDB) {
+            await this.indexedDB.clear()
+        }
+        this.canvasPool.clear()
+    }
+
+    getMode(): StorageMode {
+        return this.config.mode
+    }
+
+    close(): void {
+        if (this.indexedDB) {
+            this.indexedDB.close()
+        }
+        this.canvasPool.clear()
+    }
+}
+
+// ============================================
+// Singleton Instance
+// ============================================
+
+let storageManagerInstance: StorageManager | null = null
+
+export function getStorageManager(config?: StorageConfig): StorageManager {
+    if (!storageManagerInstance || (config && config.mode !== storageManagerInstance.getMode())) {
+        if (storageManagerInstance) {
+            storageManagerInstance.close()
+        }
+        storageManagerInstance = new StorageManager(config || { mode: 'memory' })
+    }
+    return storageManagerInstance
+}
+
+export function resetStorageManager(): void {
+    if (storageManagerInstance) {
+        storageManagerInstance.close()
+        storageManagerInstance = null
+    }
+}