Upload folder using huggingface_hub

Dockerfile

@@ -24,5 +24,5 @@ ENV NODE_ENV=development
 # Expose port 7860
 EXPOSE 7860
 
-# Run disk check, init, restore, then start dev server and backup worker concurrently
-CMD ["sh", "-c", "df -h && node cmd/hf_init.mjs && node cmd/hf_restore.mjs && npx concurrently \"npm run dev\" \"node cmd/hf_backup.mjs\""]
+# Run disk check, init, restore, then start dev server, backup worker, and PDF convert server concurrently
+CMD ["sh", "-c", "df -h && node cmd/hf_init.mjs && node cmd/hf_restore.mjs && npx concurrently \"npm run dev\" \"node cmd/hf_backup.mjs\" \"node cmd/pdf_convert_server.mjs\""]

cmd/pdf_convert_server.mjs

@@ -0,0 +1,352 @@
+/**
+ * PDF to SVG Conversion Server
+ *
+ * Runs alongside the Vite dev server on HuggingFace Spaces.
+ * Handles PDF to SVG conversion using the pdf2svg binary.
+ *
+ * Endpoints:
+ * - POST /convert/pdf - Upload PDF and get SVG pages
+ * - GET /convert/status/:jobId - Check job status
+ * - GET /convert/page/:jobId/:pageNum - Download a converted SVG page
+ */
+
+import http from 'http';
+import { spawn, execSync } from 'child_process';
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+
+const PORT = process.env.PDF_CONVERT_PORT || 7861;
+const TEMP_DIR = path.join(os.tmpdir(), 'pdf_convert');
+
+// Ensure temp directory exists
+if (!fs.existsSync(TEMP_DIR)) {
+    fs.mkdirSync(TEMP_DIR, { recursive: true });
+}
+
+// Job storage
+const jobs = new Map();
+
+/**
+ * Parse multipart form data (simple implementation for file upload)
+ */
+function parseMultipart(buffer, boundary) {
+    const parts = [];
+    const boundaryBuffer = Buffer.from(`--${boundary}`);
+    let start = buffer.indexOf(boundaryBuffer);
+
+    while (start !== -1) {
+        const end = buffer.indexOf(boundaryBuffer, start + boundaryBuffer.length);
+        if (end === -1) break;
+
+        const part = buffer.slice(start + boundaryBuffer.length, end);
+        const headerEnd = part.indexOf('\r\n\r\n');
+        if (headerEnd !== -1) {
+            const headers = part.slice(0, headerEnd).toString();
+            const content = part.slice(headerEnd + 4, part.length - 2); // -2 for trailing \r\n
+
+            const nameMatch = headers.match(/name="([^"]+)"/);
+            const filenameMatch = headers.match(/filename="([^"]+)"/);
+
+            if (nameMatch) {
+                parts.push({
+                    name: nameMatch[1],
+                    filename: filenameMatch ? filenameMatch[1] : null,
+                    content: content
+                });
+            }
+        }
+        start = end;
+    }
+    return parts;
+}
+
+/**
+ * Get PDF page count using pdfinfo or pdf2svg
+ */
+function getPdfPageCount(pdfPath) {
+    try {
+        // Try pdfinfo first
+        const output = execSync(`pdfinfo "${pdfPath}" 2>/dev/null | grep -i "Pages:" | awk '{print $2}'`, { encoding: 'utf8' });
+        const count = parseInt(output.trim(), 10);
+        if (!isNaN(count)) return count;
+    } catch (e) {
+        // pdfinfo not available, try alternative method
+    }
+
+    try {
+        // Try using pdf2svg on page 1 to check if it works, then binary search for count
+        // This is a fallback if pdfinfo isn't available
+        let maxPage = 1;
+        let testPage = 1;
+
+        // Test increasing pages until we fail
+        while (testPage <= 1000) {
+            const testOutput = path.join(TEMP_DIR, `test_${Date.now()}.svg`);
+            try {
+                execSync(`pdf2svg "${pdfPath}" "${testOutput}" ${testPage} 2>/dev/null`, { encoding: 'utf8' });
+                fs.unlinkSync(testOutput);
+                maxPage = testPage;
+                testPage++;
+            } catch (e) {
+                break;
+            }
+        }
+        return maxPage;
+    } catch (e) {
+        console.error('Failed to get page count:', e.message);
+        return 1;
+    }
+}
+
+/**
+ * Convert a single PDF page to SVG
+ */
+async function convertPage(pdfPath, pageNum, outputPath) {
+    return new Promise((resolve, reject) => {
+        const proc = spawn('pdf2svg', [pdfPath, outputPath, String(pageNum)]);
+
+        let stderr = '';
+        proc.stderr.on('data', (data) => {
+            stderr += data.toString();
+        });
+
+        proc.on('close', (code) => {
+            if (code === 0 && fs.existsSync(outputPath)) {
+                resolve(outputPath);
+            } else {
+                reject(new Error(`pdf2svg failed: ${stderr || 'Unknown error'}`));
+            }
+        });
+
+        proc.on('error', (err) => {
+            reject(err);
+        });
+    });
+}
+
+/**
+ * Process a PDF conversion job
+ */
+async function processJob(jobId) {
+    const job = jobs.get(jobId);
+    if (!job) return;
+
+    job.status = 'processing';
+    job.updatedAt = Date.now();
+
+    try {
+        // Get page count
+        const pageCount = getPdfPageCount(job.pdfPath);
+        job.pageCount = pageCount;
+
+        // Convert each page
+        for (let i = 1; i <= pageCount; i++) {
+            const outputPath = path.join(job.outputDir, `page_${i}.svg`);
+            await convertPage(job.pdfPath, i, outputPath);
+            job.processedPages = i;
+            job.updatedAt = Date.now();
+            console.log(`[PDF Convert] Job ${jobId}: Page ${i}/${pageCount} converted`);
+        }
+
+        job.status = 'completed';
+        job.updatedAt = Date.now();
+        console.log(`[PDF Convert] Job ${jobId}: Completed - ${pageCount} pages`);
+
+    } catch (e) {
+        job.status = 'failed';
+        job.error = e.message;
+        job.updatedAt = Date.now();
+        console.error(`[PDF Convert] Job ${jobId}: Failed -`, e.message);
+    }
+}
+
+/**
+ * Handle HTTP requests
+ */
+async function handleRequest(req, res) {
+    const url = new URL(req.url, `http://localhost:${PORT}`);
+
+    // CORS headers
+    res.setHeader('Access-Control-Allow-Origin', '*');
+    res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
+    res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
+
+    if (req.method === 'OPTIONS') {
+        res.writeHead(200);
+        res.end();
+        return;
+    }
+
+    // Health check
+    if (url.pathname === '/convert/health') {
+        res.writeHead(200, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({ status: 'ok', pdf2svg: true }));
+        return;
+    }
+
+    // Upload PDF
+    if (req.method === 'POST' && url.pathname === '/convert/pdf') {
+        const chunks = [];
+        req.on('data', chunk => chunks.push(chunk));
+        req.on('end', async () => {
+            try {
+                const buffer = Buffer.concat(chunks);
+                const contentType = req.headers['content-type'] || '';
+
+                let pdfBuffer;
+
+                if (contentType.includes('multipart/form-data')) {
+                    const boundary = contentType.split('boundary=')[1];
+                    const parts = parseMultipart(buffer, boundary);
+                    const filePart = parts.find(p => p.filename && p.filename.endsWith('.pdf'));
+                    if (!filePart) {
+                        res.writeHead(400, { 'Content-Type': 'application/json' });
+                        res.end(JSON.stringify({ error: 'No PDF file found' }));
+                        return;
+                    }
+                    pdfBuffer = filePart.content;
+                } else if (contentType === 'application/pdf') {
+                    pdfBuffer = buffer;
+                } else {
+                    res.writeHead(400, { 'Content-Type': 'application/json' });
+                    res.end(JSON.stringify({ error: 'Invalid content type' }));
+                    return;
+                }
+
+                // Create job
+                const jobId = `job_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+                const jobDir = path.join(TEMP_DIR, jobId);
+                fs.mkdirSync(jobDir, { recursive: true });
+
+                const pdfPath = path.join(jobDir, 'source.pdf');
+                fs.writeFileSync(pdfPath, pdfBuffer);
+
+                const job = {
+                    id: jobId,
+                    status: 'pending',
+                    pageCount: 0,
+                    processedPages: 0,
+                    pdfPath: pdfPath,
+                    outputDir: jobDir,
+                    createdAt: Date.now(),
+                    updatedAt: Date.now()
+                };
+
+                jobs.set(jobId, job);
+
+                // Start processing async
+                processJob(jobId);
+
+                res.writeHead(200, { 'Content-Type': 'application/json' });
+                res.end(JSON.stringify({
+                    jobId,
+                    status: 'pending',
+                    statusUrl: `/convert/status/${jobId}`
+                }));
+
+            } catch (e) {
+                console.error('Upload error:', e);
+                res.writeHead(500, { 'Content-Type': 'application/json' });
+                res.end(JSON.stringify({ error: e.message }));
+            }
+        });
+        return;
+    }
+
+    // Check job status
+    const statusMatch = url.pathname.match(/^\/convert\/status\/(.+)$/);
+    if (req.method === 'GET' && statusMatch) {
+        const jobId = statusMatch[1];
+        const job = jobs.get(jobId);
+
+        if (!job) {
+            res.writeHead(404, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ error: 'Job not found' }));
+            return;
+        }
+
+        const pages = [];
+        if (job.status === 'completed' || job.processedPages > 0) {
+            for (let i = 1; i <= job.processedPages; i++) {
+                pages.push({
+                    page: i,
+                    url: `/convert/page/${jobId}/${i}`
+                });
+            }
+        }
+
+        res.writeHead(200, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({
+            id: job.id,
+            status: job.status,
+            pageCount: job.pageCount,
+            processedPages: job.processedPages,
+            error: job.error,
+            pages: pages
+        }));
+        return;
+    }
+
+    // Download page
+    const pageMatch = url.pathname.match(/^\/convert\/page\/(.+)\/(\d+)$/);
+    if (req.method === 'GET' && pageMatch) {
+        const jobId = pageMatch[1];
+        const pageNum = parseInt(pageMatch[2], 10);
+        const job = jobs.get(jobId);
+
+        if (!job) {
+            res.writeHead(404, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ error: 'Job not found' }));
+            return;
+        }
+
+        const svgPath = path.join(job.outputDir, `page_${pageNum}.svg`);
+        if (!fs.existsSync(svgPath)) {
+            res.writeHead(404, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ error: 'Page not found' }));
+            return;
+        }
+
+        const svgContent = fs.readFileSync(svgPath, 'utf8');
+        res.writeHead(200, { 'Content-Type': 'image/svg+xml' });
+        res.end(svgContent);
+        return;
+    }
+
+    // 404 for unknown routes
+    res.writeHead(404, { 'Content-Type': 'application/json' });
+    res.end(JSON.stringify({ error: 'Not found' }));
+}
+
+// Create server
+const server = http.createServer(handleRequest);
+
+server.listen(PORT, () => {
+    console.log(`[PDF Convert Server] Running on port ${PORT}`);
+    console.log(`[PDF Convert Server] Endpoints:`);
+    console.log(`  POST /convert/pdf - Upload PDF file`);
+    console.log(`  GET /convert/status/:jobId - Check job status`);
+    console.log(`  GET /convert/page/:jobId/:pageNum - Download SVG page`);
+});
+
+// Cleanup old jobs periodically (every 30 minutes)
+setInterval(() => {
+    const now = Date.now();
+    const maxAge = 2 * 60 * 60 * 1000; // 2 hours
+
+    for (const [jobId, job] of jobs.entries()) {
+        if (now - job.createdAt > maxAge) {
+            // Clean up files
+            try {
+                if (fs.existsSync(job.outputDir)) {
+                    fs.rmSync(job.outputDir, { recursive: true, force: true });
+                }
+            } catch (e) {
+                console.error(`[PDF Convert] Failed to cleanup job ${jobId}:`, e.message);
+            }
+            jobs.delete(jobId);
+            console.log(`[PDF Convert] Cleaned up old job: ${jobId}`);
+        }
+    }
+}, 30 * 60 * 1000);

public/scripts/modules/ColorRmSession.js

@@ -342,7 +342,8 @@ export const ColorRmSession = {
 
     /**
      * Import PDF file - opens file picker and initiates server-side conversion
-     * This is an experimental feature that converts PDF pages to SVG
+     * This is an experimental feature that converts PDF pages to SVG using pdf2svg
+     * The PDF conversion server runs locally on HuggingFace Spaces (port 7861)
      */
     async importPdf() {
         const input = document.createElement('input');
@@ -356,14 +357,18 @@ export const ColorRmSession = {
             this.ui.toggleLoader(true, 'Uploading PDF...');
 
             try {
-                // Upload PDF to server
+                // Get the base URL for the PDF convert server
+                // On HF Spaces, it runs on port 7861 on the same host
+                const currentHost = window.location.hostname;
+                const pdfServerBase = currentHost.includes('hf.space') || currentHost.includes('huggingface')
+                    ? `${window.location.protocol}//${currentHost}:7861`
+                    : 'http://localhost:7861';
+
+                // Upload PDF to local PDF convert server
                 const formData = new FormData();
                 formData.append('file', file);
 
-                const uploadUrl = window.Config?.apiUrl(`/api/color_rm/pdf/${this.state.sessionId}`)
-                    || `/api/color_rm/pdf/${this.state.sessionId}`;
-
-                const response = await fetch(uploadUrl, {
+                const response = await fetch(`${pdfServerBase}/convert/pdf`, {
                     method: 'POST',
                     body: formData
                 });
@@ -374,8 +379,8 @@ export const ColorRmSession = {
 
                 const result = await response.json();
 
-                // Show conversion status dialog
-                this._showPdfConversionDialog(result.jobId, file.name);
+                // Show conversion status dialog (using local server)
+                this._showPdfConversionDialog(result.jobId, file.name, pdfServerBase);
 
             } catch (e) {
                 console.error('PDF upload error:', e);
@@ -392,8 +397,9 @@ export const ColorRmSession = {
      * Shows PDF conversion status dialog with polling
      * @param {string} jobId - The conversion job ID
      * @param {string} fileName - Original file name
+     * @param {string} serverBase - Base URL for PDF convert server
      */
-    _showPdfConversionDialog(jobId, fileName) {
+    _showPdfConversionDialog(jobId, fileName, serverBase) {
         const modal = document.createElement('div');
         modal.className = 'overlay';
         modal.style.cssText = 'position:fixed; top:0; left:0; width:100%; height:100%; background:rgba(0,0,0,0.8); z-index:300; display:flex; align-items:center; justify-content:center;';
@@ -401,14 +407,14 @@ export const ColorRmSession = {
         modal.innerHTML = `
             <div class="card" style="max-width:450px; width:90%; background:var(--bg-panel); border:1px solid var(--border); padding:20px;">
                 <h3 style="margin:0 0 15px; color:white; font-size:1.1rem;">
-                    <i class="bi bi-file-pdf" style="color:#ff6b6b;"></i> PDF Import (Experimental)
+                    <i class="bi bi-file-pdf" style="color:#ff6b6b;"></i> PDF Import (pdf2svg)
                 </h3>
                 <p style="color:#aaa; margin:0 0 15px; font-size:0.9rem;">${fileName}</p>
 
                 <div id="pdfStatus" style="padding:15px; background:var(--bg-dark); border-radius:8px; margin-bottom:15px;">
                     <div style="display:flex; align-items:center; gap:10px; color:#888;">
                         <div class="spinner" style="width:20px; height:20px; border:2px solid #333; border-top-color:#4dabf7; border-radius:50%; animation:spin 1s linear infinite;"></div>
-                        <span>Processing PDF...</span>
+                        <span>Converting with pdf2svg...</span>
                     </div>
                     <div id="pdfProgress" style="margin-top:10px; font-size:0.85rem; color:#666;"></div>
                 </div>
@@ -423,7 +429,7 @@ export const ColorRmSession = {
                 </div>
 
                 <p style="font-size:0.75rem; color:#666; margin:15px 0 0; text-align:center;">
-                    ⚠️ Experimental: PDF conversion quality may vary
+                    Using pdf2svg for high-quality vector conversion
                 </p>
             </div>
             <style>
@@ -438,10 +444,7 @@ export const ColorRmSession = {
 
         const pollStatus = async () => {
             try {
-                const statusUrl = window.Config?.apiUrl(`/api/color_rm/pdf/${this.state.sessionId}/job/${jobId}`)
-                    || `/api/color_rm/pdf/${this.state.sessionId}/job/${jobId}`;
-
-                const response = await fetch(statusUrl);
+                const response = await fetch(`${serverBase}/convert/status/${jobId}`);
                 if (!response.ok) throw new Error('Status check failed');
 
                 jobData = await response.json();
@@ -451,23 +454,23 @@ export const ColorRmSession = {
                 const importAllBtn = modal.querySelector('#pdfImportAll');
 
                 if (jobData.status === 'processing') {
-                    progressEl.textContent = `Processing page ${jobData.processedPages} of ${jobData.pageCount || '?'}...`;
+                    progressEl.textContent = `Converting page ${jobData.processedPages} of ${jobData.pageCount || '?'}...`;
                 } else if (jobData.status === 'completed') {
                     clearInterval(pollInterval);
 
                     statusEl.innerHTML = `
                         <div style="color:#51cf66; display:flex; align-items:center; gap:10px;">
                             <i class="bi bi-check-circle-fill"></i>
-                            <span>Conversion complete!</span>
+                            <span>Conversion complete! ${jobData.pageCount} pages ready.</span>
                         </div>
                     `;
 
-                    if (jobData.availablePages && jobData.availablePages.length > 0) {
+                    if (jobData.pages && jobData.pages.length > 0) {
                         pagesEl.style.display = 'block';
-                        pagesEl.innerHTML = jobData.availablePages.map(pageNum => `
-                            <div class="pdf-page-item" data-page="${pageNum}" style="padding:10px; border:1px solid var(--border); margin-bottom:5px; border-radius:4px; cursor:pointer; display:flex; justify-content:space-between; align-items:center;">
-                                <span>Page ${pageNum}</span>
-                                <button class="btn btn-sm pdf-import-page" data-page="${pageNum}">Import</button>
+                        pagesEl.innerHTML = jobData.pages.map(p => `
+                            <div class="pdf-page-item" data-page="${p.page}" style="padding:10px; border:1px solid var(--border); margin-bottom:5px; border-radius:4px; cursor:pointer; display:flex; justify-content:space-between; align-items:center;">
+                                <span>Page ${p.page}</span>
+                                <button class="btn btn-sm pdf-import-page" data-page="${p.page}" data-url="${serverBase}${p.url}">Import</button>
                             </div>
                         `).join('');
 
@@ -477,7 +480,7 @@ export const ColorRmSession = {
                         pagesEl.querySelectorAll('.pdf-import-page').forEach(btn => {
                             btn.onclick = (e) => {
                                 e.stopPropagation();
-                                this._importPdfPage(jobId, parseInt(btn.dataset.page));
+                                this._importPdfPageFromUrl(btn.dataset.url, parseInt(btn.dataset.page));
                             };
                         });
                     }
@@ -490,7 +493,7 @@ export const ColorRmSession = {
                         </div>
                     `;
                 } else if (jobData.status === 'pending') {
-                    progressEl.textContent = 'Waiting in queue...';
+                    progressEl.textContent = 'Starting conversion...';
                 }
 
             } catch (e) {
@@ -499,14 +502,16 @@ export const ColorRmSession = {
         };
 
         // Start polling
-        pollInterval = setInterval(pollStatus, 2000);
+        pollInterval = setInterval(pollStatus, 1000);
         pollStatus(); // Initial check
 
         // Import all pages button
         modal.querySelector('#pdfImportAll').onclick = async () => {
-            if (jobData?.availablePages) {
-                for (const pageNum of jobData.availablePages) {
-                    await this._importPdfPage(jobId, pageNum);
+            if (jobData?.pages) {
+                modal.querySelector('#pdfImportAll').disabled = true;
+                modal.querySelector('#pdfImportAll').textContent = 'Importing...';
+                for (const p of jobData.pages) {
+                    await this._importPdfPageFromUrl(`${serverBase}${p.url}`, p.page);
                 }
                 document.body.removeChild(modal);
             }
@@ -528,18 +533,15 @@ export const ColorRmSession = {
     },
 
     /**
-     * Import a single PDF page as SVG
-     * @param {string} jobId - The conversion job ID
-     * @param {number} pageNum - Page number to import
+     * Import a PDF page from a URL (from local pdf2svg server)
+     * @param {string} svgUrl - URL to fetch SVG content
+     * @param {number} pageNum - Page number for display
      */
-    async _importPdfPage(jobId, pageNum) {
+    async _importPdfPageFromUrl(svgUrl, pageNum) {
         this.ui.toggleLoader(true, `Importing page ${pageNum}...`);
 
         try {
-            const pageUrl = window.Config?.apiUrl(`/api/color_rm/pdf/${this.state.sessionId}/job/${jobId}/page/${pageNum}`)
-                || `/api/color_rm/pdf/${this.state.sessionId}/job/${jobId}/page/${pageNum}`;
-
-            const response = await fetch(pageUrl);
+            const response = await fetch(svgUrl);
             if (!response.ok) throw new Error('Failed to download page');
 
             const svgContent = await response.text();
@@ -562,15 +564,32 @@ export const ColorRmSession = {
             await this.dbPut('pages', pageObj);
             this.state.images.push(pageObj);
 
-            // CRITICAL: Upload the page blob to R2 so other users can fetch it
-            const uploadSuccess = await this._uploadPageBlob(pageObj.pageId, pageObj.blob);
+            // Upload base history to R2 for SVG items
+            if (toSync.length > 0 && this.state.sessionId && pageObj.pageId) {
+                try {
+                    const historyUrl = window.Config?.apiUrl(`/api/color_rm/history/${this.state.sessionId}/${pageObj.pageId}`)
+                        || `/api/color_rm/history/${this.state.sessionId}/${pageObj.pageId}`;
+                    await fetch(historyUrl, {
+                        method: 'POST',
+                        headers: { 'Content-Type': 'application/json' },
+                        body: JSON.stringify(toSync)
+                    });
+                    pageObj.hasBaseHistory = true;
+                    pageObj._baseHistory = [...toSync];
+                    await this.dbPut('pages', pageObj);
+                } catch (e) {
+                    console.error('[PDF Import] Failed to upload history:', e);
+                }
+            }
 
-            // Sync page structure to R2 and Liveblocks
+            // Upload page blob and sync structure
+            const uploadSuccess = await this._uploadPageBlob(pageObj.pageId, pageObj.blob);
             await this._syncPageStructureToServer();
             this._syncPageStructureToLive();
 
-            if (this.liveSync && !this.liveSync.isInitializing && toSync.length > 0) {
-                this.liveSync.setHistory(newPageIndex, toSync);
+            if (this.liveSync && !this.liveSync.isInitializing) {
+                this.liveSync.setHistory(newPageIndex, []);
+                this.liveSync.updatePageMetadata(newPageIndex, { hasBaseHistory: true, baseHistoryCount: toSync.length });
             }
 
             // Navigate to the new page
@@ -579,7 +598,7 @@ export const ColorRmSession = {
 
             if (this.state.activeSideTab === 'pages') this.renderPageSidebar();
 
-            const syncStatus = uploadSuccess ? '✓ Synced' : '⚠ Local only';
+            const syncStatus = uploadSuccess ? '✓' : '⚠';
             this.ui.showToast(`Imported PDF page ${pageNum} ${syncStatus}`);
 
         } catch (e) {