// @ts-nocheck import express from 'express'; import cors from 'cors'; import multer from 'multer'; import { commit } from '@huggingface/hub'; import path from 'path'; import { fileURLToPath } from 'url'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const app = express(); const port = process.env.PORT || 3001; app.use(cors()); // Increase body limit for large batch JSON/Form data app.use(express.json({ limit: '500mb' })); app.use(express.urlencoded({ limit: '500mb', extended: true })); // Configure Multer with higher limits const storage = multer.memoryStorage(); const upload = multer({ storage: storage, limits: { fileSize: 500 * 1024 * 1024, // 500MB per file max fieldSize: 500 * 1024 * 1024, // 500MB per field } }); const SERVER_CONFIG = { TOKEN: process.env.HF_TOKEN || '', REPO: process.env.HF_REPO || 'TwanAPI/DataTwan', TYPE: 'dataset' }; app.post('/api/upload', upload.array('files'), async (req, res) => { try { const files = req.files; let paths = req.body.paths; // Normalize paths to array if it's a single string if (!Array.isArray(paths)) { paths = [paths].filter(Boolean); } if (!files || files.length === 0) { return res.status(400).json({ error: 'No files provided' }); } if (files.length !== paths.length) { return res.status(400).json({ error: `Mismatch: ${files.length} files vs ${paths.length} paths` }); } if (!SERVER_CONFIG.TOKEN) { return res.status(500).json({ error: 'Server misconfiguration: HF_TOKEN secret is missing.' }); } console.log(`[SERVER] Processing batch of ${files.length} files...`); const operations = files.map((file, index) => ({ operation: 'addOrUpdate', path: paths[index], content: new Blob([file.buffer]) })); // Using commit is atomic and faster for datasets than individual LFS uploads for small-medium files const response = await commit({ credentials: { accessToken: SERVER_CONFIG.TOKEN, }, repo: { type: SERVER_CONFIG.TYPE, name: SERVER_CONFIG.REPO }, operations: operations, title: `Batch upload of ${files.length} files` }); const commitHash = response.oid; const urlPrefix = "https://huggingface.co/datasets"; const urls = paths.map(p => `${urlPrefix}/${SERVER_CONFIG.REPO}/blob/${commitHash}/${p}`); console.log(`[SERVER] Batch Committed: ${commitHash}`); res.json({ success: true, count: files.length, urls: urls }); } catch (error) { console.error('[SERVER] Error:', error); res.status(500).json({ success: false, error: error.message || 'Internal Server Error' }); } }); app.use(express.static(path.join(__dirname, 'dist'))); app.get('*', (req, res) => { res.sendFile(path.join(__dirname, 'dist', 'index.html')); }); app.listen(port, () => { console.log(`✅ Server running on port ${port} | Repo: ${SERVER_CONFIG.REPO}`); });