File size: 3,530 Bytes
da6bb80 d10c686 da6bb80 d10c686 da6bb80 d10c686 9e4d07b d10c686 da6bb80 d10c686 da6bb80 d10c686 da6bb80 d10c686 da6bb80 d10c686 da6bb80 d10c686 da6bb80 d10c686 da6bb80 d10c686 da6bb80 d10c686 da6bb80 d10c686 da6bb80 d10c686 da6bb80 d10c686 da6bb80 d10c686 da6bb80 d10c686 da6bb80 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 |
// @ts-nocheck
import express from 'express';
import cors from 'cors';
import multer from 'multer';
import { commit } from '@huggingface/hub'; // Use 'commit' for batch operations
import path from 'path';
import { fileURLToPath } from 'url';
// Fix __dirname for ES Modules
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const app = express();
// Hugging Face Spaces requires port 7860
const port = process.env.PORT || 3001;
// Middleware
app.use(cors());
app.use(express.json());
// Configure Multer for memory storage
// No limit on file count here, but we limit payload size via express/nginx usually
const storage = multer.memoryStorage();
const upload = multer({ storage: storage });
// SERVER CONFIGURATION
const SERVER_CONFIG = {
TOKEN: process.env.HF_TOKEN || '',
REPO: process.env.HF_REPO || 'TwanAPI/DataTwan',
TYPE: 'dataset'
};
// --- API ROUTES ---
// Batch Upload Endpoint
// Accepts multiple files in the 'files' field
app.post('/api/upload', upload.array('files'), async (req, res) => {
try {
const files = req.files;
// req.body.paths can be a string (if 1 file) or array (if multiple)
// We normalize it to an array
const paths = [].concat(req.body.paths || []);
if (!files || files.length === 0) {
return res.status(400).json({ error: 'No files provided' });
}
if (files.length !== paths.length) {
return res.status(400).json({ error: 'Mismatch between file count and path count' });
}
// Security Check
if (!SERVER_CONFIG.TOKEN) {
console.error('[SERVER] Upload Failed: HF_TOKEN missing');
return res.status(500).json({ error: 'Server misconfiguration: HF_TOKEN secret is missing.' });
}
console.log(`[SERVER] Processing batch of ${files.length} files...`);
// Prepare operations for Hugging Face 'commit'
const operations = files.map((file, index) => ({
operation: 'addOrUpdate',
path: paths[index],
content: new Blob([file.buffer])
}));
// Execute single commit for all files
const response = await commit({
credentials: {
accessToken: SERVER_CONFIG.TOKEN,
},
repo: {
type: SERVER_CONFIG.TYPE,
name: SERVER_CONFIG.REPO
},
operations: operations,
title: `Upload batch of ${files.length} files via DataTwan`
});
const commitHash = response.oid; // 'commit' returns { oid: string, ... }
const urlPrefix = "https://huggingface.co/datasets";
// Generate public URLs for response
const urls = paths.map(p => `${urlPrefix}/${SERVER_CONFIG.REPO}/blob/${commitHash}/${p}`);
console.log(`[SERVER] Batch Success: ${files.length} files committed (Hash: ${commitHash})`);
res.json({
success: true,
count: files.length,
urls: urls
});
} catch (error) {
console.error('[SERVER] Batch Upload Error:', error);
res.status(500).json({
success: false,
error: error.message || 'Internal Server Error'
});
}
});
// --- SERVE FRONTEND (Production) ---
app.use(express.static(path.join(__dirname, 'dist')));
app.get('*', (req, res) => {
res.sendFile(path.join(__dirname, 'dist', 'index.html'));
});
// Start Server
app.listen(port, () => {
console.log(`--------------------------------------------------`);
console.log(`✅ Server running on port ${port}`);
console.log(` Target Repo: ${SERVER_CONFIG.REPO}`);
console.log(`--------------------------------------------------`);
});
|