|
|
|
|
|
|
|
|
import express from 'express'; |
|
|
import cors from 'cors'; |
|
|
import multer from 'multer'; |
|
|
import { commit } from '@huggingface/hub'; |
|
|
import path from 'path'; |
|
|
import { fileURLToPath } from 'url'; |
|
|
|
|
|
|
|
|
const __filename = fileURLToPath(import.meta.url); |
|
|
const __dirname = path.dirname(__filename); |
|
|
|
|
|
const app = express(); |
|
|
|
|
|
const port = process.env.PORT || 7860; |
|
|
|
|
|
|
|
|
app.use(cors()); |
|
|
app.use(express.json()); |
|
|
|
|
|
|
|
|
|
|
|
const storage = multer.memoryStorage(); |
|
|
const upload = multer({ storage: storage }); |
|
|
|
|
|
|
|
|
const SERVER_CONFIG = { |
|
|
TOKEN: process.env.HF_TOKEN || '', |
|
|
REPO: process.env.HF_REPO || 'TwanAPI/DataTwan', |
|
|
TYPE: 'dataset' |
|
|
}; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app.post('/api/upload', upload.array('files'), async (req, res) => { |
|
|
try { |
|
|
const files = req.files; |
|
|
|
|
|
|
|
|
const paths = [].concat(req.body.paths || []); |
|
|
|
|
|
if (!files || files.length === 0) { |
|
|
return res.status(400).json({ error: 'No files provided' }); |
|
|
} |
|
|
|
|
|
if (files.length !== paths.length) { |
|
|
return res.status(400).json({ error: 'Mismatch between file count and path count' }); |
|
|
} |
|
|
|
|
|
|
|
|
if (!SERVER_CONFIG.TOKEN) { |
|
|
console.error('[SERVER] Upload Failed: HF_TOKEN missing'); |
|
|
return res.status(500).json({ error: 'Server misconfiguration: HF_TOKEN secret is missing.' }); |
|
|
} |
|
|
|
|
|
console.log(`[SERVER] Processing batch of ${files.length} files...`); |
|
|
|
|
|
|
|
|
const operations = files.map((file, index) => ({ |
|
|
operation: 'addOrUpdate', |
|
|
path: paths[index], |
|
|
content: new Blob([file.buffer]) |
|
|
})); |
|
|
|
|
|
|
|
|
const response = await commit({ |
|
|
credentials: { |
|
|
accessToken: SERVER_CONFIG.TOKEN, |
|
|
}, |
|
|
repo: { |
|
|
type: SERVER_CONFIG.TYPE, |
|
|
name: SERVER_CONFIG.REPO |
|
|
}, |
|
|
operations: operations, |
|
|
title: `Upload batch of ${files.length} files via DataTwan` |
|
|
}); |
|
|
|
|
|
const commitHash = response.oid; |
|
|
const urlPrefix = "https://huggingface.co/datasets"; |
|
|
|
|
|
|
|
|
const urls = paths.map(p => `${urlPrefix}/${SERVER_CONFIG.REPO}/blob/${commitHash}/${p}`); |
|
|
|
|
|
console.log(`[SERVER] Batch Success: ${files.length} files committed (Hash: ${commitHash})`); |
|
|
|
|
|
res.json({ |
|
|
success: true, |
|
|
count: files.length, |
|
|
urls: urls |
|
|
}); |
|
|
|
|
|
} catch (error) { |
|
|
console.error('[SERVER] Batch Upload Error:', error); |
|
|
res.status(500).json({ |
|
|
success: false, |
|
|
error: error.message || 'Internal Server Error' |
|
|
}); |
|
|
} |
|
|
}); |
|
|
|
|
|
|
|
|
|
|
|
app.use(express.static(path.join(__dirname, 'dist'))); |
|
|
|
|
|
app.get('*', (req, res) => { |
|
|
res.sendFile(path.join(__dirname, 'dist', 'index.html')); |
|
|
}); |
|
|
|
|
|
|
|
|
app.listen(port, () => { |
|
|
console.log(`--------------------------------------------------`); |
|
|
console.log(`✅ Server running on port ${port}`); |
|
|
console.log(` Target Repo: ${SERVER_CONFIG.REPO}`); |
|
|
console.log(`--------------------------------------------------`); |
|
|
}); |
|
|
|