Spaces:
Sleeping
Sleeping
| import { Router } from 'express'; | |
| import { verifyToken } from './auth.js'; | |
| import axios from 'axios'; | |
| import db from '../lib/db.js'; | |
| const router = Router(); | |
| let isSyncing = false; | |
| // Background Sync Function | |
| async function syncProjects() { | |
| if (isSyncing) return; | |
| isSyncing = true; | |
| console.log('[HF] Starting background sync...'); | |
| const tokens = (process.env.HF_TOKENS || '').split(',').map(t => t.trim()).filter(Boolean); | |
| const projectsMap = new Map<string, any>(); | |
| const orgsToSync = new Map<string, string>(); | |
| try { | |
| // 1. Fetch user projects and identify organizations | |
| await Promise.all(tokens.map(async (token) => { | |
| try { | |
| const userRes = await axios.get('https://hf-mirror.com/api/whoami-v2', { | |
| headers: { Authorization: `Bearer ${token}` }, | |
| timeout: 10000 | |
| }); | |
| const username = userRes.data.name; | |
| if (userRes.data.orgs && Array.isArray(userRes.data.orgs)) { | |
| userRes.data.orgs.forEach((org: any) => { | |
| if (!orgsToSync.has(org.name)) { | |
| orgsToSync.set(org.name, token); | |
| } | |
| }); | |
| } | |
| const spacesRes = await axios.get(`https://hf-mirror.com/api/spaces?author=${username}&full=true&limit=1000`, { | |
| headers: { Authorization: `Bearer ${token}` }, | |
| timeout: 20000 | |
| }); | |
| processSpaces(spacesRes.data, projectsMap); | |
| } catch (innerErr: any) { | |
| console.error(`[HF] Error fetching for token:`, innerErr.message); | |
| } | |
| })); | |
| // 2. Fetch organization projects | |
| if (orgsToSync.size > 0) { | |
| await Promise.all(Array.from(orgsToSync.entries()).map(async ([orgName, token]) => { | |
| try { | |
| const spacesRes = await axios.get(`https://hf-mirror.com/api/spaces?author=${orgName}&full=true&limit=1000`, { | |
| headers: { Authorization: `Bearer ${token}` }, | |
| timeout: 20000 | |
| }); | |
| processSpaces(spacesRes.data, projectsMap); | |
| } catch (err: any) { | |
| console.error(`[HF] Error fetching for org ${orgName}:`, err.message); | |
| } | |
| })); | |
| } | |
| // 3. Update Database (Transaction) | |
| const upsertStmt = db.prepare(` | |
| INSERT OR REPLACE INTO hf_projects ( | |
| id, full_name, name, title, description, url, iframe_url, type, created_at_hf, likes, sdk, synced_at | |
| ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) | |
| `); | |
| const projects = Array.from(projectsMap.values()); | |
| db.transaction(() => { | |
| if (projects.length > 0) { | |
| // Truncate to remove deleted projects | |
| db.prepare('DELETE FROM hf_projects').run(); | |
| projects.forEach(p => { | |
| upsertStmt.run( | |
| p.id, p.full_name, p.name, p.title, p.description, p.url, p.iframeUrl, p.type, p.createdAt, p.likes, p.sdk | |
| ); | |
| }); | |
| } | |
| })(); | |
| console.log(`[HF] Sync complete. Total projects: ${projects.length}`); | |
| } catch (err: any) { | |
| console.error('[HF] Sync failed:', err.message); | |
| } finally { | |
| isSyncing = false; | |
| } | |
| } | |
| function processSpaces(spaces: any[], map: Map<string, any>) { | |
| spaces.forEach((space: any) => { | |
| if (map.has(space._id)) return; | |
| const subdomain = space.id.replace(/[\/_.]/g, '-').toLowerCase(); | |
| map.set(space._id, { | |
| id: space._id, | |
| full_name: space.id, | |
| name: space.id.split('/')[1], | |
| title: space.cardData?.title || space.id.split('/')[1], | |
| description: space.cardData?.short_description || '', | |
| url: `https://hf-mirror.com/spaces/${space.id}`, | |
| iframeUrl: `https://${subdomain}.hf.space`, | |
| type: 'space', | |
| createdAt: space.createdAt, | |
| likes: space.likes, | |
| sdk: space.sdk | |
| }); | |
| }); | |
| } | |
| router.get('/list', verifyToken, async (req: any, res) => { | |
| try { | |
| // 1. Get from DB | |
| const projects = db.prepare('SELECT * FROM hf_projects ORDER BY created_at_hf DESC').all(); | |
| // 2. Return immediately | |
| const mappedProjects = projects.map((p: any) => ({ | |
| id: p.id, | |
| full_name: p.full_name, | |
| name: p.name, | |
| title: p.title, | |
| description: p.description, | |
| url: p.url, | |
| iframeUrl: p.iframe_url, | |
| type: p.type, | |
| createdAt: p.created_at_hf, | |
| likes: p.likes, | |
| sdk: p.sdk | |
| })); | |
| res.json({ success: true, projects: mappedProjects, syncing: isSyncing }); | |
| // 3. Trigger background sync | |
| syncProjects(); | |
| } catch (err: any) { | |
| res.status(500).json({ success: false, error: err.message }); | |
| } | |
| }); | |
| // Manual sync endpoint | |
| router.post('/sync', verifyToken, async (req: any, res) => { | |
| if (isSyncing) return res.json({ success: true, message: 'Sync already in progress' }); | |
| syncProjects(); // Start background sync | |
| res.json({ success: true, message: 'Sync started' }); | |
| }); | |
| export default router; | |