Create server.js

server.js

@@ -0,0 +1,183 @@
+const express = require('express');
+const app = express();
+const PORT = 7860;
+
+// Environment Variables from HF Space Settings
+const HF_TOKEN = process.env.HF_TOKEN;
+const REPO_ID = process.env.SIMBA_DATA; 
+
+// In-memory cache
+let CACHED_DATA = null;
+
+// Serve static files from 'public' folder
+app.use(express.static('public'));
+
+// --- Helper: Download File from HF ---
+async function fetchHFFile(filename) {
+    const url = `https://huggingface.co/datasets/${REPO_ID}/resolve/main/${filename}`;
+    console.log(`Downloading: ${filename}...`);
+    const res = await fetch(url, { headers: { "Authorization": `Bearer ${HF_TOKEN}` } });
+    if (!res.ok) {
+        console.warn(`Failed to fetch ${filename}: ${res.statusText}`);
+        return [];
+    }
+    const text = await res.text();
+    // Parse JSONL
+    return text.split('\n').filter(line => line.trim()).map(line => {
+        try { return JSON.parse(line); } catch(e) { return null; }
+    }).filter(x => x);
+}
+
+// --- Helper: List Files in Folder ---
+async function listHFFiles(folder) {
+    const url = `https://huggingface.co/api/datasets/${REPO_ID}/tree/main/${folder}`;
+    const res = await fetch(url, { headers: { "Authorization": `Bearer ${HF_TOKEN}` } });
+    if (!res.ok) return [];
+    const files = await res.json();
+    return files.filter(f => f.path.endsWith('.jsonl')).map(f => f.path);
+}
+
+// --- Main Processor ---
+async function loadAndProcessData() {
+    console.log("Starting Data Sync...");
+    
+    // 1. Load Metadata
+    const metaList = await fetchHFFile("final_results/metadata.jsonl");
+    const langMap = {};
+    metaList.forEach(m => { langMap[m.lang_code_key] = m; });
+
+    // 2. Load TTS & SLID
+    const ttsData = await fetchHFFile("final_results/tts_results.jsonl");
+    const slidData = await fetchHFFile("final_results/slid_results.jsonl");
+
+    // 3. Load ASR Files
+    const files = await listHFFiles("final_results");
+    let asrRecords = [];
+    for (const file of files) {
+        // Skip metadata/tts/slid if they appear in list
+        if (file.includes('metadata') || file.includes('tts_') || file.includes('slid_')) continue;
+        const records = await fetchHFFile(file);
+        asrRecords.push(...records);
+    }
+
+    // 4. Process ASR Data
+    const families = new Set();
+    const models = new Set();
+    const familyData = {}; // { Family: { Model: { Lang: {wer, cer} } } }
+
+    asrRecords.forEach(r => {
+        if (r.task !== 'asr') return;
+        
+        // Enrich
+        const meta = langMap[r.lang_code] || { name: r.lang_code, family: 'Unknown' };
+        const fam = meta.family;
+        const mod = r.model_name;
+
+        families.add(fam);
+        models.add(mod);
+
+        // Init structure
+        if (!familyData[fam]) familyData[fam] = {};
+        if (!familyData[fam][mod]) familyData[fam][mod] = {};
+
+        familyData[fam][mod][meta.name] = {
+            wer: (r.datasets_avg_wer || 0) * 100,
+            cer: (r.datasets_avg_cer || 0) * 100
+        };
+    });
+
+    // 5. Format ASR By Family
+    const asrByFamily = {};
+    Array.from(families).sort().forEach(fam => {
+        const modData = familyData[fam];
+        
+        // Find all languages for this family
+        const langsSet = new Set();
+        Object.values(modData).forEach(m => Object.keys(m).forEach(l => langsSet.add(l)));
+        const langs = Array.from(langsSet).sort();
+
+        // Build Rows
+        const rows = Object.keys(modData).map(modName => {
+            const scores = modData[modName];
+            let sW=0, sC=0, n=0;
+            
+            langs.forEach(l => {
+                if(scores[l]) { sW += scores[l].wer; sC += scores[l].cer; n++; }
+            });
+
+            const row = {
+                Model: modName,
+                Avg_WER: n ? sW/n : 0,
+                Avg_CER: n ? sC/n : 0
+            };
+            
+            // Add individual lang scores
+            langs.forEach(l => {
+                if(scores[l]) {
+                    row[`WER_${l}`] = scores[l].wer;
+                    row[`CER_${l}`] = scores[l].cer;
+                }
+            });
+            return row;
+        });
+
+        // Sort by WER
+        rows.sort((a,b) => a.Avg_WER - b.Avg_WER);
+        asrByFamily[fam] = { data: rows, languages: langs };
+    });
+
+    // 6. Format ASR By Model
+    const asrByModel = {};
+    Array.from(models).forEach(mod => {
+        const rows = [];
+        asrRecords.forEach(r => {
+            if(r.model_name === mod && r.task === 'asr') {
+                const meta = langMap[r.lang_code] || { name: r.lang_code, family: 'Unknown' };
+                rows.push({
+                    Language: meta.name,
+                    Family: meta.family,
+                    WER: (r.datasets_avg_wer || 0) * 100,
+                    CER: (r.datasets_avg_cer || 0) * 100
+                });
+            }
+        });
+        rows.sort((a,b) => a.Language.localeCompare(b.Language));
+        asrByModel[mod] = rows;
+    });
+
+    // 7. Format TTS
+    const ttsGrouped = {};
+    const ttsModels = new Set();
+    ttsData.forEach(r => {
+        ttsModels.add(r.model);
+        if(!ttsGrouped[r.model]) ttsGrouped[r.model] = [];
+        ttsGrouped[r.model].push(r);
+    });
+
+    CACHED_DATA = {
+        metadata: {
+            families: Array.from(families).sort(),
+            models: Array.from(models).sort(),
+            tts_models: Array.from(ttsModels).sort()
+        },
+        asr: { by_family: asrByFamily, by_model: asrByModel },
+        tts: ttsGrouped,
+        slid: slidData
+    };
+    console.log("Data Processing Complete.");
+}
+
+// API Route
+app.get('/api/data', async (req, res) => {
+    if (!CACHED_DATA) {
+        try { await loadAndProcessData(); } 
+        catch (e) { return res.status(500).json({error: e.message}); }
+    }
+    res.json(CACHED_DATA);
+});
+
+// Start
+app.listen(PORT, '0.0.0.0', () => {
+    console.log(`Server listening on port ${PORT}`);
+    loadAndProcessData(); // Initial load
+});