Update server.js

server.js

@@ -10,83 +10,114 @@ let CACHED_DATA = null;
 app.use(express.static('public'));
 
 // --- Fetch Helper ---
-async function fetchHFFile(filename) {
-    const url = `https://huggingface.co/datasets/${REPO_ID}/resolve/main/${filename}`;
-    console.log(`Downloading: ${filename}...`);
-    const res = await fetch(url, { headers: { "Authorization": `Bearer ${HF_TOKEN}` } });
-    if (!res.ok) return [];
-    const text = await res.text();
-    return text.split('\n').filter(line => line.trim()).map(line => {
-        try { return JSON.parse(line); } catch(e) { return null; }
-    }).filter(x => x);
+async function fetchHFFile(path) {
+    const url = `https://huggingface.co/datasets/${REPO_ID}/resolve/main/${path}`;
+    console.log(`[Loading] ${path}...`);
+    try {
+        const res = await fetch(url, { headers: { "Authorization": `Bearer ${HF_TOKEN}` } });
+        if (!res.ok) {
+            console.error(`[Error] Failed to fetch ${path}: ${res.status} ${res.statusText}`);
+            return [];
+        }
+        const text = await res.text();
+        return text.split('\n')
+            .filter(line => line.trim())
+            .map(line => {
+                try { return JSON.parse(line); } catch(e) { return null; }
+            })
+            .filter(x => x);
+    } catch (err) {
+        console.error(`[Error] Network error fetching ${path}:`, err);
+        return [];
+    }
 }
 
 // --- List Files Helper ---
 async function listHFFiles(folder) {
     const url = `https://huggingface.co/api/datasets/${REPO_ID}/tree/main/${folder}`;
-    const res = await fetch(url, { headers: { "Authorization": `Bearer ${HF_TOKEN}` } });
-    if (!res.ok) return [];
-    const files = await res.json();
-    return files.filter(f => f.path.endsWith('.jsonl')).map(f => f.path);
+    try {
+        const res = await fetch(url, { headers: { "Authorization": `Bearer ${HF_TOKEN}` } });
+        if (!res.ok) {
+            console.error(`[Error] Failed to list files: ${res.status}`);
+            return [];
+        }
+        const files = await res.json();
+        // Filter only .jsonl files
+        return files.filter(f => f.path.endsWith('.jsonl')).map(f => f.path);
+    } catch (err) {
+        console.error("[Error] Network error listing files:", err);
+        return [];
+    }
 }
 
 // --- Processor ---
 async function loadAndProcessData() {
-    console.log("Starting Data Sync...");
+    console.log("--- Starting Data Sync ---");
     
-    // 1. Metadata
+    // 1. Fetch Metadata (Crucial for language names)
     const metaList = await fetchHFFile("final_results/metadata.jsonl");
     const langMap = {};
-    // Map code key to full metadata object
     metaList.forEach(m => { langMap[m.lang_code_key] = m; });
 
-    // 2. Load TTS & SLID
-    const ttsData = await fetchHFFile("final_results/tts_results.jsonl");
-    const slidRecords = await fetchHFFile("final_results/slid_results.jsonl");
+    // 2. Scan ALL files in final_results
+    const files = await listHFFiles("final_results");
+    
+    let allRecords = [];
+    
+    // Load every file found (except metadata which we already loaded)
+    for (const file of files) {
+        if (file.endsWith('metadata.jsonl')) continue;
+        const records = await fetchHFFile(file);
+        allRecords.push(...records);
+    }
+
+    console.log(`[Processing] Loaded ${allRecords.length} total records. Sorting by task...`);
 
-    // --- NEW SLID PROCESSING LOGIC ---
-    // Pivot from Long Format -> Wide Format for Frontend
+    // 3. Bucket Records by Task
+    const ttsRecords = [];
+    const slidRecords = [];
+    const asrRecords = [];
+
+    allRecords.forEach(r => {
+        if (!r.task) return; 
+        const task = r.task.toLowerCase().trim();
+        
+        if (task === 'tts') ttsRecords.push(r);
+        else if (task === 'slid') slidRecords.push(r);
+        else if (task === 'asr') asrRecords.push(r);
+    });
+
+    // --- PROCESS SLID ---
     const slidMap = {};
     slidRecords.forEach(r => {
-        // r = { task: "SLID", lang_code: "bin", model_name: "MMS...", f1_score: 6.25 }
         const code = r.lang_code;
-        
-        // Find language name from metadata, fallback to code if missing
         const meta = langMap[code] || { name: code, family: 'Unknown' };
         
         if (!slidMap[code]) {
-            slidMap[code] = {
-                Language: meta.name,
-                ISO: code
-                // Model columns will be added dynamically below
-            };
+            slidMap[code] = { Language: meta.name, ISO: code };
         }
-        
-        // Assign score to the model key (e.g., row["MMS-LID-1024"] = 6.25)
         slidMap[code][r.model_name] = r.f1_score;
     });
-    // Convert Map back to Array
     const slidFinal = Object.values(slidMap);
 
+    // --- PROCESS TTS ---
+    const ttsGrouped = {};
+    const ttsModels = new Set();
+    ttsRecords.forEach(r => {
+        ttsModels.add(r.model);
+        if(!ttsGrouped[r.model]) ttsGrouped[r.model] = [];
+        const meta = langMap[r.lang_code] || { lang_code_key: r.lang_code };
+        r.iso = meta.lang_code_key;
+        r.Language = meta.name; // Ensure Language name is attached
+        ttsGrouped[r.model].push(r);
+    });
 
-    // 3. Load ASR Files
-    const files = await listHFFiles("final_results");
-    let asrRecords = [];
-    for (const file of files) {
-        if (file.includes('metadata') || file.includes('tts_') || file.includes('slid_')) continue;
-        const records = await fetchHFFile(file);
-        asrRecords.push(...records);
-    }
-
-    // 4. Process ASR
+    // --- PROCESS ASR ---
     const families = new Set();
     const models = new Set();
     const familyData = {}; 
 
     asrRecords.forEach(r => {
-        if (r.task !== 'asr') return;
-        
-        // Use metadata to get Family and ISO
         const meta = langMap[r.lang_code] || { name: r.lang_code, family: 'Unknown', lang_code_key: r.lang_code };
         const fam = meta.family;
         const mod = r.model_name;
@@ -97,7 +128,6 @@ async function loadAndProcessData() {
         if (!familyData[fam]) familyData[fam] = {};
         if (!familyData[fam][mod]) familyData[fam][mod] = {};
 
-        // Store by Language Name
         familyData[fam][mod][meta.name] = {
             wer: (r.datasets_avg_wer || 0) * 100,
             cer: (r.datasets_avg_cer || 0) * 100,
@@ -105,17 +135,14 @@ async function loadAndProcessData() {
         };
     });
 
-    // 5. Format ASR By Family
+    // Format ASR By Family
     const asrByFamily = {};
     Array.from(families).sort().forEach(fam => {
         const modData = familyData[fam];
-        
-        // Collect Languages with their ISO codes
         const langMapForFamily = new Map();
         Object.values(modData).forEach(m => {
             Object.keys(m).forEach(langName => {
-                const iso = m[langName].iso;
-                langMapForFamily.set(langName, iso);
+                langMapForFamily.set(langName, m[langName].iso);
             });
         });
         
@@ -127,17 +154,10 @@ async function loadAndProcessData() {
         const rows = Object.keys(modData).map(modName => {
             const scores = modData[modName];
             let sW=0, sC=0, n=0;
-            
             languages.forEach(l => {
                 if(scores[l.name]) { sW += scores[l.name].wer; sC += scores[l.name].cer; n++; }
             });
-
-            const row = {
-                Model: modName,
-                Avg_WER: n ? sW/n : 0,
-                Avg_CER: n ? sC/n : 0
-            };
-            
+            const row = { Model: modName, Avg_WER: n ? sW/n : 0, Avg_CER: n ? sC/n : 0 };
             languages.forEach(l => {
                 if(scores[l.name]) {
                     row[`WER_${l.name}`] = scores[l.name].wer;
@@ -146,17 +166,16 @@ async function loadAndProcessData() {
             });
             return row;
         });
-
         rows.sort((a,b) => a.Avg_WER - b.Avg_WER);
         asrByFamily[fam] = { data: rows, languages: languages };
     });
 
-    // 6. Format ASR By Model
+    // Format ASR By Model
     const asrByModel = {};
     Array.from(models).forEach(mod => {
         const rows = [];
         asrRecords.forEach(r => {
-            if(r.model_name === mod && r.task === 'asr') {
+            if(r.model_name === mod) {
                 const meta = langMap[r.lang_code] || { name: r.lang_code, family: 'Unknown', lang_code_key: r.lang_code };
                 rows.push({
                     Language: meta.name,
@@ -171,17 +190,6 @@ async function loadAndProcessData() {
         asrByModel[mod] = rows;
     });
 
-    // 7. Format TTS
-    const ttsGrouped = {};
-    const ttsModels = new Set();
-    ttsData.forEach(r => {
-        ttsModels.add(r.model);
-        if(!ttsGrouped[r.model]) ttsGrouped[r.model] = [];
-        const meta = langMap[r.lang_code] || { lang_code_key: r.lang_code };
-        r.iso = meta.lang_code_key;
-        ttsGrouped[r.model].push(r);
-    });
-
     CACHED_DATA = {
         metadata: {
             families: Array.from(families).sort(),
@@ -190,9 +198,9 @@ async function loadAndProcessData() {
         },
         asr: { by_family: asrByFamily, by_model: asrByModel },
         tts: ttsGrouped,
-        slid: slidFinal // Pass the pivoted data
+        slid: slidFinal
     };
-    console.log("Data Ready.");
+    console.log("--- Data Ready ---");
 }
 
 app.get('/api/data', async (req, res) => {