Update server.js

server.js

@@ -2,33 +2,26 @@ const express = require('express');
 const app = express();
 const PORT = 7860;
 
-// Environment Variables from HF Space Settings
 const HF_TOKEN = process.env.HF_TOKEN;
 const REPO_ID = process.env.SIMBA_DATA; 
 
-// In-memory cache
 let CACHED_DATA = null;
 
-// Serve static files from 'public' folder
 app.use(express.static('public'));
 
-// --- Helper: Download File from HF ---
+// --- Fetch Helper ---
 async function fetchHFFile(filename) {
     const url = `https://huggingface.co/datasets/${REPO_ID}/resolve/main/${filename}`;
     console.log(`Downloading: ${filename}...`);
     const res = await fetch(url, { headers: { "Authorization": `Bearer ${HF_TOKEN}` } });
-    if (!res.ok) {
-        console.warn(`Failed to fetch ${filename}: ${res.statusText}`);
-        return [];
-    }
+    if (!res.ok) return [];
     const text = await res.text();
-    // Parse JSONL
     return text.split('\n').filter(line => line.trim()).map(line => {
         try { return JSON.parse(line); } catch(e) { return null; }
     }).filter(x => x);
 }
 
-// --- Helper: List Files in Folder ---
+// --- List Files Helper ---
 async function listHFFiles(folder) {
     const url = `https://huggingface.co/api/datasets/${REPO_ID}/tree/main/${folder}`;
     const res = await fetch(url, { headers: { "Authorization": `Bearer ${HF_TOKEN}` } });
@@ -37,52 +30,53 @@ async function listHFFiles(folder) {
     return files.filter(f => f.path.endsWith('.jsonl')).map(f => f.path);
 }
 
-// --- Main Processor ---
+// --- Processor ---
 async function loadAndProcessData() {
     console.log("Starting Data Sync...");
     
-    // 1. Load Metadata
+    // 1. Metadata
     const metaList = await fetchHFFile("final_results/metadata.jsonl");
     const langMap = {};
+    // Map code key to full metadata object
     metaList.forEach(m => { langMap[m.lang_code_key] = m; });
 
-    // 2. Load TTS & SLID
+    // 2. Load Others
     const ttsData = await fetchHFFile("final_results/tts_results.jsonl");
     const slidData = await fetchHFFile("final_results/slid_results.jsonl");
 
-    // 3. Load ASR Files
+    // 3. Load ASR
     const files = await listHFFiles("final_results");
     let asrRecords = [];
     for (const file of files) {
-        // Skip metadata/tts/slid if they appear in list
         if (file.includes('metadata') || file.includes('tts_') || file.includes('slid_')) continue;
         const records = await fetchHFFile(file);
         asrRecords.push(...records);
     }
 
-    // 4. Process ASR Data
+    // 4. Process ASR
     const families = new Set();
     const models = new Set();
-    const familyData = {}; // { Family: { Model: { Lang: {wer, cer} } } }
+    const familyData = {}; 
 
     asrRecords.forEach(r => {
         if (r.task !== 'asr') return;
         
-        // Enrich
-        const meta = langMap[r.lang_code] || { name: r.lang_code, family: 'Unknown' };
+        // Use metadata to get Family and ISO
+        const meta = langMap[r.lang_code] || { name: r.lang_code, family: 'Unknown', lang_code_key: r.lang_code };
         const fam = meta.family;
         const mod = r.model_name;
 
         families.add(fam);
         models.add(mod);
 
-        // Init structure
         if (!familyData[fam]) familyData[fam] = {};
         if (!familyData[fam][mod]) familyData[fam][mod] = {};
 
+        // Store by Language Name
         familyData[fam][mod][meta.name] = {
             wer: (r.datasets_avg_wer || 0) * 100,
-            cer: (r.datasets_avg_cer || 0) * 100
+            cer: (r.datasets_avg_cer || 0) * 100,
+            iso: meta.lang_code_key // Store ISO here
         };
     });
 
@@ -91,18 +85,28 @@ async function loadAndProcessData() {
     Array.from(families).sort().forEach(fam => {
         const modData = familyData[fam];
         
-        // Find all languages for this family
-        const langsSet = new Set();
-        Object.values(modData).forEach(m => Object.keys(m).forEach(l => langsSet.add(l)));
-        const langs = Array.from(langsSet).sort();
+        // Collect Languages with their ISO codes
+        const langMapForFamily = new Map();
+        Object.values(modData).forEach(m => {
+            Object.keys(m).forEach(langName => {
+                // Get ISO from the stored data object
+                const iso = m[langName].iso;
+                langMapForFamily.set(langName, iso);
+            });
+        });
+        
+        // Create array of objects: { name: "Amharic", iso: "amh" }
+        const languages = Array.from(langMapForFamily.keys()).sort().map(name => ({
+            name: name,
+            iso: langMapForFamily.get(name)
+        }));
 
-        // Build Rows
         const rows = Object.keys(modData).map(modName => {
             const scores = modData[modName];
             let sW=0, sC=0, n=0;
             
-            langs.forEach(l => {
-                if(scores[l]) { sW += scores[l].wer; sC += scores[l].cer; n++; }
+            languages.forEach(l => {
+                if(scores[l.name]) { sW += scores[l.name].wer; sC += scores[l.name].cer; n++; }
             });
 
             const row = {
@@ -111,19 +115,17 @@ async function loadAndProcessData() {
                 Avg_CER: n ? sC/n : 0
             };
             
-            // Add individual lang scores
-            langs.forEach(l => {
-                if(scores[l]) {
-                    row[`WER_${l}`] = scores[l].wer;
-                    row[`CER_${l}`] = scores[l].cer;
+            languages.forEach(l => {
+                if(scores[l.name]) {
+                    row[`WER_${l.name}`] = scores[l.name].wer;
+                    row[`CER_${l.name}`] = scores[l.name].cer;
                 }
             });
             return row;
         });
 
-        // Sort by WER
         rows.sort((a,b) => a.Avg_WER - b.Avg_WER);
-        asrByFamily[fam] = { data: rows, languages: langs };
+        asrByFamily[fam] = { data: rows, languages: languages };
     });
 
     // 6. Format ASR By Model
@@ -132,9 +134,10 @@ async function loadAndProcessData() {
         const rows = [];
         asrRecords.forEach(r => {
             if(r.model_name === mod && r.task === 'asr') {
-                const meta = langMap[r.lang_code] || { name: r.lang_code, family: 'Unknown' };
+                const meta = langMap[r.lang_code] || { name: r.lang_code, family: 'Unknown', lang_code_key: r.lang_code };
                 rows.push({
                     Language: meta.name,
+                    ISO: meta.lang_code_key, // Include ISO
                     Family: meta.family,
                     WER: (r.datasets_avg_wer || 0) * 100,
                     CER: (r.datasets_avg_cer || 0) * 100
@@ -151,6 +154,9 @@ async function loadAndProcessData() {
     ttsData.forEach(r => {
         ttsModels.add(r.model);
         if(!ttsGrouped[r.model]) ttsGrouped[r.model] = [];
+        // Add ISO if available in metadata, else guess or use code
+        const meta = langMap[r.lang_code] || { lang_code_key: r.lang_code };
+        r.iso = meta.lang_code_key;
         ttsGrouped[r.model].push(r);
     });
 
@@ -164,10 +170,9 @@ async function loadAndProcessData() {
         tts: ttsGrouped,
         slid: slidData
     };
-    console.log("Data Processing Complete.");
+    console.log("Data Ready.");
 }
 
-// API Route
 app.get('/api/data', async (req, res) => {
     if (!CACHED_DATA) {
         try { await loadAndProcessData(); } 
@@ -176,8 +181,7 @@ app.get('/api/data', async (req, res) => {
     res.json(CACHED_DATA);
 });
 
-// Start
 app.listen(PORT, '0.0.0.0', () => {
-    console.log(`Server listening on port ${PORT}`);
-    loadAndProcessData(); // Initial load
+    console.log(`Server running on ${PORT}`);
+    loadAndProcessData();
 });