Spaces:
Running
Running
| // Scans the data directory for .jsonl benchmark files and writes an | |
| // index.json manifest inside it, so the static site knows which datasets | |
| // exist (static hosts cannot list directories). Runs automatically before | |
| // dev/build; can also be run standalone against any data directory: | |
| // | |
| // node scripts/update-data-index.mjs [data-dir] (default: ../data) | |
| // | |
| // Layout (category dirs are walked recursively): | |
| // data/<name>.jsonl -> "models" category (default) | |
| // data/<category>/<name>.jsonl -> <category> category (legacy flat) | |
| // data/<category>/<host>/<date>/<name>.jsonl -> <category> category (partitioned) | |
| import { readdirSync, writeFileSync, rmSync, existsSync, statSync } from 'node:fs' | |
| import { join, resolve } from 'node:path' | |
| import { fileURLToPath } from 'node:url' | |
| const root = resolve(fileURLToPath(new URL('..', import.meta.url))) | |
| const dataDir = process.argv[2] ? resolve(process.argv[2]) : join(root, 'data') | |
| const indexFile = join(dataDir, 'index.json') | |
| /** Recursively collect .jsonl paths under dir, prefixed with prefix. */ | |
| function collectJsonl(dir, prefix, out) { | |
| for (const entry of readdirSync(dir).sort()) { | |
| const full = join(dir, entry) | |
| if (statSync(full).isDirectory()) collectJsonl(full, `${prefix}${entry}/`, out) | |
| else if (entry.endsWith('.jsonl')) out.push(`${prefix}${entry}`) | |
| } | |
| return out | |
| } | |
| /** @type {Record<string, string[]>} category -> paths relative to data/ */ | |
| const categories = {} | |
| if (existsSync(dataDir)) { | |
| for (const entry of readdirSync(dataDir).sort()) { | |
| const full = join(dataDir, entry) | |
| if (statSync(full).isDirectory()) { | |
| const files = collectJsonl(full, `${entry}/`, []) | |
| if (files.length > 0) categories[entry] = files | |
| } else if (entry.endsWith('.jsonl')) { | |
| ;(categories.models ??= []).push(entry) | |
| } | |
| } | |
| } | |
| if (Object.keys(categories).length > 0) { | |
| writeFileSync(indexFile, JSON.stringify({ categories }, null, 2) + '\n') | |
| } else { | |
| rmSync(indexFile, { force: true }) | |
| } | |
| const summary = Object.entries(categories) | |
| .map(([cat, files]) => `${cat}: ${files.length}`) | |
| .join(', ') | |
| console.log(`[data-index] ${dataDir}: ${summary || 'no datasets found'}`) | |