// Scans the data directory for .jsonl benchmark files and writes an
// index.json manifest inside it, so the static site knows which datasets
// exist (static hosts cannot list directories). Runs automatically before
// dev/build; can also be run standalone against any data directory:
//
//   node scripts/update-data-index.mjs [data-dir]   (default: ../data)
//
// Layout (category dirs are walked recursively):
//   data/<name>.jsonl                              -> "models" category (default)
//   data/<category>/<name>.jsonl                   -> <category> category (legacy flat)
//   data/<category>/<host>/<date>/<name>.jsonl     -> <category> category (partitioned)
import { readdirSync, writeFileSync, rmSync, existsSync, statSync } from 'node:fs'
import { join, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'

const root = resolve(fileURLToPath(new URL('..', import.meta.url)))
const dataDir = process.argv[2] ? resolve(process.argv[2]) : join(root, 'data')
const indexFile = join(dataDir, 'index.json')

/** Recursively collect .jsonl paths under dir, prefixed with prefix. */
function collectJsonl(dir, prefix, out) {
  for (const entry of readdirSync(dir).sort()) {
    const full = join(dir, entry)
    if (statSync(full).isDirectory()) collectJsonl(full, `${prefix}${entry}/`, out)
    else if (entry.endsWith('.jsonl')) out.push(`${prefix}${entry}`)
  }
  return out
}

/** @type {Record<string, string[]>} category -> paths relative to data/ */
const categories = {}

if (existsSync(dataDir)) {
  for (const entry of readdirSync(dataDir).sort()) {
    const full = join(dataDir, entry)
    if (statSync(full).isDirectory()) {
      const files = collectJsonl(full, `${entry}/`, [])
      if (files.length > 0) categories[entry] = files
    } else if (entry.endsWith('.jsonl')) {
      ;(categories.models ??= []).push(entry)
    }
  }
}

if (Object.keys(categories).length > 0) {
  writeFileSync(indexFile, JSON.stringify({ categories }, null, 2) + '\n')
} else {
  rmSync(indexFile, { force: true })
}
const summary = Object.entries(categories)
  .map(([cat, files]) => `${cat}: ${files.length}`)
  .join(', ')
console.log(`[data-index] ${dataDir}: ${summary || 'no datasets found'}`)