Blame for data.js

// Reading the index in the browser. Every file comes from one commit of the dataset, so every number agrees.
import { asyncBufferFromUrl, parquetMetadataAsync, parquetReadObjects } from 'https://cdn.jsdelivr.net/npm/hyparquet@1.31.5/+esm';
import { compressors } from 'https://cdn.jsdelivr.net/npm/hyparquet-compressors@1.1.2/+esm';
import { HF, loadRevision as askRevision, REPO } from './commit.js';
import { t } from './i18n.js';

export { HF, REPO };
export const DATASET_URL = `${HF}/datasets/${REPO}`;

export const fileUrl = (sha, path) => `${DATASET_URL}/resolve/${sha}/${path}`;
export const num = (v) => (v == null ? null : typeof v === 'bigint' ? Number(v) : Number(v));

// the two layers, the values of the catalog column kind (#952, #1083): a material set (task material) and a corpus
export const MATERIAL_SET = 'material_set';
export const CORPUS = 'corpus';
export const KINDS = [MATERIAL_SET, CORPUS];

async function getJson(url) {
  const r = await fetch(url);
  if (!r.ok) throw new Error(`${url}: HTTP ${r.status}`);
  return r.json();
}

async function getText(url) {
  const r = await fetch(url);
  if (!r.ok) throw new Error(`${url}: HTTP ${r.status}`);
  return r.text();
}

async function getBytes(url) {
  const r = await fetch(url);
  if (!r.ok) throw new Error(`${url}: HTTP ${r.status}`);
  return r.arrayBuffer();
}

/** A small file read whole (the meta tables): one request instead of a size probe and ranges. */
export async function readParquet(url, options = {}) {
  return parquetReadObjects({ file: await getBytes(url), compressors, utf8: false, ...options });
}

export async function parquetRows(url) {
  const file = await asyncBufferFromUrl({ url });
  return num((await parquetMetadataAsync(file)).num_rows);
}

/** The columns of cols that the file has, read with hyparquet (byte ranges); range: {rowStart, rowEnd}. */
export async function readColumns(url, cols, range = {}) {
  const file = await asyncBufferFromUrl({ url });
  const metadata = await parquetMetadataAsync(file);
  const names = new Set(metadata.schema.slice(1).map((s) => s.name));
  const columns = cols.filter((c) => names.has(c));
  const rows = await parquetReadObjects({ file, metadata, columns, compressors, utf8: false, ...range });
  return { rows, total: num(metadata.num_rows) };
}

/** The commit to read: ?rev=<sha or branch>, else main. Returns {sha, date, files}. */
export async function loadRevision(rev) {
  const r = await askRevision(rev);
  if (!r.sha) throw new Error(t('error.noCommit'));
  return r;
}

/** Batch folders of one source with the date of their last commit (Hub tree API). */
export async function loadBatchDates(sha, source) {
  const j = await getJson(`${HF}/api/datasets/${REPO}/tree/${sha}/data/${encodeURIComponent(source)}?expand=true`);
  const out = new Map();
  for (const e of j) {
    if (e.type !== 'directory') continue;
    out.set(e.path.split('/').pop(), e.lastCommit?.date || null);
  }
  return out;
}

// ---- the classification contract (#252): domain is a list of codes (D1 to D10, with leaves), or multi, or
// unclassified; form is one id; names come from meta/vocabulary.json, in English (name) and in Chinese (zh). Without
// them everything shows as unclassified.
const pick = (o, keys) => {
  if (!o || typeof o !== 'object') return undefined;
  for (const k of keys) if (o[k] != null) return o[k];
  return undefined;
};
const NAME_KEYS = ['name', 'label', 'title', 'en'];
const CODE_KEYS = ['code', 'id', 'key', 'value'];
const KIDS_KEYS = ['children', 'leaves', 'subdomains', 'items'];
const ABOUT_KEYS = ['description', 'about', 'definition'];
const ZH_KEYS = ['zh'];

function vocabSection(node) {
  const names = new Map();
  const zh = new Map();
  const about = new Map();
  const order = [];
  const add = (code, name, text, zhName) => {
    if (code == null) return;
    code = String(code);
    if (!names.has(code)) order.push(code);
    if (typeof name === 'string' && name.trim()) names.set(code, name.trim());
    else if (!names.has(code)) names.set(code, null);
    if (typeof text === 'string' && text.trim()) about.set(code, text.trim());
    if (typeof zhName === 'string' && zhName.trim()) zh.set(code, zhName.trim());
  };
  const walk = (n) => {
    if (!n) return;
    if (Array.isArray(n)) {
      for (const it of n) {
        if (typeof it === 'string') add(it, null);
        else if (it && typeof it === 'object') {
          add(pick(it, CODE_KEYS), pick(it, NAME_KEYS), pick(it, ABOUT_KEYS), pick(it, ZH_KEYS));
          walk(pick(it, KIDS_KEYS));
        }
      }
    } else if (typeof n === 'object') {
      for (const [k, v] of Object.entries(n)) {
        if (typeof v === 'string') add(k, v);
        else if (v && typeof v === 'object') {
          add(k, pick(v, NAME_KEYS), pick(v, ABOUT_KEYS), pick(v, ZH_KEYS));
          walk(pick(v, KIDS_KEYS));
        } else add(k, null);
      }
    }
  };
  walk(node);
  return { names, zh, about, order };
}

// meta/vocabulary.json (#304, #1052): domains [{code, name, zh, about, parent}], forms [{id, zh, about}]. A domain shows
// its name; a form has no English name, so in English it shows its id written as words (test-suites -> test suites).
// zh is the name in Chinese. #1083: layers [{kind, name, zh, about}] (task material, then corpora), relation {about,
// zh} (the sentence that says how the two layers differ) and collections [{code, name, zh, about, campaigns}] (the
// lines of admit_collections.tsv, in their order; campaigns: the collection and its earlier names, the campaign tags
// of its material sets).
export function parseVocabulary(j) {
  const form = vocabSection(pick(j, ['form', 'forms']));
  for (const [id, name] of form.names) if (!name) form.names.set(id, id.replace(/[-_]+/g, ' '));
  const layers = new Map((Array.isArray(j?.layers) ? j.layers : []).map((l) => [l.kind, l]));
  const collections = (Array.isArray(j?.collections) ? j.collections : []).map((c) => ({
    code: String(c.code), name: c.name || String(c.code), zh: c.zh || '', about: c.about || '',
    campaigns: Array.isArray(c.campaigns) ? c.campaigns.map(String) : [],
  }));
  return { domain: vocabSection(pick(j, ['domain', 'domains'])), form, layers, relation: j?.relation || null, collections };
}

/** meta/vocabulary.json (#1052), not_included [{category, requirement, zh}]: category -> {en, zh}, the requirement
 * sentence of each category of the not-included list in English and in Chinese. */
export function parseRequirements(j) {
  const out = new Map();
  for (const c of Array.isArray(j?.not_included) ? j.not_included : []) {
    if (c && typeof c.category === 'string') out.set(c.category, { en: c.requirement || '', zh: c.zh || '' });
  }
  return out;
}

export const topOf = (code) => String(code).split('.')[0];

function domainOf(v) {
  let list = v;
  if (typeof list === 'string') list = [list];
  if (list && typeof list.toArray === 'function') list = list.toArray();
  list = Array.isArray(list) ? list.map(String).map((s) => s.trim()).filter(Boolean) : [];
  if (!list.length || list.includes('unclassified')) return { kind: 'none', codes: [] };
  if (list.includes('multi')) return { kind: 'multi', codes: [] };
  return { kind: 'codes', codes: [...new Set(list)] };
}

// ---- small parsers of the index's own text files
const intOf = (s) => Number(String(s).replace(/,/g, ''));

/** CHANGELOG.md -> [{date, title, kind, sourcesAdded, sourcesRemoved, rowsAdded, rowsRemoved, sources, rows, added,
 * rowsBy}]; added: the ids of the sources the entry adds (at the start, every source it lists), rowsBy: id -> rows
 * added (at the start, the rows of each source), so the page can count each kind apart. */
export function parseChangelog(md) {
  const out = [];
  const parts = md.split(/^## /m).slice(1);
  for (const part of parts) {
    const [head, ...lines] = part.split('\n');
    const m = head.match(/^(\d{4}-\d{2}-\d{2}):\s*(.+)$/);
    if (!m) continue;
    const e = { date: m[1], title: m[2].trim(), kind: 'other', sourcesAdded: null, sourcesRemoved: null, rowsAdded: 0, rowsRemoved: 0,
      added: [], rowsBy: new Map() };
    const body = lines.join('\n');
    const start = body.match(/lists ([\d,]+) sources with ([\d,]+) rows/);
    if (start) {
      e.kind = 'start';
      e.sources = intOf(start[1]);
      e.rows = intOf(start[2]);
      for (const r of body.matchAll(/^\s+- (\S+): ([\d,]+) rows?\s*$/gm)) {
        e.added.push(r[1]);
        e.rowsBy.set(r[1], intOf(r[2]));
      }
    }
    const sa = body.match(/^- Sources added \((\d+)\)(?::\s*(.*))?$/m);
    const sr = body.match(/^- Sources removed \((\d+)\)/m);
    if (sa || sr) {
      e.kind = 'update';
      e.sourcesAdded = sa ? Number(sa[1]) : 0;
      e.sourcesRemoved = sr ? Number(sr[1]) : 0;
      if (sa && Number(sa[1]) > 0 && sa[2]) e.added = sa[2].split(/,\s*/).map((s) => s.trim()).filter(Boolean);
    }
    for (const r of body.matchAll(/^\s+- (\S+): .*?\(([\d,]+) to ([\d,]+)\)\s*$/gm)) {
      const d = intOf(r[3]) - intOf(r[2]);
      if (d > 0) e.rowsAdded += d;
      else e.rowsRemoved -= d;
      e.rowsBy.set(r[1], d);
    }
    out.push(e);
  }
  return out;
}

/** What every row of a source allows, from sources.json license_rows: 'yes' (every row), 'cond' (yes or
 * conditional for every row) or 'no' (some rows are no or unknown). */
function permission(lr) {
  if (!lr) return 'no';
  const keys = Object.entries(lr).filter(([, v]) => num(v?.rows) > 0).map(([k]) => k);
  if (!keys.length) return 'no';
  if (keys.every((k) => k === 'yes')) return 'yes';
  if (keys.every((k) => k === 'yes' || k === 'conditional')) return 'cond';
  return 'no';
}

/** The catalog's rows and its file as read, for the page's DuckDB table (linked.js): {rows, file: {format, bytes}}. */
async function readCatalog(url) {
  if (/\.json(\?|$)/i.test(url)) {
    const j = await getJson(url);
    const rows = Array.isArray(j) ? j : j.sources || j.rows || [];
    return { rows, file: { format: 'json', bytes: new TextEncoder().encode(JSON.stringify(rows)) } };
  }
  const buf = await getBytes(url);
  // DuckDB-WASM takes its own copy: the rows hyparquet returns may still point into buf
  const bytes = new Uint8Array(buf.slice(0));
  return { rows: await parquetReadObjects({ file: buf, compressors, utf8: false }), file: { format: 'parquet', bytes } };
}

/** The counts the page shows in more than one place (the hero, the overview, the heading of the material band),
 * computed once here and read by all of them, each for one layer, never added together (#1083): task material
 * (collections: the distinct collections of the material sets; ready_*: material sets, in sets and units; answers:
 * units with an answer file)
 * and corpora (raw_*: corpora, in rows; their sizes, sizeOf). portal.py first_screen writes first.json with the same
 * rule. */
export function totals(S) {
  const T = S.filter((s) => s.isTask);
  return {
    collections: new Set(T.map((s) => s.collection).filter(Boolean)).size,
    ready_sources: T.length,
    ready_units: T.reduce((a, s) => a + s.readyUnits, 0),
    answers: T.reduce((a, s) => a + s.answered, 0),
    raw_sources: S.length - T.length,
    raw_rows: S.reduce((a, s) => a + s.rawRows, 0),
    ...sizeOf(S.filter((s) => !s.isTask)),
  };
}

/** The size of S, sources of one layer: distinct content (null when none of them records it) and the rows without a
 * size, not in it (#364). */
export function sizeOf(S) {
  const known = S.filter((s) => s.distinct != null);
  return {
    distinct_bytes: known.length ? known.reduce((a, s) => a + s.distinct, 0) : null,
    unsized_rows: S.reduce((a, s) => a + s.rowsNoSize, 0),
  };
}

/** Per collection of the vocabulary, in its order: the collection with its material sets in S (sets), their units and
 * their units with an answer file. */
export function collectionTotals(S, collections) {
  const T = S.filter((s) => s.isTask);
  return collections.map((c) => {
    const sets = T.filter((s) => s.collection === c.code);
    return { ...c, sets, units: sets.reduce((a, s) => a + s.readyUnits, 0), answers: sets.reduce((a, s) => a + s.answered, 0) };
  });
}

/** Everything the page shows, from one commit. params: URLSearchParams (rev, catalog, vocabulary); onNote(key, values):
 * a note for the reader, as a key of the locale files. */
export async function loadIndex(params, onNote) {
  const rev = await loadRevision(params.get('rev'));
  const at = (p) => fileUrl(rev.sha, p);
  const catalogUrl = params.get('catalog') || at('meta/catalog.parquet');
  const has = (p) => rev.files.includes(p);
  const vocabUrl = params.get('vocabulary') || (has('meta/vocabulary.json') ? at('meta/vocabulary.json') : null);

  const [cat, sourcesJson, coverage, batches, notIncluded, changelog, vocabJson] = await Promise.all([
    readCatalog(catalogUrl),
    getJson(at('meta/sources.json')),
    getJson(at('meta/coverage.json')),
    readParquet(at('meta/batches.parquet')).catch(() => []),
    readParquet(at('meta/not_included.parquet')).catch(() => []),
    getText(at('CHANGELOG.md')).catch(() => ''),
    vocabUrl ? getJson(vocabUrl).catch((e) => { onNote?.('notice.vocabulary', { error: e.message }); return null; }) : null,
  ]);

  const vocab = parseVocabulary(vocabJson || {});
  const srcInfo = new Map((sourcesJson.sources || []).map((e) => [e.source, e]));
  const cov = coverage.by_source || {};
  const batchesBy = new Map();
  for (const b of batches) {
    if (!batchesBy.has(b.source)) batchesBy.set(b.source, []);
    batchesBy.get(b.source).push(b);
  }
  // the catalog column kind (#952); an index commit before it has none, and the page computes the kind below
  const hasKind = cat.rows.some((r) => r.kind != null && r.kind !== '');
  const filesBy = new Map();
  for (const f of rev.files) {
    const m = f.match(/^data\/([^/]+)\/([^/]+)\/[^/]+\.parquet$/);
    if (!m) continue;
    if (!filesBy.has(m[1])) filesBy.set(m[1], []);
    filesBy.get(m[1]).push({ path: f, batch: m[2] });
  }

  const sources = cat.rows.map((r) => {
    const info = srcInfo.get(r.source) || {};
    const lic = r.license || r.licence || {};
    const dom = domainOf(r.domain);
    const rows = num(r.files ?? info.rows) || 0;
    const bytes = num(r.bytes);
    const form = r.form == null || r.form === '' || r.form === 'unclassified' ? null : String(r.form);
    // The kind is the catalog column kind: material_set (task-ready) or corpus (raw). An index commit without it gets
    // the rule the index writes it by: rows of a collection carry roles (sources.json unit: unit, companion, answer),
    // or it has answers, or fewer units than rows; such a source is a material set, every other one a corpus.
    // A unit is one unit group (a test case, one workbook); files are counted apart. Once the catalog carries the
    // columns units (distinct units; a corpus counts rows), answers (units that have an answer) and files (files of
    // every role), they are read as they are. Until then a material set's units are its answer files when it has any
    // (one answer per unit group), else its unit rows; a corpus is counted in rows, each row one unit to mine.
    const roles = info.unit || {};
    const has = (k) => r[k] != null && r[k] !== '' && Number.isFinite(num(r[k]));
    const answers = has('answers') ? num(r.answers) : num(roles.answer) || 0;
    const isTask = hasKind ? r.kind === MATERIAL_SET : num(roles.unit) > 0 || answers > 0 || (has('units') && num(r.units) < rows);
    const units = has('units') ? num(r.units) : isTask ? (answers || num(roles.unit)) : rows;
    const lr = info.license_rows?.redistribution_ok || {};
    const shipRows = (num(lr.yes?.rows) || 0) + (num(lr.conditional?.rows) || 0);
    const shipShare = rows ? Math.min(1, shipRows / rows) : 0;
    return {
      kind: isTask ? MATERIAL_SET : CORPUS,
      isTask,
      // the collection of a material set (#1083): a line of admit_collections.tsv; none for a corpus
      collection: isTask && r.collection ? String(r.collection) : null,
      readyUnits: isTask ? units : 0,
      rawRows: isTask ? 0 : rows,
      fromCatalog: has('units'),
      roles: { unit: num(roles.unit) || 0, companion: num(roles.companion) || 0, answer: answers },
      kinds: Object.keys(roles).filter((k) => !['unit', 'companion', 'answer'].includes(k)),
      units,
      answered: answers,
      shipShare,
      id: r.source,
      title: r.title || r.source,
      description: r.description || info.description || '',
      summary: r.summary || '',
      unit: r.unit || '',
      status: r.status || null,
      reasons: Array.isArray(r.status_reasons) ? r.status_reasons : r.status_reasons?.toArray?.() || [],
      domainKind: dom.kind,
      codes: dom.codes,
      tops: [...new Set(dom.codes.map(topOf))],
      form,
      tier: lic.tier || 'unknown',
      ship: lic.ship || null,
      spdx: lic.spdx || null,
      licenceNote: lic.note || null,
      contamination: r.contamination || {},
      pii: r.pii || {},
      rows,
      bytes: bytes && bytes > 0 ? bytes : null,
      // distinct content (#364): bytes minus the sizes of the rows that repeat another row's content; null when not recorded
      distinct: has('distinct_bytes') ? num(r.distinct_bytes) : null,
      rowsNoSize: num(r.files_without_size) || 0,
      topTypes: Array.isArray(r.top_types) ? r.top_types : r.top_types?.toArray?.() || [],
      needsAuth: r.needs_auth || null,
      train: permission(info.license_rows?.training_ok),
      redist: permission(info.license_rows?.redistribution_ok),
      info,
      cov: cov[r.source] || null,
      batches: batchesBy.get(r.source) || [],
      files: (filesBy.get(r.source) || []).sort((a, b) => a.path.localeCompare(b.path)),
    };
  });

  return {
    sha: rev.sha,
    date: rev.date,
    overrides: { catalog: params.get('catalog'), vocabulary: params.get('vocabulary') },
    hasVocabulary: Boolean(vocabJson),
    hasKind,
    vocab,
    sources,
    totals: totals(sources),
    collections: collectionTotals(sources, vocab.collections),
    catalogFile: cat.file,
    coverage,
    notIncluded,
    requirements: parseRequirements(vocabJson),
    changelog: parseChangelog(changelog),
  };
}