basicpitch/basicpitch.js

// basicpitch.js - Spotify's Basic Pitch in the browser: audio in, notes and MIDI out.
//
//   const output = await transcribe(samples);   // mono Float32Array at SAMPLE_RATE
//   const notes = findNotes(output);            // [{ pitchMidi, startTimeSeconds, durationSeconds, amplitude }]
//   const midi = toMidi(notes);                 // the bytes of a .mid file

import { notesFromOutput } from "./notes.js";

export const SAMPLE_RATE = 22050;
const NOTES = 88, FPS = Math.floor(SAMPLE_RATE / 256);
const WINDOW = 43844, FRAMES = 172, EDGE = 15, ROWS = FRAMES - 2 * EDGE;   // as in bp.c
const OVERLAP = 2 * EDGE * 256, STEP = WINDOW - OVERLAP;

let pool = null, jobs = 0;

// Starts the workers (transcribe does too). The first call sets the number of threads.
export function preload({ threads = Math.min(8, navigator.hardwareConcurrency || 4) } = {}) {
  if (pool) return pool.ready;
  const workers = Array.from({ length: threads }, () => new Worker(import.meta.url, { type: "module", name: "basicpitch" }));
  const handlers = new Map();   // job id -> its result handler
  const ready = loadFiles().then((files) => Promise.all(workers.map((worker) => new Promise((resolve, reject) => {
    worker.onmessage = ({ data }) => {
      if (data.ready) resolve();
      else if (data.job === undefined) reject(new Error(data.error));
      else handlers.get(data.job)?.(worker, data);
    };
    worker.onerror = (e) => {
      e.preventDefault();
      const error = e.message || "Basic Pitch failed";
      reject(new Error(error));
      for (const handle of handlers.values()) handle(worker, { error });
    };
    worker.postMessage(files);
  }))));
  pool = { workers, handlers, ready };
  ready.catch(() => { for (const worker of workers) worker.terminate(); pool = null; });
  return ready;
}

async function loadFiles() {
  const simd = WebAssembly.validate(new Uint8Array([0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10, 10, 1, 8, 0, 65, 0, 253, 15, 253, 98, 11]));
  const get = (name) => fetch(new URL(name, import.meta.url)).then((r) => {
    if (!r.ok) throw new Error(`can't load ${name} (${r.status})`);
    return r.arrayBuffer();
  });
  const [wasm, plan, weights] = await Promise.all([get(simd ? "bp.wasm" : "bp-plain.wasm"), get("plan.bin"), get("weights.bin")]);
  return { module: await WebAssembly.compile(wasm), plan, weights };
}

// The model's output: { frames, onsets, rows }, frames[row * 88 + key] (key 0 is MIDI note 21).
export async function transcribe(samples, { threads, onProgress, signal } = {}) {
  await preload(threads ? { threads } : undefined);
  const job = ++jobs, rows = Math.floor(samples.length * FPS / SAMPLE_RATE), count = Math.ceil((samples.length + OVERLAP / 2) / STEP);
  const frames = new Float32Array(rows * NOTES), onsets = new Float32Array(rows * NOTES);
  let next = 0, done = 0;
  try {
    await new Promise((resolve, reject) => {
      if (signal?.aborted) return reject(new DOMException("Aborted", "AbortError"));
      signal?.addEventListener("abort", () => reject(new DOMException("Aborted", "AbortError")), { once: true });
      const send = (worker) => {
        if (next >= count || signal?.aborted) return;
        const index = next++, start = index * STEP - OVERLAP / 2, window = new Float32Array(WINDOW);
        window.set(samples.subarray(Math.max(0, start), Math.min(samples.length, start + WINDOW)), Math.max(0, -start));
        worker.postMessage({ job, index, window }, [window.buffer]);
      };
      pool.handlers.set(job, (worker, data) => {
        if (data.error) return reject(new Error(data.error));
        const at = data.index * ROWS, n = Math.min(ROWS, rows - at) * NOTES;
        if (n > 0) { frames.set(data.frames.subarray(0, n), at * NOTES); onsets.set(data.onsets.subarray(0, n), at * NOTES); }
        done++;   // not inside onProgress?.(), which skips its argument without a callback
        onProgress?.(done / count);
        if (done === count) resolve();
        else send(worker);
      });
      for (const worker of pool.workers) { send(worker); send(worker); }
    });
  } finally {
    pool.handlers.delete(job);
  }
  return { frames, onsets, rows };
}

// The settings and defaults are basicpitch.spotify.com's (ms, Hz, 0 for no limit).
export function findNotes(output, { onsetThreshold = 0.5, frameThreshold = 0.3, minNoteLength = 128, minFrequency = 0, maxFrequency = 3000, boostOnsets = true } = {}) {
  return notesFromOutput(output, onsetThreshold, frameThreshold, Math.round(minNoteLength / 1000 * FPS), minFrequency, maxFrequency, boostOnsets);
}

// The notes as the MIDI file holds them (ticks, seconds, velocity), for playback.
export function midiNotes(notes, { tempo = 120 } = {}) {
  const ticks = 8 * tempo;   // a second, at 480 a beat
  return notes.map((n) => {
    const on = Math.round(n.startTimeSeconds * ticks), off = Math.max(on + 1, Math.round((n.startTimeSeconds + n.durationSeconds) * ticks));
    return { pitch: n.pitchMidi, on, off, start: on / ticks, end: off / ticks, velocity: Math.min(127, Math.max(1, Math.round(n.amplitude * 127))) };
  });
}

// The tempo only sets the beat grid, the notes keep their times.
export function toMidi(notes, { tempo = 120 } = {}) {
  const events = [];
  for (const n of midiNotes(notes, { tempo })) events.push([n.on, 1, n.pitch, n.velocity], [n.off, 0, n.pitch, 0]);
  events.sort((a, b) => a[0] - b[0] || a[1] - b[1]);   // offs before ons
  const us = Math.floor(60e6 / tempo);
  const track = [0x00, 0xff, 0x51, 0x03, (us >> 16) & 255, (us >> 8) & 255, us & 255];
  let last = 0;
  for (const [tick, on, pitch, velocity] of events) {
    track.push(...varint(tick - last), on ? 0x90 : 0x80, pitch, velocity);
    last = tick;
  }
  track.push(0x00, 0xff, 0x2f, 0x00);
  const n = track.length;
  return new Uint8Array([
    0x4d, 0x54, 0x68, 0x64, 0, 0, 0, 6, 0, 0, 0, 1, 0x01, 0xe0,   // MThd: format 0, 1 track, 480 ticks a beat
    0x4d, 0x54, 0x72, 0x6b, n >>> 24, (n >> 16) & 255, (n >> 8) & 255, n & 255, ...track,
  ]);
}

function varint(value) {
  const bytes = [value & 0x7f];
  while ((value >>= 7)) bytes.unshift((value & 0x7f) | 0x80);
  return bytes;
}

// In a worker

let model = null;

async function setUp({ module, plan, weights }) {
  const bp = (await WebAssembly.instantiate(module, { env: { emscripten_notify_memory_growth() {} } })).exports;
  bp._initialize?.();
  const copy = (bytes) => { const p = bp.malloc(bytes.byteLength); new Uint8Array(bp.memory.buffer, p, bytes.byteLength).set(new Uint8Array(bytes)); return p; };
  const handle = bp.bp_load(copy(plan), plan.byteLength / 4, copy(weights));
  if (!handle) throw new Error("plan.bin is not a Basic Pitch plan");
  model = { bp, handle, input: bp.malloc(WINDOW * 4), frames: bp.malloc(FRAMES * NOTES * 4), onsets: bp.malloc(FRAMES * NOTES * 4) };
  // warm up, so the browser compiles the optimised code before real work
  for (let i = 0; i < 2; i++) bp.bp_run(handle, model.input, model.frames, model.onsets, 0);
}

function run({ job, index, window }) {
  const { bp, handle, input, frames, onsets } = model;
  new Float32Array(bp.memory.buffer, input, WINDOW).set(window);
  const err = bp.bp_run(handle, input, frames, onsets, 0);
  if (err) throw new Error(`Basic Pitch error ${err}`);
  const rows = (p) => new Float32Array(bp.memory.buffer, p + EDGE * NOTES * 4, ROWS * NOTES).slice();
  const f = rows(frames), o = rows(onsets);
  self.postMessage({ job, index, frames: f, onsets: o }, [f.buffer, o.buffer]);
}

if (typeof WorkerGlobalScope !== "undefined" && self.name === "basicpitch")
  self.onmessage = ({ data }) => {
    if (data.module) setUp(data).then(() => self.postMessage({ ready: true }), (error) => self.postMessage({ error: error.message }));
    else try { run(data); } catch (error) { self.postMessage({ job: data.job, error: error.message }); }
  };