// SPDX-License-Identifier: AGPL-3.0-or-later package dev.micr0.cavaalign import ai.onnxruntime.OnnxTensor import ai.onnxruntime.OrtEnvironment import ai.onnxruntime.OrtSession import java.nio.FloatBuffer import java.nio.LongBuffer /** * Runs cavaalign.onnx with ONNX Runtime (`com.microsoft.onnxruntime:onnxruntime-android`). * * Audio is processed in 6.4 s windows with 3.2 s of preceding context, cut on the model's * 0.64 s chunk boundaries, so memory stays around 0.3–0.4 GB for any length. This is the same * recipe as `log_probs` in `align.py`. */ class CavaAlignModel private constructor( private val env: OrtEnvironment, private val session: OrtSession, ) : AutoCloseable { /** Load the model from a file path. */ constructor(modelPath: String, threads: Int = 4) : this(OrtEnvironment.getEnvironment(), modelPath, threads) /** Load the model from bytes (e.g. read from Android assets). */ constructor(modelBytes: ByteArray, threads: Int = 4) : this(OrtEnvironment.getEnvironment(), modelBytes, threads) private constructor(env: OrtEnvironment, path: String, threads: Int) : this(env, env.createSession(path, options(threads))) private constructor(env: OrtEnvironment, bytes: ByteArray, threads: Int) : this(env, env.createSession(bytes, options(threads))) /** Log-probabilities for 16 kHz mono float audio in [-1, 1]; 50 frames per second. */ fun logProbs(audio: FloatArray): Emissions { val rows = ArrayList() var pos = 0 while (pos < audio.size) { val st = maxOf(0, pos - CONTEXT_CHUNKS * CHUNK) val end = minOf(audio.size, pos + WINDOW_CHUNKS * CHUNK + 80) // last frame spans 400 samples val x = audio.copyOfRange(st, end) OnnxTensor.createTensor(env, FloatBuffer.wrap(x), longArrayOf(1, x.size.toLong())).use { wav -> OnnxTensor.createTensor(env, LongBuffer.wrap(longArrayOf(x.size.toLong())), longArrayOf(1)).use { len -> session.run(mapOf("waveform" to wav, "lengths" to len)).use { r -> @Suppress("UNCHECKED_CAST") val lp = (r[0].value as Array>)[0] val n = when (val v = r[1].value) { is LongArray -> v[0].toInt() is IntArray -> v[0] else -> lp.size } val skip = (pos - st) / 320 for (f in skip until minOf(n, skip + WINDOW_CHUNKS * 32)) rows.add(lp[f]) } } } pos += WINDOW_CHUNKS * CHUNK } if (rows.isEmpty()) return Emissions(0, VOCAB) return Emissions.fromMatrix(rows.toTypedArray()) } override fun close() = session.close() companion object { private const val CHUNK = 32 * 320 // attention chunk: 32 frames of 20 ms private const val WINDOW_CHUNKS = 10 private const val CONTEXT_CHUNKS = 5 private const val VOCAB = 29 private fun options(threads: Int) = OrtSession.SessionOptions().apply { setIntraOpNumThreads(threads) setCPUArenaAllocator(false) // keeps memory flat on long recordings setMemoryPatternOptimization(false) } } }