File size: 3,410 Bytes
8f8b0fa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | // SPDX-License-Identifier: AGPL-3.0-or-later
package dev.micr0.cavaalign
import ai.onnxruntime.OnnxTensor
import ai.onnxruntime.OrtEnvironment
import ai.onnxruntime.OrtSession
import java.nio.FloatBuffer
import java.nio.LongBuffer
/**
* Runs cavaalign.onnx with ONNX Runtime (`com.microsoft.onnxruntime:onnxruntime-android`).
*
* Audio is processed in 6.4 s windows with 3.2 s of preceding context, cut on the model's
* 0.64 s chunk boundaries, so memory stays around 0.3–0.4 GB for any length. This is the same
* recipe as `log_probs` in `align.py`.
*/
class CavaAlignModel private constructor(
private val env: OrtEnvironment,
private val session: OrtSession,
) : AutoCloseable {
/** Load the model from a file path. */
constructor(modelPath: String, threads: Int = 4) :
this(OrtEnvironment.getEnvironment(), modelPath, threads)
/** Load the model from bytes (e.g. read from Android assets). */
constructor(modelBytes: ByteArray, threads: Int = 4) :
this(OrtEnvironment.getEnvironment(), modelBytes, threads)
private constructor(env: OrtEnvironment, path: String, threads: Int) :
this(env, env.createSession(path, options(threads)))
private constructor(env: OrtEnvironment, bytes: ByteArray, threads: Int) :
this(env, env.createSession(bytes, options(threads)))
/** Log-probabilities for 16 kHz mono float audio in [-1, 1]; 50 frames per second. */
fun logProbs(audio: FloatArray): Emissions {
val rows = ArrayList<FloatArray>()
var pos = 0
while (pos < audio.size) {
val st = maxOf(0, pos - CONTEXT_CHUNKS * CHUNK)
val end = minOf(audio.size, pos + WINDOW_CHUNKS * CHUNK + 80) // last frame spans 400 samples
val x = audio.copyOfRange(st, end)
OnnxTensor.createTensor(env, FloatBuffer.wrap(x), longArrayOf(1, x.size.toLong())).use { wav ->
OnnxTensor.createTensor(env, LongBuffer.wrap(longArrayOf(x.size.toLong())), longArrayOf(1)).use { len ->
session.run(mapOf("waveform" to wav, "lengths" to len)).use { r ->
@Suppress("UNCHECKED_CAST")
val lp = (r[0].value as Array<Array<FloatArray>>)[0]
val n = when (val v = r[1].value) {
is LongArray -> v[0].toInt()
is IntArray -> v[0]
else -> lp.size
}
val skip = (pos - st) / 320
for (f in skip until minOf(n, skip + WINDOW_CHUNKS * 32)) rows.add(lp[f])
}
}
}
pos += WINDOW_CHUNKS * CHUNK
}
if (rows.isEmpty()) return Emissions(0, VOCAB)
return Emissions.fromMatrix(rows.toTypedArray())
}
override fun close() = session.close()
companion object {
private const val CHUNK = 32 * 320 // attention chunk: 32 frames of 20 ms
private const val WINDOW_CHUNKS = 10
private const val CONTEXT_CHUNKS = 5
private const val VOCAB = 29
private fun options(threads: Int) = OrtSession.SessionOptions().apply {
setIntraOpNumThreads(threads)
setCPUArenaAllocator(false) // keeps memory flat on long recordings
setMemoryPatternOptimization(false)
}
}
}
|