File size: 3,410 Bytes
8f8b0fa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
// SPDX-License-Identifier: AGPL-3.0-or-later
package dev.micr0.cavaalign

import ai.onnxruntime.OnnxTensor
import ai.onnxruntime.OrtEnvironment
import ai.onnxruntime.OrtSession
import java.nio.FloatBuffer
import java.nio.LongBuffer

/**
 * Runs cavaalign.onnx with ONNX Runtime (`com.microsoft.onnxruntime:onnxruntime-android`).
 *
 * Audio is processed in 6.4 s windows with 3.2 s of preceding context, cut on the model's
 * 0.64 s chunk boundaries, so memory stays around 0.3–0.4 GB for any length. This is the same
 * recipe as `log_probs` in `align.py`.
 */
class CavaAlignModel private constructor(
    private val env: OrtEnvironment,
    private val session: OrtSession,
) : AutoCloseable {

    /** Load the model from a file path. */
    constructor(modelPath: String, threads: Int = 4) :
        this(OrtEnvironment.getEnvironment(), modelPath, threads)

    /** Load the model from bytes (e.g. read from Android assets). */
    constructor(modelBytes: ByteArray, threads: Int = 4) :
        this(OrtEnvironment.getEnvironment(), modelBytes, threads)

    private constructor(env: OrtEnvironment, path: String, threads: Int) :
        this(env, env.createSession(path, options(threads)))

    private constructor(env: OrtEnvironment, bytes: ByteArray, threads: Int) :
        this(env, env.createSession(bytes, options(threads)))

    /** Log-probabilities for 16 kHz mono float audio in [-1, 1]; 50 frames per second. */
    fun logProbs(audio: FloatArray): Emissions {
        val rows = ArrayList<FloatArray>()
        var pos = 0
        while (pos < audio.size) {
            val st = maxOf(0, pos - CONTEXT_CHUNKS * CHUNK)
            val end = minOf(audio.size, pos + WINDOW_CHUNKS * CHUNK + 80)  // last frame spans 400 samples
            val x = audio.copyOfRange(st, end)
            OnnxTensor.createTensor(env, FloatBuffer.wrap(x), longArrayOf(1, x.size.toLong())).use { wav ->
                OnnxTensor.createTensor(env, LongBuffer.wrap(longArrayOf(x.size.toLong())), longArrayOf(1)).use { len ->
                    session.run(mapOf("waveform" to wav, "lengths" to len)).use { r ->
                        @Suppress("UNCHECKED_CAST")
                        val lp = (r[0].value as Array<Array<FloatArray>>)[0]
                        val n = when (val v = r[1].value) {
                            is LongArray -> v[0].toInt()
                            is IntArray -> v[0]
                            else -> lp.size
                        }
                        val skip = (pos - st) / 320
                        for (f in skip until minOf(n, skip + WINDOW_CHUNKS * 32)) rows.add(lp[f])
                    }
                }
            }
            pos += WINDOW_CHUNKS * CHUNK
        }
        if (rows.isEmpty()) return Emissions(0, VOCAB)
        return Emissions.fromMatrix(rows.toTypedArray())
    }

    override fun close() = session.close()

    companion object {
        private const val CHUNK = 32 * 320       // attention chunk: 32 frames of 20 ms
        private const val WINDOW_CHUNKS = 10
        private const val CONTEXT_CHUNKS = 5
        private const val VOCAB = 29

        private fun options(threads: Int) = OrtSession.SessionOptions().apply {
            setIntraOpNumThreads(threads)
            setCPUArenaAllocator(false)          // keeps memory flat on long recordings
            setMemoryPatternOptimization(false)
        }
    }
}