Duplicate from unity/inference-engine-whisper-tiny

Co-authored-by: PB Unity <UnityPaul@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,73 @@
+# This .gitignore file should be placed at the root of your Unity project directory
+#
+# Get latest from https://github.com/github/gitignore/blob/main/Unity.gitignore
+#
+/[Ll]ibrary/
+/[Tt]emp/
+/[Oo]bj/
+/[Bb]uild/
+/[Bb]uilds/
+/[Ll]ogs/
+/[Uu]ser[Ss]ettings/
+
+# MemoryCaptures can get excessive in size.
+# They also could contain extremely sensitive data
+/[Mm]emoryCaptures/
+
+# Recordings can get excessive in size
+/[Rr]ecordings/
+
+# Uncomment this line if you wish to ignore the asset store tools plugin
+# /[Aa]ssets/AssetStoreTools*
+
+# Autogenerated Jetbrains Rider plugin
+/[Aa]ssets/Plugins/Editor/JetBrains*
+
+# Visual Studio cache directory
+.vs/
+
+# Gradle cache directory
+.gradle/
+
+# Autogenerated VS/MD/Consulo solution and project files
+ExportedObj/
+.consulo/
+*.csproj
+*.unityproj
+*.sln
+*.suo
+*.tmp
+*.user
+*.userprefs
+*.pidb
+*.booproj
+*.svd
+*.pdb
+*.mdb
+*.opendb
+*.VC.db
+
+# Unity3D generated meta files
+*.pidb.meta
+*.pdb.meta
+*.mdb.meta
+
+# Unity3D generated file on crash reports
+sysinfo.txt
+
+# Builds
+*.apk
+*.aab
+*.unitypackage
+*.app
+
+# Crashlytics generated file
+crashlytics-build.properties
+
+# Packed Addressables
+/[Aa]ssets/[Aa]ddressable[Aa]ssets[Dd]ata/*/*.bin*
+
+# Temporary auto-generated Android Assets
+/[Aa]ssets/[Ss]treamingAssets/aa.meta
+/[Aa]ssets/[Ss]treamingAssets/aa/*
+.idea

README.md

@@ -0,0 +1,30 @@
+---
+license: apache-2.0
+library_name: unity-sentis
+pipeline_tag: automatic-speech-recognition
+tags:
+  - unity-inference-engine
+---
+
+# Whisper-Tiny model in Unity 6 with Inference Engine
+
+This is the [Whisper Tiny](https://huggingface.co/openai/whisper-tiny) model running in Unity 6 with Inference Engine. It is a speech-to-text model that transcribes 16kHz wav audio to text.
+
+## How to Use
+
+* Create a new scene in Unity 6;
+* Install `com.unity.ai.inference` from the package manager;
+* Install `com.unity.nuget.newtonsoft-json` from the package manager;
+* Add the `RunWhisper.cs` script to the Main Camera;
+* Drag the `decoder_model.onnx` asset from the `models` folder into the `Audio Decoder 1` field;
+* Drag the `decoder_with_past_model.onnx` asset from the `models` folder into the `Audio Decoder 2` field;
+* Drag the `encoder_model.onnx` asset from the `models` folder into the `Audio Encoder` field;
+* Drag the `logmel_spectrogram.onnx` asset from the `models` folder into the `Log Mel Spectro` field;
+* Drag the `vocab.json` asset from the `data` folder into the `Vocab Asset` field;
+* Drag an audio asset, e.g. `data/answering-machine16kHz.wav` to the `Audio Clip` field. Ensure the `Normalize` flag is set on asset import for best results.
+
+## Preview
+Enter play mode. If working correctly the transcribed audio will be logged to the console.
+
+## Inference Engine
+Inference Engine is a neural network inference library for Unity. Find out more [here](https://docs.unity3d.com/Packages/com.unity.ai.inference@latest).

RunWhisper.cs

@@ -0,0 +1,257 @@
+using System.Collections.Generic;
+using UnityEngine;
+using Unity.InferenceEngine;
+using System.Text;
+using Unity.Collections;
+using Newtonsoft.Json;
+
+public class RunWhisper : MonoBehaviour
+{
+    Worker decoder1, decoder2, encoder, spectrogram;
+    Worker argmax;
+
+    public AudioClip audioClip;
+
+    // This is how many tokens you want. It can be adjusted.
+    const int maxTokens = 100;
+
+    // Special tokens see added tokens file for details
+    const int END_OF_TEXT = 50257;
+    const int START_OF_TRANSCRIPT = 50258;
+    const int ENGLISH = 50259;
+    const int GERMAN = 50261;
+    const int FRENCH = 50265;
+    const int TRANSCRIBE = 50359; //for speech-to-text in specified language
+    const int TRANSLATE = 50358;  //for speech-to-text then translate to English
+    const int NO_TIME_STAMPS = 50363;
+    const int START_TIME = 50364;
+
+    int numSamples;
+    string[] tokens;
+
+    int tokenCount = 0;
+    NativeArray<int> outputTokens;
+
+    // Used for special character decoding
+    int[] whiteSpaceCharacters = new int[256];
+
+    Tensor<float> encodedAudio;
+
+    bool transcribe = false;
+    string outputString = "";
+
+    // Maximum size of audioClip (30s at 16kHz)
+    const int maxSamples = 30 * 16000;
+
+    public ModelAsset audioDecoder1, audioDecoder2;
+    public ModelAsset audioEncoder;
+    public ModelAsset logMelSpectro;
+
+    public async void Start()
+    {
+        SetupWhiteSpaceShifts();
+        GetTokens();
+
+        decoder1 = new Worker(ModelLoader.Load(audioDecoder1), BackendType.GPUCompute);
+        decoder2 = new Worker(ModelLoader.Load(audioDecoder2), BackendType.GPUCompute);
+
+        FunctionalGraph graph = new FunctionalGraph();
+        var input = graph.AddInput(DataType.Float, new DynamicTensorShape(1, 1, 51865));
+        var amax = Functional.ArgMax(input, -1, false);
+        var selectTokenModel = graph.Compile(amax);
+        argmax = new Worker(selectTokenModel, BackendType.GPUCompute);
+
+        encoder = new Worker(ModelLoader.Load(audioEncoder), BackendType.GPUCompute);
+        spectrogram = new Worker(ModelLoader.Load(logMelSpectro), BackendType.GPUCompute);
+
+        outputTokens = new NativeArray<int>(maxTokens, Allocator.Persistent);
+
+        outputTokens[0] = START_OF_TRANSCRIPT;
+        outputTokens[1] = ENGLISH;// GERMAN;//FRENCH;//
+        outputTokens[2] = TRANSCRIBE; //TRANSLATE;//
+        //outputTokens[3] = NO_TIME_STAMPS;// START_TIME;//
+        tokenCount = 3;
+
+        LoadAudio();
+        EncodeAudio();
+        transcribe = true;
+
+        tokensTensor = new Tensor<int>(new TensorShape(1, maxTokens));
+        ComputeTensorData.Pin(tokensTensor);
+        tokensTensor.Reshape(new TensorShape(1, tokenCount));
+        tokensTensor.dataOnBackend.Upload<int>(outputTokens, tokenCount);
+
+        lastToken = new NativeArray<int>(1, Allocator.Persistent); lastToken[0] = NO_TIME_STAMPS;
+        lastTokenTensor = new Tensor<int>(new TensorShape(1, 1), new[] { NO_TIME_STAMPS });
+
+        while (true)
+        {
+            if (!transcribe || tokenCount >= (outputTokens.Length - 1))
+                return;
+            m_Awaitable = InferenceStep();
+            await m_Awaitable;
+        }
+    }
+    Awaitable m_Awaitable;
+
+    NativeArray<int> lastToken;
+    Tensor<int> lastTokenTensor;
+    Tensor<int> tokensTensor;
+    Tensor<float> audioInput;
+
+    void LoadAudio()
+    {
+        numSamples = audioClip.samples;
+        var data = new float[maxSamples];
+
+        // Handle stereo to mono conversion
+        if (audioClip.channels == 2)
+        {
+            var stereoData = new float[numSamples * 2];
+            audioClip.GetData(stereoData, 0);
+
+            int monoSamples = Mathf.Min(numSamples, maxSamples);
+            for (int i = 0; i < monoSamples; i++)
+            {
+                data[i] = (stereoData[i * 2] + stereoData[i * 2 + 1]) / 2f;
+            }
+        }
+        else
+        {
+            numSamples = Mathf.Min(numSamples, maxSamples);
+            audioClip.GetData(data, 0);
+        }
+
+        numSamples = maxSamples;
+        audioInput = new Tensor<float>(new TensorShape(1, numSamples), data);
+    }
+
+    void EncodeAudio()
+    {
+        spectrogram.Schedule(audioInput);
+        var logmel = spectrogram.PeekOutput() as Tensor<float>;
+        encoder.Schedule(logmel);
+        encodedAudio = encoder.PeekOutput() as Tensor<float>;
+    }
+    async Awaitable InferenceStep()
+    {
+        decoder1.SetInput("input_ids", tokensTensor);
+        decoder1.SetInput("encoder_hidden_states", encodedAudio);
+        decoder1.Schedule();
+
+        var past_key_values_0_decoder_key = decoder1.PeekOutput("present.0.decoder.key") as Tensor<float>;
+        var past_key_values_0_decoder_value = decoder1.PeekOutput("present.0.decoder.value") as Tensor<float>;
+        var past_key_values_1_decoder_key = decoder1.PeekOutput("present.1.decoder.key") as Tensor<float>;
+        var past_key_values_1_decoder_value = decoder1.PeekOutput("present.1.decoder.value") as Tensor<float>;
+        var past_key_values_2_decoder_key = decoder1.PeekOutput("present.2.decoder.key") as Tensor<float>;
+        var past_key_values_2_decoder_value = decoder1.PeekOutput("present.2.decoder.value") as Tensor<float>;
+        var past_key_values_3_decoder_key = decoder1.PeekOutput("present.3.decoder.key") as Tensor<float>;
+        var past_key_values_3_decoder_value = decoder1.PeekOutput("present.3.decoder.value") as Tensor<float>;
+
+        var past_key_values_0_encoder_key = decoder1.PeekOutput("present.0.encoder.key") as Tensor<float>;
+        var past_key_values_0_encoder_value = decoder1.PeekOutput("present.0.encoder.value") as Tensor<float>;
+        var past_key_values_1_encoder_key = decoder1.PeekOutput("present.1.encoder.key") as Tensor<float>;
+        var past_key_values_1_encoder_value = decoder1.PeekOutput("present.1.encoder.value") as Tensor<float>;
+        var past_key_values_2_encoder_key = decoder1.PeekOutput("present.2.encoder.key") as Tensor<float>;
+        var past_key_values_2_encoder_value = decoder1.PeekOutput("present.2.encoder.value") as Tensor<float>;
+        var past_key_values_3_encoder_key = decoder1.PeekOutput("present.3.encoder.key") as Tensor<float>;
+        var past_key_values_3_encoder_value = decoder1.PeekOutput("present.3.encoder.value") as Tensor<float>;
+
+        decoder2.SetInput("input_ids", lastTokenTensor);
+        decoder2.SetInput("past_key_values.0.decoder.key", past_key_values_0_decoder_key);
+        decoder2.SetInput("past_key_values.0.decoder.value", past_key_values_0_decoder_value);
+        decoder2.SetInput("past_key_values.1.decoder.key", past_key_values_1_decoder_key);
+        decoder2.SetInput("past_key_values.1.decoder.value", past_key_values_1_decoder_value);
+        decoder2.SetInput("past_key_values.2.decoder.key", past_key_values_2_decoder_key);
+        decoder2.SetInput("past_key_values.2.decoder.value", past_key_values_2_decoder_value);
+        decoder2.SetInput("past_key_values.3.decoder.key", past_key_values_3_decoder_key);
+        decoder2.SetInput("past_key_values.3.decoder.value", past_key_values_3_decoder_value);
+
+        decoder2.SetInput("past_key_values.0.encoder.key", past_key_values_0_encoder_key);
+        decoder2.SetInput("past_key_values.0.encoder.value", past_key_values_0_encoder_value);
+        decoder2.SetInput("past_key_values.1.encoder.key", past_key_values_1_encoder_key);
+        decoder2.SetInput("past_key_values.1.encoder.value", past_key_values_1_encoder_value);
+        decoder2.SetInput("past_key_values.2.encoder.key", past_key_values_2_encoder_key);
+        decoder2.SetInput("past_key_values.2.encoder.value", past_key_values_2_encoder_value);
+        decoder2.SetInput("past_key_values.3.encoder.key", past_key_values_3_encoder_key);
+        decoder2.SetInput("past_key_values.3.encoder.value", past_key_values_3_encoder_value);
+
+        decoder2.Schedule();
+
+        var logits = decoder2.PeekOutput("logits") as Tensor<float>;
+        argmax.Schedule(logits);
+        using var t_Token = await argmax.PeekOutput().ReadbackAndCloneAsync() as Tensor<int>;
+        int index = t_Token[0];
+
+        outputTokens[tokenCount] = lastToken[0];
+        lastToken[0] = index;
+        tokenCount++;
+        tokensTensor.Reshape(new TensorShape(1, tokenCount));
+        tokensTensor.dataOnBackend.Upload<int>(outputTokens, tokenCount);
+        lastTokenTensor.dataOnBackend.Upload<int>(lastToken, 1);
+
+        if (index == END_OF_TEXT)
+        {
+            transcribe = false;
+        }
+        else if (index < tokens.Length)
+        {
+            outputString += GetUnicodeText(tokens[index]);
+        }
+
+        Debug.Log(outputString);
+    }
+
+    // Tokenizer
+    public TextAsset vocabAsset;
+    void GetTokens()
+    {
+        var vocab = JsonConvert.DeserializeObject<Dictionary<string, int>>(vocabAsset.text);
+        tokens = new string[vocab.Count];
+        foreach (var item in vocab)
+        {
+            tokens[item.Value] = item.Key;
+        }
+    }
+
+    string GetUnicodeText(string text)
+    {
+        var bytes = Encoding.GetEncoding("ISO-8859-1").GetBytes(ShiftCharacterDown(text));
+        return Encoding.UTF8.GetString(bytes);
+    }
+
+    string ShiftCharacterDown(string text)
+    {
+        string outText = "";
+        foreach (char letter in text)
+        {
+            outText += ((int)letter <= 256) ? letter : (char)whiteSpaceCharacters[(int)(letter - 256)];
+        }
+        return outText;
+    }
+
+    void SetupWhiteSpaceShifts()
+    {
+        for (int i = 0, n = 0; i < 256; i++)
+        {
+            if (IsWhiteSpace((char)i)) whiteSpaceCharacters[n++] = i;
+        }
+    }
+
+    bool IsWhiteSpace(char c)
+    {
+        return !(('!' <= c && c <= '~') || ('�' <= c && c <= '�') || ('�' <= c && c <= '�'));
+    }
+
+    private void OnDestroy()
+    {
+        decoder1.Dispose();
+        decoder2.Dispose();
+        encoder.Dispose();
+        spectrogram.Dispose();
+        argmax.Dispose();
+        audioInput.Dispose();
+        lastTokenTensor.Dispose();
+        tokensTensor.Dispose();
+    }
+}

info.json

@@ -0,0 +1,17 @@
+{
+    "code": [
+      "RunWhisper.cs"
+    ],
+    "models": [
+        "models/decoder_model.onnx",
+        "models/decoder_with_past_model.onnx",
+        "models/encoder_model.onnx",
+        "models/logmel_spectrogram.onnx"
+    ],
+    "data": [
+        "data/vocab.json"
+    ],
+    "version": [
+        "2.2.0"
+    ]
+}

models/decoder_model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d09dbf0425be54e6ed5218574ae3686a7d58b0b2867bf1d60785fab2ebb23ef3
+size 198062130

models/decoder_with_past_model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18d7280f11e1a4c171005da10e3a178f0603092598660941875c69dcc5cb9645
+size 193303641

models/encoder_model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b43d4d85fb76e260791ec30bdc6784cdfa109e82e62874d27ab592e035525da3
+size 32904958

models/logmel_spectrogram.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fb448b82bde665099d8532502dd1d7f95751f3afcd3760e7b30a94ca0bddebf
+size 1354556