ear 0.1.0: spoken language identification, Core ML and LiteRT

Derived from openai/whisper-tiny (MIT), stripped to the subgraph that predicts a language: 37.8M parameters to 17.9M, identical predictions.

Core ML ships as two artifacts on purpose. The frontend runs in fp32 because its power spectrum is squared magnitudes clamped at 1e-10 and 80% of those bins fall below fp16's smallest normal number; fusing it into the fp16 detector scores 84.2% routing where the split scores 97.5%. LiteRT is one file because it runs the frontend in fp32 anyway.

Detector is 256 ops, 100% Neural Engine, 15.8 ms per 30 s window.

README.md

@@ -0,0 +1,129 @@
+---
+license: other
+license_name: desert-ant-labs-source-available-1.0
+license_link: https://license.desertant.com/1.0
+language:
+- en
+- zh
+- de
+- es
+- ru
+- ko
+- fr
+- ja
+- pt
+- tr
+- pl
+- ca
+- nl
+- ar
+- sv
+- it
+- id
+- hi
+- fi
+- vi
+- he
+- uk
+- el
+- ms
+- cs
+- ro
+- da
+- hu
+- ta
+- no
+- th
+- ur
+- hr
+- bg
+- lt
+- la
+- mi
+- ml
+- cy
+- sk
+- te
+- fa
+- lv
+- bn
+- sr
+- az
+- sl
+- kn
+- et
+- mk
+- br
+- eu
+- is
+- hy
+- ne
+- mn
+- bs
+- kk
+- sq
+- sw
+- gl
+- mr
+- pa
+- si
+- km
+- sn
+- yo
+- so
+- af
+- oc
+- ka
+- be
+- tg
+- sd
+- gu
+- am
+- yi
+- lo
+- uz
+- fo
+- ht
+- ps
+- tk
+- nn
+- mt
+- sa
+- lb
+- my
+- bo
+- tl
+- mg
+- as
+- tt
+- haw
+- ln
+- ha
+- ba
+- jw
+- su
+tags:
+- language-identification
+- language-detection
+- langid
+- spoken-language-identification
+- audio-classification
+- speech
+- on-device
+- edge-ai
+- mobile
+- offline
+- core-ml
+- apple-neural-engine
+- litert
+- multilingual
+pipeline_tag: audio-classification
+---
+
+# Ear
+
+Name the language from thirty seconds.
+
+On-device spoken language identification across 99 languages.
+
+- **SDKs and documentation:** https://github.com/Desert-Ant-Labs/desert-ant-core

THIRD_PARTY_NOTICES.md

@@ -0,0 +1,40 @@
+# Third-party notices
+
+## openai/whisper-tiny
+
+The weights in this repository are derived from
+[`openai/whisper-tiny`](https://huggingface.co/openai/whisper-tiny), released by
+OpenAI under the MIT License. The derivation removes the parameters this task
+does not use, keeps the subgraph that predicts a language, and compresses the
+result for on-device execution.
+
+```
+MIT License
+
+Copyright (c) 2022 OpenAI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+```
+
+## Evaluation corpora
+
+Published accuracy figures are measured on
+[FLEURS](https://huggingface.co/datasets/google/fleurs) (CC BY 4.0, Google) and
+[VoxPopuli](https://huggingface.co/datasets/facebook/voxpopuli) (CC0, Meta).
+Neither corpus is redistributed here.

detector.mlmodelc/analytics/coremldata.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed952e53694c407af19c56cc16e371ed3bc22af3260baec41cbe9cbdf7522ddd
+size 243

detector.mlmodelc/coremldata.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bff545265c96ee0d12a127a6ed818422e6af9edd065ffbc9bd3294e1d1d0587c
+size 369

detector.mlmodelc/weights/weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac2f45edb6654b49bfda1d2160276bebb73c325e5deb2f0deee03df0267956cf
+size 13696070

ear.tflite

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eafb390863c39b28b607094fbbb7ae9202ece7ce0d5f5a28839fdc69b5770853
+size 23947568

ear_meta.json

@@ -0,0 +1,202 @@
+{
+ "model": "ear",
+ "version": "0.1.0",
+ "task": "spoken-language-identification",
+ "audio": {
+  "sample_rate": 16000,
+  "channels": 1,
+  "window_seconds": 30,
+  "recommended_windows": 3,
+  "note": "Sample windows spread across the recording, not the first 90 seconds: openings are intros and music. Average the probabilities across windows rather than counting votes."
+ },
+ "frontend": {
+  "n_fft": 400,
+  "hop_length": 160,
+  "n_mels": 80,
+  "frames": 3000,
+  "window": "hann_periodic",
+  "log": "log10",
+  "clamp_min": 1e-10,
+  "floor_below_peak_db": 8,
+  "affine": {
+   "add": 4.0,
+   "divide": 4.0
+  },
+  "input_shape": [
+   1,
+   160,
+   1,
+   3002
+  ],
+  "input_layout": "waveform reflect-padded by n_fft/2, reshaped to [1, hop, 1, rows]; framing is host-side"
+ },
+ "artifacts": {
+  "coreml": {
+   "mel": {
+    "file": "mel.mlmodelc",
+    "precision": "fp32",
+    "input": "audio_rows",
+    "output": "mel",
+    "note": "fp32 is required, not a default. The power spectrum is squared magnitudes clamped at 1e-10, and 80% of its bins fall below fp16's smallest normal number. In fp16 this stage measures 27 dB on real audio against 200 dB in fp32, and end-to-end routing drops from 97.5% to 84.2%."
+   },
+   "detector": {
+    "file": "detector.mlmodelc",
+    "precision": "fp16",
+    "compression": "palette6-group32",
+    "input": "mel",
+    "output": "logits",
+    "ops": 256,
+    "ane_residency": 1.0
+   }
+  },
+  "litert": {
+   "model": {
+    "file": "ear.tflite",
+    "quantization": "dynamic_wi8c_hr_afp32",
+    "input": "audio_rows",
+    "output": "logits",
+    "note": "One file: LiteRT runs the frontend in fp32, so it is fused. Hadamard rotation is what makes 8-bit work here; plain min-max int8 loses 11 points of routing."
+   }
+  }
+ },
+ "output": {
+  "logits": 99,
+  "order": "languages.json",
+  "decision": "argmax over averaged softmax across windows"
+ },
+ "languages": [
+  "en",
+  "zh",
+  "de",
+  "es",
+  "ru",
+  "ko",
+  "fr",
+  "ja",
+  "pt",
+  "tr",
+  "pl",
+  "ca",
+  "nl",
+  "ar",
+  "sv",
+  "it",
+  "id",
+  "hi",
+  "fi",
+  "vi",
+  "he",
+  "uk",
+  "el",
+  "ms",
+  "cs",
+  "ro",
+  "da",
+  "hu",
+  "ta",
+  "no",
+  "th",
+  "ur",
+  "hr",
+  "bg",
+  "lt",
+  "la",
+  "mi",
+  "ml",
+  "cy",
+  "sk",
+  "te",
+  "fa",
+  "lv",
+  "bn",
+  "sr",
+  "az",
+  "sl",
+  "kn",
+  "et",
+  "mk",
+  "br",
+  "eu",
+  "is",
+  "hy",
+  "ne",
+  "mn",
+  "bs",
+  "kk",
+  "sq",
+  "sw",
+  "gl",
+  "mr",
+  "pa",
+  "si",
+  "km",
+  "sn",
+  "yo",
+  "so",
+  "af",
+  "oc",
+  "ka",
+  "be",
+  "tg",
+  "sd",
+  "gu",
+  "am",
+  "yi",
+  "lo",
+  "uz",
+  "fo",
+  "ht",
+  "ps",
+  "tk",
+  "nn",
+  "mt",
+  "sa",
+  "lb",
+  "my",
+  "bo",
+  "tl",
+  "mg",
+  "as",
+  "tt",
+  "haw",
+  "ln",
+  "ha",
+  "ba",
+  "jw",
+  "su"
+ ],
+ "untrusted": [
+  "no",
+  "sv",
+  "da"
+ ],
+ "untrusted_note": "The base model reads Norwegian as Swedish about 40% of the time. Route this group to a fallback recognizer rather than acting on the answer.",
+ "measured": {
+  "clips": {
+   "corpus": "FLEURS",
+   "n": 640,
+   "languages": 32,
+   "routing": 0.975,
+   "exact": 0.862
+  },
+  "continuous": {
+   "corpus": "VoxPopuli",
+   "files": 113,
+   "segment_seconds": 30,
+   "routing_per_segment": 0.909,
+   "routing_per_file": 0.923,
+   "routing_in_scope": 1.0
+  },
+  "latency_ms": {
+   "coreml_m3_ultra": 15.8,
+   "litert_desktop_cpu_4_threads": 45.0
+  },
+  "note": "Read and parliamentary speech only. No music, crosstalk, or phone audio, and no on-device measurement."
+ },
+ "derived_from": {
+  "model": "openai/whisper-tiny",
+  "license": "MIT",
+  "params_before": 37800000,
+  "params_after": 17900000
+ }
+}

languages.json

@@ -0,0 +1 @@
+["en", "zh", "de", "es", "ru", "ko", "fr", "ja", "pt", "tr", "pl", "ca", "nl", "ar", "sv", "it", "id", "hi", "fi", "vi", "he", "uk", "el", "ms", "cs", "ro", "da", "hu", "ta", "no", "th", "ur", "hr", "bg", "lt", "la", "mi", "ml", "cy", "sk", "te", "fa", "lv", "bn", "sr", "az", "sl", "kn", "et", "mk", "br", "eu", "is", "hy", "ne", "mn", "bs", "kk", "sq", "sw", "gl", "mr", "pa", "si", "km", "sn", "yo", "so", "af", "oc", "ka", "be", "tg", "sd", "gu", "am", "yi", "lo", "uz", "fo", "ht", "ps", "tk", "nn", "mt", "sa", "lb", "my", "bo", "tl", "mg", "as", "tt", "haw", "ln", "ha", "ba", "jw", "su"]

mel.mlmodelc/analytics/coremldata.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e926bc1b285e259baaa9b564155ff49c46b5cd027837dda998945b580ba20389
+size 243

mel.mlmodelc/coremldata.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ce5ca49f387402b03f700a2f15138b6daadecc8bdd36e90d50bebf0f2d889c2
+size 377

mel.mlmodelc/model.mil

@@ -0,0 +1,55 @@
+program(1.3)
+[buildInfo = dict<string, string>({{"coremlc-component-MIL", "3600.16.1"}, {"coremlc-version", "3600.25.2"}, {"coremltools-component-torch", "2.13.0"}, {"coremltools-source-dialect", "TorchScript"}, {"coremltools-version", "9.0"}})]
+{
+    func main<ios18>(tensor<fp32, [1, 160, 1, 3002]> audio_rows) {
+            fp32 var_3 = const()[name = string("op_3"), val = fp32(0x1.b7cdfep-34)];
+            bool var_14 = const()[name = string("op_14"), val = bool(true)];
+            tensor<fp32, [80, 201, 1, 1]> self_mel_weight = const()[name = string("self_mel_weight"), val = tensor<fp32, [80, 201, 1, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(64)))];
+            tensor<fp32, [402, 160, 1, 3]> self_stft_weight = const()[name = string("self_stft_weight"), val = tensor<fp32, [402, 160, 1, 3]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(64448)))];
+            string spectrum_pad_type_0 = const()[name = string("spectrum_pad_type_0"), val = string("valid")];
+            tensor<int32, [2]> spectrum_strides_0 = const()[name = string("spectrum_strides_0"), val = tensor<int32, [2]>([1, 1])];
+            tensor<int32, [4]> spectrum_pad_0 = const()[name = string("spectrum_pad_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [2]> spectrum_dilations_0 = const()[name = string("spectrum_dilations_0"), val = tensor<int32, [2]>([1, 1])];
+            int32 spectrum_groups_0 = const()[name = string("spectrum_groups_0"), val = int32(1)];
+            tensor<fp32, [1, 402, 1, 3000]> spectrum = conv(dilations = spectrum_dilations_0, groups = spectrum_groups_0, pad = spectrum_pad_0, pad_type = spectrum_pad_type_0, strides = spectrum_strides_0, weight = self_stft_weight, x = audio_rows)[name = string("spectrum")];
+            tensor<int32, [4]> real_begin_0 = const()[name = string("real_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> real_end_0 = const()[name = string("real_end_0"), val = tensor<int32, [4]>([1, 201, 1, 3000])];
+            tensor<bool, [4]> real_end_mask_0 = const()[name = string("real_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<fp32, [1, 201, 1, 3000]> real = slice_by_index(begin = real_begin_0, end = real_end_0, end_mask = real_end_mask_0, x = spectrum)[name = string("real")];
+            tensor<int32, [4]> imag_begin_0 = const()[name = string("imag_begin_0"), val = tensor<int32, [4]>([0, 201, 0, 0])];
+            tensor<int32, [4]> imag_end_0 = const()[name = string("imag_end_0"), val = tensor<int32, [4]>([1, 402, 1, 3000])];
+            tensor<bool, [4]> imag_end_mask_0 = const()[name = string("imag_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp32, [1, 201, 1, 3000]> imag = slice_by_index(begin = imag_begin_0, end = imag_end_0, end_mask = imag_end_mask_0, x = spectrum)[name = string("imag")];
+            tensor<fp32, [1, 201, 1, 3000]> var_21 = mul(x = real, y = real)[name = string("op_21")];
+            tensor<fp32, [1, 201, 1, 3000]> var_22 = mul(x = imag, y = imag)[name = string("op_22")];
+            tensor<fp32, [1, 201, 1, 3000]> input = add(x = var_21, y = var_22)[name = string("input")];
+            string var_24_pad_type_0 = const()[name = string("op_24_pad_type_0"), val = string("valid")];
+            tensor<int32, [2]> var_24_strides_0 = const()[name = string("op_24_strides_0"), val = tensor<int32, [2]>([1, 1])];
+            tensor<int32, [4]> var_24_pad_0 = const()[name = string("op_24_pad_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [2]> var_24_dilations_0 = const()[name = string("op_24_dilations_0"), val = tensor<int32, [2]>([1, 1])];
+            int32 var_24_groups_0 = const()[name = string("op_24_groups_0"), val = int32(1)];
+            tensor<fp32, [1, 80, 1, 3000]> var_24 = conv(dilations = var_24_dilations_0, groups = var_24_groups_0, pad = var_24_pad_0, pad_type = var_24_pad_type_0, strides = var_24_strides_0, weight = self_mel_weight, x = input)[name = string("op_24")];
+            tensor<int32, [4]> var_27_begin_0 = const()[name = string("op_27_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_27_end_0 = const()[name = string("op_27_end_0"), val = tensor<int32, [4]>([1, 80, 1, 3000])];
+            tensor<bool, [4]> var_27_end_mask_0 = const()[name = string("op_27_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
+            tensor<bool, [4]> var_27_squeeze_mask_0 = const()[name = string("op_27_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
+            tensor<fp32, [1, 80, 3000]> var_27 = slice_by_index(begin = var_27_begin_0, end = var_27_end_0, end_mask = var_27_end_mask_0, squeeze_mask = var_27_squeeze_mask_0, x = var_24)[name = string("op_27")];
+            fp32 const_0 = const()[name = string("const_0"), val = fp32(0x1.fffffep+127)];
+            tensor<fp32, [1, 80, 3000]> clip_0 = clip(alpha = var_3, beta = const_0, x = var_27)[name = string("clip_0")];
+            fp32 log_0_epsilon_0 = const()[name = string("log_0_epsilon_0"), val = fp32(0x1p-149)];
+            tensor<fp32, [1, 80, 3000]> log_0 = log(epsilon = log_0_epsilon_0, x = clip_0)[name = string("log_0")];
+            fp32 mul_0_y_0 = const()[name = string("mul_0_y_0"), val = fp32(0x1.bcb7b2p-2)];
+            tensor<fp32, [1, 80, 3000]> mul_0 = mul(x = log_0, y = mul_0_y_0)[name = string("mul_0")];
+            tensor<int32, [2]> var_31_axes_0 = const()[name = string("op_31_axes_0"), val = tensor<int32, [2]>([1, 2])];
+            tensor<fp32, [1, 1, 1]> var_31 = reduce_max(axes = var_31_axes_0, keep_dims = var_14, x = mul_0)[name = string("op_31")];
+            fp32 var_32 = const()[name = string("op_32"), val = fp32(0x1p+3)];
+            tensor<fp32, [1, 1, 1]> var_33 = sub(x = var_31, y = var_32)[name = string("op_33")];
+            tensor<fp32, [1, 80, 3000]> log = maximum(x = mul_0, y = var_33)[name = string("log")];
+            fp32 var_35 = const()[name = string("op_35"), val = fp32(0x1p+2)];
+            tensor<fp32, [1, 80, 3000]> var_36 = add(x = log, y = var_35)[name = string("op_36")];
+            fp32 _inversed_38_y_0 = const()[name = string("_inversed_38_y_0"), val = fp32(0x1p-2)];
+            tensor<fp32, [1, 80, 3000]> _inversed_38 = mul(x = var_36, y = _inversed_38_y_0)[name = string("_inversed_38")];
+            string cast_2_dtype_0 = const()[name = string("cast_2_dtype_0"), val = string("fp16")];
+            tensor<fp16, [1, 80, 3000]> mel = cast(dtype = cast_2_dtype_0, x = _inversed_38)[name = string("cast_3")];
+        } -> (mel);
+}

mel.mlmodelc/weights/weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea98097dba120b4236368cca2bd6ae5202d246faddc9d46b91964646b48a9f10
+size 836352