Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+plots/ug20_multilingual_mswc38.png filter=lfs diff=lfs merge=lfs -text
+plots/uh03b_confusion_probs_heatmap_libri_dev_en.png filter=lfs diff=lfs merge=lfs -text
+plots/where_they_went_timeline.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,2 +1,91 @@
 # CUPE: Contextless Universal Phoneme Encoder
-pytorch model for contexless-phoneme prediction from speech audio
+
+A PyTorch model for contextless phoneme prediction from speech audio. CUPE processes 120ms frames independently, ensuring each frame's embeddings are acoustically pure—unlike transformer models that mix context across frames.
+
+## Trained Models
+
+Two 30.1M parameter models are available in the [checkpoints directory](https://huggingface.co/Tabahi/CUPE-2i/resolve/main/ckpt).
+
+## Datasets
+
+- **LibriSpeech ASR corpus (SR12):** 960 hours of English speech from train-100, train-360, and train-500 splits.
+- **Multilingual LibriSpeech (MLS) (SLR94):** 800 hours total, with 100 hours each for 8 languages: `pl`, `pt`, `it`, `es`, `fr`, `nl`, `de`, `en`. Dataset's train/test/val splits.
+- **MSWC Multilingual Spoken Words Corpus:** 240 hours from 50 languages (max 10 hours/language).
+    - **Training:** 38 languages (`en`, `de`, `fr`, `ca`, `es`, `fa`, `it`, `ru`, `pl`, `eu`, `cy`, `eo`, `nl`, `pt`, `tt`, `cs`, `tr`, `et`, `ky`, `id`, `sv-SE`, `ar`, `el`, `ro`, `lv`, `sl`, `zh-CN`, `ga-IE`, `ta`, `vi`, `gn`, `or`)
+    - **Testing:** 6 languages (`lt`, `mt`, `ia`, `sk`, `ka`, `as`)
+
+## Metrics
+
+**English ([en_libri1000_uj01d](https://huggingface.co/Tabahi/CUPE-2i/resolve/main/ckpt/en_libri1000_uj01d_e199_val_GER=0.2307.ckpt)):**
+- **PER:** 0.25 (Phoneme Error Rate)
+- **GER:** 0.23 (Phoneme Group Error Rate)
+
+
+**Multilingual MLS ([multi_MLS8_uh02](https://huggingface.co/Tabahi/CUPE-2i/resolve/main/ckpt/multi_MLS8_uh02_e36_val_GER=0.2334.ckpt)):**
+- **PER:** 0.31
+- **GER:** 0.26
+
+**Multilingual MSWC ([multi_mswc38_ug20](https://huggingface.co/Tabahi/CUPE-2i/resolve/main/ckpt/multi_mswc38_ug20_e59_val_GER=0.5611.ckpt)):**
+- **PER:** 0.49
+- **GER:** 0.39
+
+---
+
+# Usage
+
+See [run.py](https://huggingface.co/Tabahi/CUPE-2i/blob/main/run.py) for a complete example.
+
+```python
+import torch
+import torchaudio
+from model2i import CUPEEmbeddingsExtractor  # Main CUPE model feature extractor
+import windowing  # Provides slice_windows, stich_window_predictions
+
+cupe_ckpt_path = "./ckpt/en_libri1000_uj01d_e199_val_GER=0.2307.ckpt"
+extractor = CUPEEmbeddingsExtractor(cupe_ckpt_path, device="cuda")
+
+dummy_wav = torch.zeros(1, max_wav_len, dtype=torch.float32, device="cpu")
+audio_batch = dummy_wav.unsqueeze(0)  # Add batch dimension
+
+# Window the audio
+windowed_audio = windowing.slice_windows(
+        audio_batch.to("cuda"),
+        sample_rate,
+        window_size_ms,
+        stride_ms
+)
+batch_size, num_windows, window_size = windowed_audio.shape
+windows_flat = windowed_audio.reshape(-1, window_size)
+
+logits, _ = extractor.predict(windows_flat, return_embeddings=False, groups_only=False)
+
+# Reshape and stitch window predictions
+logits = logits.reshape(batch_size, num_windows, frames_per_window, -1)
+logits = windowing.stich_window_predictions(
+        logits,
+        original_audio_length=audio_batch.size(2),
+        cnn_output_size=frames_per_window,
+        sample_rate=sample_rate,
+        window_size_ms=window_size_ms,
+        stride_ms=stride_ms
+)
+
+print(logits.shape)  # [B, T, 66]
+```
+
+# Use Cases
+
+- Timestamp alignment (examples coming soon)
+- Speech analysis
+
+## Sample probabilties timeline
+
+![Sample output logits plot](plots/where_they_went_timeline.png)
+
+## Multilingual Confusion Plot (Counts)
+
+![Multilingual Confusion Plot (counts)](plots/ug20_multilingual_mswc38.png)
+
+## English-only Confusion Plot (Probabilities)
+
+![English-only Confusion Plot (probabiltities)](plots/uh03b_confusion_probs_heatmap_libri_dev_en.png)

ckpt/en_libri1000_uj01d_e199_val_GER=0.2307.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9129933e707c5ad4da3213f832b81f7b0a57df8597ff57babfac25493bcf8a7
+size 120485062

ckpt/en_libri1000_uj01d_e62_val_GER=0.2438.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7925c404640eae093b584c40123426ef28c5aedc85bddee478e0bc5d2db32c92
+size 120485062

ckpt/multi_MLS8_uh02_e36_val_GER=0.2334.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf4fb0a387074514db0a17871e059ef08e21fc057b8ee283d5229c14f9c4933a
+size 120485126

ckpt/multi_mswc38_ug20_e59_val_GER=0.5611.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f7a070462b4ae73be05f41ac7f7ee5734b1396da10ab1a2c6e7c72df4ee2f1a
+size 120488006

run.py

@@ -262,7 +262,7 @@ if __name__ == "__main__":
 
     torch.manual_seed(42)
 
-    cupe_ckpt_path = "ckpt/m_uj01d_epoch=62_step=326088_val_GER=0.2438copy.ckpt"
+    cupe_ckpt_path = "ckpt/en_libri1000_uj01d_e199_val_GER=0.2307.ckpt"
     pipeline = EmbeddingsExtractionPipeline(cupe_ckpt_path, max_duration=10, device="cpu", verbose=False)
     
     audio_clip1_path = "samples/109867__timkahn__butterfly.wav.wav"