G2P (byT5)

byT5/ppisljar/slo_g2p_byt5/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

byT5/ppisljar/slo_g2p_byt5/README.md

@@ -0,0 +1,19 @@
+---
+license: cc-by-3.0
+language:
+- sl
+metrics:
+- wer
+- cer
+library_name: nemo
+---
+
+
+Slovenian G2P model
+
+google/byt5-small trained on G2P task, with sentence level dataset
+
+CER: 0.25%
+
+check infer.py for example usage
+

byT5/ppisljar/slo_g2p_byt5/g2p_t5.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e15c8d2249cc940232f58442a2e93fe9e27fbaaaebdfb356f5f7a3a0fb7ec9c5
+size 1208441138

byT5/ppisljar/slo_g2p_byt5/g2p_t5.quant.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c1db9b6a431e0ba841a5758a872b7041354220139adfcf698adb43b33a69fbe
+size 311079866

byT5/ppisljar/slo_g2p_byt5/infer.py

@@ -0,0 +1,27 @@
+import onnxruntime
+import torch
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained('google/byt5-small')
+
+sentence = "Kupil sem bicikel in mu zamenjal stol.".lower()
+
+ort_session = onnxruntime.InferenceSession("g2p_t5.onnx", providers=["CPUExecutionProvider"])
+
+
+def g2p(sentence, onnx_session, tokenizer):
+    input_ids = [sentence]
+    input_encoding = tokenizer(
+        input_ids, padding='longest', max_length=512, truncation=True, return_tensors='pt',
+    )
+    input_ids, attention_mask = input_encoding.input_ids, input_encoding.attention_mask
+    ort_inputs = {'input_ids': input_ids.numpy()}
+    ort_outs = ort_session.run(None, ort_inputs)
+    generated_ids = [ort_outs[0]]
+    generated_texts = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+    return generated_texts
+
+    
+result = g2p(sentence, ort_session, tokenizer)
+print(result)

byT5/ppisljar/slo_g2p_byt5/source.txt

@@ -0,0 +1 @@
+https://huggingface.co/ppisljar/slo_g2p_byt5

byT5/ppisljar/slo_g2p_norm_byt5/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

byT5/ppisljar/slo_g2p_norm_byt5/README.md

@@ -0,0 +1,16 @@
+---
+license: cc-by-3.0
+language:
+- sl
+metrics:
+- cer
+- wer
+---
+
+Slovenian normalization & G2P model
+
+google/byt5-small trained on normalization and G2P task, with sentence level dataset
+
+CER: 0.33%
+
+check infer.py for example usage

byT5/ppisljar/slo_g2p_norm_byt5/g2p_t5_norm.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f41fe672a16bab8814e7686ff73acd775dc78602f9227a0ab6f1fa90f392117
+size 1208441138

byT5/ppisljar/slo_g2p_norm_byt5/g2p_t5_norm.quant.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfd8452989e4accc0fd2304b5eaab4415ee5936fbb4d9af9f4a02e613641c70e
+size 311079866

byT5/ppisljar/slo_g2p_norm_byt5/infer.py

@@ -0,0 +1,30 @@
+import onnxruntime
+import torch
+
+from transformers import AutoTokenizer
+
+# setup GPU
+if torch.cuda.is_available():
+    device = [0]  # use 0th CUDA device
+    accelerator = 'gpu'
+else:
+    device = 1
+    accelerator = 'cpu'
+
+map_location = torch.device('cuda:{}'.format(device[0]) if accelerator == 'gpu' else 'cpu')
+
+tokenizer = AutoTokenizer.from_pretrained('google/byt5-small')
+
+sentence = "Kupil sem bicikel in mu zamenjal stol.".lower()
+
+ort_session = onnxruntime.InferenceSession("g2p_norm_t5.onnx", providers=["CPUExecutionProvider"])
+input_ids = [sentence]
+input_encoding = tokenizer(
+    input_ids, padding='longest', max_length=512, truncation=True, return_tensors='pt',
+)
+input_ids, attention_mask = input_encoding.input_ids, input_encoding.attention_mask
+ort_inputs = {'input_ids': input_ids.numpy()}
+ort_outs = ort_session.run(None, ort_inputs)
+generated_ids = [ort_outs[0]]
+generated_texts = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+print(generated_texts)

byT5/ppisljar/slo_g2p_norm_byt5/source.txt

@@ -0,0 +1 @@
+https://huggingface.co/ppisljar/slo_g2p_norm_byt5