Commit ·
815b0ad
1
Parent(s): eb9a32d
Just fixing evaluation script, model has not been updated
Browse files
README.md
CHANGED
|
@@ -29,7 +29,7 @@ model-index:
|
|
| 29 |
metrics:
|
| 30 |
- name: Test WER
|
| 31 |
type: wer
|
| 32 |
-
value:
|
| 33 |
---
|
| 34 |
|
| 35 |
|
|
@@ -78,23 +78,27 @@ print("Reference:", test_dataset["sentence"][:2])
|
|
| 78 |
|
| 79 |
The model can be evaluated as follows on the Portuguese test data of Common Voice.
|
| 80 |
|
|
|
|
|
|
|
| 81 |
|
| 82 |
```python
|
| 83 |
import torch
|
| 84 |
import torchaudio
|
| 85 |
from datasets import load_dataset, load_metric
|
| 86 |
from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
|
|
|
|
| 87 |
import re
|
| 88 |
|
| 89 |
test_dataset = load_dataset("common_voice", "pt", split="test")
|
| 90 |
wer = load_metric("wer")
|
| 91 |
|
| 92 |
-
processor = Wav2Vec2Processor.from_pretrained("joorock12/wav2vec2-large-xlsr-portuguese")
|
| 93 |
-
model = Wav2Vec2ForCTC.from_pretrained("joorock12/wav2vec2-large-xlsr-portuguese")
|
| 94 |
model.to("cuda")
|
| 95 |
|
| 96 |
chars_to_ignore_regex = '[\,\?\.\!\-\;\:\"\“\'\�]'
|
| 97 |
resampler = torchaudio.transforms.Resample(48_000, 16_000)
|
|
|
|
| 98 |
|
| 99 |
# Preprocessing the datasets.
|
| 100 |
# We need to read the aduio files as arrays
|
|
@@ -115,7 +119,7 @@ def evaluate(batch):
|
|
| 115 |
logits = model(inputs.input_values.to("cuda"), attention_mask=inputs.attention_mask.to("cuda")).logits
|
| 116 |
|
| 117 |
pred_ids = torch.argmax(logits, dim=-1)
|
| 118 |
-
batch["pred_strings"] = processor.batch_decode(pred_ids)
|
| 119 |
return batch
|
| 120 |
|
| 121 |
result = test_dataset.map(evaluate, batched=True, batch_size=8)
|
|
@@ -123,7 +127,7 @@ result = test_dataset.map(evaluate, batched=True, batch_size=8)
|
|
| 123 |
print("WER: {:2f}".format(100 * wer.compute(predictions=result["pred_strings"], references=result["sentence"])))
|
| 124 |
```
|
| 125 |
|
| 126 |
-
**Test Result (wer)**:
|
| 127 |
|
| 128 |
|
| 129 |
## Training
|
|
|
|
| 29 |
metrics:
|
| 30 |
- name: Test WER
|
| 31 |
type: wer
|
| 32 |
+
value: 13.766801%
|
| 33 |
---
|
| 34 |
|
| 35 |
|
|
|
|
| 78 |
|
| 79 |
The model can be evaluated as follows on the Portuguese test data of Common Voice.
|
| 80 |
|
| 81 |
+
You need to install Enelvo, an open-source spell correction trained with Twitter user posts
|
| 82 |
+
`pip install enelvo`
|
| 83 |
|
| 84 |
```python
|
| 85 |
import torch
|
| 86 |
import torchaudio
|
| 87 |
from datasets import load_dataset, load_metric
|
| 88 |
from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
|
| 89 |
+
from enelvo import normaliser
|
| 90 |
import re
|
| 91 |
|
| 92 |
test_dataset = load_dataset("common_voice", "pt", split="test")
|
| 93 |
wer = load_metric("wer")
|
| 94 |
|
| 95 |
+
processor = Wav2Vec2Processor.from_pretrained("joorock12/wav2vec2-large-xlsr-portuguese-a")
|
| 96 |
+
model = Wav2Vec2ForCTC.from_pretrained("joorock12/wav2vec2-large-xlsr-portuguese-a")
|
| 97 |
model.to("cuda")
|
| 98 |
|
| 99 |
chars_to_ignore_regex = '[\,\?\.\!\-\;\:\"\“\'\�]'
|
| 100 |
resampler = torchaudio.transforms.Resample(48_000, 16_000)
|
| 101 |
+
norm = normaliser.Normaliser()
|
| 102 |
|
| 103 |
# Preprocessing the datasets.
|
| 104 |
# We need to read the aduio files as arrays
|
|
|
|
| 119 |
logits = model(inputs.input_values.to("cuda"), attention_mask=inputs.attention_mask.to("cuda")).logits
|
| 120 |
|
| 121 |
pred_ids = torch.argmax(logits, dim=-1)
|
| 122 |
+
batch["pred_strings"] = [norm.normalise(i) for i in processor.batch_decode(pred_ids)]
|
| 123 |
return batch
|
| 124 |
|
| 125 |
result = test_dataset.map(evaluate, batched=True, batch_size=8)
|
|
|
|
| 127 |
print("WER: {:2f}".format(100 * wer.compute(predictions=result["pred_strings"], references=result["sentence"])))
|
| 128 |
```
|
| 129 |
|
| 130 |
+
**Test Result (wer)**: 13.766801%
|
| 131 |
|
| 132 |
|
| 133 |
## Training
|