File size: 5,738 Bytes
248bc6b 6b86b4a 248bc6b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 | import os
import gradio as gr
import torch
import numpy
import librosa
import languages_dic
from transformers import WhisperForConditionalGeneration, WhisperProcessor, pipeline
title = "Multilanguage Transcription and Translation"
availableLang = "Afrikaans, Arabic, Armenian, Azerbaijani, Belarusian, Bosnian, Bulgarian, Catalan, Chinese, Croatian, Czech, Danish, Dutch, English, Estonian, Finnish, French, Galician, German, Greek, Hebrew, Hindi, Hungarian, Icelandic, Indonesian, Italian, Japanese, Kannada, Kazakh, Korean, Latvian, Lithuanian, Macedonian, Malay, Marathi, Maori, Nepali, Norwegian, Persian, Polish, Portuguese, Romanian, Russian, Serbian, Slovak, Slovenian, Spanish, Swahili, Swedish, Tagalog, Tamil, Thai, Turkish, Ukrainian, Urdu, Vietnamese, and Welsh."
description1 = """<p style='font-size: 18px;'> Transcribe an audio file containing a speech in any of the languages listed below and translate it to English. </p>
<p style='font-size: 16px;'> This demo uses the ASR system Whisper and runs on CPU basis hence responses might be slow. </p> \n
""" + availableLang
description2 ="""<p style='font-size: 18px;'> Transcribe a recording with your microphone of a speech in any of the languages listed below and translate it to English. </p>
<p style='font-size: 16px;'> This demo uses the ASR system Whisper and runs on CPU basis hence responses might be slow. </p> \n
""" + availableLang
device = "cuda:0" if torch.cuda.is_available() else "cpu"
#modelType = "openai/whisper-small"
class LM:
model={}
processor={}
pipe={}
#LMsizes = ["tiny", "base", "small", "medium", "large"]
LMsizes = ["base", "small", "medium"]
myLM = LM()
for LMsize in myLM.LMsizes:
modelType = "openai/whisper-"+LMsize
myLM.model[LMsize] = WhisperForConditionalGeneration.from_pretrained(modelType).to(device)
myLM.processor[LMsize] = WhisperProcessor.from_pretrained(modelType)
myLM.pipe[LMsize] = pipeline(task="automatic-speech-recognition", model=modelType, device=device, chunk_length_s=29, stride_length_s=[5,0])
def detect_language(audio_path, model, processor, asr_pipe_whisper):
#Is not possible to retrieve the predicted language directly or using a pipeline. Instead:
# Loads and resample the audio file to 16kHz, convert to mono and control the duration of the audio input to 20sec
speech_data, sampling_rate = librosa.load(audio_path, sr=16000, mono=True, duration=20)
#get the input features using the feature extractor on the raw speech data
input_features = processor.feature_extractor(speech_data, return_tensors="pt", sampling_rate=sampling_rate).input_features.to(device)
#transcribe the input tensor of features obtained from function preAudioPath
predicted_ids = model.generate(input_features, task="transcribe")
#decode the second entry from the output array which conatins the detected language
detected_lang = asr_pipe_whisper.tokenizer.decode(predicted_ids[0,1])
#looks up in the dictionary to retrieve the expanded language name. E.g. detected_lang = "<|ge|>" returns detected_lang = "german"
detected_lang = languages_dic.LANGUAGES.get(detected_lang.strip("<|>"))
return detected_lang
# def transcribe(inputs):
# # predicted_ids = model.generate(inputs, language="<|es|>", task="transcribe")
# predicted_ids = model.generate(inputs, task="transcribe")
# transcription = processor.tokenizer.batch_decode(predicted_ids, skip_special_tokens=True)[0]
# return transcription
def getLM(modelsize):
modelsize = modelsize.split(" ")
if len(modelsize) > 0:
modelsize = modelsize[0]
return (myLM.model[modelsize], myLM.processor[modelsize], myLM.pipe[modelsize])
def processAudio(audio_path, modelsize):
model, processor, asr_pipe_whisper = getLM(modelsize)
translation = asr_pipe_whisper(audio_path, max_new_tokens=256, generate_kwargs={"task":"translate"})
transcription = asr_pipe_whisper(audio_path, generate_kwargs={"task":"transcribe"})
#transcription = transcribe(preprocessAudioPath(audio_path))
inputLang = detect_language(audio_path, model, processor, asr_pipe_whisper)
return (inputLang, transcription["text"], translation["text"])
modelsizeInfo = "Try out the performance for different model sizes. Larger models are more robust and deliver better results but are also slower."
app1 = gr.Interface(
fn=processAudio,
# inputs=[gr.Audio(source="upload", type="filepath",label="Audio Input"),
# gr.Radio(["tiny - 39M", "base - 74M", "small -244M", "medium - 769M", "large - 1550M"],
# label="Select the model size", info=modelsizeInfo, value="small -244M")],
inputs=[gr.Audio(source="upload", type="filepath",label="Audio Input"),
gr.Radio(["base - 74M", "small -244M", "medium - 769M"], label="Select the model size", info=modelsizeInfo, value="small -244M")],
outputs=[gr.Textbox(label="Detected input language"), gr.Textbox(label="Transcription"), gr.Textbox(label="Translation to english")],
title=title,
description=description1
)
app2 = gr.Interface(
fn=processAudio,
inputs=[gr.Audio(source="microphone", type="filepath",label="Audio Input"),
gr.Radio(["base - 74M", "small -244M", "medium - 769M"], label="Select the model size", info=modelsizeInfo, value="small -244M")],
outputs=[gr.Textbox(label="Detected input language"), gr.Textbox(label="Transcription"), gr.Textbox(label="Translation to english")],
title=title,
description=description2
)
demo = gr.TabbedInterface([app1, app2], ["Audio File", "Microphone"])
if __name__ == "__main__":
demo.launch()
|