Upload app.py

app.py

@@ -2,7 +2,7 @@ from transformers import VisionEncoderDecoderModel, ViTFeatureExtractor, AutoTok
 import torch
 from PIL import Image
 
-import gradio as gr 
+import gradio as gr
 
 from fairseq.checkpoint_utils import load_model_ensemble_and_task_from_hf_hub
 from fairseq.models.text_to_speech.hub_interface import TTSHubInterface
@@ -12,13 +12,16 @@ feature_extractor = ViTFeatureExtractor.from_pretrained("nlpconnect/vit-gpt2-ima
 tokenizer = AutoTokenizer.from_pretrained("nlpconnect/vit-gpt2-image-captioning")
 
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-model.to(device)
+model = model.to(device)
 
 models, cfg, task = load_model_ensemble_and_task_from_hf_hub(
     "facebook/fastspeech2-en-ljspeech",
-    arg_overrides={"vocoder": "hifigan", "fp16": False}
+    arg_overrides={"vocoder": "hifigan", "fp16": True}
 )
+
 model1 = models[0]
+model1 = model1.to(device)
+
 TTSHubInterface.update_cfg_with_data_cfg(cfg, task.data_cfg)
 generator = task.build_generator(models, cfg)
 
@@ -27,32 +30,27 @@ num_beams = 4
 gen_kwargs = {"max_length": max_length, "num_beams": num_beams}
 
 
-def predict_step(image_paths):
-  images = []
-  text = ""
+def inference(image_paths):
   
-  for image_path in image_paths:
-    i_image = Image.fromarray(image_path)
-    if i_image.mode != "RGB":
+  #for image_path in image_paths:
+  i_image = Image.fromarray(image_paths)
+  if i_image.mode != "RGB":
       i_image = i_image.convert(mode="RGB")
-      print(image_path)
 
-    images.append(i_image)
-    print(images)
-  pixel_values = feature_extractor(images=images, return_tensors="pt").pixel_values
+  pixel_values = feature_extractor(images=i_image, return_tensors="pt").pixel_values
   pixel_values = pixel_values.to(device)
 
   output_ids = model.generate(pixel_values, **gen_kwargs)
 
   preds = tokenizer.batch_decode(output_ids, skip_special_tokens=True)
   preds = [pred.strip() for pred in preds]
-  preds = ' '.join(str(e) for e in preds)
-  text = text + preds
-  sample = TTSHubInterface.get_model_input(task, text)
+
+
+  sample = TTSHubInterface.get_model_input(task, preds)
+
   wav, rate = TTSHubInterface.get_prediction(task, model1, generator, sample)
-  return wav#, rate, text
-  #return ipd.Audio(wav, rate=rate)
+  return wav
 
 
-interface = gr.Interface(predict_step, gr.Image(), "audio")
+interface = gr.Interface(inference, gr.Image(), "audio")
 interface.launch()