added link

app.py

@@ -10,7 +10,7 @@ from transformers import VisionEncoderDecoderModel
 from transformers import AutoTokenizer
 import torch
 
-
+# https://github.com/NielsRogge/Transformers-Tutorials/blob/master/HuggingFace_vision_ecosystem_overview_(June_2022).ipynb
 # option 1: load with randomly initialized weights (train from scratch)
 
 config = ViTConfig(num_hidden_layers=12, hidden_size=768)
@@ -28,22 +28,6 @@ model = PerceiverForImageClassificationConvProcessing.from_pretrained("deepmind/
 
 image_pipe = ImageClassificationPipeline(model=model, feature_extractor=feature_extractor)
 
-'''
-
-
-# initialize a vit-bert from a pretrained ViT and a pretrained BERT model. Note that the cross-attention layers will be randomly initialized
-model = VisionEncoderDecoderModel.from_encoder_decoder_pretrained(
-    "google/vit-base-patch16-224-in21k", "bert-base-uncased"
-)
-# saving model after fine-tuning
-model.save_pretrained("./vit-bert")
-# load fine-tuned model
-model = VisionEncoderDecoderModel.from_pretrained("./vit-bert")
-
-
-'''
-
-
 
 def self_caption(image):
   repo_name = "ydshieh/vit-gpt2-coco-en"
@@ -62,13 +46,8 @@ def self_caption(image):
   # autoregressively generate text (using beam search or other decoding strategy)
   generated_ids = model2.generate(pixel_values, max_length=16, num_beams=4, return_dict_in_generate=True)
   
-  
   # decode into text
   preds = tokenizer.batch_decode(generated_ids[0], skip_special_tokens=True)
-  #generated_sentences = tokenizer.batch_decode(encoder_outputs, skip_special_tokens=True)
-
-  #return(generated_sentences[0].split('.')[0])
-
   preds = [pred.strip() for pred in preds]
   print("Predictions")
   print(preds)
@@ -79,8 +58,9 @@ def self_caption(image):
   pred_dictionary = dict(zip(pred_keys, pred_value))
   print("Pred dictionary")
   print(pred_dictionary)
-  return(pred_dictionary)
-  #return(preds[0].split('.')[0])
+  #return(pred_dictionary)
+  return preds
+
 
 def classify_image(image):
   results = image_pipe(image)
@@ -99,21 +79,18 @@ def classify_image(image):
 
 
 image = gr.inputs.Image(type="pil")
-image_piped = ""
 label = gr.outputs.Label(num_top_classes=5)
 examples = [["cats.jpg"]]
 title = "Generate a Story from an Image"
-description = "Demo for classifying images with Perceiver IO. To use it, simply upload an image and click 'submit' to let the model predict the 5 most probable ImageNet classes. Results will show up in a few seconds."  + image_piped
+description = "Demo for classifying images with Perceiver IO. To use it, simply upload an image and click 'submit', a caption is autogenerated as well"
 article = "<p style='text-align: center'></p>"
 
-#gr.Interface(fn=classify_image, inputs=image, outputs=label, title=title, description=description, examples="", enable_queue=True).launch(debug=True)
-print("img_info1")
 img_info1 = gr.Interface(
     fn=classify_image,
     inputs=image,
     outputs=label,
 )
-print("img_info2")
+
 img_info2 = gr.Interface(
     fn=self_caption,
     inputs=image,
@@ -122,7 +99,7 @@ img_info2 = gr.Interface(
     gr.outputs.Textbox(label = 'Caption')
 ],
 )
-print("running parallel call")
+
 Parallel(img_info1,img_info2, inputs=image, title=title, description=description, examples=examples, enable_queue=True).launch(debug=True)
 #Parallel(img_info1,img_info2, inputs=image, outputs=label, title=title, description=description, examples=examples, enable_queue=True).launch(debug=True)