Update app.py

app.py

@@ -190,10 +190,19 @@ def getAudioArray(audio_path):
     speech, rate = librosa.load(audio_path, sr=16000)
     return speech
 
+# Start text before putting image embedding
+start_text = "<|system|> \n You are an assistant good at understanding the context.<|end|> \n <|user|> \n"
+# Prepare text input for causal language modeling
+end_text = "\n Describe the objects and their relationship in the given context.<|end|> \n <|assistant|> \n"
+
+words = nltk.word_tokenize(start_text) + nltk.word_tokenize(end_text)
+input_words = list(set(words))
+# print("Input words:",input_words)
+
+
 def getInputs(image_path, question, answer=""):
 
     image_features = None
-    speech_text = ""
     num_image_tokens = 0
 
     if image_path is not None:
@@ -210,10 +219,10 @@ def getInputs(image_path, question, answer=""):
         num_image_tokens = image_features.shape[1]
 
     # Start text before putting image embedding
-    start_text = f"<|system|>\nYou are an assistant good at understanding the objects and their relationship from the context.<|end|>\n<|user|>\n"
+    start_text = f"<|system|>\nYou are an assistant good at understanding the context.<|end|>\n<|user|>\n "
 
     # Prepare text input for causal language modeling
-    end_text = f"\nPlease describe the objects and their relationship from the context.<|end|>\n<|assistant|>\n{answer}"
+    end_text = f" .\n  Describe the objects and their relationship from the context. <|end|>\n<|assistant|>\n {answer}"
 
     # Tokenize the full texts
     start_tokens = tokenizer(start_text, padding=True, truncation=True, max_length=512, return_tensors="pt")
@@ -240,12 +249,12 @@ model_location = "./MM_FT_C1"
 
 model = MultimodalPhiModel.from_pretrained(model_location).to(device)
 
-model_name = "microsoft/Phi-3.5-mini-instruct"
-base_phi_model = AutoModelForCausalLM.from_pretrained(
-    model_name,
-    torch_dtype=torch.bfloat16,
-    trust_remote_code=True,
-).to(device)
+# model_name = "microsoft/Phi-3.5-mini-instruct"
+# base_phi_model = AutoModelForCausalLM.from_pretrained(
+#     model_name,
+#     torch_dtype=torch.bfloat16,
+#     trust_remote_code=True,
+# ).to(device)
 
 def getStringAfter(output, start_str):
     if start_str in output:
@@ -256,17 +265,12 @@ def getStringAfter(output, start_str):
     answer = preprocess_text(answer)
     return answer
 
+def getAnswerPart(output):
+    output_words = nltk.word_tokenize(output)
+    filtered_words = [word for word in output_words if word.lower() not in [w.lower() for w in input_words]]
+    return ' '.join(filtered_words)
 
-def getStringAfterAnswer(output):
-    if "<|assistant|>" in output:
-        answer = output.split("<|assistant|>")[1]
-    else:
-        answer = output
-
-    answer = preprocess_text(answer)
-    return answer
-
-def generateOutput(image_path, audio_path, context_text, question, max_length=2):
+def generateOutput(image_path, audio_path, context_text, question, max_length=3):
     answerPart = ""
     speech_text = ""
     if image_path is not None:
@@ -279,7 +283,8 @@ def generateOutput(image_path, audio_path, context_text, question, max_length=2)
                 tokens[0],
                 skip_special_tokens=True
                 )
-            answerPart = getStringAfter(output, "<|assistant|>")
+            # answerPart = getStringAfter(output, "<|assistant|>")
+            answerPart = getAnswerPart(output)
         print("Answerpart:", answerPart)
 
     if audio_path is not None:
@@ -287,20 +292,21 @@ def generateOutput(image_path, audio_path, context_text, question, max_length=2)
         print("Speech Text:", speech_text)
 
     if (question is None) or (question == ""):
-        question = "Provide only in 1 sentence to describe the objects and their relationships in it."
+        question = " Describe the objects and their relationships in 1 sentence."
 
     input_text = (
-        "<|system|>\nPlease understand the context "
-        "and answer the question based on the context in 1 or 2 summarized sentences.\n"
-        f"<|end|>\n<|user|>\n<|context|>{answerPart}\n{speech_text}\n{context_text}"
-        f"\n<|question|>: {question}\n<|end|>\n<|assistant|>\n"
+        "<|system|>\n Please understand the context "
+        "and answer the question in 1 or 2 summarized sentences.\n"
+        f"<|end|>\n<|user|>\n<|context|> {answerPart} \n {speech_text} \n {context_text} "
+        f"\n<|question|>: {question} \n<|end|>\n<|assistant|>\n"
     )
     print("input_text:", input_text)
     start_tokens = tokenizer(input_text, padding=True, truncation=True, max_length=1024, return_tensors="pt")['input_ids'].to(device)
+    attention_mask = tokens['attention_mask'].to(device)
     # base_phi_model.generate(start_tokens, max_length=2, do_sample=False, pad_token_id=tokenizer.pad_token_id)
 
     output_text = tokenizer.decode(
-        base_phi_model.generate(start_tokens, max_length=1024, do_sample=False, pad_token_id=tokenizer.pad_token_id)[0],
+        model.base_phi_model.generate(start_tokens, attention_mask=attention_mask, max_length=1024, do_sample=False, pad_token_id=tokenizer.pad_token_id)[0],
         skip_special_tokens=True
     )
 
@@ -309,12 +315,6 @@ def generateOutput(image_path, audio_path, context_text, question, max_length=2)
 
 title = "Created Fine Tuned MultiModal model"
 description = "Test the fine tuned multimodal model created using clip, phi3.5 mini instruct, whisper models"
-examples = [
-    ["./images/COCO_train2014_000000581181.jpg", None, None, None, None, "Describe what is happening in this image."],
-    [None, "Audio File", "./audio/03-01-01-01-01-01-01.wav", None, None, "Describe what is the person trying to tell in this audio."],
-]
-
-#    [None, "Microphone", None, "example_audio_mic.wav", "Context without image.", "What is the result?"],
 
 demo = gr.Blocks()
 
@@ -332,18 +332,20 @@ def process_inputs(image, audio_source, audio_file, audio_mic, context_text, que
     return answer
 
 with demo:
+    gr.Markdown(f"# {title}")
+    gr.Markdown(f" {description}")
+
     with gr.Row():
         with gr.Column(scale=1, min_width=300):
             image_input = gr.Image(type="filepath", label="Upload Image")
         with gr.Column(scale=2, min_width=300):
             question = gr.Textbox(label="Question")
+            with gr.Row():
+                audio_source = gr.Radio(choices=["Microphone", "Audio File"], label="Select Audio Source")
+                audio_file = gr.Audio(sources="upload", type="filepath", visible=False)
+                audio_mic = gr.Audio(sources="microphone", type="filepath", visible=False)
+            context_text = gr.Textbox(label="Context Text")
             output_text = gr.Textbox(label="Output")
-    with gr.Row():
-        audio_source = gr.Radio(choices=["Microphone", "Audio File"], label="Select Audio Source")
-        audio_file = gr.Audio(sources="upload", type="filepath", visible=False)
-        audio_mic = gr.Audio(sources="microphone", type="filepath", visible=False)
-    with gr.Row():
-        context_text = gr.Textbox(label="Context Text")
 
     def update_audio_input(source):
         if source == "Microphone":