File size: 3,179 Bytes
3c53d13 2ede072 3c53d13 2ede072 24d11ee 2ede072 3c53d13 2ede072 3c53d13 2ede072 9671a63 2ede072 9671a63 2ede072 9671a63 2ede072 3c53d13 2ede072 3c53d13 2ede072 3c53d13 9671a63 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 |
import gradio as gr
from PIL import Image
from project_module import process_inputs, VisualQAState
# Create a session object to manage conversation state per image
session = VisualQAState()
# Global variables to keep track of the current image and the conversation history
current_image = None
chat_history = []
# Unified handler for new questions or new images
def handle_inputs(new_image, audio, followup_text, tts_enabled):
global current_image, chat_history
# If a new image is uploaded, reset current session state
if new_image is not None:
current_image = new_image
chat_history.clear() # Clear previous Q&A history
question = followup_text.strip() if followup_text else "Describe the image"
# Process the new image and question/audio
response, audio_output = process_inputs(
session, image=current_image, question=question, audio_path=audio, enable_tts=tts_enabled
)
elif current_image is not None and (followup_text or audio):
# Follow-up question for current image
question = followup_text.strip() if followup_text else ""
response, audio_output = process_inputs(
session, image=None, question=question, audio_path=audio, enable_tts=tts_enabled
)
else:
# No input given
return "Please upload an image and ask a question.", None, None, ""
# Append the Q&A to chat history
chat_history.append(f"🗨️ **Q:** {question}\n🧠 **A:** {response}")
history_text = "\n\n".join(chat_history) # Format chat as markdown
return response, current_image, audio_output if tts_enabled else None, history_text
# Build the Gradio UI
with gr.Blocks() as demo:
gr.Markdown("## 👁️🎙️ Multimodal Visual Q&A with Audio Output + History")
with gr.Row():
with gr.Column():
# User inputs
image_input = gr.Image(
label="Upload or Capture New Image", sources=["upload", "webcam"], type="pil"
)
audio_input = gr.Audio(
label="Record Voice (Optional)", sources=["microphone"], type="filepath"
)
followup_text = gr.Textbox(
label="Type a Question or Follow-up", placeholder="e.g., What’s going on?"
)
tts_toggle = gr.Checkbox(
label="Enable Audio Response", value=True
)
submit_btn = gr.Button("Ask")
with gr.Column():
# Outputs: Answer, image shown back, audio (if enabled), and chat history
status_output = gr.Textbox(label="Answer", interactive=False)
image_display = gr.Image(label="Current Image")
audio_output = gr.Audio(label="Answer Audio", interactive=False)
chat_box = gr.Markdown(label="Chat History")
# Link the submit button to the handler function
submit_btn.click(
fn=handle_inputs,
inputs=[image_input, audio_input, followup_text, tts_toggle],
outputs=[status_output, image_display, audio_output, chat_box]
)
# Launch the app
if __name__ == "__main__":
demo.launch(show_error=True, share=True)
|