Create app.py

app.py

@@ -0,0 +1,406 @@
+import os
+from dotenv import load_dotenv
+import google.generativeai as genai
+import google.ai.generativelanguage as glm
+from PIL import Image
+import gradio as gr
+import numpy as np
+import io
+import speech_recognition as sr
+import edge_tts
+import asyncio
+import pygame
+import threading
+from deep_translator import GoogleTranslator
+import cv2
+from gradio.themes import Base
+
+# Load environment variables from .env file
+load_dotenv()
+API_KEY = os.getenv("API_KEY")
+
+if API_KEY is None:
+    gr.Chatbot("API Key is not set. Please set the API key in the .env file.")
+else:
+    genai.configure(api_key=API_KEY)
+
+# GIF paths
+GIF_LISTENING = "Images/listening.gif"
+GIF_THINKING = "Images/Thinking1.gif"
+GIF_SPEAKING = "Images/speaking.gif"
+GIF_NEUTRAL = "Images/Neutral.gif"
+
+custom_theme = gr.themes.Default(
+    primary_hue="indigo",
+    secondary_hue="blue",
+    neutral_hue="gray",
+    font=("Helvetica", "sans-serif")
+)
+# Available voices and languages for TTS
+VOICES = {
+    "English": {"Female": "en-US-JennyNeural", "Male": "en-US-GuyNeural"},
+    "Hindi": {"Female": "hi-IN-SwaraNeural", "Male": "hi-IN-MadhurNeural"},
+    "Spanish": {"Female": "es-ES-ElviraNeural", "Male": "es-ES-AlvaroNeural"},
+    "French": {"Female": "fr-FR-DeniseNeural", "Male": "fr-FR-HenriNeural"},
+    "German": {"Female": "de-DE-KatjaNeural", "Male": "de-DE-ConradNeural"},
+    "Japanese": {"Female": "ja-JP-NanamiNeural", "Male": "ja-JP-KeitaNeural"},
+    "Telugu": {"Female": "te-IN-ShrutiNeural", "Male": "te-IN-MohanNeural"},
+    "Mandarin Chinese": {"Female": "zh-CN-XiaoxiaoNeural", "Male": "zh-CN-YunyangNeural"},
+    "Arabic": {"Female": "ar-EG-SalmaNeural", "Male": "ar-EG-HamedNeural"},
+    "Russian": {"Female": "ru-RU-DariyaNeural", "Male": "ru-RU-NikitaNeural"},
+    "Portuguese": {"Female": "pt-BR-FranciscaNeural", "Male": "pt-BR-AntonioNeural"},
+    "Italian": {"Female": "it-IT-ElsaNeural", "Male": "it-IT-IsmaeleNeural"},
+    "Korean": {"Female": "ko-KR-SunHyeNeural", "Male": "ko-KR-InJoonNeural"},
+    "Dutch": {"Female": "nl-NL-ColetteNeural", "Male": "nl-NL-FennNeural"},
+    "Swedish": {"Female": "sv-SE-HilleviNeural", "Male": "sv-SE-MattiasNeural"},
+    "Polish": {"Female": "pl-PL-AgnieszkaNeural", "Male": "pl-PL-MarekNeural"},
+    "Turkish": {"Female": "tr-TR-EmelNeural", "Male": "tr-TR-AhmetNeural"},
+    "Indonesian": {"Female": "id-ID-GadisNeural", "Male": "id-ID-ArdiNeural"},
+    "Thai": {"Female": "th-TH-AcharaNeural", "Male": "th-TH-PremwutNeural"},
+    "Vietnamese": {"Female": "vi-VN-HoaiMyNeural", "Male": "vi-VN-NamMinhNeural"},
+     "Greek": {"Female": "el-GR-AthinaNeural", "Male": "el-GR-NestorasNeural"},
+    "Hebrew": {"Female": "he-IL-HilaNeural", "Male": "he-IL-AvriNeural"},
+    "Hungarian": {"Female": "hu-HU-NoemiNeural", "Male": "hu-HU-TamasNeural"},
+    "Romanian": {"Female": "ro-RO-AlinaNeural", "Male": "ro-RO-EmilNeural"},
+    "Czech": {"Female": "cs-CZ-VlastaNeural", "Male": "cs-CZ-AntoninNeural"},
+    "Finnish": {"Female": "fi-FI-SelmaNeural", "Male": "fi-FI-HarriNeural"},
+    "Danish": {"Female": "da-DK-ChristelNeural", "Male": "da-DK-JeppeNeural"},
+    "Norwegian": {"Female": "nb-NO-IselinNeural", "Male": "nb-NO-FinnNeural"},
+    "Slovak": {"Female": "sk-SK-ViktoriaNeural", "Male": "sk-SK-LukasNeural"},
+    "Bulgarian": {"Female": "bg-BG-KalinaNeural", "Male": "bg-BG-BorislavNeural"},
+    # Add more languages and voices as needed
+}
+
+LANGUAGE_CODES = {
+    "English": "en", "Hindi": "hi", "Spanish": "es",
+    "French": "fr", "German": "de", "Japanese": "ja",
+    "Telugu": "te", "Mandarin Chinese": "zh-CN", "Arabic": "ar",
+    "Russian": "ru", "Portuguese": "pt-BR", "Italian": "it",
+    "Korean": "ko", "Dutch": "nl", "Swedish": "sv",
+    "Polish": "pl", "Turkish": "tr", "Indonesian": "id",
+    "Thai": "th", "Vietnamese": "vi",
+    "Greek": "el",
+    "Hebrew": "he",
+    "Hungarian": "hu",
+    "Romanian": "ro",
+    "Czech": "cs",
+    "Finnish": "fi",
+    "Danish": "da",
+    "Norwegian": "nb",
+    "Slovak": "sk",
+    "Bulgarian": "bg",
+    # Add more language codes as needed
+}
+
+conversation_history = []
+
+def text_chat(text, max_output_tokens):
+    global conversation_history
+    model = genai.GenerativeModel(model_name="gemini-1.5-flash")
+    
+    response = model.generate_content(
+        glm.Content(
+            parts=[
+                glm.Part(text=text),
+            ],
+        ),
+        generation_config = {
+            "temperature" : 0.7,
+            "max_output_tokens" : max_output_tokens,
+        } ,
+        stream=True
+    )
+    response.resolve()
+    conversation_history.append(("You", text))
+    conversation_history.append(("Assistant", response.text))
+    return conversation_history
+def image_analysis(image, prompt, input_language, output_language, voice_input=None):
+    input_language = input_language or "English"
+    output_language = output_language or "English"
+
+    if voice_input:
+        # Convert voice input to text using the selected input language
+        recognizer = sr.Recognizer()
+        with sr.AudioFile(voice_input) as source:
+            audio = recognizer.record(source)
+        prompt = recognizer.recognize_google(audio, language=LANGUAGE_CODES[input_language])
+    
+    if input_language != "English":
+        prompt = GoogleTranslator(source=LANGUAGE_CODES[input_language], target='en').translate(prompt)
+
+    if isinstance(image, np.ndarray):
+        pil_image = Image.fromarray(image)
+    elif isinstance(image, Image.Image):
+        pil_image = image
+    else:
+        bytes_data = image
+
+    if 'pil_image' in locals():
+        img_byte_arr = io.BytesIO()
+        pil_image.save(img_byte_arr, format='JPEG')
+        bytes_data = img_byte_arr.getvalue()
+
+    model = genai.GenerativeModel(model_name="gemini-1.5-flash")
+    
+    response = model.generate_content(
+        glm.Content(
+            parts=[glm.Part(text=prompt), glm.Part(inline_data=glm.Blob(mime_type='image/jpeg', data=bytes_data))],
+        ),
+        generation_config = {
+            "temperature" : 0.7,
+            "max_output_tokens" : 100,
+        } ,
+    )
+    
+    response.resolve()
+    
+    if output_language != "English":
+        response_text = GoogleTranslator(source='en', target=LANGUAGE_CODES[output_language]).translate(response.text)
+    else:
+        response_text = response.text
+
+    return response_text  # Return only the text response
+class VoiceInteraction:
+    def __init__(self):
+        self.is_running = False
+        self.recognizer = sr.Recognizer()
+        self.model = genai.GenerativeModel(model_name="gemini-1.5-flash")
+        self.conversation = []
+        self.current_state = None
+        self.current_image = None
+        self.input_language = "English"
+        self.output_language = "English"
+        self.voice = VOICES["English"]["Female"]
+        
+        # Adjust the recognizer's settings for better sensitivity
+        self.recognizer.energy_threshold = 300  # Lower energy threshold for detecting speech
+        self.recognizer.dynamic_energy_threshold = True  # Dynamically adjust for ambient noise
+        self.recognizer.pause_threshold = 0.5 
+
+    async def text_to_speech_and_play(self, text):
+        self.current_state = "Speaking"
+        communicate = edge_tts.Communicate(text, self.voice)
+        audio_path = "output.mp3"
+        await communicate.save(audio_path)
+
+        pygame.mixer.init()
+        pygame.mixer.music.load(audio_path)
+        pygame.mixer.music.play()
+        while pygame.mixer.music.get_busy():
+            pygame.time.Clock().tick(10)
+        pygame.mixer.quit()
+
+    def listen_and_respond(self):
+        with sr.Microphone() as source:
+            self.recognizer.adjust_for_ambient_noise(source, duration=1)
+            
+            while self.is_running:
+                try:
+                    self.current_state = "Listening"
+                    audio = self.recognizer.listen(source, timeout=3, phrase_time_limit=10)
+                    text = self.recognizer.recognize_google(audio, language=LANGUAGE_CODES[self.input_language])
+                   
+                    if self.input_language != "English":
+                        text = GoogleTranslator(source=LANGUAGE_CODES[self.input_language], target='en').translate(text)
+                    
+                    self.conversation.append(("You", text))
+
+                    self.current_state = "Thinking"
+                    if self.current_image:
+                        response = image_analysis(self.current_image, text)
+                    else:
+                        response = self.model.generate_content(
+                            glm.Content(parts=[glm.Part(text=text + "Give the Response in only plain text and without any Markdown Formatting ")]), 
+                            generation_config = {
+                                "temperature" : 0.7,
+                                "max_output_tokens" : 500,
+                            } ,
+                        )
+                        response.resolve()
+                        response = response.text
+
+                    if self.output_language != "English":
+                        response = GoogleTranslator(source='en', target=LANGUAGE_CODES[self.output_language]).translate(response)
+
+                    self.conversation.append(("Assistant", response))
+                    
+                    # Update the conversation output after each cycle
+                    gr.update(value=self.conversation)
+                    
+                    asyncio.run(self.text_to_speech_and_play(response))
+                except sr.WaitTimeoutError:
+                    continue
+                except sr.UnknownValueError:
+                    gr.Markdown("Could not understand audio")
+                except sr.RequestError as e:
+                    print(f"Error: {str(e)}")
+
+    def start(self):
+        self.is_running = True
+        self.thread = threading.Thread(target=self.listen_and_respond)
+        self.thread.start()
+
+    def stop(self):
+        self.is_running = False
+        if hasattr(self, 'thread'):
+            self.thread.join()
+
+    def set_current_image(self, image):
+        self.current_image = image
+
+    def set_languages_and_voice(self, input_language, output_language, voice_gender):
+        self.input_language = input_language
+        self.output_language = output_language
+        self.voice = VOICES[output_language][voice_gender]
+
+voice_interaction = VoiceInteraction()
+
+def start_voice_interaction():
+    voice_interaction.start()
+    return "Voice interaction started. Speak now!", voice_interaction.conversation
+
+def stop_voice_interaction():
+    voice_interaction.stop()
+    return "Voice interaction stopped.", voice_interaction.conversation
+
+def update_conversation():
+    return voice_interaction.conversation
+
+def get_current_gif():
+    state = voice_interaction.current_state
+    if state == "Listening":
+        return GIF_LISTENING
+    elif state == "Thinking":
+        return GIF_THINKING
+    elif state == "Speaking":
+        return GIF_SPEAKING
+    else:
+        return GIF_NEUTRAL
+def analyze_and_speak(image, prompt, input_language, output_language, voice_input=None):
+    response_text = image_analysis(image, prompt, input_language, output_language, voice_input)
+    
+    # Generate audio response
+    voice = VOICES[output_language]["Female"]
+    communicate = edge_tts.Communicate(response_text, voice)
+    audio_path = "image_analysis_response.mp3"
+    asyncio.run(communicate.save(audio_path))
+    
+    return response_text, audio_path
+def set_image_for_voice(image):
+    voice_interaction.set_current_image(image)
+    return "Image set for voice interaction. You can now ask questions about it."
+
+def set_languages_and_voice(input_language, output_language, voice_gender):
+    voice_interaction.set_languages_and_voice(input_language, output_language, voice_gender)
+    return f"Input language set to {input_language}, output language set to {output_language} with {voice_gender} voice"
+
+with gr.Blocks(theme=gr.themes.Soft()) as demo:
+    gr.Markdown("# 🌐 Conerstional Image Recognition Chatbot")
+    gr.Markdown("## Developed by AI INTENTS ")
+    with gr.Tab("🏠 Home"):
+        gr.Markdown("""
+        ## Welcome to the Conversational Image Recognition Chatbot!
+        
+        This application offers three main features:
+        
+        1. **💬 Text Chat**: Engage in a text-based conversation with our AI assistant.
+        2. **🖼️ Image Analysis**: Upload or capture images for AI-powered analysis and description.
+        3. **🎙️ Voice Interaction**: Have a voice conversation with our AI, with support for multiple languages.
+        
+        Choose a tab above to get started!
+        
+        ### How to use:
+        - In Text Chat, simply type your message and click 'Send'.
+        - For Image Analysis, upload an image or use your webcam, then type or speak your question about the image.
+        - In Voice Interaction, click 'Start' and begin speaking. The AI will respond audibly.
+        
+        Enjoy exploring the capabilities of our AI-powered chatbot!
+        """)
+    with gr.Tab("💬 Text Chat"):
+        with gr.Row():
+            with gr.Column(scale=4):
+                text_input = gr.Textbox(label="Your message", placeholder="Type your message here...")
+                max_tokens_slider = gr.Slider(minimum=100, maximum=1000, value=500, step=50, label="Max Output Tokens")
+            with gr.Column(scale=1):
+                text_button = gr.Button("Send", variant="primary")
+        text_output = gr.Chatbot(height=400, elem_id="text-chat-output")
+        text_button.click(text_chat, inputs=[text_input, max_tokens_slider], outputs=text_output)
+    with gr.Tab("🖼️ Image Analysis"):
+        with gr.Row():
+            with gr.Column(scale=1):
+                with gr.Row():
+                    image_input = gr.Image(label="Upload Image", type="pil")
+                    webcam_button = gr.Button("Capture from Webcam")
+            with gr.Column(scale=1):
+                image_prompt = gr.Textbox(label="Prompt", placeholder="Ask about the image...")
+                input_language_dropdown = gr.Dropdown(choices=list(VOICES.keys()), label="Input Language", value="English")
+                output_language_dropdown = gr.Dropdown(choices=list(VOICES.keys()), label="Output Language", value="English")
+                image_prompt_voice = gr.Audio(label="Prompt via Voice", sources=["microphone"], type="filepath")
+                image_prompt_audio = gr.Audio(label="Upload Audio File", sources=["upload"], type="filepath")
+                image_button = gr.Button("Analyze", variant="primary")
+        image_output = gr.Markdown(label="Analysis Result", elem_id="image-analysis-output")
+        audio_output = gr.Audio(label="Audio Response", elem_id="image-analysis-audio-output")
+
+        def capture_image_from_webcam():
+              cap = cv2.VideoCapture(0)
+              ret, frame = cap.read()
+              cap.release()
+              return Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
+
+        webcam_button.click(capture_image_from_webcam, outputs=image_input)
+        image_button.click(
+            analyze_and_speak,
+            inputs=[image_input, image_prompt, input_language_dropdown, output_language_dropdown],
+            outputs=[image_output, audio_output]
+        )
+
+        # Add events for voice and audio file inputs
+        image_prompt_voice.change(
+            image_analysis,
+            inputs=[image_input, image_prompt_voice, input_language_dropdown, output_language_dropdown, image_prompt_voice],
+            outputs=[image_output, audio_output]
+        )
+
+        image_prompt_audio.change(
+            image_analysis,
+            inputs=[image_input, image_prompt_audio, input_language_dropdown, output_language_dropdown, image_prompt_audio],
+            outputs=[image_output, audio_output]
+        )
+        image_button.click(image_analysis, inputs=[image_input, image_prompt], outputs=image_output)
+        image_prompt_voice.change(lambda x: image_analysis(image_input.value, x), inputs=image_prompt_voice, outputs=image_output)
+    with gr.Tab("🎙️ Voice Interaction"):
+        with gr.Row():
+            start_button = gr.Button("Start Voice Interaction", variant="primary")
+            stop_button = gr.Button("Stop Voice Interaction", variant="secondary")
+        with gr.Row():
+            input_language_dropdown = gr.Dropdown(choices=list(VOICES.keys()), label="Input Language", value="English")
+            output_language_dropdown = gr.Dropdown(choices=list(VOICES.keys()), label="Output Language", value="English")
+            voice_gender_dropdown = gr.Dropdown(choices=["Female", "Male"], label="Voice Gender", value="Female")
+        set_language_button = gr.Button("Set Languages and Voice")
+        sensitivity_slider = gr.Slider(minimum=100, maximum=1000, value=300, step=50, label="Microphone Sensitivity")
+        with gr.Row():
+            with gr.Column(scale=1):
+                gif_output = gr.Image(label="Status", visible=True, elem_id="voice-interaction-gif")
+                status_output = gr.Markdown(label="Status", elem_id="voice-interaction-status")
+            with gr.Column(scale=1):
+                conversation_output = gr.Chatbot(label="Conversation", height=400, elem_id="voice-interaction-output")
+                demo.load(update_conversation, inputs=[], outputs=[conversation_output], every=1)
+        
+        start_button.click(start_voice_interaction, inputs=[], outputs=[status_output, conversation_output])
+        stop_button.click(stop_voice_interaction, inputs=[], outputs=[status_output, conversation_output])
+        set_language_button.click(set_languages_and_voice, 
+                                  inputs=[input_language_dropdown, output_language_dropdown, voice_gender_dropdown], 
+                                  outputs=status_output)
+        
+        demo.load(get_current_gif, inputs=[], outputs=gif_output, every=1)
+
+        def update_sensitivity(value):
+            voice_interaction.recognizer.energy_threshold = value
+            return f"Microphone sensitivity set to {value}"
+
+        sensitivity_slider.change(update_sensitivity, inputs=[sensitivity_slider], outputs=[status_output])
+        gr.Markdown("## Developed by AI INTENTS ")
+        
+demo.launch(share=False)
+
+