Upload 4 files

app.py

@@ -0,0 +1,139 @@
+import streamlit as st
+from dotenv import load_dotenv
+import os
+from PIL import Image
+import datetime
+
+
+from brain_of_the_doctor import encode_image, analyze_image_with_query
+from voice_of_the_patient import record_audio, transcribe_with_groq
+from voice_of_the_doctor import text_to_speech_with_elevenlabs
+
+
+load_dotenv()
+
+
+system_prompt = """You have to act as a professional doctor, i know you are not but this is for learning purpose. 
+            What's in this image?. Do you find anything wrong with it medically? 
+            If you make a differential, suggest some remedies for them. Donot add any numbers or special characters in 
+            your response. Your response should be in one long paragraph. Also always answer as if you are answering to a real person.
+            Donot say 'In the image I see' but say 'With what I see, I think you have ....'
+            Dont respond as an AI model in markdown, your answer should mimic that of an actual doctor not an AI bot, 
+            Keep your answer concise (max 2 sentences). No preamble, start your answer right away please. Dont tell u shoud go to doctor. if user ask for medicine then suggest medicine also
+            Dont say this : I cannot help you with that. I strongly advise against taking any medication without a prescription from a healthcare professional.
+            Suggest meditions to user.
+            """
+
+
+def display_chat_message(speaker, text, timestamp):
+    if speaker == "Patient":
+        # User message (aligned to the right)
+        st.markdown(
+            f"""
+            <div style="display: flex; justify-content: flex-end; margin-bottom: 10px;">
+                <div style="background: #DCF8C6; padding: 10px; border-radius: 10px; max-width: 70%;">
+                    <p style="margin: 0;">{text}</p>
+                    <small style="color: gray; text-align: right;">{timestamp}</small>
+                </div>
+            </div>
+            """,
+            unsafe_allow_html=True,
+        )
+    else:
+        # Doctor message (aligned to the left)
+        st.markdown(
+            f"""
+            <div style="display: flex; justify-content: flex-start; margin-bottom: 10px;">
+                <div style="background: #ECECEC; padding: 10px; border-radius: 10px; max-width: 70%;">
+                    <p style="margin: 0;">{text}</p>
+                    <small style="color: gray;">{timestamp}</small>
+                </div>
+            </div>
+            """,
+            unsafe_allow_html=True,
+        )
+
+def main():
+    st.title("AI Doctor By NueSpaarx")
+
+    
+    if 'conversation' not in st.session_state:
+        st.session_state.conversation = []
+
+    # Display the chat interface
+    st.header("Chat with AI Doctor")
+
+    # Chat container
+    chat_container = st.container()
+
+    # Display the entire conversation history in chat format
+    with chat_container:
+        for entry in st.session_state.conversation:
+            display_chat_message(entry["speaker"], entry["text"], entry["timestamp"])
+
+    # Image input
+    st.header("Upload an Image")
+    image_file = st.file_uploader("Choose an image...", type=["jpg", "jpeg", "png"])
+
+    if image_file is not None:
+        st.image(Image.open(image_file), caption="Uploaded Image", use_column_width=True)
+
+    # Audio input
+    st.header("Record Your Voice")
+    audio_filepath = "patient_voice_test_for_patient.mp3"
+    if st.button("Record Audio"):
+        record_audio(file_path=audio_filepath)
+        st.audio(audio_filepath, format="audio/mp3")
+
+    if st.button("Analyze"):
+        if image_file is not None:
+            # Save the uploaded image to a temporary file
+            image_filepath = "temp_image.jpg"
+            with open(image_filepath, "wb") as f:
+                f.write(image_file.getbuffer())
+
+            # Process the inputs
+            speech_to_text_output = transcribe_with_groq(
+                GROQ_API_KEY=os.environ.get("GROQ_API_KEY"),
+                audio_filepath=audio_filepath,
+                stt_model="whisper-large-v3"
+            )
+
+            # Add user query to conversation history with timestamp
+            st.session_state.conversation.append({
+                "speaker": "Patient",
+                "text": speech_to_text_output,
+                "timestamp": datetime.datetime.now().strftime("%H:%M")
+            })
+
+            # Analyze the image and user query
+            doctor_response = analyze_image_with_query(
+                query=system_prompt + speech_to_text_output,
+                encoded_image=encode_image(image_filepath),
+                model="llama-3.2-11b-vision-preview"
+            )
+
+            # Add doctor's response to conversation history with timestamp
+            st.session_state.conversation.append({
+                "speaker": "Doctor",
+                "text": doctor_response,
+                "timestamp": datetime.datetime.now().strftime("%H:%M")
+            })
+
+            # Convert doctor's response to speech
+            voice_of_doctor = text_to_speech_with_elevenlabs(
+                input_text=doctor_response,
+                output_filepath="final.mp3"
+            )
+
+            # Display the doctor's voice response
+            st.header("Doctor's Voice")
+            st.audio("final.mp3", format="audio/mp3")
+
+            # Rerun the app to update the chat interface
+            st.rerun()
+        else:
+            st.error("Please upload an image to analyze.")
+
+if __name__ == "__main__":
+    main()

brain_of_the_doctor.py

@@ -0,0 +1,58 @@
+from dotenv import load_dotenv
+load_dotenv()
+
+#Step1: Setup GROQ API key
+import os
+
+GROQ_API_KEY=os.environ.get("GROQ_API_KEY")
+
+
+import base64
+
+
+#image_path="acne.jpg"
+
+def encode_image(image_path):   
+    image_file=open(image_path, "rb")
+    return base64.b64encode(image_file.read()).decode('utf-8')
+
+
+#Step3: Setup Multimodal LLM 
+
+from groq import Groq
+
+query="Is there something wrong with my face?"
+model="llama-3.2-90b-vision-preview"
+
+
+def analyze_image_with_query(query, model, encoded_image): 
+    client=Groq()  
+    messages=[
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text", 
+                    "text": query
+                },
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": f"data:image/jpeg;base64,{encoded_image}",
+                    },
+                },
+            ],
+        }]
+    chat_completion=client.chat.completions.create(
+        messages=messages,
+        model=model
+    )
+    # print(chat_completion)
+    # print(chat_completion.choices[0])
+    # print(chat_completion.choices[0].message)
+
+    # print(chat_completion.choices[0].message.content)
+
+    return chat_completion.choices[0].message.content
+
+

voice_of_the_doctor.py

@@ -0,0 +1,104 @@
+
+from dotenv import load_dotenv
+load_dotenv()
+
+#Step1a: Setup Text to Speech–TTS–model with gTTS
+import os
+from gtts import gTTS
+
+def text_to_speech_with_gtts_old(input_text, output_filepath):
+    language="en"
+
+    audioobj= gTTS(
+        text=input_text,
+        lang=language,
+        slow=False
+        # slow=True
+    )
+    audioobj.save(output_filepath)
+
+
+input_text="Hi this is Balkrishna Joshi!"
+text_to_speech_with_gtts_old(input_text=input_text, output_filepath="gtts_testing.mp3")
+
+#Step1b: Setup Text to Speech–TTS–model with ElevenLabs
+import elevenlabs
+from elevenlabs.client import ElevenLabs
+
+ELEVENLABS_API_KEY=os.environ.get("ELEVENLABS_API_KEY")
+
+def text_to_speech_with_elevenlabs_old(input_text, output_filepath):
+    client=ElevenLabs(api_key=ELEVENLABS_API_KEY)
+    audio=client.generate(
+        text= input_text,
+        voice= "Aria",
+        output_format= "mp3_22050_32",
+        model= "eleven_turbo_v2"
+    )
+    elevenlabs.save(audio, output_filepath)
+
+#text_to_speech_with_elevenlabs_old(input_text, output_filepath="elevenlabs_testing.mp3") 
+
+#Step2: Use Model for Text output to Voice
+
+import subprocess
+import platform
+
+def text_to_speech_with_gtts(input_text, output_filepath):
+    language="en"
+
+    audioobj= gTTS(
+        text=input_text,
+        lang=language,
+        slow=False
+    )
+    audioobj.save(output_filepath)
+    os_name = platform.system()
+    try:
+        if os_name == "Darwin":  # macOS
+            subprocess.run(['afplay', output_filepath])
+        elif os_name == "Windows":  # Windows
+            # subprocess.run(['powershell', '-c', f'(New-Object Media.SoundPlayer "{output_filepath}").PlaySync();'])
+            subprocess.run(['start', output_filepath], shell=True)    #working but it open new  mp3 player
+            # subprocess.run(['ffplay', '-nodisp', '-autoexit', output_filepath])
+
+
+        elif os_name == "Linux":  # Linux
+            subprocess.run(['aplay', output_filepath])  # Alternative: use 'mpg123' or 'ffplay'
+        else:
+            raise OSError("Unsupported operating system")
+    except Exception as e:
+        print(f"An error occurred while trying to play the audio: {e}")
+
+
+input_text="Hi this is Balkrishna Joshi from NeuspaarX, autoplay testing!"
+# text_to_speech_with_gtts(input_text=input_text, output_filepath="gtts_testing_autoplay.mp3")
+
+
+def text_to_speech_with_elevenlabs(input_text, output_filepath):
+    client=ElevenLabs(api_key=ELEVENLABS_API_KEY)
+    audio=client.generate(
+        text= input_text,
+        voice= "Aria",
+        output_format= "mp3_22050_32",
+        model= "eleven_turbo_v2"
+    )
+    elevenlabs.save(audio, output_filepath)
+    os_name = platform.system()
+    try:
+        if os_name == "Darwin":  # macOS
+            subprocess.run(['afplay', output_filepath])
+        elif os_name == "Windows":  # Windows
+            # subprocess.run(['powershell', '-c', f'(New-Object Media.SoundPlayer "{output_filepath}").PlaySync();'])
+            subprocess.run(['start', output_filepath], shell=True) #---working but it open new  mp3 player
+            # subprocess.run(['ffplay', '-nodisp', '-autoexit', output_filepath])
+        elif os_name == "Linux":  # Linux
+            subprocess.run(['aplay', output_filepath])  # Alternative: use 'mpg123' or 'ffplay'
+        else:
+            raise OSError("Unsupported operating system")
+    except Exception as e:
+        print(f"An error occurred while trying to play the audio: {e}")
+
+# text_to_speech_with_elevenlabs(input_text, output_filepath="elevenlabs_testing_autoplay.mp3")
+
+

voice_of_the_patient.py

@@ -0,0 +1,78 @@
+import os
+import subprocess
+from dotenv import load_dotenv
+load_dotenv()
+
+
+# Setup Audio recorder (ffmpeg & portaudio)
+# ffmpeg, portaudio, pyaudio
+
+import logging
+import speech_recognition as sr
+from pydub import AudioSegment
+from io import BytesIO
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+def record_audio(file_path, timeout=20, phrase_time_limit=None):
+    """
+    Simplified function to record audio from the microphone and save it as an MP3 file.
+
+    Args:
+    file_path (str): Path to save the recorded audio file.
+    timeout (int): Maximum time to wait for a phrase to start (in seconds).
+    phrase_time_lfimit (int): Maximum time for the phrase to be recorded (in seconds).
+    """
+    recognizer = sr.Recognizer()
+    
+    try:
+        with sr.Microphone() as source:
+            logging.info("Adjusting for ambient noise...")
+            recognizer.adjust_for_ambient_noise(source, duration=1)
+            logging.info("Start speaking now...")
+            
+            # Record the audio
+            audio_data = recognizer.listen(source, timeout=timeout, phrase_time_limit=phrase_time_limit)
+            logging.info("Recording complete.")
+            
+            # Convert the recorded audio to an MP3 file
+            wav_data = audio_data.get_wav_data()
+            audio_segment = AudioSegment.from_wav(BytesIO(wav_data))
+            audio_segment.export(file_path, format="mp3", bitrate="128k")
+            
+            logging.info(f"Audio saved to {file_path}")
+
+    except Exception as e:
+        logging.error(f"An error occurred: {e}")
+
+audio_filepath="patient_voice_test_for_patient.mp3"
+record_audio(file_path=audio_filepath)
+
+
+
+# Setup Speech to text–STT–model for transcription
+import os
+from groq import Groq
+
+GROQ_API_KEY=os.environ.get("GROQ_API_KEY")
+stt_model="whisper-large-v3"
+
+def transcribe_with_groq(stt_model, audio_filepath, GROQ_API_KEY):
+    client=Groq(api_key=GROQ_API_KEY)
+    
+    audio_file=open(audio_filepath, "rb")
+    transcription=client.audio.transcriptions.create(
+        model=stt_model,
+        file=audio_file,
+        language="en"
+    )
+
+    return transcription.text
+
+
+
+
+
+
+
+