Garvitj commited on
Commit
8e09e41
·
verified ·
1 Parent(s): 55e5347

Update src/app.py

Browse files
Files changed (1) hide show
  1. src/app.py +219 -132
src/app.py CHANGED
@@ -1,28 +1,207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import streamlit as st
2
- import cv2
3
  import numpy as np
4
- import os
5
- from dotenv import load_dotenv
6
- from st_audiorec import st_audiorec
7
- from analysis import (
8
- get_facial_emotion,
9
- get_voice_emotion,
10
- get_transcript,
11
- get_llm_response
12
- )
13
-
14
- # --- LOAD .ENV FILE ---
15
- load_dotenv()
16
 
17
- # Page configuration
18
  st.set_page_config(
19
- page_title="Empathetic AI Assistant",
20
- page_icon="🤖",
21
  layout="wide"
22
  )
23
 
24
  # --- SESSION STATE INITIALIZATION ---
25
- # Initialize all our state variables securely
26
  if "camera_bytes" not in st.session_state:
27
  st.session_state.camera_bytes = None
28
  if "audio_bytes" not in st.session_state:
@@ -30,48 +209,37 @@ if "audio_bytes" not in st.session_state:
30
  if "user_query" not in st.session_state:
31
  st.session_state.user_query = ""
32
 
33
- # --- CALLBACK FUNCTIONS ---
34
- # These functions run *immediately* when a widget changes.
35
- # This is the key to saving the data before a re-run erases it.
36
-
37
  def camera_on_change():
38
- """Saves the raw bytes of the photo or clears it."""
39
- # 'camera_widget_buffer' is the key we give to the camera
40
  if st.session_state.camera_widget_buffer is not None:
41
  st.session_state.camera_bytes = st.session_state.camera_widget_buffer.getvalue()
42
  else:
43
- # User must have clicked "Clear photo"
44
  st.session_state.camera_bytes = None
45
 
46
  def query_on_change():
47
  """Saves the text query."""
48
- # 'query_widget_buffer' is the key we give to the text_area
49
  st.session_state.user_query = st.session_state.query_widget_buffer
50
 
51
- # --- UI LAYOUT ---
52
-
53
  st.title("🤖 Empathetic AI Assistant")
54
- st.markdown("""
55
- This AI assistant analyzes your emotional state through:
56
- - 📸 **Facial Expression** (from camera)
57
- - 🎤 **Vocal Tone** (from microphone)
58
- - 💬 **Spoken Words** (transcribed from audio)
59
-
60
- Then provides an empathetic, context-aware response to your query.
61
- """)
62
  st.divider()
63
 
64
  col1, col2 = st.columns([1, 1])
65
 
66
  with col1:
67
  st.subheader("📸 1. Capture Your Expression")
68
- # Use the 'key' and 'on_change' parameters.
69
  st.camera_input(
70
  "Take a snapshot",
71
  key="camera_widget_buffer",
72
  on_change=camera_on_change
73
  )
74
 
 
 
 
 
75
  with col2:
76
  st.subheader("💭 2. Your Query")
77
  st.text_area(
@@ -81,109 +249,28 @@ with col2:
81
  key="query_widget_buffer",
82
  on_change=query_on_change
83
  )
 
 
 
 
84
 
85
  st.divider()
86
 
87
- st.subheader("🎙️ 3. Record Your Voice")
88
- st.write("Click the microphone to record your voice, then click 'Analyze' below.")
89
- audio_data = st_audiorec()
90
-
91
- # We manually save the audio data if it's new
92
- if audio_data is not None:
93
- st.session_state.audio_bytes = audio_data
94
-
95
- st.divider()
96
-
97
- # --- ANALYSIS BUTTON ---
98
- if st.button("🧠 Analyze My Emotion & Answer", type="primary", use_container_width=True):
99
-
100
- # Now, we check our *secure* session state variables
101
- if not st.session_state.camera_bytes:
102
- st.error("❌ Please take a snapshot using the camera first!")
103
- elif not st.session_state.audio_bytes:
104
- st.error("❌ Please record your voice first!")
105
- elif not st.session_state.user_query.strip():
106
- st.error("❌ Please enter your query!")
107
- else:
108
- # --- PROCESSING ---
109
- # All data is guaranteed to be safe in session_state
110
-
111
- # Step 1: Process camera image
112
- with st.spinner("📸 Processing facial expression..."):
113
- try:
114
- file_bytes = np.asarray(bytearray(st.session_state.camera_bytes), dtype=np.uint8)
115
- image = cv2.imdecode(file_bytes, cv2.IMREAD_COLOR)
116
- cv2.imwrite("temp_image.jpg", image)
117
- facial_emotion = get_facial_emotion("temp_image.jpg")
118
-
119
- except Exception as e:
120
- st.error(f"Error processing image: {e}")
121
- facial_emotion = "neutral"
122
-
123
- # Step 2: Process recorded audio
124
- with st.spinner("🎵 Saving and analyzing audio..."):
125
- try:
126
- with open("temp_audio.wav", "wb") as f:
127
- f.write(st.session_state.audio_bytes)
128
-
129
- voice_emotion = get_voice_emotion("temp_audio.wav")
130
- transcript = get_transcript("temp_audio.wav")
131
 
132
- except Exception as e:
133
- st.error(f"Error processing audio: {e}")
134
- voice_emotion = "neutral"
135
- transcript = ""
136
-
137
- # Display analysis results
138
- st.divider()
139
- st.subheader("📊 Emotional Analysis Results")
140
- col_a, col_b, col_c = st.columns(3)
141
- with col_a: st.metric(label="😊 Facial Emotion", value=facial_emotion.capitalize())
142
- with col_b: st.metric(label="🎤 Vocal Tone", value=voice_emotion.capitalize())
143
- with col_c: st.metric(label="💬 Speech Detected", value="Yes" if transcript else "No")
144
- if transcript: st.info(f"**Transcription:** {transcript}")
145
-
146
- # Step 5: Get empathetic AI response
147
- st.divider()
148
- with st.spinner("🤖 Empathetic AI is thinking..."):
149
- ai_response = get_llm_response(
150
- user_query=st.session_state.user_query, # Use query from state
151
- face=facial_emotion,
152
- voice=voice_emotion,
153
- text=transcript
154
- )
155
-
156
- st.subheader("💙 Empathetic Response")
157
- st.markdown(ai_response)
158
- st.balloons()
159
 
160
- # --- SIDEBAR (Unchanged) ---
161
  with st.sidebar:
162
- st.header("ℹ️ How to Use")
163
  st.markdown("""
164
  1. **Take a snapshot**
165
  2. **Type your query**
166
- 3. **Click the mic** to record (click again to stop)
167
- 4. **Click the 'Analyze' button**
168
- 5. **Receive** your response
169
- """)
170
- st.divider()
171
- st.header("🔑 Setup Requirements")
172
- st.markdown("""
173
- Make sure these environment variables are set in your
174
- Hugging Face Space **Settings > Repository secrets**:
175
- ```
176
- ROBOFLOW_API_KEY="your_key"
177
- GROQ_API_KEY="your_key"
178
- ```
179
- """)
180
- st.divider()
181
- st.header("🛠️ Tech Stack")
182
- st.markdown("""
183
- - **Frontend:** Streamlit
184
- - **Audio:** `streamlit-audiorec`
185
- - **Facial Analysis:** Roboflow
186
- - **Voice Analysis:** Hugging Face
187
- - **Speech-to-Text:** Google SR
188
- - **LLM:** Groq (Llama 3)
189
  """)
 
1
+ # import streamlit as st
2
+ # import cv2
3
+ # import numpy as np
4
+ # import os
5
+ # from dotenv import load_dotenv
6
+ # from st_audiorec import st_audiorec
7
+ # from analysis import (
8
+ # get_facial_emotion,
9
+ # get_voice_emotion,
10
+ # get_transcript,
11
+ # get_llm_response
12
+ # )
13
+
14
+ # # --- LOAD .ENV FILE ---
15
+ # load_dotenv()
16
+
17
+ # # Page configuration
18
+ # st.set_page_config(
19
+ # page_title="Empathetic AI Assistant",
20
+ # page_icon="🤖",
21
+ # layout="wide"
22
+ # )
23
+
24
+ # # --- SESSION STATE INITIALIZATION ---
25
+ # # Initialize all our state variables securely
26
+ # if "camera_bytes" not in st.session_state:
27
+ # st.session_state.camera_bytes = None
28
+ # if "audio_bytes" not in st.session_state:
29
+ # st.session_state.audio_bytes = None
30
+ # if "user_query" not in st.session_state:
31
+ # st.session_state.user_query = ""
32
+
33
+ # # --- CALLBACK FUNCTIONS ---
34
+ # # These functions run *immediately* when a widget changes.
35
+ # # This is the key to saving the data before a re-run erases it.
36
+
37
+ # def camera_on_change():
38
+ # """Saves the raw bytes of the photo or clears it."""
39
+ # # 'camera_widget_buffer' is the key we give to the camera
40
+ # if st.session_state.camera_widget_buffer is not None:
41
+ # st.session_state.camera_bytes = st.session_state.camera_widget_buffer.getvalue()
42
+ # else:
43
+ # # User must have clicked "Clear photo"
44
+ # st.session_state.camera_bytes = None
45
+
46
+ # def query_on_change():
47
+ # """Saves the text query."""
48
+ # # 'query_widget_buffer' is the key we give to the text_area
49
+ # st.session_state.user_query = st.session_state.query_widget_buffer
50
+
51
+ # # --- UI LAYOUT ---
52
+
53
+ # st.title("🤖 Empathetic AI Assistant")
54
+ # st.markdown("""
55
+ # This AI assistant analyzes your emotional state through:
56
+ # - 📸 **Facial Expression** (from camera)
57
+ # - 🎤 **Vocal Tone** (from microphone)
58
+ # - 💬 **Spoken Words** (transcribed from audio)
59
+
60
+ # Then provides an empathetic, context-aware response to your query.
61
+ # """)
62
+ # st.divider()
63
+
64
+ # col1, col2 = st.columns([1, 1])
65
+
66
+ # with col1:
67
+ # st.subheader("📸 1. Capture Your Expression")
68
+ # # Use the 'key' and 'on_change' parameters.
69
+ # st.camera_input(
70
+ # "Take a snapshot",
71
+ # key="camera_widget_buffer",
72
+ # on_change=camera_on_change
73
+ # )
74
+
75
+ # with col2:
76
+ # st.subheader("💭 2. Your Query")
77
+ # st.text_area(
78
+ # "What would you like to ask?",
79
+ # placeholder="Type your question or concern here...",
80
+ # height=100,
81
+ # key="query_widget_buffer",
82
+ # on_change=query_on_change
83
+ # )
84
+
85
+ # st.divider()
86
+
87
+ # st.subheader("🎙️ 3. Record Your Voice")
88
+ # st.write("Click the microphone to record your voice, then click 'Analyze' below.")
89
+ # audio_data = st_audiorec()
90
+
91
+ # # We manually save the audio data if it's new
92
+ # if audio_data is not None:
93
+ # st.session_state.audio_bytes = audio_data
94
+
95
+ # st.divider()
96
+
97
+ # # --- ANALYSIS BUTTON ---
98
+ # if st.button("🧠 Analyze My Emotion & Answer", type="primary", use_container_width=True):
99
+
100
+ # # Now, we check our *secure* session state variables
101
+ # if not st.session_state.camera_bytes:
102
+ # st.error("❌ Please take a snapshot using the camera first!")
103
+ # elif not st.session_state.audio_bytes:
104
+ # st.error("❌ Please record your voice first!")
105
+ # elif not st.session_state.user_query.strip():
106
+ # st.error("❌ Please enter your query!")
107
+ # else:
108
+ # # --- PROCESSING ---
109
+ # # All data is guaranteed to be safe in session_state
110
+
111
+ # # Step 1: Process camera image
112
+ # with st.spinner("📸 Processing facial expression..."):
113
+ # try:
114
+ # file_bytes = np.asarray(bytearray(st.session_state.camera_bytes), dtype=np.uint8)
115
+ # image = cv2.imdecode(file_bytes, cv2.IMREAD_COLOR)
116
+ # cv2.imwrite("temp_image.jpg", image)
117
+ # facial_emotion = get_facial_emotion("temp_image.jpg")
118
+
119
+ # except Exception as e:
120
+ # st.error(f"Error processing image: {e}")
121
+ # facial_emotion = "neutral"
122
+
123
+ # # Step 2: Process recorded audio
124
+ # with st.spinner("🎵 Saving and analyzing audio..."):
125
+ # try:
126
+ # with open("temp_audio.wav", "wb") as f:
127
+ # f.write(st.session_state.audio_bytes)
128
+
129
+ # voice_emotion = get_voice_emotion("temp_audio.wav")
130
+ # transcript = get_transcript("temp_audio.wav")
131
+
132
+ # except Exception as e:
133
+ # st.error(f"Error processing audio: {e}")
134
+ # voice_emotion = "neutral"
135
+ # transcript = ""
136
+
137
+ # # Display analysis results
138
+ # st.divider()
139
+ # st.subheader("📊 Emotional Analysis Results")
140
+ # col_a, col_b, col_c = st.columns(3)
141
+ # with col_a: st.metric(label="😊 Facial Emotion", value=facial_emotion.capitalize())
142
+ # with col_b: st.metric(label="🎤 Vocal Tone", value=voice_emotion.capitalize())
143
+ # with col_c: st.metric(label="💬 Speech Detected", value="Yes" if transcript else "No")
144
+ # if transcript: st.info(f"**Transcription:** {transcript}")
145
+
146
+ # # Step 5: Get empathetic AI response
147
+ # st.divider()
148
+ # with st.spinner("🤖 Empathetic AI is thinking..."):
149
+ # ai_response = get_llm_response(
150
+ # user_query=st.session_state.user_query, # Use query from state
151
+ # face=facial_emotion,
152
+ # voice=voice_emotion,
153
+ # text=transcript
154
+ # )
155
+
156
+ # st.subheader("💙 Empathetic Response")
157
+ # st.markdown(ai_response)
158
+ # st.balloons()
159
+
160
+ # # --- SIDEBAR (Unchanged) ---
161
+ # with st.sidebar:
162
+ # st.header("ℹ️ How to Use")
163
+ # st.markdown("""
164
+ # 1. **Take a snapshot**
165
+ # 2. **Type your query**
166
+ # 3. **Click the mic** to record (click again to stop)
167
+ # 4. **Click the 'Analyze' button**
168
+ # 5. **Receive** your response
169
+ # """)
170
+ # st.divider()
171
+ # st.header("🔑 Setup Requirements")
172
+ # st.markdown("""
173
+ # Make sure these environment variables are set in your
174
+ # Hugging Face Space **Settings > Repository secrets**:
175
+ # ```
176
+ # ROBOFLOW_API_KEY="your_key"
177
+ # GROQ_API_KEY="your_key"
178
+ # ```
179
+ # """)
180
+ # st.divider()
181
+ # st.header("🛠️ Tech Stack")
182
+ # st.markdown("""
183
+ # - **Frontend:** Streamlit
184
+ # - **Audio:** `streamlit-audiorec`
185
+ # - **Facial Analysis:** Roboflow
186
+ # - **Voice Analysis:** Hugging Face
187
+ # - **Speech-to-Text:** Google SR
188
+ # - **LLM:** Groq (Llama 3)
189
+ # """)
190
+
191
+
192
  import streamlit as st
 
193
  import numpy as np
194
+ import cv2
 
 
 
 
 
 
 
 
 
 
 
195
 
196
+ # --- Page Config (MUST be the first Streamlit command) ---
197
  st.set_page_config(
198
+ page_title="Step 1: Capture",
199
+ page_icon="📸",
200
  layout="wide"
201
  )
202
 
203
  # --- SESSION STATE INITIALIZATION ---
204
+ # Initialize all variables that need to be passed between pages
205
  if "camera_bytes" not in st.session_state:
206
  st.session_state.camera_bytes = None
207
  if "audio_bytes" not in st.session_state:
 
209
  if "user_query" not in st.session_state:
210
  st.session_state.user_query = ""
211
 
212
+ # --- Callback Functions ---
 
 
 
213
  def camera_on_change():
214
+ """Saves the raw bytes of the photo."""
 
215
  if st.session_state.camera_widget_buffer is not None:
216
  st.session_state.camera_bytes = st.session_state.camera_widget_buffer.getvalue()
217
  else:
 
218
  st.session_state.camera_bytes = None
219
 
220
  def query_on_change():
221
  """Saves the text query."""
 
222
  st.session_state.user_query = st.session_state.query_widget_buffer
223
 
224
+ # --- Page Layout ---
 
225
  st.title("🤖 Empathetic AI Assistant")
226
+ st.header("Step 1: Provide Your Inputs")
 
 
 
 
 
 
 
227
  st.divider()
228
 
229
  col1, col2 = st.columns([1, 1])
230
 
231
  with col1:
232
  st.subheader("📸 1. Capture Your Expression")
 
233
  st.camera_input(
234
  "Take a snapshot",
235
  key="camera_widget_buffer",
236
  on_change=camera_on_change
237
  )
238
 
239
+ # --- Show a confirmation that the photo is saved ---
240
+ if st.session_state.camera_bytes:
241
+ st.success("✅ Photo captured and saved!")
242
+
243
  with col2:
244
  st.subheader("💭 2. Your Query")
245
  st.text_area(
 
249
  key="query_widget_buffer",
250
  on_change=query_on_change
251
  )
252
+
253
+ # --- Show a confirmation that the query is saved ---
254
+ if st.session_state.user_query:
255
+ st.success("✅ Query saved!")
256
 
257
  st.divider()
258
 
259
+ # --- Navigation Button ---
260
+ st.write("Once your photo is captured and your query is typed, proceed to the next step.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
261
 
262
+ # Check if inputs are ready
263
+ if st.session_state.camera_bytes and st.session_state.user_query:
264
+ if st.button("Go to Step 2: Record Voice 🎤", type="primary", use_container_width=True):
265
+ st.switch_page("pages/2_Analyze.py")
266
+ else:
267
+ st.warning("Please take a photo and enter a query to continue.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
268
 
269
+ # --- Sidebar ---
270
  with st.sidebar:
271
+ st.header("Instructions")
272
  st.markdown("""
273
  1. **Take a snapshot**
274
  2. **Type your query**
275
+ 3. Click **'Go to Step 2'**
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
276
  """)