Update app.py

app.py

@@ -1,19 +1,17 @@
 from flask import Flask, render_template_string, request, jsonify
+import speech_recognition as sr
 from tempfile import NamedTemporaryFile
-import whisper
-import ffmpeg
-from fuzzywuzzy import process
 import os
+import ffmpeg
+from fuzzywuzzy import process, fuzz
+import phonetics
+from metaphone import doublemetaphone
 import logging
 
-# Flask app
 app = Flask(__name__)
 logging.basicConfig(level=logging.INFO)
 
-# Whisper Model
-model = whisper.load_model("base")  # Use 'base' or 'large' for better accuracy
-
-# Global Variables
+# Global variables
 cart = {}
 menu_preferences = "all"
 prices = {
@@ -65,19 +63,39 @@ def process_audio():
 
         converted_file = NamedTemporaryFile(delete=False, suffix=".wav")
         ffmpeg.input(temp_file.name).output(
-            converted_file.name, acodec="pcm_s16le", ac=1, ar="16000"
+            converted_file.name, acodec="pcm_s16le", ac=1, ar="16000", af="highpass=f=200,lowpass=f=3000,acompressor"
         ).run(overwrite_output=True)
 
-        # Use Whisper for transcription
-        result = model.transcribe(converted_file.name)
-        raw_command = result["text"].lower()
+        recognizer = sr.Recognizer()
+        recognizer.dynamic_energy_threshold = True
+        recognizer.energy_threshold = 100
+        recognizer.pause_threshold = 1.0
+        recognizer.phrase_threshold = 0.3
+        recognizer.non_speaking_duration = 0.2
+
+        with sr.AudioFile(converted_file.name) as source:
+            recognizer.adjust_for_ambient_noise(source, duration=1)
+            audio_data = recognizer.record(source)
+
+        raw_command = None
+        for _ in range(2):  # Retry recognition twice if needed
+            try:
+                raw_command = recognizer.recognize_google(audio_data).lower()
+                break
+            except sr.UnknownValueError:
+                response = "Sorry, I couldn't catch that. Could you repeat?"
+                logging.info(response)
+
+        if raw_command is None:
+            return jsonify({"response": "Sorry, I couldn't understand. Please try again."})
+
         logging.info(f"Raw recognized command: {raw_command}")
 
         # Preprocess command
         all_menu_items = menus["all"]
         command = preprocess_command(raw_command, all_menu_items)
 
-        # Process command
+        # Pass preprocessed command to process_command
         response = process_command(command)
 
     except Exception as e:
@@ -92,45 +110,65 @@ def preprocess_command(command, menu_items):
     """
     Preprocess the user command:
     - Normalize speech for accents and speed using fuzzy matching.
+    - Phonetically match menu items.
     """
-    closest_match = process.extractOne(command, menu_items)
-    if closest_match and closest_match[1] > 70:  # Adjust fuzzy match threshold
+    def phonetic_match(word, options):
+        word_phonetic = doublemetaphone(word)[0]
+        for option in options:
+            if doublemetaphone(option)[0] == word_phonetic:
+                return option
+        return None
+
+    # First, try fuzzy matching
+    closest_match = process.extractOne(command, menu_items, scorer=fuzz.token_set_ratio)
+    if closest_match and closest_match[1] > 60:
         return closest_match[0]
+
+    # Fallback to phonetic matching
+    words = command.split()
+    for word in words:
+        match = phonetic_match(word, menu_items)
+        if match:
+            return match
+
     return command
 
 def process_command(command):
     global cart, menu_preferences
     command = command.lower()
 
+    # Recognize menu preferences explicitly
     if menu_preferences == "all":
         if "non-vegetarian" in command:
             menu_preferences = "non-vegetarian"
-            return "You have chosen the Non-Vegetarian menu. To view menu say menu."
+            return "You have chosen the Non-Vegetarian menu. To view menu say menu"
         elif "vegetarian" in command and "non-vegetarian" not in command:
             menu_preferences = "vegetarian"
-            return "You have chosen the Vegetarian menu. To view menu say menu."
+            return "You have chosen the Vegetarian menu. To view menu say menu"
         elif "guilt-free" in command:
             menu_preferences = "guilt-free"
-            return "You have chosen the Guilt-Free menu. To view menu say menu."
+            return "You have chosen the Guilt-Free menu. To view menu say menu"
         elif "all" in command:
             menu_preferences = "all"
-            return "You have chosen the complete menu. To view menu say menu."
+            return "You have chosen the complete menu. To view menu say menu"
 
+    # Filtered menu based on preference
     menu = menus.get(menu_preferences, menus["all"])
 
     if "menu" in command:
         return f"Here is your menu: {', '.join(menu)}. To select an item say item name."
     elif "price of" in command:
         item = command.replace("price of", "").strip()
-        closest_match = process.extractOne(item, prices.keys())
-        if closest_match and closest_match[1] > 70:
+        closest_match = process.extractOne(item, prices.keys(), scorer=fuzz.token_set_ratio)
+        if closest_match and closest_match[1] > 60:
             matched_item = closest_match[0]
             return f"The price of {matched_item} is ${prices[matched_item]}."
         return "Sorry, I couldn't find that item in the menu."
     elif "remove" in command:
+        # Extract the item name after "remove"
         item = command.replace("remove", "").strip()
-        closest_match = process.extractOne(item, list(cart.keys()))
-        if closest_match and closest_match[1] > 70:
+        closest_match = process.extractOne(item, list(cart.keys()), scorer=fuzz.token_set_ratio)
+        if closest_match and closest_match[1] > 60:
             matched_item = closest_match[0]
             if cart[matched_item] > 1:
                 cart[matched_item] -= 1
@@ -140,16 +178,16 @@ def process_command(command):
                 return f"{matched_item.capitalize()} has been removed from your cart. Current cart: {dict(cart)}."
         return "Sorry, that item is not in your cart."
     elif any(item in command for item in menu):
-        closest_match = process.extractOne(command, menu)
-        if closest_match and closest_match[1] > 70:
+        closest_match = process.extractOne(command, menu, scorer=fuzz.token_set_ratio)
+        if closest_match and closest_match[1] > 60:
             matched_item = closest_match[0]
             cart[matched_item] = cart.get(matched_item, 0) + 1
-            return f"{matched_item.capitalize()} added to your cart. Current cart: {dict(cart)}. To finalize say final order."
+            return f"{matched_item.capitalize()} added to your cart. Current cart: {dict(cart)}. To finalize say final order"
         return "Sorry, I couldn't recognize the item. Could you try again?"
     elif "final order" in command:
         if cart:
             total = sum(prices[item] * count for item, count in cart.items())
-            response = f"Your final order is: {', '.join(f'{item} x{count}' for item, count in cart.items())}. Your total bill is ${total}. Thank you for ordering! To exit this conversation say nothing or goodbye."
+            response = f"Your final order is: {', '.join(f'{item} x{count}' for item, count in cart.items())}. Your total bill is ${total}. Thank you for ordering! To exit this conversation say nothing or goodbye!"
             cart.clear()
             return response
         return "Your cart is empty. Please add items to your cart first."
@@ -159,26 +197,137 @@ def process_command(command):
         return "Goodbye! Thank you for using AI Dining Assistant."
     return "Sorry, I couldn't understand that. Please try again."
 
-html_code = """<!DOCTYPE html>
+html_code = """
+<!DOCTYPE html>
 <html lang="en">
 <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>AI Dining Assistant</title>
+    <style>
+        body {
+            display: flex;
+            flex-direction: column;
+            align-items: center;
+            justify-content: center;
+            min-height: 100vh;
+            margin: 0;
+            font-family: Arial, sans-serif;
+            background-color: #f4f4f9;
+        }
+        h1 {
+            color: #333;
+        }
+        .mic-button {
+            font-size: 2rem;
+            padding: 1rem 2rem;
+            color: white;
+            background-color: #007bff;
+            border: none;
+            border-radius: 50px;
+            cursor: pointer;
+            transition: background-color 0.3s;
+        }
+        .mic-button:hover {
+            background-color: #0056b3;
+        }
+        .status, .response {
+            margin-top: 1rem;
+            text-align: center;
+            color: #555;
+            font-size: 1.2rem;
+        }
+        .response {
+            background-color: #e8e8ff;
+            padding: 1rem;
+            border-radius: 10px;
+            box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2);
+            display: none;
+        }
+    </style>
 </head>
 <body>
     <h1>AI Dining Assistant</h1>
-    <button id="mic-button">🎤 Speak</button>
-    <div id="response"></div>
+    <button class="mic-button" id="mic-button">🎤</button>
+    <div class="status" id="status">Press the mic button to start...</div>
+    <div class="response" id="response">Response will appear here...</div>
     <script>
         const micButton = document.getElementById('mic-button');
-        const responseDiv = document.getElementById('response');
+        const status = document.getElementById('status');
+        const response = document.getElementById('response');
+        let mediaRecorder;
+        let audioChunks = [];
+        let isConversationActive = false;
         micButton.addEventListener('click', () => {
-            alert('Audio capture functionality not implemented in this sample HTML.');
+            if (!isConversationActive) {
+                isConversationActive = true;
+                startConversation();
+            }
         });
+        function startConversation() {
+            const utterance = new SpeechSynthesisUtterance('Please choose your preference: All, Vegetarian, Non-Vegetarian, or Guilt-Free.');
+            speechSynthesis.speak(utterance);
+            utterance.onend = () => {
+                status.textContent = 'Listening...';
+                startListening();
+            };
+        }
+        function startListening() {
+            navigator.mediaDevices.getUserMedia({ audio: true }).then(stream => {
+                mediaRecorder = new MediaRecorder(stream, { mimeType: 'audio/webm;codecs=opus' });
+                mediaRecorder.start();
+                audioChunks = [];
+                mediaRecorder.ondataavailable = event => audioChunks.push(event.data);
+                mediaRecorder.onstop = async () => {
+                    const audioBlob = new Blob(audioChunks, { type: 'audio/webm' });
+                    const formData = new FormData();
+                    formData.append('audio', audioBlob);
+                    status.textContent = 'Processing...';
+                    try {
+                        const result = await fetch('/process-audio', { method: 'POST', body: formData });
+                        const data = await result.json();
+                        response.textContent = data.response;
+                        response.style.display = 'block';
+                        const utterance = new SpeechSynthesisUtterance(data.response);
+                        speechSynthesis.speak(utterance);
+                        utterance.onend = () => {
+                            console.log("Speech synthesis completed.");
+                            if (data.response.includes("Goodbye")) {
+                                status.textContent = 'Conversation ended. Press the mic button to start again.';
+                                isConversationActive = false;
+                                fetch('/reset-cart'); // Reset the cart dynamically on end
+                            } else if (data.response.includes("Your order is complete")) {
+                                status.textContent = 'Order complete. Thank you for using AI Dining Assistant.';
+                                isConversationActive = false;
+                                fetch('/reset-cart'); // Reset the cart after final order
+                            } else {
+                                status.textContent = 'Listening...';
+                                setTimeout(() => {
+                                    startListening();
+                                }, 100);
+                            }
+                        };
+                        utterance.onerror = (e) => {
+                            console.error("Speech synthesis error:", e.error);
+                            status.textContent = 'Error with speech output.';
+                            isConversationActive = false;
+                        };
+                    } catch (error) {
+                        response.textContent = 'Sorry, I could not understand. Please try again.';
+                        response.style.display = 'block';
+                        status.textContent = 'Press the mic button to restart the conversation.';
+                        isConversationActive = false;
+                    }
+                };
+                setTimeout(() => mediaRecorder.stop(), 5000);
+            }).catch(() => {
+                status.textContent = 'Microphone access denied.';
+                isConversationActive = false;
+            });
+        }
     </script>
 </body>
-</html>"""
-
+</html>
+"""
 if __name__ == "__main__":
     app.run(host="0.0.0.0", port=7860)