from flask import Flask, render_template_string, request, jsonify import speech_recognition as sr from tempfile import NamedTemporaryFile import os import ffmpeg from fuzzywuzzy import process import phonetics import logging app = Flask(__name__) logging.basicConfig(level=logging.INFO) # Global variables cart = {} menu_preferences = "all" prices = { "samosa": 9, "onion pakoda": 10, "chilli gobi": 12, "chicken biryani": 14, "mutton biryani": 16, "veg biryani": 12, "panner butter": 10, "fish curry": 12, "chicken manchurian": 14, "veg manchurian": 12, "chilli chicken": 14, "panner biryani": 13, "chicken curry": 14 } menus = { "all": list(prices.keys()), "vegetarian": [ "samosa", "onion pakoda", "chilli gobi", "veg biryani", "panner butter", "veg manchurian", "panner biryani" ], "non-vegetarian": [ "chicken biryani", "mutton biryani", "fish curry", "chicken manchurian", "chilli chicken", "chicken curry" ], "guilt-free": ["samosa", "onion pakoda"] } @app.route("/") def index(): return render_template_string(html_code) @app.route("/reset-cart", methods=["GET"]) def reset_cart(): global cart, menu_preferences cart = {} menu_preferences = "all" return "Cart reset successfully." @app.route("/process-audio", methods=["POST"]) def process_audio(): try: audio_file = request.files.get("audio") if not audio_file: return jsonify({"response": "No audio file provided."}), 400 temp_file = NamedTemporaryFile(delete=False, suffix=".webm") audio_file.save(temp_file.name) converted_file = NamedTemporaryFile(delete=False, suffix=".wav") ffmpeg.input(temp_file.name).output( converted_file.name, acodec="pcm_s16le", ac=1, ar="16000" ).run(overwrite_output=True) recognizer = sr.Recognizer() recognizer.dynamic_energy_threshold = True recognizer.energy_threshold = 100 # Sensitive for low audio levels with sr.AudioFile(converted_file.name) as source: audio_data = recognizer.record(source) raw_command = recognizer.recognize_google(audio_data).lower() logging.info(f"Raw recognized command: {raw_command}") # Preprocess command all_menu_items = menus["all"] command = preprocess_command(raw_command, all_menu_items) # Pass preprocessed command to process_command response = process_command(command) except sr.UnknownValueError: response = "Sorry, I couldn't understand. Please try again." except Exception as e: response = f"An error occurred: {str(e)}" finally: os.unlink(temp_file.name) os.unlink(converted_file.name) return jsonify({"response": response}) def preprocess_command(command, menu_items): """ Preprocess the user command: - Normalize speech for accents and speed using fuzzy matching. - Phonetically match menu items. """ def phonetic_match(word, options): word_phonetic = phonetics.metaphone(word) for option in options: if phonetics.metaphone(option) == word_phonetic: return option return None # First, try fuzzy matching closest_match = process.extractOne(command, menu_items) if closest_match and closest_match[1] > 70: # Adjust fuzzy match threshold return closest_match[0] # Fallback to phonetic matching words = command.split() for word in words: match = phonetic_match(word, menu_items) if match: return match return command def process_command(command): global cart, menu_preferences command = command.lower() # Recognize menu preferences explicitly if menu_preferences == "all": if "non-vegetarian" in command: menu_preferences = "non-vegetarian" return "You have chosen the Non-Vegetarian menu. To view menu say menu" elif "vegetarian" in command and "non-vegetarian" not in command: menu_preferences = "vegetarian" return "You have chosen the Vegetarian menu. To view menu say menu" elif "guilt-free" in command: menu_preferences = "guilt-free" return "You have chosen the Guilt-Free menu. To view menu say menu" elif "all" in command: menu_preferences = "all" return "You have chosen the complete menu. To view menu say menu" # Filtered menu based on preference menu = menus.get(menu_preferences, menus["all"]) if "menu" in command: return f"Here is your menu: {', '.join(menu)}. To select an item say item name." elif "price of" in command: item = command.replace("price of", "").strip() closest_match = process.extractOne(item, prices.keys()) if closest_match and closest_match[1] > 70: matched_item = closest_match[0] return f"The price of {matched_item} is ${prices[matched_item]}." return "Sorry, I couldn't find that item in the menu." elif "remove" in command: # Extract the item name after "remove" item = command.replace("remove", "").strip() closest_match = process.extractOne(item, list(cart.keys())) if closest_match and closest_match[1] > 70: matched_item = closest_match[0] if cart[matched_item] > 1: cart[matched_item] -= 1 return f"One {matched_item} has been removed from your cart. Current cart: {dict(cart)}." else: del cart[matched_item] return f"{matched_item.capitalize()} has been removed from your cart. Current cart: {dict(cart)}." return "Sorry, that item is not in your cart." elif any(item in command for item in menu): closest_match = process.extractOne(command, menu) if closest_match and closest_match[1] > 70: matched_item = closest_match[0] cart[matched_item] = cart.get(matched_item, 0) + 1 return f"{matched_item.capitalize()} added to your cart. Current cart: {dict(cart)}. To finalize say final order" return "Sorry, I couldn't recognize the item. Could you try again?" elif "final order" in command: if cart: total = sum(prices[item] * count for item, count in cart.items()) response = f"Your final order is: {', '.join(f'{item} x{count}' for item, count in cart.items())}. Your total bill is ${total}. Thank you for ordering! To exist this conversation say nothing or good bye!" cart.clear() return response return "Your cart is empty. Please add items to your cart first." elif "no" in command or "nothing" in command or "goodbye" in command: cart.clear() menu_preferences = "all" return "Goodbye! Thank you for using AI Dining Assistant." return "Sorry, I couldn't understand that. Please try again." html_code = """ AI Dining Assistant

AI Dining Assistant

Press the mic button to start...
Response will appear here...
""" if __name__ == "__main__": app.run(host="0.0.0.0", port=7860)