import streamlit as st import cv2 import numpy as np from PIL import Image import torch import torchvision.transforms as transforms from PIL import Image import sys from transformers import FlaxAutoModelForSeq2SeqLM, AutoTokenizer import time import requests sys.path.append("F:\\Desktop\\DL\\Project\\bhanucha_charviku_final_project\\bhanucha_charviku_final\\application\\yolov7") from models.experimental import attempt_load model_path ="F:\\Desktop\\DL\\Project\\bhanucha_charviku_final_project\\bhanucha_charviku_final\\application\\best.pt" API_TOKEN="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" # RECIPE GENERATION METHODS MODEL_NAME_OR_PATH = "flax-community/t5-recipe-generation" tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME_OR_PATH, use_fast=True) model = FlaxAutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME_OR_PATH) prefix = "items: " generation_kwargs = { "max_length": 512, "min_length": 64, "no_repeat_ngram_size": 3, "do_sample": True, "top_k": 60, "top_p": 0.95 } special_tokens = tokenizer.all_special_tokens tokens_map = { "": "--", "
": "\n" } def skip_special_tokens(text, special_tokens): for token in special_tokens: text = text.replace(token, "") return text def target_postprocessing(texts, special_tokens): if not isinstance(texts, list): texts = [texts] new_texts = [] for text in texts: text = skip_special_tokens(text, special_tokens) for k, v in tokens_map.items(): text = text.replace(k, v) new_texts.append(text) return new_texts @st.cache_data def generation_function(texts): _inputs = texts if isinstance(texts, list) else [texts] inputs = [prefix + inp for inp in _inputs] print(inputs) inputs = tokenizer(inputs, max_length=256, padding="max_length", truncation=True, return_tensors="jax") input_ids = inputs.input_ids attention_mask = inputs.attention_mask output_ids = model.generate(input_ids=input_ids, attention_mask=attention_mask, **generation_kwargs) generated = tokenizer.batch_decode(output_ids.sequences, skip_special_tokens=False) generated_recipe = target_postprocessing(generated, special_tokens) return generated_recipe def print_recipe(recipe_text): sections = recipe_text.split("\n") for section in sections: section = section.strip() if section.startswith("title:"): section = section.replace("title:", "").strip() headline = "TITLE" elif section.startswith("ingredients:"): section = section.replace("ingredients:", "").strip() headline = "INGREDIENTS" elif section.startswith("directions:"): section = section.replace("directions:", "").strip() headline = "DIRECTIONS" if headline == "TITLE": st.write(f"[{headline}]: {section.capitalize()}") else: section_info = [f" - {i+1}: {info.strip().capitalize()}" for i, info in enumerate(section.split("--"))] st.write(f"[{headline}]:") for info in section_info: st.write(info) st.write("-" * 130) # RECIPE GENERATION METHODS # OBJECT DETCTION METHODS @st.cache_data def load_model(model_path): device = torch.device('cpu') model = attempt_load(model_path, map_location=device) model.eval() return model def detect_ingredients(_image, _model): image = np.array(_image) transform = transforms.Compose([ transforms.ToPILImage(), transforms.Resize((640, 640)), transforms.ToTensor(), # transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) image = transform(image).unsqueeze(0) with torch.no_grad(): outputs = _model(image) predictions = outputs[0] print("Predictions Shape:", predictions.shape) return predictions def parse_predictions(predictions, class_labels, threshold=0.1): predictions = predictions.squeeze(0) detected_ingredients = [] for prediction in predictions: conf = prediction[4] if conf > threshold: class_probs = prediction[5:] class_id = class_probs.argmax() class_confidence = class_probs[class_id] # print("Class ID:", class_id) # print("Class Confidence:", class_confidence) # print("Raw Prediction:", prediction) if class_confidence > threshold: ingredient = class_labels.get(int(class_id), "Unknown") # print("Detected Ingredient:", ingredient) if ingredient not in detected_ingredients: detected_ingredients.append(ingredient) return detected_ingredients @st.cache_data def load_class_labels(filepath): class_labels = {} with open(filepath, 'r') as file: for idx, line in enumerate(file): class_labels[idx] = line.strip() return class_labels # OBJECT DETCTION METHODS # NUTRITIONAL VALUE def get_nutritional_values(ingredients): print("INGred: ", ingredients) url = "https://api-inference.huggingface.co/models/sgarbi/bert-fda-nutrition-ner" headers = {"Authorization": "Bearer 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"} payload = { "inputs": ingredients } print("PAYLoad: ",payload) response = requests.post(url, json=payload, headers=headers) print("RESPONSE: ",response) if response.status_code == 200: return response.json() else: print() return None #NUTRITIONAL VALUE def main(): st.set_option("deprecation.showfileUploaderEncoding", False) st.title("CSE 676: Deep Learning") st.header("EYES ON EATS: From Image to Formula") st.header("Snap, Cook, Enjoy: Satisfy Your Cravings!") st.write("What's in Your Kitchen? Let's Create a Dish") st.write("MODEL: Object Detection using YOLO") model = load_model(model_path) class_labels = load_class_labels("F:\\Desktop\\DL\\Project\\bhanucha_charviku_final_project\\bhanucha_charviku_final\\application\\Final_classes.txt") upload_option = st.radio("Upload Image Option:", ("Upload from File", "Capture from Camera")) ingredients = [] # List to store detected ingredients if upload_option == "Upload from File": uploaded_file = st.file_uploader("Upload an image of the ingredients", type=["jpg", "jpeg"]) if uploaded_file is not None: try: image = Image.open(uploaded_file) st.image(image, caption="Uploaded Image", use_column_width=True) predictions = detect_ingredients(image, model) new_ingredients = parse_predictions(predictions, class_labels) print("detected ingredients: ", new_ingredients) st.write("Predicted Ingredients:", new_ingredients) print("NEW INGREDIENTS:", new_ingredients) ingredients += new_ingredients # Append new ingredients to the existing list print("INGREDIENTS STORED: ", ingredients) ingredients_string = ", ".join(new_ingredients) generate_recipe_button = st.button("Generate Recipe", key="generate_recipe") if generate_recipe_button: st.write("Loading Recipe...") generated_recipes = generation_function(ingredients_string) st.header("Generated Recipe:") for recipe_text in generated_recipes: print_recipe(recipe_text) st.write("----------------------------------------------------------------------------------------------------------------------------------") # Retrieve nutritional values nutrition_recipe_button = st.button("Get Nutritional Content", key="nutritional_value") if nutrition_recipe_button: nutritional_info = get_nutritional_values(", ".join(new_ingredients)) if nutritional_info is not None: st.write("Nutritional Values:") st.write(nutritional_info) else: st.error("Failed to retrieve nutritional values.") except Exception as e: st.error(f"Error processing the recipe: {str(e)}") else: st.write("Please allow access to your camera.") camera = cv2.VideoCapture(0) if st.button("Capture Image"): st.write("Get ready to take a snap!") time.sleep(3) # Add a delay of 3 seconds _, frame = camera.read() frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) image = Image.fromarray(frame) st.image(image, caption="Captured Image", use_column_width=True) predictions = detect_ingredients(image, model) new_ingredients = parse_predictions(predictions, class_labels) st.write("Predicted Ingredients:", new_ingredients) ingredients += new_ingredients # Append new ingredients to the existing list generate_recipe_button = st.button("Generate Recipe", key="generate_recipe") if generate_recipe_button: st.write("Loading Recipe...") recipe = generation_function(ingredients) st.header("Generated Recipe:") st.write(recipe) st.markdown("**Made by: Tarun and Charvi**") if __name__ == '__main__': main()