Update app.py

app.py

@@ -1,22 +1,15 @@
 # -*- coding: utf-8 -*-
-"""Flipkart Frontend.ipynb
+"""Complete_3_model_code.ipynb
 
 Automatically generated by Colab.
 
 Original file is located at
-    https://colab.research.google.com/github/Abhinav-gh/404NotFound/blob/main/Flipkart%20Frontend.ipynb
+    https://colab.research.google.com/drive/1Ivlv1jHXwoldi9Mb-quvDBlCNeIIAhc9
 
 # 1. Install Gradio and Required Libraries
 ### Start by installing Gradio if it's not already installed.
 """
 
-# ! pip install gradio
-# ! pip install cv
-# ! pip install ultralytics
-# ! pip install supervision
-# !pip install google-generativeai
-# !pip install paddleocr
-# !pip install paddlepaddle
 
 """# 2. Import Libraries
 ### Getting all the necessary Libraries
@@ -29,48 +22,35 @@ from PIL import Image
 import cv2
 import time
 from ultralytics import YOLO
-import supervision as sv
 import pandas as pd
+from IPython.display import clear_output
 from collections import defaultdict, deque
 import matplotlib.pyplot as plt
+import torch
+from PIL import Image
+from torchvision import transforms, models, datasets, transforms
+from torch.utils.data import DataLoader
+import torch.nn as nn
+import matplotlib.pyplot as plt
 import google.generativeai as genai
 from datetime import datetime
 from paddleocr import PaddleOCR
 import os
 
-"""# Path Variables
-
-### Path used in OCR
-"""
-
-OCR_M3="best.pt"
-GOOGLE_API_KEY = os.getenv("GEMINI_API")
-GEMINI_MODEL = 'models/gemini-1.5-flash'
-
-"""### Path used in  Brand Recognition  model"""
-
-Brand_Recognition_Model ='kitkat_s.pt'
-annotatedOpFile= 'annotated_output.mp4'
-
 """# 3. Import Drive
 
 """
 
 # from google.colab import drive
-
 # drive.mount('/content/drive')
 
 """# 4. Brand Recognition Backend
 
-### Model for Grocery Detection
+### Image uploading for Grocery detection
 """
 
-model_path = Brand_Recognition_Model
-model = YOLO(model_path)
-
-"""### Image uploading for Grocery detection"""
-
 def detect_grocery_items(image):
+    model = YOLO('kitkat_s.pt')
     image = np.array(image)[:, :, ::-1]
     results = model(image)
     annotated_image = results[0].plot()
@@ -109,7 +89,6 @@ def detect_grocery_items(image):
 """### Detect Grovcery brand from video"""
 
 def iou(box1, box2):
-    # Calculate intersection over union
     x1 = max(box1[0], box2[0])
     y1 = max(box1[1], box2[1])
     x2 = min(box1[2], box2[2])
@@ -128,22 +107,19 @@ def smooth_box(box_history):
     return np.mean(box_history, axis=0)
 
 def process_video(input_path, output_path):
+    model = YOLO('kitkat_n.pt')
     cap = cv2.VideoCapture(input_path)
 
-    # Get video properties
     width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
     height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
     fps = int(cap.get(cv2.CAP_PROP_FPS))
 
-    # Initialize video writer
     fourcc = cv2.VideoWriter_fourcc(*'mp4v')
     out = cv2.VideoWriter(output_path, fourcc, fps, (width, height))
 
-    # Initialize variables for tracking
     detected_items = {}
     frame_count = 0
 
-    # For result confirmation
     detections_history = defaultdict(lambda: defaultdict(int))
 
     while cap.isOpened():
@@ -153,7 +129,6 @@ def process_video(input_path, output_path):
 
         frame_count += 1
 
-        # Run YOLO detection every 5th frame
         if frame_count % 5 == 0:
             results = model(frame)
 
@@ -169,7 +144,6 @@ def process_video(input_path, output_path):
 
                     current_frame_detections.append((brand, [x1, y1, x2, y2], conf))
 
-            # Match current detections with existing items
             for brand, box, conf in current_frame_detections:
                 matched = False
                 for item_id, item_info in detected_items.items():
@@ -203,7 +177,6 @@ def process_video(input_path, output_path):
                 del detected_items[item_id]
                 continue
 
-            # Interpolate box position
             if item_info['smoothed_box'] is not None:
                 alpha = 0.3
                 current_box = item_info['smoothed_box']
@@ -224,7 +197,6 @@ def process_video(input_path, output_path):
     cap.release()
     out.release()
 
-    # Calculate final counts and confirm results
     total_frames = frame_count
     confirmed_items = {}
     for brand, frame_counts in detections_history.items():
@@ -236,7 +208,7 @@ def process_video(input_path, output_path):
     return confirmed_items
 
 def annotate_video(input_video):
-    output_path = annotatedOpFile
+    output_path = 'annotated_output.mp4'
     confirmed_items = process_video(input_video, output_path)
 
     item_list = [(brand, quantity) for brand, quantity in confirmed_items.items()]
@@ -281,6 +253,7 @@ def draw_bounding_boxes(image_path):
     return all_text_data
 
 # Set your API key securely (store it in Colab’s userdata)
+GOOGLE_API_KEY= os.getenv("GEMINI_API")
 genai.configure(api_key=GOOGLE_API_KEY)
 
 def gemini_context_correction(text):
@@ -301,14 +274,9 @@ def gemini_context_correction(text):
 
     return response.text
 
-# Test Gemini with example text (replace with actual OCR output)
-sample_text = "EXP 12/2024 MFD 08/2023 Best Before 06/2025 MRP Rs. 250/-"
-refined_output = gemini_context_correction(sample_text)
-print("[DEBUG] Gemini Refined Output:\n", refined_output)
-
 def validate_dates_with_gemini(mfg_date, exp_date):
     """Use Gemini API to validate and correct the manufacturing and expiration dates."""
-    model = genai.GenerativeModel(GEMINI_MODEL)
+    model = genai.GenerativeModel('models/gemini-1.5-flash')
     response = model.generate_content = (
         f"Input Manufacturing Date: {mfg_date}, Expiration Date: {exp_date}. "
         f"If either date is '-1', leave it as is. "
@@ -332,7 +300,7 @@ def extract_and_validate_with_gemini(refined_text):
     """
     Use Gemini API to extract, validate, and correct manufacturing and expiration dates.
     """
-    model = genai.GenerativeModel(GEMINI_MODEL)
+    model = genai.GenerativeModel('models/gemini-1.5-flash')
 
     # Correctly call the generate_content method
     response = model.generate_content(
@@ -381,7 +349,7 @@ def extract_and_validate_with_gemini(refined_text):
     """
     Use Gemini API to extract, validate, correct, and swap dates in 'yyyy/mm/dd' format if necessary.
     """
-    model = genai.GenerativeModel(GEMINI_MODEL)
+    model = genai.GenerativeModel('models/gemini-1.5-flash')
 
     # Generate content using Gemini with the refined prompt
     response = model.generate_content(
@@ -455,9 +423,6 @@ def extract_date(refined_text, date_type):
             return '-1'  # Return -1 if the date is not found
     return '-1'  # Return -1 if the date type is not in the text
 
-
-
-
 """### **Model 3**
 Using Yolov8 x-large model trained till about 75 epochs
 and
@@ -466,9 +431,7 @@ Gradio as user interface
 
 """
 
-model_path = OCR_M3
-model = YOLO(model_path)
-
+model = YOLO('best.pt')
 """## Driver code to be run after selecting from Model 2 or 3.
 (Note: not needed for model 1)
 """
@@ -601,6 +564,310 @@ def handle_processing(validated_output):
     print("[DEBUG] Hiding the 'Further Processing' button.")  # Debug print
     return gr.update(visible=False)  # Hide button if dates are valid
 
+"""# Freshness Backend"""
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+
+class EfficientNet_FeatureExtractor(nn.Module):
+
+    def __init__(self):
+        super(EfficientNet_FeatureExtractor, self).__init__()
+        self.efficientnet = models.efficientnet_b0(pretrained=True)
+        self.efficientnet = nn.Sequential(*list(self.efficientnet.children())[:-1])
+
+    def forward(self, x):
+        x = self.efficientnet(x)
+        x = x.view(x.size(0), -1)
+
+        return x
+
+# Calculating the mean and variance of the images whose features will be extracted
+
+
+transform = transforms.Compose([
+    transforms.Resize(256),
+    transforms.CenterCrop(224),
+    transforms.ToTensor(),
+])
+
+dataset = datasets.ImageFolder(root='Datasets/Bananas', transform=transform)
+
+# Create a DataLoader
+loader = DataLoader(dataset, batch_size=32, shuffle=False)
+
+# Initialize variables to calculate the mean and std
+mean = 0.0
+std = 0.0
+total_images = 0
+
+# Iterate over the dataset to compute mean and std
+for images, _ in loader:
+    batch_samples = images.size(0)
+    images = images.view(batch_samples, images.size(1), -1)  # Flatten each image (C, H*W)
+
+    # Calculate mean and std for this batch and add to the running total
+    mean += images.mean(2).sum(0)
+    std += images.std(2).sum(0)
+    total_images += batch_samples
+
+# Final mean and std across all images in the dataset
+mean /= total_images
+std /= total_images
+
+print(f"Mean: {mean}")
+print(f"Std: {std}")
+
+# Transforming the images into the format so that they can be passes through the EfficientNet model
+# Define the transform for your dataset, including normalization with custom mean and std
+transform = transforms.Compose([
+    transforms.Resize(256),
+    transforms.CenterCrop(224),
+    transforms.ToTensor(),
+    transforms.Normalize(mean=mean, std=std)
+])
+
+test_dataset = datasets.ImageFolder(root='/Dataset/Bananas', transform=transform)
+
+# Extracting features from Efficientnet model
+def extract_features(test_dataset):
+
+  # Initialize the feature extractor model
+  model = EfficientNet_FeatureExtractor().to(device)
+  model.eval()  # Set to evaluation mode
+
+  # Create a DataLoader for the test dataset
+  test_loader = DataLoader(test_dataset, batch_size=50, shuffle=False)
+
+
+  # Store the extracted features
+  all_features = []
+
+  # Loop over the test dataset and extract features
+  with torch.no_grad():  # Disable gradient calculation for efficiency
+      for images, _ in test_loader:
+          # Send the images to the same device as the model
+          images = images.to(device)
+
+          # Pass the images through the feature extractor
+          features = model(images)
+
+          # Move features to CPU and convert to NumPy (optional)
+          features = features.cpu().numpy()
+
+          # Append the features for further use
+          all_features.append(features)
+  return all_features
+
+all_features = extract_features(test_dataset)
+
+# Print the shape of each batch stored in the list
+for i, features in enumerate(all_features):
+    print(f"Shape of batch {i}: {features.shape}")
+
+# Calculating the mean and varinance of the entire distribution
+
+# Stack all the feature vectors into a single tensor
+all_features_tensor = torch.cat([torch.tensor(batch) for batch in all_features], dim=0)
+
+# Calculate the mean and variance along the feature dimension
+feature_mean = all_features_tensor.mean(dim=0)
+feature_mean = feature_mean.to(device)
+feature_variance = all_features_tensor.var(dim=0)
+
+print(f"Feature Mean Shape: {feature_mean.shape}")
+
+all_features_tensor = torch.cat([torch.tensor(f) for f in all_features], dim=0)
+all_features_tensor = all_features_tensor.to(device)
+feature_mean_temp = all_features_tensor.mean(dim=0)
+centered_features = all_features_tensor - feature_mean_temp
+
+# Calculate the covariance matrix
+# Covariance matrix: (num_features, num_features)
+covariance_matrix = torch.cov(centered_features.T)
+covariance_matrix = covariance_matrix.to(device)
+
+print(f"All Feature Tensor Shape: {all_features_tensor.shape}")
+print(f"Covariance Matrix Shape: {covariance_matrix.shape}")
+
+# Defining the function to calculate the Mahalanobis distance
+
+import torch
+
+def mahalanobis(x=None, feature_mean=None, feature_cov=None):
+    """Compute the Mahalanobis Distance between each row of x and the data
+    x             : tensor of shape [batch_size, num_features], feature vectors of test data
+    feature_mean  : tensor of shape [num_features], mean of the training feature vectors
+    feature_cov   : tensor of shape [num_features, num_features], covariance matrix of the training feature vectors
+    """
+
+    # Subtract the mean from x
+    x_minus_mu = x - feature_mean
+
+    # Invert the covariance matrix
+    inv_covmat = torch.inverse(feature_cov)
+
+    # Mahalanobis distance computation: (x - mu)^T * inv_cov * (x - mu)
+    left_term = torch.matmul(x_minus_mu, inv_covmat)
+    mahal = torch.matmul(left_term, x_minus_mu.T)
+    return mahal.diag()
+
+
+transform = transforms.Compose([
+    transforms.Resize(256),
+    transforms.CenterCrop(224),
+    transforms.ToTensor(),
+    transforms.Normalize(mean=mean, std=std)
+])
+
+def classify_banana_by_distance(distance):
+    """
+    Classifies the banana's freshness based on the Mahalanobis distance.
+
+    Args:
+        distance (float): Mahalanobis distance of the banana.
+
+    Returns:
+        dict: A dictionary containing the classification and relevant details.
+    """
+
+    # Define thresholds for classification based on the provided distances
+    if distance >= 9:
+        # Case 1: Completely Fresh Banana
+        return {
+            "Classification": "Completely Fresh",
+            "Freshness Index": 10,
+            "Color": "Mostly yellow, little to no brown spots",
+            "Dark Spots": "0-10%",
+            "Shelf Life": "5-7 days",
+            "Ripeness Stage": "Just ripe",
+            "Texture": "Firm and smooth"
+        }
+    elif -90 <= distance < 0:
+        # Case 2: Banana with 40% Dark Brown Spots
+        return {
+            "Classification": "Moderately Ripe",
+            "Freshness Index": 6,
+            "Color": "60% yellow, 40% dark spots",
+            "Dark Spots": "40% dark spots",
+            "Shelf Life": "2-3 days",
+            "Ripeness Stage": "Moderately ripe",
+            "Texture": "Some softness, still edible"
+        }
+    else:
+        # Case 3: Almost Rotten Banana
+        return {
+            "Classification": "Almost Rotten",
+            "Freshness Index": 2,
+            "Color": "Mostly brown or black, very few yellow patches",
+            "Dark Spots": "80-100% dark spots",
+            "Shelf Life": "0-1 days",
+            "Ripeness Stage": "Overripe",
+            "Texture": "Very soft, mushy, may leak moisture"
+        }
+
+    return result
+
+def classify_banana(image):
+
+    model = EfficientNet_FeatureExtractor().to(device)
+    model.eval()  # Set to evaluation mode
+
+    # Load and transform the image
+    img = Image.fromarray(image)
+    img_transformed = transform(img).unsqueeze(0).to(device)
+
+    # Feature extraction
+    with torch.no_grad():
+        features = model(img_transformed)
+
+    # Calculate Mahalanobis distance
+    distance = mahalanobis(features, feature_mean, covariance_matrix)
+    distance = (distance) / 1e8
+
+    return classify_banana_by_distance(distance)
+
+"""## Freshness Detect Using image"""
+
+def detect_objects(image):
+
+
+    # Load the YOLO model
+    model = YOLO('Yash_Best.pt')
+    # Run inference on the image
+    result = model(image)
+
+    # Get the image from the result
+    img = result[0].orig_img  # Original image
+
+    # If bounding boxes are detected, loop over them and draw them
+    if result[0].boxes is not None:
+        for i, box in enumerate(result[0].boxes.xyxy):  # Bounding boxes (x1, y1, x2, y2)
+            x1, y1, x2, y2 = map(int, box[:4])
+            conf = result[0].boxes.conf[i].item()  # Confidence score
+            cls = int(result[0].boxes.cls[i].item())  # Class ID
+
+            # Get the label name
+            label = f'{result[0].names[cls]} {conf:.2f}'
+
+            # Draw the bounding box
+            cv2.rectangle(img, (x1, y1), (x2, y2), (0, 255, 0), 2)  # Green box
+            cv2.putText(img, label, (x1, y1 + 20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 2)
+
+    # Convert image to RGB for displaying in Gradio
+    img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+
+    return img_rgb
+
+"""## Freshness Detect using Video"""
+
+def detect_objects_video(video_file):
+
+    # Load the YOLO model
+    model = YOLO('Yash_Best.pt')
+    # Open the video file
+    cap = cv2.VideoCapture(video_file.name)
+
+    # Get video properties
+    width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    fps = int(cap.get(cv2.CAP_PROP_FPS))
+
+    # Output video writer to save the results
+    output_video_path = 'output_detected_video.mp4'
+    fourcc = cv2.VideoWriter_fourcc(*'mp4v')
+    out = cv2.VideoWriter(output_video_path, fourcc, fps, (width, height))
+
+    # Process each frame from the video
+    while cap.isOpened():
+        ret, frame = cap.read()
+        if not ret:
+            break  # Exit if there are no more frames
+
+        # Run object detection on the frame
+        results = model(frame)
+
+        # Loop over detection results and draw bounding boxes with labels
+        if results[0].boxes is not None:
+            for i, box in enumerate(results[0].boxes.xyxy):  # Bounding boxes (x1, y1, x2, y2)
+                x1, y1, x2, y2 = map(int, box[:4])
+                conf = results[0].boxes.conf[i].item()  # Confidence score
+                cls = int(results[0].boxes.cls[i].item())  # Class ID
+                label = f'{results[0].names[cls]} {conf:.2f}'
+
+                # Draw bounding box and label
+                cv2.rectangle(frame, (x1, y1), (x2, y2), (0, 255, 0), 2)
+                cv2.putText(frame, label, (x1, y1 + 20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 2)
+
+        # Write the processed frame to the output video
+        out.write(frame)
+
+    # Release resources
+    cap.release()
+    out.release()
+
+    return output_video_path
+
 """# 5. Frontend Of Brand Recognition
 
 ## Layout for Image interface
@@ -712,14 +979,66 @@ def create_ocr_interface():
 ocr_interface = create_ocr_interface()
 # ocr_interface.launch(share=True, debug=True)
 
+
+"""# Frontend for Fruit Freshness
+
+## Layout for Freshness Index
+"""
+
+def create_banana_classifier_interface():
+    return gr.Interface(
+        fn=classify_banana,  # Your classification function
+        inputs=gr.Image(type="numpy", label="Upload a Banana Image"),  # Removed tool argument
+        outputs=gr.JSON(label="Classification Result"),
+        title="Banana Freshness Classifier",
+        description="Upload an image of a banana to classify its freshness.",
+        css="#component-0 { width: 300px; height: 300px; }"  # Keep your CSS for fixed size
+    )
+
+def image_freshness_interface():
+    return gr.Interface(
+        fn=detect_objects,  # Your detection function
+        inputs=gr.Image(type="numpy", label="Upload an Image"),  # Removed tool argument
+        outputs=gr.Image(type="numpy", label="Detected Image"),
+        live=True,
+        title="Image Freshness Detection",
+        description="Upload an image of fruit to detect freshness.",
+        css="#component-0 { width: 300px; height: 300px; }"  # Keep your CSS for fixed size
+    )
+
+def video_freshness_interface():
+    return gr.Interface(
+        fn=process_video,  # Your video processing function
+        inputs=gr.Video(label="Upload a Video"),
+        outputs=gr.Video(label="Processed Video"),
+        title="Video Freshness Detection",
+        description="Upload a video of fruit to detect freshness.",
+        css="#component-0 { width: 300px; height: 300px; }"  # Keep your CSS for fixed size
+    )
+
+def create_fruit_interface():
+    with gr.Blocks() as demo:
+        gr.Markdown("# Flipkart Grid Robotics Track - Fruits Interface")
+        with gr.Tabs():
+            with gr.Tab("Banana"):
+                create_banana_classifier_interface()  # Call the banana classifier interface
+            with gr.Tab("Image Freshness"):
+                image_freshness_interface()  # Call the image freshness interface
+            with gr.Tab("Video Freshness"):
+                video_freshness_interface()  # Call the video freshness interface
+    return demo
+
+
+Fruit = create_fruit_interface()
+
 """# 6. Create a Tabbed Interface for Both Image and Video
 ### Here, we combine the image and video interfaces into a tabbed structure so users can switch between them easily.
 """
 
 def create_tabbed_interface():
     return gr.TabbedInterface(
-        [Brand_recog,  ocr_interface ],
-        ["Brand Recongnition", "OCR"]
+        [Brand_recog,  ocr_interface,Fruit ],
+        ["Brand Recongnition", "OCR" , "Fruit Freshness"]
     )
 
 tabbed_interface = create_tabbed_interface()
@@ -728,4 +1047,4 @@ tabbed_interface = create_tabbed_interface()
 ### Finally, launch the Gradio interface to make it interactable.
 """
 
-tabbed_interface.launch()
+tabbed_interface.launch()