Final update

app.py

@@ -4,12 +4,15 @@ import openpyxl
 from openpyxl.styles import Font, Alignment
 import tempfile
 from roboflow import Roboflow
+from inference import get_model
+import supervision as sv
 from PIL import Image
 import cv2
 import numpy as np
 import os
 from gradio_calendar import Calendar
 import datetime
+import google.generativeai as genai
 
 excel_tempfile_state = gr.State()
 roll_number_state = gr.State()
@@ -19,6 +22,33 @@ sno_state.value=1
 
 str_nums = {"zero":0,"one":1,"two":2,"three":3,"four":4,"five":5,"six":6,"seven":7,"eight":8,"nine":9}
 
+genai.configure(api_key="AIzaSyBXu8JrmDtmuK0FQCGZMyB3T-hb35UCtGM")
+
+def upload_to_gemini(path, mime_type=None):
+    try:
+        file = genai.upload_file(path, mime_type=mime_type)
+        print(f"Uploaded file '{file.display_name}' as: {file.uri}")
+        return file
+    except Exception as e:
+        print(f"Error uploading file: {e}")
+        return None
+
+# Create the model
+generation_config = {
+  "temperature": 1,
+  "top_p": 0.95,
+  "top_k": 64,
+  "max_output_tokens": 8192,
+  "response_mime_type": "text/plain",
+}
+
+
+model = genai.GenerativeModel(
+  model_name="gemini-2.0-flash-lite-preview-02-05",
+  generation_config=generation_config,
+  system_instruction="Identify the single handwritten digit on the given image, some times there will be blank images given.In that case give blank as output",
+)
+
 def task1(Examination, Date_Of_Exam, Program, Branch, Course, Name_Of_Faculty, Academic_Year):
   with tempfile.NamedTemporaryFile(suffix=".xlsx", delete=False) as temp_file:
         workbook = openpyxl.Workbook()
@@ -194,24 +224,51 @@ iface1 = gr.Interface(
 )
 
 
-def predict_and_crop(image_np, api_key, project_name, model_version, confidence=40, overlap=30):
-    img = Image.fromarray(image_np)
-    rf = Roboflow(api_key=api_key)
-    project = rf.workspace().project(project_name)
-    model = project.version(model_version).model
-    corners = model.predict(image_np, confidence=confidence, overlap=overlap).json()
-    predictions = corners["predictions"][0]
-    prediction = {key: int(value) for key, value in predictions.items() if key in ['x', 'y', 'width', 'height']}
-    x1 = int(prediction['x'] - prediction['width'] / 2)
-    y1 = int(prediction['y'] - prediction['height'] / 2)
-    x2 = int(prediction['x'] + prediction['width'] / 2)
-    y2 = int(prediction['y'] + prediction['height'] / 2)
-    cropped_img = img.crop((x1, y1, x2, y2))
-    cropped_img_np = np.array(cropped_img)
-    h, w, c = cropped_img_np.shape
-    if h > w:
-        cropped_img_np = cv2.rotate(cropped_img_np, cv2.ROTATE_90_CLOCKWISE)
-    return cropped_img_np, img  # Return both cropped image and original image
+def predict_and_crop(image_np, api_key, model_id, confidence=40, overlap=30):
+    # Convert the image to the format expected by the model (e.g., numpy array)
+    image = image_np
+
+    # Get the Roboflow model and run inference
+    model = get_model(model_id=model_id, api_key=api_key)
+    results = model.infer(image)[0]
+
+    # Print and process results
+    print("Results:", results)
+
+    # Load the results into the supervision Detections API
+    detections = sv.Detections.from_inference(results)
+    print("detections:", detections)
+
+    # Create supervision annotators
+    bounding_box_annotator = sv.BoxAnnotator()
+    label_annotator = sv.LabelAnnotator()
+
+    # Annotate the image with bounding boxes and labels
+    annotated_image = bounding_box_annotator.annotate(scene=image, detections=detections)
+    annotated_image = label_annotator.annotate(scene=annotated_image, detections=detections)
+
+    # Display the image with annotations
+    sv.plot_image(annotated_image)
+
+    # Now, extract the bounding box details from detections for cropping
+    for detection in detections.xyxy:
+        # Extract coordinates from detections (xyxy format: [x1, y1, x2, y2])
+        x1, y1, x2, y2 = map(int, detection)
+
+        # Crop the image based on the bounding box
+        cropped_img = image[y1:y2, x1:x2]
+
+        # Convert cropped image to PIL for rotation handling (if needed)
+        cropped_img_pil = Image.fromarray(cropped_img)
+        cropped_img_np = np.array(cropped_img_pil)
+
+        # Rotate the image if height is greater than width
+        h, w, c = cropped_img_np.shape
+        if h > w:
+            cropped_img_np = cv2.rotate(cropped_img_np, cv2.ROTATE_90_CLOCKWISE)
+        print(cropped_img_np)
+        return cropped_img_np, image  # Return both cropped image and original image
+
 
 # Function to resize and insert cropped image
 def resize_and_insert(cropped_image, output_image_path):
@@ -274,18 +331,21 @@ def append_to_workbook(cells_data, excel_file_path):
 def task2(image_np):
   api_key = "UyAumhQJOJpo7vUu3LaK"
   project_name = "marks_table_detection_lbrce"
+  model_id = "marks_table_detection_lbrce/2"
   model_version = 1
-  base_image_path = "base_img.png"
-  temp_image_path = "temp_image.jpg"
-  output_image_path = "merged_image.jpg"
+  base_image_path = "/content/base_img.png"
+  temp_image_path = "/content/temp_image.jpg"
+  output_image_path = "/content/merged_image.jpg"
 
   #API and requirements for OCR Model
-  rf = Roboflow(api_key="XsMt3y86MNDGihOYcWDY")
-  project = rf.workspace().project("mnist-cjkff")
-  model = project.version(2).model
+  # rf = Roboflow(api_key="XsMt3y86MNDGihOYcWDY")
+  # project = rf.workspace().project("mnist-cjkff")
+  # model = project.version(2).model
+  # model = get_model(model_id="mnist-cjkff", api_key="XsMt3y86MNDGihOYcWDY")
 
   # Predict and crop
-  cropped_image, original_image = predict_and_crop(image_np, api_key, project_name, model_version)
+  # cropped_image, original_image = predict_and_crop(image_np, api_key, project_name, model_version)
+  cropped_image, original_image = predict_and_crop(image_np, api_key, model_id)
 
   # Resize and insert
   result_image = resize_and_insert(cropped_image, output_image_path)
@@ -305,12 +365,33 @@ def task2(image_np):
       _, temp_thresholded_path = tempfile.mkstemp(suffix=".jpg")
       cv2.imwrite(temp_thresholded_path, thresholded_cell)
 
-      # res = model.predict(temp_thresholded_path).json()
-      # var = res["predictions"][0]["predictions"][0]["class"]
-      # conf = res["predictions"][0]["predictions"][0]["confidence"]
-      var = 2
-      conf = 97
-      cells_data.append(convert_str_int(var, conf))
+      image_path = "/content/temp_image_12.jpg"  # Make sure the image is in the correct path
+      mime_type = "image/jpeg"  # Use the appropriate MIME type
+
+      # Upload the image to Gemini
+      file = upload_to_gemini(temp_thresholded_path, mime_type)
+
+      # Start a chat session and send the image as input
+      chat_session = model.start_chat(
+          history=[
+      {
+        "role": "user",
+        "parts": [
+          file,
+          "Identify the digit",
+        ],
+      },
+      ]
+      )
+
+      # Send a message to process the image
+      response = chat_session.send_message("Identify the digit")
+      print(response.text)  # Output the model's response
+      dt = response.text[::2]
+      if dt.isdigit():
+        cells_data.append(int(dt))
+      else:
+        cells_data.append(0)
       qno += 1
   excel_file_path = excel_tempfile_state.value
   append_to_workbook(cells_data,excel_file_path)
@@ -336,6 +417,4 @@ iface2 = gr.Interface(
 demo = gr.TabbedInterface([iface1, iface2], ["Configure Excel Sheet Data", "Extract marks from Answer Sheets"])
 
 # Run the interface
-demo.launch(share=True,debug=True)
-
-
+demo.launch(share=True,debug=True)