Updated Application File

app.py

@@ -4,11 +4,11 @@ from PIL import Image
 import mediapipe as mp
 import time
 import gradio as gr
+import glob
 
 DOMINANT_HAND = "Right"
 
-# width, height = 1280, 720
-width_, height_, = 256, 144
+width_, height_, = 144, 96
 
 drawing_flag = False
 sleepy_time = time.time()
@@ -17,25 +17,31 @@ output_frames = []
 
 
 def find_hands(brain, img):
-    img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)  # opencv image is in BGR form but mp is trained with RGB
-    results = brain.process(
-        img_rgb)  # process finds the hands and outputs classification and 21 landmarks for each hand
-    all_hands = []  # initializing array to hold the dictionary for the hands
-    h, w, _ = img.shape  # get height and width of image for scaling
-    if results.multi_hand_landmarks:
-        for hand_type, hand_lms in zip(results.multi_handedness,
-                                       results.multi_hand_landmarks):  # elegant solution for mp list object traversal
-            hand = {}  # initializing dict for each hand
-            lm_list = []  # landmarks array for all 21 point of the hand
-            for lm in hand_lms.landmark:
-                px, py, pz = int(lm.x * w), int(lm.y * h), int(
-                    lm.z * w)  # scaling landmark points to image size for frame coordinates
-                lm_list.append([px, py, pz])
-
-            hand["lm_list"] = lm_list  # add "lm_list" key for all landmark points of the hand
-            hand["type"] = hand_type.classification[0].label  # adds the label (left/right) for the hand
-            all_hands.append(hand)  # appends the dict
-    return all_hands
+    if img is not None:
+        # print(type(img))
+        img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)  # opencv image is in BGR form but mp is trained with RGB
+        results = brain.process(
+            img_rgb)  # process finds the hands and outputs classification and 21 landmarks for each hand
+        all_hands = []  # initializing array to hold the dictionary for the hands
+        h, w, _ = img.shape  # get height and width of image for scaling
+        if results.multi_hand_landmarks:
+            for hand_type, hand_lms in zip(results.multi_handedness,
+                                           results.multi_hand_landmarks):  # elegant solution for mp list object traversal
+                hand = {}  # initializing dict for each hand
+                lm_list = []  # landmarks array for all 21 point of the hand
+                for lm in hand_lms.landmark:
+                    px, py, pz = int(lm.x * w), int(lm.y * h), int(
+                        lm.z * w)  # scaling landmark points to image size for frame coordinates
+                    lm_list.append([px, py, pz])
+
+                hand["lm_list"] = lm_list  # add "lm_list" key for all landmark points of the hand
+                hand["type"] = hand_type.classification[0].label  # adds the label (left/right) for the hand
+                all_hands.append(hand)  # appends the dict
+        return all_hands
+
+    else:
+        return 0
+
 
 
 def is_drawing(index, thumb):  # proximity function with arbitrary threshold
@@ -75,7 +81,7 @@ def show(video): # main
     paper.fill(255)
 
     past_holder = ()  # hold previous index coordinates
-    palette = cv2.imread('palette.jpg')
+    palette = cv2.imread('palette_small.jpg')
 
     page_num = 0  # iterating for saving (not a viable function for gradio)
 
@@ -83,11 +89,10 @@ def show(video): # main
 
     global sleepy_time  # get sleep time for multiple gestures
 
-    # runny = 1
     while cam.isOpened():
         # runny -= 1
         x, rgb_image = cam.read()
-        rgb_image_f = cv2.flip(np.asanyarray(rgb_image), 1)  # mirrored video
+        rgb_image_f = cv2.flip(rgb_image, 1)  # mirrored video
 
         hands = find_hands(detector, rgb_image_f)
 
@@ -130,16 +135,15 @@ def show(video): # main
                             # paper[idx_coords[0]][idx_coords[1]][3] = 255
                             cv2.circle(rgb_image_f, idx_coords, 5, color, cv2.FILLED)
 
-                        if save(lm_list1) and time.time() - sleepy_time > 3: # save / output
-                            paper[0:height_, w - width_: w] = 255
-                            paper = cv2.cvtColor(paper, cv2.COLOR_BGR2RGB)
-                            im = Image.fromarray(paper)
-                            im.save("paper%s.png" % page_num)
-                            print("saved")
-                            sleepy_time = time.time()
-                            paper = cv2.cvtColor(paper, cv2.COLOR_RGB2BGR)
-                            page_num += 1
-                            return paper
+                        # if save(lm_list1) and time.time() - sleepy_time > 3: # save / output
+                        #     paper[0:height_, w - width_: w] = 255 # presenter eraser
+                        #     paper = cv2.cvtColor(paper, cv2.COLOR_BGR2RGB)
+                        #     im = Image.fromarray(paper)
+                        #     im.save("paper%s.png" % page_num)
+                        #     print("saved")
+                        #     sleepy_time = time.time()
+                        #     paper = cv2.cvtColor(paper, cv2.COLOR_RGB2BGR)
+                        #     page_num += 1
 
                         if clear(lm_list1) and time.time() - sleepy_time > 3: # reset paper
                             paper = np.zeros((height, width, 3), dtype=np.uint8)
@@ -158,23 +162,35 @@ def show(video): # main
                 pass
 
             finally:
-                rgb_image_f[0:72, ] = palette
-                presenter = cv2.resize(rgb_image_f, (width_, height_))
-                h, w, _ = rgb_image_f.shape
-                paper[0:height_, w - width_: w] = presenter
-
-                # output_frames.append(paper)
-
-                # cv2.imshow("Image", rgb_image_f)
-                # cv2.imshow("paper", paper)
-                # key = cv2.waitKey(1)
-                # if key & 0xFF == ord('q') or key == 27:  # Press esc or 'q' to close the image window
-                #     break
+                if True:
+                    rgb_image_f[0:48, ] = palette # 48 small
+                    presenter = cv2.resize(rgb_image_f, (width_, height_))
+                    h, w, _ = rgb_image_f.shape
+                    paper[0:height_, w - width_: w] = presenter
 
         else:
             break
 
+        paper = cv2.cvtColor(paper, cv2.COLOR_RGB2BGR)
+        im = Image.fromarray(paper)
+        output_frames.append(paper)
+        im.save("paper%s.png" % page_num)
+        page_num += 1
+
+    img_array = []
+    for filename in glob.glob('*.png'):
+        imggg = cv2.imread(filename)
+        img_array.append(imggg)
+
+    video_output = cv2.VideoWriter('any.webm', cv2.VideoWriter_fourcc(*'VP80'), 30, (640, 480))
+
+    for i in range(len(img_array)):
+        video_output.write(img_array[i])
+    video_output.release()
+
+    return 'any.webm'
+
 
-iface = gr.Interface(fn=show, inputs=gr.inputs.Video(source="webcam", type="mp4"), outputs='image')
+iface = gr.Interface(fn=show, inputs=gr.inputs.Video(source="webcam"), outputs='video')
 
-iface.launch(share=True)
+iface.launch(share=True, enable_queue=True)