testing new app for realtime pred

app.py

@@ -6,7 +6,7 @@ from huggingface_hub import snapshot_download
 
 
 REPO_ID='SharkSpace/videos_examples'
-snapshot_download(repo_id=REPO_ID, token= os.environ.get('SHARK_MODEL'),repo_type='dataset',local_dir='videos_example')
+snapshot_download(repo_id=REPO_ID, token=os.environ.get('SHARK_MODEL'),repo_type='dataset',local_dir='videos_example')
 
 
 if os.getenv('SYSTEM') == 'spaces':
@@ -27,7 +27,7 @@ dotenv.load_dotenv()
 import numpy as np
 import gradio as gr
 import glob
-from inference import inference_frame
+from inference import inference_frame,inference_frame_serial
 from inference import inference_frame_par_ready
 from inference import process_frame
 import os
@@ -36,69 +36,42 @@ import multiprocessing as mp
 
 from time import time
 
-def analyze_video(x, skip_frames = 5, frame_rate_out = 8):
-    print(x)
 
-    #Define path to saved images
-    path = 'tmp/test/'
+def analize_video_serial(x):
+    print(x)
+    path = '/tmp/test/'
     os.makedirs(path, exist_ok=True)
-    
-    # Define name of current video as number of videos in path
-    n_videos_in_path = len(os.listdir(path))
-    path = f'{path}{n_videos_in_path}'
+    videos = len(os.listdir(path))
+    path = f'{path}{videos}'
     os.makedirs(path, exist_ok=True)
-    
-    # Define name of output video
     outname = f'{path}_processed.mp4'
-    
     if os.path.exists(outname):
         print('video already processed')
         return outname
-    
     cap = cv2.VideoCapture(x)
     counter = 0
-    
+    import pdb;pdb.set_trace()
     while(cap.isOpened()):
-        frames = []
-        start = time()
-        for i in range(16):
-            start = time()
-            ret, frame = cap.read()
-            frames.append(frame)
-            if ret == False:
-                break
-        print(f'read time: {time()-start}')
-        
-        #if ret==True:
-
-            #if counter % skip_frames == 0:
-        name = os.path.join(path,f'{counter:05d}.png')
-        # Get timing for inference
-        start = time()
-        frames = inference_frame(frames)
-        print(f'inference time: {time()-start}')
-        # write the flipped frame
-        
-        start = time()
-        for frame in frames:
+        ret, frame = cap.read()
+        yield None, frame 
+        if ret==True:
             name = os.path.join(path,f'{counter:05d}.png')
+            frame = inference_frame_serial(frame)
+            # write the flipped frame
+            
             cv2.imwrite(name, frame)
             counter +=1
-        print(f'write time: {time()-start}')
-        # else: 
-            
-        # print(counter)
-        # counter +=1
-        # else:
-        #     break
-    
+           
+            #yield None,frame 
+        else:
+            break
     # Release everything if job is finished
-    cap.release()
-
-    # Create video from predicted images
     print(path)
-    os.system(f'''ffmpeg -framerate {frame_rate_out} -pattern_type glob -i '{path}/*.png'  -c:v libx264 -pix_fmt yuv420p {outname} -y''')
-    return outname
+    os.system(f'''ffmpeg -framerate 20 -pattern_type glob -i '{path}/*.png'  -c:v libx264 -pix_fmt yuv420p {outname} -y''')
+    return outname,frame 
+
+
+
 
 
 def analyze_video_parallel(x, skip_frames = 5, 
@@ -174,12 +147,14 @@ def analyze_video_parallel(x, skip_frames = 5,
         name = os.path.join(path,f'{counter:05d}.png')
         cv2.imwrite(name, pool_out_tmp)
         counter +=1
+        yield None,pool_out_tmp
+        
     print(f'write time: {time()-start}')
 
     # Create video from predicted images
     print(path)
     os.system(f'''ffmpeg -framerate {frame_rate_out} -pattern_type glob -i '{path}/*.png'  -c:v libx264 -pix_fmt yuv420p {outname} -y''')
-    return outname
+    return outname, pool_out_tmp
   
 
 def set_example_image(example: list) -> dict:
@@ -207,7 +182,10 @@ with gr.Blocks(title='Shark Patrol',theme=gr.themes.Soft(),live=True,) as demo:
         with gr.Row():
             video_input = gr.Video(source='upload',include_audio=False)
             #video_input.style(witdh='50%',height='50%')
+            image_temp = gr.Image()
+        with gr.Row():
             video_output = gr.Video()
+            
             #video_output.style(witdh='50%',height='50%')
         
         video_button = gr.Button("Analyze your Video")
@@ -215,14 +193,17 @@ with gr.Blocks(title='Shark Patrol',theme=gr.themes.Soft(),live=True,) as demo:
             paths = sorted(pathlib.Path('videos_example/').rglob('*.mp4'))
             example_images = gr.Dataset(components=[video_input],
                                     samples=[[path.as_posix()]
-                                             for path in paths if 'videos_side_by_side' not in str(path)])
+                                             for path in paths if 'raw_videos'  in str(path)])
 
-    video_button.click(analyze_video_parallel, inputs=video_input, outputs=video_output)
+    video_button.click(analize_video_serial, inputs=video_input, outputs=[video_output,image_temp])
 
     example_images.click(fn=set_example_image,
                          inputs=example_images,
                          outputs=video_input)
 
+         
 demo.queue()
-#if os.getenv('SYSTEM') == 'spaces':
-demo.launch(width='40%',auth=(os.environ.get('SHARK_USERNAME'), os.environ.get('SHARK_PASSWORD')))
+if os.getenv('SYSTEM') == 'spaces':
+    demo.launch(width='40%',auth=(os.environ.get('SHARK_USERNAME'), os.environ.get('SHARK_PASSWORD')))
+else: 
+    demo.launch()

app3.py

@@ -0,0 +1,79 @@
+
+import subprocess
+import os  
+if os.getenv('SYSTEM') == 'spaces':
+
+    subprocess.call('pip install -U openmim'.split())
+    subprocess.call('pip install python-dotenv'.split())
+    subprocess.call('pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu113'.split())
+    subprocess.call('mim install mmcv>=2.0.0'.split())
+    subprocess.call('mim install mmengine'.split())
+    subprocess.call('mim install mmdet'.split())
+    subprocess.call('pip install opencv-python-headless==4.5.5.64'.split())
+    subprocess.call('pip install git+https://github.com/cocodataset/panopticapi.git'.split())
+
+import gradio as gr
+
+from huggingface_hub import snapshot_download
+import cv2 
+import dotenv 
+dotenv.load_dotenv()
+import numpy as np
+import gradio as gr
+import glob
+from inference import inference_frame,inference_frame_serial
+from inference import inference_frame_par_ready
+from inference import process_frame
+import os
+import pathlib
+import multiprocessing as mp
+from time import time
+
+
+REPO_ID='SharkSpace/videos_examples'
+snapshot_download(repo_id=REPO_ID, token=os.environ.get('SHARK_MODEL'),repo_type='dataset',local_dir='videos_example')
+
+
+
+    
+
+
+
+def process_video(input_video):
+    cap = cv2.VideoCapture(input_video)
+
+    output_path = "output.mp4"
+
+    fps = int(cap.get(cv2.CAP_PROP_FPS))
+    width  = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+
+    video = cv2.VideoWriter(output_path, cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height))
+
+    iterating, frame = cap.read()
+    while iterating:
+        # flip frame vertically
+        display_frame = inference_frame_serial(frame)
+        video.write(frame)
+        yield display_frame, None
+        iterating, frame = cap.read()
+
+    video.release()
+    yield display_frame, output_path
+
+with gr.Blocks() as demo:
+    with gr.Row():
+        input_video = gr.Video(label="input")
+        processed_frames = gr.Image(label="last frame")
+        output_video = gr.Video(label="output")
+
+    with gr.Row():
+        paths = sorted(pathlib.Path('videos_example/').rglob('*.mp4'))
+        samples=[[path.as_posix()] for path in paths if 'raw_videos'  in str(path)]
+        examples = gr.Examples(samples, inputs=input_video)
+        process_video_btn = gr.Button("process video")
+
+    process_video_btn.click(process_video, input_video, [processed_frames, output_video])
+
+demo.queue()
+demo.launch()

inference.py

@@ -80,7 +80,7 @@ classes= ['Beach',
 REPO_ID = "SharkSpace/maskformer_model"
 FILENAME = "mask2former"
 
-# snapshot_download(repo_id=REPO_ID, token= os.environ.get('SHARK_MODEL'),local_dir='model/')
+snapshot_download(repo_id=REPO_ID, token= os.environ.get('SHARK_MODEL'),local_dir='model/')
 
 
 
@@ -105,12 +105,23 @@ print(dir(visualizer))
 # the dataset_meta is loaded from the checkpoint and
 # then pass to the model in init_detector
 visualizer.dataset_meta = model.dataset_meta
+def inference_frame_serial(image):
+    result = inference_detector(model, image)
+    # show the results
+    visualizer.add_datasample(
+    'result',
+    image,
+    data_sample=result,
+    draw_gt = None,
+    show=False
+    )
+    frame = visualizer.get_image()
+    return frame
+
 
 def inference_frame(image):
-    #import ipdb; ipdb.set_trace()
     result = inference_detector(model, image)
     # show the results
-    #import ipdb; ipdb.set_trace()
     frames = []
     cnt=0
     
@@ -127,8 +138,7 @@ def inference_frame(image):
         cnt+=1
 
     #frames = process_frames(result, image, visualizer)
-    end = time()
-    print("Time taken for drawing: ", end-start)
+ 
     return frames
 
 def inference_frame_par_ready(image):