prepare for testing

README.md

@@ -1,5 +1,5 @@
 ---
-title: NextJump
+title: NextJump + Single Rope Contest
 emoji: 🦘
 colorFrom: blue
 colorTo: green

app.py

@@ -46,9 +46,11 @@ if torch.cuda.is_available():
     sess_options = ort.SessionOptions()
     #sess_options.log_severity_level = 0
     ort_sess = ort.InferenceSession(onnx_file, sess_options=sess_options, providers=providers)
+    use_cuda = True
 else:
     print("Using CPU")
     ort_sess = ort.InferenceSession(onnx_file)
+    use_cuda = False
 
 # warmup inference
 ort_sess.run(None, {'video': np.zeros((4, 64, 3, IMG_SIZE, IMG_SIZE), dtype=np.float32)})
@@ -182,121 +184,6 @@ def detect_beeps(video_path, target_event_length=30, beep_height=0.8):
     return event_start, event_end
 
 
-def detect_relay_beeps(video_path, event_start, relay_length=30, n_jumpers=4, beep_height=0.8):
-    reference_file = 'relay_beep.WAV'
-    fs, beep = wavfile.read(reference_file)
-    beep = beep[:, 0] + beep[:, 1]  # combine stereo to mono
-    video = cv2.VideoCapture(video_path)
-    try:
-        os.remove('temp.wav')
-    except FileNotFoundError:
-        pass
-    audio_convert_command = f'ffmpeg -i {video_path} -vn -acodec pcm_s16le -ar {fs} -ac 2 temp.wav'
-    subprocess.call(audio_convert_command, shell=True)
-    length = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
-    fps = int(video.get(cv2.CAP_PROP_FPS))
-    audio = wavfile.read('temp.wav')[1]
-    audio = (audio[:, 0] + audio[:, 1]) / 2  # combine stereo to mono
-    corr = correlate(audio, beep, mode='same') / audio.size
-    # min max scale to -1, 1
-    corr = 2 * (corr - np.min(corr)) / (np.max(corr) - np.min(corr)) - 1
-    
-    # Calculate total event length in frames
-    total_event_length_frames = fps * relay_length * n_jumpers
-    print(event_start, total_event_length_frames)
-    expected_event_end = event_start + total_event_length_frames
-    
-    # Find all significant peaks in the correlation
-    peaks, _ = find_peaks(corr, height=beep_height, distance=fs)
-    
-    # Convert peaks from sample indices to frame indices
-    peak_frames = [int(peak / fs * fps) for peak in peaks]
-    
-    # For debugging
-    plt.plot(corr)
-    plt.plot(peaks, corr[peaks], "x")
-    plt.savefig('beep.png')
-    plt.close()
-
-    starts = []
-    ends = []
-    
-    # Add the event start for the first jumper
-    starts.append(event_start)
-    
-    # Convert event_start back to sample index for comparison
-    event_start_sample = int(event_start * fs / fps)
-    
-    # Find peaks that come after the event start but before the expected end
-    # Convert expected_event_end to sample index
-    expected_event_end_sample = int(expected_event_end * fs / fps)
-    relevant_peaks = [p for p in peaks if event_start_sample < p < expected_event_end_sample]
-    
-    # If we don't have enough peaks, try lowering the threshold
-    if len(relevant_peaks) < n_jumpers - 1:  # We need n_jumpers-1 transitions
-        for lower_height in [0.7, 0.6, 0.5, 0.4, 0.3]:
-            peaks, _ = find_peaks(corr, height=lower_height, distance=fs)
-            relevant_peaks = [p for p in peaks if event_start_sample < p < expected_event_end_sample]
-            if len(relevant_peaks) >= n_jumpers - 1:
-                break
-    
-    # If we still don't have enough peaks, we'll need to estimate some transitions
-    relay_length_frames = fps * relay_length
-    
-    # Process peaks to identify jumper transitions
-    if len(relevant_peaks) >= n_jumpers - 1:
-        # Ideal case: we found enough beeps for transitions
-        # Sort peaks by time to ensure correct order
-        relevant_peaks.sort()
-        
-        # Use the first n_jumpers-1 peaks as transition points
-        transition_frames = [int(p / fs * fps) for p in relevant_peaks[:n_jumpers-1]]
-        
-        # Set ends for jumpers based on transition points
-        for i in range(n_jumpers - 1):
-            ends.append(transition_frames[i])
-            starts.append(transition_frames[i])
-        
-        # Add end for the last jumper
-        ends.append(expected_event_end)
-    else:
-        # Not enough peaks detected, use expected relay_length to estimate
-        for i in range(n_jumpers):
-            if i == 0:
-                # First jumper starts at event_start (already added to starts)
-                jumper_end = event_start + relay_length_frames
-                ends.append(jumper_end)
-                if i < n_jumpers - 1:
-                    starts.append(jumper_end)
-            elif i < n_jumpers - 1:
-                jumper_end = starts[i] + relay_length_frames
-                ends.append(jumper_end)
-                starts.append(jumper_end)
-            else:
-                # Last jumper
-                jumper_end = starts[i] + relay_length_frames
-                ends.append(jumper_end)
-    
-    # Validate and adjust if necessary
-    # Make sure all intervals are close to relay_length
-    for i in range(n_jumpers):
-        interval = ends[i] - starts[i]
-        # If an interval is significantly different from relay_length, adjust it
-        if abs(interval - relay_length_frames) > relay_length_frames * 0.2:  # 20% tolerance
-            # Adjust the end time to match expected relay_length
-            ends[i] = starts[i] + relay_length_frames
-            # If not the last jumper, adjust the next start time
-            if i < n_jumpers - 1:
-                starts[i + 1] = ends[i]
-    
-    # Final check: ensure the total length matches expected
-    if ends[-1] != expected_event_end:
-        # Adjust the last end to match the expected total event end
-        ends[-1] = expected_event_end
-    
-    return starts, ends
-
-
 def upload_video(out_text, in_video):
     if out_text != '':
         # generate a timestamp name for the video
@@ -336,38 +223,16 @@ def count_phases(phase_sin, phase_cos, threshold=0.5):
 
 
 
-def inference(in_video, stream_url, start_time, end_time, use_60fps, model_choice, 
-              beep_detection_on, event_length, relay_detection_on, relay_length, switch_delay,
+def inference(in_video, use_60fps, 
+              beep_detection_on, event_length,
               count_only_api, api_key, seq_len=64, stride_length=32, stride_pad=3, batch_size=2,
               miss_threshold=0.5, marks_threshold=0.5, median_pred_filter=True, both_feet=True, 
               api_call=False,
               progress=gr.Progress()):
-    global current_model, ort_sess
     print(in_video)
-    if model_choice != current_model:
-        current_model = model_choice
-        onnx_file = hf_hub_download(repo_id="lumos-motion/nextjump", filename=f"{current_model}.onnx", repo_type="model", token=os.environ['DATASET_SECRET'])
-        #onnx_file = f'{current_model}.onnx'
-
-        if torch.cuda.is_available():
-            print("Using CUDA")
-            providers = [("CUDAExecutionProvider", {"device_id": torch.cuda.current_device(),
-                                                    "user_compute_stream": str(torch.cuda.current_stream().cuda_stream)})]
-            sess_options = ort.SessionOptions()
-            #sess_options.log_severity_level = 0
-            ort_sess = ort.InferenceSession(onnx_file, sess_options=sess_options, providers=providers)
-        else:
-            print("Using CPU")
-            ort_sess = ort.InferenceSession(onnx_file)
-
-        # warmup inference
-        ort_sess.run(None, {'video': np.zeros((4, 64, 3, IMG_SIZE, IMG_SIZE), dtype=np.float32)})
     if in_video is None:
-        print("No video input provided.")
-        in_video = download_clips(stream_url, os.path.join(os.getcwd(), 'clips'), start_time, end_time, use_60fps=use_60fps)
-    else:  # local uploaded video (still resize with ffmpeg)
-        print("Using uploaded video input.")
-        in_video = download_clips(in_video, os.path.join(os.getcwd(), 'clips'), start_time, end_time, use_60fps=use_60fps)
+        return "No video input provided."
+    in_video = download_clips(in_video, os.path.join(os.getcwd(), 'clips'), '00:00:00', '', use_60fps=use_60fps, use_cuda=use_cuda)
     progress(0, desc="Running inference...")
     has_access = False
     if api_call:
@@ -379,10 +244,6 @@ def inference(in_video, stream_url, start_time, end_time, use_60fps, model_choic
         event_length = int(event_length)
         event_start, event_end = detect_beeps(in_video, event_length)
         print(event_start, event_end)
-        if relay_detection_on:
-            n_jumpers = int(int(event_length) / int(relay_length))
-            relay_starts, relay_ends = detect_relay_beeps(in_video, event_start, int(relay_length), n_jumpers)
-            print(relay_starts, relay_ends)
     
     cap = cv2.VideoCapture(in_video)
     length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
@@ -521,12 +382,6 @@ def inference(in_video, stream_url, start_time, end_time, use_60fps, model_choic
             print(f"Event detected: {event_start} - {event_end}")
             periodicity[:event_start] = 0
             periodicity[event_end:] = 0
-        if relay_detection_on:
-            for start, end in zip(relay_starts, relay_ends):
-                if start > 0 and end > 0:
-                    print(f"Relay Event detected: {start} - {end}")
-                    # immediately after the beep set periodicity to 0 for switch_delay seconds
-                    periodicity[start:start + int(float(switch_delay) * fps)] = 0
     pred_marks_peaks, _ = find_peaks(full_marks, distance=3, height=marks_threshold)
     full_marks_mask = np.zeros(len(full_marks))
     full_marks_mask[pred_marks_peaks] = 1
@@ -672,7 +527,7 @@ def inference(in_video, stream_url, start_time, end_time, use_60fps, model_choic
                 periodicity_str = np.array2string(periodicity, formatter={'float_kind':lambda x: "%.2f" % x}, threshold=np.inf).replace('\n', '')
                 periodLength_str = np.array2string(periodLength, formatter={'float_kind':lambda x: "%.2f" % x}, threshold=np.inf).replace('\n', '')
                 full_marks_str = np.array2string(full_marks, formatter={'float_kind':lambda x: "%.2f" % x}, threshold=np.inf).replace('\n', '')
-                f.write(f"{stream_url}\t{start_time}\t{end_time}\t{beep_detection_on}\t{event_length}\t{periodicity_str}\t{periodLength_str}\t{full_marks_str}\t{count_pred}\t{total_confidence}\n")
+                f.write(f"{in_video}\t{beep_detection_on}\t{event_length}\t{periodicity_str}\t{periodLength_str}\t{full_marks_str}\t{count_pred}\t{total_confidence}\n")
         if count_only_api:
             return f"{count_pred:.2f} (conf: {total_confidence:.2f})"
         else:
@@ -707,9 +562,6 @@ def inference(in_video, stream_url, start_time, end_time, use_60fps, model_choic
             if beep_detection_on:
                 results_dict['event_start'] = event_start
                 results_dict['event_end'] = event_end
-            if relay_detection_on:
-                results_dict['relay_starts'] = relay_starts
-                results_dict['relay_ends'] = relay_ends
             return json.dumps(results_dict)
 
    
@@ -795,12 +647,6 @@ def inference(in_video, stream_url, start_time, end_time, use_60fps, model_choic
     if beep_detection_on:
         # add vertical lines for beep event
         fig.add_vrect(x0=event_start / fps, x1=event_end / fps, fillcolor="LightSalmon", opacity=0.25, layer="below", line_width=0)
-        if relay_detection_on:
-            for start, end in zip(relay_starts, relay_ends):
-                start += 10  # add some padding
-                end -= 10
-                fig.add_vrect(x0=start / fps, x1=end / fps, fillcolor="LightGreen", opacity=0.25, layer="below",
-                              line_width=0)
 
     
     fig.update_layout(legend=dict(
@@ -904,20 +750,12 @@ def inference(in_video, stream_url, start_time, end_time, use_60fps, model_choic
                         histnorm='percent',
                         title="Distribution of jumping speed (jumps-per-second)")
     
-    # make a bar plot of the event type distribution
-
-    bar = px.bar(x=['single rope', 'double dutch', 'double unders', 'single bounces', 'double bounces', 'triple unders', 'other'], 
-                 y=event_type_probs,
-                 template="plotly_dark",
-                 title="Event Type Distribution",
-                 labels={'x': 'event type', 'y': 'probability'},
-                 range_y=[0, 1])
     try:
         os.remove('temp.wav')
     except FileNotFoundError:
         pass
     
-    return count_msg, fig, fig_phase_spiral, fig_phase_spiral_marks, hist, bar
+    return count_msg, fig, fig_phase_spiral, fig_phase_spiral_marks, hist
         
 #css = '#phase-spiral {transform: rotate(0.25turn);}\n#phase-spiral-marks {transform: rotate(0.25turn);}'
 with gr.Blocks() as demo:
@@ -926,30 +764,15 @@ with gr.Blocks() as demo:
                                 width=400, height=400, interactive=True, container=True,
                                 max_length=300)
     with gr.Row():
-        with gr.Column():
-            gr.Markdown(
-                """
-                ### Stream Input Options
-                Either upload a video file above, or provide a stream URL below.
-                """,
-                elem_id='stream-input-options',
-            )
-            in_stream_url = gr.Textbox(label="Stream URL", elem_id='stream-url', visible=True)
-            in_stream_start = gr.Textbox(label="Start Time", elem_id='stream-start', visible=True, value='00:00:00')
-            in_stream_end = gr.Textbox(label="End Time", elem_id='stream-end', visible=True)
         with gr.Column():
             gr.Markdown(
                 """
                 ### Inference Options
-                Select the model and framerate for inference.
+                Select the framerate for inference.
                 """,
                 elem_id='inference-options',
             )
             use_60fps = gr.Checkbox(label="Use 60 FPS", elem_id='use-60fps', visible=True)
-            model_choice =  gr.Dropdown(
-                ["nextjump_speed", "nextjump_all", "nextjump_both_feet"], label="Model Choice", info="For now just speed-only or general model",
-                value="nextjump_speed", elem_id='model-choice'
-            )
         with gr.Column():
             gr.Markdown(
                 """
@@ -960,9 +783,6 @@ with gr.Blocks() as demo:
             )
             beep_detection_on = gr.Checkbox(label="Detect Beeps", elem_id='detect-beeps', visible=True)
             event_length = gr.Textbox(label="Expected Event Length (s)", elem_id='event-length', visible=True)
-            relay_detection_on = gr.Checkbox(label="Relay Event", elem_id='relay-beeps', visible=True)
-            relay_length = gr.Textbox(label="Relay Length (s)", elem_id='relay-length', visible=True, value='30')
-            switch_delay = gr.Textbox(label="Expected Switch Delay (s)", elem_id='event-length', visible=True, value='0.2')
             
     with gr.Row():
         run_button = gr.Button(value="Run", elem_id='run-button', scale=1)
@@ -986,26 +806,21 @@ with gr.Blocks() as demo:
         with gr.Row():
             with gr.Column():
                 out_hist = gr.Plot(label="Speed Histogram", elem_id='output-hist')
-            with gr.Column():
-                out_event_type_dist = gr.Plot(label="Event Type Distribution", elem_id='output-event-type-dist')
               
 
     demo_inference = partial(inference, count_only_api=False, api_key=None)
     
-    run_button.click(demo_inference, [in_video, in_stream_url, in_stream_start, in_stream_end, use_60fps, model_choice, beep_detection_on, event_length, relay_detection_on, relay_length, switch_delay], 
-                     outputs=[out_text, out_plot, out_phase_spiral, out_phase, out_hist, out_event_type_dist])
+    run_button.click(demo_inference, [in_video, use_60fps, beep_detection_on, event_length], 
+                     outputs=[out_text, out_plot, out_phase_spiral, out_phase, out_hist])
     api_inference = partial(inference, api_call=True)
-    api_dummy_button.click(api_inference, [in_video, in_stream_url, in_stream_start, in_stream_end, use_60fps, model_choice, beep_detection_on, event_length, relay_detection_on, relay_length, switch_delay, count_only, api_token], 
+    api_dummy_button.click(api_inference, [in_video, use_60fps, beep_detection_on, event_length, count_only, api_token], 
                            outputs=[period_length], api_name='inference')
     examples = [
-        #['https://hiemdall-dev2.azurewebsites.net/api/clip/clp_vrpWTyjM/mp4', '00:00:00', '00:01:10', True, 60],
-        ['files/wc2023.mp4', '', '00:00:00', '', True, 'nextjump_speed', True, 30, False, '30', '0.2'],
-        #['https://hiemdall-dev2.azurewebsites.net/api/playlist/rec_rd2FAyUo/vod', '01:24:22', '01:25:35', True, 60]
-        #['https://hiemdall-dev2.azurewebsites.net/api/playlist/rec_PY5Ukaua/vod, '00:52:53', '00:55:00', True, 120]
+        ['files/wc2023.mp4', True, True, 30],
     ]
     gr.Examples(examples, 
-                inputs=[in_video, in_stream_url, in_stream_start, in_stream_end, use_60fps, model_choice, beep_detection_on, event_length, relay_detection_on, relay_length, switch_delay], 
-                outputs=[out_text, out_plot, out_phase_spiral, out_phase, out_hist, out_event_type_dist],
+                inputs=[in_video, use_60fps, beep_detection_on, event_length], 
+                outputs=[out_text, out_plot, out_phase_spiral, out_phase, out_hist],
                 fn=demo_inference, cache_examples=False)
 
 

hls_download.py

@@ -1,26 +1,19 @@
 import subprocess
 import os
 
-def download_clips(stream_url, out_dir, start_time, end_time, resize=True, use_60fps=False):
+def download_clips(stream_url, out_dir, start_time, end_time, resize=True, use_60fps=False, use_cuda=True):
     # remove all .mp4 files in out_dir to avoid confusion
     if len(os.listdir(out_dir)) > 5:
         for f in os.listdir(out_dir):
             if f.endswith('.mp4'):
                 os.remove(os.path.join(out_dir, f))
-    os.makedirs(out_dir, exist_ok=True)
     output_file = os.path.join(out_dir, f"train_{len(os.listdir(out_dir))}.mp4")
-    try:
-        os.remove(output_file)
-    except FileNotFoundError:
-        pass
     if resize:  # resize and convert to 30 fps
         ffmpeg_cmd = [
                 'ffmpeg',
-                #'-hwaccel', 'cuda',
-                '-allowed_extensions', 'ALL',
-                '-extension_picky', '0',  # https://github.com/yt-dlp/yt-dlp/issues/12700#issuecomment-2745400091
+                '-hwaccel', 'cuda' if use_cuda else 'none',
                 '-i', stream_url,
-                '-c:v', 'libx264',
+                #'-c:v', 'libx264',
                 '-crf', '23',
                 '-r', '30' if not use_60fps else '60',
                 '-maxrate', '2M',
@@ -38,7 +31,6 @@ def download_clips(stream_url, out_dir, start_time, end_time, resize=True, use_6
         except subprocess.CalledProcessError as e:
             print(f"Error occurred: {e}")
             print(f"ffmpeg output: {e.output}")
-            print(f"ffmpeg stderr: {e.stderr}")
             return output_file
     # else:
     #     os.rename(tmp_file, output_file)