Upload old results

annotation_editor.js

@@ -25,6 +25,7 @@
 
         show_frame();
     }
+    
     window.prev_frame = () => {
         window.frame_index = Math.max(window.frame_index - 1, 0);
         show_frame();
@@ -105,7 +106,9 @@
         prettify_annotation();
     }
     color_from_id = (id) => {
-        return 'hsl(' + Math.floor((id*id)*57 % 360) + ', 100%, 50%)'
+        c = Math.ceil(Math.log2(id)); 
+        hue = (2*id - Math.pow(2,c) - 1) / Math.pow(2,c);
+        return 'hsl(' + Math.floor(hue*359) + ', 100%, 50%)'
     }
 
     window.prettify_annotation = () => {

annotation_handler.py

@@ -6,7 +6,7 @@ import base64
 
 VIDEO_HEIGHT = 700
 
-def load_frames(video_path, preds_path):
+def load_frames(video_path, preds):
     """Load frames for annotation editing
     """
 
@@ -15,11 +15,11 @@ def load_frames(video_path, preds_path):
     didson = dataset.didson
     
     frames = didson.load_frames(start_frame=0)
-    frame_info, h, w = get_frame_info(frames, preds_path)
+    frame_info, h, w = get_frame_info(frames, preds)
     
     return frame_info
 
-def get_frame_info(frames, preds_path):
+def get_frame_info(frames, preds):
     """Get visualized video frames ready for output, given raw ARIS/DIDSON frames.
     Warning: all frames in frames will be stored in memory - careful of OOM errors. Consider processing large files
     in batches, such as in generate_video_batches()
@@ -27,7 +27,6 @@ def get_frame_info(frames, preds_path):
     Returns:
         list(np.ndarray), height (int), width (int)
     """
-    preds = json.load(open(preds_path, 'r'))
     color_map = { fish['id'] : fish['color'] for fish in preds['fish'] }
     
     frame_info = []

app.py

@@ -1,5 +1,5 @@
 import gradio as gr
-from uploader import save_data
+from uploader import save_data, create_data_dir
 from main import predict_task
 from state_handler import load_example_result, reset_state
 from file_reader import File
@@ -11,29 +11,29 @@ import base64
 from bbox import draggable_js
 from annotation_handler import load_frames
 import json
+from zipfile import ZipFile
+import os
 
 max_tabs = 10
 table_headers = ["TOTAL" , "FRAME_NUM", "DIR", "R", "THETA", "L", "TIME", "DATE", "SPECIES"]
 info_headers = [
-    "TOTAL_TIME", "DATE", "START", "END",
-    "TOTAL_FISH", "UPSTREAM_FISH", "DOWNSTREAM_FISH", "NONDIRECTIONAL_FISH", 
-    "TOTAL_FRAMES", "FRAME_RATE", 
-    "UPSTREAM_MOTION", "INTENSITY", "THRESHOLD", "WINDOW_START", "WINDOW_END", "WATER_TEMP"
+    ["TOTAL_TIME", "DATE", "START", "END", "TOTAL_FRAMES", "FRAME_RATE"],
+    ["TOTAL_FISH", "UPSTREAM_FISH", "DOWNSTREAM_FISH", "NONDIRECTIONAL_FISH"],
+    ["UPSTREAM_MOTION", "INTENSITY", "THRESHOLD", "WATER_TEMP"]
 ]
 css = """
+<style>
 #result_json {
     height: 500px;
     overflow: scroll !important;
 }
-#marking_json textarea {
-    height: 100% !important;
-}
-#marking_json label {
-    height: calc(100% - 30px) !important;
+#marking_json thead {
+    display: none !important;
 }
 #canvas {
     align-self: center;
 }
+</style>
 """
 js_update_tabs = """
     async () => {
@@ -43,7 +43,7 @@ js_update_tabs = """
         style_sheet = document.getElementById("tab_style")
         style_sheet.innerHTML = ""
         for (let i = 1; i <= idx; i++) {
-            style_sheet.innerHTML += "button.svelte-kqij2n:nth-child(" + i + "):before {content: 'Result " + i + "';}"
+            style_sheet.innerHTML += "#result_tabs button.svelte-kqij2n:nth-child(" + i + "):before {content: 'Result " + i + "';}"
         }
     }
 """
@@ -106,14 +106,15 @@ def handle_next(_, progress=gr.Progress()):
     upload_file(file_path, "fishcounting", "webapp_uploads/" + file_name)
 
     # Do inference
-    metadata, json_filepath, zip_filepath, video_filepath, marking_filepath = predict_task(file_path, gradio_progress=set_progress)
+    json_result, json_filepath, zip_filepath, video_filepath, marking_filepath = predict_task(file_path, gradio_progress=set_progress)
     
     # Store result for that file
+    result['json_result'].append(json_result)
     result["path_video"].append(video_filepath)
     result["path_zip"].append(zip_filepath)
     result["path_json"].append(json_filepath)
     result["path_marking"].append(marking_filepath)
-    fish_table, fish_info = create_metadata_table(metadata, table_headers, info_headers)
+    fish_table, fish_info = create_metadata_table(json_result, table_headers, info_headers)
     result["fish_table"].append(fish_table)
     result["fish_info"].append(fish_info)
 
@@ -146,6 +147,17 @@ def show_data():
     # Check if inference is done
     not_done = state['index'] < state['total']
 
+
+
+    frame_info = load_frames("static/example/input_file.aris", result['json_result'])
+
+    annotation_html = "<div style='display:flex'>"
+    annotation_html += "<canvas id='canvas' style='width:50%' onmousedown='mouse_down(event)' onmousemove='mouse_move(event)' onmouseup='mouse_up()' onmouseleave='mouse_up()'></canvas>"
+    annotation_html += "<div id='annotation_display' style='width:50%'></div>"
+    annotation_html += "</div>"
+    annotation_html += "<p id='annotation_info' style='display:none'>" + json.dumps(frame_info) + "</p>"
+    annotation_html += "<img id='annotation_img' onload='draw()' style='display:none'></img>"
+
     # Send update to UI, and to inference_handler to start next file inference
     return {
         zip_out: gr.update(value=result["path_zip"]),
@@ -153,83 +165,111 @@ def show_data():
         tabs[i]['video']: gr.update(value=result["path_video"][i], visible=True),
         tabs[i]['metadata']: gr.update(value=result["fish_info"][i], visible=True),
         tabs[i]['table']: gr.update(value=result["fish_table"][i], visible=True),
+        tabs[i]['annotation']: gr.update(value=annotation_html, visible=True),
         tab_parent: gr.update(selected=i),
         inference_handler: gr.update(value = str(np.random.rand()), visible=not_done)
     }
 
 
+    
 
-def init_annotation():
+def preview_result(zip_info):
+    zip_name = zip_info[0]
+    print(zip_name)
+    dir_name = create_data_dir()
+    with ZipFile(zip_name) as zip_file:
+        ZipFile.extractall(zip_file, path=dir_name)
 
-    frame_info = load_frames("static/example/input_file.aris", "static/example/640 2018-07-09_115439_results.json")
+    unzipped = os.listdir(dir_name)
+    print(unzipped)
 
-    html = "<div style='display:flex'>"
-    html += "<canvas id='canvas' style='width:50%' onmousedown='mouse_down(event)' onmousemove='mouse_move(event)' onmouseup='mouse_up()' onmouseleave='mouse_up()'></canvas>"
-    html += "<div id='annotation_display' style='width:50%'></div>"
-    html += "</div>"
-    html += "<p id='annotation_info' style='display:none'>" + json.dumps(frame_info) + "</p>"
-    html += "<img id='annotation_img' onload='draw()' style='display:none'></img>"
-    return html
+    reset_state(result, state)
+    state["index"] = 1
+    for file in unzipped:
+        if (file.endswith("_results.mp4")):
+            result["path_video"].append(os.path.join(dir_name, file))
+        elif (file.endswith("_results.json")):
+            result["path_json"].append(os.path.join(dir_name, file))
+        elif (file.endswith("_marking.txt")):
+            result["path_marking"].append(os.path.join(dir_name, file))
     
-
+    with open(result['path_json'][0]) as f:
+        json_result = json.load(f)
+        result['json_result'] = json_result
+    fish_table, fish_info = create_metadata_table(json_result, table_headers, info_headers)
+    result["fish_table"].append(fish_table)
+    result["fish_info"].append(fish_info)
+    
+    return {
+        result_handler: gr.update(value = str(state["index"])),
+        inference_handler: gr.update()
+    }
 
 
 
 demo = gr.Blocks()
 with demo:
-    with gr.Blocks(css=css) as inner_body:
+    with gr.Blocks() as inner_body:
 
         # Title of page
         gr.HTML(
             """
-        <h1 align="center" style="font-size:xxx-large">Caltech Fisheye</h1>
-        <p align="center">Submit an .aris file to analyze result.</p>
-        <style id="tab_style"></style>
+            <h1 align="center" style="font-size:xxx-large">Caltech Fisheye</h1>
+            """ + css + """
+            <style id="tab_style"></style>
         """
         )
 
-        annotation_editor = gr.HTML("""""")
-        with open('annotation_editor.js', 'r') as f:
-            js = f.read()
-            print(js)
-            annotation_editor.change(lambda x: gr.update(), None, annotation_editor, _js=js)
-        gr.Button("Edit Annotation").click(init_annotation, None, annotation_editor)
 
     
-        
-        #Input field for aris submission
-        input = File(file_types=[".aris", ".ddf"], type="binary", label="ARIS Input", file_count="multiple")
+        with gr.Tabs():
+            with gr.Tab("Infer ARIS"):
+                gr.HTML("<p align='center' style='font-size: large;font-style: italic;'>Submit an .aris file to analyze result.</p>")
+                #Input field for aris submission
+                input = File(file_types=[".aris", ".ddf"], type="binary", label="ARIS Input", file_count="multiple")
+
+                # Dummy element to call inference events, this also displays the inference progress
+                inference_handler = gr.Text(value=str(np.random.rand()), visible=False)
+
+                # Zip file output
+                zip_out = gr.File(label="ZIP Output", interactive=False)
+
+            with gr.Tab("Review Results"):
+                # Title of page
+                gr.HTML("<p align='center' style='font-size: large;font-style: italic;'>Submit an old zip file of results to visualize.</p>")
+                result_input = File(file_types=[".zip"], type="binary", label="Upload result file")
+                preview_result_btn = gr.Button("Preview Result")
 
-        # Dummy element to call inference events, this also displays the inference progress
-        inference_handler = gr.Text(value=str(np.random.rand()), visible=False)
 
         # Dummy element to call UI events
         result_handler = gr.Text(value="LOADING", visible=False, elem_id="result_handler")
 
         # List of all UI components that will recieve outputs from the result_handler
         UI_components = []
-
-        # Zip file output
-        zip_out = gr.File(label="ZIP Output", interactive=False)
         UI_components.append(zip_out)
 
         # Create result tabs
         tabs = []
-        with gr.Tabs() as tab_parent:
+        with gr.Tabs(elem_id="result_tabs") as tab_parent:
             UI_components.append(tab_parent)
             for i in range(max_tabs):
                 with gr.Tab(label="", id=i, elem_id="result_tab"+str(i)) as tab:
-                    with gr.Row():
-                        metadata_out = gr.JSON(label="Info", visible=False, elem_id="marking_json")
-                        video_out = gr.Video(label='Annotated Video', interactive=False, visible=False)
+                    metadata_out = gr.Matrix(label="Info", interactive=False, headers=[""]*6, datatype="markdown", visible=False, elem_id="marking_json")
                     table_out = gr.Matrix(label='Indentified Fish', headers=table_headers, interactive=False, visible=False)
+                    video_out = gr.Video(label='Annotated Video', interactive=False, visible=False)
+                    annotation_editor = gr.HTML("""""", visible=False)
+
+                    with open('annotation_editor.js', 'r') as f:
+                        js = f.read()
+                        annotation_editor.change(lambda x: gr.update(), None, annotation_editor, _js=js)
                     tabs.append({
                         'tab': tab,
                         'metadata': metadata_out,
                         'video': video_out,
-                        'table': table_out
+                        'table': table_out,
+                        'annotation': annotation_editor
                     })
-                    UI_components.extend([tab, metadata_out, video_out, table_out])
+                    UI_components.extend([tab, metadata_out, video_out, table_out, annotation_editor])
 
     # Button to show example result
     #gr.Button(value="Show Example Result").click(show_example_data, None, result_handler)
@@ -253,6 +293,9 @@ with demo:
 
     # Send UI changes based on the new results to the UI_components, and tell the inference_handler to start next inference
     result_handler.change(show_data, None, UI_components + [inference_handler], _js=js_update_tabs)
+    
+    # Button to load a previous result and view visualization
+    preview_result_btn.click(preview_result, result_input, [result_handler, inference_handler])
 
 demo.queue().launch()
 

aris.py

@@ -13,6 +13,7 @@ import json
 import pytz
 from copy import deepcopy
 from multiprocessing import Pool
+import math
 
 import pyARIS
 from tracker import Tracker
@@ -257,13 +258,11 @@ def prep_for_mm(json_data):
     return json_data
 
 
-def create_metadata_dictionary(aris_fp, json_fp, beam_width_dir=BEAM_WIDTH_DIR):
+def add_metadata_to_result(aris_fp, json_data, beam_width_dir=BEAM_WIDTH_DIR):
     """
     Return:
         dictionary, for manual marking
     """
-    with open(json_fp) as json_file:
-        json_data = json.load(json_file)
 
     metadata = {}
     metadata["FILE_NAME"] = aris_fp
@@ -319,7 +318,7 @@ def create_metadata_dictionary(aris_fp, json_fp, beam_width_dir=BEAM_WIDTH_DIR):
 
     metadata["COUNT_FILE_NAME"] = 'N/A'
     metadata["EDITOR_ID"] = 'N/A'
-    metadata["INTENSITY"] = f'{round(frame.threshold, 1):.1f} dB' # Missing
+    metadata["INTENSITY"] = f'{round(frame.intensity, 1):.1f} dB' # Missing
     metadata["THRESHOLD"] = f'{round(frame.threshold, 1):.1f} dB' # Missing
     metadata["WINDOW_START"] = round(frame.windowstart, 2)
     metadata["WINDOW_END"] = round(frame.windowstart + frame.windowlength, 2)
@@ -404,7 +403,9 @@ def create_metadata_dictionary(aris_fp, json_fp, beam_width_dir=BEAM_WIDTH_DIR):
         fish_entry['LATITUDE'] = frame.latitude or 'N 00 d  0.00000 m'
         fish_entry['LONGITUDE'] = frame.longitude or 'E 000 d  0.00000 m'
         fish_entry['PAN'] = frame.sonarpan
+        if math.isnan(fish_entry['PAN']): fish_entry['PAN'] = "nan"
         fish_entry['TILT'] = frame.sonartilt
+        if math.isnan(fish_entry['TILT']): fish_entry['TILT'] = "nan"
         fish_entry['ROLL'] = frame.roll # May be wrong number but sonarroll was NaN
         fish_entry['SPECIES'] = 'Unknown'
         fish_entry['MOTION'] = 'Running <-->'
@@ -424,9 +425,17 @@ def create_metadata_dictionary(aris_fp, json_fp, beam_width_dir=BEAM_WIDTH_DIR):
     metadata["START"] = start_time
     metadata["END"] = end_time
 
-    return metadata
+    json_data['metadata'] = metadata
 
-def create_metadata_table(metadata, table_headers, info_headers):
+    return json_data
+
+def create_metadata_table(result, table_headers, info_headers):
+    if 'metadata' in result:
+        metadata = result['metadata']
+    else:
+        metadata = { 'FISH': [] }
+    
+    # Create fish table
     table = []
     for fish in metadata["FISH"]:
         row = []
@@ -440,17 +449,38 @@ def create_metadata_table(metadata, table_headers, info_headers):
             row.append("-")
         table.append(row)
 
-    info = {}
-    for header in info_headers:
-        info[header] = metadata[header]
-
+    # Create info table
+    stacked_info = []
+    max_col = 0
+    for column in info_headers:
+        column_res = []
+        for field in column:
+            if field in metadata:
+                column_res.append([field, metadata[field]])
+        stacked_info.append(column_res)
+        if len(column_res) > max_col:
+            max_col = len(column_res)
+    
+    info = []
+    for i in range(max_col):
+        row = []
+        for column in stacked_info:
+            if i < len(column):
+                entry = column[i]
+                row.append("**" + entry[0] + "**")
+                row.append(str(entry[1]))
+            else: 
+                row.append(" ")
+                row.append(" ")
+        info.append(row)
     return table, info
 
-def create_manual_marking(metadata, out_path=None):
+def create_manual_marking(results, out_path=None):
     """
     Return:
         string, full contents of manual marking
     """
+    metadata = results['metadata']
 
     s = f'''
 Total Fish       =      {metadata["TOTAL_FISH"]}
@@ -480,7 +510,14 @@ File  Total  Frame#  Dir   R (m)   Theta   L(cm)  dR(cm)    L/dR  Aspect    Time
 ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 '''
     
-    for entry in metadata["FISH"]:
+    for fish in metadata["FISH"]:
+        entry = {}
+        for field in fish.keys():
+            if type(fish[field]) == str: 
+                entry[field] = math.nan
+            else:
+                entry[field] = fish[field]
+
         s += f'{entry["FILE"]:>4}  {entry["TOTAL"]:>5}  {entry["FRAME_NUM"]:>6}  {entry["DIR"]:>3}  {entry["R"]:>6.2f}  {entry["THETA"]:>6.1f}  {entry["L"]:>6.1f}  {entry["DR"]:>6.1f}  {entry["LDR"]:>6.2f}  {entry["ASPECT"]:>6.1f}  {entry["TIME"]:>8}  {entry["DATE"]:>10}  {entry["LATITUDE"]:>17}  {entry["LONGITUDE"]:>18}  {entry["PAN"]:>7.2f}  {entry["TILT"]:>7.2f}  {entry["ROLL"]:>7.2f}  {entry["SPECIES"]:>8}  {entry["MOTION"]:>37}  {entry["Q"]:>5}  {entry["N"]:>2}  {entry["COMMENT"]}\n'
 
     s += f'''

main.py

@@ -2,7 +2,7 @@ import os
 import torch
 from zipfile import ZipFile
 
-from aris import create_manual_marking, BEAM_WIDTH_DIR, create_metadata_dictionary, prep_for_mm
+from aris import create_manual_marking, BEAM_WIDTH_DIR, add_metadata_to_result, prep_for_mm
 from dataloader import create_dataloader_aris
 from inference import do_full_inference, json_dump_round_float
 from visualizer import generate_video_batches
@@ -52,17 +52,16 @@ def predict_task(filepath, weights=WEIGHTS, gradio_progress=None):
 
     # re-index results if desired - this should be done before writing the file
     results = prep_for_mm(results)
+    results = add_metadata_to_result(filepath, results)
     
     # write output to disk
     json_dump_round_float(results, results_filepath)
     
-    metadata = None
     if dataset.didson.info['version'][3] == 5: # ARIS only
-        metadata = create_metadata_dictionary(filepath, results_filepath)
-        create_manual_marking(metadata, out_path=marking_filepath)
+        create_manual_marking(results, out_path=marking_filepath)
     
     # generate a video with tracking results
-    generate_video_batches(dataset.didson, results_filepath, frame_rate, video_filepath, 
+    generate_video_batches(dataset.didson, results, frame_rate, video_filepath, 
                    image_meter_width=image_meter_width, image_meter_height=image_meter_height, gp=gradio_progress)
     
     # zip up the results
@@ -74,4 +73,4 @@ def predict_task(filepath, weights=WEIGHTS, gradio_progress=None):
     # release GPU memory 
     torch.cuda.empty_cache()
     
-    return metadata, results_filepath, zip_filepath, video_filepath, marking_filepath
+    return results, results_filepath, zip_filepath, video_filepath, marking_filepath

state_handler.py

@@ -363,6 +363,7 @@ def load_example_result(result, table_headers, info_headers):
 def reset_state(result, state):
 
     # Reset Result
+    result["json_result"] = []
     result["path_video"] = []
     result["path_zip"] = []
     result["path_json"] = []

visualizer.py

@@ -20,18 +20,21 @@ BOX_THICKNESS = 2
 FONT_SCALE = 0.65
 FONT_THICKNESS = 1
 
-def generate_video_batches(didson, preds_path, frame_rate, video_out_path, gp=None, image_meter_width=None, image_meter_height=None, batch_size=1000):
+def generate_video_batches(didson, preds, frame_rate, video_out_path, gp=None, image_meter_width=None, image_meter_height=None, batch_size=1000):
     """Write a visualized video to video_out_path, given a didson object.
     """
     if (gp): gp(0, "Generating results video...")
     end_frame = didson.info['endframe'] or didson.info['numframes']
     out = None # need to wait til we have height and width to instantiate video file
+
+    print(preds)
+    print(preds.keys())
     
     with tqdm(total=end_frame, desc="Generating results video", ncols=0) as pbar:
         for i in range(0, end_frame, batch_size):
             batch_end = min(end_frame, i+batch_size)
             frames = didson.load_frames(start_frame=i, end_frame=batch_end)
-            vid_frames, h, w = get_video_frames(frames, preds_path, frame_rate, image_meter_width, image_meter_height, start_frame=i)
+            vid_frames, h, w = get_video_frames(frames, preds, frame_rate, image_meter_width, image_meter_height, start_frame=i)
 
             if out is None:
                 out = cv2.VideoWriter(video_out_path, cv2.VideoWriter_fourcc(*'mp4v'), frame_rate, [ int(1.5*w), h ] )
@@ -46,7 +49,7 @@ def generate_video_batches(didson, preds_path, frame_rate, video_out_path, gp=No
     
     out.release()
     
-def get_video_frames(frames, preds_path, frame_rate, image_meter_width=None, image_meter_height=None, start_frame=0):
+def get_video_frames(frames, preds, frame_rate, image_meter_width=None, image_meter_height=None, start_frame=0):
     """Get visualized video frames ready for output, given raw ARIS/DIDSON frames.
     Warning: all frames in frames will be stored in memory - careful of OOM errors. Consider processing large files
     in batches, such as in generate_video_batches()
@@ -54,7 +57,6 @@ def get_video_frames(frames, preds_path, frame_rate, image_meter_width=None, ima
     Returns:
         list(np.ndarray), height (int), width (int)
     """
-    preds = json.load(open(preds_path, 'r'))
     pred_lengths = { fish['id'] : "%.2fm" % fish['length'] for fish in preds['fish'] }
     clip_pr_counts = Tracker.count_dirs(preds)
     color_map = { fish['id'] : fish['color'] for fish in preds['fish'] }