update

app.py

@@ -23,8 +23,6 @@ sys.path.append(".")
 from config_parser import *
 from inferencer import *
 
-
-
 gr.close_all()
 
 
@@ -91,6 +89,7 @@ def create_interface(task_number, product, model_number):
     model_name = (
         f'Résultst of {model_info["name"]}' if "name" in model_info else "Résultats"
     )
+    model_uuid = model_info["path"].split("/")[-2:-1][0]
     task_info = config["tasks"][f"task{task_number+1}"]
     product_examples = (
         [
@@ -111,7 +110,7 @@ def create_interface(task_number, product, model_number):
     created_interface = gr.Interface(
         title=title,
         description=description,
-        fn=inferencer_arr[f"task{task_number+1}"][product][str(model_number)][
+        fn=inferencer_arr[f"task{task_number+1}"][product][str(model_number)][model_uuid][
             "function"
         ],
         # fn=lambda x: x,
@@ -211,5 +210,5 @@ if __name__ == "__main__":
     iface_generated.launch(
         server_name="0.0.0.0"
     )
-    shutil.rmtree("examples")
+    # shutil.rmtree("examples")
     logging.log(level=logging.INFO, msg="Stopping the Gradio server...")

benchmarker.py

@@ -0,0 +1,85 @@
+"""
+Author : Bastien GUILLAUME
+Version : 0.0.1
+Date : 2023-03-16
+
+Title : Benchmark ONNX model from a config file made for gradio_interfacer
+"""
+
+import os
+
+from config_parser import *
+from inferencer import *
+from pathlib import Path
+
+def format_examples(task_number, product, product_example):
+    response = requests.get(product_example)
+    examples_folder = Path(f"examples/{product}")
+    os.makedirs(examples_folder, exist_ok=True)
+    filepath = Path(examples_folder / f'{product_example.split("/")[-1]}')
+    if filepath.exists():
+        pass
+    else:
+        with open(filepath, "wb") as f:
+            f.write(response.content)
+    return [f"task{task_number+1}", product, filepath]
+
+def benchmark_models(task_number, product):
+    logging.log(level=logging.INFO, msg=f"Entering benchmark_models")
+    models_to_benchamrk = config["tasks"][f"task{task_number+1}"]["models"][product]
+    number_of_model = len(models_to_benchamrk)
+    task_info = config["tasks"][f"task{task_number+1}"]
+    result = []
+    product_examples = (
+        [
+            format_examples(task_number, product, product_example)
+            for product_example in task_info["examples"][product]
+        ]
+        if "examples" in task_info and product in task_info["examples"]
+        else []
+    )
+    for model in models_to_benchamrk:
+        for product_example in product_examples:
+            result.append(inference(task_number, product, product_example, number_of_model))
+    return result
+
+benchmark_builder_list = []
+benchmark_builder_dict = {}
+logging.log(level=logging.INFO, msg=f"Building Interfaces")
+logging.log(level=logging.INFO, msg=f"Number of task(s) : {len(tasks)}")
+for task_number in range(0, len(tasks)):
+        logging.log(level=logging.INFO, msg=f"Treating task n°{task_number+1}")
+        benchmark_builder_dict[tasks[task_number]] = {}
+        product_list = list(config["tasks"][f"task{task_number+1}"]["models"].keys())
+        logging.log(level=logging.DEBUG, msg=f"Products : {product_list}")
+        benchmark_builder_product_level_list = []
+        for product in product_list:
+            logging.log(level=logging.INFO, msg=f"Product : {product}")
+            benchmark_builder_dict[tasks[task_number]][product] = []
+            if len(config["tasks"][f"task{task_number+1}"]["models"][product]) > 1:
+                generated_parralel_interface = benchmark_models(
+                    task_number, product
+                )
+                benchmark_builder_dict[tasks[task_number]][product].append(
+                    generated_parralel_interface
+                )
+                benchmark_builder_product_level_list.append(
+                    generated_parralel_interface
+                )
+            else:
+                generated_interface = create_interface(
+                    task_number=task_number, product=product, model_number=0
+                )
+                benchmark_builder_dict[tasks[task_number]][product].append(
+                    generated_interface
+                )
+                benchmark_builder_product_level_list.append(generated_interface)
+        benchmark_builder_list.append(
+            gr.TabbedInterface(
+                interface_list=benchmark_builder_product_level_list,
+                tab_names=product_list,
+            )
+        )
+    logging.log(level=logging.INFO, msg=f"Interfaces ready\n")
+    logging.log(level=logging.DEBUG, msg=f"Interfaces List {benchmark_builder_list}")
+    # logging.log(level=logging.INFO, msg=f"Interfaces Dict {interface_dict}")

build_image.sh

@@ -0,0 +1,8 @@
+#!/bin/bash -e
+image_name=gcr.io/tough-variety-310920/openvino_inference
+image_tag=1.0.1
+full_image_name=${image_name}:${image_tag}
+
+cd "$(dirname "$0")" 
+docker build -t "${full_image_name}" .
+docker push "$full_image_name"

inferencer.py

@@ -9,6 +9,8 @@ Date : 2023-03-16
 Title : Inference With Gradio running an onnxruntime backend
 """
 
+from pathlib import Path
+
 import numpy as np
 import onnxruntime as ort
 import requests
@@ -27,7 +29,9 @@ def make_func(task, product, model_number):
         Returns:
             - String including label and confidence of the model
         """
-        input_image = pre_process_all(task=task, product=product, model_number=model_number, image=image)
+        input_image = pre_process_all(
+            task=task, product=product, model_number=model_number, image=image
+        )
         result = inference(task, product, input_image, model_number=model_number)
         logging.log(level=logging.DEBUG, msg=result)
         return result
@@ -35,6 +39,22 @@ def make_func(task, product, model_number):
     return _analysis
 
 
+def download_models(product, model, model_uuid):
+    logging.log(level=logging.DEBUG, msg=model)
+    response = requests.get(model, stream=True).content
+    models_folder = Path(f"models/{product}/{model_uuid}")
+    os.makedirs(models_folder, exist_ok=True)
+    filepath = Path(models_folder / f'{model.split("/")[-1]}')
+    logging.log(level=logging.DEBUG, msg=filepath)
+    if filepath.exists():
+        pass
+    else:
+        with open(filepath, "xb") as f:
+            f.write(response)
+    # return [f"task{task_number+1}", product, filepath]
+    return filepath
+
+
 corck_screwing_metadata = {
     "image_threshold": 0.9247307181358337,
     "pixel_threshold": 0.9247307181358337,
@@ -52,26 +72,34 @@ for task in config["tasks"].keys():
         for model_number in range(len(config["tasks"][task]["models"][product])):
             model = config["tasks"][task]["models"][product][model_number]
             model_path = model["path"]
+            model_uuid = model_path.split("/")[-2:-1][0]
             logging.log(
                 level=logging.INFO,
                 msg=f"Loading model for product {product}, version {model_number}",
             )
+            logging.log(
+                level=logging.INFO,
+                msg=f"Model UUID {model_uuid}",
+            )
             inferencer_arr[task][product][str(model_number)] = {}
             if model_path.startswith("http"):
-                r = requests.get(model_path, stream=True).content
-            inferencer_arr[task][product][str(model_number)][
+                # r = requests.get(model_path, stream=True).content
+                model_path = download_models(product, model_path, model_uuid)
+            inferencer_arr[task][product][str(model_number)][model_uuid] = {}
+            inferencer_arr[task][product][str(model_number)][model_uuid][
                 "model"
-            ] = ort.InferenceSession(r if model_path.startswith("http") else model_path)
-            inferencer_arr[task][product][str(model_number)]["function"] = make_func(
+            # ] = ort.InferenceSession(r if model_path.startswith("http") else model_path)
+            ] = ort.InferenceSession(model_path.as_posix())
+            inferencer_arr[task][product][str(model_number)][model_uuid]["function"] = make_func(
                 task, product, model_number
             )
-            inferencer_arr[task][product][str(model_number)]["input_name"] = (
-                inferencer_arr[task][product][str(model_number)]["model"]
+            inferencer_arr[task][product][str(model_number)][model_uuid]["input_name"] = (
+                inferencer_arr[task][product][str(model_number)][model_uuid]["model"]
                 .get_inputs()[0]
                 .name
             )
-            inferencer_arr[task][product][str(model_number)]["output_name"] = (
-                inferencer_arr[task][product][str(model_number)]["model"]
+            inferencer_arr[task][product][str(model_number)][model_uuid]["output_name"] = (
+                inferencer_arr[task][product][str(model_number)][model_uuid]["model"]
                 .get_outputs()[0]
                 .name
             )
@@ -155,10 +183,11 @@ def inference(task, product, input_image, model_number):
     logging.log(level=logging.INFO, msg=f"Task {task}")
     logging.log(level=logging.INFO, msg=f"Product {product}")
     logging.log(level=logging.INFO, msg=f"Model {model_number}")
+    model_uuid = config["tasks"][task]["models"][product][int(model_number)]["path"].split("/")[-2:-1][0]
     result = "Algorithm not yet supported"
-    prediction = inferencer_arr[task][product][str(model_number)]["model"].run(
-        [inferencer_arr[task][product][str(model_number)]["output_name"]],
-        {inferencer_arr[task][product][str(model_number)]["input_name"]: input_image},
+    prediction = inferencer_arr[task][product][str(model_number)][model_uuid]["model"].run(
+        [inferencer_arr[task][product][str(model_number)][model_uuid]["output_name"]],
+        {inferencer_arr[task][product][str(model_number)][model_uuid]["input_name"]: input_image},
     )
     prediction = prediction[0].squeeze()
     model_type = config["tasks"][task]["models"][product][int(model_number)]["type"]