Add MVP localizer, quiet logging to stdout, config aliasing, and updated documentation

.github/workflows/docker-publish.yaml

@@ -73,6 +73,7 @@ jobs:
         run: |
           docker buildx build \
               -t wildme/scoutbot:${{ env.IMAGE_TAG }} \
+              -t wildme/scoutbot:latest \
               --platform linux/amd64 \
               --push \
               .

Dockerfile

@@ -22,4 +22,4 @@ RUN pip3 install --no-cache-dir -r requirements.txt \
  && pip3 uninstall -y onnxruntime \
  && pip3 install onnxruntime-gpu
 
-CMD python3 app.py
+CMD python3 app2.py

README.rst

@@ -138,7 +138,7 @@ There is Sphinx documentation in the ``docs/`` folder, which can be built by run
 Logging
 -------
 
-The script uses Python's built-in logging functionality called ``logging``.  All print functions are replaced with :func:``log.info``, which sends the output to two places:
+The script uses Python's built-in logging functionality called ``logging``.  All print functions are replaced with ``log.info()``, which sends the output to two places:
 
     - 1. the terminal window, and
     - 2. the file `scoutbot.log`

app.py

@@ -1,4 +1,6 @@
 # -*- coding: utf-8 -*-
+import time
+
 import cv2
 import gradio as gr
 import numpy as np
@@ -6,18 +8,29 @@ import numpy as np
 from scoutbot import loc, wic
 
 
-def predict(filepath, wic_thresh, loc_thresh, nms_thresh):
+def predict(filepath, config, wic_thresh, loc_thresh, nms_thresh):
+    start = time.time()
+
+    if config == 'MVP':
+        config = 'mvp'
+    elif config == 'Phase 1':
+        config = 'phase1'
+    else:
+        raise ValueError()
+
     wic_thresh /= 100.0
     loc_thresh /= 100.0
     nms_thresh /= 100.0
 
+    nms_thresh = 1.0 - nms_thresh
+
     # Load data
     img = cv2.imread(filepath)
     img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
-    inputs = [filepath]
 
     # Run WIC
-    outputs = wic.post(wic.predict(wic.pre(inputs)))
+    inputs = [filepath]
+    outputs = wic.post(wic.predict(wic.pre(inputs, config=config)))
 
     # Get WIC confidence
     output = outputs[0]
@@ -28,7 +41,9 @@ def predict(filepath, wic_thresh, loc_thresh, nms_thresh):
 
         # Run Localizer
         outputs = loc.post(
-            loc.predict(loc.pre(inputs)), loc_thresh=loc_thresh, nms_thresh=nms_thresh
+            loc.predict(loc.pre(inputs, config=config)),
+            loc_thresh=loc_thresh,
+            nms_thresh=nms_thresh,
         )
 
         # Format and render results
@@ -50,7 +65,11 @@ def predict(filepath, wic_thresh, loc_thresh, nms_thresh):
                 loc_detections.append(f'{label}: {conf:0.04f}')
     loc_detections = '\n'.join(loc_detections)
 
-    return img, wic_confidence, loc_detections
+    end = time.time()
+    duration = end - start
+    speed = f'{duration:0.02f} seconds)'
+
+    return img, speed, wic_confidence, loc_detections
 
 
 interface = gr.Interface(
@@ -58,26 +77,33 @@ interface = gr.Interface(
     title='Wild Me Scout - Tile ML Demo',
     inputs=[
         gr.Image(type='filepath'),
-        gr.Slider(label='WIC Confidence Threshold', value=20),
-        gr.Slider(label='Localizer Confidence Threshold', value=48),
-        gr.Slider(label='Localizer NMS Threshold', value=20),
+        gr.Radio(
+            label='Model Configuration',
+            type='value',
+            choices=['Phase 1', 'MVP'],
+            value='MVP',
+        ),
+        gr.Slider(label='WIC Confidence Threshold', value=7),
+        gr.Slider(label='Localizer Confidence Threshold', value=14),
+        gr.Slider(label='Localizer NMS Threshold', value=80),
     ],
     outputs=[
         gr.Image(type='numpy'),
+        gr.Textbox(label='Prediction Speed', interactive=False),
         gr.Number(label='Predicted WIC Confidence', precision=5, interactive=False),
         gr.Textbox(label='Predicted Localizer Detections', interactive=False),
     ],
     examples=[
-        ['examples/07a4b8db-f31c-261d-4580-e9402768fd45.true.jpg', 20, 48, 20],
-        ['examples/15e815d9-5aad-fa53-d1ed-33429020e15e.true.jpg', 10, 48, 20],
-        ['examples/1bb79811-3149-7a60-2d88-613dc3eeb261.true.jpg', 20, 48, 20],
-        ['examples/1e8372e4-357d-26e6-d7fd-0e0ae402463a.true.jpg', 20, 48, 20],
-        ['examples/201bc65e-d64e-80d3-2610-5865a22d04b4.false.jpg', 20, 48, 20],
-        ['examples/3affd8b6-9722-f2d5-9171-639615b4c38f.true.jpg', 20, 48, 20],
-        ['examples/4aedb818-f2f4-e462-8b75-5c8e34a01a59.false.jpg', 20, 48, 20],
-        ['examples/474bc2b6-dc51-c1b5-4612-efe810bbe091.true.jpg', 20, 48, 20],
-        ['examples/c3014107-3464-60b5-e04a-e4bfafdf8809.false.jpg', 20, 48, 20],
-        ['examples/f835ce33-292a-9116-794e-f8859b5956ec.true.jpg', 20, 48, 20],
+        ['examples/07a4b8db-f31c-261d-4580-e9402768fd45.true.jpg', 'MVP', 7, 14, 80],
+        ['examples/15e815d9-5aad-fa53-d1ed-33429020e15e.true.jpg', 'MVP', 7, 14, 80],
+        ['examples/1bb79811-3149-7a60-2d88-613dc3eeb261.true.jpg', 'MVP', 7, 14, 80],
+        ['examples/1e8372e4-357d-26e6-d7fd-0e0ae402463a.true.jpg', 'MVP', 7, 14, 80],
+        ['examples/201bc65e-d64e-80d3-2610-5865a22d04b4.false.jpg', 'MVP', 7, 14, 80],
+        ['examples/3affd8b6-9722-f2d5-9171-639615b4c38f.true.jpg', 'MVP', 7, 14, 80],
+        ['examples/4aedb818-f2f4-e462-8b75-5c8e34a01a59.false.jpg', 'MVP', 7, 14, 80],
+        ['examples/474bc2b6-dc51-c1b5-4612-efe810bbe091.true.jpg', 'MVP', 7, 14, 80],
+        ['examples/c3014107-3464-60b5-e04a-e4bfafdf8809.false.jpg', 'MVP', 7, 14, 80],
+        ['examples/f835ce33-292a-9116-794e-f8859b5956ec.true.jpg', 'MVP', 7, 14, 80],
     ],
     cache_examples=True,
     allow_flagging='never',

app2.py

@@ -8,15 +8,27 @@ import numpy as np
 import scoutbot
 
 
-def predict(filepath, wic_thresh, loc_thresh, agg_thresh, loc_nms_thresh, agg_nms_thresh):
+def predict(
+    filepath, config, wic_thresh, loc_thresh, agg_thresh, loc_nms_thresh, agg_nms_thresh
+):
     start = time.time()
 
+    if config == 'MVP':
+        config = 'mvp'
+    elif config == 'Phase 1':
+        config = 'phase1'
+    else:
+        raise ValueError()
+
     wic_thresh /= 100.0
     loc_thresh /= 100.0
     loc_nms_thresh /= 100.0
     agg_thresh /= 100.0
     agg_nms_thresh /= 100.0
 
+    loc_nms_thresh = 1.0 - loc_nms_thresh
+    agg_nms_thresh = 1.0 - agg_nms_thresh
+
     # Load data
     img = cv2.imread(filepath)
     img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
@@ -26,7 +38,13 @@ def predict(filepath, wic_thresh, loc_thresh, agg_thresh, loc_nms_thresh, agg_nm
     megapixels = pixels / 1e6
 
     wic_, detects = scoutbot.pipeline(
-        filepath, wic_thresh, loc_thresh, loc_nms_thresh, agg_thresh, agg_nms_thresh
+        filepath,
+        config=config,
+        wic_thresh=wic_thresh,
+        loc_thresh=loc_thresh,
+        loc_nms_thresh=loc_nms_thresh,
+        agg_thresh=agg_thresh,
+        agg_nms_thresh=agg_nms_thresh,
     )
 
     output = []
@@ -52,7 +70,7 @@ def predict(filepath, wic_thresh, loc_thresh, agg_thresh, loc_nms_thresh, agg_nm
     speed = duration / megapixels
     speed = f'{speed:0.02f} seconds per megapixel (total: {megapixels:0.02f} megapixels, {duration:0.02f} seconds)'
 
-    return img, speed, output
+    return img, speed, wic_, output
 
 
 interface = gr.Interface(
@@ -60,28 +78,35 @@ interface = gr.Interface(
     title='Wild Me Scout - Image ML Demo',
     inputs=[
         gr.Image(type='filepath'),
-        gr.Slider(label='WIC Confidence Threshold', value=20),
-        gr.Slider(label='Localizer Confidence Threshold', value=48),
+        gr.Radio(
+            label='Model Configuration',
+            type='value',
+            choices=['Phase 1', 'MVP'],
+            value='MVP',
+        ),
+        gr.Slider(label='WIC Confidence Threshold', value=7),
+        gr.Slider(label='Localizer Confidence Threshold', value=14),
         gr.Slider(label='Aggregation Confidence Threshold', value=51),
-        gr.Slider(label='Localizer NMS Threshold', value=20),
-        gr.Slider(label='Aggregation NMS Threshold', value=20),
+        gr.Slider(label='Localizer NMS Threshold', value=80),
+        gr.Slider(label='Aggregation NMS Threshold', value=80),
     ],
     outputs=[
         gr.Image(type='numpy'),
         gr.Textbox(label='Prediction Speed', interactive=False),
+        gr.Number(label='Predicted WIC Confidence', precision=5, interactive=False),
         gr.Textbox(label='Predicted Detections', interactive=False),
     ],
     examples=[
-        ['examples/0d4e4df2-7b69-91b1-1985-c8421f2f3253.jpg', 20, 48, 51, 20, 20],
-        ['examples/18cef191-74ed-2b5e-55a5-f58bd3d483ff.jpg', 10, 48, 51, 20, 20],
-        ['examples/1be4d40a-6fd0-42ce-da6c-294e45781f41.jpg', 20, 48, 51, 20, 20],
-        ['examples/1d3c85e9-ee24-f290-e7e1-6e338f2eaebb.jpg', 20, 48, 51, 20, 20],
-        ['examples/3e043302-af1c-75a7-4057-3a2f25c123bf.jpg', 20, 48, 51, 20, 20],
-        ['examples/43ecc08d-502a-7a51-9d68-3e40a76439a2.jpg', 20, 48, 51, 20, 20],
-        ['examples/479058af-e774-e6aa-a2b0-9a42dd6ff8b1.jpg', 20, 48, 51, 20, 20],
-        ['examples/7c910b87-ae3a-f580-d431-03cd89793803.jpg', 20, 48, 51, 20, 20],
-        ['examples/8fa04489-cd94-7d8f-7e2e-5f0fe2f7ae76.jpg', 20, 48, 51, 20, 20],
-        ['examples/bb7b4345-b98a-c727-4c94-6090f0aa4355.jpg', 20, 48, 51, 20, 20],
+        ['examples/0d4e4df2-7b69-91b1-1985-c8421f2f3253.jpg', 'MVP', 7, 14, 51, 80, 80],
+        ['examples/18cef191-74ed-2b5e-55a5-f58bd3d483ff.jpg', 'MVP', 7, 14, 51, 80, 80],
+        ['examples/1be4d40a-6fd0-42ce-da6c-294e45781f41.jpg', 'MVP', 7, 14, 51, 80, 80],
+        ['examples/1d3c85e9-ee24-f290-e7e1-6e338f2eaebb.jpg', 'MVP', 7, 14, 51, 80, 80],
+        ['examples/3e043302-af1c-75a7-4057-3a2f25c123bf.jpg', 'MVP', 7, 14, 51, 80, 80],
+        ['examples/43ecc08d-502a-7a51-9d68-3e40a76439a2.jpg', 'MVP', 7, 14, 51, 80, 80],
+        ['examples/479058af-e774-e6aa-a2b0-9a42dd6ff8b1.jpg', 'MVP', 7, 14, 51, 80, 80],
+        ['examples/7c910b87-ae3a-f580-d431-03cd89793803.jpg', 'MVP', 7, 14, 51, 80, 80],
+        ['examples/8fa04489-cd94-7d8f-7e2e-5f0fe2f7ae76.jpg', 'MVP', 7, 14, 51, 80, 80],
+        ['examples/bb7b4345-b98a-c727-4c94-6090f0aa4355.jpg', 'MVP', 7, 14, 51, 80, 80],
     ],
     cache_examples=True,
     allow_flagging='never',

docs/_static/theme.css

@@ -0,0 +1,3 @@
+.wy-nav-content {
+    max-width: 900px !important;
+}

docs/conf.py

@@ -86,3 +86,6 @@ html_sidebars = {
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
+html_css_files = [
+    'theme.css',
+]

docs/environment.rst

@@ -4,12 +4,15 @@ Environment Variables
 The Scoutbot API and CLI have two environment variables (envars) that allow you to configure global settings
 and configurations.
 
-   - ``CONFIG`` (default: phase1)
+   - ``CONFIG`` (default: mvp)
       The configuration setting for which machine lerning models to use.
-      Must be one of ``phase1`` or ``mvp``.
+      Must be one of ``phase1`` or ``mvp``, or their respective aliases as ``old`` or ``new``.
    - ``WIC_BATCH_SIZE`` (default: 256)
       The configuration setting for how many tiles to send to the GPU in a single batch during the WIC
       prediction (forward inference).  The LOC model has a fixed batch size (16 for ``phase1`` and
       32 for ``mvp``) and cannot be adjusted.  This setting can be used to control how fast the pipeline
       runs, as a trade-off of faster compute for more memory usage.  It is highly suggested to set this
       value as high as possible to fit into the GPU.
+   - ``VERBOSE`` (default: not set)
+      A verbosity flag that can be set to turn on debug logging.  Defaults to "not set", which translates
+      to no debug logging.

docs/onnx.rst

@@ -16,12 +16,14 @@ To pre-download the models for a specific config (e.g., ``mvp``), you can specif
 These functions will download the following files and will store them in your Operating System's default
 cache folder:
 
-   - Phase 1
-      - ``WIC``: ``https://wildbookiarepository.azureedge.net/models/scout.wic.5fbfff26.3.0.onnx`` (81MB)
+   - Phase 1: ``phase1``
+      - WIC: ``https://wildbookiarepository.azureedge.net/models/scout.wic.5fbfff26.3.0.onnx`` (81MB)
          SHA256 checksum: ``cbc7f381fa58504e03b6510245b6b2742d63049429337465d95663a6468df4c1``
-      - ``LOC``: ``https://wildbookiarepository.azureedge.net/models/scout.loc.5fbfff26.0.onnx`` (209MB)
+      - LOC: ``https://wildbookiarepository.azureedge.net/models/scout.loc.5fbfff26.0.onnx`` (194M)
          SHA256 checksum: ``85a9378311d42b5143f74570136f32f50bf97c548135921b178b46ba7612b216``
 
-   - MVP
-      - ``WIC``: ``https://wildbookiarepository.azureedge.net/models/scout.wic.mvp.2.0.onnx`` (97MB)
+   - MVP: ``mvp``
+      - WIC: ``https://wildbookiarepository.azureedge.net/models/scout.wic.mvp.2.0.onnx`` (97MB)
          SHA256 checksum: ``3ff3a192803e53758af5e112526ba9622f1dedc55e2fa88850db6f32af160f32``
+      - LOC: ``https://wildbookiarepository.azureedge.net/models/scout.loc.mvp.0.onnx`` (194M)
+         SHA256 checksum: ``f5bd22fbacc91ba4cf5abaef5197d1645ae5bc4e63e88839e6848c48b3710c58``

scoutbot/__init__.py

@@ -19,7 +19,11 @@ how the entire pipeline can be run on tiles or images, respectively.
     img_shape, tile_grids, tile_filepaths = tile.compute(filepath)
 
     # Run WIC
-    wic_outputs = wic.post(wic.predict(wic.pre(tile_filepaths, config=config)))
+    wic_outputs = wic.post(wic.predict(wic.pre(
+        tile_filepaths,
+        config=config,
+        # batch_size=wic_batch_size,  # Optional override of config
+    )))
 
     # Threshold for WIC
     flags = [wic_output.get('positive') >= wic_thresh for wic_output in wic_outputs]
@@ -31,8 +35,8 @@ how the entire pipeline can be run on tiles or images, respectively.
         loc.predict(
             loc.pre(loc_tile_filepaths, config=config)
         ),
-        loc_thresh=loc_thresh,
-        nms_thresh=loc_nms_thresh
+        # loc_thresh=loc_thresh,  # Optional override of config
+        # nms_thresh=loc_nms_thresh,  # Optional override of config
     )
 
     # Run Aggregation and get final detections
@@ -41,8 +45,8 @@ how the entire pipeline can be run on tiles or images, respectively.
         loc_tile_grids,
         loc_outputs,
         config=config,
-        agg_thresh=agg_thresh,
-        nms_thresh=agg_nms_thresh,
+        # agg_thresh=agg_thresh,  # Optional override of config
+        # nms_thresh=agg_nms_thresh,  # Optional override of config
     )
 '''
 from os.path import exists
@@ -53,11 +57,12 @@ import utool as ut
 from scoutbot import utils
 
 log = utils.init_logging()
+QUIET = not utils.VERBOSE
 
 
 from scoutbot import agg, loc, tile, wic  # NOQA
 
-VERSION = '0.1.15'
+VERSION = '0.1.16'
 version = VERSION
 __version__ = VERSION
 
@@ -73,7 +78,7 @@ def fetch(pull=False, config=None):
         pull (bool, optional): If :obj:`True`, force using the downloaded versions
             stored in the local system's cache.  Defaults to :obj:`False`.
         config (str or None, optional): the configuration to use, one of ``phase1``
-            or ``mvp``.  Defaults to :obj:`None` (the ``phase1`` model).
+            or ``mvp``.  Defaults to :obj:`None`.
 
     Returns:
         None
@@ -115,17 +120,17 @@ def pipeline(
     Args:
         filepath (str): image filepath (relative or absolute)
         config (str or None, optional): the configuration to use, one of ``phase1``
-            or ``mvp``.  Defaults to :obj:`None` (the ``phase1`` model).
+            or ``mvp``.  Defaults to :obj:`None`.
         wic_thresh (float or None, optional): the confidence threshold for the WIC's
-            predictions.  Defaults to the ``phase1`` configuration setting.
+            predictions.  Defaults to the default configuration setting.
         loc_thresh (float or None, optional): the confidence threshold for the localizer's
-            predictions.  Defaults to the ``phase1`` configuration setting.
+            predictions.  Defaults to the default configuration setting.
         nms_thresh (float or None, optional): the non-maximum suppression (NMS) threshold
-            for the localizer's predictions.  Defaults to the ``phase1`` configuration setting.
+            for the localizer's predictions.  Defaults to the default configuration setting.
         agg_thresh (float or None, optional): the confidence threshold for the aggregated
-            localizer predictions.  Defaults to the ``phase1`` configuration setting.
+            localizer predictions. Defaults to the default configuration setting.
         agg_nms_thresh (float or None, optional): the non-maximum suppression (NMS) threshold
-            for the aggregated localizer's predictions.  Defaults to the ``phase1``
+            for the aggregated localizer's predictions.  Defaults to the default
             configuration setting.
         clean (bool, optional): a flag to clean up any on-disk tiles that were generated.
             Defaults to :obj:`True`.
@@ -147,7 +152,7 @@ def pipeline(
     loc_tile_grids = ut.compress(tile_grids, flags)
     loc_tile_filepaths = ut.compress(tile_filepaths, flags)
 
-    log.info(f'Filtered to {len(loc_tile_filepaths)} tiles')
+    log.debug(f'Filtered to {len(loc_tile_filepaths)} tiles')
 
     # Run localizer
     loc_outputs = loc.post(
@@ -207,17 +212,17 @@ def batch(
     Args:
         filepaths (list): list of str image filepath (relative or absolute)
         config (str or None, optional): the configuration to use, one of ``phase1``
-            or ``mvp``.  Defaults to :obj:`None` (the ``phase1`` model).
+            or ``mvp``.  Defaults to :obj:`None`.
         wic_thresh (float or None, optional): the confidence threshold for the WIC's
-            predictions.  Defaults to the ``phase1`` configuration setting.
+            predictions.  Defaults to the default configuration setting.
         loc_thresh (float or None, optional): the confidence threshold for the localizer's
-            predictions.  Defaults to the ``phase1`` configuration setting.
+            predictions.  Defaults to the default configuration setting.
         nms_thresh (float or None, optional): the non-maximum suppression (NMS) threshold
-            for the localizer's predictions.  Defaults to the ``phase1`` configuration setting.
+            for the localizer's predictions.  Defaults to the default configuration setting.
         agg_thresh (float or None, optional): the confidence threshold for the aggregated
-            localizer predictions.  Defaults to the ``phase1`` configuration setting.
+            localizer predictions.  Defaults to the default configuration setting.
         agg_nms_thresh (float or None, optional): the non-maximum suppression (NMS) threshold
-            for the aggregated localizer's predictions.  Defaults to the ``phase1``
+            for the aggregated localizer's predictions.  Defaults to the default
             configuration setting.
         clean (bool, optional): a flag to clean up any on-disk tiles that were generated.
             Defaults to :obj:`True`.
@@ -271,7 +276,7 @@ def batch(
     loc_tile_grids = ut.compress(tile_grids, flags)
     loc_tile_filepaths = ut.compress(tile_filepaths, flags)
 
-    log.info(f'Filtered to {len(loc_tile_filepaths)} tiles')
+    log.debug(f'Filtered to {len(loc_tile_filepaths)} tiles')
 
     # Run localizer
     loc_outputs = loc.post(
@@ -335,8 +340,8 @@ def example():
     )
     assert exists(img_filepath)
 
-    log.info(f'Running pipeline on image: {img_filepath}')
+    log.debug(f'Running pipeline on image: {img_filepath}')
 
     wic_, detects = pipeline(img_filepath)
 
-    log.info(ut.repr3(detects))
+    log.debug(ut.repr3(detects))

scoutbot/agg/__init__.py

@@ -15,7 +15,7 @@ from scoutbot import log
 
 MARGIN = 32.0
 
-DEFAULT_CONFIG = os.getenv('CONFIG', 'phase1').strip().lower()
+DEFAULT_CONFIG = os.getenv('CONFIG', 'mvp').strip().lower()
 CONFIGS = {
     'phase1': {
         'thresh': 0.4,
@@ -27,6 +27,8 @@ CONFIGS = {
     },
 }
 CONFIGS[None] = CONFIGS[DEFAULT_CONFIG]
+CONFIGS['old'] = CONFIGS['phase1']
+CONFIGS['new'] = CONFIGS['mvp']
 assert DEFAULT_CONFIG in CONFIGS
 
 
@@ -199,13 +201,11 @@ def compute(
         tile_grids (list of dict): a list of tile coordinates
         loc_output (list of list of dict): the output predictions from the Localizer.
         config (str or None, optional): the configuration to use, one of ``phase1``
-            or ``mvp``.  Defaults to :obj:`None` (the ``phase1`` model).
+            or ``mvp``.  Defaults to :obj:`None`.
         agg_thresh (float or None, optional): the confidence threshold for the aggregated
-            localizer predictions.  Defaults to None.  Defaults to :obj:`None`
-            (the ``phase1`` model's settings).
+            localizer predictions.  Defaults to None.  Defaults to :obj:`None`.
         nms_thresh (float or None, optional): the non-maximum suppression (NMS) threshold
-            for the aggregated localizer's predictions.  Defaults to :obj:`None`
-            (the ``phase1`` model's settings).
+            for the aggregated localizer's predictions.  Defaults to :obj:`None`.
 
     Returns:
         list ( dict ): list of Localizer predictions
@@ -219,7 +219,7 @@ def compute(
     if nms_thresh is None:
         nms_thresh = CONFIGS[config]['nms']
 
-    log.info(f'Aggregating {len(tile_grids)} tiles onto {img_shape} canvas')
+    log.debug(f'Aggregating {len(tile_grids)} tiles onto {img_shape} canvas')
 
     if len(tile_grids) == 0:
         final = []
@@ -251,6 +251,6 @@ def compute(
             final = ut.take(detects, keeps)
             final.sort(key=lambda val: val['c'], reverse=True)
 
-    log.info(f'Found {len(final)} detections')
+    log.debug(f'Found {len(final)} detections')
 
     return final

scoutbot/loc/__init__.py

@@ -8,6 +8,7 @@ output into usable detection bounding boxes with class labels and confidence
 scores.
 '''
 import os
+import warnings
 from os.path import exists, join
 from pathlib import Path
 
@@ -20,7 +21,7 @@ import torchvision
 import tqdm
 import utool as ut
 
-from scoutbot import log
+from scoutbot import QUIET, log
 from scoutbot.loc.transforms import (
     Compose,
     GetBoundingBoxes,
@@ -36,7 +37,7 @@ INPUT_SIZE = (416, 416)
 INPUT_SIZE_H, INPUT_SIZE_W = INPUT_SIZE
 NETWORK_SIZE = (INPUT_SIZE_H, INPUT_SIZE_W, 3)
 
-DEFAULT_CONFIG = os.getenv('CONFIG', 'phase1').strip().lower()
+DEFAULT_CONFIG = os.getenv('CONFIG', 'mvp').strip().lower()
 CONFIGS = {
     'phase1': {
         'batch': 16,
@@ -58,7 +59,7 @@ CONFIGS = {
         'batch': 32,
         'name': 'scout.loc.mvp.0.onnx',
         'path': join(PWD, 'models', 'onnx', 'scout.loc.mvp.0.onnx'),
-        'hash': 'AAA',
+        'hash': 'f5bd22fbacc91ba4cf5abaef5197d1645ae5bc4e63e88839e6848c48b3710c58',
         'classes': [
             'buffalo',
             'camel',
@@ -100,7 +101,7 @@ CONFIGS = {
             'wildebeest',
             'zebra',
         ],
-        'thresh': 0.4,
+        'thresh': 0.14,
         'nms': 0.8,
         'anchors': [
             (1.3221, 1.73145),
@@ -112,6 +113,8 @@ CONFIGS = {
     },
 }
 CONFIGS[None] = CONFIGS[DEFAULT_CONFIG]
+CONFIGS['old'] = CONFIGS['phase1']
+CONFIGS['new'] = CONFIGS['mvp']
 assert DEFAULT_CONFIG in CONFIGS
 
 
@@ -126,7 +129,7 @@ def fetch(pull=False, config=DEFAULT_CONFIG):
         pull (bool, optional): If :obj:`True`, force using the downloaded versions
             stored in the local system's cache.  Defaults to :obj:`False`.
         config (str or None, optional): the configuration to use, one of ``phase1``
-            or ``mvp``.  Defaults to :obj:`None` (the ``phase1`` model).
+            or ``mvp``.  Defaults to :obj:`None`.
 
     Returns:
         str: local ONNX model file path.
@@ -144,11 +147,11 @@ def fetch(pull=False, config=DEFAULT_CONFIG):
         onnx_model = pooch.retrieve(
             url=f'https://wildbookiarepository.azureedge.net/models/{model_name}',
             known_hash=model_hash,
-            progressbar=True,
+            progressbar=not QUIET,
         )
         assert exists(onnx_model)
 
-    log.info(f'LOC Model: {onnx_model}')
+    log.debug(f'LOC Model: {onnx_model}')
 
     return onnx_model
 
@@ -165,7 +168,7 @@ def pre(inputs, config=DEFAULT_CONFIG):
     Args:
         inputs (list(str)): list of tile image filepaths (relative or absolute)
         config (str or None, optional): the configuration to use, one of ``phase1``
-            or ``mvp``.  Defaults to :obj:`None` (the ``phase1`` model).
+            or ``mvp``.  Defaults to :obj:`None`.
 
     Returns:
         generator ( np.ndarray<np.float32>, list ( tuple ( int ) ), int, str ):
@@ -179,7 +182,7 @@ def pre(inputs, config=DEFAULT_CONFIG):
         return [], config
 
     batch_size = CONFIGS[config]['batch']
-    log.info(f'Preprocessing {len(inputs)} LOC inputs in batches of {batch_size}')
+    log.debug(f'Preprocessing {len(inputs)} LOC inputs in batches of {batch_size}')
 
     transform = torchvision.transforms.ToTensor()
 
@@ -221,11 +224,11 @@ def predict(gen):
             - - list of each tile's original size
             - - model configuration
     """
-    log.info('Running LOC inference')
+    log.debug('Running LOC inference')
 
     ort_sessions = {}
 
-    for chunk, sizes, trim, config in tqdm.tqdm(gen):
+    for chunk, sizes, trim, config in tqdm.tqdm(gen, disable=QUIET):
         assert len(chunk) == len(sizes)
 
         if len(chunk) == 0:
@@ -236,10 +239,13 @@ def predict(gen):
             if ort_session is None:
                 onnx_model = fetch(config=config)
 
-                ort_session = ort.InferenceSession(
-                    onnx_model,
-                    providers=['CUDAExecutionProvider', 'CPUExecutionProvider'],
-                )
+                with warnings.catch_warnings():
+                    warnings.filterwarnings('ignore', category=UserWarning)
+                    ort_session = ort.InferenceSession(
+                        onnx_model,
+                        providers=['CUDAExecutionProvider', 'CPUExecutionProvider'],
+                    )
+
                 ort_sessions[config] = ort_session
 
             assert trim <= len(chunk)
@@ -286,16 +292,14 @@ def post(gen, loc_thresh=None, nms_thresh=None):
         gen (generator): generator of batches of raw ONNX model outputs and sizes,
             the return of :meth:`scoutbot.loc.predict`
         loc_thresh (float or None, optional): the confidence threshold for the localizer's
-            predictions.  Defaults to None.  Defaults to :obj:`None`
-            (the ``phase1`` model).
+            predictions.  Defaults to None.  Defaults to :obj:`None`.
         nms_thresh (float or None, optional): the non-maximum suppression (NMS) threshold
-            for the localizer's predictions.  Defaults to :obj:`None`
-            (the ``phase1`` model).
+            for the localizer's predictions.  Defaults to :obj:`None`.
 
     Returns:
         list ( list ( dict ) ): nested list of Localizer predictions
     """
-    log.info('Postprocessing LOC outputs')
+    log.debug('Postprocessing LOC outputs')
 
     # Exhaust generator and format output
     outputs = []
@@ -321,12 +325,25 @@ def post(gen, loc_thresh=None, nms_thresh=None):
 
         preds = postprocess(torch.tensor(preds))
 
+        if config in ['phase1']:
+            class_map = {}
+        elif config in [None, 'mvp']:
+            class_map = {
+                'dead_animalwhite_bones': 'white_bones',
+                'deadbones': 'white_bones',
+                'elecarcass_old': 'white_bones',
+                'gazelle_gr': 'gazelle_grants',
+                'gazelle_th': 'gazelle_thomsons',
+            }
+        else:
+            raise ValueError()
+
         for pred, size in zip(preds, sizes):
             output = ReverseLetterbox.apply([pred], INPUT_SIZE, size)
             output = output[0]
             output = [
                 {
-                    'l': detect.class_label,
+                    'l': class_map.get(detect.class_label, detect.class_label),
                     'c': detect.confidence,
                     'x': detect.x_top_left,
                     'y': detect.y_top_left,

scoutbot/loc/convert.mvp.py

@@ -0,0 +1,345 @@
+# -*- coding: utf-8 -*-
+"""
+pip install torch torchvision onnx onnxruntime-gpu tqdm wbia-utool scikit-learn numpy
+"""
+import random
+import time
+from os.path import exists, join, split, splitext
+
+import cv2
+import lightnet as ln
+import numpy as np
+import onnx
+import onnxruntime as ort
+import sklearn
+import torch
+import torchvision
+import tqdm
+import utool as ut
+import vtool as vt
+import wbia
+
+WITH_GPU = False
+BATCH_SIZE = 32
+
+
+ibs = wbia.opendb(dbdir='/data/db')
+
+
+pkl_path = 'scout.pkl'
+if not exists(pkl_path):
+    if False:
+        pass
+        # tids = ibs.get_valid_gids(is_tile=True)
+    else:
+        imageset_text_list = ['TEST_SET']
+        imageset_rowid_list = ibs.get_imageset_imgsetids_from_text(imageset_text_list)
+        gids_list = ibs.get_imageset_gids(imageset_rowid_list)
+        gids = ut.flatten(gids_list)
+        flags = ibs.get_tile_flags(gids)
+        test_gids = ut.filterfalse_items(gids, flags)
+        assert sum(ibs.get_tile_flags(test_gids)) == 0
+        tids = ibs.scout_get_valid_tile_rowids(gid_list=test_gids)
+
+    random.shuffle(tids)
+    positive, negative = [], []
+    for chunk_tids in tqdm.tqdm(ut.ichunks(tids, 1000)):
+        _, _, chunk_flags = ibs.scout_tile_positive_cumulative_area(chunk_tids)
+        chunk_filepaths = ibs.get_image_paths(chunk_tids)
+        for index, (tid, flag, filepath) in enumerate(
+            zip(chunk_tids, chunk_flags, chunk_filepaths)
+        ):
+            if not exists(filepath):
+                continue
+            if flag:
+                positive.append(tid)
+            else:
+                negative.append(tid)
+        if len(positive) >= 100 and len(negative) >= 100:
+            break
+        print(len(positive), len(negative))
+
+    random.shuffle(positive)
+    random.shuffle(negative)
+    positive = positive[:100]
+    negative = negative[:100]
+    data = positive + negative
+    filepaths = ibs.get_image_paths(data)
+    labels = [True] * len(positive) + [False] * len(negative)
+    ut.save_cPkl(pkl_path, (data, labels))
+
+    OUTPUT_PATH = '/data/db/checks'
+    ut.delete(OUTPUT_PATH)
+    ut.ensuredir(OUTPUT_PATH)
+    for filepath, label in zip(filepaths, labels):
+        path, filename = split(filepath)
+        name, ext = splitext(filename)
+        tag = 'true' if label else 'false'
+        filename_ = f'{name}.{tag}{ext}'
+        filepath_ = join(OUTPUT_PATH, filename_)
+        if not exists(filepath_):
+            ut.copy(filepath, filepath_)
+
+assert exists(pkl_path)
+data, labels = ut.load_cPkl(pkl_path)
+
+filepaths = ibs.get_image_paths(data)
+orients = ibs.get_image_orientation(data)
+
+assert len(data) == len(set(data))
+assert set(ibs.get_image_sizes(data)) == {(256, 256)}
+assert sum(map(exists, filepaths)) == len(filepaths)
+assert sum(orients) == 0
+
+##########
+
+INDEX = 0
+
+config_path = f'/cache/lightnet/detect.lightnet.scout.mvp.{INDEX}.py'
+weights_path = f'/cache/lightnet/detect.lightnet.scout.mvp.{INDEX}.weights'
+conf_thresh = 0.0
+nms_thresh = 0.2
+
+assert exists(config_path)
+assert exists(weights_path)
+
+params = ln.engine.HyperParameters.from_file(config_path)
+params.load(weights_path)
+
+model = params.network
+
+# Update conf_thresh and nms_thresh in postpsocess
+model.postprocess[0].conf_thresh = conf_thresh
+model.postprocess[1].nms_thresh = nms_thresh
+
+if WITH_GPU:
+    model = model.cuda()
+model.eval()
+
+INPUT_SIZE = params.input_dimension
+INPUT_SIZE_H, INPUT_SIZE_W = INPUT_SIZE
+
+#############
+
+dataloader = list(zip(filepaths, orients, labels))
+
+transform = torchvision.transforms.ToTensor()
+
+time_pytorch = 0.0
+inputs = []
+sizes = []
+outputs = []
+targets = []
+for chunk in ut.ichunks(dataloader, BATCH_SIZE):
+
+    filepaths_ = ut.take_column(chunk, 0)
+    orients_ = ut.take_column(chunk, 1)
+    targets_ = ut.take_column(chunk, 2)
+
+    inputs_ = []
+    sizes_ = []
+    for filepath, orient in zip(filepaths_, orients_):
+        img = vt.imread(filepath, orient=orient)
+        size = img.shape[:2][::-1]
+
+        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+        img = ln.data.transform.Letterbox.apply(img, dimension=INPUT_SIZE)
+        img = transform(img)
+
+        inputs_.append(img)
+        sizes_.append(size)
+    inputs_ = torch.stack(inputs_)
+
+    if WITH_GPU:
+        inputs_ = inputs_.cuda()
+
+    time_start = time.time()
+    with torch.set_grad_enabled(False):
+        output_ = model(inputs_)
+    time_end = time.time()
+    time_pytorch += time_end - time_start
+
+    output_transform_ = []
+    for out_, size_ in zip(output_, sizes_):
+        out_transform_ = ln.data.transform.ReverseLetterbox.apply(
+            [out_], INPUT_SIZE, size_
+        )
+        output_transform_.append(out_transform_[0])
+
+    inputs += inputs_.tolist()
+    sizes += sizes_
+    outputs += output_transform_
+    targets += targets_
+
+predictions_pytorch = outputs
+
+#############
+
+threshs = list(np.arange(0.0, 1.01, 0.01))
+best_thresh = None
+best_accuracy = 0.0
+best_confusion = None
+for thresh in tqdm.tqdm(threshs):
+    globals().update(locals())
+    values = [
+        [prediction for prediction in predictions if prediction.confidence >= thresh]
+        for predictions in predictions_pytorch
+    ]
+    values = [len(value) > 0 for value in values]
+    accuracy = sklearn.metrics.accuracy_score(targets, values)
+    confusion = sklearn.metrics.confusion_matrix(targets, values)
+    if accuracy > best_accuracy:
+        best_thresh = thresh
+        best_accuracy = accuracy
+        best_confusion = confusion
+
+tn, fp, fn, tp = best_confusion.ravel()
+print(f'Thresh:    {best_thresh}')
+print(f'Accuracy:  {best_accuracy}')
+print(f'TP:        {tp}')
+print(f'TN:        {tn}')
+print(f'FP:        {fp}')
+print(f'FN:        {fn}')
+
+# Thresh:    0.14
+# Accuracy:  0.895
+# TP:        89
+# TN:        90
+# FP:        10
+# FN:        11
+
+#############
+
+dummy_input = torch.randn(BATCH_SIZE, 3, INPUT_SIZE_H, INPUT_SIZE_W, device='cpu')
+input_names = ['input']
+output_names = ['output']
+
+model.onnx = True
+onnx_filename = f'scout.loc.mvp.{INDEX}.onnx'
+output = torch.onnx.export(
+    model,
+    dummy_input,
+    onnx_filename,
+    verbose=True,
+    input_names=input_names,
+    output_names=output_names,
+    dynamic_axes={
+        'input': {0: 'batch_size'},  # variable length axes
+        'output': {0: 'batch_size'},
+    },
+)
+
+###########
+
+model = onnx.load(onnx_filename)
+onnx.checker.check_model(model)
+print(onnx.helper.printable_graph(model.graph))
+
+###########
+
+ort_session = ort.InferenceSession(onnx_filename, providers=['CPUExecutionProvider'])
+
+num_classes = params.network.num_classes
+anchors = params.network.anchors
+network_size = (INPUT_SIZE_H, INPUT_SIZE_W, 3)
+class_label_map = params.class_label_map
+conf_thresh = 0.0
+nms_thresh = 0.2
+
+postprocess = ln.data.transform.Compose(
+    [
+        ln.data.transform.GetBoundingBoxes(num_classes, anchors, conf_thresh),
+        ln.data.transform.NonMaxSupression(nms_thresh),
+        ln.data.transform.TensorToBrambox(network_size, class_label_map),
+    ]
+)
+
+zipped = list(zip(inputs, sizes))
+
+time_onnx = 0.0
+outputs = []
+for chunk in ut.ichunks(zipped, BATCH_SIZE):
+
+    imgs = ut.take_column(chunk, 0)
+    sizes_ = ut.take_column(chunk, 1)
+
+    trim = len(imgs)
+    while (len(imgs)) < BATCH_SIZE:
+        imgs.append(np.random.randn(3, INPUT_SIZE_H, INPUT_SIZE_W).astype(np.float32))
+        sizes_.append(INPUT_SIZE)
+    input_ = np.array(imgs, dtype=np.float32)
+
+    time_start = time.time()
+    outputs_ = ort_session.run(
+        None,
+        {'input': input_},
+    )
+    output_ = postprocess(torch.tensor(outputs_[0]))
+    time_end = time.time()
+    time_onnx += time_end - time_start
+
+    output_transform_ = []
+    for out_, size_ in zip(output_, sizes_):
+        out_transform_ = ln.data.transform.ReverseLetterbox.apply(
+            [out_], INPUT_SIZE, size_
+        )
+        output_transform_.append(out_transform_[0])
+
+    outputs += output_transform_[:trim]
+
+predictions_onnx = outputs
+
+###########
+
+globals().update(locals())
+values_pytorch = [
+    [prediction for prediction in predictions if prediction.confidence >= best_thresh]
+    for predictions in predictions_pytorch
+]
+values_onnx = [
+    [prediction for prediction in predictions if prediction.confidence >= best_thresh]
+    for predictions in predictions_onnx
+]
+
+deviations = []
+for value_pytorch, value_onnx in zip(values_pytorch, values_onnx):
+    assert len(value_pytorch) == len(value_onnx)
+    for value_p, value_o in zip(value_pytorch, value_onnx):
+        assert value_p.class_label == value_o.class_label
+        for attr in ['x_top_left', 'y_top_left', 'width', 'height', 'confidence']:
+            deviation = abs(getattr(value_p, attr) - getattr(value_o, attr))
+            deviations.append(deviation)
+
+print(f'Min:  {np.min(deviations):0.08f}')
+print(f'Max:  {np.max(deviations):0.08f}')
+print(f'Mean: {np.mean(deviations):0.08f} +/- {np.std(deviations):0.08f}')
+print(f'Time Pytorch: {time_pytorch:0.02f} sec.')
+print(f'Time ONNX:    {time_onnx:0.02f} sec.')
+
+values = [
+    [prediction for prediction in predictions if prediction.confidence >= best_thresh]
+    for predictions in predictions_onnx
+]
+values = [len(value) > 0 for value in values]
+accuracy = sklearn.metrics.accuracy_score(targets, values)
+confusion = sklearn.metrics.confusion_matrix(targets, values)
+tn, fp, fn, tp = best_confusion.ravel()
+
+print(f'Thresh:    {best_thresh}')
+print(f'Accuracy:  {best_accuracy}')
+print(f'TP:        {tp}')
+print(f'TN:        {tn}')
+print(f'FP:        {fp}')
+print(f'FN:        {fn}')
+
+# Min:  0.00000000
+# Max:  0.00027231
+# Mean: 0.00001667 +/- 0.00002650
+# Time Pytorch: 19.77 sec.
+# Time ONNX:    10.52 sec.
+# Thresh:    0.14
+# Accuracy:  0.895
+# TP:        89
+# TN:        90
+# FP:        10
+# FN:        11

scoutbot/loc/models/onnx/scout.loc.mvp.0.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5bd22fbacc91ba4cf5abaef5197d1645ae5bc4e63e88839e6848c48b3710c58
+size 203171952

scoutbot/loc/models/pytorch/detect.lightnet.scout.mvp.0.py

@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+import lightnet as ln
+import torch
+
+__all__ = ['params']
+
+
+params = ln.engine.HyperParameters(
+    # Network
+    class_label_map=[
+        'buffalo',
+        'camel',
+        'canoe',
+        'car',
+        'cow',
+        'crocodile',
+        'dead_animalwhite_bones',
+        'deadbones',
+        'eland',
+        'elecarcass_old',
+        'elephant',
+        'gazelle_gr',
+        'gazelle_grants',
+        'gazelle_th',
+        'gazelle_thomsons',
+        'gerenuk',
+        'giant_forest_hog',
+        'giraffe',
+        'goat',
+        'hartebeest',
+        'hippo',
+        'impala',
+        'kob',
+        'kudu',
+        'motorcycle',
+        'oribi',
+        'oryx',
+        'ostrich',
+        'roof_grass',
+        'roof_mabati',
+        'sheep',
+        'test',
+        'topi',
+        'vehicle',
+        'warthog',
+        'waterbuck',
+        'white_bones',
+        'wildebeest',
+        'zebra',
+    ],
+    input_dimension=(416, 416),
+    batch_size=1024,
+    mini_batch_size=512,
+    max_batches=30000,
+    # Dataset
+    _train_set='/data/db/_ibsdb/_ibeis_cache/training/lightnet/lightnet-training-mvp-892b8c24f52400ff/data/train.pkl',
+    _valid_set=None,
+    _test_set='/data/db/_ibsdb/_ibeis_cache/training/lightnet/lightnet-training-mvp-892b8c24f52400ff/data/test.pkl',
+    _filter_anno='ignore',
+    # Data Augmentation
+    jitter=0.3,
+    flip=0.5,
+    hue=0.1,
+    saturation=1.5,
+    value=1.5,
+)
+
+
+# Network
+def init_weights(m):
+    if isinstance(m, torch.nn.Conv2d):
+        torch.nn.init.kaiming_normal_(m.weight, nonlinearity='leaky_relu')
+
+
+params.network = ln.models.Yolo(
+    len(params.class_label_map),
+    conf_thresh=0.001,
+    nms_thresh=0.5,
+)
+params.network.postprocess.append(
+    ln.data.transform.TensorToBrambox(params.input_dimension, params.class_label_map)
+)
+params.network.apply(init_weights)
+
+# Optimizers
+params.add_optimizer(
+    torch.optim.SGD(
+        params.network.parameters(),
+        lr=0.001 / params.batch_size,
+        momentum=0.9,
+        weight_decay=0.0005 * params.batch_size,
+        dampening=0,
+    )
+)
+
+# Schedulers
+burn_in = torch.optim.lr_scheduler.LambdaLR(
+    params.optimizers[0],
+    lambda b: (b / 1000) ** 4,
+)
+step = torch.optim.lr_scheduler.MultiStepLR(
+    params.optimizers[0],
+    milestones=[20000, 40000],
+    gamma=0.1,
+)
+params.add_scheduler(
+    ln.engine.SchedulerCompositor(
+        #   batch   scheduler
+        (0, burn_in),
+        (1000, step),
+    )
+)

scoutbot/loc/models/pytorch/detect.lightnet.scout.mvp.0.weights

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1986d74d0d5d3d102fe91fc36a63049da63289f0b171659ed3a1558447b1c9da
+size 406179399

scoutbot/scoutbot.py

@@ -25,7 +25,7 @@ def pipeline_filepath_validator(ctx, param, value):
     '--config',
     help='Which ML models to use for inference',
     default=None,
-    type=click.Choice(['phase1', 'mvp']),
+    type=click.Choice(['phase1', 'mvp', 'old', 'new']),
 )
 def fetch(config):
     """
@@ -45,7 +45,7 @@ def fetch(config):
     '--config',
     help='Which ML models to use for inference',
     default=None,
-    type=click.Choice(['phase1', 'mvp']),
+    type=click.Choice(['phase1', 'mvp', 'old', 'new']),
 )
 @click.option(
     '--output',
@@ -94,9 +94,30 @@ def pipeline(
     agg_nms_thresh,
 ):
     """
-    Run the ScoutBot pipeline on an input image filepath
+    Run the ScoutBot pipeline on an input image filepath.  An example output of the JSON
+    can be seen below.
+
+    .. code-block:: javascript
+
+            {
+                '/path/to/image.ext': {
+                    'wic': 0.5,
+                    'loc': [
+                        {
+                            'l': 'elephant',
+                            'c': 0.9,
+                            'x': 100,
+                            'y': 100,
+                            'w': 50,
+                            'h': 10
+                        },
+                        ...
+                    ],
+                }
+            }
     """
-    config = config.strip().lower()
+    if config is not None:
+        config = config.strip().lower()
     wic_thresh /= 100.0
     loc_thresh /= 100.0
     loc_nms_thresh /= 100.0
@@ -113,19 +134,18 @@ def pipeline(
         agg_nms_thresh=agg_nms_thresh,
     )
 
+    data = {
+        filepath: {
+            'wic': wic_,
+            'loc': detects,
+        }
+    }
+
     if output:
         with open(output, 'w') as outfile:
-            data = {
-                filepath: {
-                    'wic': wic_,
-                    'loc': detects,
-                }
-            }
             json.dump(data, outfile)
     else:
-        log.info(filepath)
-        log.info(f'WIC: {wic_:0.04f}')
-        log.info('LOC: {}'.format(ut.repr3(detects)))
+        print(ut.repr3(data))
 
 
 @click.command('batch')
@@ -138,7 +158,7 @@ def pipeline(
     '--config',
     help='Which ML models to use for inference',
     default=None,
-    type=click.Choice(['phase1', 'mvp']),
+    type=click.Choice(['phase1', 'mvp', 'old', 'new']),
 )
 @click.option(
     '--output',
@@ -187,16 +207,52 @@ def batch(
     agg_nms_thresh,
 ):
     """
-    Run the ScoutBot pipeline in batch on a list of input image filepaths
+    Run the ScoutBot pipeline in batch on a list of input image filepaths.
+    An example output of the JSON can be seen below.
+
+    .. code-block:: javascript
+
+            {
+                '/path/to/image1.ext': {
+                    'wic': 0.5,
+                    'loc': [
+                        {
+                            'l': 'elephant',
+                            'c': 0.9,
+                            'x': 100,
+                            'y': 100,
+                            'w': 50,
+                            'h': 10
+                        },
+                        ...
+                    ],
+                },
+                '/path/to/image2.ext': {
+                    'wic': 0.5,
+                    'loc': [
+                        {
+                            'l': 'elephant',
+                            'c': 0.9,
+                            'x': 100,
+                            'y': 100,
+                            'w': 50,
+                            'h': 10
+                        },
+                        ...
+                    ],
+                },
+                ...
+            }
     """
-    config = config.strip().lower()
+    if config is not None:
+        config = config.strip().lower()
     wic_thresh /= 100.0
     loc_thresh /= 100.0
     loc_nms_thresh /= 100.0
     agg_thresh /= 100.0
     agg_nms_thresh /= 100.0
 
-    log.info(f'Running batch on {len(filepaths)} files...')
+    log.debug(f'Running batch on {len(filepaths)} files...')
 
     wic_list, detects_list = scoutbot.batch(
         filepaths,
@@ -209,20 +265,18 @@ def batch(
     )
     results = zip(filepaths, wic_list, detects_list)
 
+    data = {}
+    for filepath, wic_, detects in results:
+        data[filepath] = {
+            'wic': wic,
+            'loc': detects,
+        }
+
     if output:
         with open(output, 'w') as outfile:
-            data = {}
-            for filepath, wic_, detects in results:
-                data[filepath] = {
-                    'wic': wic,
-                    'loc': detects,
-                }
-                json.dump(data, outfile)
+            json.dump(data, outfile)
     else:
-        for filepath, wic_, detects in results:
-            log.info(filepath)
-            log.info(f'WIC: {wic_:0.04f}')
-            log.info('LOC: {}'.format(ut.repr3(detects)))
+        print(ut.repr3(data))
 
 
 @click.command('example')

scoutbot/tile/__init__.py

@@ -44,7 +44,7 @@ def compute(img_filepath, grid1=True, grid2=True, ext=None, **kwargs):
     img = cv2.imread(img_filepath)
     shape = img.shape
 
-    log.info(f'Computing tiles (grid1={grid1}, grid2={grid2}) on {img_filepath}')
+    log.debug(f'Computing tiles (grid1={grid1}, grid2={grid2}) on {img_filepath}')
 
     grids = []
     if grid1:
@@ -56,7 +56,7 @@ def compute(img_filepath, grid1=True, grid2=True, ext=None, **kwargs):
     for grid, filepath in zip(grids, filepaths):
         assert tile_write(img, grid, filepath)
 
-    log.info(f'Rendered {len(filepaths)} tiles')
+    log.debug(f'Rendered {len(filepaths)} tiles')
 
     return shape, grids, filepaths
 

scoutbot/utils.py

@@ -3,9 +3,12 @@
 Scoutbot utilities file for common and handy functions.
 '''
 import logging
+import os
 from logging.handlers import TimedRotatingFileHandler
 
 DAYS = 21
+VERBOSE = os.getenv('VERBOSE', None) is not None
+DEFAULT_LOG_LEVEL = logging.DEBUG if VERBOSE else logging.INFO
 
 
 def init_logging():
@@ -43,7 +46,7 @@ def init_logging():
         'tracebacks_show_locals': True,
     }
     logging_kwargs = {
-        'level': logging.INFO,
+        'level': DEFAULT_LOG_LEVEL,
         'format': '[%(name)s] %(message)s',
         'datefmt': '[%X]',
     }
@@ -64,7 +67,7 @@ def init_logging():
     # Setup global logger with the handlers and set the default level to INFO
     logging.basicConfig(handlers=handlers, **logging_kwargs)
     logger = logging.getLogger()
-    logger.setLevel(logging.INFO)
+    logger.setLevel(DEFAULT_LOG_LEVEL)
     log = logging.getLogger(name)
 
     return log

scoutbot/wic/__init__.py

@@ -7,6 +7,7 @@ WIC ONNX model on this input, and finally how to convert this raw CNN output
 into usable confidence scores.
 '''
 import os
+import warnings
 from os.path import exists, join
 from pathlib import Path
 
@@ -16,7 +17,7 @@ import pooch
 import torch
 import tqdm
 
-from scoutbot import log
+from scoutbot import QUIET, log
 from scoutbot.wic.dataloader import (  # NOQA
     BATCH_SIZE,
     INPUT_SIZE,
@@ -27,7 +28,7 @@ from scoutbot.wic.dataloader import (  # NOQA
 PWD = Path(__file__).absolute().parent
 
 
-DEFAULT_CONFIG = os.getenv('CONFIG', 'phase1').strip().lower()
+DEFAULT_CONFIG = os.getenv('CONFIG', 'mvp').strip().lower()
 CONFIGS = {
     'phase1': {
         'name': 'scout.wic.5fbfff26.3.0.onnx',
@@ -45,6 +46,8 @@ CONFIGS = {
     },
 }
 CONFIGS[None] = CONFIGS[DEFAULT_CONFIG]
+CONFIGS['old'] = CONFIGS['phase1']
+CONFIGS['new'] = CONFIGS['mvp']
 assert DEFAULT_CONFIG in CONFIGS
 
 
@@ -59,7 +62,7 @@ def fetch(pull=False, config=DEFAULT_CONFIG):
         pull (bool, optional): If :obj:`True`, force using the downloaded versions
             stored in the local system's cache.  Defaults to :obj:`False`.
         config (str or None, optional): the configuration to use, one of ``phase1``
-            or ``mvp``.  Defaults to :obj:`None` (the ``phase1`` model).
+            or ``mvp``.  Defaults to :obj:`None`.
 
     Returns:
         str: local ONNX model file path.
@@ -77,11 +80,11 @@ def fetch(pull=False, config=DEFAULT_CONFIG):
         onnx_model = pooch.retrieve(
             url=f'https://wildbookiarepository.azureedge.net/models/{model_name}',
             known_hash=model_hash,
-            progressbar=True,
+            progressbar=not QUIET,
         )
         assert exists(onnx_model)
 
-    log.info(f'WIC Model: {onnx_model}')
+    log.debug(f'WIC Model: {onnx_model}')
 
     return onnx_model
 
@@ -100,7 +103,7 @@ def pre(inputs, batch_size=BATCH_SIZE, config=DEFAULT_CONFIG):
         batch_size (int, optional): the maximum number of images to load in a
             single batch.  Defaults to the environment variable ``WIC_BATCH_SIZE``.
         config (str or None, optional): the configuration to use, one of ``phase1``
-            or ``mvp``.  Defaults to :obj:`None` (the ``phase1`` model).
+            or ``mvp``.  Defaults to :obj:`None`.
 
     Returns:
         generator ( np.ndarray<np.float32>, str ):
@@ -111,7 +114,7 @@ def pre(inputs, batch_size=BATCH_SIZE, config=DEFAULT_CONFIG):
     if len(inputs) == 0:
         return [], config
 
-    log.info(f'Preprocessing {len(inputs)} WIC inputs in batches of {batch_size}')
+    log.debug(f'Preprocessing {len(inputs)} WIC inputs in batches of {batch_size}')
 
     transform = _init_transforms()
     dataset = ImageFilePathList(inputs, transform=transform)
@@ -137,19 +140,23 @@ def predict(gen):
             - - list of raw ONNX model outputs as shape ``(b, n)``
             - - model configuration
     """
-    log.info('Running WIC inference')
+    log.debug('Running WIC inference')
 
     ort_sessions = {}
 
-    for chunk, config in tqdm.tqdm(gen):
+    for chunk, config in tqdm.tqdm(gen, disable=QUIET):
 
         ort_session = ort_sessions.get(config)
         if ort_session is None:
             onnx_model = fetch(config=config)
 
-            ort_session = ort.InferenceSession(
-                onnx_model, providers=['CUDAExecutionProvider', 'CPUExecutionProvider']
-            )
+            with warnings.catch_warnings():
+                warnings.filterwarnings('ignore', category=UserWarning)
+                ort_session = ort.InferenceSession(
+                    onnx_model,
+                    providers=['CUDAExecutionProvider', 'CPUExecutionProvider'],
+                )
+
             ort_sessions[config] = ort_session
 
         if len(chunk) == 0:
@@ -178,7 +185,7 @@ def post(gen):
         list ( dict ): list of WIC predictions
     """
     # Exhaust generator and format output
-    log.info('Postprocessing WIC outputs')
+    log.debug('Postprocessing WIC outputs')
 
     outputs = []
     for preds, config in gen:

tests/test_agg.py

@@ -19,7 +19,7 @@ def test_agg_compute_phase1():
 
     # Threshold for WIC
     flags = [
-        wic_output.get('positive') >= wic.CONFIGS[None]['thresh']
+        wic_output.get('positive') >= wic.CONFIGS['phase1']['thresh']
         for wic_output in wic_outputs
     ]
     loc_tile_grids = ut.compress(tile_grids, flags)
@@ -49,3 +49,55 @@ def test_agg_compute_phase1():
                 assert abs(output.get(key) - target.get(key)) < 1e-2
             else:
                 assert abs(output.get(key) - target.get(key)) < 3
+
+
+def test_agg_compute_mvp():
+    img_filepath = abspath(join('examples', '1be4d40a-6fd0-42ce-da6c-294e45781f41.jpg'))
+
+    # Run tiling
+    img_shape, tile_grids, tile_filepaths = tile.compute(img_filepath)
+    assert len(tile_filepaths) == 1252
+
+    # Run WIC
+    wic_outputs = wic.post(wic.predict(wic.pre(tile_filepaths, config='mvp')))
+    assert len(wic_outputs) == len(tile_filepaths)
+
+    # Threshold for WIC
+    flags = [
+        wic_output.get('positive') >= wic.CONFIGS['mvp']['thresh']
+        for wic_output in wic_outputs
+    ]
+    loc_tile_grids = ut.compress(tile_grids, flags)
+    loc_tile_filepaths = ut.compress(tile_filepaths, flags)
+    assert sum(flags) == 125
+
+    # Run localizer
+    loc_outputs = loc.post(loc.predict(loc.pre(loc_tile_filepaths, config='mvp')))
+    assert len(loc_tile_grids) == len(loc_outputs)
+
+    # Aggregate
+    detects = agg.compute(img_shape, loc_tile_grids, loc_outputs, config='mvp')
+
+    assert len(detects) == 8
+
+    # fmt: off
+    targets = [
+        {'l': 'elephant', 'c': 0.6795, 'x': 4593, 'y': 2300, 'w': 78, 'h': 201},
+        {'l': 'elephant', 'c': 0.6126, 'x': 4813, 'y': 2452, 'w': 54, 'h': 87},
+        {'l': 'kob',      'c': 0.6058, 'x': 3391, 'y': 1076, 'w': 33, 'h': 32},
+        {'l': 'elephant', 'c': 0.5933, 'x': 4873, 'y': 2428, 'w': 80, 'h': 99},
+        {'l': 'kob',      'c': 0.4767, 'x': 1601, 'y': 1729, 'w': 53, 'h': 55},
+        {'l': 'warthog',  'c': 0.4571, 'x': 4199, 'y': 2109, 'w': 31, 'h': 45},
+        {'l': 'kob',      'c': 0.4193, 'x': 1441, 'y': 3377, 'w': 30, 'h': 38},
+        {'l': 'elephant', 'c': 0.4178, 'x': 3891, 'y': 3641, 'w': 60, 'h': 84},
+    ]
+    # fmt: on
+
+    for output, target in zip(detects, targets):
+        for key in target.keys():
+            if key == 'l':
+                assert output.get(key) == target.get(key)
+            elif key == 'c':
+                assert abs(output.get(key) - target.get(key)) < 1e-2
+            else:
+                assert abs(output.get(key) - target.get(key)) < 3

tests/test_loc.py

@@ -17,6 +17,19 @@ def test_loc_onnx_load_phase1():
     assert graph.count('\n') == 107
 
 
+def test_loc_onnx_load_mvp():
+    from scoutbot.loc import fetch
+
+    onnx_model = fetch(config='mvp')
+    model = onnx.load(onnx_model)
+    assert exists(onnx_model)
+
+    onnx.checker.check_model(model)
+
+    graph = onnx.helper.printable_graph(model.graph)
+    assert graph.count('\n') == 107
+
+
 def test_loc_onnx_pipeline_phase1():
     from scoutbot.loc import CONFIGS, INPUT_SIZE, post, pre, predict
 
@@ -27,7 +40,7 @@ def test_loc_onnx_pipeline_phase1():
     assert exists(inputs[0])
 
     data = pre(inputs, config='phase1')
-    batch_size = CONFIGS[None]['batch']
+    batch_size = CONFIGS['phase1']['batch']
 
     temp, sizes, trim, config = next(data)
     assert temp.shape == (batch_size, 3, INPUT_SIZE[0], INPUT_SIZE[1])
@@ -51,49 +64,180 @@ def test_loc_onnx_pipeline_phase1():
 
     assert len(outputs) == 1
     assert len(outputs[0]) == 5
+    # assert len(outputs[0]) == 7
 
     # fmt: off
     targets = [
         {
             'l': 'elephant_savanna',
+            'c':   0.77065581,
             'x': 206.00893930,
             'y': 189.09138371,
             'w':  53.78145658,
             'h':  66.46106896,
-            'c':   0.77065581,
         },
         {
             'l': 'elephant_savanna',
+            'c':   0.61152166,
             'x': 216.61065204,
             'y': 193.30525090,
             'w':  42.83404541,
             'h':  62.44728440,
-            'c':   0.61152166,
         },
         {
             'l': 'elephant_savanna',
+            'c':   0.50862342,
             'x':  51.61210749,
             'y': 235.37819260,
             'w':  79.69709660,
             'h':  17.41258826,
-            'c':   0.50862342,
         },
         {
             'l': 'elephant_savanna',
+            'c':   0.44841822,
             'x':  57.47630427,
             'y': 236.92587515,
             'w':  94.69935960,
             'h':  16.03246718,
-            'c':   0.44841822,
         },
         {
             'l': 'elephant_savanna',
+            'c':   0.44012001,
             'x':  37.07233605,
             'y': 230.39122596,
             'w': 105.40560208,
             'h':  24.81017362,
-            'c':   0.44012001,
         },
+        # {
+        #     'l': 'elephant_savanna',
+        #     'c':   0.38498798,
+        #     'x':  56.43274395,
+        #     'y': 232.00978440,
+        #     'w':  99.98320124,
+        #     'h':  22.50272075,
+        # },
+        # {
+        #     'l': 'elephant_savanna',
+        #     'c':   0.37786528,
+        #     'x': 202.67217548,
+        #     'y': 178.77696814,
+        #     'w':  58.69518573,
+        #     'h':  71.09806941,
+        # },
+    ]
+    # fmt: on
+
+    for output, target in zip(outputs[0], targets):
+        for key in target.keys():
+            if key == 'l':
+                assert output.get(key) == target.get(key)
+            elif key == 'c':
+                assert abs(output.get(key) - target.get(key)) < 1e-2
+            else:
+                assert abs(output.get(key) - target.get(key)) < 3
+
+
+def test_loc_onnx_pipeline_mvp():
+    from scoutbot.loc import CONFIGS, INPUT_SIZE, post, pre, predict
+
+    inputs = [
+        abspath(join('examples', '0d01a14e-311d-e153-356f-8431b6996b84.true.jpg')),
+    ]
+
+    assert exists(inputs[0])
+
+    data = pre(inputs, config='mvp')
+    batch_size = CONFIGS['mvp']['batch']
+
+    temp, sizes, trim, config = next(data)
+    assert temp.shape == (batch_size, 3, INPUT_SIZE[0], INPUT_SIZE[1])
+    assert len(temp) == len(sizes)
+    assert sizes[0] == (256, 256)
+    assert set(sizes[1:]) == {(0, 0)}
+    assert config == 'mvp'
+
+    data = pre(inputs, config='mvp')
+    preds = predict(data)
+
+    temp, sizes, config = next(preds)
+    assert temp.shape == (1, 220, 13, 13)
+    assert len(temp) == len(sizes)
+    assert sizes == [(256, 256)]
+    assert config == 'mvp'
+
+    data = pre(inputs, config='mvp')
+    preds = predict(data)
+    outputs = post(preds)
+
+    assert len(outputs) == 1
+    assert len(outputs[0]) == 8
+
+    # fmt: off
+    targets = [
+        {
+            'l': 'elephant',
+            'c':   0.78486251,
+            'x': 205.34572190,
+            'y': 198.39648437,
+            'w':  52.55188457,
+            'h':  56.18781456,
+        },
+        {
+            'l': 'elephant',
+            'c':   0.54303294,
+            'x': 213.27392578,
+            'y': 195.15114182,
+            'w':  48.83143498,
+            'h':  61.92804424,
+        },
+        {
+            'l': 'elephant',
+            'c':   0.25485479,
+            'x':  39.34061373,
+            'y': 227.89024939,
+            'w':  99.23480694,
+            'h':  26.51788095,
+        },
+        {
+            'l': 'elephant',
+            'c':   0.24082227,
+            'x':  56.96651517,
+            'y': 229.90174278,
+            'w':  62.85778339,
+            'h':  23.15211838,
+        },
+        {
+            'l': 'elephant',
+            'c':   0.22669222,
+            'x': 213.39426832,
+            'y': 200.48779296,
+            'w':  36.94954974,
+            'h':  57.41221266,
+        },
+        {
+            'l': 'elephant',
+            'c':   0.19940485,
+            'x': 219.36613581,
+            'y': 205.06403996,
+            'w':  41.39131986,
+            'h':  46.13519756,
+        },
+        {
+            'l': 'kob',
+            'c':   0.17925532,
+            'x':   6.99571814,
+            'y':   0.92224179,
+            'w':  43.32685734,
+            'h':  18.18345876,
+        },
+        {
+            'l': 'elephant',
+            'c':   0.15872234,
+            'x': 160.69904972,
+            'y': 235.63134765,
+            'w':  51.77306659,
+            'h':  19.74641535,
+        }
     ]
     # fmt: on
 

tests/test_scoutbot.py

@@ -21,12 +21,13 @@ def test_pipeline_phase1():
     wic_, detects = scoutbot.pipeline(img_filepath, config='phase1')
 
     assert abs(wic_ - 1.0) < 1e-2
-    assert len(detects) == 3
+    assert len(detects) == 4
 
     targets = [
         {'l': 'elephant_savanna', 'c': 0.9299, 'x': 4597, 'y': 2322, 'w': 72, 'h': 149},
         {'l': 'elephant_savanna', 'c': 0.8739, 'x': 4865, 'y': 2422, 'w': 97, 'h': 109},
         {'l': 'elephant_savanna', 'c': 0.7115, 'x': 4806, 'y': 2476, 'w': 66, 'h': 119},
+        {'l': 'elephant_savanna', 'c': 0.5236, 'x': 3511, 'y': 1228, 'w': 47, 'h': 78},
     ]
 
     for output, target in zip(detects, targets):
@@ -51,12 +52,13 @@ def test_batch_phase1():
     detects = detects_list[0]
 
     assert abs(wic_ - 1.0) < 1e-2
-    assert len(detects) == 3
+    assert len(detects) == 4
 
     targets = [
         {'l': 'elephant_savanna', 'c': 0.9299, 'x': 4597, 'y': 2322, 'w': 72, 'h': 149},
         {'l': 'elephant_savanna', 'c': 0.8739, 'x': 4865, 'y': 2422, 'w': 97, 'h': 109},
         {'l': 'elephant_savanna', 'c': 0.7115, 'x': 4806, 'y': 2476, 'w': 66, 'h': 119},
+        {'l': 'elephant_savanna', 'c': 0.5236, 'x': 3511, 'y': 1228, 'w': 47, 'h': 78},
     ]
 
     for output, target in zip(detects, targets):
@@ -69,5 +71,73 @@ def test_batch_phase1():
                 assert abs(output.get(key) - target.get(key)) < 3
 
 
+def test_pipeline_mvp():
+    img_filepath = abspath(join('examples', '1be4d40a-6fd0-42ce-da6c-294e45781f41.jpg'))
+
+    wic_, detects = scoutbot.pipeline(img_filepath, config='mvp')
+
+    assert abs(wic_ - 1.0) < 1e-2
+    assert len(detects) == 8
+
+    # fmt: off
+    targets = [
+        {'l': 'elephant', 'c': 0.6795, 'x': 4593, 'y': 2300, 'w': 78, 'h': 201},
+        {'l': 'elephant', 'c': 0.6126, 'x': 4813, 'y': 2452, 'w': 54, 'h': 87},
+        {'l': 'kob',      'c': 0.6058, 'x': 3391, 'y': 1076, 'w': 33, 'h': 32},
+        {'l': 'elephant', 'c': 0.5933, 'x': 4873, 'y': 2428, 'w': 80, 'h': 99},
+        {'l': 'kob',      'c': 0.4767, 'x': 1601, 'y': 1729, 'w': 53, 'h': 55},
+        {'l': 'warthog',  'c': 0.4571, 'x': 4199, 'y': 2109, 'w': 31, 'h': 45},
+        {'l': 'kob',      'c': 0.4193, 'x': 1441, 'y': 3377, 'w': 30, 'h': 38},
+        {'l': 'elephant', 'c': 0.4178, 'x': 3891, 'y': 3641, 'w': 60, 'h': 84},
+    ]
+    # fmt: on
+
+    for output, target in zip(detects, targets):
+        for key in target.keys():
+            if key == 'l':
+                assert output.get(key) == target.get(key)
+            elif key == 'c':
+                assert abs(output.get(key) - target.get(key)) < 1e-2
+            else:
+                assert abs(output.get(key) - target.get(key)) < 3
+
+
+def test_batch_mvp():
+    img_filepath = abspath(join('examples', '1be4d40a-6fd0-42ce-da6c-294e45781f41.jpg'))
+
+    img_filepaths = [img_filepath]
+    wic_list, detects_list = scoutbot.batch(img_filepaths, config='mvp')
+    assert len(wic_list) == 1
+    assert len(detects_list) == 1
+
+    wic_ = wic_list[0]
+    detects = detects_list[0]
+
+    assert abs(wic_ - 1.0) < 1e-2
+    assert len(detects) == 8
+
+    # fmt: off
+    targets = [
+        {'l': 'elephant', 'c': 0.6795, 'x': 4593, 'y': 2300, 'w': 78, 'h': 201},
+        {'l': 'elephant', 'c': 0.6126, 'x': 4813, 'y': 2452, 'w': 54, 'h': 87},
+        {'l': 'kob',      'c': 0.6058, 'x': 3391, 'y': 1076, 'w': 33, 'h': 32},
+        {'l': 'elephant', 'c': 0.5933, 'x': 4873, 'y': 2428, 'w': 80, 'h': 99},
+        {'l': 'kob',      'c': 0.4767, 'x': 1601, 'y': 1729, 'w': 53, 'h': 55},
+        {'l': 'warthog',  'c': 0.4571, 'x': 4199, 'y': 2109, 'w': 31, 'h': 45},
+        {'l': 'kob',      'c': 0.4193, 'x': 1441, 'y': 3377, 'w': 30, 'h': 38},
+        {'l': 'elephant', 'c': 0.4178, 'x': 3891, 'y': 3641, 'w': 60, 'h': 84},
+    ]
+    # fmt: on
+
+    for output, target in zip(detects, targets):
+        for key in target.keys():
+            if key == 'l':
+                assert output.get(key) == target.get(key)
+            elif key == 'c':
+                assert abs(output.get(key) - target.get(key)) < 1e-2
+            else:
+                assert abs(output.get(key) - target.get(key)) < 3
+
+
 def test_example():
     scoutbot.example()

tests/test_wic.py

@@ -61,7 +61,7 @@ def test_wic_onnx_pipeline_phase1():
 
     assert len(outputs) == 1
     output = outputs[0]
-    classes = CONFIGS[None]['classes']
+    classes = CONFIGS['phase1']['classes']
     assert output.keys() == set(classes)
     assert output['positive'] > output['negative']
     assert abs(output['negative'] - 0.00001503) < 1e-4
@@ -101,7 +101,7 @@ def test_wic_onnx_pipeline_mvp():
 
     assert len(outputs) == 1
     output = outputs[0]
-    classes = CONFIGS[None]['classes']
+    classes = CONFIGS['mvp']['classes']
     assert output.keys() == set(classes)
     assert output['positive'] > output['negative']
     assert abs(output['negative'] - 0.00000000) < 1e-4