Convert WIC and LOC to use generators during the pre() and predict() functions

.gitignore

@@ -5,7 +5,7 @@ output.*.jpg
 *.egg-info/
 
 examples/*_w_256_h_256.jpg
-.coverage
+.coverage*
 coverage/
 
 gradio_cached_examples/

app.py

@@ -27,9 +27,9 @@ def predict(filepath, wic_thresh, loc_thresh, nms_thresh):
     if wic_confidence > wic_thresh:
 
         # Run Localizer
-        data, sizes = loc.pre(inputs)
-        preds = loc.predict(data)
-        outputs = loc.post(preds, sizes, loc_thresh=loc_thresh, nms_thresh=nms_thresh)
+        outputs = loc.post(
+            loc.predict(loc.pre(inputs)), loc_thresh=loc_thresh, nms_thresh=nms_thresh
+        )
 
         # Format and render results
         detects = outputs[0]

scoutbot/__init__.py

@@ -26,11 +26,10 @@ how the entire pipeline can be run on tiles or images, respectively.
     loc_tile_filepaths = ut.compress(tile_filepaths, flags)
 
     # Run localizer
-    loc_data, loc_sizes = loc.pre(loc_tile_filepaths)
-    loc_preds = loc.predict(loc_data)
     loc_outputs = loc.post(
-        loc_preds,
-        loc_sizes,
+        loc.predict(
+            loc.pre(loc_tile_filepaths)
+        ),
         loc_thresh=loc_thresh,
         nms_thresh=loc_nms_thresh
     )
@@ -56,7 +55,7 @@ log = utils.init_logging()
 
 from scoutbot import agg, loc, tile, wic  # NOQA
 
-VERSION = '0.1.11'
+VERSION = '0.1.12'
 version = VERSION
 __version__ = VERSION
 
@@ -89,6 +88,7 @@ def pipeline(
     loc_nms_thresh=loc.NMS_THRESH,
     agg_thresh=agg.AGG_THRESH,
     agg_nms_thresh=agg.NMS_THRESH,
+    clean=True,
 ):
     """
     Run the ML pipeline on a given image filepath and return the detections
@@ -126,11 +126,13 @@ def pipeline(
     loc_tile_grids = ut.compress(tile_grids, flags)
     loc_tile_filepaths = ut.compress(tile_filepaths, flags)
 
+    log.info(f'Filtered to {len(loc_tile_filepaths)} tiles')
+
     # Run localizer
-    loc_data, loc_sizes = loc.pre(loc_tile_filepaths)
-    loc_preds = loc.predict(loc_data)
     loc_outputs = loc.post(
-        loc_preds, loc_sizes, loc_thresh=loc_thresh, nms_thresh=loc_nms_thresh
+        loc.predict(loc.pre(loc_tile_filepaths)),
+        loc_thresh=loc_thresh,
+        nms_thresh=loc_nms_thresh,
     )
     assert len(loc_tile_grids) == len(loc_outputs)
 
@@ -143,6 +145,11 @@ def pipeline(
         nms_thresh=agg_nms_thresh,
     )
 
+    if clean:
+        for tile_filepath in tile_filepaths:
+            if exists(tile_filepath):
+                ut.delete(tile_filepath, verbose=False)
+
     return detects
 
 
@@ -153,6 +160,7 @@ def batch(
     loc_nms_thresh=loc.NMS_THRESH,
     agg_thresh=agg.AGG_THRESH,
     agg_nms_thresh=agg.NMS_THRESH,
+    clean=True,
 ):
     """
     Run the ML pipeline on a given batch of image filepaths and return the detections
@@ -202,12 +210,12 @@ def batch(
     tile_filepaths = []
     for filepath in filepaths:
         data = batch[filepath]
-        grids = data['grids']
-        filepaths = data['filepaths']
-        assert len(grids) == len(filepaths)
-        tile_img_filepaths += [filepath] * len(grids)
-        tile_grids += grids
-        tile_filepaths += filepaths
+        batch_grids = data['grids']
+        batch_filepaths = data['filepaths']
+        assert len(batch_grids) == len(batch_filepaths)
+        tile_img_filepaths += [filepath] * len(batch_grids)
+        tile_grids += batch_grids
+        tile_filepaths += batch_filepaths
 
     wic_outputs = wic.post(wic.predict(wic.pre(tile_filepaths)))
 
@@ -217,11 +225,13 @@ def batch(
     loc_tile_grids = ut.compress(tile_grids, flags)
     loc_tile_filepaths = ut.compress(tile_filepaths, flags)
 
+    log.info(f'Filtered to {len(loc_tile_filepaths)} tiles')
+
     # Run localizer
-    loc_data, loc_sizes = loc.pre(loc_tile_filepaths)
-    loc_preds = loc.predict(loc_data)
     loc_outputs = loc.post(
-        loc_preds, loc_sizes, loc_thresh=loc_thresh, nms_thresh=loc_nms_thresh
+        loc.predict(loc.pre(loc_tile_filepaths)),
+        loc_thresh=loc_thresh,
+        nms_thresh=loc_nms_thresh,
     )
     assert len(loc_tile_grids) == len(loc_outputs)
 
@@ -250,10 +260,18 @@ def batch(
         )
         detects_list.append(detects)
 
+    if clean:
+        for tile_filepath in tile_filepaths:
+            if exists(tile_filepath):
+                ut.delete(tile_filepath, verbose=False)
+
     return detects_list
 
 
 def example():
+    """
+    Run the pipeline on an example image
+    """
     TEST_IMAGE = 'scout.example.jpg'
     TEST_IMAGE_HASH = (
         '786a940b062a90961f409539292f09144c3dbdbc6b6faa64c3e764d63d55c988'  # NOQA

scoutbot/loc/__init__.py

@@ -16,6 +16,7 @@ import onnxruntime as ort
 import pooch
 import torch
 import torchvision
+import tqdm
 import utool as ut
 
 from scoutbot import log
@@ -96,73 +97,84 @@ def pre(inputs):
         inputs (list(str)): list of tile image filepaths (relative or absolute)
 
     Returns:
-        tuple ( list ( list ( list ( list ( float ) ) ) ), list ( tuple ( int ) ) ):
-            - list of transformed image data.
-            - list of each tile's original size.
+        generator ( tuple ( list ( list ( list ( list ( float ) ) ) ), list ( tuple ( int ) ) ) ):
+            - generator ->
+            - - list of transformed image data.
+            - - list of each tile's original size.
     """
     assert len(inputs) > 0
 
+    log.info(f'Preprocessing {len(inputs)} LOC inputs in batches of {BATCH_SIZE}')
+
     transform = torchvision.transforms.ToTensor()
 
-    data = []
-    sizes = []
-    for filepath in inputs:
-        img = cv2.imread(filepath)
-        size = img.shape[:2][::-1]
+    for filepaths in ut.ichunks(inputs, BATCH_SIZE):
+        data = np.zeros((BATCH_SIZE, 3, INPUT_SIZE_H, INPUT_SIZE_W), dtype=np.float32)
+        sizes = []
+        trim = len(filepaths)
+
+        for index, filepath in enumerate(filepaths):
+            img = cv2.imread(filepath)
+            size = img.shape[:2][::-1]
 
-        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
-        img = Letterbox.apply(img, dimension=INPUT_SIZE)
-        img = transform(img)
+            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+            img = Letterbox.apply(img, dimension=INPUT_SIZE)
+            img = transform(img)
+            img = img.numpy().astype(np.float32)
 
-        data.append(img.tolist())
-        sizes.append(size)
+            data[index] = img
+            sizes.append(size)
 
-    return data, sizes
+        while len(sizes) < BATCH_SIZE:
+            sizes.append((0, 0))
 
+        yield data, sizes, trim
 
-def predict(data, fill=True):
+
+def predict(gen):
     """
     Run neural network inference using the Localizer's ONNX model on preprocessed data.
 
     Args:
-        data (list): list of transformed image data, the first return of :meth:`scoutbot.loc.pre`
-        fill (bool, optional): If :obj:`True`, fill any partial batches to the LOC `BATCH_SIZE`,
-            and then trim them after inference.  Defaults to :obj:`True`.
+        gen (generator): generator of batches of transformed image data, the return of
+            :meth:`scoutbot.loc.pre`
 
     Returns:
-        list ( list ( float ) ): list of raw ONNX model outputs
+        generator ( list ( list ( float ) ), list ( tuple ( int ) ) ) ):
+            - generator ->
+            - - list of raw ONNX model outputs.
+            - - list of each tile's original size.
     """
     onnx_model = fetch()
 
-    log.info(f'Running LOC inference on {len(data)} tiles')
-
-    if len(data) == 0:
-        return []
+    log.info('Running LOC inference')
 
     ort_session = ort.InferenceSession(
         onnx_model, providers=['CUDAExecutionProvider', 'CPUExecutionProvider']
     )
 
-    preds = []
-    for chunk in ut.ichunks(data, BATCH_SIZE):
-        trim = len(chunk)
-        if fill:
-            while (len(chunk)) < BATCH_SIZE:
-                chunk.append(
-                    np.random.randn(3, INPUT_SIZE_H, INPUT_SIZE_W).astype(np.float32)
-                )
-        input_ = np.array(chunk, dtype=np.float32)
-
-        pred_ = ort_session.run(
-            None,
-            {'input': input_},
-        )
-        preds += pred_[0].tolist()[:trim]
+    for chunk, sizes, trim in tqdm.tqdm(gen):
+        assert len(chunk) == len(sizes)
+
+        if len(chunk) == 0:
+            preds = []
+            sizes = []
+        else:
+            assert trim <= len(chunk)
+
+            pred = ort_session.run(
+                None,
+                {'input': chunk},
+            )
+            preds = pred[0]
 
-    return preds
+            preds = preds[:trim]
+            sizes = sizes[:trim]
 
+        yield preds, sizes
 
-def post(preds, sizes, loc_thresh=LOC_THRESH, nms_thresh=NMS_THRESH):
+
+def post(gen, loc_thresh=LOC_THRESH, nms_thresh=NMS_THRESH):
     """
     Apply a post-processing normalization of the raw ONNX network outputs.
 
@@ -189,16 +201,13 @@ def post(preds, sizes, loc_thresh=LOC_THRESH, nms_thresh=NMS_THRESH):
     The ``x``, ``y``, ``w``, ``h`` bounding box keys are in real pixel values.
 
     Args:
-        preds (list): list of raw ONNX model outputs, the return of :meth:`scoutbot.loc.predict`
-        sizes (list): list of original tile sizes, the second return of :meth:`scoutbot.loc.pre`
+        gen (generator): generator of batches of raw ONNX model outputs and sizes,
+            the return of :meth:`scoutbot.loc.predict`
 
     Returns:
         list ( list ( dict ) ): nested list of Localizer predictions
     """
-    assert len(preds) == len(sizes)
-
-    if len(preds) == 0:
-        return []
+    log.info('Postprocessing LOC outputs')
 
     postprocess = Compose(
         [
@@ -208,23 +217,29 @@ def post(preds, sizes, loc_thresh=LOC_THRESH, nms_thresh=NMS_THRESH):
         ]
     )
 
-    preds = postprocess(torch.tensor(preds))
-
+    # Exhaust generator and format output
     outputs = []
-    for pred, size in zip(preds, sizes):
-        output = ReverseLetterbox.apply([pred], INPUT_SIZE, size)
-        output = output[0]
-        output = [
-            {
-                'l': detect.class_label,
-                'c': detect.confidence,
-                'x': detect.x_top_left,
-                'y': detect.y_top_left,
-                'w': detect.width,
-                'h': detect.height,
-            }
-            for detect in output
-        ]
-        outputs.append(output)
+    for preds, sizes in gen:
+        assert len(preds) == len(sizes)
+        if len(preds) == 0:
+            continue
+
+        preds = postprocess(torch.tensor(preds))
+
+        for pred, size in zip(preds, sizes):
+            output = ReverseLetterbox.apply([pred], INPUT_SIZE, size)
+            output = output[0]
+            output = [
+                {
+                    'l': detect.class_label,
+                    'c': detect.confidence,
+                    'x': detect.x_top_left,
+                    'y': detect.y_top_left,
+                    'w': detect.width,
+                    'h': detect.height,
+                }
+                for detect in output
+            ]
+            outputs.append(output)
 
     return outputs

scoutbot/scoutbot.py

@@ -39,36 +39,36 @@ def fetch():
     '--output',
     help='Path to output JSON (if unspecified, results are printed to screen)',
     default=None,
-    type=click.IntRange(0, 100, clamp=True),
+    type=str,
 )
 @click.option(
     '--wic_thresh',
     help='Whole Image Classifier (WIC) confidence threshold',
-    default=wic.WIC_THRESH,
+    default=int(wic.WIC_THRESH * 100),
     type=click.IntRange(0, 100, clamp=True),
 )
 @click.option(
     '--loc_thresh',
     help='Localizer (LOC) confidence threshold',
-    default=loc.LOC_THRESH,
+    default=int(loc.LOC_THRESH * 100),
     type=click.IntRange(0, 100, clamp=True),
 )
 @click.option(
     '--loc_nms_thresh',
     help='Localizer (LOC) non-maximum suppression (NMS) threshold',
-    default=loc.NMS_THRESH,
+    default=int(loc.NMS_THRESH * 100),
     type=click.IntRange(0, 100, clamp=True),
 )
 @click.option(
     '--agg_thresh',
     help='Aggregation (AGG) confidence threshold',
-    default=agg.AGG_THRESH,
+    default=int(agg.AGG_THRESH * 100),
     type=click.IntRange(0, 100, clamp=True),
 )
 @click.option(
     '--agg_nms_thresh',
     help='Aggregation (AGG) non-maximum suppression (NMS) threshold',
-    default=agg.NMS_THRESH,
+    default=int(agg.NMS_THRESH * 100),
     type=click.IntRange(0, 100, clamp=True),
 )
 def pipeline(
@@ -99,7 +99,7 @@ def pipeline(
         log.info(ut.repr3(detects))
 
 
-@click.command()
+@click.command('batch')
 @click.argument(
     'filepaths',
     nargs=-1,
@@ -109,43 +109,43 @@ def pipeline(
     '--output',
     help='Path to output JSON (if unspecified, results are printed to screen)',
     default=None,
-    type=click.IntRange(0, 100, clamp=True),
+    type=str,
 )
 @click.option(
     '--wic_thresh',
     help='Whole Image Classifier (WIC) confidence threshold',
-    default=wic.WIC_THRESH,
+    default=int(wic.WIC_THRESH * 100),
     type=click.IntRange(0, 100, clamp=True),
 )
 @click.option(
     '--loc_thresh',
     help='Localizer (LOC) confidence threshold',
-    default=loc.LOC_THRESH,
+    default=int(loc.LOC_THRESH * 100),
     type=click.IntRange(0, 100, clamp=True),
 )
 @click.option(
     '--loc_nms_thresh',
     help='Localizer (LOC) non-maximum suppression (NMS) threshold',
-    default=loc.NMS_THRESH,
+    default=int(loc.NMS_THRESH * 100),
     type=click.IntRange(0, 100, clamp=True),
 )
 @click.option(
     '--agg_thresh',
     help='Aggregation (AGG) confidence threshold',
-    default=agg.AGG_THRESH,
+    default=int(agg.AGG_THRESH * 100),
     type=click.IntRange(0, 100, clamp=True),
 )
 @click.option(
     '--agg_nms_thresh',
     help='Aggregation (AGG) non-maximum suppression (NMS) threshold',
-    default=agg.NMS_THRESH,
+    default=int(agg.NMS_THRESH * 100),
     type=click.IntRange(0, 100, clamp=True),
 )
 def batch(
     filepaths, output, wic_thresh, loc_thresh, loc_nms_thresh, agg_thresh, agg_nms_thresh
 ):
     """
-    Run the ScoutBot pipeline on an input image filepath
+    Run the ScoutBot pipeline in batch on a list of input image filepaths
     """
     wic_thresh /= 100.0
     loc_thresh /= 100.0

scoutbot/tile/__init__.py

@@ -28,7 +28,7 @@ def compute(img_filepath, grid1=True, grid2=True, ext=None, **kwargs):
         grid1 (bool, optional): If :obj:`True`, create a dense grid of tiles on the image.
             Defaults to :obj:`True`.
         grid2 (bool, optional): If :obj:`True`, create a secondary dense grid of tiles
-            on the image with a 50% offset.  Defaults to :obj:`True`.
+            on the image with a 50% offset.  Defaults to :obj:`False`.
         ext (str, optional): The file extension of the resulting tile files.  If this value is
             not specified, it will use the same extension as `img_filepath`.  Passed as input
             to :meth:`scoutbot.tile.tile_filepath`.  Defaults to :obj:`None`.

scoutbot/wic/__init__.py

@@ -13,10 +13,11 @@ import numpy as np
 import onnxruntime as ort
 import pooch
 import torch
+import tqdm
 import utool as ut
 
 from scoutbot import log
-from scoutbot.wic.dataloader import (
+from scoutbot.wic.dataloader import (  # NOQA
     BATCH_SIZE,
     INPUT_SIZE,
     ImageFilePathList,
@@ -65,7 +66,7 @@ def fetch(pull=False):
     return onnx_model
 
 
-def pre(inputs):
+def pre(inputs, batch_size=BATCH_SIZE):
     """
     Load a list of filepaths and return a corresponding list of the image
     data as a 4-D list of floats.  The image data is loaded from disk, transformed
@@ -78,66 +79,56 @@ def pre(inputs):
         inputs (list(str)): list of tile image filepaths (relative or absolute)
 
     Returns:
-        list ( list ( list ( list ( float ) ) ) ): list of transformed image data
+        generator ( list ( list ( list ( list ( float ) ) ) ) ) : generator ->
+        list of transformed image data
     """
     assert len(inputs) > 0
 
+    log.info(f'Preprocessing {len(inputs)} WIC inputs in batches of {batch_size}')
+
     transform = _init_transforms()
     dataset = ImageFilePathList(inputs, transform=transform)
     dataloader = torch.utils.data.DataLoader(
-        dataset, batch_size=BATCH_SIZE, num_workers=0, pin_memory=False
+        dataset, batch_size=batch_size, num_workers=8, pin_memory=False
     )
 
-    data = []
-    for (data_,) in dataloader:
-        data += data_.tolist()
-
-    return data
+    for (data,) in dataloader:
+        yield data.numpy().astype(np.float32)
 
 
-def predict(data, fill=False):
+def predict(gen):
     """
     Run neural network inference using the WIC's ONNX model on preprocessed data.
 
     Args:
-        data (list): list of transformed image data, the return of :meth:`scoutbot.wic.pre`
-        fill (bool, optional): If :obj:`True`, fill any partial batches to the WIC `BATCH_SIZE`,
-            and then trim them after inference.  Defaults to :obj:`False`.
+        gen (generator): generator of batches of transformed image data, the
+            return of :meth:`scoutbot.wic.pre`
 
     Returns:
-        list ( list ( float ) ): list of raw ONNX model outputs
+        generator ( list ( list ( float ) ) ): generator -> list of raw ONNX
+        model outputs
     """
     onnx_model = fetch()
 
-    log.info(f'Running WIC inference on {len(data)} tiles')
-
-    if len(data) == 0:
-        return []
+    log.info('Running WIC inference')
 
     ort_session = ort.InferenceSession(
         onnx_model, providers=['CUDAExecutionProvider', 'CPUExecutionProvider']
     )
 
-    preds = []
-    for chunk in ut.ichunks(data, BATCH_SIZE):
-        trim = len(chunk)
-        if fill:
-            while (len(chunk)) < BATCH_SIZE:
-                chunk.append(
-                    np.random.randn(3, INPUT_SIZE, INPUT_SIZE).astype(np.float32)
-                )
-        input_ = np.array(chunk, dtype=np.float32)
-
-        pred_ = ort_session.run(
-            None,
-            {'input': input_},
-        )
-        preds += pred_[0].tolist()[:trim]
-
-    return preds
+    for chunk in tqdm.tqdm(gen):
+        if len(chunk) == 0:
+            preds = []
+        else:
+            pred = ort_session.run(
+                None,
+                {'input': chunk},
+            )
+            preds = pred[0]
+        yield preds
 
 
-def post(preds):
+def post(gen):
     """
     Apply a post-processing normalization of the raw ONNX network outputs.
 
@@ -145,10 +136,14 @@ def post(preds):
     and the values are their corresponding confidence values.
 
     Args:
-        preds (list): list of raw ONNX model outputs, the return of :meth:`scoutbot.wic.predict`
+        gen (generator): generator of batches of raw ONNX model
+            outputs, the return of :meth:`scoutbot.wic.predict`
 
     Returns:
         list ( dict ): list of WIC predictions
     """
-    outputs = [dict(zip(ONNX_CLASSES, pred)) for pred in preds]
+    # Exhaust generator and format output
+    log.info('Postprocessing WIC outputs')
+
+    outputs = [dict(zip(ONNX_CLASSES, pred.tolist())) for pred in ut.flatten(gen)]
     return outputs

tests/test_agg.py

@@ -24,10 +24,10 @@ def test_agg_compute():
     assert sum(flags) == 15
 
     # Run localizer
-    loc_data, loc_sizes = loc.pre(loc_tile_filepaths)
-    loc_preds = loc.predict(loc_data)
     loc_outputs = loc.post(
-        loc_preds, loc_sizes, loc_thresh=loc.LOC_THRESH, nms_thresh=loc.NMS_THRESH
+        loc.predict(loc.pre(loc_tile_filepaths)),
+        loc_thresh=loc.LOC_THRESH,
+        nms_thresh=loc.NMS_THRESH,
     )
     assert len(loc_tile_grids) == len(loc_outputs)
 

tests/test_loc.py

@@ -18,7 +18,7 @@ def test_loc_onnx_load():
 
 
 def test_loc_onnx_pipeline():
-    from scoutbot.loc import INPUT_SIZE, post, pre, predict
+    from scoutbot.loc import BATCH_SIZE, INPUT_SIZE, post, pre, predict
 
     inputs = [
         abspath(join('examples', '0d01a14e-311d-e153-356f-8431b6996b84.true.jpg')),
@@ -26,20 +26,25 @@ def test_loc_onnx_pipeline():
 
     assert exists(inputs[0])
 
-    data, sizes = pre(inputs)
+    data = pre(inputs)
 
-    assert len(data) == 1
-    assert len(data[0]) == 3
-    assert len(data[0][0]) == INPUT_SIZE[0]
-    assert len(data[0][0][0]) == INPUT_SIZE[1]
-    assert sizes == [(256, 256)]
+    temp, sizes, trim = next(data)
+    assert temp.shape == (BATCH_SIZE, 3, INPUT_SIZE[0], INPUT_SIZE[1])
+    assert len(temp) == len(sizes)
+    assert sizes[0] == (256, 256)
+    assert set(sizes[1:]) == {(0, 0)}
 
+    data = pre(inputs)
     preds = predict(data)
 
-    assert len(preds) == 1
-    assert len(preds[0]) == 30
+    temp, sizes = next(preds)
+    assert temp.shape == (1, 30, 13, 13)
+    assert len(temp) == len(sizes)
+    assert sizes == [(256, 256)]
 
-    outputs = post(preds, sizes)
+    data = pre(inputs)
+    preds = predict(data)
+    outputs = post(preds)
 
     assert len(outputs) == 1
     assert len(outputs[0]) == 5

tests/test_wic.py

@@ -28,19 +28,20 @@ def test_wic_onnx_pipeline():
 
     data = pre(inputs)
 
-    assert len(data) == 1
-    assert len(data[0]) == 3
-    assert len(data[0][0]) == INPUT_SIZE
-    assert len(data[0][0][0]) == INPUT_SIZE
+    temp = next(data)
+    assert temp.shape == (1, 3, INPUT_SIZE, INPUT_SIZE)
 
+    data = pre(inputs)
     preds = predict(data)
 
-    assert len(preds) == 1
-    assert len(preds[0]) == 2
-    assert preds[0][1] > preds[0][0]
-    assert abs(preds[0][0] - 0.00001503) < 1e-4
-    assert abs(preds[0][1] - 0.99998497) < 1e-4
+    temp = next(preds)
+    assert temp.shape == (1, 2)
+    assert temp[0][1] > temp[0][0]
+    assert abs(temp[0][0] - 0.00001503) < 1e-4
+    assert abs(temp[0][1] - 0.99998497) < 1e-4
 
+    data = pre(inputs)
+    preds = predict(data)
     outputs = post(preds)
 
     assert len(outputs) == 1
@@ -49,3 +50,5 @@ def test_wic_onnx_pipeline():
     assert output['positive'] > output['negative']
     assert abs(output['negative'] - 0.00001503) < 1e-4
     assert abs(output['positive'] - 0.99998497) < 1e-4
+    assert isinstance(output['negative'], float)
+    assert isinstance(output['positive'], float)