util/upsample.py

from scipy.ndimage.filters import gaussian_filter
from scipy.interpolate import RectBivariateSpline
from scipy.ndimage.interpolation import zoom
import numpy

def upsampleL(fieldmap, activation_data, reduction=1, shape=None,
        scaleshape=None, out=None):
    '''
    Applies a bilinear upsampling.
    '''
    offset, size, step = fieldmap
    input_count = activation_data.shape[0]
    if len(activation_data.shape) == 2:
        ay, ax = centered_arange(fieldmap, activation_data.shape, reduction)
        if shape is None:
            shape = upsampled_shape(
                fieldmap, activation_data.shape, reduction)
    else:
        ay, ax = centered_arange(fieldmap, activation_data.shape[1:], reduction)
        if shape is None:
            shape = upsampled_shape(
                fieldmap, activation_data.shape[1:], reduction)
    if scaleshape is not None:
        iy, ix = full_arange(scaleshape)
        # TODO: consider treaing each point as a center of a pixel
        iy *= shape[0] / scaleshape[0]
        ix *= shape[1] / scaleshape[1]
    else:
        iy, ix = full_arange(shape)
    if out is None:
        out = numpy.empty((input_count, len(iy), len(ix)),
                dtype=activation_data.dtype)
    if len(activation_data.shape) == 2:
        f = RectBivariateSpline(ay, ax, activation_data, kx=1, ky=1)
        return f(iy, ix, grid=True)
    else:
        for z in range(input_count):
            f = RectBivariateSpline(ay, ax, activation_data[z], kx=1, ky=1)
            out[z] = f(iy, ix, grid=True)
    return out

def upsampleC(fieldmap, activation_data, shape=None, out=None):
    '''
    Applies a bicubic upsampling.
    '''
    offset, size, step = fieldmap
    input_count = activation_data.shape[0]
    ay, ax = centered_arange(fieldmap, activation_data.shape[1:])
    if shape is None:
        shape = upsampled_shape(fieldmap, activation_data.shape[1:])
    iy, ix = full_arange(shape)
    if out is None:
        out = numpy.empty((input_count,) + shape,
                dtype=activation_data.dtype)
    for z in range(input_count):
        f = RectBivariateSpline(ay, ax, activation_data[z], kx=3, ky=3)
        out[z] = f(iy, ix, grid=True)
    return out

def upsampleG(fieldmap, activation_data, shape=None):
    '''
    Upsampling utility functions
    '''
    offset, size, step = fieldmap
    input_count = activation_data.shape[0]
    if shape is None:
        shape = upsampled_shape(fieldmap, activation_data.shape[1:])
    activations = numpy.zeros((input_count,) + shape)
    activations[(slice(None),) +
            centered_slice(fieldmap, activation_data.shape[1:])] = (
        activation_data * numpy.prod(step))
    blurred = gaussian_filter(
        activations,
        sigma=(0, ) + tuple(t // 1.414 for o, s, t in zip(*fieldmap)),
        mode='constant')
    return blurred

def topo_sort(layers):
    # First, build a links-from and also a links-to graph
    links_from = {}
    links_to = {}
    for layer in layers:
        for bot in layer.bottom:
            if bot not in links_from:
                links_from[bot] = []
            links_from[bot].append(layer)
        for top in layer.top:
            if top not in links_to:
                links_to[top] = []
            links_to[top].append(layer)
    # Now do a DFS to figure out the ordering (using links-from)
    visited = set()
    ordering = []
    stack = []
    for seed in links_from:
        if seed not in visited:
            stack.append((seed, True))
            stack.append((seed, False))
            visited.add(seed)
            while stack:
                (blob, completed) = stack.pop()
                if completed:
                    ordering.append(blob)
                elif blob in links_from:
                    for layer in links_from[blob]:
                        for t in layer.top:
                            if t not in visited:
                                stack.append((t, True))
                                stack.append((t, False))
                                visited.add(t)
    # Return a result in front-to-back order, with incoming links for each
    return list((blob, links_to[blob] if blob in links_to else [])
            for blob in reversed(ordering))

def composed_fieldmap(layers, end):
    ts = topo_sort(layers)
    fm_record = {}
    for blob, layers in ts:
        # Compute fm's on all the edges that go to this blob.
        all_fms = [
            (compose_fieldmap(fm_record[bot][0], layer_fieldmap(layer)),
                fm_record[bot][1] + [(bot, layer)])
            for layer in layers for bot in layer.bottom if bot != blob]
        # And take the max fieldmap.
        fm_record[blob] = max_fieldmap(all_fms)
        if blob == end:
            return fm_record[blob]

def max_fieldmap(maps):
    biggest, bp = None, None
    for fm, path in maps:
        if biggest is None:
            biggest, bp = fm, path
        elif fm[1][0] > biggest[1][0]:
            biggest, bp = fm, path
    # When there is no biggest, for example when maps is the empty array,
    # use the trivial identity fieldmap with no path.
    if biggest is None:
        return ((0, 0), (1, 1), (1, 1)), []
    return biggest, bp

def shortest_layer_path(start, end, layers):
    # First, build a blob-to-outgoing-layer graph
    links_from = {}
    for layer in layers:
        for bot in layer.bottom:
            if bot not in links_from:
                links_from[bot] = []
            links_from[bot].append(layer)
    # Then do a BFS on the graph to find the shortest path to 'end'
    queue = [(s, []) for s in start]
    visited = set(start)
    while queue:
        (blob, path) = queue.pop(0)
        for layer in links_from[blob]:
            for t in layer.top:
                if t == end:
                    return path + [layer]
                if t not in visited:
                    queue.append((t, path + [layer]))
                    visited.add(t)
    return None

def upsampled_shape(fieldmap, shape, reduction=1):
    # Given the shape of a layer's activation and a fieldmap describing
    # the transformation to original image space, returns the shape of
    # the input size
    return tuple(((w - 1) * t + s + 2 * o) // reduction
            for (o, s, t), w in zip(zip(*fieldmap), shape))

def make_mask_set(image_shape, fieldmap, activation_data,
              output=None, sigma=0.1, threshold=0.5, percentile=None):
    """Creates a set of receptive field masks with uniform thresholds
    over a range of inputs.
    """
    offset, shape, step = fieldmap
    input_count = activation_data.shape[0]
    activations = numpy.zeros((input_count,) + image_shape)
    activations[(slice(None),) +
            centered_slice(fieldmap, activation_data.shape[1:])] = (
        activation_data)
    blurred = gaussian_filter(
        activations,
        sigma=(0, ) + tuple(s * sigma for s in shape),
        mode='constant')
    if percentile is not None:
        limit = blurred.ravel().percentile(percentile)
        return blurred > limit
    else:
        maximum = blurred.ravel().max()
        return (blurred > maximum * threshold)

def safezoom(array, ratio, output=None, order=0):
    '''Like numpy.zoom, but does not crash when the first dimension
    of the array is of size 1, as happens often with segmentations'''
    dtype = array.dtype
    if array.dtype == numpy.float16:
        array = array.astype(numpy.float32)
    if array.shape[0] == 1:
        if output is not None:
            output = output[0,...]
        result = zoom(array[0,...], ratio[1:],
                output=output, order=order)
        if output is None:
            output = result[numpy.newaxis]
    else:
        result = zoom(array, ratio, output=output, order=order)
        if output is None:
            output = result
    return output.astype(dtype)

def receptive_field(location, fieldmap):
    """Computes the receptive field of a specific location.

    Parameters
    ----------
    location: tuple
        The x-y position of the unit being queried.
    fieldmap:
        The (offset, size, step) tuple fieldmap representing the
        receptive field map for the layer being queried.
    """
    return compose_fieldmap(fieldmap, (location, (1, 1), (1, 1)))[:2]


def proto_getattr(p, a, d):
    hf = True
    # Try using HasField to detect the presence of a field;
    # if there is no HasField, then just use getattr.
    try:
        hf = p.HasField(a)
    except:
        pass
    if hf:
        return getattr(p, a, d)
    return d

def wh_attr(layer, attrname, default=0, minval=0):
    if not hasattr(default, '__len__'):
        default = (default, default)
    val = proto_getattr(layer, attrname, None)
    if val is None or val == []:
        h = max(minval, getattr(layer, attrname + '_h', default[0]))
        w = max(minval, getattr(layer, attrname + '_w', default[1]))
    elif hasattr(val, '__len__'):
        h = val[0]
        w = val[1] if len(val) >= 2 else h
    else:
        h = val
        w = val
    return (h, w)

def layer_fieldmap(layer):
    # Only convolutional and pooling layers affect geometry.
    if layer.type == 'Convolution' or layer.type == 'Pooling':
        if layer.type == 'Pooling':
            config = layer.pooling_param
            if config.global_pooling:
                return ((0, 0), (None, None), (1, 1))
        else:
            config = layer.convolution_param
        size = wh_attr(config, 'kernel_size', wh_attr(config, 'kernel', 1))
        stride = wh_attr(config, 'stride', 1, minval=1)
        padding = wh_attr(config, 'pad', 0)
        neg_padding = tuple((-x) for x in padding)
        return (neg_padding, size, stride)
    # All other layers just pass through geometry unchanged.
    return ((0, 0), (1, 1), (1, 1))

def layerarray_fieldmap(layerarray):
    fieldmap = ((0, 0), (1, 1), (1, 1))
    for layer in layerarray:
        fieldmap = compose_fieldmap(fieldmap, layer_fieldmap(layer))
    return fieldmap

# rf1 is the lower layer, rf2 is the higher layer
def compose_fieldmap(rf1, rf2):
    """Composes two stacked fieldmap maps.

    Field maps are represented as triples of (offset, size, step),
    where each is an (x, y) pair.

    To find the pixel range corresponding to output pixel (x, y), just
    do the following:
       start_x = x * step[0] + offset[1]
       limit_x = start_x + size[0]
       start_y = y * step[1] + offset[1]
       limit_y = start_y + size[1]

    Parameters
    ----------
    rf1: tuple
        The lower-layer receptive fieldmap, a tuple of (offset, size, step).
    rf2: tuple
        The higher-layer receptive fieldmap, a tuple of (offset, size, step).
    """
    if rf1 == None:
        import pdb; pdb.set_trace()
    offset1, size1, step1 = rf1
    offset2, size2, step2 = rf2

    size = tuple((size2c - 1) * step1c + size1c
            for size1c, step1c, size2c in zip(size1, step1, size2))
    offset = tuple(offset2c * step1c + offset1c
            for offset2c, step1c, offset1c in zip(offset2, step1, offset1))
    step = tuple(step2c * step1c
            for step1c, step2c in zip(step1, step2))
    return (offset, size, step)

def _cropped_slices(offset, size, limit):
    corner = 0
    if offset < 0:
        size += offset
        offset = 0
    if limit - offset < size:
        corner = limit - offset
        size -= corner
    return (slice(corner, corner + size), slice(offset, offset + size))

def crop_field(image_data, fieldmap, location):
    """Crops image_data to the specified receptive field.

    Together fieldmap and location specify a receptive field on the image,
    which may overlap the edge. This returns a crop to that shape, including
    any zero padding necessary to fill out the shape beyond the image edge.
    """
    offset, size = receptive_field(fieldmap, location)
    return crop_rectangle(image_data, offset, size)

def crop_rectangle(image_data, offset, size):
    coloraxis = 0 if image_data.size <= 2 else 1
    allcolors = () if not coloraxis else (slice(None),) * coloraxis
    colordepth = () if not coloraxis else (image_data.size[0], )
    result = numpy.zeros(colordepth + size)
    (xto, xfrom), (yto, yfrom) = (_cropped_slices(
        o, s, l) for o, s, l in zip(offset, size, image_data.size[coloraxis:]))
    result[allcolors + (xto, yto)] = image_data[allcolors + (xfrom, yfrom)]
    return result

def center_location(fieldmap, location):
    if isinstance(location, numpy.ndarray):
        offset, size, step = fieldmap
        broadcast = (numpy.newaxis, ) * (len(location.shape) - 1) + (
                        slice(None),)
        step = numpy.array(step)[broadcast]
        offset = numpy.array(offset)[broadcast]
        size = numpy.array(size)[broadcast]
        return location * step + offset + size // 2
    else:
        offset, shape = receptive_field(location, fieldmap)
        return tuple(o + s // 2 for o, s in zip(offset, shape))

def centered_slice(fieldmap, activation_shape, reduction=1):
    offset, size, step = fieldmap
    r = reduction
    return tuple(slice((s // 2 + o) // r, (s // 2 + o + a * t) // r, t // r)
            for o, s, t, a in zip(offset, size, step, activation_shape))

def centered_arange(fieldmap, activation_shape, reduction=1):
    offset, size, step = fieldmap
    r = reduction
    return tuple(numpy.arange(
        (s // 2 + o) // r, (s // 2 + o + a * t) // r, t // r)[:a] # Hack to avoid a+1 points
            for o, s, t, a in zip(offset, size, step, activation_shape))

def full_arange(output_shape):
    return tuple(numpy.arange(o) for o in output_shape)