<filename>setup_scripts/burn_streams.py
import os
from tabnanny import check
import time
import pickle

# import pandas as pd
import numpy as np

# import shapely
# from shapely.geometry import Polygon, Point
import geopandas as gpd

import xarray as xr
import rioxarray as rxr
import rasterio  as rio
from rasterio import features

from numba import jit

from skimage.morphology import skeletonize
from scipy import ndimage

from pysheds.grid import Grid
from pysheds.view import Raster, ViewFinder

from pyproj import CRS, Proj

import warnings
warnings.filterwarnings('ignore')

t0 = time.time()

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DATA_DIR = os.path.join(BASE_DIR, 'source_data/')

processed_data_dir = os.path.join(BASE_DIR, 'processed_data')
data_dir = '/media/danbot/Samsung_T5/geospatial_data/'

# dem_dir = os.path.join(DATA_DIR, 'dem_data/')
# dem_dir = os.path.join(data_dir, 'DEM_data/')

# processed_dem_dir = os.path.join(dem_dir, 'processed_dem/')

processed_dem_dir = '/home/danbot/Documents/code/hysets_validation/source_data/dem_data/processed_dem/'

# specify the DEM source
# either 'EarthEnv_DEM90' or 'USGS_3DEP'
DEM_source = 'EarthEnv_DEM90'
# DEM_source = 'USGS_3DEP'

def retrieve_and_preprocess_raster(region_code):
    # load the region DEM once and iterate through all
    # region_dem_path = os.path.join(processed_dem_dir, f'{region_code}_DEM_3005_{resolution}.tif')
    if DEM_source == 'EarthEnv_DEM90':
        region_dem_path = os.path.join(processed_dem_dir, f'{region_code}_{DEM_source}_3005_{resolution}.tif')
    else:
        region_dem_path = os.path.join(processed_dem_dir, f'{region_code}_DEM_3005_{resolution}.tif')
    assert os.path.exists(region_dem_path)

    rds = rxr.open_rasterio(region_dem_path, mask_and_scale=True, dtype=np.float32)

    grid = Grid.from_raster(region_dem_path)
    dem = grid.read_raster(region_dem_path, dtype=np.float64)

    viewfinder = ViewFinder(affine=dem.affine, shape=dem.shape, crs=dem.crs, nodata=dem.nodata)

    dat = rds.data[0]

    raster = Raster(dat, viewfinder=viewfinder)
    return raster, rds

def get_river_mask(region_code, rds):
    nhn_grouped_vector_path = os.path.join(processed_data_dir, 'grouped_hydrographic_features/')
    vector_rivers_path = os.path.join(nhn_grouped_vector_path, f'{region_code}/NLFLOW/')

    # mask = create_mask(vector_rivers_path)
    rivers = gpd.read_file(vector_rivers_path, mask_and_scale=True)
    affine = rds.rio.transform(recalc=False)
    # Rasterize river shapefile
    river_raster = features.rasterize(rivers.geometry, out_shape=rds.shape[1:],
                                  transform=affine, all_touched=False)

    # Skeletonize river raster
    river_raster = skeletonize(river_raster).astype(np.uint8)

    # Create boolean mask based on rasterized river shapes
    mask = river_raster.astype(bool)
    return mask

# now create polygons using the raster just generated
def retrieve_raster(fpath):
    dem = rxr.open_rasterio(fpath, mask_and_scale=True, dtype=np.float32)
    crs = dem.rio.crs
    affine = dem.rio.transform(recalc=False)
    return dem, crs, affine

@jit(nopython=True)
def flatten_streams_windowed(dem, max_depth=1):

    n_adjustments = 0
    tot_adjustment = 0

    rows = dem.shape[0] # # of steps in y direction
    cols = dem.shape[1] # # of steps in x direction
    # print(f' dem shape = {dem.shape}')
    # create an array to track indices of all stream ends.
    stream_ends = np.empty((3,))
    stream_ends.fill(np.nan)
    # stream_nodes = np.empty((1,2))

    for i in range(rows):
        for j in range(cols):
            px_el = dem[i, j]

            # instead of iterating through the whole image,
            # just get a list of pixel indices (streams are sparse)
            if np.isfinite(px_el):
                # print(i, j, px_el)
                c1, c2 = max(0, j - max_depth), min(j + max_depth + 1, cols)
                r1, r2 = max(0, i - max_depth), min(i + max_depth + 1, rows)

                window = dem[r1:r2, c1:c2]
                # print('---------')
                # print(f'target cell ({i},{j}) el = {px_el:.1f}')
                # print(window)

                # the target cell is the centre of the flattened matrix
                # or different indices based on whether or not it's at an edge
                target_idx_coords, flat_index_loc = find_del_loc(i, j, window.shape[1], max_depth)

                # get the elevations surrounding the target pixel
                outer_vals = np.delete(window, flat_index_loc)

                # faster method of sorting when we just want the two smallest
                # and don't care about order
                two_smallest = np.partition(outer_vals, 2)[:2]

                # print(outer_vals)
                # print(np.count_nonzero(~np.isnan(outer_vals)))

                # print('')

                if np.count_nonzero(~np.isnan(outer_vals)) == 1:
                    # append the target index to track streamline terminus
                    stream_ends = np.append(stream_ends,(i, j, px_el))

                # print('outer vals and two smallest: ')
                # print(outer_vals, two_smallest)
                # if the centre pixel is higher or lower than both lowest
                # neighbours, set the pixel value to the average
                if np.isfinite(two_smallest).all():
                    if np.less(two_smallest, np.array(px_el)).all() | np.less(np.array(px_el), two_smallest).all():
                        new_el = np.mean(two_smallest)
                        dem[i, j] = new_el
                        n_adjustments += 1
                        tot_adjustment += px_el - new_el
                #         print('edited window')
                #         print(dem[r1:r2, c1:c2])
                #         print('')
                # print('')
                # print('_______')

    return dem, stream_ends, n_adjustments, tot_adjustment

@jit(nopython=True)
def find_del_loc(i, j, window_width, max_depth=1):
    if i <= max_depth:
        ti = i
    else:
        ti = max_depth
    if j <= max_depth:
        tj = j
    else:
        tj = max_depth
    # get the index of the target pixel
    # if the matrix is flattened
    flat_index = window_width * ti + tj
    # print(f'(ti, tj) = ({ti},{tj})')
    # print(f'flat_index: {flat_index}  {window_width}')
    return (int(ti), int(tj)), int(flat_index)

@jit(nopython=True)
def get_min_outer_pixel_indices(window, outer_vals, flat_index_loc, xs, ys, ix, jx, checked_indices, prevent_inf_loop, rows, cols, max_depth):

    # print(f'outer vals: {outer_vals}')
    # get the index of the smallest outer value
    total_len = len(window.flatten())

    # print('outer vals: ', outer_vals)
    min_outer_val_idx = np.nanargmin(outer_vals)
    # print('min outer val idx: ', min_outer_val_idx)

    # the middle pixel was deleted from the outer_vals array,
    # so add 1 if the minimum index is in the back half of the
    # array so we can reconstruct its 2d position
    if min_outer_val_idx >= flat_index_loc:
        min_outer_val_idx += 1

    # reconstruct the 2d indices of the outer pixel with min elevation
    # retrieve the row index that the min outer value falls in
    min_row_idx = int(np.floor(min_outer_val_idx / window.shape[1]))
    # retrieve the column index that the min outer value falls in
    min_col_idx = int(min_outer_val_idx - (min_row_idx) * window.shape[1])
    # test that we referenced the correct elevation.
    min_outer_el = window[min_row_idx, min_col_idx]

    # print(f'min idx ({min_row_idx},{min_col_idx}): el: {min_outer_el}')
    # print(f'(ix, jx)=({ix},{jx})')

    # convert the min neighbor's window index to dem index
    new_dem_ix = ix + (min_row_idx - max_depth)
    if ix < max_depth:
        new_dem_ix = ix + min_row_idx

    new_dem_jx = jx + (min_col_idx - max_depth)
    if jx < max_depth:
        new_dem_jx = jx + min_col_idx

    new_dem_idx = (new_dem_ix, new_dem_jx)

    # print('new dem idx: ', new_dem_idx)

    indices_idx = np.where(np.logical_and(xs == new_dem_idx[0], ys == new_dem_idx[1]))[0][0]

    if not np.any(np.in1d(checked_indices, indices_idx)):
        # print(f'outer val index: {min_outer_val_idx}')
        if min_outer_val_idx >= flat_index_loc:
            min_outer_val_idx -= 1

        # print('already checked cell. Update outer vals: ', outer_vals)
        outer_vals[min_outer_val_idx] = np.nan
        # print('already checked cell. Update outer vals: ', outer_vals)
        # print('are all outer vals nan?: ', np.all(np.isnan(outer_vals)))
        if np.all(np.isnan(outer_vals)):
            return new_dem_idx, min_outer_el, indices_idx, prevent_inf_loop, True
        else:
            new_dem_idx, min_outer_el, indices_idx, prevent_inf_loop, end_of_line = get_min_outer_pixel_indices(window, outer_vals, flat_index_loc, xs, ys, ix, jx, checked_indices, prevent_inf_loop, rows, cols, max_depth)
            prevent_inf_loop += 1
            if prevent_inf_loop >= 4:
                raise Exception; 'infinite loop!'

    return new_dem_idx, min_outer_el, indices_idx, prevent_inf_loop, False

def get_windows(raster, dem, ix, jx, rows, cols):
    i1, i2 = max(0, ix - 1), min(ix + 2, rows)
    j1, j2 = max(0, jx - 1), min(jx + 2, cols)
    # don't let window indices go beyond raster edges
    dem_window = dem[i1:i2, j1:j2]
    raster_window = raster[i1:i2, j1:j2]
    return raster_window, dem_window

def check_adj_slope_elevations(raster_window, dem_window, ix, jx, rows, cols):
    ta = time.time()
    # sometimes the stream vector will not line
    # up with the thalweg in the dem
    # look at surrounding (nan) cells and
    # replace the target cell elevation with a lower
    # value if there is one (not on the headwater cell)
    pairs = [[1, 0], [0, 1], [1, 2], [2, 1]]
    if ix == 0: # top row
        if jx == 0: # top left
            pairs = [[1, 0], [0, 1]]
        elif jx == cols: # top right
            pairs = [[0, 0], [1, 1]]
        else: # top middle
            pairs = [[0, 0], [1, 1], [0, 2]]

    if ix == rows: # bottom row
        if jx == cols: # bottom right
            pairs = [[1, 0], [0, 1]]
        elif jx == 0: # bottom left
            pairs = [[0, 0], [1, 1]]
        else:
            pairs = [[1, 0], [0, 1], [1, 2]]

    nan_ixs = np.argwhere(np.isnan(raster_window)).tolist()
    ics = [e for e in nan_ixs if e in pairs]
    min_adjacent_el = 1E9
    if len(ics) > 0:
        els = [raster_window[ic[0], ic[1]] for ic in ics]
        if len(els) > 0:
            min_adjacent_el = min(els)

    return min_adjacent_el

def check_adj_stream_els(dem, ci, cj):
    stream_cells = np.argwhere(np.isfinite(dem))
    adj_px = [e for e in stream_cells if tuple(e) != (ci, cj)]
    # max_adj_idx = np.argmax(dem)
    adj_els = [dem[p[0], p[1]] for p in adj_px]
    if len(adj_els) > 0:
        return (min(adj_els), max(adj_els))
    else:
        return None

def travel_stream(raster, dem, indices, check, tot_adjustment, n_adjustments, max_depth=1):
    # don't check the first (headwater) cell
    headwater_cell_unchecked = False
    n_checks = 0
    while len(indices) > 0:
        check += 1
        # if check >= 10:
        #     break

        (ix, jx) = indices.pop()

        px_el = dem[ix, jx]
        px_el_og = px_el

        rows, cols = raster.shape[0], raster.shape[1]

        (ci, cj), flat_idx = find_del_loc(ix, jx, dem.shape[1])

        raster_window, dem_window = get_windows(raster, dem, ix, jx, rows, cols)

        min_adjacent_slope_el, next_idx = check_adj_slope_elevations(raster_window, dem_window, ix, jx, rows, cols)

        if not min_adjacent_slope_el:
            pass

        # print(f'current: {i}, el: {px_el:.1f} checked_indices', checked_indices)
        # checked_el[i] = px_el
            # if there is an outer pixel with a lower elevation
        # that isn't in the stream, change the current target
        # elevation to a value slightly smaller
        if headwater_cell_unchecked & (min_adjacent_slope_el < px_el):
            # print(f'    Updating target cell el from {px_el:.1f} to {min_outer_el - 0.1:.1f}')
            dem[ix, jx] = min_adjacent_slope_el - 0.1
            headwater_cell_unchecked = False
            px_el = min_adjacent_slope_el - 0.1
            tot_adjustment +=  px_el_og - px_el
            n_adjustments += 1
            n_checks += 1

        neighbor_stream_cells = check_adj_stream_els(dem_window, ci, cj)

        headwater_cell_unchecked = True

    return dem, check, n_adjustments, tot_adjustment, n_checks

def find_and_sort_stream_cells(dem):
    # indices of stream elements
    nzidx = np.where(np.isfinite(dem))
    # ordered stream pixel indices by elevation descending
    el_ranking = np.argsort(dem[nzidx])[::-1]
    xs, ys = tuple(np.array(nzidx)[:, el_ranking])
    stream_cell_indices = np.array(tuple(zip(xs, ys)))
    return stream_cell_indices

def find_end_cells(dem, raster, stream_cell_indices, adjust_dem=True):
    """Find all terminations of the stream network,
    either headwaters or outlets.

    Args:
        dem (_type_): _description_
        raster (_type_): _description_
        stream_cell_indices (_type_): _description_
        adjust_dem (bool, optional): _description_. Defaults to True.
    """
    # n_cells = len(stream_cell_indices)
    rows, cols = dem.shape[0], dem.shape[1]
    for (ix, jx) in stream_cell_indices:
        raster_window, dem_window = get_windows(raster, dem, ix, jx, rows, cols)

def flatten_streams_streamwise(raster, dem, max_depth=1):
    n_adjustments = 0
    tot_adjustment = 0

    # ta = time.time()
    # # stream_cell_indices = find_and_sort_stream_cells(raster, dem)
    # tb = time.time()
    # print(f'Time to sort indices: {tb-ta:.1e}s')

    # headwater_indices = find_headwater_cells(dem, raster, stream_cell_indices)

    # rows, cols = dem.shape[0], dem.shape[1]
    # checked_indices = np.empty(0)

    # print(dem)
    check = 0
    # dem, check, n_adjustments, tot_adjustment, n_checks = travel_stream(raster, dem, indices, tot_adjustment, n_adjustments, check)

    # print(dem)
    # print('')
    # print('')

    # the target cell is the centre of the flattened 3x3 matrix
    # or different indices based on whether or not it's at an edge
    # which makes 2x3 (left or right edge), 3x2 (top or bottom),
    # or 2x2 (corners)
    # target_idx_coords, flat_index_loc = find_del_loc(ix, jx, window.shape[1], rows, cols, max_depth)

    # # print(f' delete flat index loc: {flat_index_loc}')

    # # get the elevations of STREAM PIXELS
    # # surrounding the target pixel
    # outer_vals = np.delete(window, flat_index_loc)

    # prevent_inf_loop = 0
    # new_dem_idx, min_outer_el, indices_idx, prevent_inf_loop, end_of_line = get_min_outer_pixel_indices(window, outer_vals, flat_index_loc, xs, ys, ix, jx, checked_indices, prevent_inf_loop, rows, cols, max_depth)

    # # if we've reached the end of the current streamline
    # # skip the rest of the loop and go to the next highest index
    # if end_of_line:
    #     # print('')
    #     # print('END OF CURRENT STREAMLINE')
    #     # print('')
    #     # reset the headwater check status so we don't modify
    #     # the headwater cell at the start of the next stream
    #     headwater_cell_unchecked = False
    #     continue

    # indices = np.delete(indices, indices_idx, axis=0)
    # if i == n_cells-1:
    #     # print(f'before append: {indices[-5:]}')
    #     indices = np.append(indices, new_dem_idx)
    #     # print(f'new index appended: {indices[-5:]}')
    # else:
    #     indices = np.insert(indices, i+1, new_dem_idx, axis=0)
    # # track how the order of indices ends up being checked

    # if not np.any(np.in1d(checked_indices, indices_idx)):
    #     checked_indices = np.append(checked_indices, int(indices_idx))

    # if np.all(np.isnan(outer_vals)):
    #     continue

    # # if the minimum outer elevation is greater than
    # # the current pixel, change it to slightly less than
    # # the current pixel
    # if min_outer_el >= px_el:
    #     # print(f'     outer px el reduced from {min_outer_el:.1f} to {px_el-0.1:.1f}')
    #     dem[new_dem_idx[0], new_dem_idx[1]] = round(px_el - 0.1, 1)
    #     tot_adjustment += px_el_og - round(px_el - 0.1, 1)
    #     n_adjustments += 1

    # # remove the updated cell from the main list
    # # so we don't loop around it again
    # # find the min neighbor's index in the ordered indices array
    # el_new = dem[new_dem_idx]

    return dem, n_adjustments, tot_adjustment, n_checks

dir_method = 'D8' # D8, DINF
delineation_method = 'PYSHEDS'
# for region in code

resolution = 'res1'

dem_files = os.listdir(processed_dem_dir)
dem_files = [e for e in dem_files if resolution in e]

region_codes = sorted(list(set([e.split('_')[0] for e in dem_files])))

i = 1
for region_code in region_codes:
    print('___________________________________________________')
    print('')
    print(f'Starting stream burn on region {region_code} {i}/{len(region_codes)}.')

    # get the covering region for the station
    t_start = time.time()

    if 'EarthEnv' in DEM_source:
        resolution = 'res1'

    load_start = time.time()
    raster, rds = retrieve_and_preprocess_raster(region_code)
    load_end = time.time()
    print(f'    Time to load raster = {load_end-load_start:.1f}s')
    print(f'        --raster is {raster.shape[0]}px high by {raster.shape[1]}px wide')

    river_mask = get_river_mask(region_code, rds)
    mask_end = time.time()
    print(f'    Time to get river mask = {mask_end-load_end:.1f}s')

    # Blur mask using a gaussian filter
    blurred_mask = ndimage.filters.gaussian_filter(river_mask.astype(np.float64), sigma=2.5)

    # Set central channel to max to prevent pits
    blurred_mask[river_mask.astype(np.bool)] = blurred_mask.max()

    mask = blurred_mask

    # Create a view onto the DEM array
    # dem = grid.view('dem', dtype=np.float64, nodata=np.nan)
    # Set elevation change for burned cells

    dz = 6.5

    # n_px = 6

    masked_dem = np.empty_like(raster)
    masked_dem.fill(np.nan)
    # masked_dem = masked_dem.reshape()
    center = (int(raster.shape[0] / 2), int(raster.shape[1]*0.3))

    # dem = raster.view('dem', dtype=np.float64, nodata=np.nan)
    # mask the DEM by the river pixels
    masked_dem[river_mask > 0] = raster[river_mask > 0]

    t_start = time.time()

    # two-diagonal hump
    # test_dem = masked_dem[center[0]-10:center[0]+10, center[1]-70:center[1]-60].round(0)

    # # cut a sample window to test the algorithm
    # v1, v2 = -50, 50
    # h1, h2 = -50, 50
    # top_px, bot_px = max(center[0]+v1, 0), min(center[0]+v2, raster.shape[0])
    # left_px, right_px = max(center[0]+h1, 0), min(center[0]+h2, raster.shape[1])
    # test_dem = masked_dem[top_px:bot_px, left_px:right_px].round(1)
    # test_raster = raster[top_px:bot_px, left_px:right_px].round(1)

    # test_size = test_dem.shape[0] * test_dem.shape[1]
    # print(f'test raster size = ({test_dem.shape}) ({test_size:.1e}px)')

    # if (test_dem.shape == masked_dem.shape):
    #     print('***************************')
    #     print('Maximum test size reached.')
    #     print('***************************')
    # print(test_dem)
    modified_dem, stream_ends, n_adjustments, tot_adjustment = flatten_streams_windowed(masked_dem)

    # print('raster range: ', np.nanmin(raster),np.nanmax(raster))
    # print('min el in masked dem')
    # print(np.nanmin(masked_dem))

    # remove the first nan entry of a 3-tuple
    # stream_ends = stream_ends[3:]
    # stream_ends = stream_ends.reshape(int(len(stream_ends)/3),3)
    # n_ends = len(stream_ends)
    # print(f'Found {n_ends} stream ends.')
    # # sort the array by elevations (2nd index)
    # stream_ends = stream_ends[stream_ends[:,2].argsort()]

    # t_win = time.time()
    # raster_size = masked_dem.shape[0] * masked_dem.shape[1]
    # print(f'   {raster_size} px raster processed in {t_win-t_start:.2e}s')

    # modified_dem, n_adjustments, tot_adjustment, n_checks = flatten_streams_streamwise(test_raster, test_dem)
    # modified_dem, n_adjustments, tot_adjustment = flatten_streams_streamwise(raster, masked_dem)
    # print('')
    # t_end = time.time()

    # print(modified_dem)
    # print('')
    # print(f'{n_adjustments}/{test_size} cells adjusted in {n_checks} checks, total adjustment = {tot_adjustment:.1f}m')
    # print(f'   {test_size} px raster processed in {t_end-t_start:.2e}s')

    # raster_size = raster.shape[0] * raster.shape[1]
    # print(f'   Stream flattening completed in {t_end -t_start:.2e} for {raster_size:.2e} pixels. {n_adjustments} pixels adjusted, {tot_adjustment} m total adjustment.')

    # replace the river pixels in the original raster
    # with the modified values
    # raster[river_mask > 0] = modified_dem[river_mask > 0]

    # Subtract a constant dz from all stream pixels
    raster[river_mask > 0] -= dz

    out_fname = f'{region_code}_{DEM_source}_burned_streams_{resolution}.tif'
    processed_out_dir = f'/media/danbot/Samsung_T5/geospatial_data/DEM_data/processed_dem/'
    out_path = os.path.join(processed_out_dir, out_fname)

    rds.data[0] = raster.data
    # rds.rio.write_nodata(-32768, inplace=True)

    rds.rio.to_raster(out_path)#, nodata=-32768)
    # dem.to_raster(grid, out_path)

    t_end_cond = time.time()
    t_cond = t_end_cond - t_start
    print(f'    ...stream burn for {region_code} in {t_cond:.1f}.  Created {out_fname}')
    i += 1


<filename>silx/opencl/sift/test/test_transform.py<gh_stars>1-10
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#    Project: Sift implementation in Python + OpenCL
#             https://github.com/silx-kit/silx
#
#    Copyright (C) 2013-2017  European Synchrotron Radiation Facility, Grenoble, France
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.

"""
Test suite for transformation kernel
"""

from __future__ import division, print_function

__authors__ = ["<NAME>", "<NAME>"]
__contact__ = "<EMAIL>"
__license__ = "MIT"
__copyright__ = "2013 European Synchrotron Radiation Facility, Grenoble, France"
__date__ = "25/06/2018"

import os
import unittest
import time
import logging
import numpy
try:
    import scipy.misc
    import scipy.ndimage
except ImportError:
    scipy = None

from silx.opencl import ocl, kernel_workgroup_size
if ocl:
    import pyopencl.array
# for Python implementation of tested functions
# from .test_image_functions import
# from .test_image_setup import
from ..utils import calc_size, get_opencl_code, matching_correction
from ..plan import SiftPlan
from ..match import MatchPlan
from silx.test.utils import test_options
logger = logging.getLogger(__name__)

@unittest.skipUnless(scipy and ocl, "scipy or ocl missing")
class TestTransform(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        super(TestTransform, cls).setUpClass()
        if ocl:
            cls.ctx = ocl.create_context()
            if logger.getEffectiveLevel() <= logging.INFO:
                cls.PROFILE = True
                cls.queue = pyopencl.CommandQueue(cls.ctx, properties=pyopencl.command_queue_properties.PROFILING_ENABLE)
            else:
                cls.PROFILE = False
                cls.queue = pyopencl.CommandQueue(cls.ctx)
            device = cls.ctx.devices[0]
            device_id = device.platform.get_devices().index(device)
            platform_id = pyopencl.get_platforms().index(device.platform)
            cls.maxwg = ocl.platforms[platform_id].devices[device_id].max_work_group_size

    @classmethod
    def tearDownClass(cls):
        super(TestTransform, cls).tearDownClass()
        cls.ctx = None
        cls.queue = None

    def setUp(self):
        kernel_src = get_opencl_code(os.path.join("sift", "transform"))
        self.program = pyopencl.Program(self.ctx, kernel_src).build()  # .build('-D WORKGROUP_SIZE=%s' % wg_size)
        self.wg = (1, 128)
        if hasattr(scipy.misc, "ascent"):
            self.image = scipy.misc.ascent().astype(numpy.float32)
        else:
            self.image = scipy.misc.lena().astype(numpy.float32)

    def tearDown(self):
        self.program = None

    def image_reshape(self, img, output_height, output_width, image_height, image_width):
        '''
        Reshape the image to get a bigger image with the input image in the center

        '''
        image3 = numpy.zeros((output_height, output_width), dtype=numpy.float32)
        d1 = (output_width - image_width) // 2
        d0 = (output_height - image_height) // 2
        image3[d0:-d0, d1:-d1] = numpy.copy(img)
        image = image3
        image_height, image_width = output_height, output_width
        return image, image_height, image_width

    @unittest.skipIf(test_options.TEST_LOW_MEM, "low mem")
    def test_transform(self):
        '''
        tests transform kernel
        '''

        # Transformation
        # ---------------
        matrix = numpy.array([[1.0, -0.75], [0.7, 0.5]], dtype=numpy.float32)
        offset_value = numpy.array([250.0, -150.0], dtype=numpy.float32)
        transformation = lambda img: scipy.ndimage.interpolation.affine_transform(img, matrix, offset=offset_value, order=1, mode="constant")
        image_transformed = transformation(self.image)

        fill_value = numpy.float32(0.0)
        mode = numpy.int32(1)

        # computing keypoints matching with SIFT
        sift_plan = SiftPlan(template=self.image, block_size=self.maxwg)
        kp1 = sift_plan.keypoints(self.image)
        kp2 = sift_plan.keypoints(image_transformed)  # image2 and image must have the same size
        match_plan = MatchPlan()  # cls.ctx
        matching = match_plan.match(kp2, kp1)

        # Retrieve the linear transformation from the matching pairs
        sol = matching_correction(matching)
        logger.info(sol)

        # Compute the correction matrix (inverse of transformation)
        correction_matrix = numpy.zeros((2, 2), dtype=numpy.float32)
        correction_matrix[0] = sol[0:2, 0]
        correction_matrix[1] = sol[3:5, 0]
        matrix_for_gpu = correction_matrix.reshape(4, 1)  # for float4 struct
        offset_value[0] = sol[2, 0]
        offset_value[1] = sol[5, 0]

        # Prepare the arguments for the "transform" kernel call
        maxwg = kernel_workgroup_size(self.program, "transform")
        wg = maxwg, 1
        shape = calc_size(self.image.shape[::-1], wg)
        gpu_image = pyopencl.array.to_device(self.queue, image_transformed)
        gpu_output = pyopencl.array.empty(self.queue, self.image.shape, dtype=numpy.float32, order="C")
        gpu_matrix = pyopencl.array.to_device(self.queue, matrix_for_gpu)
        gpu_offset = pyopencl.array.to_device(self.queue, offset_value)
        image_height, image_width = numpy.int32(self.image.shape)
        output_height, output_width = numpy.int32(gpu_output.shape)
        kargs = [
            gpu_image.data,
            gpu_output.data,
            gpu_matrix.data,
            gpu_offset.data,
            image_width,
            image_height,
            output_width,
            output_height,
            fill_value, mode
        ]

        # Call the kernel
        t0 = time.time()
        k1 = self.program.transform(self.queue, shape, wg, *kargs)
        res = gpu_output.get()

        # Reference result
        t1 = time.time()
        ref = scipy.ndimage.interpolation.affine_transform(image_transformed, correction_matrix,
                                                           offset=offset_value,
                                                           output_shape=(output_height, output_width),
                                                           order=1,
                                                           mode="constant",
                                                           cval=fill_value)
        t2 = time.time()

        # Compare the implementations
        delta = numpy.abs(res - ref)
        delta_arg = delta.argmax()
        delta_max = delta.max()
        at_0, at_1 = delta_arg / output_width, delta_arg % output_width
        logger.info("Max difference wrt scipy : %f at (%d, %d)", delta_max, at_0, at_1)

        if self.PROFILE:
            logger.info("Global execution time: CPU %.3fms, GPU: %.3fms.", 1000.0 * (t2 - t1), 1000.0 * (t1 - t0))
            logger.info("Transformation took %.3fms", 1e-6 * (k1.profile.end - k1.profile.start))

def suite():
    testSuite = unittest.TestSuite()
    testSuite.addTest(TestTransform("test_transform"))
    return testSuite


from PyQt5 import QtWidgets
from PyQt5.QtCore import QThread
from scipy.ndimage import center_of_mass
import numpy as np
from pymodaq.daq_utils.daq_utils import linspace_step
from pymodaq_pid.utils import PIDModelGeneric, OutputToActuator

class PIDModelBeamSteering(PIDModelGeneric):
    limits = dict(max=dict(state=True, value=1),
                  min=dict(state=True, value=-1),)
    konstants = dict(kp=4, ki=0.05, kd=0.001)

    Nsetpoint = 2
    setpoint_ini = [600. for ind in range(Nsetpoint)]

    actuators_name = ["Haxis", "Vaxis"]
    detectors_name = ['Camera']
    params = [
        {'title': 'Threshold', 'name': 'threshold', 'type': 'float', 'value': 10.},
        {'title': 'Correction sign:', 'name': 'correction_sign', 'type': 'int', 'value': 1, 'min': -1, 'max': 1,
         'step': 2},
        {'title': 'Calibration', 'name': 'calibration', 'type': 'group', 'expanded': True, 'visible': True,
         'children': [
            {'title': 'Start calibration:', 'name': 'start_calibration', 'type': 'bool', 'value': False},
            {'title': 'Timeout:', 'name': 'timeout', 'type': 'int', 'value': 10000},
            {'title': 'Calibration scan', 'name': 'calibration_scan', 'type': 'group', 'expanded': True,
             'visible': True, 'children': [
                {'title': 'Number of steps:', 'name': 'number_of_steps', 'type': 'int', 'value': 5},
                {'title': 'Step size:', 'name': 'step_size', 'type': 'int', 'value': 2}]},
            {'title': 'Fit parameters', 'name': 'fit_parameters', 'type': 'group', 'expanded': True, 'visible': True,
             'children': [
                {'title': 'Slope:', 'name': 'slope', 'type': 'float', 'value': 0.0},
                {'title': 'Offset:', 'name': 'offset', 'type': 'float', 'value': 0.0}]}]}
    ]

    def __init__(self, pid_controller):
        super().__init__(pid_controller)
        self.pid_controller = pid_controller

    def update_settings(self, param):
        """
        Get a parameter instance whose value has been modified by a user on the UI
        Parameters
        ----------
        param: (Parameter) instance of Parameter object
        """
        if param.name() == '':
            pass

    def get_beam_position(self, image):
        """
        Get the center of mass of the beam from the image of the camera.
        All the pixels that are under the threshold are nulled so they do not contribute to the center of mass
        calculation.
        Parameters
        ----------
        image: (numpy 2d array) Image from the camera.

        Returns
        -------
        float: the pixel number corresponding to the center of mass of the beam along this axis.
        """
        image = image - self.settings.child('threshold').value()
        image[image < 0] = 0
        x, y = center_of_mass(image)

        return x, y

    def ini_model(self):
        super().ini_model()

    def convert_input(self, measurements):
        """
        Convert the image of the camera into x and y positions of the center of the beam.
        Parameters
        ----------
        measurements: (Ordereddict) Data from the camera

        Returns
        -------
        tuple: the coordinate of the center of the beam
        """
        image = measurements['Camera']['data2D']['Camera_Mock2DPID_CH000']['data']
        x, y = self.get_beam_position(image)
        self.curr_input = [x, y]
        return self.get_beam_position(image)

    def convert_output(self, output, dt, stab=True):
        """
        Convert the output of the PID in units to be fed into the actuator
        Parameters
        ----------
        output: (float) output value from the PID from which the model extract a value of the same units as the actuator

        Returns
        -------
        list: the converted output as a list (if there are a few actuators)

        """
        #print('output converted')

        self.curr_output = output
        return OutputToActuator(mode='rel', values=[output])

def main():
    from pymodaq.dashboard import DashBoard
    from pymodaq.daq_utils.daq_utils import get_set_preset_path
    from pymodaq.daq_utils import gui_utils as gutils
    from pathlib import Path
    from PyQt5 import QtWidgets
    from pymodaq_pid.pid_controller import DAQ_PID

    import sys
    app = QtWidgets.QApplication(sys.argv)
    win = QtWidgets.QMainWindow()
    area = gutils.DockArea()
    win.setCentralWidget(area)
    win.resize(1000, 500)
    win.setWindowTitle('PyMoDAQ Dashboard')

    dashboard = DashBoard(area)
    file = Path(get_set_preset_path()).joinpath("BeamSteering.xml")
    if file.exists():
        dashboard.set_preset_mode(file)
        # prog.load_scan_module()
        pid_area = gutils.DockArea()
        pid_window = QtWidgets.QMainWindow()
        pid_window.setCentralWidget(pid_area)

        prog = DAQ_PID(pid_area, dashboard.modules_manager)
        pid_window.show()
        pid_window.setWindowTitle('PidController')
        QtWidgets.QApplication.processEvents()

    else:
        msgBox = QtWidgets.QMessageBox()
        msgBox.setText(f"The default file specified in the configuration file does not exists!\n"
                       f"{file}\n"
                       f"Impossible to load the DAQ_PID Module")
        msgBox.setStandardButtons(msgBox.Ok)
        ret = msgBox.exec()

    sys.exit(app.exec_())

if __name__ == '__main__':
    main()


from scipy.optimize import Bounds, LinearConstraint, minimize, minimize_scalar,fmin_powell,fmin,basinhopping
from math import pi
import cantera as ct
import numpy as np
import pickle
from keras.layers import Input, Dense, Activation, concatenate
from keras.models import Model
from keras import losses, metrics
from keras import optimizers
from keras import callbacks
from keras.callbacks import CSVLogger
from keras.utils import plot_model
from keras import backend as K
from scipy.optimize.optimize import fminbound
import tensorflow as tf
import itertools
from functools import partial
NAME='0606_FPC_modelV11'
def objective_function(delta_T, hf_obj, prev_y,raw_T_list,mfr=53.053,p=13,ccl4=1000,
                       mole_cracking_heat=171, Cp=0.29, Area=pi*11.1*2.54*18/100):

    T_list = raw_T_list + [raw_T_list[-1]+delta_T]
    print(f"\nTubes {len(T_list)-1}:")
    print(list(np.round(np.hstack(T_list),2)))

    compos,t,t_sum = EDC_cracking(T_list,p,ccl4,mfr,n_pfr=len(T_list)-1)
    with open(f'../results/{NAME}/clf.pickle', 'rb') as f:
        scaler = pickle.load(f)
    # Load model
    model = build_model()
    model.load_weights(f"../results/{NAME}/model.h5")
    x_predict=np.hstack([T_list[-2], T_list[-1], compos,
                     p, ccl4, t_sum, t, prev_y]).reshape(1,-1)
    rescaled_X_predict = scaler.transform(x_predict[:, :-1])
    x_predict = [rescaled_X_predict[:, 0:2],
                    rescaled_X_predict[:, 2:], x_predict[:, -1]]
    X = float(model.predict(x_predict))
    print(f"prev_y:{prev_y*100:.2f}%")
    print(f"X:{X*100:.2f}%")

    print(f"delta_X:{(X-prev_y)*100:.2f}%")
    print(f"delta_T:{delta_T}")
    mfr *= 1000  # T/H to kg/H
    EDC_cracked = (X-prev_y)*mfr  # already / 100

    Q1 = mfr * Cp * delta_T

    print(f"Q1:{Q1}")

    Q2 = EDC_cracked * mole_cracking_heat

    print(f"Q2:{Q2}")
    print(f"hf_obj:{hf_obj}")
    hf = (Q1+Q2)/Area  # surface area
    print(f"hf_calculated:{hf}")
    if delta_T > 25:
        Q1 *= 1e4
    if X <= prev_y:
        Q2 = 1e15*abs(X-prev_y)
    hf = (Q1+Q2)/Area
    print(f"hf loss: {abs(hf-hf_obj)}")
    if(abs(hf-hf_obj)< 500):
        return 0
    else:
        return abs(hf-hf_obj)/1000

def build_model(lr=0.001):
    first_input = Input(shape=(2,), name='Input_layer_1')
    second_input = Input(shape=(31,), name='Input_layer_2')
    third_input = Input(shape=(1,), name='Prev_cracking')

    layer = Dense(13, name='Hinden_layer_1')(first_input)
    layer = Activation('relu')(layer)
    layer = Dense(15, name='Hinden_layer_2')(layer)
    layer = Activation('relu')(layer)
    # layer = Dense(16, name='Hinden_layer_3')(layer)
    # layer = Activation('relu')(layer)
    layer = concatenate([layer, second_input], name='Concatenate_layer')
    layer = Activation('relu')(layer)
    # layer = Dense(11, name='Hinden_layer_4')(layer)
    # layer = Activation('relu')(layer)
    layer = Dense(12, name='Hinden_layer_4')(layer)
    layer = Activation('relu')(layer)
    layer = concatenate([layer, third_input], name='Concatenate_layer_2')
    layer = Dense(1, name='Hinden_layer_5')(layer)
    output = Activation('sigmoid')(layer)
    model = Model(inputs=[first_input, second_input, third_input],
                  outputs=output)
    model.compile(optimizer=optimizers.Adam(lr=lr),
                  loss=losses.mean_absolute_error,
                  metrics=['accuracy', 'mae'])
    # model.compile(optimizer=optimizers.Adam(lr=lr),
    #               loss=losses.mean_absolute_error,
    #               metrics=[metrics.MeanAbsoluteError()])

    # model.compile(optimizer=optimizers.Adam(lr=lr),
    #               loss='mse',
    #               metrics=[tf.keras.metrics.MeanSquaredError()])
    return model

def EDC_cracking(
        T_list,
        pressure_0,
        CCl4_X_0,
        mass_flow_rate,
        reaction_mech='../../KM/2009_Schirmeister_EDC/test.cti',
        n_steps=1000,
        n_pfr=22,
        length=18,
        area=3.14 * (262 / 1000) ** 2 / 4
):
    """
    Module that runs a single PFR Cantera simulation via a series of CSTRs.
    The Plug flow reactor is represented by a linear chain of zero-dimensional
    reactors. The gas at the inlet to the first one has the specified inlet
    composition, and for all others the inlet composition is fixed at the
    composition of the reactor immediately upstream. Since in a PFR model there
    is no diffusion, the upstream reactors are not affected by any downstream
    reactors, and therefore the problem may be solved by simply marching from
    the first to last reactor, integrating each one to steady state.
    Parameters
    =============== =============================================================
    Attribute       Description
    =============== =============================================================
    `reaction_mech` Cantera reaction mechanism (.cti file)
    `T_list`        Temperature profile (°C)
    `pressure_0`    Initial pressue (atm)
    `CCl4_X_0`      Initial CCl4 concentration (mass fraction)
    `mass_flow_rate`Mass flow rate of input gas (T/H)
    `n_steps`       Number of iterations/number of CSTRs
    `n_pfr`         Number of PFRs
    `length`        Length of each PFR (m)
    `area`          Cross-sectional area (m**2)
    `label`         Label of this mechanism
    =============== =============================================================

    """
    #######################################################################
    # Input Parameters
    #######################################################################
    if CCl4_X_0 > 1:  # ppm
        CCl4_X_0 = float(CCl4_X_0) / 1000000
    T_0 = 273.15 + T_list[0]  # inlet temperature [K]
    pressure_0 *= ct.one_atm
    spcs = ct.Species.listFromFile(reaction_mech)
    for spc in spcs[::-1]:
        if spc.composition == {'C': 2.0, 'Cl': 2.0, 'H': 4.0} and spc.charge == 0:
            EDC_label = spc.name
        if spc.composition == {'C': 1.0, 'Cl': 4.0} and spc.charge == 0:
            CCl4_label = spc.name
    EDC_X_0 = 1 - CCl4_X_0
    composition_0 = '{}:{}, {}:{}'.format(
        EDC_label, EDC_X_0, CCl4_label, CCl4_X_0)
    mass_flow_rate *= 1000 / 3600  # T/H to kg/s

    # import the gas model and set the initial conditions
    model = ct.Solution(reaction_mech)
    model.TPY = T_0, pressure_0, composition_0
    dz = length / n_steps
    r_vol = area * dz

    # create a new reactor
    r = ct.IdealGasReactor(model)
    r.volume = r_vol

    # create a reservoir to represent the reactor immediately upstream. Note
    # that the gas object is set already to the state of the upstream reactor
    upstream = ct.Reservoir(model, name='upstream')

    # create a reservoir for the reactor to exhaust into. The composition of
    # this reservoir is irrelevant.
    downstream = ct.Reservoir(model, name='downstream')

    # The mass flow rate into the reactor will be fixed by using a
    # MassFlowController object.
    m = ct.MassFlowController(upstream, r, mdot=mass_flow_rate)

    # We need an outlet to the downstream reservoir. This will determine the
    # pressure in the reactor. The value of K will only affect the transient
    # pressure difference.
    v = ct.PressureController(r, downstream, master=m, K=1e-5)

    sim = ct.ReactorNet([r])

    # define time, space, and other information vectors
    z = (np.arange(n_steps) + 1) * dz
    t = np.zeros(n_pfr)  # residence time in each PFR reactor
    # compositions of output stream in each PFR reactor
    compositions = [None] * n_pfr
    states = ct.SolutionArray(r.thermo)

    cracking_rates = [0]
    for i, T in enumerate(T_list[1:]):
        Ti = T_list[i] + 273.15
        Te = T + 273.15
        dT = (Te - Ti) / n_steps
        T = Ti
        t_r = np.zeros_like(z)  # residence time in each CSTR reactor
        # iterate through the PFR cells
        for n in range(n_steps):
            # simulate the linear T-profile in each reactor
            T = Ti + (n + 1) * dT
            model.TP = T, None
            r.syncState()
            # Set the state of the reservoir to match that of the previous reactor
            model.TPY = r.thermo.TPY
            upstream.syncState()
            # integrate the reactor forward in time until steady state is reached
            sim.reinitialize()
            sim.set_initial_time(0)
            sim.advance_to_steady_state()
            # compute velocity and transform into time
            t_r[n] = r.mass / mass_flow_rate  # residence time in this reactor
            # write output data
            states.append(r.thermo.state)
        t[i] = np.sum(t_r)
        compositions[i] = model.Y[4:]
        cracking_rate = (
            EDC_X_0 - model.Y[model.species_index(EDC_label)]) / EDC_X_0
        cracking_rates.append(cracking_rate)
        t_total = np.sum(t)
    return compositions[-1], t[-1], t_total

class MinimizeStopper(object):
    def __init__(self, f=objective_function, tau=10):
        self.fun = f                     # set the objective function
        self.best_x = None
        self.best_func = np.inf
        self.tau = tau                   # set the user-desired threshold

    def __call__(self, xk, convergence=None,  *args, **kwds):
        fval = self.fun(xk, *args, **kwds)
        if fval < self.best_func:
            self.best_func = fval
            self.best_x = xk
        if self.best_func <= self.tau:
            print("Terminating optimization: objective function threshold triggered")
            print(self.best_x)
            return True
        else:
            return False

def optimize_hf(Te, Ti=350., target_X=55.0, mfr=53.053, p=13., CCl4=1000,
                mole_cracking_heat=171, Cp=0.29, SurArea=pi*11.1*2.54*18/100):
    Total_hf = (target_X * mole_cracking_heat*mfr*1000/100 + (Te - Ti)*mfr*Cp*1000)/SurArea
    # ratio of per tube to total hf
    ratio = [0.045775715, 0.052760646, 0.054246201, 0.057755337, 0.059452529, 0.060304555, 0.06103342,
             0.060060211, 0.058149083, 0.052729307, 0.048923541, 0.045456762, 0.042852356, 0.04072963,
             0.038601821, 0.036790655, 0.0348499, 0.032971834, 0.031425517, 0.029733461, 0.028328363, 0.027069159]
    T_list = [Ti]
    X=[0]
    bounds=[[8,25],[8,25],[8,25],[8,25],[5,20],[3,18],[3,18],[3,18],[1,10],[1,8],[1,6],[0,3],[0,3],[0,2],[0,2],[0,1],[0,1],[0,1]]
    for i in range(22):
        # res = fminbound(objective_function,bounds[i][0],bounds[i][1],args=(ratio[i]*Total_hf,X[-1],T_list),
            # disp=3)
        # res = basinhopping(objective_function,bounds[i][0],interval=5)
        # res = minimize(objective_function,bounds[i][0],method='Nelder-Mead',args=(ratio[i]*Total_hf,X[-1],T_list),tol=1e-2,ptions={'fatol': 10})
        res = fmin(objective_function,1,xtol=0.05,maxiter=1000,args=(ratio[i]*Total_hf,X[-1],T_list),
            disp=True)
        T_list.append(T_list[-1]+res[0])
        compos,t,t_sum = EDC_cracking(T_list,p,CCl4,mfr,n_pfr=len(T_list)-1)
        with open(f'../results/{NAME}/clf.pickle', 'rb') as f:
            scaler = pickle.load(f)
        # Load model
        model = build_model()
        model.load_weights(f"../results/{NAME}/model.h5")
        x_predict=np.hstack([T_list[-2], T_list[-1], compos,
                        p, CCl4, t_sum, t, X[-1]]).reshape(1,-1)
        rescaled_X_predict = scaler.transform(x_predict[:, :-1])
        x_predict = [rescaled_X_predict[:, 0:2],
                        rescaled_X_predict[:, 2:], x_predict[:, -1]]
        X.append(float(model.predict(x_predict)))
    print('T_list:')
    print(','.join(map(str(T_list))))
    print(X)

if __name__ == '__main__':
    optimize_hf(Te=465.)


<gh_stars>10-100
import scipy.io
import numpy as np

def main():
    mat = scipy.io.loadmat('data/calibration_dataset.mat')
    pts1 = np.array(mat['mA'], dtype=np.float)[:, :, np.newaxis]
    pts2 = np.array(mat['mB'], dtype=np.float)[:, :, np.newaxis]
    pts3 = np.array(mat['mC'], dtype=np.float)[:, :, np.newaxis]

    np.set_printoptions(suppress=True, precision=10)

    dev1 = dev2 = dev3 = 0.0
    for i in range(24):
        dev1 += np.sqrt(np.var(pts1[i * 10:(i + 1) * 10] * 1000, axis=0))
        dev2 += np.sqrt(np.var(pts2[i * 10:(i + 1) * 10] * 1000, axis=0))
        dev3 += np.sqrt(np.var(pts3[i * 10:(i + 1) * 10] * 1000, axis=0))

    dev1 /= 24
    dev2 /= 24
    dev3 /= 24

    dev = (dev1 + dev2 + dev3) / 3

    print("Coordinatewise deviation: ")
    print(dev)

    print("Average deviation")
    print(np.sum(dev) / 3)

if __name__ == '__main__':
    main()


<reponame>rgalhama/wordrep_cmcl2020
import sys, os
import pandas as pd
import numpy as np
from scipy import stats
import statsmodels.api as sm
import statsmodels.formula.api as smf
from statsmodels.stats.outliers_influence import variance_inflation_factor
import warnings
warnings.filterwarnings("ignore", category=RuntimeWarning)
from configs.config_loader import load_config, opts2record

data_fname="merged_indegree_train_th%.2faoa_wordbank_eng_understands_prop0.5_nouns_clean_means.csv"

header_svd="win;thr;"
header_sgns="dyn;thr"

def write_output(model_type, config, datafile, similarity_threshold, pearsonr, pearson_pval, outputpath):

    params_model=load_config(config)
    header, record=opts2record(params_model["counts"],params_model[model_type], params_model["post"])

    header="model;"+header
    record=model_type+";"+record

    header+="similarity_threshold;"
    record+=similarity_threshold+";"

    header+="pearsonr;pearson_pval"
    record+="{0:.4f};".format(pearsonr)
    record+="{0:.4f}".format(pearson_pval)

    outfile=os.path.join(outputpath, "params_corr_"+os.path.basename(datafile))
    with open(outfile, "w") as fh:
        fh.write(header+"\n")
        fh.write(record+"\n")

    print("Correlation saved at:\n %s\n"%outfile)

def main(model_type, config, similarity_th, datafile, outputpath):
    #Read in data with AoA and computed index
    df = pd.read_csv(datafile, sep=";")
    X=df["index"]
    Y=df["aoa"]

    #Read in frequency data and merge
    df["logfreq"] = np.log(df.freq)

    #Compute stats
    pearsonr, pearson_pval = stats.pearsonr(X, Y)

    #Write output
    write_output(model_type, config, datafile, similarity_th, pearsonr, pearson_pval, outputpath)

if __name__ == "__main__":
    args=sys.argv[1:]
    if len(args) != 5:
        print("Usage: corr_index.py <model_type> <config> <similarity_th> <datafile> <output_path>")
        exit(-1)
    main(*args)

#svd
#/home/rgalhama/Data_Research/results_wordrep/eng_0_60/svd_thr_100_win_5_dim_500_neg_1/post_eig_0_neg_1/
#0.7
#/home/rgalhama/Data_Research/results_wordrep/eng_0_60/svd_thr_100_win_5_dim_500_neg_1/post_eig_0_neg_1/evaluation_aoa_cdi_0.5_understands_nouns


import autograd.numpy as np
from autograd import grad, jacobian
import matplotlib.pyplot as plt
from scipy.interpolate import CubicSpline, PchipInterpolator

def get_cc_to_tangent(x, y, slope, radius):
    cc_x = np.zeros(np.size(x))
    cc_y = np.zeros(np.size(x))
    for idx, slopei in enumerate(slope):
        norm_vec = np.array((-slopei, 1))
        norm_vec = norm_vec / np.sqrt(np.sum(norm_vec ** 2))
        delta = norm_vec * radius
        cc_x[idx] = delta[0] + x[idx]
        cc_y[idx] = delta[1] + y[idx]

    return cc_x, cc_y

def is_local_minimum(x, z, z_func, r):
    # for a point at x, z, check if it is just a local minimum point w.r.t. the road
    # The check is performed by checking if there are points in the road surface with distance to (x,z)<radius
    #
    # the check should be valid if: 1) the grid point is fine enough and 2) the z_func is 1-1 mapping
    n_point = 100
    xs = np.linspace(x-r, x+r, n_point)
    zs = z_func(xs)

    dist = np.sqrt((xs-x)**2 + (zs-z)**2)

    return any(dist<r)

def get_distance(x, z):
    # compute the curvilinear coordinate quantities
    dx = np.diff(x)
    dz = np.diff(z)
    ds = np.sqrt(dx**2 + dz**2)
    s = np.cumsum(np.insert(ds, 0, 0))
    return s

def get_force(s, n, ndot, road_spline, k, c):
    # compute the force in cartesian coordinate, for a wheel at s, n location in the curvilinear force field
    # k is spring rate in N/m, and is +ve. So the force must be opposite to deflection.
    # c is the viscous term N*s/m, where F = -lambda*v

    # also for n > 0, the force should be zero! (no deflection, wheel in the air), but we put in a small term to
    # help the numerics (so it still sees the gradient in the n direciton , even when in the air!)

    if n > 0:
        k = k*1e-6
        c = c*1e-6

    theta = road_spline['theta'](s)

    f = -n*k - c*ndot
    fx = f*np.sin(-theta)
    fz = f*np.cos(-theta)

    return fx, fz

def sn2xz(s, n, road_spline):
    x0 = road_spline['x'](s)
    z0 = road_spline['z'](x0)

    theta = road_spline['theta'](s)
    x = x0 + np.sin(-theta)*n
    z = z0 + np.cos(-theta)*n

    return x, z

def build_force_field(z_func, x_min, x_max, r):
    # return the spline z = f(x) that represents the 0 force contour in the contact force field

    # TODO: constant? (pass from outside?)
    x_step = r/5
    # BUG: n_point must be an integer
    n_point = int(np.ceil((x_max - x_min)/x_step))

    # get gradient using autograd
    dzdx = grad(z_func)

    # get the tangent line of each surface point
    x_surf = np.linspace(x_min, x_max, n_point)

    slope = [dzdx(x) for x in x_surf]
    z_surf = z_func(x_surf)
    cc_x, cc_z = get_cc_to_tangent(x_surf, z_surf, slope, r)

    # identify the points that shouldn't be on the profile
    b_local_min = [is_local_minimum(x, z, z_func, r) for x, z in zip(cc_x, cc_z)]
    b_global_min = np.logical_not(b_local_min)

    # create cubic spline for the forcefield centerline
    x_f0 = cc_x[b_global_min]
    z_f0 = cc_z[b_global_min]

    # pchip is a lot smoother than cubic, which can be wiggly
    # TODO: find something that can be differentiated with autograd!
    zspline = PchipInterpolator(x_f0, z_f0)

    # build (s, kappa) spline

    # first resample uniformly from the spline
    xs = np.linspace(x_min, x_max, n_point)
    zs = zspline(xs)

    # compute the curvilinear coordinate quantities
    dx = np.diff(xs)
    dz = np.diff(zs)
    theta = np.arctan2(dz, dx)
    theta = np.append(theta, theta[-1])
    ds = np.sqrt(dx**2 + dz**2)
    s = np.cumsum(np.insert(ds, 0, 0))

    # plt.figure(101)
    # plt.plot(s, theta)
    # plt.show()

    # compute theta spline
    thetaspline = PchipInterpolator(s, theta)

    # x as a function of s
    xspline = PchipInterpolator(s, xs)

    # kappa is just the 1st derivative of theta w.r.t. s
    def kappaspline(ss):
        return thetaspline(ss, 1)

    # put everything in a dictinonary
    road_spline = {'z':     zspline,
                   'x':     xspline,
                   'theta': thetaspline,
                   'kappa': kappaspline}

    return road_spline

def get_road(x_min, x_max, r):
    # surface definition
    # manually assigned value, further work needs to be done
    surface_offset = 1
    step_height = 0.1
    def z_func(x):
        # zs = np.linspace(0, 0, xs.size) + surface_offset
        # zs = np.cos(x * 5 * np.pi)*0.3
        zs = (np.tanh((x-2)*20)+1)*step_height/2 + surface_offset
        return zs

    road_spline = build_force_field(z_func, x_min, x_max, r)
    return road_spline

def main():
    r = 0.4
    x_min = 1.5
    x_max = 2.5
    road_spline = get_road(x_min, x_max, r)

    n_point = 100
    x_surf = np.linspace(x_min, x_max, n_point)
    z_surf = road_spline['z'](x_surf)

    x_fine = np.linspace(x_min, x_max, n_point*10)
    z_fine = road_spline['z'](x_fine)

    s_fine = get_distance(x_fine, z_fine)

    # plot
    plt.figure(1)

    plt.plot(x_surf, z_surf, 'r')
    plt.plot(x_fine, z_fine, 'gx')

    plt.figure(2)
    plt.plot(s_fine, road_spline['theta'](s_fine))
    plt.plot(s_fine, road_spline['kappa'](s_fine), 'r')

    plt.show()

if __name__ == '__main__':
    main()


<reponame>ruiann/SignatureVerification
import random
import numpy as np
from scipy.fftpack import dct

base_path = './SVC2004/Task1'
useless_line = [0]

def get_writer_list():
    return range(1, 41)

def genuine_data_range():
    return range(1, 21)

def fake_data_range():
    return range(21, 41)

# data definition of BIT Handwriting
def read_file(path):
    try:
        file = open(path, 'r')
        lines = file.readlines()
        for line_index in useless_line:
            del lines[line_index]

        front_p = 0
        before_front_p = 0
        sample_x = []
        sample_y = []
        velocity_x = []
        velocity_y = []
        acceleration_x = []
        acceleration_y = []
        for line in lines:
            line = line.replace('\r', '')
            line = line.replace('\n', '')
            data = line.split()

            if int(data[3]) != 0:
                x = int(data[0])
                y = int(data[1])
                sample_x.append(x)
                sample_y.append(y)
                if front_p == 1:
                    velocity_x.append(x - front_x)
                    velocity_y.append(y - front_y)
                    if before_front_p == 1:
                        acceleration_x.append(velocity_x[-1] - velocity_x[-2])
                        acceleration_y.append(velocity_y[-1] - velocity_y[-2])
                front_x = x
                front_y = y
                before_front_p = front_p
                front_p = 1

            else:
                before_front_p = front_p
                front_p = 0

    except:
        print(path)
        return None

    # for dct, use velocity & acceleration only
    return [velocity_x, velocity_y, acceleration_x, acceleration_y]

def get_genuine_data(dir_path=base_path):
    data = []
    for writer in get_writer_list():
        writer_sample = []
        for index in genuine_data_range():
            writer_sample.append(read_file('{}/U{}S{}.TXT'.format(dir_path, writer, index)))
        data.append(writer_sample)
    return data

def get_fake_data(dir_path=base_path):
    data = []
    for writer in get_writer_list():
        writer_sample = []
        for index in fake_data_range():
            writer_sample.append(read_file('{}/U{}S{}.TXT'.format(dir_path, writer, index)))
        data.append(writer_sample)
    return data

class Data:
    def __init__(self, dir_path=base_path):
        self.genuine_data = get_genuine_data(dir_path)
        self.fake_data = get_fake_data(dir_path)
        self.writer_list = get_writer_list()
        self.genuine_range = genuine_data_range()
        self.fake_range = fake_data_range()

    def norm(self, sequence):
        sequence = np.array(sequence, dtype=np.float32)
        mean = sequence.mean()
        std = sequence.std()
        sequence = (sequence - mean) / std
        return dct(sequence)[0:20]

    def normalize(self, sample):
        for i in range(len(sample)):
            sample[i] = self.norm(sample[i])
        return sample

    def get_genuine_pair(self):
        writer = random.sample(self.writer_list, 1)[0] - 1
        reference_index = random.sample(self.genuine_range, 1)[0] - 1
        target_index = random.sample(self.genuine_range, 1)[0] - 1
        reference = self.normalize(self.genuine_data[writer][reference_index])
        target = self.normalize(self.genuine_data[writer][target_index])
        return reference, target

    def get_all_genuine_pair(self):
        pair = []
        for writer in self.writer_list:
            writer = writer - 1
            for reference_index in self.genuine_range:
                for target_index in self.genuine_range:
                    if target_index != reference_index:
                        reference = self.normalize(self.genuine_data[writer][reference_index - 1])
                        target = self.normalize(self.genuine_data[writer][target_index - 1])
                        pair.append((reference, target))

        return pair

    def get_fake_pair(self):
        writer = random.sample(self.writer_list, 1)[0] - 1
        reference_index = random.sample(self.genuine_range, 1)[0] - 1
        target_index = random.sample(self.genuine_range, 1)[0] - 1
        reference = self.normalize(self.genuine_data[writer][reference_index - 1])
        target = self.normalize(self.fake_data[writer][target_index - 1])
        return reference, target

    def get_all_fake_pair(self):
        pair = []
        for writer in self.writer_list:
            writer = writer - 1
            for reference_index in self.genuine_range:
                for target_index in self.genuine_range:
                    reference = self.normalize(self.genuine_data[writer][reference_index - 1])
                    target = self.normalize(self.fake_data[writer][target_index - 1])
                    pair.append((reference, target))

        return pair

if __name__ == '__main__':
    data = Data()
    reference, target = data.get_genuine_pair()
    print(reference)


import unittest

import numpy as np
from scipy import sparse

from linkedin.learner.ds.indexed_dataset import IndexedDataset
from linkedin.learner.ds.indexed_model import IndexedModel
from linkedin.learner.prediction.linear_scorer import score_linear_model
from linkedin.learner.utils.functions import sparse_diag_matrix

def mock_indexed_data():
    """Get a very small mock indexed dataset."""
    labels = np.array([1.0, -1.0, 1.0])
    weights = np.array([1, 1, 1])
    offsets = np.array([0.2, 0.3, 0.4])
    design_matrix = sparse.csc_matrix(np.array([[1.0, 1.0, 0, 0, 0, 0, 0], [1.0, 0, 0, 0, 0, 0, 0], [1.0, 0, 0, 0.5, 1.0, 1.3, 0.8]]))
    return IndexedDataset(design_matrix, labels, weights, offsets)

class LinearScorerTest(unittest.TestCase):
    def test_score_linear_model(self):
        """Test linear scoring in the general case."""
        theta = np.array([1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6])
        hessian = sparse_diag_matrix([2.0, 2.1, 2.2, 2.3, 2.4, 2.5, 2.6])
        indexed_model = IndexedModel(theta, hessian)

        indexed_data = mock_indexed_data()

        scores = score_linear_model(indexed_model, indexed_data)

        expected_scores = np.array(
            [0.2 + 1.0 * 1.0 + 1.0 * 0.1, 0.3 + 1.0 * 1.0, 0.4 + 1.0 * 1.0 + 0.5 * 0.3 + 1.0 * 0.4 + 1.3 * 0.5 + 0.8 * 0.6]
        )

        self.assertTrue(all(scores == expected_scores), f"Scores match expected scores. Actual {scores} == expected {expected_scores}.")

        scores_with_exploration = score_linear_model(indexed_model, indexed_data, exploration_threshold=1)

        self.assertTrue(
            all(scores_with_exploration != expected_scores),
            f"Scores with exploration don't exactly match expected scores. Actual {scores} == expected {expected_scores}.",
        )

    def test_score_linear_model_offsets_only(self):
        """Test linear scoring when all coefficients are zero."""
        num_features = 7
        theta = np.zeros(num_features)
        indexed_model = IndexedModel(theta)
        indexed_data = mock_indexed_data()

        scores = score_linear_model(indexed_model, indexed_data)

        expected_scores = np.array([0.2, 0.3, 0.4])

        self.assertTrue(
            all(scores == expected_scores),
            f"Offsets are used for scores when model coeffs are all zero/missing. Actual {scores} == expected {expected_scores}.",
        )


<reponame>qinyichen/UCLA-Subgraph-Matching
import sys
sys.path.append("..")

import uclasm
import numpy as np
from scipy import sparse

# First we must generate some data
# The number of edges between nodes will be iid geometric with the chosen params
# A signal will be placed in the top left block of the world graph by addition

n_channels = 2
n_tmplt_nodes = 5
n_world_nodes = 100
tmplt_p = 0.85
world_p = 0.8

# Now generate world and template graphs, inserting a signal in the top left

np.random.seed(0)

channels = list(range(n_channels))

# TODO: allow for nodes of arbitrary dtype

# Note: by design, the template nodes have the same ids as the signal nodes
tmplt_nodes = np.arange(n_world_nodes, n_world_nodes+n_tmplt_nodes)
world_nodes = np.arange(n_world_nodes, 2*n_world_nodes)

tmplt_adj_mats = []
world_adj_mats = []

for channel in channels:
    tmplt_adj = np.random.geometric(tmplt_p, (n_tmplt_nodes, n_tmplt_nodes)) - 1
    world_adj = np.random.geometric(world_p, (n_world_nodes, n_world_nodes)) - 1

    # Embed a signal in the top left block of the world graph
    world_adj[:n_tmplt_nodes, :n_tmplt_nodes] += tmplt_adj

    tmplt_adj_mats.append(sparse.csc_matrix(tmplt_adj))
    world_adj_mats.append(sparse.csc_matrix(world_adj))

# initial candidate set for template nodes is the full set of world nodes
tmplt = uclasm.Template(world_nodes, tmplt_nodes, channels, tmplt_adj_mats)
world = uclasm.World(world_nodes, channels, world_adj_mats)


import numpy as np
import matplotlib.pyplot as plt
from scipy import fftpack

f = 10
f_s = 100

t = np.linspace(0, 2, 2 * f_s, endpoint=False)
x = np.sin(2 * np.pi * f * t)

# fig, ax = plt.subplots()
# ax.plot(t, x)
# ax.set_xlabel("Time [s]")
# ax.set_ylabel("Amplitude")

X = fftpack.fft(x)
X = X[:int(len(X)/2)]
freqs = fftpack.fftfreq(len(x)) * f_s
freqs = freqs[:int(len(freqs)/2)]
print(freqs)

fig, ax = plt.subplots()

ax.stem(freqs, np.abs(X))
ax.set_xlabel('Frequency in Hertz [Hz]')
ax.set_ylabel('Frequency Domain (Spectrum) Magnitude')
ax.set_xlim(-f_s / 2, f_s / 2)
ax.set_ylim(-5, 110)

plt.show()


<gh_stars>0
'''This script demonstrates how to build a variational autoencoder with Keras.

 #Reference

 - Auto-Encoding Variational Bayes
   https://arxiv.org/abs/1312.6114
'''
from __future__ import print_function

import csv
import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import norm

from keras.layers import Input, Conv2D, MaxPooling2D, UpSampling2D, BatchNormalization, Activation
from keras.utils import plot_model
from keras.layers import Dense, Lambda, Flatten, Reshape, Layer, Conv2DTranspose
from keras.models import Model
from keras import backend as K
from keras import metrics
from util.summary_model import summary_and_png

def vae_model(channel):

    batch_size = 8
    input_shape = (256,256,3)
    img_rows,img_cols = input_shape[:2]
    latent_dim = 2
    intermediate_dim = 256
    epochs = 100
    epsilon_std = 1.0

    # encoder architecture
    input_img = Input(shape=input_shape)
    x = Conv2D(16, (3, 3), padding='same')(input_img)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = MaxPooling2D((2, 2), padding='same')(x)
    x = Conv2D(16, (3, 3), padding='same')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = MaxPooling2D((2, 2), padding='same')(x)
    x = Conv2D(32, (3, 3), padding='same')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = MaxPooling2D((2, 2), padding='same')(x)
    x = Conv2D(32, (3, 3), padding='same')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = MaxPooling2D((2, 2), padding='same')(x)
    x = Conv2D(64, (3, 3), padding='same')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = MaxPooling2D((2, 2), padding='same')(x)
    x = Conv2D(64, (3, 3), padding='same')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    encoded = MaxPooling2D((2, 2), padding='same')(x)

    flat = Flatten()(encoded)
    hidden = Dense(intermediate_dim, activation='relu')(flat)
    #2次元平面の平均と分散
    z_mean = Dense(latent_dim)(hidden)
    z_log_var = Dense(latent_dim)(hidden)

    # note that "output_shape" isn't necessary with the TensorFlow backend
    z = Lambda(_sampling, output_shape=(latent_dim,))([z_mean, z_log_var])

    # we instantiate these layers separately so as to reuse them later
    decoder_hid = Dense(intermediate_dim, activation='relu')(z)
    decoder_upsample = Dense(64 * 16 * 16, activation='relu')(decoder_hid)
    decoder_reshape = Reshape((16, 16, 64))(decoder_upsample)
    x = Conv2D(64, (3, 3), padding='same')(decoder_reshape)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = UpSampling2D((2, 2))(x)
    x = Conv2D(64, (3, 3), padding='same')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = UpSampling2D((2, 2))(x)
    x = Conv2D(32, (3, 3), padding='same')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = UpSampling2D((2, 2))(x)
    x = Conv2D(32, (3, 3), padding='same')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = UpSampling2D((2, 2))(x)
    x = Conv2D(16, (3, 3), padding='same')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = UpSampling2D((2, 2))(x)
    x = Conv2D(16, (3, 3), padding='same')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = UpSampling2D((2, 2))(x)
    x = Conv2D(channel, (3, 3), padding='same')(x)
    x = BatchNormalization()(x)
    decoded = Activation('sigmoid')(x)

    y = CustomVariationalLayer()([input_img, decoded])
    vae = Model(input_img, y)

    summary_and_png(vae, summary, to_png, png_file)

    def _sampling(self, args):
        z_mean, z_log_var = args
        epsilon = K.random_normal(shape=(K.shape(z_mean)[0], self.latent_dim), mean=0.,
                                stddev=self.epsilon_std)
        return z_mean + K.exp(z_log_var / 2) * epsilon

    # Custom loss layer
    class CustomVariationalLayer(Layer):
        def __init__(self, **kwargs):
            self.is_placeholder = True
            super(CustomVariationalLayer, self).__init__(**kwargs)

        def vae_loss(self, x, x_decoded_mean_squash):
            x = K.flatten(x)
            x_decoded_mean_squash = K.flatten(x_decoded_mean_squash)
            xent_loss = img_rows * img_cols * metrics.binary_crossentropy(x, x_decoded_mean_squash)
            kl_loss = - 0.5 * K.mean(1 + .z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
            return K.mean(xent_loss + kl_loss)

        def call(self, inputs):
            x = inputs[0]
            x_decoded_mean_squash = inputs[1]
            loss = self.vae_loss(x, x_decoded_mean_squash)
            self.add_loss(loss, inputs=inputs)
            return x

    return vae

if __name__ == '__main__':
    network = VAE(channel=3)
    model = network.get_model(to_png=True, png_file='vae_model.png')
    model.compile(optimizer='adam', loss=None)


#!python36
# -*- coding: utf-8 -*-
"""
NCams Toolbox
Copyright 2019-2020 <NAME>, <NAME>
https://github.com/CMGreenspon/NCams

"""
import os

import matplotlib.pyplot as mpl_pp
from scipy.spatial.transform import Rotation as R

import ncams

def import_kinematics():
    # Translate the triangulated data into OSim trc format
    BASE_DIR = os.path.join('C://', 'FLIR_cameras', 'PublicExample')
    proj_path = os.path.join(BASE_DIR, '2019.12.20_8camsNoMarkers-AS-2019-12-23')
    triangulated_path = os.path.join(proj_path, 'triangulated_full_rank_0.9', 'session4')
    ik_dir = os.path.join(proj_path, 'inverse_kinematics')
    if not os.path.isdir(ik_dir):
        os.mkdir(ik_dir)

    # load a csv file into a dictionary by specified column names
    marker_name_dict = ncams.utils.dic_from_csv('marker_meta.csv', 'sDlcMarker', 'sOpenSimMarker')

    triangulated_csv = os.path.join(triangulated_path, 'triangulated_points_4_smoothed.csv')

    # rotate the data from the NCams coordinate system
    # preview the rotations by loading the model and using 'File->Preview experimental data'
    # the right click on the loaded kinematics and 'Transform'. If using our model and our
    # calibration, the rotations should be as described below:
    r = R.from_euler('zyx', [0, 90, 180], degrees=True)
    # scipy.spatial.transform.Rotation.apply returns an ndarray with vertical vectors, so the
    # function is changed in the lambda
    rot = lambda v: r.apply(v)[0].tolist()

    suffixes = ['remote', 'marshmallow', 'wave', 'pen']
    frame_ranges = [(103, 140), (260, 360), (510, 650), (1919, 2019)]
    for frame_range, suffix in zip(frame_ranges, suffixes):
        trc_file = os.path.join(ik_dir, 'triangulated_4_{}.trc'.format(suffix))

        # makes an IK config
        ik_file = os.path.join(ik_dir, 'full_arm_model_IK_4_{}.xml'.format(suffix))
        ik_out_mot_file = os.path.join(ik_dir, 'out_inv_kin_4_{}.mot'.format(suffix))

        ncams.inverse_kinematics.triangulated_to_trc(
            triangulated_csv, trc_file, marker_name_dict,
            data_unit_convert=lambda x: x*100,  # dm to mm
            rate=50, zero_marker='scapula_anterior', frame_range=frame_range, rotation=rot,
            ik_file=ik_file, ik_out_mot_file=ik_out_mot_file)

def filter_joint_angles():
    # Translate the triangulated data into OSim trc format
    BASE_DIR = os.path.join('C://', 'FLIR_cameras', 'PublicExample')
    proj_path = os.path.join(BASE_DIR, '2019.12.20_8camsNoMarkers-AS-2019-12-23')
    ik_dir = os.path.join(proj_path, 'inverse_kinematics')

    suffixes = ['remote', 'marshmallow', 'wave', 'pen']
    for suffix in suffixes:
        ik_out_mot_file = os.path.join(ik_dir, 'out_inv_kin_4_{}.mot'.format(suffix))
        ik_filtered_mot_file = os.path.join(ik_dir, 'out_inv_kin_4_{}_filtered.mot'.format(suffix))
        ncams.inverse_kinematics.smooth_motion(ik_out_mot_file, ik_filtered_mot_file,
                                               median_kernel_size=11)

def make_combined_videos():
    # Translate the triangulated data into OSim trc format
    BASE_DIR = os.path.join('C://', 'FLIR_cameras', 'PublicExample')
    proj_path = os.path.join(BASE_DIR, '2019.12.20_8camsNoMarkers-AS-2019-12-23')
    config_path = os.path.join(proj_path, 'config.yaml')
    ik_dir = os.path.join(proj_path, 'inverse_kinematics')

    triangulated_path = os.path.join(proj_path, 'triangulated_full_rank_0.9', 'session4')
    triangulated_csv = os.path.join(triangulated_path, 'triangulated_points_4_smoothed.csv')

    suffixes = ['remote', 'marshmallow', 'wave', 'pen']
    frame_ranges = [(103, 140), (260, 360), (510, 650), (1919, 2019)]
    # estimate manually with an external program, e.g. MPC-HC, easier if recorded more than one loop
    # from OpenSim
    frame_offsets = [23, 0, 7, 3]
    video_path = os.path.join(BASE_DIR, 'exp_session_2019.12.20_videos', '4_cam19335177.mp4')
    for frame_range, suffix, frame_offset in zip(frame_ranges, suffixes, frame_offsets):
        # Load the motion generated during inverse kinematics and play it.
        # To record a video, press a camera button in the top right corner of the viewer. To stop
        # recording, press the button again. Save the video path to 'ik_video_path'.
        ik_video_path = os.path.join(ik_dir, '4_{}.webm'.format(suffix))  # manually set filename
        output_path = os.path.join(ik_dir, '4_{}_19335177_4.mp4'.format(suffix))
        ncams.make_triangulation_video(
            video_path, triangulated_csv, skeleton_config=config_path,
            frame_range=frame_range, output_path=output_path,
            thrd_video_path=ik_video_path,
            thrd_video_frame_offset=frame_offset,  # if the IK movement starts later
            third_video_crop_hw=[slice(0, -100), slice(350, -700)],  # crops the IK video
            figure_dpi=300,
            ranges=((-0.33, 3), (-2, 2), (-1.33, 6.74)),  # manually set ranges for 3D plot
            plot_markers=False, horizontal_subplots=True)

def make_gifs_and_timeseries():
    BASE_DIR = os.path.join('C://', 'FLIR_cameras', 'PublicExample')
    proj_path = os.path.join(BASE_DIR, '2019.12.20_8camsNoMarkers-AS-2019-12-23')
    ik_dir = os.path.join(proj_path, 'inverse_kinematics')

    filenames = ['4_marshmallow_19335177_4', '4_pen_19335177_4', '4_remote_19335177_4',
                 '4_wave_19335177_4']
    # filenames = ['4_marshmallow_19335177_4']
    for filename in filenames:
        video = os.path.join(ik_dir, '{}.mp4'.format(filename))
        video_images_dir = os.path.join(ik_dir, '{}'.format(filename))
        ncams.image_tools.video_to_images(video, output_directory=ik_dir, output_format='jpeg')
        video_images = ncams.utils.get_image_list(path=video_images_dir)
        gif = os.path.join(ik_dir, '{}.gif'.format(filename))
        ncams.image_tools.images_to_video(video_images, gif, fps=25)

        video = os.path.join(ik_dir, '{}_vertical2.mp4'.format(filename))
        fig_name = os.path.join(ik_dir, '{}_timeseries.png'.format(filename))
        ncams.image_tools.video_to_timeseries(video, fig_name, num_images=5, figure_size=(9, 5),
                                              figure_dpi=200,
                                              crop_hw=[slice(450, -350), slice(2275, -2150)])

if __name__ == '__main__':
    # import_kinematics()
    # filter_joint_angles()
    # make_combined_videos()
    make_gifs_and_timeseries()


<filename>matching_algorithms.py
from sys import float_info
from essentia.standard import Mean
import numpy as np
from scipy.ndimage.interpolation import shift
from constants import (
    PLSH_INDEX,
    NLSH_INDEX,
    COSINE_SIMILARITY,
    JACCARD_SIMILARITY,
    MANHATTAN_DISTANCE,
    LINEAR_SCALING,
    BALS,
    BALS_SHIFT_SIZE,
    RECURSIVE_ALIGNMENT,
    MAX_RA_DEPTH,
    KTRA,
    MAX_KTRA_DEPTH,
    INITIAL_KTRA_K_VALUE
)
from utils import percent

__all__ = ["apply_matching_algorithm_to_lsh"]

MAX_FLOAT = float_info.max
mean = Mean()

def _mean_substract(pitch_vector):
    pitch_vector = np.array(pitch_vector, dtype=np.float32)
    return pitch_vector - mean.compute(pitch_vector)

def calculate_cosine_similarity(query_tfidfs, song_tfidfs, **kwargs):
    """
    Cosine Similarity (d1, d2) = Dot product(d1, d2) / ||d1|| * ||d2||
    """

    # NOTE: this ignores songs longer size, and considers only the
    # first query-length pitches
    dot_product = sum(p*q for p,q in zip(query_tfidfs, song_tfidfs))

    query_norm = np.sqrt( np.sum( np.square(query_tfidfs)))
    candidate_norm = np.sqrt( np.sum( np.square(song_tfidfs)))

    norms = query_norm * candidate_norm

    cosine = 0.0
    if norms != 0:
        cosine = dot_product/norms

    return cosine

def calculate_jaccard_similarity(query_audio, candidate, **kwargs):
    """
    Calculates Jaccard distance in a generic way.
    Jaccard Similarity algorithm in steps:
    1. Get the shared members between both sets, i.e. intersection.
    2. Get the members in both sets (shared and un-shared, i.e. union).
    3. Divide the number of shared members found in (1) by the total number of
       members found in (2).
    4. Multiply the found number in (3) by 100.
    """
    intersection = np.intersect1d(candidate, query_audio)
    union = np.union1d(candidate, query_audio)

    jaccard_similarity = 0
    if union.size > 0:
        jaccard_similarity = (intersection.size / union.size) * 100

    return jaccard_similarity

def __calculate_jaccard_similarity(query_audio, candidate, **kwargs):
    '''
    WARNING: Use this algorithm ONLY for THIS LSH environment purposes. If you
    want to calculate Jaccard distance in a generic way, please use
    calculate_jaccard_similarity.

    Jaccard Similarity algorithm in steps:
    1. Get the shared members between both sets, i.e. intersection.
    2. Get the members in both sets (shared and un-shared, i.e. union).
    3. Divide the number of shared members found in (1) by the total number of
       members found in (2).
    4. Multiply the found number in (3) by 100.
    '''
    from lsh import (
        exec_plsh_pitch_extraction as extract_plsh_pitches,
        exec_nlsh_pitch_extraction as extract_nlsh_pitches
    )  # this fixes circular import error

    index_type = kwargs.get('index_type')

    exec_pitch_extraction = {
        PLSH_INDEX: extract_plsh_pitches,
        NLSH_INDEX: extract_nlsh_pitches
    }

    query_chunks = exec_pitch_extraction[index_type](query_audio.tolist())
    candidate_chunks = exec_pitch_extraction[index_type](candidate.tolist())

    jaccard_similarity = calculate_jaccard_similarity(
        query_chunks,
        candidate_chunks
    )

    return jaccard_similarity

def _rescale_audio(query_audio):
    # ------------------[0.5, 0.75, 1.0, 1.25, 1.5]
    scaling_factors = ((1, 2), (3, 4), (1, 1), (5, 4), (3, 2))
    original_len = query_audio.size
    rescaled_audios = []
    for numerator, denominator in scaling_factors:
        rescaled_audio = np.array([])
        scaling_factor = numerator / denominator
        if scaling_factor == 1.0:
            rescaled_audio = np.copy(query_audio)
        else:
            # Shorten or lenghten audio
            numerator, denominator = scaling_factor.as_integer_ratio()
            for i in range(0, original_len, denominator):
                chunck = query_audio[i:i + denominator]
                if scaling_factor > 1.0:
                    # Lenghten audio
                    # FIXME: only works for 1.25 (5,4) and 1.5 (3,2)
                    repeated = chunck[-1]
                    chunck = np.append(chunck, repeated)
                rescaled_audio = np.append(rescaled_audio, chunck[:numerator])
        rescaled_audios.append(rescaled_audio)

    return rescaled_audios

def _calculate_manhattan_distance(rescaled_audio, candidate):
    additional_length = candidate.size - rescaled_audio.size

    # Equalize size
    if additional_length > 0:
        rescaled_audio = np.append(rescaled_audio, np.zeros(additional_length))
    elif additional_length < 0:
        rescaled_audio = rescaled_audio[:candidate.size]

    # Calculate distance
    result = np.absolute(
        np.subtract(candidate, rescaled_audio)
    )
    return sum(result)

def _calculate_linear_scaling(
    rescaled_query_audios, candidate, **kwargs
):
    '''
    Implemented as explained in "Query-By-Singing-and-Hummimg" from CHIAO-WEI
    LIN.
    '''
    include_zero_distance = kwargs.get('include_zero_distance')
    if not isinstance(rescaled_query_audios, list):
        rescaled_query_audios = [rescaled_query_audios]
    distances = []
    for rescaled_query_audio in rescaled_query_audios:
        distance = _calculate_manhattan_distance(
            rescaled_query_audio,
            candidate
        )
        if distance > 0.0 or include_zero_distance:
            distances.append((distance, rescaled_query_audio))
    if not distance:
        # Ignoring zero distance. (It's likely a noise)
        min_distance, query = MAX_FLOAT, None
    else:
        min_distance, query = min(distances, key=lambda t: t[0])
    return min_distance, query

def _get_candidate_neighbourhood(**kwargs):
    # FIXME: Looks a little bit (or a lot) wrong
    candidate = kwargs.get('candidate')
    candidate_name = kwargs.get('candidate_name')
    original_positions_mapping = kwargs.get('original_positions_mapping')
    neighbours = []
    # Apply shift, shorthen, lengthen operations
    # Left moved vector
    left_moved_vector = shift(candidate, BALS_SHIFT_SIZE)
    # Right moved vector
    right_moved_vector = shift(candidate, -BALS_SHIFT_SIZE)
    # TODO: Left shortened vector
    # TODO: Right shortened vector
    # Left lenghtened vector
    left_lenghtened = shift(candidate, BALS_SHIFT_SIZE, mode='nearest')
    # Right lenghtened vector
    right_lenghtened = shift(candidate, -BALS_SHIFT_SIZE, mode='nearest')

    # Tem algo que parece não estar certo... É para aplicar as operações sobre
    # cada trecho do áudio candidato ou sobre ele como um todo? Ou ainda,
    # sobre um único fragmento candidato?
    neighbours = [
        left_moved_vector,
        right_moved_vector,
        left_lenghtened,
        right_lenghtened
    ]

    return neighbours

def _calculate_bals(rescaled_query_audios, candidate, **kwargs):
    '''
    Explore candidates neighbourhood
       - For each candidate, lenghten or shorten it.
    For each neighbor, measure LS distance.
    Retain the fragment with the shortest distance.
    '''
    candidate_distance, _query = _calculate_linear_scaling(
        rescaled_query_audios, candidate, include_zero_distance=True
    )

    kwargs['candidate'] = candidate

    neighbours = _get_candidate_neighbourhood(**kwargs)
    # Starts with the similar audio
    nearest_neighbour_distance = candidate_distance
    for neighbour in neighbours:
        distance, _query = _calculate_linear_scaling(
            rescaled_query_audios,
            neighbour,
            include_zero_distance=True
        )
        if distance < nearest_neighbour_distance:
            nearest_neighbour_distance = distance

    return nearest_neighbour_distance

def _recursive_align(query_audio, candidate, **kwargs):
    # Compute the linear distance of the corresponding part
    min_distance, rescaled_query_audio = _calculate_linear_scaling(
        query_audio,
        candidate=candidate,
        include_zero_distance=False
    )

    depth = kwargs.get('depth', 0)

    try:
        if rescaled_query_audio.size == 0 or candidate.size == 0:
            return min_distance
        # raise Exception('size zero detected!!!')
    except AttributeError:
        return min_distance

    if depth < MAX_RA_DEPTH:
        query_size = rescaled_query_audio.size
        candidate_size = candidate.size
        query_portion_size = int((query_size / 2) + 1)
        # portion_percents = [10, 20, 30, 40, 50, 60, 70, 80, 90] # Too slow
        portion_percents = [40, 50, 60]
        for portion_percent in portion_percents:
            size = int(
                percent(portion_percent, candidate_size) + 1
            )
            complement_size = candidate_size + 1 - size
            left_query_portion = rescaled_query_audio[:query_portion_size]
            right_query_portion = rescaled_query_audio[query_portion_size:]
            left_similar_portion = candidate[:size]
            right_similar_portion = candidate[complement_size:]

            left_distance = _recursive_align(
                left_query_portion,
                left_similar_portion,
                depth=depth + 1
            )

            right_distance = _recursive_align(
                right_query_portion,
                right_similar_portion,
                depth=depth + 1
            )

            min_distance = min([left_distance, right_distance, min_distance])

    return min_distance

def _calculate_ktra(query_audio, candidate, **kwargs):
    depth = kwargs.get('depth', 0)

    if depth == 0:
        query_audio = _mean_substract(query_audio)
        candidate = _mean_substract(candidate)

    k = kwargs.get('k', INITIAL_KTRA_K_VALUE)

    d_minus = _recursive_align(query_audio - k, candidate, depth=0)
    d_zero = _recursive_align(query_audio, candidate, depth=0)
    d_plus = _recursive_align(query_audio + k, candidate, depth=0)

    min_distance = min([d_minus, d_zero, d_plus])
    if depth < MAX_KTRA_DEPTH:
        # FIXME: Needs to treat equal distances?
        if d_minus == min_distance:
            query_audio = query_audio - k
        elif d_plus == min_distance:
            query_audio = query_audio + k

        min_distance = _calculate_ktra(
            query_audio,
            candidate,
            k=k / 2,
            depth=depth + 1
        )

    return min_distance

def apply_matching_algorithm_to_lsh(
    choosed_algorithm, query, candidates, index_type, original_positions_mapping, use_ls
):
    """
    Applies mathing algorithm in LSH search context. It means
    this function should be used after LSH retrieval, in order to filter
    the found candidates.
    """
    matching_algorithms = {
        JACCARD_SIMILARITY: __calculate_jaccard_similarity,
        LINEAR_SCALING: _calculate_linear_scaling,
        BALS: _calculate_bals,
        RECURSIVE_ALIGNMENT: _recursive_align,
        KTRA: _calculate_ktra
    }

    all_queries_distances = {}
    for query_audio_name, query_audio in query:
        query_audio = np.array(query_audio)
        query_audio = np.trim_zeros(query_audio)
        query_distance = dict()
        if use_ls or choosed_algorithm in [LINEAR_SCALING, BALS]:
            # Rescaling here to optmize time consumption
            query_audios = _rescale_audio(query_audio)
        else:
            # not an array for jaccard, ra and ktra with use_ls=False
            query_audios = query_audio
        for candidate_tuple in candidates:
            candidate_filename, candidate = candidate_tuple
            candidate = np.array(candidate)
            candidate = np.trim_zeros(candidate)

            if use_ls and choosed_algorithm == KTRA:
                # NOTE: returns just one query audio here!
                _min_distance, query_audios = _calculate_linear_scaling(
                    query_audios,
                    candidate,
                    include_zero_distance=True
                )
            ##
            distance_or_similarity = matching_algorithms[choosed_algorithm](
                query_audios,
                candidate,
                query_audio_name=query_audio_name,
                index_type=index_type,  # For Jaccard
                include_zero_distance=True,  # For LS and BALS
                original_positions_mapping=original_positions_mapping,
                depth=0,  # For RA and KTRA
                k=INITIAL_KTRA_K_VALUE  # For KTRA
            )
            ##
            if choosed_algorithm == LINEAR_SCALING:
                query_distance[candidate_filename] = distance_or_similarity[0]
            else:
                query_distance[candidate_filename] = distance_or_similarity

        reverse_order = False
        if choosed_algorithm == JACCARD_SIMILARITY:
            reverse_order = True

        query_distance = sorted(
            query_distance.items(),
            key=lambda res: res[1],
            reverse=reverse_order
        )
        all_queries_distances[query_audio_name] = query_distance

    return all_queries_distances

def apply_matching_algorithm_to_tfidf(choosed_algorithm, **kwargs):
    """
    Applies mathing algorithm in TF-IDF search context.
    """
    matching_algorithms = {
        JACCARD_SIMILARITY: calculate_jaccard_similarity,
        COSINE_SIMILARITY: calculate_cosine_similarity,
        # MANHATTAN_DISTANCE: _calculate_manhattan_distance,

        # TODO: Adapt methods for this context
        LINEAR_SCALING: _calculate_linear_scaling,
        BALS: _calculate_bals,
        RECURSIVE_ALIGNMENT: _recursive_align,
        KTRA: _calculate_ktra
    }

    if choosed_algorithm in [LINEAR_SCALING, BALS]:
        raise(f'{LINEAR_SCALING} and {BALS} are not implemented for tfidf.')

    # if choosed_algorithm in [LINEAR_SCALING, BALS]:
    #     _query = _rescale_audio(query)
    # else:
    #     _query = query

    if choosed_algorithm == COSINE_SIMILARITY:
        result = calculate_cosine_similarity(
            query_tfidfs=kwargs.get('query_tfidfs'),
            song_tfidfs=kwargs.get('song_tfidfs')
        )
    else:
        result = matching_algorithms[choosed_algorithm](
            query_audio=kwargs.get('query'),
            candidate=kwargs.get('song')
        )

    # if choosed_algorithm == LINEAR_SCALING:
    #     # Returns distance and query. Needs only the query
    #     distance_or_similarity = distance_or_similarity[0]

    return result

def normalize_distance_to_similarity(queries_expected_songs_and_distances):
    """
    Tries to "normalize" distance to similarity range [0-100].
    Bigger distances must result in the smaller similarities, and the
    other way around.
    Exponential function: f(x) = (e^(1/x)) - 1
    f(x) = similarity
    x = distance
    """

    similarities = {}
    for query_name, expected_song_name, distance in queries_expected_songs_and_distances:
        similarity = np.exp(1/distance) - 1
        similarities[query_name] = 100.0 if similarity > 100 else similarity
        print(f"{query_name} : distance={distance}\tsimilarity={similarities[query_name]}")

    return similarities