#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jun 26 14:46:56 2017
@author: luohao
"""

"""
CVPR2017 paper:<NAME>, <NAME>, <NAME>, et al. Re-ranking Person Re-identification with k-reciprocal Encoding[J]. 2017.
url:http://openaccess.thecvf.com/content_cvpr_2017/papers/Zhong_Re-Ranking_Person_Re-Identification_CVPR_2017_paper.pdf
Matlab version: https://github.com/zhunzhong07/person-re-ranking
"""

"""
API
probFea: all feature vectors of the query set, shape = (image_size, feature_dim)
galFea: all feature vectors of the gallery set, shape = (image_size, feature_dim)
k1,k2,lambda: parameters, the original paper is (k1=20,k2=6,lambda=0.3)
MemorySave: set to 'True' when using MemorySave mode
Minibatch: avaliable when 'MemorySave' is 'True'
"""

import numpy as np
from scipy.spatial.distance import cdist
from scipy.stats import entropy # for KL divergence
from math import log

def compute_metadata_distance_hard(q_metadatas, g_metadatas, metadata_prob_ranges):
    q_num = q_metadatas.shape[0]
    g_num = g_metadatas.shape[0]
    dist = np.zeros((q_num, g_num), dtype=np.float32)
    for iq in range(q_num):
        for ig in range(g_num):
            for p_begin, p_end in metadata_prob_ranges:
                cq = np.argmax(q_metadatas[iq][p_begin:p_end])
                cg = np.argmax(g_metadatas[ig][p_begin:p_end])
                if cq != cg:
                    dist[iq, ig] += 1
                    break
    return dist

def compute_metadata_distance_semihard(q_metadatas, g_metadatas, metadata_prob_ranges):
    q_num = q_metadatas.shape[0]
    g_num = g_metadatas.shape[0]
    dist = np.zeros((q_num, g_num), dtype=np.float32)
    for iq in range(q_num):
        for ig in range(g_num):
            for p_begin, p_end in metadata_prob_ranges:
                cq = np.argmax(q_metadatas[iq][p_begin:p_end])
                cg = np.argmax(g_metadatas[ig][p_begin:p_end])
                if cq != cg and cq != (p_end - p_begin - 1) and cg != (p_end - p_begin - 1): # the last class is "other"
                    dist[iq, ig] += 1
                    break
    return dist

def compute_metadata_distance_easy(q_metadatas, g_metadatas, metadata_prob_ranges):
    q_num = q_metadatas.shape[0]
    g_num = g_metadatas.shape[0]
    dist = np.ones((q_num, g_num), dtype=np.float32)
    for iq in range(q_num):
        for ig in range(g_num):
            for p_begin, p_end in metadata_prob_ranges:
                cq = np.argmax(q_metadatas[iq][p_begin:p_end])
                cg = np.argmax(g_metadatas[ig][p_begin:p_end])
                if cq == cg:
                    dist[iq, ig] = 0
                    break
    return dist

def compute_KL_divergence(q_metadatas, g_metadatas, metadata_prob_ranges = [(0,6), (6,18), (18,26)]):
    q_num = q_metadatas.shape[0]
    g_num = g_metadatas.shape[0]
    m_num = len(metadata_prob_ranges)
    KL_div = np.zeros((q_num, g_num, m_num), dtype=np.float32)
    epsilon = 1e-4
    for iq in range(q_num):
        for ig in range(g_num):
            for im, (p_begin, p_end) in enumerate(metadata_prob_ranges):
                KL_div[iq, ig, im] = entropy(q_metadatas[iq][p_begin:p_end]+epsilon, g_metadatas[ig][p_begin:p_end]+epsilon)
    return KL_div

def compute_pred(metadatas, metadata_prob_ranges):
    all_num = metadatas.shape[0]
    m_num = len(metadata_prob_ranges)
    pred = np.zeros((all_num, m_num), dtype=np.int32)
    for im, (p_begin, p_end) in enumerate(metadata_prob_ranges):
        pred[:,im] = np.argmax(metadatas[:,p_begin:p_end], axis=1)
    return pred

def compute_confusion_weight_old(q_pred, g_pred, confusion_mat):
    q_num = q_pred.shape[0]
    g_num = g_pred.shape[0]
    c_num = confusion_mat.shape[0]

    confusion_mat = confusion_mat + 1e-4*np.ones((c_num, c_num), dtype=np.float32)

    c_weight = np.transpose(confusion_mat)*np.diag(confusion_mat)
    c_weight += np.transpose(c_weight)
    c_sum = np.sum(confusion_mat, axis=0).reshape(1,-1)
    c_sum = np.matmul(np.transpose(c_sum), c_sum)
    c_weight = c_weight * np.reciprocal(c_sum)
    #c_weight[range(c_num),range(c_num)]/=2
    np.fill_diagonal(c_weight, 1) # no penalty for the same class
    #print('c_weight = ')
    #print(c_weight)

    confusion_weight = np.ones((q_num, g_num), dtype=np.float32)
    for iq in range(q_num):
        for ig in range(g_num):
            confusion_weight[iq, ig] = c_weight[q_pred[iq], g_pred[ig]]
    return confusion_weight

def compute_confusion_weight(q_pred, g_pred, confusion_mat):
    q_num = q_pred.shape[0]
    g_num = g_pred.shape[0]
    c_num = confusion_mat.shape[0]

    #print('confusion_mat = ')
    #print(confusion_mat)
    confusion_mat = confusion_mat + 1e-4*np.ones((c_num, c_num), dtype=np.float32)
    c_sum = np.sum(confusion_mat, axis=0)
    confusion_mat_norm = confusion_mat * np.reciprocal(c_sum)
    #print('confusion_mat_norm = ')
    #print(confusion_mat_norm)
    c_weight = np.matmul(np.transpose(confusion_mat_norm), confusion_mat_norm)
    np.fill_diagonal(c_weight, 1) # no penalty for the same class
    #print('c_weight = ')
    #print(c_weight)

    confusion_weight = np.ones((q_num, g_num), dtype=np.float32)
    for iq in range(q_num):
        for ig in range(g_num):
            confusion_weight[iq, ig] = c_weight[q_pred[iq], g_pred[ig]]
    return confusion_weight

def cluster_gallery_soft(gf, g_metadatas, metadata_prob_ranges = [(0,6), (6,18), (18,26)], k=20, learning_rate=0.5, num_iter=20, MemorySave=False, Minibatch=2000):
    '''
    return new gallery feature gf_new
    '''
    gf = gf.copy() # make a copy since it will be updated in each iteration
    g_num = gf.shape[0]
    # meta data penalty
    '''dist_meta = np.zeros((g_num, g_num), dtype=np.float16)
    epsilon = 1e-4
    for i in range(g_num):
        metaI = g_metadatas[i]
        for j in range(g_num):
            metaJ = g_metadatas[j]
            for prob_range_begin, prob_ranges_end in metadata_prob_ranges:
                if entropy (metaI[prob_range_begin:prob_ranges_end] + epsilon, metaJ[prob_range_begin:prob_ranges_end] + epsilon) > 0.5:
                    dist_meta[i][j] = 1
                    break'''
    dist_meta = compute_metadata_distance_hard(g_metadatas, g_metadatas, metadata_prob_ranges)
    for iter in range(num_iter):
        #print('iter: %d' % iter)
        #print('computing original distance')
        if MemorySave:
            g_g_dist = np.zeros(shape=[g_num, g_num], dtype=np.float16)
            i = 0
            while True:
                it = i + Minibatch
                if it < np.shape(gf)[0]:
                    g_g_dist[i:it, ] = np.power(cdist(gf[i:it, ], gf), 2).astype(np.float16)
                else:
                    g_g_dist[i:, :] = np.power(cdist(gf[i:, ], gf), 2).astype(np.float16)
                    break
                i = it
        else:
            g_g_dist = cdist(gf, gf).astype(np.float16)
            g_g_dist = np.power(g_g_dist, 2).astype(np.float16)
        dist_min = np.min(g_g_dist[np.triu_indices(g_num,1)])
        dist_max = np.max(g_g_dist[np.triu_indices(g_num,1)])
        #print('dist_min = %f, dist_max = %f' % (dist_min, dist_max))
        #g_g_dist = np.transpose(g_g_dist / np.max(g_g_dist, axis=0))
        # apply meta data
        g_g_dist += np.transpose(dist_meta * np.max(g_g_dist, axis=1))
        initial_rank = np.argsort(g_g_dist).astype(np.int32)
        # apply mean field
        gf_new = gf.copy()
        sigma = dist_min / 2 + 1
        for i in range(g_num):
            k_neigh_index = initial_rank[i, :k + 1]
            sigma = np.min(g_g_dist[i, k_neigh_index[1:]]) + 1
            weight = np.exp(-g_g_dist[i, k_neigh_index] / sigma)
            weight /= np.sum(weight)
            if i % 100 == 0 and False:
                print(i)
                print(k_neigh_index)
                print(g_g_dist[i, k_neigh_index])
                print(weight)
            gf_new[i] = np.dot(np.transpose(gf[k_neigh_index]), weight)
        gf = gf * (1 - learning_rate) +  gf_new * (learning_rate)
    return gf

def re_ranking_metadata_soft_v3(original_dist, metadata_dist, query_num, all_num, r_metadata, k1, k2, lambda_value):
    '''
    input:
        original_dist: pre-compute distmat
        metadata_dist: metadata distance
        r_metadata: weight for metadata distance
    return:
        final_dist
    '''

    # The following naming, e.g. gallery_num, is different from outer scope.
    # Don't care about it.
    gallery_num = all_num
    original_dist = np.transpose(original_dist / np.max(original_dist, axis=0))
    ### additional scaling
    scaling = False
    if scaling:
        tmp_rank = np.argsort(original_dist).astype(np.int32)
        min_dist = original_dist[range(all_num), tmp_rank[:,1]]
        metadata_dist = np.transpose(metadata_dist * min_dist)
        #print('min_dist = ')
        #print(min_dist)
    ###
    original_dist += r_metadata * metadata_dist

    print('starting re_ranking')
    initial_rank = np.argsort(original_dist).astype(np.int32)
    V = np.zeros_like(original_dist).astype(np.float16)
    for i in range(all_num):
        # k-reciprocal neighbors
        forward_k_neigh_index = initial_rank[i, :k1 + 1]
        backward_k_neigh_index = initial_rank[forward_k_neigh_index, :k1 + 1]
        fi = np.where(backward_k_neigh_index == i)[0]
        k_reciprocal_index = forward_k_neigh_index[fi]
        k_reciprocal_expansion_index = k_reciprocal_index
        for j in range(len(k_reciprocal_index)):
            candidate = k_reciprocal_index[j]
            candidate_forward_k_neigh_index = initial_rank[candidate, :int(np.around(k1 / 2)) + 1]
            candidate_backward_k_neigh_index = initial_rank[candidate_forward_k_neigh_index,
                                               :int(np.around(k1 / 2)) + 1]
            fi_candidate = np.where(candidate_backward_k_neigh_index == candidate)[0]
            candidate_k_reciprocal_index = candidate_forward_k_neigh_index[fi_candidate]
            if len(np.intersect1d(candidate_k_reciprocal_index, k_reciprocal_index)) > 2 / 3 * len(
                    candidate_k_reciprocal_index):
                k_reciprocal_expansion_index = np.append(k_reciprocal_expansion_index, candidate_k_reciprocal_index)

        k_reciprocal_expansion_index = np.unique(k_reciprocal_expansion_index)
        weight = np.exp(-original_dist[i, k_reciprocal_expansion_index])
        V[i, k_reciprocal_expansion_index] = weight / np.sum(weight)
    original_dist = original_dist[:query_num, ]
    if k2 != 1:
        V_qe = np.zeros_like(V, dtype=np.float16)
        for i in range(all_num):
            V_qe[i, :] = np.mean(V[initial_rank[i, :k2], :], axis=0)
        V = V_qe
        del V_qe
    del initial_rank
    invIndex = []
    for i in range(gallery_num):
        invIndex.append(np.where(V[:, i] != 0)[0])

    jaccard_dist = np.zeros_like(original_dist, dtype=np.float16)

    for i in range(query_num):
        temp_min = np.zeros(shape=[1, gallery_num], dtype=np.float16)
        indNonZero = np.where(V[i, :] != 0)[0]
        indImages = []
        indImages = [invIndex[ind] for ind in indNonZero]
        for j in range(len(indNonZero)):
            temp_min[0, indImages[j]] = temp_min[0, indImages[j]] + np.minimum(V[i, indNonZero[j]], V[indImages[j], indNonZero[j]])
        jaccard_dist[i] = 1 - temp_min / (2 - temp_min)

    final_dist = jaccard_dist * (1 - lambda_value) + original_dist * lambda_value
    del original_dist
    del V
    del jaccard_dist
    final_dist = final_dist[:query_num, query_num:]

    # np.save('final_dist.npy', final_dist)

    return final_dist

def re_ranking_metadata_soft_v2(qf, gf, q_metadatas, g_metadatas, confusion_mats, metadata_prob_ranges, k1=4, k2=4, lambda_value=0.5, MemorySave=False, Minibatch=2000):

    m_num = len(metadata_prob_ranges)
    for p_begin, p_end in metadata_prob_ranges:
        assert (p_begin, p_end) in confusion_mats

    query_num = qf.shape[0]
    all_num = query_num + gf.shape[0]
    feat = np.append(qf, gf, axis=0)
    all_metadatas = np.append(q_metadatas, g_metadatas, axis=0)
    ###feat = np.concatenate((feat, all_metadatas*20), axis=1)
    # feat = np.append(probFea, galFea)
    # feat = np.vstack((probFea, galFea))
    feat = feat.astype(np.float16)
    print('computing original distance')
    if MemorySave:
        original_dist = np.zeros(shape=[all_num, all_num], dtype=np.float16)
        i = 0
        while True:
            it = i + Minibatch
            if it < np.shape(feat)[0]:
                original_dist[i:it, ] = np.power(cdist(feat[i:it, ], feat), 2).astype(np.float16)
            else:
                original_dist[i:, :] = np.power(cdist(feat[i:, ], feat), 2).astype(np.float16)
                break
            i = it
    else:
        original_dist = cdist(feat, feat).astype(np.float16)
        original_dist = np.power(original_dist, 2).astype(np.float16)
    del feat
    gallery_num = original_dist.shape[0]
    original_dist = np.transpose(original_dist / np.max(original_dist, axis=0))
    V = np.zeros_like(original_dist).astype(np.float16)

    # apply meta data
    print('computing KL divergence')

    KL_div = compute_KL_divergence(all_metadatas, all_metadatas, metadata_prob_ranges)
    KL_div_U = compute_KL_divergence(all_metadatas, np.ones(all_metadatas.shape, dtype=np.float32), metadata_prob_ranges)
    conf_pred = np.zeros((all_num, all_num, m_num), dtype=np.float32)
    for im, (p_begin, p_end) in enumerate(metadata_prob_ranges):
        conf_pred[:,:,im] = KL_div_U[:,:,im] * np.transpose(KL_div_U[:,:,im]) / (np.log(p_end - p_begin)*np.log(p_end - p_begin))
    pred = compute_pred(all_metadatas, metadata_prob_ranges)
    confusion_dist = np.zeros((all_num, all_num, m_num), dtype=np.float32)
    for im, (p_begin, p_end) in enumerate(metadata_prob_ranges):
        confusion_weight = compute_confusion_weight(pred[:,im], pred[:,im], confusion_mats[(p_begin, p_end)])
        confusion_dist[:,:,im] = -np.log(confusion_weight + 1e-4) / np.log(p_end-p_begin)

    pred_weight = conf_pred * confusion_dist# * KL_div
    pred_weight = np.sum(pred_weight, axis=2)
    #print('confusion_dist = ')
    #print(confusion_dist)

    tmp_rank = np.argsort(original_dist).astype(np.int32)
    min_dist = original_dist[range(all_num), tmp_rank[:,1]]
    #print('min_dist = ')
    #print(min_dist)
    pred_dist = np.transpose(pred_weight * min_dist)
    #print('pred_dist = ')
    #print(pred_dist)

    r_KL = 10#0.5#20.0
    #print('original_dist = ')
    #print(original_dist)
    #original_dist_no_meta = original_dist.copy()
    original_dist += pred_dist*r_KL
    #original_dist = np.clip(original_dist, 0, 1) # not meaningful
    #print('original_dist = ')
    #print(original_dist)

    initial_rank = np.argsort(original_dist).astype(np.int32)
    #original_dist_no_query = original_dist.copy()
    #original_dist_no_query[:,:query_num] = 1000.0
    #initial_rank = np.argsort(original_dist_no_query).astype(np.int32)

    print('starting re_ranking')
    for i in range(all_num):
        # k-reciprocal neighbors
        forward_k_neigh_index = initial_rank[i, :k1 + 1]
        backward_k_neigh_index = initial_rank[forward_k_neigh_index, :k1 + 1]
        fi = np.where(backward_k_neigh_index == i)[0]
        k_reciprocal_index = forward_k_neigh_index[fi]
        k_reciprocal_expansion_index = k_reciprocal_index
        for j in range(len(k_reciprocal_index)):
            candidate = k_reciprocal_index[j]
            candidate_forward_k_neigh_index = initial_rank[candidate, :int(np.around(k1 / 2)) + 1]
            candidate_backward_k_neigh_index = initial_rank[candidate_forward_k_neigh_index,
                                               :int(np.around(k1 / 2)) + 1]
            fi_candidate = np.where(candidate_backward_k_neigh_index == candidate)[0]
            candidate_k_reciprocal_index = candidate_forward_k_neigh_index[fi_candidate]
            if len(np.intersect1d(candidate_k_reciprocal_index, k_reciprocal_index)) > 2 / 3 * len(
                    candidate_k_reciprocal_index):
                k_reciprocal_expansion_index = np.append(k_reciprocal_expansion_index, candidate_k_reciprocal_index)

        k_reciprocal_expansion_index = np.unique(k_reciprocal_expansion_index)
        weight = np.exp(-original_dist[i, k_reciprocal_expansion_index])
        V[i, k_reciprocal_expansion_index] = weight / np.sum(weight)
    original_dist = original_dist[:query_num, ]
    if k2 != 1:
        V_qe = np.zeros_like(V, dtype=np.float16)
        for i in range(all_num):
            V_qe[i, :] = np.mean(V[initial_rank[i, :k2], :], axis=0)
        V = V_qe
        del V_qe
    del initial_rank
    invIndex = []
    for i in range(gallery_num):
        invIndex.append(np.where(V[:, i] != 0)[0])

    jaccard_dist = np.zeros_like(original_dist, dtype=np.float16)

    for i in range(query_num):
        temp_min = np.zeros(shape=[1, gallery_num], dtype=np.float16)
        indNonZero = np.where(V[i, :] != 0)[0]
        indImages = []
        indImages = [invIndex[ind] for ind in indNonZero]
        for j in range(len(indNonZero)):
            temp_min[0, indImages[j]] = temp_min[0, indImages[j]] + np.minimum(V[i, indNonZero[j]], V[indImages[j], indNonZero[j]])
        jaccard_dist[i] = 1 - temp_min / (2 - temp_min)

    final_dist = jaccard_dist * (1 - lambda_value) + original_dist * lambda_value

    #original_dist_easy_meta = original_dist_no_meta + 100*compute_metadata_distance_easy(all_metadatas, all_metadatas, metadata_prob_ranges)
    #original_dist_easy_meta = original_dist_easy_meta[:query_num, ]
    #final_dist = jaccard_dist * (1 - lambda_value) + original_dist_easy_meta * lambda_value

    #original_dist_no_meta = original_dist_no_meta[:query_num, ]
    #final_dist = jaccard_dist * (1 - lambda_value) + original_dist_no_meta * lambda_value

    del original_dist
    del V
    del jaccard_dist
    final_dist = final_dist[:query_num, query_num:]

    # np.save('final_dist.npy', final_dist)

    return final_dist

def re_ranking_metadata_soft(qf, gf, q_metadatas, g_metadatas, metadata_prob_ranges, k1=4, k2=4, lambda_value=0.5, MemorySave=False, Minibatch=2000):
    query_num = qf.shape[0]
    all_num = query_num + gf.shape[0]
    feat = np.append(qf, gf, axis=0)
    #meta = np.append(q_metadatas, g_metadatas, axis=0)
    ###feat = np.concatenate((feat, meta*20), axis=1)
    # feat = np.append(probFea, galFea)
    # feat = np.vstack((probFea, galFea))
    feat = feat.astype(np.float16)
    print('computing original distance')
    if MemorySave:
        original_dist = np.zeros(shape=[all_num, all_num], dtype=np.float16)
        i = 0
        while True:
            it = i + Minibatch
            if it < np.shape(feat)[0]:
                original_dist[i:it, ] = np.power(cdist(feat[i:it, ], feat), 2).astype(np.float16)
            else:
                original_dist[i:, :] = np.power(cdist(feat[i:, ], feat), 2).astype(np.float16)
                break
            i = it
    else:
        original_dist = cdist(feat, feat).astype(np.float16)
        original_dist = np.power(original_dist, 2).astype(np.float16)
    del feat
    gallery_num = original_dist.shape[0]
    original_dist = np.transpose(original_dist / np.max(original_dist, axis=0))
    V = np.zeros_like(original_dist).astype(np.float16)
    # apply meta data
    print('computing KL divergence')
    KL_div = np.zeros((all_num, all_num), dtype=np.float16)
    tmp_rank = np.argsort(original_dist).astype(np.int32)
    for i in range(all_num):
        if i < query_num:
            metaI = q_metadatas[i]
        else:
            metaI = g_metadatas[i - query_num]
        d_min = original_dist[i][tmp_rank[i,1]]
        #print('d_min: %f' % d_min)
        for j in range(all_num):
            if j < query_num:
                metaJ = q_metadatas[j]
            else:
                metaJ = g_metadatas[j - query_num]
            for prob_range_begin, prob_range_end in metadata_prob_ranges:
                hard_threshold = True
                epsilon = 1e-4
                pk = metaI[prob_range_begin:prob_range_end] + epsilon
                qk = metaJ[prob_range_begin:prob_range_end] + epsilon
                if hard_threshold:
                    if np.argmax(pk) != np.argmax(qk):
                        KL_div[i][j] += 100
                        break
                    else:
                        continue
                #s = entropy(pk, qk)*0.5 + entropy(qk, pk)*0.5
                s = min(entropy(pk, qk), entropy(qk, pk))
                #print('%d: %f' % (num_classes, s))
                #KL_div[i][j] += max(s/log(num_classes) - 1, 0) * d_min
                KL_div[i][j] += s * d_min
    print('KL_div min: %f' % np.min(KL_div[np.triu_indices(all_num,1)]))
    print('KL_div max: %f' % np.max(KL_div[np.triu_indices(all_num,1)]))
    r_KL = 1.0
    original_dist = np.clip(original_dist+KL_div*r_KL, 0, 1)

    initial_rank = np.argsort(original_dist).astype(np.int32)
    #original_dist_no_query = original_dist.copy()
    #original_dist_no_query[:,:query_num] = 1000.0
    #initial_rank = np.argsort(original_dist_no_query).astype(np.int32)

    print('starting re_ranking')
    for i in range(all_num):
        # k-reciprocal neighbors
        forward_k_neigh_index = initial_rank[i, :k1 + 1]
        backward_k_neigh_index = initial_rank[forward_k_neigh_index, :k1 + 1]
        fi = np.where(backward_k_neigh_index == i)[0]
        k_reciprocal_index = forward_k_neigh_index[fi]
        k_reciprocal_expansion_index = k_reciprocal_index
        for j in range(len(k_reciprocal_index)):
            candidate = k_reciprocal_index[j]
            candidate_forward_k_neigh_index = initial_rank[candidate, :int(np.around(k1 / 2)) + 1]
            candidate_backward_k_neigh_index = initial_rank[candidate_forward_k_neigh_index,
                                               :int(np.around(k1 / 2)) + 1]
            fi_candidate = np.where(candidate_backward_k_neigh_index == candidate)[0]
            candidate_k_reciprocal_index = candidate_forward_k_neigh_index[fi_candidate]
            if len(np.intersect1d(candidate_k_reciprocal_index, k_reciprocal_index)) > 2 / 3 * len(
                    candidate_k_reciprocal_index):
                k_reciprocal_expansion_index = np.append(k_reciprocal_expansion_index, candidate_k_reciprocal_index)

        k_reciprocal_expansion_index = np.unique(k_reciprocal_expansion_index)
        weight = np.exp(-original_dist[i, k_reciprocal_expansion_index])
        V[i, k_reciprocal_expansion_index] = weight / np.sum(weight)
    original_dist = original_dist[:query_num, ]
    if k2 != 1:
        V_qe = np.zeros_like(V, dtype=np.float16)
        for i in range(all_num):
            V_qe[i, :] = np.mean(V[initial_rank[i, :k2], :], axis=0)
        V = V_qe
        del V_qe
    del initial_rank
    invIndex = []
    for i in range(gallery_num):
        invIndex.append(np.where(V[:, i] != 0)[0])

    jaccard_dist = np.zeros_like(original_dist, dtype=np.float16)

    for i in range(query_num):
        temp_min = np.zeros(shape=[1, gallery_num], dtype=np.float16)
        indNonZero = np.where(V[i, :] != 0)[0]
        indImages = []
        indImages = [invIndex[ind] for ind in indNonZero]
        for j in range(len(indNonZero)):
            temp_min[0, indImages[j]] = temp_min[0, indImages[j]] + np.minimum(V[i, indNonZero[j]], V[indImages[j], indNonZero[j]])
        jaccard_dist[i] = 1 - temp_min / (2 - temp_min)

    final_dist = jaccard_dist * (1 - lambda_value) + original_dist * lambda_value
    del original_dist
    del V
    del jaccard_dist
    final_dist = final_dist[:query_num, query_num:]

    # np.save('final_dist.npy', final_dist)

    return final_dist


import matplotlib as mpl
import numpy as np
import os
import matplotlib.pyplot as plt
import tkinter

import datetime
import struct
import ftplib
import glob
import tempfile
import pymongo
import requests
import sys

import scipy.signal as signal
import scipy.io.wavfile
import pywt

from unophysics import ladc, wavefuncs
from pathlib import Path
from scipy.io import savemat
from sshtunnel import SSHTunnelForwarder

root = tkinter.Tk()
root.title('LADC-GEMM Interactive')
root.geometry('540x260')
root.grid_columnconfigure(1, minsize=135)

# SET UP INFORMATION PAGE FOR EACH LOCATION
localcalls = []
def info_location_change():
    currentlocation.set(ladc.Stuff.brydes_calls[info_location.get()]['Location'])
    locationlabel.grid(column=1, row=0, sticky=tkinter.N+tkinter.W)

    global localcalls
    localcalls = ladc.Stuff.brydes_calls[info_location.get()]['Call(s) recorded']
    numcallsstring = ', '.join(localcalls)
    current_numcalls.set(numcallsstring)
    numcalls_label.grid(column=1, row=0, sticky=tkinter.N+tkinter.W, pady=5)
    callmenu.delete(0, tkinter.END)
    for x in localcalls:
        callmenu.insert(tkinter.END, x)
    current_date.set(ladc.Stuff.brydes_calls[info_location.get()]['Date'])
    date_label.grid(column=1, row=0, sticky=tkinter.N+tkinter.W, pady=5)
    current_minmax.set('')

# LOCATION BUTTONS
infobutton_box = tkinter.LabelFrame(root)
info_location = tkinter.StringVar()
info_location.set('ETP')

currentlocation=tkinter.StringVar()
locationlabel = tkinter.Label(root, textvariable=currentlocation, font=('bold', 13))

etp = tkinter.Radiobutton(infobutton_box, text='Eastern Tropical Pacific', variable=info_location, value='ETP', command=info_location_change)
scarib = tkinter.Radiobutton(infobutton_box, text='Southern Caribbean', variable=info_location, value='SCaribbean', command=info_location_change)
nwpac = tkinter.Radiobutton(infobutton_box, text='Northwest Pacific', variable=info_location, value='NWPacific', command=info_location_change)
goc = tkinter.Radiobutton(infobutton_box, text='Gulf of California', variable=info_location, value='GoC', command=info_location_change)
cabo = tkinter.Radiobutton(infobutton_box, text='Cabo Frio, Brazil', variable=info_location, value='CaboFrio', command=info_location_change)
gom = tkinter.Radiobutton(infobutton_box, text='Gulf of Mexico', variable=info_location, value='GoM', command=info_location_change)

# REGIONAL CALL INFORMATION WIDGETS
numcallsbox = tkinter.LabelFrame(root)
numcalls = tkinter.Label(numcallsbox, text='Call(s) recorded: ')
current_numcalls = tkinter.StringVar()
numcalls_label = tkinter.Label(numcallsbox, textvariable=current_numcalls, wraplength=200, justify='left')

datebox = tkinter.LabelFrame(root)
date = tkinter.Label(datebox, text='Date: ')
current_date = tkinter.StringVar()
date_label = tkinter.Label(datebox, textvariable=current_date, wraplength=200, justify='left')

# FREQUENCY INFORMATION WIDGETS
freqbox = tkinter.LabelFrame(root)
freq = tkinter.Label(freqbox, text='Frequency information: ')

def change_facts(event):
    selecttuple = event.widget.curselection()
    selectindex = selecttuple[0]
    callname = localcalls[selectindex]
    minmaxrange = ladc.Stuff.frequency_info[info_location.get()][callname]
    minbookmark = minmaxrange[0]
    maxbookmark = minmaxrange[1]
    minmax_string = f'{minbookmark} - {maxbookmark} Hz'
    current_minmax.set(minmax_string)

callbox = tkinter.Frame(freqbox)
call = tkinter.Label(callbox, text='Call: ')
callmenu = tkinter.Listbox(callbox, selectmode='SINGLE', height=6)
callmenu.bind('<<ListboxSelect>>', change_facts)

minmaxbox = tkinter.Frame(freqbox)
minmax = tkinter.Label(minmaxbox, text='Min/max frequencies: ')
current_minmax = tkinter.StringVar()
minmax_label = tkinter.Label(minmaxbox, textvariable=current_minmax)

# MENU WIDGET CHANGES
def datapage():
    reset_inputs()

    root.grid_columnconfigure(index=1, minsize=135)
    root.grid_rowconfigure(index=0, minsize=10)
    root.grid_rowconfigure(index=2, minsize=10)

    widgetlist = [minmax_label, callbox, minmaxbox, freqbox, freq, call, callmenu, minmax, infobutton_box, etp, scarib, nwpac, goc, cabo, gom, datebox, date, numcallsbox, numcalls, locationlabel, numcalls_label, date_label]
    for widget in widgetlist:
        widget.grid_remove()

    onefilename_box.grid(column=1, row=0, sticky=tkinter.W+tkinter.E, padx=10, pady=10)
    enter_onefile.grid(column=1, row=0, pady=5, padx=3)

    skipval_box.grid(column=2, row=0, sticky=tkinter.W+tkinter.E, padx=10, pady=10)
    enter_skipval.grid(column=2, row=0, pady=5, padx=3)

    cmap_box.grid(column=1, row=1, padx=10, pady=10, sticky=tkinter.E+tkinter.W)
    colorschemes.grid(column=1, row=1, sticky=tkinter.E+tkinter.W)

    fileamount_box.grid(column=2, row=1, padx=10, pady=10, sticky=tkinter.E+tkinter.W)
    fileamounts.grid(column=2, row=1, sticky=tkinter.E+tkinter.W)

    reset.grid(column=1, row=2, sticky=tkinter.E+tkinter.W, pady=10, padx=10)

    show_button.grid(column=2, row=2, sticky=tkinter.E+tkinter.W, padx=10, pady=10)

    databutton_box.grid(column=0, row=0, rowspan=4, padx=10, pady=10, sticky=tkinter.N+tkinter.S)
    spec_plot.grid(row=0, sticky=tkinter.W, pady=15, padx=5)
    amp_plot.grid(row=1, sticky=tkinter.W, pady=15, padx=5)
    search_interesting.grid(row=2, sticky=tkinter.W, pady=15, padx=5)

def infopage():
    info_location.set(None)

    root.grid_columnconfigure(index=1, minsize=330)
    root.grid_rowconfigure(index=0, minsize=30)
    root.grid_rowconfigure(index=2, minsize=20)

    widgetlist = [onefilename_box, enter_onefile, skipval_box, enter_skipval, cmap_box, colorschemes, fileamount_box, fileamounts, reset, show_button, databutton_box, spec_plot, amp_plot, search_interesting]
    for widget in widgetlist:
        widget.grid_remove()

    infobutton_box.grid(column=0, row=0, padx=10, pady=10, rowspan=6, sticky=tkinter.N+tkinter.S)

    etp.grid(column=0, row=0, sticky=tkinter.W, padx=10, pady=5)
    scarib.grid(column=0, row=1, sticky=tkinter.W, padx=10, pady=5)
    nwpac.grid(column=0, row=2, sticky=tkinter.W, padx=10, pady=5)
    goc.grid(column=0, row=3, sticky=tkinter.W, padx=10, pady=5)
    cabo.grid(column=0, row=4, sticky=tkinter.W, padx=10, pady=5)
    gom.grid(column=0, row=5, sticky=tkinter.W, padx=10, pady=5)

    numcallsbox.grid(column=1, row=1, columnspan=2, sticky=tkinter.W+tkinter.E+tkinter.N+tkinter.S)
    numcalls.grid(column=0, row=0, sticky=tkinter.W+tkinter.N, padx=10, pady=5)
    datebox.grid(column=1, row=2, columnspan=2, sticky=tkinter.W+tkinter.E+tkinter.N+tkinter.S)
    date.grid(column=0, row=0, sticky=tkinter.W+tkinter.N, padx=10, pady=5)

    freqbox.grid(column=1, row=3, columnspan=2, sticky=tkinter.W+tkinter.E+tkinter.N+tkinter.S)
    freq.grid(column=0, row=0)

    callbox.grid(column=0, row=1)
    call.grid(column=0, row=0)
    callmenu.grid(column=1, row=0)

    minmaxbox.grid(column=1, row=1)
    minmax.grid(column=0, row=0)
    minmax_label.grid(column=1, row=0)
    callmenu.delete(0, tkinter.END)
    current_minmax.set('')

# CREATING MENU
menu = tkinter.Menu(root)
root.config(menu=menu)
datamenu = tkinter.Menu(menu)
menu.add_cascade(label='Data', menu=datamenu)
datamenu.add_command(label='Data', command=datapage)
infomenu = tkinter.Menu(menu)
menu.add_cascade(label='Information', menu=infomenu)
infomenu.add_command(label='Information', command=infopage)

# TYPE IN FILE NAME
onefilename_box = tkinter.LabelFrame(root, text='Filename')
onefilename_box.grid(column=1, row=0, sticky=tkinter.W+tkinter.E, padx=10, pady=10)
onefilename = tkinter.StringVar()
onefilename.set('')
enter_onefile = tkinter.Entry(onefilename_box, textvariable=onefilename, width=16)
enter_onefile.grid(column=1, row=0, pady=5, padx=3)

# TYPE IN SKIP VALUE
def skip_error_check(number):
    valid = False
    if number.isdigit():
        if (int(number) <= 10000) and (int(number) >= 0):
            valid = True
    elif number == '':
        valid = True
    return valid
validate_skip = (root.register(skip_error_check), '%P')
skipval_box = tkinter.LabelFrame(root, text='Skip Value')
skipval_box.grid(column=2, row=0, sticky=tkinter.W+tkinter.E, padx=10, pady=10)
skipval = tkinter.StringVar()
skipval.set(0)
enter_skipval = tkinter.Spinbox(skipval_box, from_=0, to=10000, textvariable=skipval, width=16, validate='all', validatecommand=validate_skip)
enter_skipval.grid(column=2, row=0, pady=5, padx=3)

# OPTION MENUS
def show_options():
    global cmap_box, colorschemes, fileamount_box, fileamounts, fileamount_str, enter_onefile
    if current_page.get() == 'spec':
        enter_onefile.configure(state='normal')
        fileamounts.configure(state='disabled')
        colorschemes.configure(state='normal')
        colorscheme_str.set(cmaps[2])
        skipval.set(0)
        fileamount_str.set(number_of_files[0])
        onefilename.set('')
    else:
        colorschemes.configure(state='disabled')

    if current_page.get() == 'amp':
        enter_onefile.configure(state='normal')
        fileamounts.configure(state='disabled')
        colorschemes.configure(state='disabled')
        skipval.set(0)
        fileamount_str.set(number_of_files[0])
        onefilename.set('')

    if current_page.get() == 'interesting':
        enter_onefile.configure(state='disabled')
        fileamounts.configure(state='normal')
        colorschemes.configure(state='disabled')
        skipval.set(0)
        fileamount_str.set(number_of_files[0])
    else:
        fileamounts.configure(state='disabled')

    if current_page.get() != 'spec' and current_page.get() != 'amp' and current_page.get() != 'interesting':
        enter_onefile.configure(state='disabled')
        fileamounts.configure(state='disabled')
        colorschemes.configure(state='disabled')
        enter_skipval.configure(state='disabled')
        reset.configure(state='disabled')
        show_button.configure(state='disabled')

cmaps = ['hsv', 'Greys', 'nipy_spectral']
cmap_box = tkinter.LabelFrame(root, text='Color Scheme')
colorscheme_str = tkinter.StringVar()
colorscheme_str.set(cmaps[2])
colorschemes = tkinter.OptionMenu(cmap_box, colorscheme_str, *cmaps)
cmap_box.grid(column=1, row=1, padx=10, pady=10, sticky=tkinter.E+tkinter.W)
colorschemes.grid(column=1, row=1, sticky=tkinter.E+tkinter.W)

number_of_files = ['4','9','16']
fileamount_box = tkinter.LabelFrame(root, text='Number of Files')
fileamount_str = tkinter.StringVar()
fileamount_str.set(number_of_files[0])
fileamounts = tkinter.OptionMenu(fileamount_box, fileamount_str, *number_of_files)
fileamount_box.grid(column=2, row=1, padx=10, pady=10, sticky=tkinter.E+tkinter.W)
fileamounts.grid(column=2, row=1, sticky=tkinter.E+tkinter.W)

# PLOT TYPE RADIOBUTTONS
# RADIOBUTTON LABELFRAME
databutton_box = tkinter.LabelFrame(root)
databutton_box.grid(column=0, row=0, rowspan=4, padx=10, pady=10, sticky=tkinter.N+tkinter.S)

current_page = tkinter.StringVar()
current_page.set(None)
spec_plot = tkinter.Radiobutton(databutton_box, text='Create Spectrogram', command=show_options, variable=current_page, value='spec')
spec_plot.grid(row=0, sticky=tkinter.W, pady=15, padx=5)

amp_plot = tkinter.Radiobutton(databutton_box, text='Create Time Series', command=show_options, variable=current_page, value='amp')
amp_plot.grid(row=1, sticky=tkinter.W, pady=15, padx=5)

search_interesting = tkinter.Radiobutton(databutton_box, text= 'Find Interesting', command=show_options, variable=current_page, value='interesting')
search_interesting.grid(row=2, sticky=tkinter.W, pady=15, padx=5)

# RESET BUTTON
def reset_inputs():
    global onefilename, skipval, current_page, cmap_box, colorschemes, fileamount_box, fileamounts, fileamount_str, colorscheme_str
    onefilename.set('')
    skipval.set(0)
    current_page.set(None)
    fileamount_str.set(number_of_files[0])
    colorscheme_str.set(cmaps[2])
    enter_onefile.configure(state='normal')
    fileamounts.configure(state='normal')
    colorschemes.configure(state='normal')

reset = tkinter.Button(root, text='Reset All', command=reset_inputs)
reset.grid(column=1, row=2, sticky=tkinter.E+tkinter.W, pady=10, padx=10)

# FORMAT USER INPUT FOR _ladc FUNCTIONS
trueskip = 0
truefilename = ''
fileamount_int = 0

def recordswindow(recordfn=None, recordskip=None, recordnumber=None):
    global trueskip, truefilename, fileamount_int

    recordswindow = tkinter.Toplevel(root)
    recordswindow.title('Records')
    recordswindow.geometry('630x300')

    truerecords = tkinter.StringVar()
    truerecords.set('')

    trueheader = tkinter.StringVar()
    trueheader.set('')

    recordlabel = tkinter.Label(recordswindow, textvariable=truerecords, wraplength=600, font=('bold', 12), justify='left')
    headerlabel = tkinter.Label(recordswindow, textvariable=trueheader, font=(6))

    if current_page.get() == 'spec' or current_page.get() == 'amp':
        if recordfn is None:
            if recordskip is None:
                detect = ladc.find()
            elif recordskip is not None:
                detect = ladc.find(skip=recordskip)
        elif recordfn is not None:
            detect = ladc.find(filename=recordfn)

        detectbookmark = str(detect)
        truerecords.set(detectbookmark)

        filenamebookmark = detect['filename']
        headerbookmark = (f'File {filenamebookmark} Records')
        trueheader.set(headerbookmark)

    if current_page.get() == 'interesting':
        if recordnumber is not None:
            truerecords.set(ladc.find_interesting(skip_start=recordskip, number_of_files=recordnumber, Type=6, Buoy='13', Disk='0'))
            trueheader.set('File Records')

    recordlabel.grid(column=0, row=1, sticky=tkinter.E+tkinter.W, padx=5, pady=5)
    headerlabel.grid(column=0, row=0, sticky=tkinter.N+tkinter.W, padx=5, pady=5)
    print('window')

# POPUP WINDOW
def build_plot():
    global trueskip, truefilename, fileamount_int
    print('built plot')

    truefilename = onefilename.get()
    if truefilename == '':
        truefilename = None

    fileamount_int = int(fileamount_str.get())

    trueskip = int(skipval.get())
    if trueskip == 0:
        trueskip = None

    if current_page.get() == 'spec' or current_page.get() == 'amp':
        if (trueskip is not None) and (truefilename is not None):
            errorwindow = tkinter.Toplevel(root)
            errorwindow.title('Error')
            errormessage = tkinter.Label(errorwindow, text='Please enter only a filename OR a skip value.')
            errormessage.grid(column=0, row=0, sticky=tkinter.E+tkinter.W, padx=10, pady=10)
            okbutton = tkinter.Button(errorwindow, text='Okay', command=errorwindow.destroy)
            okbutton.grid(column=0, row=1, sticky=tkinter.E+tkinter.W, padx=10, pady=10)
        else:
            try:
                recordswindow(recordfn=truefilename, recordskip=trueskip, recordnumber=None)
                if current_page.get() == 'spec': # use the output from the window building function instead of getting it from the widget
                    ladc.create_spec(skip=trueskip, cmap=(colorscheme_str.get()), figsize=(6,4), save_fig=None, show_plt=True, filename=truefilename)
                if current_page.get() == 'amp':
                    ladc.create_timeseries(filename=truefilename, skip=trueskip, show_plt=True)
            except FileNotFoundError:
                errorwindow = tkinter.Toplevel(root)
                errorwindow.title('Error')
                errormessage = tkinter.Label(errorwindow,text='Sorry, that file does not exist.')
                errormessage.grid(column=0, row=0, sticky=tkinter.E+tkinter.W, padx=10, pady=10)
                okbutton = tkinter.Button(errorwindow, text='Okay', command=errorwindow.destroy)
                okbutton.grid(column=0, row=1, sticky=tkinter.E+tkinter.W, padx=10, pady=10)

    if current_page.get() == 'interesting':
        recordswindow(recordfn=None, recordskip=trueskip, recordnumber=fileamount_int)
        ladc.find_interesting(skip_start=trueskip, number_of_files=fileamount_int, Type=6, Buoy='13', Disk='0')
        ladc.MATLAB_format(plot=True, show_plt=True, save_plt=False, clip_length=577, number_of_files=fileamount_int, directory='data', records=None, Type=6, Buoy='13', Disk='0', skip_start=trueskip)

show_button = tkinter.Button(root, text='Create Figure(s)', command=build_plot)
show_button.grid(column=2, row=2, sticky=tkinter.E+tkinter.W, padx=10, pady=10)

root.mainloop()


<reponame>syys96/digital_rf<filename>python/tools/drf_sti.py
#!python
# ----------------------------------------------------------------------------
# Copyright (c) 2017 Massachusetts Institute of Technology (MIT)
# All rights reserved.
#
# Distributed under the terms of the BSD 3-clause license.
#
# The full license is in the LICENSE file, distributed with this software.
# ----------------------------------------------------------------------------
"""Create a spectral time intensity summary plot for a data set."""

import datetime
import optparse
import os
import sys
import time
import traceback

import dateutil
import digital_rf as drf
import matplotlib.gridspec
import matplotlib.mlab
import matplotlib.pyplot
import numpy as np
import pytz
import scipy
import scipy.signal

class DataPlotter(object):
    def __init__(self, control):
        """Initialize a data plotter for STI plotting."""
        self.control = control
        ch = self.control.channel.split(":")
        self.channel = ch[0]
        self.sub_channel = int(ch[1])

        # open digital RF path
        self.dio = drf.DigitalRFReader(self.control.path)

        if self.control.verbose:
            print("channel bounds:", self.dio.get_bounds(self.channel))

        self.bounds = self.dio.get_bounds(self.channel)

        print("bounds ", self.bounds)

        # Figure setup

        self.f = matplotlib.pyplot.figure(
            figsize=(7, np.min([np.max([4, self.control.frames]), 7])), dpi=128
        )

        self.gridspec = matplotlib.gridspec.GridSpec(self.control.frames, 1)

        self.subplots = []

        """ Setup the subplots for this display """
        for n in np.arange(self.control.frames):
            ax = self.f.add_subplot(self.gridspec[n])
            self.subplots.append(ax)

    def plot(self):
        """Iterate over the data set and plot the STI into the subplot panels.

        Each panel is divided into a provided number of bins of a given
        integration length. Strides between the panels are made between
        integrations.

        """
        # initialize outside the loop to avoid memory leak

        # initial plotting scales
        vmin = 0
        vmax = 0

        sr = self.dio.get_properties(self.channel)["samples_per_second"]

        if self.control.verbose:
            print("sample rate: ", sr)

        # initial time info
        b = self.dio.get_bounds(self.channel)

        if self.control.verbose:
            print("data bounds: ", b)

        if self.control.start:
            dtst0 = dateutil.parser.parse(self.control.start)
            st0 = (
                dtst0 - datetime.datetime(1970, 1, 1, tzinfo=pytz.utc)
            ).total_seconds()
            st0 = int(st0 * sr)
        else:
            st0 = int(b[0])

        if self.control.end:
            dtst0 = dateutil.parser.parse(self.control.end)
            et0 = (
                dtst0 - datetime.datetime(1970, 1, 1, tzinfo=pytz.utc)
            ).total_seconds()
            et0 = int(et0 * sr)
        else:
            et0 = int(b[1])

        if self.control.verbose:

            print("start sample st0: ", st0)
            print("end sample et0: ", et0)

        blocks = self.control.bins * self.control.frames

        samples_per_stripe = (
            self.control.num_fft * self.control.integration * self.control.decimation
        )
        total_samples = blocks * samples_per_stripe

        if total_samples > (et0 - st0):
            print(
                "Insufficient samples for %d samples per stripe and %d blocks between %ld and %ld"
                % (samples_per_stripe, blocks, st0, et0)
            )
            return

        stripe_stride = (et0 - st0) / blocks

        bin_stride = stripe_stride / self.control.bins

        start_sample = st0

        print("first ", start_sample)

        # get metadata
        # this could be done better to ensure we catch frequency or sample rate
        # changes
        mdt = self.dio.read_metadata(st0, et0, self.channel)
        try:
            md = mdt[list(mdt.keys())[0]]
            cfreq = md["center_frequencies"].ravel()[self.sub_channel]
        except (IndexError, KeyError):
            cfreq = 0.0

        if self.control.verbose:
            print(
                "processing info : ",
                self.control.frames,
                self.control.bins,
                samples_per_stripe,
                bin_stride,
            )

        for p in np.arange(self.control.frames):
            sti_psd_data = np.zeros([self.control.num_fft, self.control.bins], np.float)
            sti_times = np.zeros([self.control.bins], np.complex128)

            for b in np.arange(self.control.bins, dtype=np.int_):

                if self.control.verbose:
                    print(
                        "read vector :", self.channel, start_sample, samples_per_stripe
                    )

                data = self.dio.read_vector(
                    start_sample, samples_per_stripe, self.channel, self.sub_channel
                )
                if self.control.decimation > 1:
                    data = scipy.signal.decimate(data, self.control.decimation)
                    sample_freq = sr / self.control.decimation
                else:
                    sample_freq = sr

                if self.control.mean:
                    detrend_fn = matplotlib.mlab.detrend_mean
                else:
                    detrend_fn = matplotlib.mlab.detrend_none

                try:
                    psd_data, freq_axis = matplotlib.mlab.psd(
                        data,
                        NFFT=self.control.num_fft,
                        Fs=float(sample_freq),
                        detrend=detrend_fn,
                        scale_by_freq=False,
                    )
                except:
                    traceback.print_exc(file=sys.stdout)

                sti_psd_data[:, b] = np.real(10.0 * np.log10(np.abs(psd_data) + 1e-12))

                sti_times[b] = start_sample / sr

                start_sample += stripe_stride

            # Now Plot the Data
            ax = self.subplots[p]

            # determine image x-y extent
            extent = (
                0,
                self.control.bins,
                np.min(freq_axis) / 1e3,
                np.max(freq_axis) / 1e3,
            )

            # determine image color extent in log scale units
            Pss = sti_psd_data

            if self.control.zaxis:
                vmin = int(self.control.zaxis.split(":")[0])
                vmax = int(self.control.zaxis.split(":")[1])
            else:
                med_Pss = np.nanmedian(Pss)
                max_Pss = np.nanmax(Pss)
                vmin = np.real(med_Pss - 6.0)
                vmax = np.real(med_Pss + (max_Pss - med_Pss) * 0.61803398875 + 50.0)

            im = ax.imshow(
                sti_psd_data,
                cmap="jet",
                origin="lower",
                extent=extent,
                interpolation="nearest",
                vmin=vmin,
                vmax=vmax,
                aspect="auto",
            )

            ax.set_ylabel("f (kHz)", fontsize=8)

            # plot dates

            tick_spacing = np.arange(
                self.control.bins / 8,
                self.control.bins,
                self.control.bins / 8,
                dtype=np.int_,
            )
            ax.set_xticks(tick_spacing)
            tick_labels = []

            for s in tick_spacing:
                tick_time = sti_times[s]

                if tick_time == 0:
                    tick_string = ""
                else:
                    gm_tick_time = time.gmtime(np.real(tick_time))
                    tick_string = "%02d:%02d:%02d" % (
                        gm_tick_time[3],
                        gm_tick_time[4],
                        gm_tick_time[5],
                    )
                    tick_labels.append(tick_string)

            ax.set_xticklabels(tick_labels)

            # set the font sizes
            tl = ax.get_xticklabels()

            for tk in tl:
                tk.set_size(8)
            del tl

            tl = ax.get_yticklabels()

            for tk in tl:
                tk.set_size(8)
            del tl

        print("last ", start_sample)

        # create a time stamp
        start_time = st0 / sr
        srt_time = time.gmtime(start_time)
        sub_second = int(round((start_time - int(start_time)) * 100))

        timestamp = "%d-%02d-%02d %02d:%02d:%02d.%02d UT" % (
            srt_time[0],
            srt_time[1],
            srt_time[2],
            srt_time[3],
            srt_time[4],
            srt_time[5],
            sub_second,
        )

        self.f.suptitle(
            "%s %s %4.2f MHz (%s)"
            % (self.control.title, timestamp, cfreq / 1e6, self.control.path),
            fontsize=10,
        )

        # ax.legend(fontsize=8)
        ax.set_xlabel("time (UTC)", fontsize=8)

        # fixup ticks

        tl = ax.get_xticklabels()
        for tk in tl:
            tk.set_size(8)
        del tl
        tl = ax.get_yticklabels()
        for tk in tl:
            tk.set_size(8)
        del tl

        self.gridspec.update()

        self.f.tight_layout()

        self.f.subplots_adjust(top=0.95, right=0.88)
        cax = self.f.add_axes([0.9, 0.12, 0.02, 0.80])
        self.f.colorbar(im, cax=cax)
        if self.control.outname:
            fname, ext = os.path.splitext(self.control.outname)
            if ext == "":
                ext = ".png"
            print("Save plot as {}".format(fname + ext))
            matplotlib.pyplot.savefig(fname + ext)
        if self.control.appear or not self.control.outname:
            print("Show plot")
            matplotlib.pyplot.show()

def parse_command_line(str_input=None):
    # if str_input is None:
    #     parser = optparse.OptionParser()
    # else:
    #     parser = optparse.OptionParser(str_input)
    parser = optparse.OptionParser()

    parser.add_option(
        "-t",
        "--title",
        dest="title",
        default="Digital RF Data",
        help="Use title provided for the data.",
    )
    parser.add_option(
        "-s",
        "--start",
        dest="start",
        default=None,
        help="Use the provided start time instead of the first time in the data. format is ISO8601: 2015-11-01T15:24:00Z",
    )
    parser.add_option(
        "-e",
        "--end",
        dest="end",
        default=None,
        help="Use the provided end time for the plot. format is ISO8601: 2015-11-01T15:24:00Z",
    )

    parser.add_option(
        "-p",
        "--path",
        dest="path",
        help="Use data from the provided digital RF data <path>.",
    )
    parser.add_option(
        "-c",
        "--channel",
        dest="channel",
        default="ch0:0",
        help="Use data from the provided digital RF channel <channel>:<subchannel>.",
    )
    parser.add_option(
        "-l",
        "--length",
        dest="length",
        default=0.04,
        type="float",
        help="The default data length in seconds for unframed data.",
    )
    parser.add_option(
        "-b",
        "--bins",
        dest="bins",
        default=128,
        type="int",
        help="The number of time bins for the STI.",
    )
    parser.add_option(
        "-f",
        "--frames",
        dest="frames",
        default=4,
        type="int",
        help="The number of sub-panel frames in the plot.",
    )
    parser.add_option(
        "-n",
        "--num_fft",
        dest="num_fft",
        default=128,
        type="int",
        help="The number of FFT bints for the STI.",
    )
    parser.add_option(
        "-i",
        "--integration",
        dest="integration",
        default=1,
        type="int",
        help="The number of rasters to integrate for each plot.",
    )
    parser.add_option(
        "-d",
        "--decimation",
        dest="decimation",
        default=1,
        type="int",
        help="The decimation factor for the data (integer).",
    )
    parser.add_option(
        "-m",
        "--mean",
        dest="mean",
        action="store_true",
        default=False,
        help="Remove the mean from the data at the PSD processing step.",
    )
    parser.add_option(
        "-z",
        "--zaxis",
        dest="zaxis",
        default=None,
        type="string",
        help="zaxis colorbar setting e.g. -50:50",
    )
    parser.add_option(
        "-v",
        "--verbose",
        action="store_true",
        dest="verbose",
        default=False,
        help="Print status messages to stdout.",
    )
    parser.add_option(
        "-o",
        "--outname",
        dest="outname",
        default=None,
        type=str,
        help="Name of file that figure will be saved under.",
    )
    parser.add_option(
        "-a",
        "--appear",
        action="store_true",
        dest="appear",
        default=False,
        help="Makes the plot appear through pyplot show.",
    )
    if str_input is None:
        (options, args) = parser.parse_args()
    else:
        (options, args) = parser.parse_args(str_input)

    return (options, args)

#
# MAIN PROGRAM
#

# Setup Defaults
if __name__ == "__main__":
    """
    Needed to add main function to use outside functions outside of module.
    """

    # Parse the Command Line for configuration
    (options, args) = parse_command_line()

    if options.path is None:
        print("Please provide an input source with the -p option!")
        sys.exit(1)

    # Activate the DataPlotter
    dpc = DataPlotter(options)

    dpc.plot()


<reponame>JohnStarich/python-pool-performance<filename>pools/pool.py
from requests.adapters import HTTPAdapter
from collections.abc import Mapping, Sequence
from types import FunctionType
from tqdm import tqdm
import time
import sys
import gc

class PoolTest(object):
    def __init__(self, worker_count: int):
        self.worker_count = worker_count
        self.pool = self.init_pool(worker_count)
        self.compute_resource = self.init_compute_resource()
        self.network_resource = self.init_network_resource()

    def init_pool(self, worker_count: int) -> object:
        raise NotImplementedError("{} does not implement init_pool"
                                  .format(self.__class__.__name__))

    def destroy_pool(self):
        pass

    def map(self, work_func: FunctionType, inputs: Sequence) -> Sequence:
        raise NotImplementedError("{} does not implement map"
                                  .format(self.__class__.__name__))

    def init_compute_resource(self) -> object:
        from cmath import sqrt
        return sqrt

    def init_network_resource(self) -> object:
        import requests
        return requests.Session

    @staticmethod
    def do_compute_work(args) -> None:
        compute_resource, num, *_ = args
        sqrt = compute_resource
        sqrt(sqrt(sqrt(num)))

    @staticmethod
    def do_network_work(args) -> None:
        network_resource, *_ = args
        Session = network_resource
        with Session() as s:
            adapter = HTTPAdapter(max_retries=3)
            s.mount('http://', adapter)
            s.get('http://localhost:8080/')

    def run_compute_test(self, jobs: int, trials: int,
                         show_progress: bool=False) -> Mapping:
        return self._run_test(self.do_compute_work, self.compute_resource,
                              jobs, trials, show_progress=show_progress)

    def run_network_test(self, jobs: int, trials: int,
                         show_progress: bool=False) -> Mapping:
        return self._run_test(self.do_network_work, self.network_resource,
                              jobs, trials, show_progress=show_progress)

    def _run_test(self, work_func: FunctionType, work_resource: object,
                  jobs: int, trials: int,
                  show_progress: bool=False) -> Mapping:
        results = {
            'jobs': jobs,
            'trials': trials,
            'time': [],
            'blocks': [],
        }
        # Forcibly evaluate the inputs to prevent time/resources taken up later
        inputs = list(zip(
            [work_resource] * jobs,
            range(jobs)
        ))
        trial_iter = range(trials)
        if show_progress is True and trials > 2:
            trial_iter = tqdm(trial_iter, desc='trials')
        gc.collect()
        for _ in trial_iter:
            # Run trial of pool map function and measure it
            gc.collect()
            blocks_start = sys.getallocatedblocks()
            time_start = time.time()
            list(self.map(work_func, inputs))
            time_end = time.time()
            results['time'].append(time_end - time_start)
            # Get allocated blocks before garbage collection to show peak usage
            blocks_end = sys.getallocatedblocks()
            results['blocks'].append(blocks_end - blocks_start)
        return results


import numpy as np
import math
from ..utils import Monitor
from scipy.optimize import basinhopping, minimize, Bounds

class OverbudgetException(Exception):
    def __init__(self):
        super(Exception, self).__init__()

def get_variable_bounds(problem):
    lbs = problem.lbs()
    ubs = problem.ubs()

    return Bounds(lbs, ubs)

def optimize_basinhopping(problem, max_evals, T=1.0, stepsize=0.5, localmethod="L-BFGS-B", log=None, verbose=True):
    vt = problem.vartype()
    lbs = problem.lbs()
    ubs = problem.ubs()
    mon = Monitor(f"scipy.basinhopping/{localmethod}", problem, log=log)
    def f(x):
        # This approach does not stay within its evaluation budget (it has little to no way to enforce this!)
        # As such. raise an exception if we are over the limit
        if mon.num_iters > max_evals:
            raise OverbudgetException()
        # scipy.optimize
        xvec = x.copy()
        # Round non-continuous variables
        xvec[vt != 'cont'] = np.round(xvec[vt != 'cont'])
        # Clamp variable values to bounds
        np.clip(xvec, lbs, ubs, out=xvec)
        mon.commit_start_eval()
        r = problem.evaluate(xvec)
        mon.commit_end_eval(xvec, r)
        return r

    def budget_check_global(x, f, accept):
        # Callback used to stop basin hopping when evaluation limit is reached.
        # x -- local minimum solution
        # f -- corresponding fitness
        # accept -- whether this local optima was accepted as the new reference solution
        return mon.num_iters >= max_evals

    def budget_check_local(x):
        # Callback used to stop local optimization when evaluation limit is reached.
        # x -- local minimum solution
        return mon.num_iters >= max_evals

    minimizer_kwargs = {
        'method': localmethod,
        'bounds': get_variable_bounds(problem),
        'callback': budget_check_local
    }

    # Generate initial point
    lb = problem.lbs()
    ub = problem.ubs()
    d = len(lb)
    x0 = np.random.rand(d)*(ub-lb) + lb
    x0[vt != 'cont'] = np.round(x0[vt != 'cont'])

    mon.start()
    try:
        optim_result = basinhopping(func=f, x0=x0, niter=max_evals, T=T, stepsize=stepsize, minimizer_kwargs=minimizer_kwargs, callback=budget_check_global)
    except OverbudgetException as e:
        pass
    mon.end()

    solX = mon.best_x #optim_result['x']
    solY = mon.best_fitness #optim_result['fun']

    return solX, solY, mon

def optimize_scipy_local(problem, max_evals, method="BFGS", log=None, verbose=False):

    vt = problem.vartype()
    mon = Monitor(f"scipy.{method}", problem, log=log)
    def f(x):
        # This approach does not stay within its evaluation budget (it has little to no way to enforce this!)
        # As such. raise an exception if we are over the limit
        if mon.num_iters > max_evals:
            raise OverbudgetException()
        # scipy.optimize
        xvec = x.copy()
        # Round non-continuous variables
        xvec[vt != 'cont'] = np.round(xvec[vt != 'cont'])
        mon.commit_start_eval()
        r = problem.evaluate(xvec)
        mon.commit_end_eval(xvec, r)
        return r

    def budget_check_local(x):
        # Callback used to stop local optimization when evaluation limit is reached.
        # x -- local minimum solution
        return mon.num_iters >= max_evals

    # Generate initial point, round the integers.
    lb = problem.lbs()
    ub = problem.ubs()
    d = len(lb)
    x0 = np.random.rand(d)*(ub-lb) + lb
    x0[vt != 'cont'] = np.round(x0[vt != 'cont'])

    mon.start()
    try:
        optim_result = minimize(fun=f, x0=x0, method=method, bounds=get_variable_bounds(problem), options={'maxiter': max_evals}, callback=budget_check_local)
    except OverbudgetException as e:
        pass
    mon.end()

    solX = mon.best_x #optim_result['x']
    solY = mon.best_fitness #optim_result['fun']

    return solX, solY, mon


<reponame>prise-3d/IPFML
"""
Functions which can be used to extract information from image or reduce it
"""

# main imports
import os
import random
import numpy as np

# image processing imports
from numpy.linalg import svd
from scipy import misc
from sklearn import preprocessing
from skimage import io, color
import cv2
from PIL import Image

# ipfml imports
from ipfml.processing import compression

def get_LAB(image):
    """Transforms RGB  Image into Lab

    Args:
        image: image to convert

    Returns:
        Lab information

    Usage:

    >>> from PIL import Image
    >>> from ipfml.processing import transform
    >>> img = Image.open('./images/test_img.png')
    >>> Lab = transform.get_LAB(img)
    >>> Lab.shape
    (200, 200, 3)
    """

    return color.rgb2lab(image)

def get_LAB_L(image):
    """Transforms RGB Image into Lab and returns L

    Args:
        image: image to convert

    Returns:
        The L chanel from Lab information

    >>> from PIL import Image
    >>> from ipfml.processing import transform
    >>> img = Image.open('./images/test_img.png')
    >>> L = transform.get_LAB_L(img)
    >>> L.shape
    (200, 200)
    """

    lab = get_LAB(image)
    return lab[:, :, 0]

def get_LAB_a(image):
    """Transforms RGB Image into LAB and returns a

    Args:
        image: image to convert

    Returns:
        The a chanel from Lab information

    Usage:

    >>> from PIL import Image
    >>> from ipfml.processing import transform
    >>> img = Image.open('./images/test_img.png')
    >>> a = transform.get_LAB_a(img)
    >>> a.shape
    (200, 200)
    """

    lab = get_LAB(image)
    return lab[:, :, 1]

def get_LAB_b(image):
    """Transforms RGB Image into LAB and returns b

    Args:
        image: image to convert

    Returns:
        The b chanel from Lab information

    Usage :

    >>> from PIL import Image
    >>> from ipfml.processing import transform
    >>> img = Image.open('./images/test_img.png')
    >>> b = transform.get_LAB_b(img)
    >>> b.shape
    (200, 200)
    """

    lab = get_LAB(image)
    return lab[:, :, 2]

def get_XYZ(image):
    """Transforms RGB Image into XYZ

    Args:
        image: image to convert

    Returns:
        XYZ information obtained from transformation

    Usage:

    >>> from PIL import Image
    >>> from ipfml.processing import transform
    >>> img = Image.open('./images/test_img.png')
    >>> transform.get_XYZ(img).shape
    (200, 200, 3)
    """

    return color.rgb2xyz(image)

def get_XYZ_X(image):
    """Transforms RGB Image into XYZ and returns X

    Args:
        image: image to convert

    Returns:
        The X chanel from XYZ information

    Usage:

    >>> from PIL import Image
    >>> from ipfml.processing import transform
    >>> img = Image.open('./images/test_img.png')
    >>> x = transform.get_XYZ_X(img)
    >>> x.shape
    (200, 200)
    """

    xyz = color.rgb2xyz(image)
    return xyz[:, :, 0]

def get_XYZ_Y(image):
    """Transforms RGB Image into XYZ and returns Y

    Args:
        image: image to convert

    Returns:
        The Y chanel from XYZ information

    Usage:

    >>> from PIL import Image
    >>> from ipfml.processing import transform
    >>> img = Image.open('./images/test_img.png')
    >>> y = transform.get_XYZ_Y(img)
    >>> y.shape
    (200, 200)
    """

    xyz = color.rgb2xyz(image)
    return xyz[:, :, 1]

def get_XYZ_Z(image):
    """Transforms RGB Image into XYZ and returns Z

    Args:
        image: image to convert

    Returns:
        The Z chanel from XYZ information

    Raises:
        ValueError: If `nb_bits` has unexpected value. `nb_bits` needs to be in interval [1, 8].

    Usage:

    >>> from PIL import Image
    >>> from ipfml.processing import transform
    >>> img = Image.open('./images/test_img.png')
    >>> z = transform.get_XYZ_Z(img)
    >>> z.shape
    (200, 200)
    """

    xyz = color.rgb2xyz(image)
    return xyz[:, :, 2]

def get_low_bits_img(image, nb_bits=4):
    """Returns Image or Numpy array with data information reduced using only low bits

    Args:
        image: image to convert
        nb_bits: optional parameter which indicates the number of bits to keep

    Returns:
        Numpy array with reduced values

    Usage:

    >>> from PIL import Image
    >>> from ipfml.processing import transform
    >>> img = Image.open('./images/test_img.png')
    >>> low_bits_img = transform.get_low_bits_img(img, 5)
    >>> low_bits_img.shape
    (200, 200, 3)
    """

    if nb_bits <= 0:
        raise ValueError(
            "unexpected value of number of bits to keep. @nb_bits needs to be positive and greater than 0."
        )

    if nb_bits > 8:
        raise ValueError(
            "Unexpected value of number of bits to keep. @nb_bits needs to be in interval [1, 8]."
        )

    img_arr = np.array(image)

    bits_values = sum([pow(2, i - 1) for i in range(1, nb_bits + 1)])

    return img_arr & bits_values

def get_bits_img(image, interval):
    """Returns only bits specified into the interval

    Args:
        image: image to convert using this interval of bits value to keep
        interval: (begin, end) of bits values

    Returns:
        Numpy array with reduced values

    Raises:
        ValueError: If min value from interval is not >= 1.
        ValueError: If max value from interval is not <= 8.
        ValueError: If min value from interval >= max value.

    Usage:

    >>> from PIL import Image
    >>> from ipfml.processing import transform
    >>> img = Image.open('./images/test_img.png')
    >>> bits_img = transform.get_bits_img(img, (2, 5))
    >>> bits_img.shape
    (200, 200, 3)
    """

    img_arr = np.array(image)
    begin, end = interval

    if begin < 1:
        raise ValueError(
            "Unexpected value of interval. Interval min value needs to be >= 1."
        )

    if end > 8:
        raise ValueError(
            "Unexpected value of interval. Interval min value needs to be <= 8."
        )

    if begin >= end:
        raise ValueError("Unexpected interval values order.")

    bits_values = sum([pow(2, i - 1) for i in range(begin, end + 1)])

    return img_arr & bits_values

def gray_to_mscn(image):
    """Convert Grayscale Image into Mean Subtracted Contrast Normalized (MSCN)

    Args:
        image: grayscale image

    Returns:
        MSCN matrix obtained from transformation

    Usage:

    >>> from PIL import Image
    >>> from ipfml.processing import transform
    >>> img = Image.open('./images/test_img.png')
    >>> img = transform.get_LAB_L(img)
    >>> img_mscn = transform.gray_to_mscn(img)
    >>> img_mscn.shape
    (200, 200)
    """

    s = 7 / 6
    blurred = cv2.GaussianBlur(image, (7, 7),
                               s)  # apply gaussian blur to the image
    blurred_sq = blurred * blurred
    sigma = cv2.GaussianBlur(image * image, (7, 7), s)
    sigma = abs(sigma - blurred_sq)**0.5
    sigma = sigma + 1.0 / 255  # avoid DivideByZero Exception
    mscn = (image - blurred) / sigma  # MSCN(i, j) image

    return mscn

def rgb_to_mscn(image):
    """Convert RGB Image into Mean Subtracted Contrast Normalized (MSCN)

    Args:
        image: 3D RGB image Numpy array or PIL RGB image

    Returns:
        2D Numpy array with MSCN information

    Example:

    >>> from PIL import Image
    >>> from ipfml.processing import transform
    >>> img = Image.open('./images/test_img.png')
    >>> img_mscn = transform.rgb_to_mscn(img)
    >>> img_mscn.shape
    (200, 200)
    """

    # check if PIL image or not
    img_arr = np.array(image)

    # convert rgb image to gray
    im = np.array(color.rgb2gray(img_arr) * 255, 'uint8')

    return gray_to_mscn(im)

def get_mscn_coefficients(image):
    """Compute the Mean Substracted Constrast Normalized coefficients of an image

    Args:
        image: PIL Image, Numpy array or path of image

    Returns:
        MSCN coefficients

    Raises:
        FileNotFoundError: If `image` is set as str path and image was not found
        ValueError: If `image` numpy shape are not correct

    Example:

    >>> from PIL import Image
    >>> import numpy as np
    >>> from ipfml.processing import transform
    >>> image_values = Image.open('./images/test_img.png')
    >>> mscn_coefficients = transform.get_mscn_coefficients(image_values)
    >>> mscn_coefficients.shape
    (200, 200)
    """

    if isinstance(image, str):
        if os.path.exists(image):
            # open image directly as grey level image
            imdist = cv2.imread(image, 0)
        else:
            raise FileNotFoundError('Image not found in your system')

    elif isinstance(image, np.ndarray):
        # convert if necessary to grey level numpy array
        if image.ndim == 2:
            imdist = image
        if image.ndim == 3:
            imdist = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
        else:
            raise ValueError('Incorrect image shape')
    else:
        # if PIL Image
        image = np.asarray(image)

        if image.ndim == 2:
            imdist = image
        if image.ndim == 3:
            imdist = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
        else:
            raise ValueError('Incorrect image shape')

    imdist = imdist.astype(np.float64)
    imdist = imdist / 255.0

    # calculating MSCN coefficients
    mu = cv2.GaussianBlur(imdist, (7, 7),
                          7 / 6,
                          borderType=cv2.BORDER_CONSTANT)
    mu_sq = mu * mu
    sigma = cv2.GaussianBlur(imdist * imdist, (7, 7),
                             7 / 6,
                             borderType=cv2.BORDER_CONSTANT)
    sigma = np.sqrt(abs((sigma - mu_sq)))
    structdis = (imdist - mu) / (sigma + 1)
    return structdis

def get_LAB_L_SVD(image):
    """Returns Singular values from LAB L Image information

    Args:
        image: PIL Image or Numpy array

    Returns:
        U, s, V information obtained from SVD compression using Lab

    Example:

    >>> from PIL import Image
    >>> from ipfml.processing import transform
    >>> img = Image.open('./images/test_img.png')
    >>> U, s, V = transform.get_LAB_L_SVD(img)
    >>> U.shape
    (200, 200)
    >>> len(s)
    200
    >>> V.shape
    (200, 200)
    """
    L = get_LAB_L(image)
    return compression.get_SVD(L)

def get_LAB_L_SVD_s(image):
    """Returns s (Singular values) SVD from L of LAB Image information

    Args:
        image: PIL Image or Numpy array

    Returns:
        vector of singular values

    Example:

    >>> from PIL import Image
    >>> from ipfml.processing import transform
    >>> img = Image.open('./images/test_img.png')
    >>> s = transform.get_LAB_L_SVD_s(img)
    >>> len(s)
    200
    """
    L = get_LAB_L(image)
    return compression.get_SVD_s(L)

def get_LAB_L_SVD_U(image):
    """Returns U SVD from L of LAB Image information

    Args:
        image: PIL Image or Numpy array

    Returns:
        U matrix of SVD compression

    Example:

    >>> from PIL import Image
    >>> from ipfml.processing import transform
    >>> img = Image.open('./images/test_img.png')
    >>> U = transform.get_LAB_L_SVD_U(img)
    >>> U.shape
    (200, 200)
    """
    L = get_LAB_L(image)
    return compression.get_SVD_U(L)

def get_LAB_L_SVD_V(image):
    """Returns V SVD from L of LAB Image information

    Args:
        image: PIL Image or Numpy array

    Returns:
        V matrix of SVD compression

    Example:

    >>> from PIL import Image
    >>> from ipfml.processing import transform
    >>> img = Image.open('./images/test_img.png')
    >>> V = transform.get_LAB_L_SVD_V(img)
    >>> V.shape
    (200, 200)
    """

    L = get_LAB_L(image)
    return compression.get_SVD_V(L)

def rgb_to_grey_low_bits(image, nb_bits=4):
    """Convert RGB Image into grey image using only 4 low bits values

    Args:
        image: 3D RGB image Numpy array or PIL RGB image
        nb_bits: optional parameter which indicates the number of bits to keep (default 4)

    Returns:
        2D Numpy array with low bits information kept

    Example:

    >>> from PIL import Image
    >>> from ipfml.processing import transform
    >>> img = Image.open('./images/test_img.png')
    >>> low_bits_grey_img = transform.rgb_to_grey_low_bits(img, 5)
    >>> low_bits_grey_img.shape
    (200, 200)
    """

    img_arr = np.array(image)
    grey_block = np.array(color.rgb2gray(img_arr) * 255, 'uint8')

    return get_low_bits_img(grey_block, nb_bits)

def rgb_to_LAB_L_low_bits(image, nb_bits=4):
    """Convert RGB Image into Lab L channel image using only 4 low bits values

    Args:
        image: 3D RGB image Numpy array or PIL RGB image
        nb_bits: optional parameter which indicates the number of bits to keep (default 4)

    Returns:
        2D Numpy array with low bits information kept

    Example:

    >>> from PIL import Image
    >>> from ipfml.processing import transform
    >>> img = Image.open('./images/test_img.png')
    >>> low_bits_Lab_l_img = transform.rgb_to_LAB_L_low_bits(img, 5)
    >>> low_bits_Lab_l_img.shape
    (200, 200)
    """

    L_block = np.asarray(get_LAB_L(image), 'uint8')

    return get_low_bits_img(L_block, nb_bits)

def rgb_to_LAB_L_bits(image, interval):
    """Returns only bits from LAB L canal specified into the interval

    Args:
        image: image to convert using this interval of bits value to keep
        interval: (begin, end) of bits values

    Returns:
        2D Numpy array with reduced values

    >>> from PIL import Image
    >>> from ipfml.processing import transform
    >>> img = Image.open('./images/test_img.png')
    >>> bits_Lab_l_img = transform.rgb_to_LAB_L_bits(img, (2, 6))
    >>> bits_Lab_l_img.shape
    (200, 200)
    """

    L_block = np.asarray(get_LAB_L(image), 'uint8')

    return get_bits_img(L_block, interval)


import numpy as np
from scipy.stats import multivariate_normal
from to.probabilistic_model import ProbabilisticModel
from evolution import chromosome as evolution
from copy import deepcopy

class MixtureModel(object):

    def __init__(self, allModels, alpha=False):
        self.model_list = allModels.copy()
        self.nModels = len(allModels)
        if alpha is False:
            self.alpha = (1/self.nModels)*np.ones(self.nModels)
        else:
            self.alpha = alpha

        self.probTable = None
        self.nSol = None
        self.__target_model_added = False

    def add_target_solutions(self, solutions, modelType):
        if not self.__target_model_added:
            self.nModels = self.nModels + 1
            self.model_list.append(ProbabilisticModel(modelType=modelType))
            self.model_list[-1].buildModel(solutions)
            self.__target_model_added = True
        else:
            raise Exception('Target model is already added.')

    def add_target_model(self, target_model):
        if not self.__target_model_added:
            self.nModels = self.nModels + 1
            self.model_list.append(target_model)
            self.target_model_added = True
        else:
            raise Exception('Target model is already added.')

    def createTable(self, solutions, CV, modelType, probs_RL=None):
        if CV:
            self.add_target_solutions(solutions, modelType)
            self.alpha = (1/self.nModels) * np.ones(self.nModels)
            nSol = solutions.shape[0]
            self.nSol = nSol
            self.probTable = np.ones([nSol, self.nModels])

            if probs_RL is None:
                for j in range(self.nModels-1):
                    self.probTable[:, j] = self.model_list[j].pdfEval(solutions)
            else:
                for j in range(0, self.nModels-1):
                    self.probTable[:, j] = self.model_list[j].pdfEval(solutions) # Time complexity: O(pd)

            for i in range(nSol):  # Leave-one-out cross validation
                x = np.concatenate((solutions[:i, :], solutions[i+1:, :]))
                tModel = ProbabilisticModel(modelType=modelType)
                tModel.buildModel(x)
                self.probTable[i, -1] = tModel.pdfEval(solutions[[i], :])
        else:
            nSol = solutions.shape[0]
            self.probTable = np.ones([nSol, self.nModels])
            for j in range(self.nModels):
                self.probTable[:, j] = self.model_list[j].pdfEval(solutions)
            self.nSol = nSol

    def EMstacking(self, iterations=1):
        for t in range(iterations):
            print(t)
            talpha = self.alpha
            probVector = np.matmul(self.probTable, talpha.T)
            if any(probVector == 0):
                print('probVector: ', probVector)
                print('self.probTable: ', self.probTable)
                print('talpha: ', talpha)
            for i in range(self.nModels):
                talpha[i] = np.sum((1/self.nSol)*talpha[i]*self.probTable[:, i]/probVector)
            self.alpha = talpha

        if np.sum(np.isnan(self.alpha)) > 0:
            print('sanity check mutate')
            self.alpha = np.zeros(self.nModels)
            self.alpha[-1] = 1

    def mutate(self, version='normal'):
        modif_alpha = None

        modif_alpha = self.alpha + np.random.rand(self.nModels)*0.01

        total_alpha = np.sum(modif_alpha)
        if total_alpha == 0:
            self.alpha = np.zeros(self.nModels)
            self.alpha[-1] = 1
        else:
            self.alpha = modif_alpha/total_alpha

        # Sanity check
        if np.sum(np.isnan(self.alpha)) > 0:
            print('sanity check mutate')
            self.alpha = np.zeros(self.nModels)
            self.alpha[-1] = 1

    def sample(self, nSol, samplesRL=None, preprocess=False):

        if preprocess:
            i = 0
            while any(self.alpha[self.alpha!=0]<(1/nSol - np.finfo(np.float32).eps)):
                self.alpha[self.alpha<(1/nSol - np.finfo(np.float32).eps)] = 0
                self.alpha = self.alpha/np.sum(self.alpha)
                i += 1

        indSamples = np.ceil(nSol*self.alpha).astype(int)
        solutions = np.array([])
        for i in range(self.nModels):
            if indSamples[i] == 0:
                pass
            elif i == self.nModels - 2 and samplesRL is not None:
                solutions = np.vstack([solutions, samplesRL]) if solutions.size else samplesRL
            else:
                sols = self.model_list[i].sample(indSamples[i])
                solutions = np.vstack([solutions, sols]) if solutions.size else sols
        solutions = solutions[np.random.permutation(solutions.shape[0]), :]
        solutions = solutions[:nSol, :]
        return solutions

    def sample_enhanced(self, nSol, problem, mutation_strength,
                        samples_count, max_sampling_num=None, solution_found=None,
                        problem_type='knapsack', net=None, s_len=None, mutation=True):
        """
        This sampling function only works for sTrEvo algorithm
        """

        if max_sampling_num is None:
            max_sampling_num = nSol
        indSamples = np.ceil(nSol*self.alpha).astype(int)

        solutions = []
        added_solutions = []
        solutions_idx = []
        for i in range(self.nModels):
            if indSamples[i] == 0:
                pass
            else:

                sampling_size = min(max_sampling_num, indSamples[i])
                sols_idx = np.ones(sampling_size) * i
                sols = self.model_list[i].sample(sampling_size)

                solutions = np.append(solutions, sols, axis=0) if len(solutions) else deepcopy(sols)
                solutions_idx = np.append(solutions_idx, sols_idx, axis=0) if len(sols_idx) else deepcopy(sols_idx)

        perm_indexes = np.random.permutation(len(solutions))
        solutions_num = min(nSol, len(solutions))
        solutions = solutions[perm_indexes][:solutions_num]
        solutions_idx = solutions_idx[perm_indexes][:solutions_num].astype(np.int)

        # Fitness Evaluation + Mutation_strength Update
        offsprings = []
        fitness_mean = 0

        func_eval_num = 0
        for solution, src_idx in zip(solutions, solutions_idx):

            if problem_type == 'knapsack':
                offsprings.append(evolution.Chromosome(solution))
                fitness = offsprings[-1].fitness_calc(problem)
            elif problem_type == 'pole':
                offsprings.append(evolution.ChromosomePole(solution))
                fitness = offsprings[-1].fitness_calc(net, problem, s_len)
                if not solution_found.value:
                    func_eval_num += 1
                if fitness - 2000 > -0.0001:
                    solution_found.value = True
            elif problem_type == 'arm':
                offsprings.append(evolution.ChromosomeKA(solution))
                fitness = offsprings[-1].fitness_calc(*problem)
            else:
                raise ValueError('Problem_type is wrong')

            fitness_mean += fitness
            if src_idx != self.nModels-1:
                samples_count[src_idx] += 1
                mutation_strength[src_idx] += (1/samples_count[src_idx])*(fitness - mutation_strength[src_idx])

        fitness_mean = fitness_mean/solutions_num

        # Sanity check
        if len(offsprings) != solutions_num:
            raise ValueError('offsprings length does not match the number of solutions')

        if solution_found is not None:
            return offsprings, mutation_strength, samples_count, fitness_mean, func_eval_num
        else:
            return offsprings, mutation_strength, samples_count, fitness_mean

    def n_samples(self, ind, nSol):
        return np.ceil(nSol * self.alpha[ind]).astype(int)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.4'
#       jupytext_version: 1.1.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # S_BackForwEwmaSD [<img src="https://www.arpm.co/lab/icons/icon_permalink.png" width=30 height=30 style="display: inline;">](https://www.arpm.co/lab/redirect.php?code=S_BackForwEwmaSD&codeLang=Python)
# For details, see [here](https://www.arpm.co/lab/redirect.php?permalink=eb-estimation-fwd-bwd-exp-smooth).

# ## Prepare the environment

# +
import os
import os.path as path
import sys

sys.path.append(path.abspath('../../functions-legacy'))

from numpy import arange, array, zeros, diff, abs, log, exp, sqrt, linspace
from numpy import sum as npsum

from scipy.io import loadmat

import matplotlib.pyplot as plt
from matplotlib.pyplot import subplots, title
import matplotlib.dates as mdates

plt.style.use('seaborn')

from CONFIG import GLOBAL_DB, TEMPORARY_DB
from ARPM_utils import save_plot, struct_to_dict, date_mtop

# Parameters
tau_HL = 30
lam = log(2) / tau_HL
i_ = 252
# -

# ## Upload database db_Stock SPX

# +
try:
    db = loadmat(os.path.join(GLOBAL_DB, 'db_Stocks'), squeeze_me=True)
except FileNotFoundError:
    db = loadmat(os.path.join(TEMPORARY_DB, 'db_Stocks'), squeeze_me=True)

SPX = struct_to_dict(db['SPX'])
# -

# ## Compute the realized compounded returns

# +
v = SPX.Price_close

x = log(v)
epsi = diff(x, 1).reshape(1,-1)
date = SPX.Date[1:]

t_ = epsi.shape[1]
# -

# ## Compute the backward-forward exponential decay probabilities

edecayprobs = exp(-lam*(abs(arange(-i_, i_ + 1)))).reshape(1,-1)
gamma = npsum(edecayprobs)  # normalization coefficient
edecayprobs = edecayprobs / gamma  # decay factors

# ## Compute the backward/forward exponentially weighted moving standard deviations

y = zeros(t_ - 2 * i_)  # start from time= i_+1 and estimate up to time= t_end -i_  (so that i_ observations are always availabe both backward and forward)
for t in arange(i_,t_-i_):
    ret = epsi[[0],t - i_:t + i_+1]
    y[t - i_] = sqrt(edecayprobs@ret.T ** 2)

# ## Display the compounded returns and the backward/forward exponentially weighted moving standard deviations

# +
date_dt = array([date_mtop(i) for i in date])
myFmt = mdates.DateFormatter('%d-%b-%Y')

f, ax = subplots(2, 1)
date_est = date_dt[i_:t_- i_]
ax[0].plot(date_est, epsi[0,i_:t_ - i_], color='b',lw=1)
ax[0].set_xlim([date_est[0], date_est[-1]])
ax[0].xaxis.set_major_formatter(myFmt)
title('Compounded returns')

date_est = date_dt[i_ :t_- i_]
ax[1].plot(date_est, y, color=[.9, .4, 0], lw = 1.5)
ax[1].set_xlim([date_est[0], date_est[-1]])
ax[1].xaxis.set_major_formatter(myFmt)
title('Estimated Exponentially Weighted Moving St. Deviation')
plt.tight_layout();
# save_plot(ax=plt.gca(), extension='png', scriptname=os.path.basename('.')[:-3], count=plt.get_fignums()[-1])
# -

# ## Display the backward/forward exponential decay probabilities

f, ax = subplots(1, 1)
ax.bar(arange(edecayprobs.shape[1]),edecayprobs[0], facecolor=[.7, .7, .7], edgecolor=[.7, .7, .7])
ax.set_xlim([1, 2 * i_ + 1])
plt.xticks(linspace(1,2*i_+1,3),[-252,0,252])
title('Exponential decay factors profile');
# save_plot(ax=plt.gca(), extension='png', scriptname=os.path.basename('.')[:-3], count=plt.get_fignums()[-1])


<reponame>ADBI-george2/AnomalyDetection
from __future__ import print_function, division
from igraph import *
import numpy as np
import numpy.linalg as la
from scipy.stats import pearsonr
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings("ignore")

__author__ = 'panzer'

FEATURES = ["degree", "clustering_coefficient", "ego_net_edges"]

def say(*lst):
  print(*lst, end="")
  sys.stdout.flush()

def list_files(folder):
  """
  List all files in a folder
  :param folder: Name of the folder
  :return: list of complete file names in folder
  """
  return ["%s/%s"%(folder, f) for f in os.listdir(folder) if f.endswith(".txt")]

def make_graph(file_name):
  """
  Make graph from a file
  :param file_name:
  :return:
  """
  with open(file_name, 'r') as f:
    lines = f.readlines()
    node_count, edge_count = map(int, lines[0].strip().split())
    edges = [map(int, line.strip().split()) for line in lines[1:]]
    graph = Graph()
    graph.add_vertices(node_count)
    graph.add_edges(edges)
  for vertex in graph.vs:
    assign_attributes(vertex, graph)
  return graph

def assign_attributes(vertex, graph):
  """
  Assign Attributes for the vertex
  :param vertex: Vertex to be assigned attributes
  :param graph: Instance of graph to which the vertex belongs
  """
  neighbors = graph.neighbors(vertex.index)
  ego_net = graph.subgraph([vertex.index]+neighbors)
  vertex["degree"] = vertex.degree()
  cc = graph.transitivity_local_undirected([vertex.index])[0]
  vertex["clustering_coefficient"] = 0 if np.isnan(cc) else cc
  vertex["ego_net_edges"] = len(ego_net.es)

def get_feature_vector(graphs, vertex_id, feature):
  return [graph.vs[vertex_id][feature] for graph in graphs]

def pearson_rho(x_vector, y_vector):
  val, _ = pearsonr(x_vector, y_vector)
  return 0 if np.isnan(val) else val

def get_principal_eigen_vector(matrix):
  _, v = la.eig(matrix)
  return v[0]

def construct_correlation_matrix(all_graphs, feature, start, window=7):
  graphs = all_graphs[start:start+window]
  vertices = range(len(graphs[0].vs))
  matrix = []
  for x in vertices:
    x_vector = get_feature_vector(graphs, x, feature)
    covariance_vector = []
    for y in vertices:
      y_vector = get_feature_vector(graphs, y, feature)
      covariance_vector.append(pearson_rho(x_vector, y_vector))
    matrix.append(covariance_vector)
  return matrix

def vector_average(vectors):
  total = vectors[0]
  count = 1
  for vector in vectors[1:]:
    total = total + vector
    count += 1
  return total / count

def construct_correlation_matrices(all_graphs, window=7):
  feature_info = {}
  for feature in FEATURES:
    matrices = []
    eigens = []
    for start in range(len(all_graphs)-window):
      say(".")
      matrix = construct_correlation_matrix(all_graphs, feature, start, window)
      matrices.append(matrix)
      eigens.append(get_principal_eigen_vector(matrix))
    feature_info[feature] = {
      "matrices" : matrices,
      "eigens" : eigens
    }
    print("%s completed"%feature)
  return feature_info

def compute_eigen_behaviour(feature_info, window=7):
  eigen_behaviours = {}
  for feature in FEATURES:
    eigens =  feature_info[feature]["eigens"]
    eigen_behaviour = []
    for start in range(len(eigens)-window):
      u_t = eigens[start+window]
      r_t1 = vector_average(eigens[start:start+window])
      eigen_behaviour.append(round(np.dot(u_t, r_t1).real, 2))
    eigen_behaviours[feature] = eigen_behaviour
  return eigen_behaviours

def save_eigen_behaviours(eigen_behaviours, file_name):
  lines = [" ".join(FEATURES)+"\n"]
  vals = []
  for feature in FEATURES:
    vals.append(eigen_behaviours[feature])
  vals = zip(*vals)
  for line in vals:
    lines.append(" ".join(map(str, line))+"\n")
  with open(file_name, 'w') as f:
    f.writelines(lines)

def plot_eigen_behaviours(eigen_behaviours, file_name, window = 7):
  xs = range(window,len(eigen_behaviours.values()[0])+window)
  colors = ["r", "g", "b"]
  f, axis_arr = plt.subplots(3, sharex=True)
  for i, feature in enumerate(FEATURES):
    ys = eigen_behaviours[feature]
    axis_arr[i].plot(xs, ys, "%s-"%colors[i])
    axis_arr[i].set_ylabel("Z Score")
  plt.xlabel("Time")
  plt.xlim(0, xs[-1]+2)
  plt.savefig(file_name)
  plt.clf()

def _main(folder):
  graphs = []
  for f in list_files(folder):
    graphs.append(make_graph(f))
  print("Graphs Processed")
  feature_info = construct_correlation_matrices(graphs)
  eigen_behaviours = compute_eigen_behaviour(feature_info)
  dataset = folder.split("/")[-1]
  ts_file_name = "%s_time_series.txt"%dataset
  ts_png_name = "%s_time_series.png"%dataset
  save_eigen_behaviours(eigen_behaviours, ts_file_name)
  plot_eigen_behaviours(eigen_behaviours, ts_png_name)

if __name__ == "__main__":
  args = sys.argv
  if len(args) != 2:
    print("USE THE COMMAND : python anomaly.py <data folder>")
    exit()
  folder_name = args[1]
  _main(folder_name)