<gh_stars>1-10
# -*-coding:UTF-8–*-

from scipy import integrate
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation
from web.rcdata import *
import os
import time

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
IMAGE_DIR = os.path.join(BASE_DIR, 'templates/web/images')

class RC:
    # def __init__(self, b, h, rc_type, A_s, steel_type, mode):
    def __init__(self, shape, rc_type, A_s, steel_type, mode):

        # 初始化RC样本
        _fcuk = [15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80]
        _fc = [7.2, 9.6, 11.9, 14.3, 16.7, 19.1, 21.2, 23.1, 25.3, 27.5, 29.7, 31.8, 33.8, 35.9]
        _fy = [270, 300, 360, 435]
        _Es = [2.1 * 1e5, 2.0 * 1e5, 2.0 * 1e5, 2.0 * 1e5]

        self.timestamp = str(round(time.time() * 1000)) # timestamp
        # shape
        self.shape = shape['shape']
        if self.shape == Shape_Type.REC:
            self.b = shape['b']
            self.h = shape['h']
        elif self.shape == Shape_Type.T:
            self.b = shape['b']
            self.h = shape['h']
            self.bf = shape['bf']
            self.hf = shape['hf']
        elif self.shape == Shape_Type.CIR:
            self.h = shape['d_out']
            self.r = shape['d_out'] / 2
        elif self.shape == Shape_Type.RING:
            self.h = shape['d_out']
            self.r_in = shape['d_in'] / 2
            self.r_out = shape['d_out'] / 2
            self.thickness = self.r_out - self.r_in

        self.index = rc_type.value
        self.f_c = _fc[self.index]
        self.f_cuk = _fcuk[self.index]
        self.A_s = A_s
        self.f_y = _fy[steel_type.value]
        self.E_s = _Es[steel_type.value]
        self.mode = mode

        # TODO
        #    修改GZH
        if mode == Mode.GZH:
            # epsilon 10^-6
            _epsilon_0 = [1370, 1470, 1560, 1640, 1720, 1790, 1850, 1920, 1980, 2030, 2080, 2130, 2190, 2240]
            _alpha_a = [2.21, 2.15, 2.09, 2.03, 1.96, 1.9, 1.84, 1.78, 1.71, 1.65, 1.65, 1.65, 1.65]
            _alpha_d = [0.41, 0.74, 1.06, 1.36, 1.65, 1.94, 2.21, 2.48, 2.74, 3.00, 3.25, 3.50, 3.75]
            # epsilon_u_over_0 = [4.2, 3.0, 2.6, 2.3, 2.1, 2.0, 1.9, 1.9, 1.8, 1.8, 1.7, 1.7, 1.7, 1.6]
            self.epsilon_0 = _epsilon_0[self.index] * 1e-6
            self.alpha_a = _alpha_a[self.index]
            self.alpha_d = _alpha_d[self.index]
            # 此处实质上为epsilon_cu 注意和epsilon_u的区别，书P33，此处按GB取
            # self.epsilon_u = 3300
            self.epsilon_u = (0.0033 - (self.f_cuk - 50) * 1e-6) if (
                    0.0033 - (self.f_cuk - 50) * 1e-6 <= 0.0033) else 0.0033

            # 插值求参数
            # self.epsilon_0 = (_epsilon_0[self.index + 1] - _epsilon_0[self.index]) / 5 * (
            #         self.f_c - _fcuk[self.index]) + \
            #                  _epsilon_0[self.index]  # /5
            # self.alpha_a = (_alpha_a[self.index + 1] - _alpha_a[self.index]) / 5 * (
            #         self.f_c - _fcuk[self.index]) + _alpha_a[self.index]
            # self.alpha_d = (_alpha_d[self.index + 1] - _alpha_d[self.index]) / 5 * (
            #         self.f_c - _fcuk[self.index]) + _alpha_d[self.index]

        elif mode == Mode.GB:
            self.n = 2 - 1 / 60 * (self.f_cuk - 50) if 2 - 1 / 60 * (self.f_cuk - 50) <= 2 else 2
            self.epsilon_0 = (0.002 + 0.5 * (self.f_cuk - 50) * 1e-6) if (0.002 + 0.5 * (
                    self.f_cuk - 50) * 1e-6 >= 0.002) else 0.002
            self.epsilon_u = (0.0033 - (self.f_cuk - 50) * 1e-6) if (
                    0.0033 - (self.f_cuk - 50) * 1e-6 <= 0.0033) else 0.0033

        self.xn = self.get_xn()  # 计算中性轴高度
        self.alpha, self.beta = self.equiv_rec()  # 计算等效矩形应力图系数

    def equiv_rec(self):
        #   计算等效应力图形系数 alpha, beta
        xn = self.xn
        k1, err1 = integrate.quad(self.area, 0, xn, args=xn)
        k2, err2 = integrate.quad(self.inertia, 0, xn, args=xn)
        beta = 2 / xn * k2 / k1
        alpha = 1 / (beta * self.f_c * xn) * k1
        ab = (round(alpha, 4), round(beta, 4))
        return ab

    def sigma_epsilon(self, epsilon):
        #   2-22过镇海曲线
        #   epsilon 为10^-6
        if self.mode == Mode.GZH:
            x = epsilon / self.epsilon_0
            if 0 <= x <= 1:
                y = self.alpha_a * x + (3 - 2 * self.alpha_a) * x ** 2 + (self.alpha_a - 2) * x ** 3
            elif x > 1:
                y = x / (self.alpha_d * (x - 1) * (x - 1) + x)
            else:
                # 对epsilon小于0的取0
                y = 0
            return y * self.f_c
        # GB2010公式
        elif self.mode == Mode.GB:
            if epsilon > self.epsilon_0:
                return self.f_c if epsilon < self.epsilon_u else 0
            else:
                # 对epsilon小于0的取0
                return self.f_c * (1 - (1 - epsilon / self.epsilon_0) ** self.n) if epsilon > 0 else 0

    def get_b(self, x):
        if self.shape == Shape_Type.REC:
            return self.b
        elif self.shape == Shape_Type.T:
            return self.bf if x <= self.hf else self.b
        elif self.shape == Shape_Type.CIR:
            return 2 * (2 * self.r * x - x * x) ** 0.5
        elif self.shape == Shape_Type.RING:
            return 2 * ((2 * self.r_out * x - x * x) ** 0.5 - (
                    2 * self.r_in * (x - self.thickness) - (x - self.thickness) ** 2) ** 0.5) if (
                    self.thickness < x < self.h - self.thickness) else 2 * (2 * self.r_out * x - x * x) ** 0.5

    # def axis_force(self, x, xn):
    #     return self.get_b(x) * self.sigma_epsilon(-1 * self.epsilon_u / xn * x + self.epsilon_u)

    def axis_force(self, x, xn, epsilon_c):
        return self.get_b(x) * self.sigma_epsilon(-1 * epsilon_c / xn * x + epsilon_c)

    def get_xn(self):
        tolerance = 1e-10
        low = 0
        high = self.h
        flag = self.A_s * self.f_y
        while low + tolerance < high:
            xn = (low + high) / 2
            # k为轴力
            # k, err = integrate.quad(self.axis_force, 0, xn, args=xn)
            k, err = integrate.quad(self.axis_force, 0, xn, args=(xn, self.epsilon_u))
            if k < flag:
                low = xn
            else:
                high = xn
        return xn

    def area(self, x, xn):
        # sigma(x)
        return self.sigma_epsilon(-1 * self.epsilon_u / xn * x + self.epsilon_u)

    def inertia(self, x, xn):
        # x*sigma(x)
        return x * self.sigma_epsilon(-1 * self.epsilon_u / xn * x + self.epsilon_u)

    def plot(self):
        # plot
        plt.switch_backend('agg')   # NOT USE GUI
        fig = plt.figure()
        ax1 = plt.subplot(2, 1, 1)
        ax2 = plt.subplot(2, 1, 2)
        fig.tight_layout()  # 调整整体空白
        plt.subplots_adjust(wspace=0, hspace=0.3)  # 调整子图间距

        x = np.arange(0, self.h, 0.01)

        # plot epsilon-x
        plt.sca(ax1)
        ax1.set_title('ε-y', fontsize=12, color='black')
        epsilon = -1 * self.epsilon_u / self.xn * x + self.epsilon_u
        plt.plot(x, epsilon, color='blue')

        # plot sigma-x
        plt.sca(ax2)
        ax2.set_title('σ-y', fontsize=12, color='black')
        sigma = np.empty([0, 1])
        for i in x:
            sigma = np.append(sigma, self.area(i, self.xn))
        plt.plot(x, sigma, color='blue')
        plt.plot([0, self.beta * self.xn], [0, 0], color='red', linestyle='--')
        plt.plot([0, self.beta * self.xn], [self.alpha * self.f_c, self.alpha * self.f_c], color='red', linestyle='--')
        plt.plot([self.beta * self.xn, self.beta * self.xn], [0, self.alpha * self.f_c], color='red', linestyle='--')
        plt.plot([0, 0], [0, self.alpha * self.f_c], color='red', linestyle='--')

        IMAGE_NAME = 'plot' + self.timestamp + '.png'  # timestamp
        # print(IMAGE_NAME)
        # print(IMAGE_DIR)
        plt.savefig(IMAGE_DIR + '/' + IMAGE_NAME)
        # plt.show()
        return IMAGE_NAME

    def get_display_xn(self, phi, display_xn):
        tolerance = 1e-3    # set a larger num to accelerate
        low = 0
        high = display_xn   # add limit
        while low + tolerance < high:
            xn = (low + high) / 2
            epsilon_c = phi * xn
            epsilon_s = -(-phi * self.h + epsilon_c)  # positive
            steel_sigma = epsilon_s * self.E_s if epsilon_s * self.E_s < self.f_y else self.f_y  # yield or not
            flag = self.A_s * steel_sigma
            # k为轴力
            # k, err = integrate.quad(self.axis_force, 0, xn, args=xn)
            k, err = integrate.quad(self.axis_force, 0, xn, args=(xn, epsilon_c))
            if k < flag:
                low = xn
            else:
                high = xn
        return xn

    def display_area(self, x, xn, phi):
        # sigma(x)
        epsilon_c = phi * xn
        return self.sigma_epsilon(-1 * epsilon_c / xn * x + epsilon_c)

    def display(self):
        # plot
        plt.switch_backend('agg')   # NOT USE GUI
        fig, ax = plt.subplots()
        # ax1 = plt.subplot(2, 1, 1)
        # ax2 = plt.subplot(2, 1, 2)
        # fig.tight_layout()  # 调整整体空白
        # plt.subplots_adjust(wspace=0, hspace=0.3)  # 调整子图间距
        # x, sigma = [], []
        line, = plt.plot([], [], color='blue')
        limit_frame = int(self.epsilon_u / self.xn * 1e6)
        ax.set_xlim(0, self.h)
        ax.set_ylim(0, 1.2 * self.f_c)

        def init():
            return line,

        def update(step):
            # plt.sca(ax1)
            phi = step * 1e-6
            title = 'σ-y (phi = ' + str(round(phi, 6)) + ')'
            ax.set_title(title, fontsize=12, color='black')

            display_xn = self.display_xn  # initial temp
            if display_xn > self.xn:
                display_xn = self.get_display_xn(phi, display_xn)
            else:
                display_xn = self.xn

            self.display_xn = display_xn
            x = np.arange(0, self.h, 0.01)
            sigma = np.empty([0, 1])

            for i in x:
                sigma = np.append(sigma, self.display_area(i, display_xn, phi))

            # for i in x:
            #     sigma = np.append(sigma, self.area(i, self.xn))

            line.set_data(x, sigma)
            return line, display_xn,

        self.display_xn = self.h
        anime = FuncAnimation(fig, update, frames=range(1, limit_frame), init_func=init, interval=10)
        # plt.show()
        Anime_NAME = 'anime' + self.timestamp + '.gif'  # timestamp
        anime.save(IMAGE_DIR + '/' + Anime_NAME, writer='imagemagick', fps=20)
        return Anime_NAME

def test():
    #   default: C30混凝土
    # shape, rc_type, A_s, steel_type, mode
    # rc_set = RC({'shape': Shape_Type.REC, 'b': 250, 'h': 500}, RC_Type.C30, 1256, Steel_Type.HRB335, Mode.GZH)
    # print(rc_set.alpha, rc_set.beta, '\n')
    # rc_set.plot()

    rc_set2 = RC({'shape': Shape_Type.T, 'b': 250, 'h': 500, 'bf': 600, 'hf': 100}, RC_Type.C40, 2147,
                 Steel_Type.HRB400, Mode.GB)
    print(rc_set2.alpha, rc_set2.beta, '\n')
    rc_set2.plot()
    # rc_set2.display()

    #
    # rc_set3 = RC({'shape': Shape_Type.CIR, 'd_out': 400}, RC_Type.C50, 1212, Steel_Type.HRB335, Mode.GB)
    # print(rc_set3.alpha, rc_set3.beta, '\n')
    # rc_set3.plot()
    #
    # rc_set4 = RC({'shape': Shape_Type.RING, 'd_in': 300, 'd_out': 500}, RC_Type.C30, 1212, Steel_Type.HRB335, Mode.GB)
    # print(rc_set4.alpha, rc_set4.beta, '\n')
    # rc_set4.plot()

    return


<reponame>mmaltafe/Learning
import numpy as np
import pandas as pd
import scipy.stats as stats
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
import matplotlib.pyplot as plt
import pickle
import SODA
import threading
from datetime import datetime
from psutil import cpu_percent, swap_memory
from progress.bar import Bar
from sklearn.neighbors import kneighbors_graph
from scipy.sparse.linalg import expm
import scipy.sparse
from sklearn.metrics.pairwise import euclidean_distances
from sklearn import preprocessing
import sklearn
from sklearn.utils.validation import check_array

class performance(threading.Thread):
    # Declares variables for perfomance analysis:

    def __init__(self):
        threading.Thread.__init__(self)
        self.control = True

    def run(self):
        cpu_p = []
        ram_p = []
        ram_u = []
        while self.control:
            cpu_p.append(cpu_percent(interval=1, percpu=True))
            ram_p.append(swap_memory().percent)
            ram_u.append(swap_memory().used/(1024**3))
        self.mean_cpu_p = np.mean(cpu_p)
        self.mean_ram_p = np.mean(ram_p)
        self.mean_ram_u = np.mean(ram_u)
        self.max_cpu_p = np.max(np.mean(cpu_p, axis=1))
        self.max_ram_p = np.max(ram_p)
        self.max_ram_u = np.max(ram_u)

    def stop(self):
        self.control = False

    def join(self):
        threading.Thread.join(self)
        out = {'mean_cpu_p': self.mean_cpu_p,
               'mean_ram_p': self.mean_ram_p,
               'mean_ram_u': self.mean_ram_u,
               'max_cpu_p': self.max_cpu_p,
               'max_ram_p': self.max_ram_p,
               'max_ram_u': self.max_ram_u}
        return out

def divide(data, n_windows = 100, n_samples = 50):
    """Divide the data in n_samples, and the samples are equaly distributed in n_windows
    -- Input
    - data = data to split
    - n_windows = number of windows to separete the data
    - n_samples = number of samples of the output
    -- Output
    - reduced_data = splited data with n_samples
    - data_sample_id = id of the splited data"""
    L, W = data.shape

    # Checking if the windows can be of the same size

    if int(L % n_windows) != 0:

        # Checking if we need to peak the same amount of data from each window

        if int(n_samples % n_windows) != 0 or (n_samples/n_windows) % 1 != 0:

            lines_per_window = L // n_windows
            samples_per_window = n_samples // n_windows
            reduced_data = np.zeros((int(n_samples),W))
            data_sample_id = np.zeros(int(n_samples))

            for i in range(n_windows):
                if i >= n_windows - 1:
                    for j in range(samples_per_window + int(n_windows*((n_samples/n_windows) % 1))):
                        sample = np.random.randint(i*lines_per_window,int((i+1)*lines_per_window + int(L % n_windows)))
                        new_line = data[sample]
                        reduced_data[j+(i*samples_per_window)] = new_line
                        data_sample_id[j+(i*samples_per_window)] = sample

                else:
                    for j in range(samples_per_window):
                        sample = np.random.randint(i*lines_per_window,(i+1)*lines_per_window)
                        new_line = data[sample]
                        reduced_data[j+(i*samples_per_window)] = new_line
                        data_sample_id[j+(i*samples_per_window)] = sample

        # Even amount of data

        else:

            lines_per_window = L // n_windows
            samples_per_window = n_samples // n_windows
            reduced_data = np.zeros((int(n_samples),W))
            data_sample_id = np.zeros(int(n_samples))

            for i in range(n_windows):
                if i >= n_windows - 1:
                    for j in range(samples_per_window):
                        sample = np.random.randint(i*lines_per_window,int((i+1)*lines_per_window + int(L % n_windows)))
                        new_line = data[sample]
                        reduced_data[j+(i*samples_per_window)] = new_line
                        data_sample_id[j+(i*samples_per_window)] = sample

                else:
                    for j in range(samples_per_window):
                        sample = np.random.randint(i*lines_per_window,(i+1)*lines_per_window)
                        new_line = data[sample]
                        reduced_data[j+(i*samples_per_window)] = new_line
                        data_sample_id[j+(i*samples_per_window)] = sample

    # Windows of same size

    else:

        # Checking if we need to pick the same amount of data from each window

        if int(n_samples % n_windows) != 0 or (n_samples/n_windows) % 1 != 0:

            lines_per_window = L // n_windows
            samples_per_window = n_samples // n_windows
            reduced_data = np.zeros((int(n_samples),W))
            data_sample_id = np.zeros(int(n_samples))

            for i in range(n_windows):
                if i >= n_windows - 1:
                    for j in range(samples_per_window + int(n_windows*((n_samples/n_windows) % 1))):
                        sample = np.random.randint(i*lines_per_window,int((i+1)*lines_per_window + int(L % n_windows)))
                        new_line = data[sample]
                        reduced_data[j+(i*samples_per_window)] = new_line
                        data_sample_id[j+(i*samples_per_window)] = sample

                else:
                    for j in range(samples_per_window):
                        sample = np.random.randint(i*lines_per_window,(i+1)*lines_per_window)
                        new_line = data[sample]
                        reduced_data[j+(i*samples_per_window)] = new_line
                        data_sample_id[j+(i*samples_per_window)] = sample

        # Even amount of data

        else:

            lines_per_window = L // n_windows
            samples_per_window = n_samples // n_windows
            reduced_data = np.zeros((int(n_samples),W))
            data_sample_id = np.zeros(int(n_samples))

            for i in range(n_windows):
                for j in range(samples_per_window):
                    sample = np.random.randint(i*lines_per_window,int((i+1)*lines_per_window))
                    new_line = data[sample]
                    reduced_data[j+(i*samples_per_window)] = new_line
                    data_sample_id[j+(i*samples_per_window)] = sample

    return reduced_data, data_sample_id

def Normalisation(data):
    """Use standart deviation to normalise the data
    -- Input
    - data
    -- Output
    - Normalised data
    """

    # Normalizing whole data
    scaler = StandardScaler().fit(data)
    norm_data = scaler.transform(data)

    return norm_data

def PCA_Analysis(mantained_variation, attributes_influence,laplace=True):
    """Create and save the PCA model
    -- Input
    - mantained_variation = variation mantained for each PC
    - attributes_influence = influence of each attribute on the model
    -- Output
    - Saves plot figures in results folder
    """

    # Plots the variation mantained by each PC

    fig = plt.figure(figsize=[16,8])
    ax = fig.subplots(1,1)
    ax.bar(x=['PC' + str(x) for x in range(1,(len(mantained_variation)+1))],height=mantained_variation)

    ax.set_ylabel('Percentage of Variance Held',fontsize=20)
    ax.set_xlabel('Principal Components',fontsize=20)
    ax.tick_params(axis='x', labelsize=14)
    ax.tick_params(axis='y', labelsize=18)
    ax.grid()

    fig.savefig('results/Percentage_of_Variance_Held.png', bbox_inches='tight')

    sorted_sensors_contribution = attributes_influence.values[:]

    # Ploting Cntribution Attributes influence

    fig = plt.figure(figsize=[25,8])

    fig.suptitle('Attributes Weighted Contribution Percentage', fontsize=16)

    ax = fig.subplots(1,1)

    sorted_sensors_contribution = sorted_sensors_contribution.ravel()
    ax.bar(x=list(attributes_influence.columns),height=sorted_sensors_contribution)
    plt.ylabel('Relevance Percentage',fontsize = 20)
    plt.xlabel('Attributes',fontsize = 20)
    plt.tick_params(axis='x', labelsize=14)
    plt.tick_params(axis='y', labelsize=18)
    plt.xticks(rotation=90)
    ax.grid()

    fig.savefig('results/Attributes_Contribution.png', bbox_inches='tight')

    return

def PCA_Projection(background_train,streaming_data, N_PCs, maintained_features=0):
    """Transform Data with PCA and normalize
    -- Input
    - Offline data
    - Streaming data
    - N_PCs = number of PCs to calculate
    -- Output
    - Projected Offline data
    - Projected Streaming data
    - Variation Mantained
    - Attributes Influence
    """

    # Calcules the PCA and projects the data-set into them

    pca= PCA(n_components = N_PCs)
    pca.fit(background_train)

    # Calculates the total variance maintained by each PCs

    pca_variation = pca.explained_variance_ratio_ * 100

    print('Normal Variation maintained: %.2f' % np.round(pca_variation.sum(), decimals = 2))

    proj_background_train = pca.transform(background_train)
    proj_streaming_data = pca.transform(streaming_data)

    ### Attributes analyses ###

    columns=["px1","py1","pz1","E1","eta1","phi1","pt1",\
                "px2","py2","pz2","E2","eta2","phi2",\
                "pt2","Delta_R","M12","MET","S","C","HT",\
                "A", "Min","Max","Mean","Var","Skw","Kurt",\
                "M2","M3","M4"] #,"Bmin","Bmax"]

    # Gets eigen vectors information from the trained pca object
    eigen_matrix = np.array(pca.components_)

    # Inverting negative signals
    eigen_matrix = pow((pow(eigen_matrix,2)),0.5)

    # Calculates the feature contribution

    for i in range (eigen_matrix.shape[0]):
        LineSum = sum(eigen_matrix[i,:])
        for j in range (eigen_matrix.shape[1]):
            eigen_matrix[i,j] = ((eigen_matrix[i,j]*100)/LineSum)

    weighted_contribution = np.zeros((eigen_matrix.shape[1]))

    for i in range (eigen_matrix.shape[1]):
        NumeratorSum = 0
        for j in range (N_PCs):
            NumeratorSum += eigen_matrix[j,i] * pca_variation[j]

        weighted_contribution[i] = NumeratorSum / sum(pca_variation)

    weighted_contribution = weighted_contribution.reshape((1,-1))

    # Sorting attributes by their contribution values

    attributes_contribution = pd.DataFrame (weighted_contribution, columns = columns)

    attributes_contribution = attributes_contribution.sort_values(by=0, axis=1,ascending=False)

    return proj_background_train, proj_streaming_data, pca_variation, attributes_contribution

def statistics_attributes(data):
    """
    When xyz_attributes=True:
       Concatenate with the data, statistics attributes for each event.
       Currently Applied:
        - Minimum Value
        - Maximum Value
        - Mean
        - Variance
        - Skewness
        - Kurtosis
        - 2nd Central Moment
        - 3rd Central Moment
        - 4th Central Moment
        - Bayesian Confidence Interval (Min and Max)
        More attributes may be added later

       -- Input
       - data [numpy.array]
       -- Output
       - output_data = data with adition of statistical features for each line [numpy.array]
       """

    momentum_data = np.concatenate((data[:, 0:3], data[:, 7:10]), axis=1)

    L, W = data.shape

    _, (min_v, max_v), mean, var, skew, kurt = stats.describe(momentum_data.transpose())

    # Minimum Value
    min_v = min_v.reshape(-1,1)

    min_v = check_array(min_v) # Minimum Value

    # Maximum Value
    max_v = max_v.reshape(-1,1)

    max_v = check_array(max_v) # Maximum Value

    # Mean
    mean = mean.reshape(-1,1)

    mean = check_array(mean) # Mean

    # Variance
    var = var.reshape(-1,1)

    var = check_array(var) # Variance

    # Skewness
    skew = skew.reshape(-1,1)

    skew = check_array(skew) # Skewness

    # Kurtosis
    kurt = kurt.reshape(-1,1)

    kurt = check_array(kurt) # Kurtosis

    # 2nd Central Moment
    moment2 = stats.moment(momentum_data.transpose(), moment=2).reshape(-1,1)

    moment2 = check_array(moment2) # 2nd Central Moment

    # 3rd Central Moment
    moment3 = stats.moment(momentum_data.transpose(), moment=3).reshape(-1,1)

    moment3 = check_array(moment3) # 3rd Central Moment

    # 4th Central Moment
    moment4 = stats.moment(momentum_data.transpose(), moment=4).reshape(-1,1)

    moment4 = check_array(moment4) # 4th Central Moment

    """
    bayes_min = np.zeros(L).reshape(-1,1)

    bayes_min = check_array(momentum_data) # bayes_min

    bayes_max = np.zeros(L).reshape(-1,1)

    bayes_max = check_array(momentum_data) # bayes_max

    for i,d in enumerate(momentum_data):
        bayes = stats.bayes_mvs(d)

        bayes = check_array(momentum_data) # bayes

        bayes_min[i] = bayes[0][1][0]

        bayes_min = check_array(momentum_data) # bayes_min

        bayes_max[i] = bayes[0][1][1]

        bayes_max = check_array(momentum_data) # bayes_max
    """
    output_data = np.concatenate((data, min_v, max_v, mean, var, skew, kurt, moment2, moment3, moment4), axis=1)
                                        #bayes_min, bayes_max
    return output_data

def SODA_Granularity_Iteration(offline_data,streaming_data,gra,target,Iteration):

    print('=> Iteration Number {}:         .Executing granularity:'.format(Iteration),gra)
    # Formmating  Data
    offline_data = np.matrix(offline_data)
    L1 = len(offline_data)

    streaming_data = np.matrix(streaming_data)

    data = np.concatenate((offline_data, streaming_data), axis=0)

    # Dreate data frames to save each iteration result.

    performance_info = pd.DataFrame(np.zeros((1,8)).reshape((1,-1)), columns=['Granularity', 'Time_Elapsed',
                                                                'Mean CPU_Percentage', 'Max CPU_Percentage',
                                                                'Mean RAM_Percentage', 'Max RAM_Percentage',
                                                                'Mean RAM_Usage_GB', 'Max RAM_Usage_GB'])

    begin = datetime.now()

    performance_thread = performance()
    performance_thread.start()

    Input = {'GridSize':gra, 'StaticData':np.vstack((offline_data,streaming_data)), 'DistanceType': 'euclidean','Target':target}

    out = SODA.SelfOrganisedDirectionAwareDataPartitioning(Input,'Offline')

    performance_thread.stop()
    performance_out = performance_thread.join()
    final = datetime.now()
    performance_info.loc[0,'Time_Elapsed'] = (final - begin)
    performance_info.loc[0,'Mean CPU_Percentage'] = performance_out['mean_cpu_p']
    performance_info.loc[0,'Max CPU_Percentage'] = performance_out['max_cpu_p']
    performance_info.loc[0,'Mean RAM_Percentage'] = performance_out['mean_ram_p']
    performance_info.loc[0,'Max RAM_Percentage'] = performance_out['max_ram_p']
    performance_info.loc[0,'Mean RAM_Usage_GB'] = performance_out['mean_ram_u']
    performance_info.loc[0,'Max RAM_Usage_GB'] = performance_out['max_ram_u']

    np.savetxt('results/SODA_labels__'+ str(gra) + '__' + str(Iteration) +'__.csv', out['IDX'],delimiter=',')
    np.savetxt('results/Density_1__'+ str(gra) + '__' + str(Iteration) +'__.csv', out['Density1'],delimiter=',')
    np.savetxt('results/Density_2__'+ str(gra) + '__' + str(Iteration) +'__.csv', out['Density2'],delimiter=',')
    performance_info.to_csv('results/performance_info__' + str(gra) + '__' + str(Iteration) + '__.csv', index=False)


# coding=utf-8
"""Face Detection and Recognition"""
# MIT License
#
# Copyright (c) 2017 <NAME>
#
# This is the work of <NAME> and shanren7 remodelled into a
# high level container. It's an attempt to simplify the use of such
# technology and provide an easy to use facial recognition package.
#
# https://github.com/davidsandberg/facenet
# https://github.com/shanren7/real_time_face_recognition
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import pickle
import os
import math
import cv2
import numpy as np
import tensorflow as tf
from scipy import misc
from sklearn.grid_search import GridSearchCV
from sklearn.svm import SVC
from sklearn.model_selection import ParameterGrid
from sklearn.linear_model import SGDClassifier
from sklearn.metrics import roc_auc_score
from sklearn.neighbors import KNeighborsClassifier
import align.detect_face
import facenet
from tqdm import tqdm
from timeit import timeit
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
import time

gpu_memory_fraction = 0.6
data_dir = '/work/MachineLearning/my_dataset/wos_event'
facenet_model_checkpoint = '/work/MachineLearning/model_checkpoints/20180402-114759/20180402-114759.pb'
#classifier_model = '/work/MachineLearning/model_checkpoints/incremental_test.pkl'
classifier_model = '/work/MachineLearning/model_checkpoints/lighter_model.pkl'

debug = False
sess = None
graph = None
model = SVC(kernel='rbf', probability=True)
labels = None
class_names = None
emb_array = None

class Face(object):
    def __init__(self):
        self.name = None
        self.bounding_box = None
        self.image = None
        self.container_image = None
        self.embedding = None
        self.confidence = None

class Recognition(object):
    def __init__(self):
        self.detect = Detection()
        self.encoder = Encoder()
        self.identifier = Identifier()

    def add_identity(self, image, person_name):
        faces = self.detect.find_faces(image)

        if len(faces) == 1:
            face = faces[0]
            face.name = person_name
            face.embedding = self.encoder.generate_embedding(face)
            return faces

    def identify(self, image):
        faces = self.detect.find_faces(image)
        for i, face in enumerate(faces):
            if debug:
                cv2.imshow("Face: " + str(i), face.image)
            face.embedding = self.encoder.generate_embedding(face)
            face.name = self.identifier.identify(face)
            #face.confidence = 1.0
            face.confidence = self.identifier.confidence_level(face)
        return faces

    # def identify(self, image):
    #     faces = self.detect.find_faces(image)

    #     if len(faces) > 0:
    #         faces = faces[0]

    #     faces.embedding = self.encoder.generate_embedding(faces)
    #     faces.name = self.identifier.identify(faces)
    #     faces.confidence = self.identifier.confidence_level(faces)
    #     return faces

class Identifier(object):
    def __init__(self):
        with open(classifier_model, 'rb') as infile:
            self.emb_array, self.labels, self.class_names = pickle.load(infile, encoding='latin1')
        self.build_model()

    def build_model(self):
        global labels
        global emb_array
        global class_names

        emb_array = self.emb_array
        labels = self.labels
        class_names = self.class_names
        #emb_array = np.concatenate((emb_array, self.emb_array))
        #labels = np.concatenate((labels , self.labels.tolist())).tolist()     #add new labels to old labels
        #class_names = np.concatenate((class_names , self.class_names))                          #append new class name to old class names

        #X_train, X_test, y_train, y_test = train_test_split(emb_array, labels, test_size=0.2)
        #self.model = SGDClassifier(loss='log', verbose=True, n_jobs=-1, n_iter=1000, alpha=1e-5,
        #    tol=None, shuffle=True, random_state=100, penalty='l2')
       #self.model = KNeighborsClassifier(n_neighbors=1, algorithm='auto')
        print('Start building model')
        start = time.time()
        #param_grid = {'C': [1e3, 5e3, 1e4, 5e4, 1e5],
        #      'gamma': [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.1], }
        #self.model = GridSearchCV(SVC(kernel='rbf', cache_size=2048, probability=True), param_grid, n_jobs=-1)
        model.fit(emb_array, labels)
        #self.model.fit(X_train, y_train)
        end = time.time()
        print ("Fit Time: {0:4f}s".format(end - start))
        print('Build model done')

    def identify(self, face):
        global model
        global class_names
        if face.embedding is not None:
            predictions = model.predict_proba([face.embedding])
            best_class_indices = np.argmax(predictions, axis=1)
            best_class_probabilities = predictions[np.arange(len(best_class_indices)), best_class_indices]
            return class_names[best_class_indices[0]]

    def confidence_level(self, face):
        global model
        if face.embedding is not None:
            predictions = model.predict_proba([face.embedding])
            best_class_indices = np.argmax(predictions, axis=1)
            best_class_probabilities = predictions[np.arange(len(best_class_indices)), best_class_indices]
            return best_class_probabilities[0]

class Encoder(object):
    def __init__(self):
        self.graph = tf.Graph()
        self.sess = tf.Session(graph=self.graph)
        with self.graph.as_default():
            with self.sess.as_default():
                facenet.load_model(facenet_model_checkpoint)
                print("Loaded model")

    def generate_embedding(self, face):
        # Get input and output tensors
        images_placeholder = self.graph.get_tensor_by_name("input:0")
        embeddings = self.graph.get_tensor_by_name("embeddings:0")
        phase_train_placeholder = self.graph.get_tensor_by_name("phase_train:0")

        prewhiten_face = facenet.prewhiten(face.image)

        # Run forward pass to calculate embeddings
        feed_dict = {images_placeholder: [prewhiten_face], phase_train_placeholder: False}
        return self.sess.run(embeddings, feed_dict=feed_dict)[0]

    def incremental_training(self, emb_array_new, labels_new, class_names_new):
        global labels
        global emb_array
        global class_names
        emb_array = np.concatenate((emb_array, emb_array_new))
        labels = np.concatenate((labels ,labels_new))     #add new labels to old labels
        class_names = np.concatenate((class_names ,class_names_new))                          #append new class name to old class names

    def retrain_model(self, incremental):
        global labels
        global emb_array
        global class_names
        if incremental is True:
            dataset, append_index = facenet.append_dataset(data_dir, class_names)
            paths, self.append_labels = facenet.get_image_paths_and_labels(dataset, append_index)
            self.append_class_names = [cls.name.replace('_', ' ') for cls in dataset]

        else:
            dataset, append_index = facenet.get_dataset(data_dir)
            paths, labels = facenet.get_image_paths_and_labels(dataset, append_index)
            class_names = [cls.name.replace('_', ' ') for cls in dataset]

        np.random.seed(seed=666)
        # Check that there are at least one training image per class
        for cls in dataset:
            assert (len(cls.image_paths) > 0, 'There must be at least one image for each class in the dataset')

        # Create a list of class names

        print('Number of classes: %d' % len(dataset))
        print('Number of images: %d' % len(paths))
        if incremental is True:
            print("new people added: ")
            print(self.append_class_names)

        # Get input and output tensors
        images_placeholder = self.graph.get_tensor_by_name("input:0")
        embeddings = self.graph.get_tensor_by_name("embeddings:0")
        phase_train_placeholder = self.graph.get_tensor_by_name("phase_train:0")
        embedding_size = embeddings.get_shape()[1]

        # Run forward pass to calculate embeddings
        print('Calculating features for images')
        nrof_images = len(paths)
        nrof_batches_per_epoch = int(math.ceil(1.0 * nrof_images / 90))
        self.append_emb_array = np.zeros((nrof_images, embedding_size))

        for i in tqdm(range(nrof_batches_per_epoch)):
            start_index = i * 90
            end_index = min((i + 1) * 90, nrof_images)
            paths_batch = paths[start_index:end_index]
            images = facenet.load_data(paths_batch, False, False, 160)
            feed_dict = {images_placeholder: images, phase_train_placeholder: False}
            self.append_emb_array[start_index:end_index, :] = self.sess.run(embeddings, feed_dict=feed_dict)

        classifier_filename_exp = os.path.expanduser(classifier_model)
        print('Training classifier')

        if incremental is True:
            self.incremental_training(self.append_emb_array, self.append_labels, self.append_class_names)

        print(len(emb_array))
        print(class_names)
        #X_train, X_test, y_train, y_test = train_test_split(emb_array, labels, test_size=0.25)
        #self.model = SGDClassifier(loss='log', verbose=True, n_jobs=-1, n_iter=1000, alpha=1e-5,
        #    tol=None, shuffle=True, random_state=100, penalty='l2')
        #self.model = SVC(kernel='rbf', probability=True, verbose=True, cache_size=1024)
        #self.model = KNeighborsClassifier(n_neighbors=1, algorithm='auto')
        print('Start building model')
        start = time.time()
        #param_grid = {'C': [1e3, 5e3, 1e4, 5e4, 1e5],
        #      'gamma': [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.1], }
        #self.model = GridSearchCV(SVC(kernel='rbf', cache_size=2048, probability=True), param_grid, n_jobs=-1)
        model.fit(emb_array, labels)
        #self.model.fit(X_train, y_train)
        end = time.time()
        print ("Fit Time: {0:4f}s".format(end - start))
        print('Build model done')

        if incremental is False:
            # Saving classifier model
            with open(classifier_filename_exp, 'wb') as outfile:
                pickle.dump((emb_array, labels, class_names), outfile)
            print('Saved classifier model to file "%s"' % classifier_filename_exp)

        return 'Success'

class Detection(object):
    # face detection parameters
    minsize = 50  # minimum size of face
    threshold = [0.6, 0.7, 0.7]  # three steps's threshold
    factor = 0.709  # scale factor

    def __init__(self, face_crop_size=160, face_crop_margin=32):
        self.pnet, self.rnet, self.onet = self._setup_mtcnn()
        self.face_crop_size = face_crop_size
        self.face_crop_margin = face_crop_margin

    def _setup_mtcnn(self):
        with tf.Graph().as_default():
            gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=gpu_memory_fraction)
            sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False))
            with sess.as_default():
                return align.detect_face.create_mtcnn(sess, None)

    def find_faces(self, image):
        faces = []

        bounding_boxes, _ = align.detect_face.detect_face(image, self.minsize,
                                                          self.pnet, self.rnet, self.onet,
                                                          self.threshold, self.factor)
        for bb in bounding_boxes:
            face = Face()
            face.container_image = image
            face.bounding_box = np.zeros(4, dtype=np.int32)

            img_size = np.asarray(image.shape)[0:2]
            face.bounding_box[0] = np.maximum(bb[0] - self.face_crop_margin / 2, 0)
            face.bounding_box[1] = np.maximum(bb[1] - self.face_crop_margin / 2, 0)
            face.bounding_box[2] = np.minimum(bb[2] + self.face_crop_margin / 2, img_size[1])
            face.bounding_box[3] = np.minimum(bb[3] + self.face_crop_margin / 2, img_size[0])
            cropped = image[face.bounding_box[1]:face.bounding_box[3], face.bounding_box[0]:face.bounding_box[2], :]
            face.image = misc.imresize(cropped, (self.face_crop_size, self.face_crop_size), interp='bilinear')

            faces.append(face)

        return faces


#CalcHilbertTrans.py
#This script opens up relevant nodes and drops in a hilbert transform for the signal.
#<NAME>, 23 Feb 2012
from MDSplus import *
from scipy import *
from scipy.signal import *
import sys

s=int(sys.argv[1])

tree=Tree('magnetics',s)

def calcHilb(parentNode) :
	y=parentNode.getData().data()
	yh=hilbert(y)
	timebase_expr='dim_of('+parentNode.getLocalPath()+')'  #Grab timebase with TDI expression so it's local - don't move over network.
	expr=Data.compile('BUILD_SIGNAL($1, *, '+timebase_expr+')', yh) #Build a TDI expression for storing signal.
	parentNode.getNode('hilbert').putData(expr)

calcHilb(tree.getNode('shoelace:ant_i'))
calcHilb(tree.getNode('shoelace:ant_v'))


<reponame>Gabriel-p/LinearBayes3<gh_stars>0

import sys
from astropy.io import ascii
import numpy as np
from scipy.optimize import differential_evolution as DE
import emcee
import matplotlib.pyplot as plt
import warnings

def main():
    """
    """

    file_name = "rgb6.poor.txt"
    xy_cols = "RA(deg)", "Dec(deg)"

    # Read data
    x, ex, y, ey = readData(file_name, xy_cols)
    # x, ex, y, ey = synthData()

    xy_data = np.array([x, y, ex, ey]).T

    # Some priors: slope, intercept, intrinsic scatter, outlier mean,
    # outlier scatter (standard deviation of outliers), outlier fraction
    priors = [0, 400, -400., 0., 0.001, 100., -10., 10., 0.001, 1000.]
    # priors = [-10., 10., -10., 10., 0.001, 100., -10., 10., 0.001, 1000.]

    # Make the fit
    samples, point_estim = fit_data(xy_data, priors)

    makePlots(x, ex, y, ey, samples, point_estim)

def readData(file_name, xy_cols):
    """
    """

    data = ascii.read(file_name)
    x, y = data[xy_cols[0]], data[xy_cols[1]]
    N = len(x)

    ex, ey = np.zeros(N), np.zeros(N)

    return x, ex, y, ey

def synthData():
    """
    """
    m, b = np.random.uniform(-10., 10., 2)
    print("True values: m={:.3f}, b={:.3f}".format(m, b))

    # Generate some synthetic data from the model.
    N = np.random.randint(10, 500)
    x = np.sort(10 * np.random.rand(N))
    ex = 0.1 + 0.5 * np.random.rand(N)

    y = m * x + b
    ey = 0.2 * (y.max() - y.min()) * np.random.rand(N)
    y += ey * np.random.randn(N)
    ey = ey * .5

    return x, ex, y, ey

def fit_data(
    data,
        priorlimits=[-10., 10., -10., 10., 0.001, 100., -10., 10., 0.001,
                     1000.], nwalkers=20, nsteps=5000, burn_frac=.25):
    """
    This code will fit a straight line with intrinsic dispersion to data with
    (optionally) covariant errors on both the independent and dependent
    variables, and takes care of outliers using a mixture model approach.

    The free parameters in the model are:

    * slope: slope of the fitted line.

    * intercept: intercept of the fitted line.

    * intrinsic scatter ('sigma_intrinsic'): Hogg, Bovy, Lang (2010):
      "intrinsic Gaussian variance V, orthogonal to the line."

    * outlier mean ('y_outlier'): mean of outliers.

    * outlier scatter ('sigma_outlier'): standard deviation of outliers.

    * outlier fraction ('outlier_fraction'): fraction of ouliers in data.
      Hogg, <NAME> (2010): "the probability that a data point is bad (or,
      more properly, the amplitude of the bad-data distribution function in the
      mixture)."

    Parameters
    ----------

    data : np.ndarray
        Should have shape (N,4) (if no covariances on errors) or (N,5) (if
        covariant errors). Should be in the order (x, y, dx, dy) or
        (x, y, dx, dy, dxy).

    priorlimits : np.ndarray
        Upper and lower values for each of the model parameters (except the
        outlier fraction which has a flat prior between 0 and 1). The limits
        should be provided in the order:
        [slope, intercept, intrinsic scatter, outlier mean, outlier scatter]
        (so that the array has 10 elements).

    nwalkers : int
        The number of emcee walkers to use in the fit.

    nsteps : int
        The number of steps each walker should take in the MCMC.

    burn_frac : 0 < float < 1
        The fraction of initial emcee walkers to discard as burn-in.

    Returns
    -------

    samples : np.array
        All the chains (flattened) minus the burn-in period.

    point_estim : np.array
        (16th, 50th, 84th) percentile for each of the 6 fitted parameters in
        the order: (slope, intercept, intrinsic scatter, outlier mean,
        outlier deviation, outlier fraction).

    """
    # Unpack and check data.
    if data.shape[1] == 4:
        # No correlations on the errors
        x, y, dx, dy = data.T
        dxy = np.zeros_like(x)
    elif data.shape[1] == 5:
        # Data with dxy correlations
        x, y, dx, dy, dxy = data.T
    else:
        raise ValueError("'data' must have 4 or 5 columns, not {}. \
                Try transposing your data.".format(data.shape[1]))

    # Supress RuntimeWarning
    warnings.filterwarnings("ignore")

    # The number of dimensions is fixed.
    ndim = 6
    # Add outlier fraction prior limits, also fixed.
    priorlimits = priorlimits + [0., 1.]

    print("Running optimization...")

    # Estimate initial values using DE algorithm.
    def minfunc(model):
        return -full_posterior(model, x, y, dx, dy, dxy, priorlimits)
    bmin, bmax = priorlimits[0::2], priorlimits[1::2]
    bounds = list(zip(*[bmin, bmax]))
    pstart = DE(minfunc, bounds, maxiter=5000).x
    print("Initial guesses: "
          "({:.3f}, {:.3f}, {:.3f}, {:.3f}, {:.3f}, {:.3f})".format(*pstart))

    # Sample ball around the max posterior point.
    p0 = emcee.utils.sample_ball(
        pstart, 0.01 * np.ones_like(pstart), size=nwalkers)
    # Make sure there are no negative outlier fractions
    p0[:, -1] = np.abs(p0[:, -1])

    print("Running emcee...")
    sampler = emcee.EnsembleSampler(
        nwalkers, ndim, full_posterior, args=[x, y, dx, dy, dxy, priorlimits])
    for i, result in enumerate(sampler.sample(p0, iterations=nsteps)):
        updt(nsteps, i)

    # Remove burn-in and flatten all chains.
    nburn = int(burn_frac * nsteps)
    samples = sampler.chain[:, nburn:, :].reshape((-1, ndim))

    # Shape: (6, 3)
    point_estim = np.percentile(samples, [16, 50, 84], axis=0).T

    return samples, point_estim

def full_posterior(params, x, y, dx, dy, dxy, priorlimits):
    """
    The log-posterior of the data given the full mixture model of the linear
    function and the outlier distribution.

    Parameters
    ----------

    params : np.ndarray or list
        [slope,intercept, intrinsic scatter, outlier mean,
            outlier standard deviation, outlier fraction]

    Returns
    -------

    float
        The posterior of the parameters given the data.
    """
    if log_priors(params, priorlimits) == -np.inf:
        return -np.inf
    else:
        return log_priors(params, priorlimits) +\
            full_log_likelihood(params, x, y, dx, dy, dxy)

def log_priors(params, priorlimits):
    """
    Prior probabilities on the parameters, given upper and lower limits on each
    parameter. Jeffreys priors are used for the intrinsic and outlier standard
    deviations, and a prior that is flat in Arctan(slope) is used for the
    slope. For everything else, priors are uniform within the given limits.

    Parameters
    ----------

    params : np.ndarray or list
        [slope, intercept, intrinsic scatter, outlier mean, outlier standard
         deviation, outlier fraction]

    Returns
    -------

    float
        The prior density of these parameters.
    """
    m, b, sigma_intrinsic, y_outlier, sigma_outlier, outlier_fraction = params
    mlo, mhi, blo, bhi, silo, sihi, yolo, yohi, solo, sohi, oflo, ofhi =\
        priorlimits

    if m < mlo or m > mhi or b < blo or b > bhi or sigma_intrinsic < silo or\
        sigma_intrinsic > sihi or sigma_outlier < solo or sigma_outlier > sohi\
        or y_outlier < yolo or y_outlier > yohi or outlier_fraction < oflo or\
            outlier_fraction > ofhi:
        return -np.inf
    else:
        return -np.log(1. + m * m) - np.log(sigma_intrinsic) -\
            np.log(sigma_outlier)

def full_log_likelihood(params, x, y, dx, dy, dxy):
    """
    The log-likelihood of the data given the full mixture model of the linear
    function and the outlier distribution.

    This is basically E1. (17) in <NAME> (2010), accounting for the
    intrinsic scatter term in 'likelihood_line()'.

    Returns
    -------

    float
        The likelihood of the data given this set of model parameters.
    """
    m, b, sigma_intrinsic, y_outlier, sigma_outlier, outlier_fraction = params

    lkl_line = likelihood_line([m, b, sigma_intrinsic], x, y, dx, dy, dxy)
    out_dist = outlier_distribution(
        [y_outlier, sigma_outlier], x, y, dx, dy, dxy)

    return np.sum(np.log(
        (1. - outlier_fraction) * lkl_line + outlier_fraction * out_dist))

def likelihood_line(params, x, y, dx, dy, dxy):
    """
    Likelihood for the linear function.

    Returns
    -------

    float
        The likelihood of the data given this set of model parameters.
    """
    m, b, sigma_intrinsic = params
    theta = np.arctan(m)

    sint, cost = np.sin(theta), np.cos(theta)

    # Perpendicular distance to the line
    delta = -sint * x + cost * y - cost * b

    # Projection of covariance matrix along line
    Sigma_dd = sint**2. * dx**2. - np.sin(2. * theta) * dxy + cost**2. * dy**2.

    lkl_line = (2. * np.pi * (Sigma_dd + sigma_intrinsic**2.))**-.5 *\
        np.exp(-delta**2. / (2. * (Sigma_dd + sigma_intrinsic**2.)))

    return lkl_line

def outlier_distribution(params, x, y, dx, dy, dxy):
    """
    The likelihood for the outlier distribution, which is modeled as a uniform
    distribution in x and a Gaussian distribution in y with some mean y0 and
    standard deviation sigma0.

    Returns
    -------

    float
        The likelihood of the data given this set of model parameters.
    """
    y_outlier, sigma_outlier = params
    sigma_total2 = sigma_outlier**2. + dy**2.

    out_dist = (2. * np.pi * sigma_total2)**-0.5 *\
        np.exp(-.5 * (y - y_outlier)**2. / sigma_total2)

    return out_dist

def updt(total, progress, extra=""):
    """
    Displays or updates a console progress bar.

    Original source: https://stackoverflow.com/a/15860757/1391441
    """
    barLength, status = 20, ""
    progress = float(progress) / float(total)
    if progress >= 1.:
        progress, status = 1, "\r\n"
    block = int(round(barLength * progress))
    text = "\r[{}] {:.0f}% {}{}".format(
        "#" * block + "-" * (barLength - block),
        round(progress * 100, 0), extra, status)
    sys.stdout.write(text)
    sys.stdout.flush()

def makePlots(x, ex, y, ey, samples, point_estim, m=None, b=None):
    """
    """
    try:
        import corner
        fig = plt.figure(figsize=(10, 10))
        corner.corner(samples)
        fig.tight_layout()
        plt.savefig("corner.png", dpi=150)
    except ModuleNotFoundError:
        print("No corner module")

    fig = plt.figure(figsize=(10, 10))
    ax = plt.subplot(111)
    ax.minorticks_on()
    ax.grid(b=True, which='both', color='gray', linestyle='--', lw=.5)

    if m is not None:
        txt = r"$True:\;m={:.3f},\,b={:.3f}$".format(m, b)
    else:
        txt = ''
    plt.errorbar(x, y, xerr=ex, yerr=ey, fmt='o', label=txt)
    xmin, xmax = ax.get_xlim()
    ymin, ymax = ax.get_ylim()

    mlo, mmed, mhi = point_estim[0]
    # mlo, mhi = mmed - mlo, mhi - mmed
    blo, bmed, bhi = point_estim[1]
    # blo, bhi = bmed - blo, bhi - bmed
    txt = r"$Estim:\;m={:.3f}_{{{:.3f}}}^{{{:.3f}}}$".format(mmed, mlo, mhi)
    txt += r"$,\;b={:.3f}_{{{:.3f}}}^{{{:.3f}}}$".format(bmed, blo, bhi)
    print('\n' + txt)

    x0 = np.linspace(min(x), max(x), 10)
    plt.plot(x0, np.poly1d((mmed, bmed))(x0), label=txt, zorder=4)

    plt.xlim(xmin, xmax)
    plt.ylim(ymin, ymax)

    plt.legend()
    fig.tight_layout()
    plt.savefig("final_fit.png", dpi=150)

if __name__ == '__main__':
    main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Apr 26 13:35:57 2018

@author: cham
"""

# %%
#%pylab qt5
import numpy as np
from ruby import get_isochrone_grid, IsoGrid
# from ruby import isoc_interp, ezinterp
# from ezpadova.parsec import get_one_isochrone, get_photometry_list
from astropy.table import Table

from ruby.isochrone_interp import ezinterp
from ruby.imf import salpeter

from ezpadova import parsec

# %%
""" 1. define grid """
grid_feh = np.arange(-4., 1.2, 0.1)  # 4
grid_logt = np.arange(6., 10.5, 0.1)  # x2

# ig12 = get_isochrone_grid(
#        grid_feh, grid_logt, model='parsec12s',
#        phot='gaiaDR2',Zsun=0.0152, n_jobs=20, verbose=10)
# ig12.unify(model='parsec12s')

""" 2. download isochrones """
ig_gaiaDR2 = get_isochrone_grid(
    grid_feh, grid_logt, model='parsec12s_r14',
    phot='gaiaDR2', Zsun=0.0152, n_jobs=100, verbose=10)
ig_gaiaDR2.Zsun

# %%
from ruby.isochrone_grid import dump_ig, load_ig

dump_ig(ig_gaiaDR2,
        "/media/cham/Seagate Expansion Drive/parsec/grid/gaiaDR2.dump")
# %%
from ruby.isochrone_grid import dump_ig, load_ig

ig_gaiaDR2 = load_ig(
    "/media/cham/Seagate Expansion Drive/parsec/grid/gaiaDR2.dump")
# %%
import copy

ig = copy.copy(ig_gaiaDR2)

""" modify columns """
from astropy.table import Column

for i in range(ig.niso):
    ig.data[i].add_column(
        Column(np.log10(ig.data[i]["Zini"] / ig.Zsun), "feh_ini"))
    # ig.data[i].remove_column("feh_ini")
    ig.data[i].add_column(Column(np.log10(ig.data[i]["Z"] / ig.Zsun), "feh"))
    # ig.data[i].remove_column("feh")
    ig.data[i].add_column(Column(np.log10(ig.data[i]["Age"]), "logt"))
    # ig.data[i].remove_column("logt")

# %%
# ig.unify(model='parsec12s_r14')

""" 3. select subset of isochrone [columns & rows]"""
# ig.sub_rows(cond=(('label', (0, 8)), ('Mini', (0.0, 8.0)), ('logTe', (3.6, 4.1)), ('logg', (2.0, 5.0))))
ig.sub_rows(cond=(('label', (0, 8)), ('Mini', (0.0, 8.0))))
colnames = ['feh_ini', 'logt', 'Mini', 'Mass', 'logL', 'logTe', 'logg',
            'label', 'Mloss', 'feh', 'mbolmag', 'Gmag', 'G_BPmag', 'G_RPmag']
ig.sub_cols(colnames=colnames)

""" 4. interpolate along Mini, logTe, logg, etc."""
for i in range(ig.niso):
    ig.data[i] = ezinterp(ig[i],
                          restrictions=(
                          ('Mini', 0.02), ('logTe', 0.02), ('logg', 0.05)),
                          mode='linear', Mini='Mini')

""" 5. define volume weight """
ig.set_dt(0.2)
ig.set_dfeh(dfeh=0.5)
ig.set_dm()
ig.set_imf(salpeter)

""" 6. calculate total weight for each point """
from astropy.table import Column

for i in range(ig.niso):
    w = ig.data[i]["dm"] * ig.data[i]["dt"] * ig.data[i]["dfeh"] * ig.data[i][
        "imf"]
    try:
        ig.data[i].add_column(Column(w, "w"))
    except ValueError:
        ig.data[i]["w"] = w

    ig.data[i].add_column(Column(10. ** ig.data[i]["logTe"], "teff"))

""" 7. combine all isochrones """
from astropy import table

combined_iso = table.vstack(list(ig.data))

# %%
# from joblib import dump, load
# dump(ig, "/media/cham/Seagate Expansion Drive/parsec/grid/gaiaDR2.dump")

# %%
""" 1. The prior from model """
rcParams.update({"font.size": 20})
H, xe, ye = np.histogram2d(
    combined_iso["logTe"], combined_iso["logg"],
    bins=(np.arange(3., 5, .005), np.arange(-1., 6., 0.05)), normed=False,
    weights=combined_iso["w"])

fig = figure(figsize=(10, 8))
imshow(np.fliplr(np.log10(H.T)), cmap=cm.jet,
       extent=(5.005, 2.995, 6.025, -1.025), aspect="auto")
colorbar()
xlim(4.5, 3.3)
ylim(5.5, -1)

title("log10(Prior)")
xlabel("$T_{\\rm eff}$ [K]")
ylabel("$\\log{g}$ [dex]")
fig.tight_layout()
fig.savefig("/home/cham/projects/gaia/figs/bayesian/log10_prior.pdf")
fig.savefig("/home/cham/projects/gaia/figs/bayesian/log10_prior.svg")

# %%
from ruby import IsoGrid

x_sun = IsoGrid.predict_from_chi2(
    combined_iso,
    var_colnames=["teff", "logg", "feh_ini"],
    tlf=np.array(np.array([5750, 4.35, 0.0])),
    tlf_err=np.array([100., 0.1, 0.1]),
    return_colnames=("Mini", "logt", "feh_ini"),
    q=(0.16, 0.50, 0.84))

tlf = np.array([5750, 4.35, 0.0])  # solar value
tlf_err = np.array([100., 0.1, 0.1])

var_colnames = ["teff", "logg", "feh_ini"]
# 1. convert isochrone(table) into array
sub_iso = np.array(combined_iso[var_colnames].to_pandas())

# 2. calculate chi2
chi2_values = 0
for i_var in range(len(var_colnames)):
    chi2_values += ((sub_iso[:, i_var] - tlf[i_var]) / tlf_err[
        i_var]) ** 2.
chi2_values *= -0.5

# 3. chi2 --> PDF
p_post = np.exp(chi2_values) * combined_iso["w"]

figure()
plt.hist(10 ** combined_iso["logTe"], weights=p_post,
         bins=np.arange(5000, 6000, 30), normed=True)

figure()
plt.hist(combined_iso["logg"], weights=p_post, bins=np.arange(4, 5, 0.02),
         normed=True)

figure()
plt.hist(combined_iso["feh_ini"], weights=p_post, bins=np.arange(-1, 1, 0.1),
         normed=True)

figure()
plt.hist2d(10 ** combined_iso["logTe"], combined_iso["logg"],
           bins=(np.arange(5200, 6250, 25), np.arange(4, 5, 0.05)),
           weights=p_post, normed=True, cmap=cm.gray_r)
colorbar()

# %%
# H, xe, ye = np.histogram2d(
#        10.**combined_iso["logTe"], combined_iso["logg"],
#        bins=(np.arange(3500., 10000, 50.), np.arange(-1., 6., 0.05)), normed=False,
#        weights=combined_iso["w"])
#
# import seaborn as sns
# Y = np.vstack((10.**combined_iso["logTe"], combined_iso["logg"])).T
# ax = sns.kdeplot(Y, shade = True, cmap = "PuBu")
#
#
# figure()
# imshow(np.fliplr(np.log10(H.T)), cmap=cm.jet, extent=(10000+25, 3500-25, 6.025, -1.025), aspect="auto")
# colorbar()

# %%
try:
    combined_iso.add_column(Column(10. ** combined_iso["logTe"], "teff"))

# %%

test_tlf = np.array([5750, 4.35, 0.0])  # solar value
err_tlf = np.array([100., 0.1, 0.1])

test_tlf = np.array([5500, 2.5, 0.0])
err_tlf = np.array([100., 0.1, 0.1])

test_tlf = np.array([10. ** 3.68, 2.44, 0.0])
err_tlf = np.array([100., 0.1, 0.1])

def chi2(x, x0, err):
    return -0.5 * ((x - x0) / err) ** 2.

# %%timeit

grid_logt = ig.grid_logt
grid_feh = ig.grid_feh
# grid_mini = np.arange(0.3, 5.1, 0.1)

grid_logt = ig.grid_logt
# grid_feh = np.arange(-2.0, 0.8, 0.8)
grid_feh = np.array([-2.0, -1.0, 0., 0.4])
grid_mini = np.logspace(-1, 1, 30)

mesh_logt, mesh_feh, mesh_mini = np.meshgrid(grid_logt, grid_feh, grid_mini)

""" ind: 1:feh, 2:logt, 3:mini """
mesh_label = np.zeros_like(mesh_logt)
# basics, mini, logt, feh
mesh_logt_est = np.zeros_like(mesh_logt)
mesh_logt_err = np.zeros_like(mesh_logt)

mesh_feh_est = np.zeros_like(mesh_logt)
mesh_feh_err = np.zeros_like(mesh_logt)

mesh_mini_est = np.zeros_like(mesh_logt)
mesh_mini_err = np.zeros_like(mesh_logt)

# spectroscopic, teff, logg
mesh_teff = np.zeros_like(mesh_logt)
mesh_teff_est = np.zeros_like(mesh_logt)
mesh_teff_err = np.zeros_like(mesh_logt)

mesh_logg = np.zeros_like(mesh_logt)
mesh_logg_est = np.zeros_like(mesh_logt)
mesh_logg_err = np.zeros_like(mesh_logt)

n_all = np.prod(mesh_logt.shape)
ijk = []
for i in range(mesh_logt.shape[0]):
    for j in range(mesh_logt.shape[1]):
        for k in range(mesh_logt.shape[2]):
            o_interp = ig.interp_mini(
                mesh_logt[i, j, k], mesh_feh[i, j, k], mesh_mini[i, j, k],
                return_colnames=(
                'Mini', 'logt', "feh_ini", 'teff', 'logg', "label"))
            mesh_teff[i, j, k] = o_interp[3]
            mesh_logg[i, j, k] = o_interp[4]
            ijk.append((i, j, k))
            mesh_label[i, j, k] = o_interp[5]
print(len(ijk))

from ipyparallel import Client

rc = Client(profile="default")
dv = rc.direct_view()
dv.push({"combined_iso": combined_iso}).get()
dv.push({"mesh_teff": mesh_teff}).get()
dv.push({"mesh_logg": mesh_logg}).get()
dv.push({"mesh_feh": mesh_feh}).get()
dv.push({"flat_mini": mesh_mini.flatten()}).get()
dv.push({"flat_logt": mesh_logt.flatten()}).get()

dv.execute("import numpy as np").get()
dv.execute("from ruby.isochrone_grid import IsoGrid").get()

cmd = """
x_all = np.zeros((len(ijk), 3, 3))
for i, _ijk in enumerate(ijk):
    _i, _j, _k = _ijk

    if mesh_teff[_i, _j, _k]>0:
        try:
            x_all[i] = IsoGrid.predict_from_chi2(
                combined_iso,
                var_colnames=["teff", "logg", "feh_ini"],
                tlf=np.array([mesh_teff[_i, _j, _k], mesh_logg[_i, _j, _k], mesh_feh[_i, _j, _k]]),
                tlf_err=np.array([100., 0.1, 0.1]),
                return_colnames=("Mini", "logt", "feh_ini"),
                q=(0.16, 0.50, 0.84))
        except ValueError as ae:
            x_all[i] = np.nan
    else:
        x_all[i] = np.nan
"""

# %% test
dv.scatter("ijk", ijk[:100]).get()

dv["ijk"]
dv["len(ijk)"]
dv["x=1"]
% % time
dv.execute(cmd).get()

# %%
dv.scatter("ijk", ijk[:]).get()

dv["ijk"]
dv["len(ijk)"]
dv["x=1"]
dv.execute(cmd).get()
# %%
x_all = dv.gather("x_all").get()
# from joblib import dump
# dump(x_all, "/home/cham/projects/gaia/data/x_all.dump")
for i, (_i, _j, _k) in enumerate(ijk):
    mesh_mini_est[_i, _j, _k] = x_all[i][1, 0]
    mesh_mini_err[_i, _j, _k] = (x_all[i][2, 0] - x_all[i][0, 0]) / 2.
    mesh_logt_est[_i, _j, _k] = x_all[i][1, 1]
    mesh_logt_err[_i, _j, _k] = (x_all[i][2, 1] - x_all[i][0, 1]) / 2.

# %%
rcParams.update({"font.size": 20})
fig = figure(figsize=(10, 8));
ax = fig.add_subplot(111)
im = ax.imshow(mesh_logt_err[2], vmin=0., vmax=0.5, cmap=cm.jet,
               origin="lower",
               extent=(-1 - 1 / 30., 1 + 1 / 30., 6. - 0.05, 10.1 + 0.05),
               aspect="auto")

xticks = np.array([0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.])
ax.set_xticks(np.log10(xticks))
ax.set_xticklabels(xticks)
colorbar(im)
ax.contour(np.log10(grid_mini), grid_logt, mesh_logt_err[2], [0.1, 0.2, 0.4],
           colors="k", linewidths=[1, 2, 3])
ax.set_xlabel("$M_{ini}$ [$M_\\odot$]")
ax.set_ylabel("$\\log_{10}$ (age / yr)")

ax.plot([0.4, 0.6], [10, 10], lw=1, c="k")
ax.text(.7, 10 - .06, "0.10")

ax.plot([0.4, 0.6], [9.75, 9.75], lw=2, c="k")
ax.text(.7, 9.75 - .06, "0.20")

ax.plot([0.4, 0.6], [9.5, 9.50], lw=3, c="k")
ax.text(.7, 9.5 - .06, "0.40")

ax.set_title("Error of $\\log_{10}$ (age / yr)")
fig.tight_layout()
fig.savefig("/home/cham/projects/gaia/figs/bayesian/logt_error.pdf")
fig.savefig("/home/cham/projects/gaia/figs/bayesian/logt_error.svg")

# %%
rcParams.update({"font.size": 20})
fig = figure(figsize=(10, 8));
ax = fig.add_subplot(111)
im = ax.imshow(np.abs(mesh_logt_est[2] - mesh_logt[2]), vmin=0, vmax=3,
               cmap=cm.jet, origin="lower",
               extent=(-1 - 1 / 30., 1 + 1 / 30., 6. - 0.05, 10.1 + 0.05),
               aspect="auto")

xticks = np.array([0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.])
ax.set_xticks(np.log10(xticks))
ax.set_xticklabels(xticks)
colorbar(im)
ax.contour(np.log10(grid_mini), grid_logt,
           np.abs(mesh_logt_est[2] - mesh_logt[2]), [0.5, 1.0, 2.0],
           colors="w", linewidths=[1, 2, 3])
ax.set_xlabel("$M_{ini}$ [$M_\\odot$]")
ax.set_ylabel("$\\log_{10}$ (age / yr)")

ax.plot([0.4, 0.6], [10, 10], lw=1, c="k")
ax.text(.7, 10 - .06, "0.5")

ax.plot([0.4, 0.6], [9.75, 9.75], lw=2, c="k")
ax.text(.7, 9.75 - .06, "1.0")

ax.plot([0.4, 0.6], [9.5, 9.50], lw=3, c="k")
ax.text(.7, 9.5 - .06, "2.0")

ax.set_title("Bias of $\\log_{10}$ (age / yr)")
fig.tight_layout()
fig.savefig("/home/cham/projects/gaia/figs/bayesian/logt_bias.pdf")
fig.savefig("/home/cham/projects/gaia/figs/bayesian/logt_bias.svg")

# %%
rcParams.update({"font.size": 20})
fig = figure(figsize=(10, 8));
ax = fig.add_subplot(111)
im = ax.imshow(mesh_mini_err[2] / mesh_mini_est[2], vmin=0., vmax=0.3,
               cmap=cm.jet, origin="lower",
               extent=(-1 - 1 / 30., 1 + 1 / 30., 6. - 0.05, 10.1 + 0.05),
               aspect="auto")

xticks = np.array([0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.])
ax.set_xticks(np.log10(xticks))
ax.set_xticklabels(xticks)
colorbar(im)
ax.contour(np.log10(grid_mini), grid_logt, mesh_mini_err[2] / mesh_mini_est[2],
           [0.05, 0.1, 0.2], colors="w", linewidths=[1, 2, 3])
ax.set_xlabel("$M_{ini}$ [$M_\\odot$]")
ax.set_ylabel("$\\log_{10}$ (age / yr)")

ax.plot([0.4, 0.6], [10, 10], lw=1, c="k")
ax.text(.7, 10 - .06, "0.05")

ax.plot([0.4, 0.6], [9.75, 9.75], lw=2, c="k")
ax.text(.7, 9.75 - .06, "0.10")

ax.plot([0.4, 0.6], [9.5, 9.50], lw=3, c="k")
ax.text(.7, 9.5 - .06, "0.20")

ax.set_title("Error of $M_{ini}$ [$M_\\odot$]")
fig.tight_layout()
fig.savefig("/home/cham/projects/gaia/figs/bayesian/mass_error.pdf")
fig.savefig("/home/cham/projects/gaia/figs/bayesian/mass_error.svg")

# %%
rcParams.update({"font.size": 20})
fig = figure(figsize=(10, 8));
ax = fig.add_subplot(111)
im = ax.imshow(np.abs(mesh_mini_est[2] - mesh_mini[2]) / mesh_mini[2], vmin=0.,
               vmax=0.1, cmap=cm.jet, origin="lower",
               extent=(-1 - 1 / 30., 1 + 1 / 30., 6. - 0.05, 10.1 + 0.05),
               aspect="auto")

xticks = np.array([0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.])
ax.set_xticks(np.log10(xticks))
ax.set_xticklabels(xticks)
colorbar(im)
ax.contour(np.log10(grid_mini), grid_logt,
           np.abs(mesh_mini_est[2] - mesh_mini[2]) / mesh_mini[2],
           [0.01, 0.05, 0.08], colors="w", linewidths=[1, 2, 3])
ax.set_xlabel("$M_{ini}$ [$M_\\odot$]")
ax.set_ylabel("$\\log_{10}$ (age / yr)")

ax.plot([0.4, 0.6], [10, 10], lw=1, c="k")
ax.text(.7, 10 - .06, "0.01")

ax.plot([0.4, 0.6], [9.75, 9.75], lw=2, c="k")
ax.text(.7, 9.75 - .06, "0.05")

ax.plot([0.4, 0.6], [9.5, 9.50], lw=3, c="k")
ax.text(.7, 9.5 - .06, "0.08")

ax.set_title("Bias of $M_{ini}$ [$M_\\odot$]")
fig.tight_layout()
fig.savefig("/home/cham/projects/gaia/figs/bayesian/mass_bias.pdf")
fig.savefig("/home/cham/projects/gaia/figs/bayesian/mass_bias.svg")

# %%
rcParams.update({"font.size": 20})
fig = figure(figsize=(10, 8));
ax = fig.add_subplot(111)
im = ax.imshow(mesh_label[2], vmin=0., vmax=7, cmap=cm.jet, origin="lower",
               extent=(-1 - 1 / 30., 1 + 1 / 30., 6. - 0.05, 10.1 + 0.05),
               aspect="auto")

xticks = np.array([0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.])
ax.set_xticks(np.log10(xticks))
ax.set_xticklabels(xticks)
colorbar(im)
ax.contour(np.log10(grid_mini), grid_logt, mesh_label[2], [0.5, 1.5, 2.5],
           colors="w", linewidths=[1, 2, 3])
ax.set_xlabel("$M_{ini}$ [$M_\\odot$]")
ax.set_ylabel("$\\log_{10}$ (age / yr)")

ax.plot([0.4, 0.6], [10, 10], lw=1, c="k")
ax.text(.7, 10 - .06, "0.5")

ax.plot([0.4, 0.6], [9.75, 9.75], lw=2, c="k")
ax.text(.7, 9.75 - .06, "1.5")

ax.plot([0.4, 0.6], [9.5, 9.50], lw=3, c="k")
ax.text(.7, 9.5 - .06, "2.5")

ax.set_title("Labels")
fig.tight_layout()
fig.savefig("/home/cham/projects/gaia/figs/bayesian/label.pdf")
fig.savefig("/home/cham/projects/gaia/figs/bayesian/label.svg")

# %%
iso = ig.get_iso(9.7, 0)
figure()
plot(iso["teff"], iso["logg"], 'x-')
for i in range(len(iso)):
    #    text(iso["teff"][i], iso["logg"][i], "{:1.0f}".format(iso["label"][i]))
    text(iso["teff"][i], iso["logg"][i], "{:1.3f}".format(iso["Mini"][i]))

# %%

figure()
plot(mesh_teff[2].flatten(), mesh_logg[2].flatten(), '.')

# %%
grid_teff = np.arange(8000., 4000, -100)
grid_logg = np.arange(6, -0.1, -0.2)
# grid_feh  = np.array([0.])

# mesh_teff, mesh_logg, mesh_feh = np.meshgrid(grid_teff, grid_logg, grid_feh)
mesh_teff, mesh_logg = np.meshgrid(grid_teff, grid_logg)

flat_teff = mesh_teff.flatten()
flat_logg = mesh_logg.flatten()

dv.scatter("flat_teff", flat_teff).get()
dv.scatter("flat_logg", flat_logg).get()
dv["flat_teff.shape"]

cmd = """
x_all = np.zeros((len(flat_teff), 3, 3))
for i, (_teff, _logg) in enumerate(zip(flat_teff, flat_logg)):
    try:
        x_all[i] = IsoGrid.predict_from_chi2(
            combined_iso,
            var_colnames=["teff", "logg", "feh_ini"],
            tlf=np.array([_teff, _logg, 0.]),
            tlf_err=np.array([100., 0.1, 0.1]),
            return_colnames=("Mini", "logt", "feh_ini"),
            q=(0.16, 0.50, 0.84))
    except ValueError as ae:
        x_all[i] = np.nan
"""
dv.execute(cmd).get()

# %%
x2 = dv.gather("x_all").get()
mesh_mini_est = x2[:, 1, 0].reshape(*mesh_teff.shape)
mesh_mini_err = ((x2[:, 2, 0] - x2[:, 0, 0]) / 2.).reshape(*mesh_teff.shape)
mesh_logt_est = x2[:, 1, 1].reshape(*mesh_teff.shape)
mesh_logt_err = ((x2[:, 2, 1] - x2[:, 0, 1]) / 2.).reshape(*mesh_teff.shape)
# %%

figure()
imshow(mesh_mini_err)
# %%
if mesh_teff[i, j, k] > 0 and i > 0 and j > 0:
    # do a chi2 matching
    try:
        x = ig.predict_from_chi2(
            combined_iso,
            var_colnames=["teff", "logg", "feh_ini"],
            tlf=np.array(
                [mesh_teff[i, j, k], mesh_logg[i, j, k], mesh_feh[i, j, k]]),
            tlf_err=np.array([100., 0.1, 0.1]),
            return_colnames=("Mini", "logt", "feh_ini"),
            q=(0.16, 0.50, 0.84))
        mesh_mini_est[i, j, k] = x[1, 0]
        mesh_mini_err[i, j, k] = (x[2, 0] - x[0, 0]) / 2.
        mesh_logt_est[i, j, k] = x[1, 1]
        mesh_logt_err[i, j, k] = (x[2, 1] - x[0, 1]) / 2.
        print(i, j, k, n_all, "success")
    except ValueError as ae:
        mesh_mini_est[i, j, k] = np.nan
        mesh_mini_err[i, j, k] = np.nan
        mesh_logt_est[i, j, k] = np.nan
        mesh_logt_err[i, j, k] = np.nan
        print(i, j, k, n_all, "failure")
else:
    mesh_mini_est[i, j, k] = np.nan
    mesh_mini_err[i, j, k] = np.nan
    mesh_logt_est[i, j, k] = np.nan
    mesh_logt_err[i, j, k] = np.nan
    print(i, j, k, n_all, "failure")

x = ig.predict_from_chi2(
    combined_iso,
    var_colnames=["teff", "logg", "feh_ini"],
    tlf=np.array([5500, 2.5, 0.0]),
    tlf_err=np.array([100., 0.1, 0.1]),
    return_colnames=("Mini", "logt", "feh_ini", "G"),
    q=(0.16, 0.50, 0.84))

# %%

var_colnames = ["teff", "logg", "feh_ini"]
sub_iso = np.array(combined_iso[var_colnames].to_pandas())

chi2_values = 0
for ivar in range(len(var_colnames)):
    chi2_values += ((sub_iso[:, ivar] - test_tlf[ivar]) / err_tlf[ivar]) ** 2.
chi2_values *= -0.5

p_post = np.exp(chi2_values) * combined_iso["w"]

u_feh_ini, inv_ind = np.unique(combined_iso["feh_ini"], return_inverse=True)

u_p_post = np.zeros(u_feh_ini.shape)
u_p_post[inv_ind] = u_p_post[inv_ind] + p_post
for i, _ in enumerate(inv_ind):
    if _ < len(u_p_post):
        u_p_post[_] += 0.5 * p_post[i]
        u_p_post[_ + 1] += 0.5 * p_post[i]
    else:
        u_p_post[_] += p_post[i]

from scipy.interpolate import interp1d

interp1d(np.cumsum(u_p_post) / np.sum(u_p_post), u_feh_ini)((0.16, 0.50, 0.84))

figure()
plot(u_feh_ini, u_p_post)
plot(u_feh_ini, np.cumsum(u_p_post))

u_teff, inv_ind = np.unique(combined_iso["teff"], return_inverse=True)
u_p_post = np.zeros_like(u_teff)
u_p_post[inv_ind] += p_post
figure()
plot(u_teff, u_p_post)
plot(u_teff, np.cumsum(u_p_post))
# %% teff
hist, bin_edges = np.histogram(combined_iso["teff"],
                               bins=np.arange(3500., 10000, 50.), normed=True,
                               weights=p_post)

figure()
plt.step(bin_edges[:-1], hist)

# %% logg
hist, bin_edges = np.histogram(combined_iso["logg"],
                               np.arange(-1., 6., 0.1), normed=True,
                               weights=p_post)

figure()
plt.step(bin_edges[:-1], hist)

# %% Mini
hist, bin_edges = np.histogram(combined_iso["Mini"],
                               np.arange(0., 6., 0.2), normed=True,
                               weights=p_post)

figure()
plt.step(bin_edges[:-1], hist)

# %% logt
hist, bin_edges = np.histogram(combined_iso["logt"],
                               np.arange(0., 13., 0.2), normed=True,
                               weights=p_post)

figure()
plt.step(bin_edges[:-1], hist)

# %%

H, xe, ye = np.histogram2d(
    combined_iso["teff"], combined_iso["logg"],
    bins=(np.arange(3500., 10000, 100.), np.arange(-1., 6., 0.1)), normed=True,
    weights=p_post)
H = np.log10(H)
figure()
imshow(np.fliplr(H.T), cmap=cm.jet,
       extent=(10000 + 25, 3500 - 25, 6.025, -1.025),
       aspect="auto", vmin=-10, vmax=np.nanmax(H))

# %%
H, xe, ye = np.histogram2d(
    combined_iso["Mini"], combined_iso["logt"],
    bins=(np.arange(0, 5, 0.1), np.arange(0, 13., 0.2)), normed=True,
    weights=p_post)
H = np.log10(H)
# H *= H>-10
figure()
imshow(H.T, cmap=cm.gray_r,
       extent=(0 - 0.025, 5 + 0.025, 6 - 0.05, 10.0 + 0.05),
       aspect="auto", vmin=-10, vmax=np.nanmax(H))

# %%

figure()
plot(sub_iso[:, 1], chi2_values, 'o', alpha=0.5)

figure()
scatter(sub_iso[:, 0], sub_iso[:, 1], s=10, c=chi2_values, alpha=0.5, vmin=-10,
        vmax=0, cmap=cm.gray_r)
colorbar()
xlim(6500, 5000)
ylim(, 5500)

# %%
figure()
plot(combined_iso["logTe"], combined_iso["logg"], '.')

from joblib import dump

dump(ig, "/media/cham/Seagate Expansion Drive/parsec/grid/gaiaDR2.dump")

# %%
def salpeter(m, ksi0=1.0):
    return ksi0 * (m) ** -2.35

x = np.arange(0.08, 12.0, 0.01)
y = salpeter(x)
figure()
plot(np.log10(x), np.log10(y))
i

d_mini

# %%

figure()
for isoc in ig.data:
    ind = (isoc["label"] >= 1) & (isoc["label"] < 8) & (isoc["Mini"] < 12)
    ind = (isoc["label"] < 9) & (isoc["Mini"] < 12)
    plot(isoc["logTe"][ind], isoc["logg"][ind], 'kx', alpha=0.05)

# %%
figure();
x = ig.get_iso(9.0, 0)
plot(x["logTe"], x["logg"], "r-.")
for i in range(len(x)):
    text(x["logTe"][i], x["logg"][i], "{}".format(x["label"][i]))

x = ig12.get_iso(9.0, 0)
plot(x["logTe"], x["logg"], "b-.")
for i in range(len(x)):
    text(x["logTe"][i], x["logg"][i], "{}".format(x["label"][i]))

# %%
"""
Process:
    0. Download grid d_logt=0.05, d_feh_ini=0.05
    1. unify colnames
    2. subcol & subrow, Mini<12.0, 1<=label<8
    3. interpolate :
        Mini 0.02
        logTe 0.01
        logg 0.01
    4. calculate weight
    5. add prior/weight

stage. The labels are:
0=PMS,
1=MS,
2=SGB,
3=RGB,
(4,5,6)=different stages of CHEB,
7=EAGB,
8=TPAGB.
"""
# %%
print(ig)
print(ig12)

# %%
ig12.get_iso(9.0, 0.0)
ig.get_iso(9.0, 0.0)["label"]

unify12(isoc)

# %%
# Mini

# Mass --> Mact
# isoc.rename_column("Mass", "Mact")

# Mass --> Mact

# define (minit, logt, feh) column names

x12 = ig12.get_iso(7.0, 0.0)
x = ig.get_iso(7.0, 0.0)

plot(isoc_interp(x, restrictions=(("Mini", 0.05),),
                 interp_config=(("label", "linear"),), M_ini="Mini"))

figure();
plot(x["Mini"][:-1], np.diff(x["Mini"]))

# %%
from ruby import IsoSet

IsoSet(isoc_list, vgrid_feh)

# %%
# get_one_isochrone(1e9, 0.0152, model="parsec12s",phot="gaia")

grid_feh = np.arange(-4., 1.2, 0.2)
grid_logt = np.arange(6., 10.5, 0.05)

vgrid_feh, vgrid_logt, grid_list, isoc_list = get_isochrone_grid(
    grid_feh, grid_logt, model='parsec12s_r14', phot='gaiaDR2',
    Zsun=0.0152, n_jobs=20, verbose=10)

np.sum([_.data.shape[0] for _ in isoc_list])

from ezpadova import parsec

print(parsec.get_one_isochrone(1e7, 0.02, model='parsec12s',
                               phot='gaia').colnames)
print(parsec.get_one_isochrone(1e7, 0.02, model='parsec12s_r14',
                               phot='gaia').colnames)

# %%

gdr1 = parsec.get_one_isochrone(1e9, 0.0152, model='parsec12s', phot='gaia')
gdr1_r14 = parsec.get_one_isochrone(1e9, 0.0152, model='parsec12s_r14',
                                    phot='gaia')
gdr2 = parsec.get_one_isochrone(1e9, 0.0152, model='parsec12s', phot='gaiaDR2')
# %%
figure()
plot(gdr1["G"] - gdr1["G_RP"], gdr1["G"], '-')
scatter(gdr1["G"] - gdr1["G_RP"], gdr1["G"], s=10, c=gdr1["stage"],
        cmap=cm.jet)

ind = gdr1_r14["label"] < 8
plot(gdr1_r14["Gmag"][ind] - gdr1_r14["G_RPmag"][ind], gdr1_r14["Gmag"][ind],
     '-')
colorbar()
ylim(30, -30)
# %%
figure()
plot(gdr1["Gmag"] - gdr1["G_RPmag"], gdr1["Gmag"], '-')
plot(gdr2["Gmag"] - gdr2["G_RPmag"], gdr2["Gmag"], '-')