<reponame>chinesefirewall/Robotics
# <NAME>
# -*- coding: utf-8 -*-

import matplotlib.pyplot as plt
import numpy as np
import scipy.optimize as opt

# Fill the lists with your data
X = [71, 78, 92, 109, 166, 261, 439] # Size [px]
Y = [1800, 1600, 1380, 1100, 700, 400, 200] # Distance [mm]

def blob_size_to_distance(blob_size, a, b):
    """
    TASK: The cost function which parameters we are looking for, replace with correct function
    """

    dist = (a/blob_size) + b

    return dist

if __name__ == "__main__":
    # The curve fitting happens here
    optimized_parameters, pcov = opt.curve_fit(
        blob_size_to_distance,
        X,
        Y,
        bounds=([-np.inf, -np.inf], [np.inf, np.inf])
    )

    print(optimized_parameters)
    a = optimized_parameters[0]
    b = optimized_parameters[1]

    print("Optimized parameters:")
    print("  a = " + str(a))
    print("  b = " + str(b))

    # Calculate points with the optimized parameters
    x_data_fit = np.linspace(min(X), max(X), 100)
    y_data_fit = blob_size_to_distance(x_data_fit, *optimized_parameters)

    # Plot the data
    plt.plot(X, Y, ".", label="measured data")
    plt.plot(x_data_fit, y_data_fit, label="fitted data")

    # Show the graph
    plt.legend()
    plt.xlabel("Blob size (px)")
    plt.ylabel("Distance (mm)")
    plt.show()


<filename>big_data/python_tools/big_data_tools/bokeh_tools/find_dist.py
from scipy import stats
from scipy.stats import probplot
import math
import scipy
import numpy as np
import pprint
from bokeh.plotting import figure, show
from bokeh.layouts import gridplot
import pprint
# /home/henry/Envs/class/lib/python3.5/site-packages/scipy/stats/distributions.py
from scipy.stats import distributions
pp = pprint.PrettyPrinter(indent = 4)

D = {'norm': stats.norm,
    'pareto':stats.pareto,
    'lognorm':stats.lognorm,
    'poisson':stats.poisson,
        }

def sse(y1, y2):
    return math.sqrt(sum((y1- y2) * (y1- y2))/len(y1))

def get_dist(dist_string):
    return D[dist_string]

def pp_plot(data, dist, sparams = None, title= None):
    if not title:
        title = "{dist} QQ Plot".format(dist = dist)
    series = probplot(data, dist=dist, sparams= sparams)
    p = figure(title=title)
    x = series[0][0]
    y = series[0][1]
    slope = series[1][0]
    intercept = series[1][1]
    y2 = [z * slope + intercept for z in x]
    p.scatter(series[0][0],series[0][1], fill_color="red")
    p.line(x,y2, line_width =2)
    e = sse(series[0][1], y2)
    return p, e

def make_pps(data, dist, sparams = None, title= None):
    if not title:
        title = "{dist} QQ Plot".format(dist = dist)
    series = probplot(data, dist=dist, sparams= sparams)
    p = figure(title=title)
    x = series[0][0]
    y = series[0][1]
    slope = series[1][0]
    intercept = series[1][1]
    y2 = [z * slope + intercept for z in x]
    p.scatter(series[0][0],series[0][1], fill_color="red")
    p.line(x,y2, line_width =2)
    return p

#this seems wrong with the fit
def _make_pps(data, dists):
    ps = []
    es = []
    for i in dists:
        f = get_dist(i)
        params = f.fit(data)
        p, e =pp_plot(data, i, params)
        ps.append(p)
        es.append((i, e))
    grid = gridplot(ps, ncols = 2, height=500,
            width = 500)
    show(grid)

def _find_best(data):
    least_sse = np.inf
    best_f = None
    ps = []
    for i in ['lognorm', 'pareto', 'norm', 'lognorm']:
        f = get_dist(i)
        p = figure(title = i)
        hist, edges = np.histogram([x for x in data if x < f.ppf(0.99, b)], density=True )
        ps.append(p)
        p.quad(top = hist, bottom=0, left=edges[:-1], right=edges[1:], alpha = .4)
        params = f.fit(data)
        x = np.linspace(f.ppf(0.01, *params), f.ppf(0.99, *params), len(data))
        pdf = f.pdf(x, *params)
        p.line(x, pdf)
        pdf = f.pdf(x, *params)
        y, x = np.histogram(data, bins=1000, density=True)
        e = sse(pdf, y)
        print(i, e)
        if e < least_sse:
            least_sse = e
            best_f = i
    grid = gridplot(ps, ncols = 2, height=500,
            width = 500)
    show(grid)
    return least_sse, best_f

def test_ppf_cdf():
    """ppf is percentiles"""
    loc = 0
    scale =1
    ppf = .9
    #inverse functions
    result = stats.norm.cdf(x= stats.norm.ppf(q= ppf, loc=loc, scale=scale), loc=loc, scale = scale)
    assert round(ppf,6) == round(result,6)
    #what is the number for which 90% lies to the left
    x = stats.norm.ppf(q= .9, loc=loc, scale=scale)
    #1.28, the Z score for a normal dist
    #what are lies to the left of 1.28?
    per = stats.norm.cdf(x=1.28, loc= loc, scale = scale)
    #.9

if __name__ == '__main__':
    #b = 2.62
    #data = stats.pareto.rvs(b, loc = 3, size= 1000)
    #dists are dir(distributions)
    data = [.83, .88, .88, 1.4, 1.09, 1.12, 1.29, 1.31,
            1.48, 1.49, 1.59, 1.62, 1.65, 1.71, 1.83]
    dists = ['pareto', 'norm', 'lognorm']
    d = stats.describe(data)
    test_ppf_cdf()
    #find_best(data, dists)


from sqlite3.dbapi2 import connect
from data import database_manager as dm
import utils
from matplotlib import pyplot
from linear_algebra import curve_fitting as cf
from numpy import arange, greater
from scipy.optimize import curve_fit

#####################################the sqlite section#########################################

# create a connection to the database
connection = dm.create_connection("longley.db")
# To-Do : retrieve rows of the table
ROWS=dm.select_all(connection)
print("here are the rows of your longley table")
dm.print_rows(ROWS)
print(len(ROWS))
dm.close_connection(connection) #close the connection to the db file

#####################################the data type section#######################################
data=utils.convert_to_floats(ROWS)
print(data) #to see the format
#####################################the data shape##############################################
print("the shape of our data is :",data.shape)
#####################################the linear algebra section#################################

# Let's check if the two variables GNP.deflator and year  are correlated
def objective(x, a, b):
    	return a * x + b

# load the dataset

x, y = data[:, 4], data[:, -1]

a, b=cf.get_result(x,y)

# plotting the result
pyplot.scatter(x, y)
# define a sequence of inputs between the smallest and largest known inputs
x_line = arange(min(x), max(x), 1)
# calculate the output for the range
y_line = cf.set_objective(x_line, a, b)
# create a line plot for the mapping function
pyplot.plot(x_line, y_line, '--', color='red')
pyplot.show()


<reponame>Polydynamical/morfeus
"""Pyramidalization code."""

import functools
import itertools
from typing import Any, Iterable, List, Optional, Sequence, Union

import numpy as np
import scipy.spatial

from morfeus.io import read_geometry
from morfeus.utils import get_connectivity_matrix

class Pyramidalization:
    """Calculates and stores results of pyramidalization and alpha angle.

    As described in Struct. Chem. 1991, 2, 107 and alternatively according to bond
    angles as in J. Comput. Chem. 2012, 33 (27), 2173–2179.

    Args:
        coordinates: Coordinates (Å)
        atom_index: Index of pyramidalized atom (1-indexed)
        neighbor_indices: Indices of neighbors to pyramidalized atom
        elements: Elements as atomic symbols or numbers
        radii: Covalent radii used to determine connectivity (Å)
        radii_type: Covalent radii type: 'pyykko'
        excluded_atoms: Indices of atoms to exclude
        method: Method for detecting neighbors: 'connectivity' or 'distance'. Ignored if
            neighbor_indices is given.

    Attributes:
        alpha: Average alpha angle (degrees)
        alphas: Alpha angles for all permutations of neighbors (degrees)
        neighbor_indices: Indices of neighbors to pyramidalized atom
        P_angle: Pyramidalization according to Gavrish
        P: Pyramidalization according to Radhakrishnan
    """

    alpha: float
    alphas: np.ndarray
    neighbor_indices: List[int]
    P_angle: float
    P: float

    def __init__(  # noqa: C901
        self,
        coordinates: Sequence[Sequence[float]],
        atom_index: int,
        neighbor_indices: Optional[Sequence[int]] = None,
        elements: Optional[Union[Iterable[int], Iterable[str]]] = None,
        radii: Optional[Sequence[float]] = None,
        radii_type: str = "pyykko",
        excluded_atoms: Optional[Sequence[int]] = None,
        method: str = "distance",
        scale_factor: float = 1.2,
    ) -> None:
        coordinates = np.array(coordinates)
        atom_coordinates = coordinates[atom_index - 1]

        if neighbor_indices is None:
            neighbor_indices = []
        else:
            neighbor_indices = list(neighbor_indices)

        if excluded_atoms is None:
            excluded_atoms = []
        excluded_atoms = excluded_atoms = np.array(excluded_atoms, dtype=bool)

        # Get 3 closest neighbors
        if len(neighbor_indices) > 0:
            if len(neighbor_indices) != 3:
                raise Exception(f"Only {len(neighbor_indices)} neighbors.")
            neighbors = np.array(neighbor_indices) - 1
        elif method == "distance":
            # Generate mask for excluded atoms
            mask = np.zeros(len(coordinates), dtype=bool)
            mask[excluded_atoms - 1] = True
            mask[atom_index - 1] = True

            # Get three closest atoms not in the excluded atoms
            distances = scipy.spatial.distance.cdist(
                atom_coordinates.reshape(1, -1), coordinates
            ).reshape(-1)
            distances[mask] = np.inf
            neighbors = np.argsort(distances)[:3]
        elif method == "connectivity":
            # Construct connectivity matrix and get closest neighbors.
            if elements is None and radii is None:
                raise Exception("Connectivity requires elements or radii.")
            # if radii is None:
            #    radii = get_radii(elements, radii_type="pyykko")
            connectivity_matrix = get_connectivity_matrix(
                coordinates,
                elements=elements,
                radii=radii,
                radii_type=radii_type,
                scale_factor=scale_factor,
            )
            connected_atoms = np.where(connectivity_matrix[atom_index - 1, :])[0]
            neighbors = connected_atoms[~np.isin(connected_atoms, excluded_atoms - 1)]
            if len(neighbors) != 3:
                raise Exception(f"{len(neighbors)} neighbors. 3 expected.")

        # Get unit vectors between central atom and neighbors
        a = coordinates[neighbors[0]] - atom_coordinates
        a /= np.linalg.norm(a)
        b = coordinates[neighbors[1]] - atom_coordinates
        b /= np.linalg.norm(b)
        c = coordinates[neighbors[2]] - atom_coordinates
        c /= np.linalg.norm(c)

        # Calculate alpha for all permutations
        alphas = []
        vectors = []
        cos_alphas = []
        thetas = []
        for v_1, v_2, v_3 in itertools.permutations([a, b, c], 3):
            # Calculate cos_alpha
            normal = np.cross(v_1, v_2)
            normal /= np.linalg.norm(normal)
            cos_alpha = np.dot(v_3, normal)

            # Test if normal vector is colinear with v_3
            if cos_alpha < 0:
                continue
            alpha = np.arccos(cos_alpha)

            # Check for "acute" pyramid and correct angle
            v_1_2 = v_1 + v_2
            v_1_2 /= np.linalg.norm(v_1_2)
            cos_angle = np.dot(v_1_2, v_3)
            if cos_angle > 0:
                alpha = -alpha
            alphas.append(alpha)
            cos_alphas.append(cos_alpha)
            vectors.append((v_1, v_2))

            # Calculate theta angle
            cos_theta = np.dot(v_1, v_2)
            theta = np.rad2deg(np.arccos(cos_theta))
            thetas.append(theta)

        # Calculate P
        v_1, v_2 = vectors[0]
        sin_theta = np.linalg.norm(np.cross(v_1, v_2))
        P = sin_theta * cos_alphas[0]

        # Correct P if pyramid is "acute" on average
        if np.mean(alphas) < 0:
            P = 2 - P

        # Calculate P according to Gavrish method
        P_angle = np.sqrt(360 - sum(thetas))

        # Store attributes
        self.P = P
        self.P_angle = P_angle
        self.alpha = np.rad2deg(np.mean(alphas))
        self.alphas = np.rad2deg(alphas)
        self.neighbor_indices = (neighbors + 1).tolist()

    def print_report(self) -> None:
        """Print report of results."""
        print(f"P: {self.P:.3f}")
        print(f"P_angle: {self.P_angle:.3f}")

    def __repr__(self) -> str:
        return f"{self.__class__.__name__}({round(self.P, 3)!r})"

def cli(file: str) -> Any:
    """CLI for pyramidalization.

    Args:
        file: Geometry file

    Returns:
        Partially instantiated class
    """
    elements, coordinates = read_geometry(file)
    return functools.partial(Pyramidalization, coordinates, elements=elements)


#!/usr/bin/env python

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from matplotlib.colors import LogNorm, ListedColormap,BoundaryNorm
import numpy as np
import datetime as dt
import sys, os, pickle, time
from scipy.ndimage.filters import gaussian_filter
import pandas as pd
from mpl_toolkits.basemap import *
from sklearn.calibration import CalibratedClassifierCV, calibration_curve
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier
from sklearn import metrics
from sklearn.model_selection import train_test_split
from keras.models import Model, save_model, load_model
from keras.layers import Dense, Activation, Conv2D, Input, AveragePooling2D, Flatten, LeakyReLU
from keras.layers import Dropout, BatchNormalization
from keras.regularizers import l2
from keras.optimizers import SGD, Adam
import keras.backend as K
import tensorflow as tf
from scipy import spatial
from netCDF4 import Dataset
import datetime as dt

from ml_functions import read_csv_files, normalize_multivariate_data, log, get_features

def readNCLcm(name):
    '''Read in NCL colormap for use in matplotlib'''
    rgb, appending = [], False
    rgb_dir_ch = '/glade/u/apps/ch/opt/ncl/6.4.0/intel/16.0.3/lib/ncarg/colormaps'
    fh = open('%s/%s.rgb'%(rgb_dir_ch,name), 'r')

    for line in list(fh.read().splitlines()):
        if appending: rgb.append(list(map(float,line.split())))
        if ''.join(line.split()) in ['#rgb',';RGB']: appending = True
    maxrgb = max([ x for y in rgb for x in y ])
    if maxrgb > 1: rgb = [ [ x/255.0 for x in a ] for a in rgb ]
    return rgb

def brier_score_keras(obs, preds):
    return K.mean((preds - obs) ** 2)

def brier_skill_score_keras(obs, preds):
    climo = K.mean((obs - K.mean(obs)) ** 2)
    bs = brier_score_keras(obs, preds)
    ratio = (bs / climo)
    return climo

def auc(obs, preds):
    auc = tf.metrics.auc(obs, preds)[1]
    K.get_session().run(tf.local_variables_initializer())
    return auc

def bss(obs, preds):
    bs = np.mean((preds - obs) ** 2)
    climo = np.mean((obs - np.mean(obs)) ** 2)
    return 1.0 - (bs/climo)

def make_gridded_forecast(predictions, labels, dates, fhr):
    ### reconstruct into grid by day (mask makes things more complex than a simple reshape)
    gridded_predictions = np.zeros((num_dates,num_fhr,65*93), dtype=np.float64)
    gridded_labels      = np.zeros((num_dates,num_fhr,65*93), dtype=np.float64)

    # just grid predictions for this class
    predictions = predictions.reshape((num_dates, num_fhr, -1))
    labels      = labels.reshape((num_dates, num_fhr, -1))

    for i, dt in enumerate(unique_forecasts):
        for j, f in enumerate(unique_fhr):
            gridded_predictions[i,j,thismask] = predictions[i,j,:]
            gridded_labels[i,j,thismask]      = labels[i,j,:]
        #print(dt, gridded_predictions[i,:].max())

    # return only predictions for US points
    return (gridded_predictions.reshape((num_dates, num_fhr, 65, 93)), gridded_labels.reshape((num_dates, num_fhr, 65, 93)))

def smooth_gridded_forecast(predictions_gridded):
    smoothed_predictions = []
    dim = predictions_gridded.shape
    for k,s in enumerate(smooth_sigma):
        if len(dim) == 4: smoothed_predictions.append(gaussian_filter(predictions_gridded, sigma=[0,0,s,s]))
        if len(dim) == 3: smoothed_predictions.append(gaussian_filter(predictions_gridded, sigma=[0,s,s]))

    # return only predictions for US points
    return np.array(smoothed_predictions)

def plot_forecast(predictions, prefix="", fhr=36):
    test = readNCLcm('MPL_Greys')[25::] + [[1,1,1]] + readNCLcm('MPL_Reds')[10::]
    #test = readNCLcm('perc2_9lev')[1::]
    cmap = ListedColormap(test)
    #cmap = plt.get_cmap('RdGy_r')
    norm = BoundaryNorm(np.arange(0,1.1,0.1), ncolors=cmap.N, clip=True)

    #print(predictions)

    #awips = Basemap(projection='lcc', llcrnrlon=-133.459, llcrnrlat=12.19, urcrnrlon=-49.38641, urcrnrlat=57.2894, lat_1=25.0, lat_2=25.0, lon_0=-95, resolution='l', area_thresh=10000.)

    #fig, axes, m  = pickle.load(open('/glade/u/home/sobash/NSC_scripts/ch_pk_files/rt2015_ch_CONUS.pk', 'r'))
    #fig, axes, m  = pickle.load(open('/glade/u/home/sobash/NSC_scripts/dav_pk_files/rt2015_ch_CONUS.pk', 'rb'))
    fig, axes, m = pickle.load(open('rt2015_ch_CONUS.pk', 'rb'))

    lats, lons = predictions['lat'].values, predictions['lon'].values
    x, y = m(lons, lats)

    # do something convoluted here to only plot each point once
    probmax = {}
    for i,p in enumerate(predictions['predict_proba'].values):
        thiskey = '%f%f'%(lats[i],lons[i])
        if thiskey in probmax:
            if p > probmax[thiskey]:
                probmax[thiskey] = p
        else:
           probmax[thiskey] = p

    # need to do this before calling text
    #m.set_axes_limits(ax=axes)

    for i,p in enumerate(predictions['predict_proba'].values):
        thiskey = '%f%f'%(lats[i],lons[i])
        thisvalue = probmax[thiskey]

        color = cmap(norm([thisvalue])[0])
        probmax[thiskey] = -999
        if x[i] < m.xmax and x[i] > m.xmin and y[i] < m.ymax and y[i] > m.ymin and thisvalue > 0.05:
        #if thisvalue >= 0.15:
            a = axes.text(x[i], y[i], int(round(thisvalue*100)), fontsize=10, ha='center', va='center', family='monospace', color=color, fontweight='bold')
           # a = axes.text(x[i], y[i], int(round(thisvalue*100)), fontsize=12, ha='center', va='center', family='monospace', color=color, fontweight='bold')
    #a = m.scatter(x, y, s=50, c=predictions['predict_proba'].values, lw=0.5, edgecolors='k', cmap=cmap, norm=norm)

    ax = plt.gca()
    cdate = sdate + dt.timedelta(hours=fhr)
    sdatestr = (cdate - dt.timedelta(hours=2)).strftime('%Y-%m-%d %H:%M:%S UTC')
    edatestr = (cdate + dt.timedelta(hours=2)).strftime('%Y-%m-%d %H:%M:%S UTC')
    #plt.text(0,1.01,'Probability of severe within 120-km of a point valid %s - %s'%(sdatestr, edatestr), fontsize=14, transform=ax.transAxes)
    plt.text(0,1.01,'Max 4-h, 120-km all-severe NNPF over all forecast hours for WRF init %s'%sdate.strftime('%Y%m%d%H'), fontsize=14, transform=ax.transAxes)

    # ADD COLORBAR
    #cax = fig.add_axes([0.02,0.1,0.02,0.3])
    #cb = plt.colorbar(a, cax=cax, orientation='vertical', extendfrac=0.0)
    #cb.outline.set_linewidth(0.5)
    #cb.ax.tick_params(labelsize=10)

    # plot reflectivity
    plot_cref = False
    if plot_cref:
        initstr = sdate.strftime('%Y%m%d00')
        wrfcdate = cdate.strftime('%Y-%m-%d_%H_%M_%S')
        fh = Dataset('/glade/p/mmm/parc/sobash/NSC/3KM_WRF_POST_12sec_ts/%s/diags_d01_%s.nc'%(initstr,wrfcdate), 'r')
        lats = fh.variables['XLAT'][0,:]
        lons = fh.variables['XLONG'][0,:]
        cref = fh.variables['REFL_COM'][0,:]
        fh.close()

        x, y = m(lons, lats)
        plt.contourf(x, y, cref, levels=[35,1000], colors='k', alpha=0.5)

    plt.savefig('forecast%s.png'%prefix, dpi=150)

def make_labels():
    #labels   = ((df['hail_rptdist'+twin] < d) & (df['hail_rptdist'+twin] > 0)) |  \
    labels   = ((df['hailone_rptdist'+twin] < d) & (df['hailone_rptdist'+twin] > 0)) |  \
               ((df['wind_rptdist'+twin] < d) & (df['wind_rptdist'+twin] > 0)) | \
               ((df['torn_rptdist'+twin] < d) & (df['torn_rptdist'+twin] > 0))

    labels_wind     =  ((df['wind_rptdist'+twin] < d) & (df['wind_rptdist'+twin] > 0))
    labels_hailone  =  ((df['hailone_rptdist'+twin] < d) & (df['hailone_rptdist'+twin] > 0))
    labels_torn     =  ((df['torn_rptdist'+twin] < d) & (df['torn_rptdist'+twin] > 0))
    labels_sighail  =  ((df['sighail_rptdist'+twin] < d) & (df['sighail_rptdist'+twin] > 0))
    labels_sigwind  =  ((df['sigwind_rptdist'+twin] < d) & (df['sigwind_rptdist'+twin] > 0))

    # labels for multi-class neural network
    if multiclass: labels = np.array([ labels, labels_wind, labels_hailone, labels_torn, labels_sighail, labels_sigwind ]).T
    else: labels = np.array([ labels ]).T

    return labels

def compute_optimal_uh():
    predictions_gridded_uh, labels_gridded = make_gridded_forecast(uh120_all, labels_all[:,0], dates_all, fhr_all)

    optimal_uh_warmseason, num_rpts_warm = pickle.load(open('/glade/work/sobash/NSC_objects/optimal_uh_warmseason', 'rb'))
    optimal_uh_coolseason, num_rpts_cool = pickle.load(open('/glade/work/sobash/NSC_objects/optimal_uh_coolseason', 'rb'))

    #months_all = months_all.reshape((num_dates, num_fhr, -1))
    #months_all = months_all[:,0,0]
    #for k,m in enumerate(months_all):

    m = sdate.month
    print(num_fhr)
    if m in [4,5,6,7]: this_uh = ( predictions_gridded_uh >= optimal_uh_warmseason[:num_fhr,:] )
    else:              this_uh = ( predictions_gridded_uh >= optimal_uh_coolseason[:num_fhr,:] )

    this_uh = this_uh.reshape((num_fhr,-1))[:,thismask]

    uh_binary = np.array(this_uh).flatten()
    return uh_binary

def output_csv(fname):
    # output probabilities for one forecast

    # output 80-km grid locations
    #awips = Basemap(projection='lcc', llcrnrlon=-133.459, llcrnrlat=12.19, urcrnrlon=-49.38641, urcrnrlat=57.2894, lat_1=25.0, lat_2=25.0, lon_0=-95, resolution=None, area_thresh=10000.)
    #lons, lats = awips.makegrid(93, 65)
    #np.savetxt('grid.out', np.array([lons.flatten(), lats.flatten(), thismask]).T, fmt='%.3f,%.3f,%.0d', header='lon,lat,mask')

    idxarray = np.tile(np.arange(0,93*65)[np.newaxis,:], (num_fhr,1)).flatten()
    #fhrarray = np.tile(np.arange(1,num_fhr+1)[:,np.newaxis], (1,93*65)).flatten()
    fhrarray = np.tile(unique_fhr[:,np.newaxis], (1,93*65)).flatten()
    usmask   = np.tile(thismask[np.newaxis,:], (num_fhr,1)).flatten()

    all_probs = 100*predictions_all_gridded.reshape((7,-1)) #should become (7,num_fhr*93*65)
    all_probs = np.where(all_probs<1, 0, all_probs)

    # want to only include areas where ANY prob is non-zero and within US mask area (smoothed UH likely has probs outside of US, maybe ML too)
    probmask =  ( np.any(all_probs, axis=0) & usmask )

    np.savetxt(fname, np.array([idxarray[probmask], fhrarray[probmask], \
                                all_probs[0,probmask], all_probs[1,probmask], all_probs[2,probmask], \
                                all_probs[3,probmask], all_probs[4,probmask], all_probs[5,probmask], all_probs[6,probmask]]).T, \
                            delimiter=',', fmt='%.0d', comments='', header='idx,fhr,psvr,pwind,phail,ptorn,psighail,psigwind,puh')

    #probarray =100*predictions_gridded[fmask,:,:].flatten()
    #probmask = (probarray >= 1)
    #np.savetxt('test.out', np.array([idxarray[probmask], fhrarray[probmask], probarray[probmask]]).T, delimiter=',', fmt='%.0d', header='idx,fhr,prob')

### NEURAL NETWORK PARAMETERS ###

nn_params = { 'num_layers': 1, 'num_neurons': [ 1024 ], 'dropout': 0.1, 'lr': 0.001, 'num_epochs': 10, \
              'report_window_space':[ 40,120 ], 'report_window_time':[ 2 ] }

rf_params = { 'ntrees': 100, 'max_depth': 20, 'min_samples_split': 20, 'min_samples_leaf': 10 }
#rf_params = { 'ntrees': 200, 'max_depth': 20, 'min_samples_split': 20 }

#years         =  [2011,2012,2013,2014,2015,2016] #k-fold cross validation for these years
#years         =  [ int(sys.argv[3]) ]
model         =  sys.argv[2]
mem           =  1

plot          =  True

multiclass    =  True
thin_data     =  True
thin_fraction =  0.5
smooth_probs  =  False
smooth_sigma  =  1
simple_features = True
dataset = 'HRRR'
scaling_dataset = 'HRRR'
scaling_dataset = 'HRRRX'
subset = 'all'
use_nschrrr_features = False
expname = 'epoch30'
expname = 'hrrrv4-epoch30'

#mask  = pickle.load(open('/glade/u/home/sobash/2013RT/usamask.pk', 'rb'))
mask  = pickle.load(open('/glade/work/sobash/NSC_objects/HRRR/usamask_mod.pk', 'rb'))
thismask = mask.flatten()

trained_models_dir = '/glade/work/sobash/NSC_objects'
trained_models_dir = '/glade/work/sobash/NSC_objects/trained_models_paper'
trained_models_dir = '/glade/work/sobash/NSC_objects/HRRR/trained_models'

#sdate   = dt.datetime(2020,3,2,0,0,0)
#edate   = dt.datetime(2020,3,2,0,0,0)
sdate = dt.datetime.strptime(sys.argv[1], '%Y%m%d%H')
edate = sdate
dateinc = dt.timedelta(days=1)

##################################

if multiclass: numclasses = 6
else: numclasses = 1
twin = "_%dhr"%nn_params['report_window_time'][0]

features = get_features(subset, use_nschrrr_features)

log('Number of features %d'%len(features))
log(nn_params)
log(rf_params)

log('Reading Data')
# read data and reassign data types to float32 to save memory
type_dict = {}
for f in features: type_dict[f]='float32'
df, numfcsts = read_csv_files(sdate, edate, dataset)

# restrict hrrr values to min/max range
#with open('hrrrv3_minmax') as fh:
#    for line in fh.read().splitlines()[1:]:
#        r = line.split(',')
#        if r[0] in ['fhr', 'xind', 'yind', 'lat', 'lon', 'year', 'month', 'hour', 'dayofyear']: continue
#        field, minv, maxv = r[0], float(r[1]), float(r[2])
#        df.loc[df[field] > maxv, field] = maxv
#        df.loc[df[field] < minv, field] = minv
#        #print(field, df[field].min(), df[field].max())

mask  = pickle.load(open('/glade/u/home/sobash/2013RT/usamask.pk', 'rb'))
mask = mask.reshape((65,93))

# what forecast points to use
forecast_mask = ( (df['fhr'] >= 1) )
these_points = df[forecast_mask]
year = sdate.year

#if year > 2016: year = 2016 #use NN without 2016 for any date past 2016
#year = 2017
year = 2020
year = 2021
#year = 2016

classes = { 0:'all', 1:'wind', 2:'hailone', 3:'torn', 4:'sighail', 5:'sigwind'}
hazard_type = 0

fhr_all  = df[forecast_mask]['fhr'].values
dates_all = df[forecast_mask]['Date'].values
unique_forecasts, unique_fhr = np.unique(dates_all), np.unique(fhr_all)
smooth_sigma, num_dates, num_fhr = [2.0], len(unique_forecasts), len(unique_fhr)

for d in nn_params['report_window_space']:
    labels_all = make_labels()

    if model == 'nn':
        scaling_values = pickle.load(open('/glade/work/sobash/NSC_objects/HRRR/scaling_values_all_%s.pk'%scaling_dataset, 'rb'))
        norm_in_data, scaling_values = normalize_multivariate_data(df[features].values.astype(np.float32), features, scaling_values=scaling_values)

        this_in_data = norm_in_data[forecast_mask,:]

        dense_model = None
        model_fname = '%s/neural_network_%s_%dkm%s_nn%d_drop%.1f_%s.h5'%(trained_models_dir,year,d,twin,\
                                                                      nn_params['num_neurons'][0],nn_params['dropout'],expname)
        dense_model = load_model(model_fname, custom_objects={'brier_score_keras': brier_score_keras, 'brier_skill_score_keras':brier_skill_score_keras, 'auc':auc })
        print('Using', model_fname)

        predictions_all = dense_model.predict(this_in_data)

        these_points['predict_proba'] = predictions_all[:,hazard_type]

    if model == 'rf':
        this_in_data = these_points[features].values

        model_fname = '%s/rf_gridded_%s_%dkm%s_n%d_d%d_m%d_l%d.pk'%(trained_models_dir,year,d,twin,rf_params['ntrees'],rf_params['max_depth'],rf_params['min_samples_split'],rf_params['min_samples_leaf'])
        rf = pickle.load(open(model_fname, 'rb'))

        predictions_all = rf.predict_proba(this_in_data)
        predictions_all = np.array(predictions_all)[:,:,1].T #needs to be in shape (examples,classes)

        these_points['predict_proba'] = predictions_all[:,hazard_type]

    if d == 40 and twin == '_2hr': uh120_all  = df[forecast_mask]['UP_HELI_MAX-N1T5'].values
    if d == 80 and twin == '_2hr': uh120_all  = df[forecast_mask]['UP_HELI_MAX80-N1T5'].values
    if d == 120 and twin == '_2hr': uh120_all = df[forecast_mask]['UP_HELI_MAX120-N1T5'].values

    #uh_binary = compute_optimal_uh()
    #predictions_gridded_uh, labels_gridded  = make_gridded_forecast((uh_binary).astype(np.int32), labels_all[:,0], dates_all, fhr_all)

    # convert predictions into grid and add UH probs
    predictions_all_gridded = []
    for i in range(6):
        predictions_gridded, labels_gridded = make_gridded_forecast(predictions_all[:,i], labels_all[:,i], dates_all, fhr_all)
        predictions_all_gridded.append(predictions_gridded)

    predictions_gridded_uh, labels_gridded  = make_gridded_forecast((uh120_all>75).astype(np.int32), labels_all[:,0], dates_all, fhr_all)
    predictions_gridded_uh_smoothed = smooth_gridded_forecast(predictions_gridded_uh)
    predictions_all_gridded.append(predictions_gridded_uh_smoothed[0,:])
    predictions_all_gridded = np.array(predictions_all_gridded)

    log('Outputting CSV')
    fname = 'probs_%s_%s_%dkm.out'%(model,sdate.strftime('%Y%m%d%H'),d)
    output_csv(fname)

    log('Outputting grib')
    import write_grib as wg
    #ofile = "./grib/hrrr_ml_%dkm_2hr_%s.grb"%(d,sdate.strftime('%Y%m%d%H'))
    #wg.write_grib(predictions_gridded[0,:,:], sdate, 0, ofile) #write all forecast hours into one file

    # write grib for each hour for each scale, combine hazard probs into one file
    for i,f in enumerate(unique_fhr):
        ofile = "./grib/hrrr_ml_%dkm_2hr_%sf%03d.grb"%(d,sdate.strftime('%Y%m%d%H'), f)
        wg.write_grib(predictions_all_gridded[:6,0,i-1,:], sdate, f, ofile) #output first 6 elements (ignore UH)

log('Finished')


"""
Following functions are specific to the analysis of the data saved
with BELLA control system
"""

import pandas as pd
import matplotlib.pyplot as plt
import os
import glob
import re
from numpy import unravel_index
import numpy as np
from scipy import stats
import json
from functions.data_analysis import df_outlier2none

def get_data(dir_date, nscan=None, para=None, trim_std=None):
    '''Get DataFrame
    dir_date: directory of a date where scan data is stored (str)
    nscan: list of scan number(int)
    para_list: list of parameters(str).No need to write the full name.
    '''
    path = get_scan_path(dir_date, nscan)
    df = get_data_from_path(path, para)

    #parameters to consider getting rid of outliers...(don't consider scan)
    para_vals = list(df.columns)
    if 'scan' in para_vals:
        para_vals.remove('scan')
    if 'DateTime Timestamp' in para_vals:
        para_vals.remove('DateTime Timestamp')
    if 'Shotnumber' in para_vals:
        para_vals.remove('Shotnumber')

    #get rid of outliers
    if trim_std:
        df_new = df_outlier2none(df, std=trim_std, columns = para_vals )
    return df

def get_files_list(dirpath,f_format):
    """
    get get path of all files with f_format in the directory
    dir_date: directory path
    f_format: ex) txt
    """
    return sorted(glob.glob(dirpath+'/*.'+f_format))

def get_notebook_name():
    """
    Return the full path of the jupyter notebook.
    """
    import ipykernel
    import requests
    from requests.compat import urljoin
    from notebook.notebookapp import list_running_servers

    kernel_id = re.search('kernel-(.*).json',
                          ipykernel.connect.get_connection_file()).group(1)
    servers = list_running_servers()
    for ss in servers:
        response = requests.get(urljoin(ss['url'], 'api/sessions'),
                                params={'token': ss.get('token', '')})
        for nn in json.loads(response.text):
            if nn['kernel']['id'] == kernel_id:
                relative_path = nn['notebook']['path']
                return os.path.join(ss['notebook_dir'], relative_path)

def save_dataframe(df, name, ipynb = None):
    '''save dataframe under data/"current ipython name"/'''

    #get the file name of ipynb
    if ipynb == None:
        ipynb_fullpath = get_notebook_name()
        ipynb = os.path.splitext(os.path.basename(ipynb_fullpath))[0]

    #Open the data folder if doesnt exist
    if not os.path.exists('data_ipynb'):
        os.makedirs('data_ipynb')
    if not os.path.exists('data_ipynb/'+ipynb):
        os.makedirs('data_ipynb/'+ipynb)
    #Save data
    df.to_pickle('data_ipynb/'+ipynb+'/'+name+'.pkl')
    print(name+' saved')
    return None

def load_dataframe(name, ipynb = None):
    """load dataframe which was saved using the function save_dataframe
    name: correspons to the name of the daframe you sppecified with save_dataframe
    ipynb: the ipynb name you are running. If None, it will be automatically aquired. (NOt working sometime).
    """
    #get the file name of ipynb
    if ipynb == None:
        ipynb_fullpath = get_notebook_name()
        ipynb = os.path.splitext(os.path.basename(ipynb_fullpath))[0]
    load_path = 'data_ipynb/'+ipynb+'/'+name+'.pkl'

    df = pd.read_pickle(load_path)
    print(name+' loaded')
    return df

def get_data_from_path(path_list, para_list = None):
    '''Get DataFrame from the file.
    path_list: a filename or list of multiple filenames. they will append all data sets.
    para_list: list of parameters (column names) you want to select from dataframe
    output: dataframe
    '''
    data_list = []
    for i in range(len(path_list)):
        data_i = pd.read_csv(path_list[i], sep='\t')
        if para_list:
            #get full name of the parameters
            para_list_full = []
            for j in para_list:
                para_full = par_full(path_list[i], j)
                if para_full:
                    para_list_full = para_list_full+[para_full]
            #If you can get all parameters, append the data of the scan
            if len(para_list_full) == len(para_list):
                data_i = data_i[para_list_full]
                data_list.append(data_i)
            else:
                print('Skip saving data from', os.path.basename(path_list[i]))
        else:
            #if there is no para_list, get all the parameters that are saved
            data_list.append(data_i)

    data = pd.concat(data_list, sort=False)

    #rename column names to alias if exists
    for col in data.columns:
        if 'Alias:' in col:
            alias = col.split('Alias:', 1)[1]
            data = data.rename(columns={col:alias})
    return data

def get_nscan_last(dir_date):
    '''Get the last scan number which is already done'''
    path = dir_date + '\\analysis'
    if not os.path.isdir(path):
        return 0
    else:
        # get last scan info file name
        files = glob.glob(path + '\\s*info.txt')
        file_last = os.path.basename(files[-1])
        # regexp. find number in the file name
        n_scans = int(re.findall(r"\d+", file_last)[0])
        return n_scans

def get_scan_path(dir_date, nscan=None):
    '''
    Get a path of the scan file s**.txt in the analysis
    nscan: List or int of scan number. if None, creat a list of all scan text paths
    '''

    #if nscan_list=None, make a list of all scan #s
    if not nscan:
        nscan_last = get_nscan_last(dir_date)
        nscan_list = range(1, nscan_last+1)
    elif isinstance(nscan, int):
        nscan_list = [nscan]
    else:
        nscan_list = nscan
    path_list = []
    #make a list of all scan file paths
    for i in nscan_list:
        path = dir_date + '\\analysis\\s' + str(i) + '.txt'
        path_list = path_list + [path]
    return path_list

def par_full(file, par):
    '''get a full name of the parameter'''

    data = pd.read_csv(file, sep='\t')
    indices = [k for k, s in enumerate(list(data)) if par in s]
    if not indices or data.empty:
        print(par, 'not found in', os.path.basename(file))
        return None
    elif len(indices) > 1:
        for j in indices:
            if list(data)[j]==par:
                return list(data)[j]
        raise NameError('Please Specify the Name. Several parameters match for ',par,list( list(data)[i] for i in indices )  )
        return None
    else:
        return list(data)[indices[0]]

def show_time_xaxis():
    '''X axis changed from timestamp to day time of california when you plot a graph with time stamp on x axis.
    '''
    from datetime import datetime
    summer20_start = datetime.timestamp(datetime(2020,3,8,3,0))
    summer20_end = datetime.timestamp(datetime(2020,11,1,2,0))
    # get current axis
    ax = plt.gca()
    # get current xtick labels
    xticks = ax.get_xticks()
    if xticks[0] > summer20_start and xticks[0] <summer20_end:
        xticks = xticks - 7*60*60
    else:
        xticks = xticks - 8*60*60
    # convert all xtick labels to selected format from ms timestamp
    ax.set_xticklabels([pd.to_datetime(tm, unit='s').strftime('%Y-%m-%d\n %H:%M:%S') for tm in xticks],
     rotation=50)
    return None

def get_calib(Dict, device, axis = None):
    '''Get paramters from calibration dictionary
    device: device name except axis (drop off the X or Y in the device name)'''

    if device in Dict:
        #get target position
        try: target = Dict[device]['target'][axis]
        except: target = 0

        #get sign (positive or negative)
        try: sign = Dict[device]['sign'][axis]
        except: sign = 1

        #get calibration
        try: calib = Dict[device]['calib'][axis]
        except:
            try: calib = Dict[device]['calib']
            except: calib = 1

        #get unit
        try: unit = ' ('+Dict[device]['unit']+')'
        except: unit = ''

        return target, sign, calib, unit

    #if 2ndmomW0, get calibration data from centroid data
    elif device.split()[1]=='2ndmomW0' or '2ndmom' or '2mdmom' or 'FWHM':
        device_centroid = device.split()[0]+' centroid'
        return get_calib(Dict, device_centroid)

    else:
        print('can not find a calibration for ', device)
        return None, None, None, None

def PT_time(lvtimestamp):
    '''
    Conert the labview timestamp to pacific time.
    lvtimestamp: labview timestamp (float). should be 10 digit (36...)
    "'''
    lv_dt = datetime.fromtimestamp(lvtimestamp) #labview time
    utc_dt = lv_dt - relativedelta(years=66, days=1) #UTC time
    #convert to Pacific time
    ca_tz = timezone('America/Los_Angeles')
    ca_date = utc_dt.astimezone(ca_tz)

    return ca_date

def df_calib(df, Dict):
    '''Return a new dataframe with calibrated values. Only those with calibration saved in Dict are stored in the
    output dataframe.
    df: DataFrame
    Dict: calibration dictionary. It's in a separate file
    '''
    #parameters of the DataFrame
    para = df.columns
    #define a new dataframe
    df_new = pd.DataFrame()

    #get device name and axis for the camera
    for i in para:
        #if camera device
        if i[:3]=='UC_':
            #if add column is used, strip ..T..., and find axis
            if i.find(' T ') > -1:
                axis = i[i.find(' T ')-1]
                device = i[:i.find(' T ')-1].rstrip() #get rid of space in the end if exist
            #if the name ends with x or y
            elif i[-1]=='x' or i[-1]=='y':
                axis = i[-1]
                device=i[:-1]
            else:
                axis=''
                device = i
                print('cant identify the parameter ',i)

            #get the calibration and convert to the calibrated data
            target, sign, calib, unit = get_calib(Dict, device, axis)
            para_new = device[3:]+axis+unit
            df_new[para_new]=sign*(df[i]-target)*calib

        #non camera device but exists in calibration dictionary
        elif i in Dict:
            if i[:2]=='U_':
                df_new[i[2:]] = df[i]
            else:
                target, sign, calib, unit = get_calib(Dict, i)
                df_new[i+unit] = sign*(df[i]-target)*calib

        #if it's timestamp
        elif i=='DateTime Timestamp':
            df_new['DateTime (Pacific Time)'] = PT_time(df[i])

        else:
            print('can not find a calibration data for ', para[i])

    if len(df_new.columns) < len(para):
        print( len(para) - len(df_new.columns), 'parameters not saved to the calibrated dataframe' )
    return df_new


"""
@ Author: ryanreadbooks
@ Time: 9/4/2020, 19:51
@ File name: metrics.py
@ File description: some metric functions for calculating the scores
"""

import math
from typing import Tuple

import numpy as np
from utils import project_3d_2d, transform_pts
import scipy.spatial as scipy_spatial

def mask_iou(pred: np.ndarray, truth: np.ndarray) -> float:
	"""
	Calculate the iou between two mask
	:param pred: the predicted mask, shape (h, w), with the value of masked region to be  1, background to be value 0
	:param truth: the ground truth mask, shape (h, w), with the value of masked region to be 1, background to be value 0
	:return: iou between these two masks
	"""
	intersection = np.logical_and(truth, pred)
	union = np.logical_or(truth, pred)
	iou = np.sum(intersection) / np.sum(union)
	return iou

def mask_miou(pred: np.ndarray, truth: np.ndarray) -> float:
	"""
	Calculate the mean IoU between masks
	:param pred: the predicted mask, shape (n, h, w)
	:param truth: the ground truth mask, shape (n, h ,w)
	:return: the mean IoU
	"""
	intersection: np.ndarray = np.logical_and(truth, pred).sum(axis=1).sum(axis=1)
	union: np.ndarray = np.logical_or(truth, pred).sum(axis=1).sum(axis=1)
	ious: np.ndarray = intersection / union
	return ious.mean()

def calculate_add(pred_pose: np.ndarray, gt_pose: np.ndarray, points: np.ndarray) -> float:
	r"""
	Calculate the ADD metric. This metrics is more strict to the predicted pose
	:math:`ADD = \sum[(R * x + T) - (R_p * x + T_p)] / m`

	:param pred_pose: the predicted pose, array with shape (3, 4), [R|t]
	:param gt_pose: the ground truth pose, array with shape (3, 4), [R|t]
	:param points: the model points to be transformed, array with shape (n, 3), n is the number of points
	:return: the ADD value
	"""
	points_pred: np.ndarray = transform_pts(points, pred_pose)
	points_gt: np.ndarray = transform_pts(points, gt_pose)

	return np.linalg.norm(points_gt - points_pred, axis=1).mean()

def calculate_add_s(pred_pose: np.ndarray, gt_pose: np.ndarray, points: np.ndarray) -> float:
	r"""
	Calculate the ADD-S metric. This metric is less strict than ADD metric
	:math:`ADD-S = \sum_{x1 \in {M}}{min_{x2 \in{M}} (R * x1 + T)-(R_p * x2 + T_p)} / m`

	:param pred_pose: the predicted pose, array with shape (3, 4), [R|t]
	:param gt_pose: the ground truth pose, array with shape (3, 4), [R|t]
	:param points: the model points to be transformed
	:return: the ADD-S value
	"""

	points_pred: np.ndarray = transform_pts(points, pred_pose)
	points_gt: np.ndarray = transform_pts(points, gt_pose)
	kdtree: scipy_spatial.cKDTree = scipy_spatial.cKDTree(points_pred)
	query_result: Tuple[np.ndarray, np.ndarray] = kdtree.query(points_gt)  # query result: distances, index at points_pred
	nearest_distances = query_result[0]
	add_s: float = nearest_distances.mean()

	return add_s

def rotation_error(r_pred: np.ndarray, r_gt: np.ndarray) -> float:
	"""
	calculate the rotation error between two rotation matrix.
	original implementation from https://github.com/thodan/bop_toolkit/blob/master/bop_toolkit_lib/pose_error.py
	math is from Rodrigue's formula.
	:param r_pred: the predicted rotation matrix, shape of (3, 3)
	:param r_gt: the ground truth rotation matrix, shape of (3, 3)
	:return: the calculated error between them
	"""
	assert (r_pred.shape == r_gt.shape == (3, 3)), 'ground truth and predicted value must be of the same shape (3, 3)'
	error_cos = float(0.5 * (np.trace(r_pred.dot(np.linalg.inv(r_gt))) - 1.0))

	# Avoid invalid values due to numerical errors.
	error_cos = min(1.0, max(-1.0, error_cos))

	error = math.acos(error_cos)
	error = 180.0 * error / np.pi  # Convert [rad] to [deg].
	return error

def translation_error(t_pred: np.ndarray, t_gt: np.ndarray) -> float:
	"""
	calculate the translation error between two translation vector
	original implementation from https://github.com/thodan/bop_toolkit/blob/master/bop_toolkit_lib/pose_error.py
	:param t_pred: predicted translation vector, size of 3
	:param t_gt: ground truth translation vector, size of 3
	:return: the computed translation error
	"""
	assert (t_gt.size == t_pred.size == 3), 'invalid size for translation vector'
	error = np.linalg.norm(t_gt - t_pred)
	return error

def projection_error(pts_3d: np.ndarray, camera_k: np.ndarray, pred_pose: np.ndarray, gt_pose: np.ndarray):
	"""
	Average distance of projections of object model vertices [px]
	:param pts_3d: model points, shape of (n, 3)
	:param camera_k: camera intrinsic matrix, shape of (3, 3)
	:param pred_pose: predicted rotation and translation, shape (3, 4), [R|t]
	:param gt_pose: ground truth rotation and translation, shape (3, 4), [R|t]
	:return: the returned error, unit is pixel
	"""
	# projection shape (n, 2)
	pred_projection: np.ndarray = project_3d_2d(pts_3d=pts_3d, camera_intrinsic=camera_k, transformation=pred_pose)
	gt_projection: np.ndarray = project_3d_2d(pts_3d=pts_3d, camera_intrinsic=camera_k, transformation=gt_pose)
	error = np.linalg.norm(gt_projection - pred_projection, axis=1).mean()
	return error

def check_pose_correct(val1: float, threshold1: float, metric: str, diameter: float = 0., val2: float = 0., threshold2: float = 0.):
	"""
	Check if the pose_pred should be considered correct with respect to pose_gt based on the metric
	:param val1: value to check
	:param threshold1: threshold that determine the values are correct or not
	:param metric: criteria, add, add-s, projection error, 5cm5°
	:param diameter: in add(-s), you need diameter of the object
	:param val2: optional argument for 5cm5° metric
	:param threshold2: optional argument for 5cm5° metric
	:return: pose_pred should be considered correct of not. correct -> True, incorrect -> False
	"""
	if metric == 'add' or metric == 'add-s':
		assert diameter != 0
		if val1 <= threshold1 * diameter:
			return True
		else:
			return False
	elif metric == '5cm5':
		if val1 <= threshold1 and val2 <= threshold2:
			return True
		else:
			return False
	elif metric == 'projection':
		if val1 < threshold1:
			return True
		else:
			return False


<reponame>PiotrBenedysiuk/qucat<filename>tests/test_core.py
import sys
import os
sys.path.append(os.path.join(os.path.join(os.path.dirname(os.path.dirname(__file__)),'src')))
import unittest
import core
from math import isclose
import numpy as np
from scipy.constants import e, pi, h, hbar

# Run plt.ion() to avoid hanging on plt.show() calls
import matplotlib.pyplot as plt
plt.ion()

def cutoff_digits(f,digits):
    float_format = '%%.%de'%digits
    return float(float_format%(np.real(f))) + 1j*float(float_format%(np.imag(f)))

class TestCaseAppended(unittest.TestCase):

    def assertRelativelyClose(self,a,b,digits = 6):
        a = cutoff_digits(a,digits)
        b = cutoff_digits(b,digits)
        self.assertEqual(a,b)

    def assertArrayRelativelyClose(self,a,b,digits = 6):
        a = np.array(a)
        b = np.array(b)
        self.assertTrue(a.shape==b.shape,msg = f'Arrays do not have the same dimension {a.shape}!={b.shape}')
        for index,_ in np.ndenumerate(a):
            a_comp = cutoff_digits(a[index],digits)
            b_comp = cutoff_digits(b[index],digits)
            self.assertEqual(a_comp,b_comp,
                    msg = f'Components with index {index} do not match {a_comp}!={b_comp}')

    def open_gui_file(self,filename, edit = False, print_network=False,plot=False):
        return core.GUI(os.path.join(
            os.path.dirname(__file__),
            'gui_testing_files',
            filename),
            edit = edit, print_network=print_network,plot=plot)

class SeriesRLC(TestCaseAppended):
    '''
    Series RLC circuit parameters
    '''

    def parameters(self,R,L,C):
        circuit = core.Network([
            core.C(0,1,C),
            core.L(1,2,L),
            core.R(0,2,R)
        ])
        return circuit.f_k_A_chi()

    def test_frequency(self):
        C = 100e-15
        L = 10e-9
        R = 100e-9
        w,k,A,chi = self.parameters(R,L,C)
        cpx_w = (1j*C*R + np.sqrt(4*C*L - C**2*R**2))/(2.*C*L)
        self.assertRelativelyClose(np.real(cpx_w)/2/np.pi,w)

    def test_frequency2(self):
        C = 1
        L = 3
        R = 0.5
        w,k,A,chi = self.parameters(R,L,C)
        cpx_w = (1j*C*R + np.sqrt(4*C*L - C**2*R**2))/(2.*C*L)
        self.assertRelativelyClose(np.real(cpx_w)/2/np.pi,w)

    def test_dissipation(self):
        C = 100e-15
        L = 10e-9
        R = 100e-9
        w,k,A,chi = self.parameters(R,L,C)
        cpx_w = (1j*C*R + np.sqrt(4*C*L - C**2*R**2))/(2.*C*L)
        self.assertRelativelyClose(np.imag(cpx_w)/2/np.pi,k)

class Other(TestCaseAppended):
    def test_LC_double_series_L_double_series_C(self):
        C = 1e-8
        L = 3
        circuit = core.Network([
            core.C(0,1,C*2),
            core.C(1,2,C*2),
            core.L(2,3,L/2),
            core.L(3,0,L/2)
        ])
        f,k,A,chi = circuit.f_k_A_chi()
        f_expected = 1/np.sqrt(L*C)/2/np.pi
        self.assertRelativelyClose(f_expected,f)

class Transmon(TestCaseAppended):
    '''
    Transmon circuit parameters
    '''

    def parameters(self,C,Lj):
        circuit = core.Network([
            core.C(0,1,C),
            core.J(0,1,Lj)
        ])
        return circuit.f_k_A_chi()

    def test_frequency(self):
        C = 100e-15
        Lj = 10e-9
        w,k,A,chi = self.parameters(C,Lj)
        self.assertRelativelyClose(1/(np.sqrt(C*Lj)*2.*pi),w)

    def test_anharmonicity(self):
        C = 100e-15
        Lj = 10e-9
        w,k,A,chi = self.parameters(C,Lj)
        self.assertRelativelyClose(e**2/2./C/h,A[0])

    def test_phi_zpf(self):
        Cj = 100e-15
        Lj = 10e-9
        junction = core.J(0,1,Lj)
        circuit = core.Network([
            core.C(0,1,Cj),
            junction,
            core.R(0,1,1e6)
        ])
        phi_0 = hbar/2/e
        Z = np.sqrt(Lj/Cj)
        phi_zpf = np.sqrt(hbar*Z/2)
        self.assertRelativelyClose(phi_zpf/phi_0,junction.zpf(mode=0,quantity = 'flux'))

    def test_q_zpf(self):
        Cj = 100e-15
        Lj = 10e-9
        junction = core.J(0,1,Lj)
        circuit = core.Network([
            core.C(0,1,Cj),
            junction,
            core.R(0,1,1e6)
        ])
        Z = np.sqrt(Lj/Cj)
        q_zpf = np.sqrt(hbar/Z/2)
        self.assertRelativelyClose(q_zpf/e,np.absolute(junction.zpf(mode=0,quantity = 'charge')))

    def test_anharmonicity_using_hamiltonian(self):
        Cj = 1e-10
        circuit = core.Network([
            core.C(0,1,Cj),
            core.J(0,1,10e-9)
        ])
        H = circuit.hamiltonian(modes = [0],taylor = 4,excitations = [10])
        ee = H.eigenenergies()
        A = np.absolute((ee[1]-ee[0])-(ee[2]-ee[1]))
        # Due to higher order terms, the mismatch with e**2/2/Cj/h is
        # (193702.3+0j) != (194712.7+0j)
        A_expected = 194712.7
        self.assertRelativelyClose(A_expected,A)

    def test_double_series_capacitor(self):
        C = 100e-15
        Lj = 10e-9
        circuit = core.Network([
            core.C(0,1,C*2),
            core.C(1,2,C*2),
            core.J(0,2,Lj)
        ])
        f,k,A,chi = circuit.f_k_A_chi()
        self.assertArrayRelativelyClose([e**2/2./C/h,1/(np.sqrt(C*Lj)*2.*pi)],[A[0],f[0]])

class ShuntedJosephsonRing(TestCaseAppended):
    '''
    Shunted Josephson ring
    '''

    def parameters(self,C,L):
        circuit = core.Network([
            core.C(0,2,C),
            core.C(1,3,C),
            core.J(0,1,L),
            core.J(1,2,L),
            core.J(2,3,L),
            core.J(3,0,L)
        ])
        return circuit.f_k_A_chi()

    def test_number_of_modes_nHz(self):
        C = 2.e9
        L = 3.e11
        w,k,A,chi = self.parameters(C,L)
        self.assertEqual(len(w),2,msg = f"f_res = {w}")
    def test_number_of_modes_Hz(self):
        C = 2.
        L = 3.
        w,k,A,chi = self.parameters(C,L)
        self.assertEqual(len(w),2,msg = f"f_res = {w}")

    def test_number_of_modes_GHz(self):
        C = 1e-13
        L = 1e-8
        w,k,A,chi = self.parameters(C,L)
        self.assertEqual(len(w),2,msg = f"f_res = {w}")

    def test_frequency_0(self):
        C = 1e-13
        L = 1e-8
        w,k,A,chi = self.parameters(C,L)
        self.assertRelativelyClose(w[0],1/np.sqrt(L*C)/2./np.pi)

    def test_frequency_1(self):
        C = 1e-13
        L = 1e-8
        w,k,A,chi = self.parameters(C,L)
        self.assertRelativelyClose(w[1],1/np.sqrt(L*C)/2./np.pi)

    def test_anharmonicity_0(self):
        C = 1e-13
        L = 1e-8
        w,k,A,chi = self.parameters(C,L)
        self.assertRelativelyClose(A[0],e**2/2./(8*C)/h)

    def test_anharmonicity_1(self):
        C = 1e-13
        L = 1e-8
        w,k,A,chi = self.parameters(C,L)
        self.assertRelativelyClose(A[1],e**2/2./(8*C)/h)

class CoupledTransmonRLC(TestCaseAppended):
    '''
    Coupled transmon/RLC
    '''
    def test_sweeping_LJ_in_fkAchi(self):
        cir = core.Network([
            core.C(0,1,100e-15),
            core.J(0,1,'L_J'),
            core.C(1,2,1e-15),
            core.C(2,0,100e-15),
            core.L(2,0,10e-9),
            core.R(2,0,1e6)
            ])
        [cir.f_k_A_chi(L_J=x) for x in [1e-9,2e-9]]

class TestGraphics(TestCaseAppended):

    def test_error_when_trying_to_plot_from_Network_show(self):
        circuit = core.Network([
            core.C(0,1,'C'),
            core.J(0,1,'Lj')
        ])
        with self.assertRaises(TypeError):
            circuit.show()

    def test_error_when_trying_to_plot_from_Network_show_normal_modes(self):
        circuit = core.Network([
            core.C(0,1,'C'),
            core.J(0,1,'Lj')
        ])
        with self.assertRaises(TypeError):
            circuit.show_normal_mode()

    def test_generate_graphics(self):
        import _generate_graphics

    def test_show_transmon_RLC(self):
        cir=self.open_gui_file('show_normal_mode_transmon_RLC_Lj_as_parameter.txt')
        cir.show()

    def test_show_normal_mode_transmon_RLC_Lj_as_parameter(self):
        cir=self.open_gui_file('show_normal_mode_transmon_RLC_Lj_as_parameter.txt')
        for quantity in ['flux','voltage','charge','current']:
            cir.show_normal_mode(0,quantity,Lj=1e-9)

class TestNetworkAnalysis(TestCaseAppended):

    def test_transfer_left_right_port_identical(self):
        '''
        Trivial cases
        '''
        net = core._Network([core.R(0,1,'Z2')])
        self.assertEqual(net.transfer(0,1,0,1),1)

    def test_transfer_left_right_port_indentical_inverted(self):
        '''
        Trivial cases
        '''
        net = core._Network([core.R(0,1,'Z2')])
        self.assertEqual(net.transfer(0,1,1,0),-1)

    def test_transfer_voltage_divider(self):
        '''
        Voltage divider, see:
        https://en.wikipedia.org/wiki/Voltage_divider
        We add an extra resistor between Vin and ground
        '''

        # Compute the bridge transfer function
        net = core._Network([
            core.R(0,1,'Z2'),
            core.R(1,2,'Z1'),
            core.R(2,0,'Zg'),
            ])
        transfer = net.transfer(0,2,0,1)

        # What the tranfer function should be
        def transfer_theory(Z1,Z2,Zg):
            return Z2/(Z1+Z2)

        # Define some numerical values
        # for the resistors
        test_parameters = {
            'Z1':1,
            'Z2':2,
            'Zg':3
        }

        self.assertRelativelyClose(
            transfer.evalf(subs = test_parameters),
            transfer_theory(**test_parameters))

    def test_transfer_wheatstone(self):
        '''
        Wheatstone bridge, see:
        https://en.wikipedia.org/wiki/Wheatstone_bridge
        Note that the correct definitions of the voltages V_G and V_S can be found here:
        http://www.ece.lsu.edu/ee4770/1999/lsli04.4up.pdf
        '''

        # Compute the bridge transfer function
        net = core._Network([
            core.R(0,1,'R_3'),
            core.R(1,2,'R_x'),
            core.R(2,3,'R_2'),
            core.R(3,0,'R_1'),
            ])
        transfer = net.transfer(0,2,3,1)

        # What the tranfer function should be
        def transfer_theory(R_1,R_2,R_3,R_x):
            return R_2/(R_1+R_2)-R_x/(R_x+R_3)

        # Define some numerical values
        # for the bridge resistors
        test_parameters = {
            'R_1':1,
            'R_2':2,
            'R_3':3,
            'R_x':4,
        }

        self.assertRelativelyClose(
            transfer.evalf(subs = test_parameters),
            transfer_theory(**test_parameters))

    def test_transfer_wheatstone_all_equal(self):
        '''
        Case where all resistors of the bridge are equal, and
        there is potential for these lines:
            A_lattice = (Ya + Yb)*(Yd + Yc)/(Ya*Yd-Yb*Yc)
            B_lattice = (Ya + Yb + Yc + Yd)/(Ya*Yd-Yb*Yc)
        to raise an error.
        Note: I made sure with a break-point that this code
        actually calls those two lines.
        '''

        # Compute the bridge transfer function
        net = core._Network([
            core.R(0,1,'R_3'),
            core.R(1,2,'R_x'),
            core.R(2,3,'R_2'),
            core.R(3,0,'R_1'),
            ])
        transfer = net.transfer(0,2,3,1)

        # Define some numerical values
        # for the bridge resistors
        test_parameters = {
            'R_1':1,
            'R_2':1,
            'R_3':1,
            'R_x':1,
        }

        # We just run this to check wether it raises an error
        transfer.evalf(subs = test_parameters)

    def test_transfer_square_symmetrical_lattice(self):
        '''
        Square symmetrical lattice
        '''

        # Compute the transfer function
        left_minus = 0
        left_plus = 1
        right_minus = 2
        right_plus = 3
        net = core._Network([
            core.R(left_minus,left_plus,'Zs'),
            core.R(right_minus,right_plus,'Zl'),
            core.Admittance(left_minus,right_plus,0),
            core.Admittance(right_minus,left_plus,0),
            core.R(left_minus,right_minus,'Za'),
            core.R(right_plus,left_plus,'Za')
            ])
        transfer = net.transfer(left_minus,left_plus,right_minus,right_plus)

        # What the tranfer function should be
        def transfer_theory(Zs,Zl,Za):
            return Zl/(2*Za+Zl)

        # Define some numerical values
        # for the resistors
        test_parameters = {
            'Zs':1,
            'Zl':2,
            'Za':3,
        }

        self.assertRelativelyClose(
            transfer.evalf(subs = test_parameters),
            transfer_theory(**test_parameters))

    def test_open_or_series_check(self):
        with self.assertRaises(ValueError):
            net = core._Network([
                core.R(0,1,'Z'),
                core.C(1,2,'Z'),
                core.J(2,3,'Z'),
                ])

    def test_connectivity_check_single_element_not_connected(self):
        with self.assertRaises(ValueError):
            net = core._Network([
                core.R(0,1,'Z'),
                core.C(1,2,'Z'),
                core.J(3,4,'Z'),
                ])
    def test_connectivity_check_subcircuit_not_connected(self):
        with self.assertRaises(ValueError):
            net = core._Network([
                core.R(0,1,'Z'),
                core.C(1,2,'Z'),
                core.J(3,4,'Z'),
                core.J(3,4,'Z'),
                core.J(4,5,'Z'),
                ])

if __name__ == "__main__":
    unittest.main()


<gh_stars>1-10
#!/usr/bin/env python
# coding: utf-8

# # MCMC Comparison
#
# ## surmise
#
# 1. Train a GP (using surmise) to emulate the $S$-factor.
# 2. Use that GP to calculate the $R$-matrix parameter posteriors.
# 3. Calculate the $R$-matrix parameter posteriors using the $R$-matrix prediction *directly*.
#
# * range of energies
# * use parameter samples near the true values *exclusively*
# * use increasingly wider distributions to generate training samples
#     * If we use flat distributions, the results are crap. With narrow (10%) Gaussians, the results are significantly better. As the Gaussians get wider, we expect to recover our original bad results. Where does this happen?
# * use fewer training points and see how things change

import os
import sys
import subprocess

import numpy as np
import matplotlib.pyplot as plt
from corner import corner
import seaborn as sns
from scipy import stats
from surmise.emulation import emulator
import emcee
import h5py
from corner import corner

from dt import cross_section, s_factor
import constants as const

rel_unc = float(sys.argv[1])
nt = int(sys.argv[2])
ntrain = 250

plt.style.use('science')
gr = (1 + np.sqrt(5)) / 2
h = 3
plt.rcParams['figure.dpi'] = 300

# Our physics model.
def f(energy, theta):
    er, gd2, gn2 = theta
    return s_factor(energy, er, er, gd2, gn2, const.AD, const.AN, const.UE, const.A)

parameter_labels = [r'$E_r$', r'$\gamma_d^2$', r'$\gamma_n^2$']

train = np.loadtxt(f'datfiles/better_training_data_{rel_unc:.2f}.txt')
test = np.loadtxt(f'datfiles/better_testing_data_{rel_unc:.2f}.txt')

ns, nd = train.shape
nk = np.unique(train[:, 0]).size # number of momenta

x = train[::ntrain, 0].reshape(-1, 1) # input/location/momentum
w = train[:nt, 1:4] # parameters that we want to emulate

y = train[:, -1].reshape(nk, ntrain) # output/cross section
y = y[:, :nt]

# Set up the surmise emulator.
args = {'epsilon' : 0.0, 'warnings' : True}
# args = {'epsilon': 0.1, ‘hypregmean’: -8}
emu = emulator(x=x, theta=w, f=y, method='PCGPwM', args=args)

bounds = np.copy(const.BOUNDS)
priors = [stats.uniform(b[0], b[1]-b[0]) for b in bounds]

energies, data, data_err = np.loadtxt('datfiles/data.txt', unpack=True)
theta_true = np.loadtxt('datfiles/theta_true.txt')[:3]

def ln_prior(theta):
    return np.sum([p.logpdf(t) for (p, t) in zip(priors, theta)])

class Model:
    def __init__(self, predict):
        self.predict_func = predict

    def ln_likelihood(self, theta):
        mu, var = self.predict_func(theta)
        var_tot = data_err**2 + var
        return np.sum(-np.log(np.sqrt(2*np.pi*var_tot)) - (data - mu)**2 / var_tot)

    def ln_posterior(self, theta):
        lnpi = ln_prior(theta)
        if lnpi == -np.inf:
            return -np.inf
        return lnpi + self.ln_likelihood(theta)

def gp_func(theta):
    p = emu.predict(theta=theta)
    mu = p.mean().T[0]
    var = p.var().T[0]
    return mu, var

def r_func(theta):
    return np.array([f(ei, theta) for ei in energies]), np.zeros(energies.size)

model_gp = Model(gp_func)
model_r = Model(r_func)

nd = 3
nw = 2*nd

p0 = np.array(
    [stats.norm(theta_true, 0.01*np.abs(theta_true)).rvs() for _ in range(nw)]
)

moves = [(emcee.moves.DEMove(), 0.2), (emcee.moves.DESnookerMove(), 0.8)]
sampler_gp = emcee.EnsembleSampler(nw, nd, model_gp.ln_posterior, moves=moves)
sampler_r = emcee.EnsembleSampler(nw, nd, model_r.ln_posterior, moves=moves)

state = sampler_gp.run_mcmc(p0, 1000, thin_by=10, tune=True, progress=True)
state = sampler_r.run_mcmc(p0, 1000, thin_by=10, tune=True, progress=True)

fig, ax = plt.subplots(ncols=2, figsize=(2*gr*h, h))
fig.patch.set_facecolor('white')

nb_gp = 100
nb_r = nb_gp

ax[0].plot(sampler_gp.get_log_prob(discard=nb_gp))
ax[1].plot(sampler_r.get_log_prob(discard=nb_r));

flat_chain_gp = sampler_gp.get_chain(flat=True, discard=nb_gp)
flat_chain_r = sampler_r.get_chain(flat=True, discard=nb_r)

# Corner plots

fig = corner(flat_chain_gp, labels=parameter_labels, show_titles=True, quantiles=[0.16, 0.5, 0.84], color='C0')
fig = corner(flat_chain_r, color='C1', fig=fig, truths=theta_true, truth_color='C3')
# fig = corner(w, color='C2', fig=fig)
fig.patch.set_facecolor('white')
plt.savefig(f'figures/corner_{rel_unc:.2f}_{nt}.pdf')

# Chains
fig, ax = plt.subplots(3, figsize=(gr*h, 3*h))
fig.patch.set_facecolor('white')

for i in range(3):
    ax[i].plot(flat_chain_gp[:, i], label='GP')
    ax[i].plot(flat_chain_r[:, i], label='R-matrix')
#     ax[i].plot(w[:, i], label='Train')
    low_68, high_68 = np.quantile(w[:, i], [0.16, 0.84])
    low_95, high_95 = np.quantile(w[:, i], [0.05, 0.95])
    ax[i].fill_between(np.arange(flat_chain_gp[:, i].size), low_68, high_68, color='C2', alpha=0.5)
    ax[i].fill_between(np.arange(flat_chain_gp[:, i].size), low_95, high_95, color='C2', alpha=0.25)
    ax[i].set_ylabel(parameter_labels[i])
    ax[i].axhline(theta_true[i], color='C3', label='True')
    ax[i].legend()

plt.savefig(f'figures/chains_{rel_unc:.2f}_{nt}.pdf')


<reponame>PerryXDeng/adversarial_mnist_attacks
import neural_network_configuration as conf
import numpy as np
import scipy.special as ss

def sigmoid(z):
  """
  sigmoid/softmax activation, uses ss.expit to avoid numeric instability
  :param z: vectorized values to be activated
  :return: vectorized activated values
  """
  return ss.expit(z)

def cross_entropy(h, y):
  """
  cross entropy cost function
  :param h: vectorized hypothesis value
  :param y: vectorized actual value
  :return: cross entropy loss
  """
  return -np.sum(y * np.log(h+1e-6))

def activation(prev, weights, bias):
  """
  a neural network layer
  :param prev: activations of previous layer
  :param weights: weights of this layer
  :param bias: biases of this layer
  :return: activations of this layer
  """
  prev_copy = np.r_[np.ones(prev.shape[1])[np.newaxis], prev]
  weights_copy = np.c_[bias, weights]
  return sigmoid(np.matmul(weights_copy, prev_copy))

def feed_forward(x, weights, bias):
  """
  feeds forward the input through the neural network
  :param x: input
  :param weights: weights
  :param bias: biases
  :return: activations for all the layers (last layer is the output)
  """
  # activation value matrices of the two twin networks and the joined network
  activations = np.ndarray(conf.LAYERS_NUM, dtype=np.matrix)

  # transposing horizontal input vectors (or matrices) into feature vectors
  if len(x.shape) == 1:
    activations[0] = x[np.newaxis].T
  else:
    activations[0] = x.T

  # forward propagation
  for i in range(1, conf.LAYERS_NUM):
    activations[i] = activation(activations[i - 1], weights[i - 1], bias[i - 1])

  return activations

def regularize(weights, cost, gradients):
  """
  regularize the gradients and costs
  :param weights: weights
  :param cost: cost function value
  :param gradients: gradients value
  :return: regularized cost and gradietns
  """
  for n in range(1, conf.LAYERS_NUM):
    weights_without_bias = np.c_[(np.zeros(weights[n - 1].shape[0]),
                                  weights[n - 1])]
    regularization_offset = conf.REG_CONST * weights_without_bias
    gradients[n - 1] += regularization_offset
    cost += conf.REG_CONST * np.sum(np.multiply(weights[n - 1], weights[n - 1])) / 2
  return cost, gradients

def cost_derivatives(x, y, weights, bias):
  """
  gets the gradients of cost w.r.t. biases & weights
  :param x: input
  :param y: vectorized labels
  :param weights: weights
  :param bias: biases
  :return: cost and gradients(for both biases and weights)
  """
  # zero initializes cost and gradients
  cost = np.float(0)
  transformations_derivatives = np.ndarray(conf.LAYERS_NUM - 1,
                                           dtype=np.ndarray)
  gradients = np.ndarray(conf.LAYERS_NUM - 1, dtype=np.matrix)
  for i in range(1, conf.LAYERS_NUM):
    gradients[i - 1] = np.matrix(
        np.zeros((conf.LAYERS_UNITS[i], conf.LAYERS_UNITS[i - 1] + 1)))

  # sum up the derivatives of cost for each sample
  activations = feed_forward(x, weights, bias)
  for i in range(0, x.shape[0]):
    cost += cross_entropy(activations[conf.LAYERS_NUM - 1][:, i], y[:, i])

    # backpropagate
    transformations_derivatives[conf.LAYERS_NUM - 2] = \
        activations[conf.LAYERS_NUM - 1][:, i] - y[:, i]

    for n in reversed(range(0, conf.LAYERS_NUM - 2)):
      # n is the n + 1 layer in the network
      next_layer_transforms_gradients = transformations_derivatives[
          n + 1]
      next_layer_weights = weights[n + 1]
      this_layer_activations_gradients = activations[n + 1][:, i] \
                                         * (1 - activations[n + 1][:, i])
      transformations_derivatives[n] = np.multiply(
          np.matmul(next_layer_weights.T, next_layer_transforms_gradients),
          this_layer_activations_gradients)

    # calculate gradients of weights in relation to their transformations
    for n in range(1, conf.LAYERS_NUM):
      ad = np.r_[np.ones(1), activations[n - 1][:, i]][np.newaxis]
      gradients[n - 1] += \
          np.matmul(transformations_derivatives[n - 1][np.newaxis].T, ad)

  # take their mean
  cost /= x.shape[0]
  for n in range(1, conf.LAYERS_NUM):
    gradients[n - 1] /= x.shape[0]

  return cost, gradients

def predict(x, weights=None, bias=None):
  """
  gives discrete output (labels) based on input (vectorized images)
  :param x: input
  :param weights: weights of the neural network
  :param bias: biases of the neural network
  :return: predicted labels
  """
  if weights is None:
    weights = np.ndarray(conf.LAYERS_NUM - 1, dtype=np.matrix)
    bias = np.ndarray(conf.LAYERS_NUM - 1, dtype=np.ndarray)
    weights[0] = np.load("neural_network/weights_1.npy")
    weights[1] = np.load("neural_network/weights_2.npy")
    bias[0] = np.load("neural_network/bias_1.npy")
    bias[1] = np.load("neural_network/bias_2.npy")
  h = feed_forward(x, weights, bias)
  return np.argmax(h[-1], axis=0)

def accuracy(x, labels, weights, bias):
  """
  a performance metric
  :param x: dataset inputs
  :param labels: dataset labels
  :param weights: neural network weights
  :param bias: neural network biases
  :return: accuracy between 0 and 1
  """
  out = predict(x, weights, bias)
  results = [(out[i] == labels[i])
             for i in range(x.shape[0])]
  return sum(result for result in results) / x.shape[0]

# def numerical_derivative_approximation(x, y, weights, bias, i, j, l, cost):
#   # make two copies of the weights and biases
#   weights_copy = np.ndarray(nn_conf.LAYERS_NUM - 1, dtype=np.matrix)
#   bias_copy = np.ndarray(nn_conf.LAYERS_NUM - 1, dtype=np.ndarray)
#   for n in range(1, nn_conf.LAYERS_NUM):
#     weights_copy[n - 1] = weights[n - 1]
#     bias_copy[n - 1] = bias[n - 1]
#
#   # copy and modify the weight/bias matrices at (i, j, l)
#   if j == 0:
#     new_bias = np.ndarray.copy(bias_copy[l])
#     new_bias[i] += nn_conf.NUMERICAL_DELTA
#     bias_copy[l] = new_bias
#   else:
#     new_weights = np.ndarray.copy(weights_copy[l])
#     # j - 1 due to lack of biases
#     new_weights[i][j - 1] += nn_conf.NUMERICAL_DELTA
#     weights_copy = new_weights
#   # forward propagate
#   out = feed_forward(x, weights_copy, bias_copy)
#   # calculate costs for both sets of weights
#   new_cost = cross_entropy(out[nn_conf.LAYERS_NUM - 1], y)
#   # print("numerical costs: " + str(cost_1) + ", " + str(cost_2))
#   return (new_cost - cost) / nn_conf.NUMERICAL_DELTA
#
#
# def num_approx_aggregate(x, y, weights, bias):
#   out = feed_forward(x, weights, bias)
#   cost = cross_entropy(out[nn_conf.LAYERS_NUM - 1], y)
#   weights_gradients = np.ndarray(nn_conf.LAYERS_NUM - 1, dtype=np.matrix)
#   for l in range(nn_conf.LAYERS_NUM - 1):
#     mat = np.zeros(shape=(nn_conf.LAYERS_UNITS[l + 1], nn_conf.LAYERS_UNITS[l] + 1))
#     for i in range(nn_conf.LAYERS_UNITS[l + 1]):
#       for j in range(nn_conf.LAYERS_UNITS[l] + 1):
#         mat[i][j] = numerical_derivative_approximation(x, y, weights, bias,
#                                                        i, j, l, cost)
#     weights_gradients[l] = mat
#   return weights_gradients


<gh_stars>0
from statistics import mean

class Match_db:

    def __init__(self, sn, name, pid, alg_len, e_val, missmatch, gaps, gaps_o, read):
        self.short_name = sn
        self.name = name
        self.count = 1
        self.pid = [pid]
        self.alg_len = [alg_len]
        self.e_val= [e_val]
        self.missmatch = [missmatch]
        self.gaps = [gaps]
        self.gap_openings = [gaps_o]
        self.read = [read]

    def add_read(self, read, pid, alg_len, e_val, missmatch, gaps, gaps_o):
        self.count += 1
        self.read.append(read)
        self.pid.append(pid)
        self.alg_len.append(alg_len)
        self.e_val.append(e_val)
        self.missmatch.append(missmatch)
        self.gaps.append(gaps)
        self.gap_openings.append(gaps_o)

    def __repr__(self):
        return '''Match_db([{sn},{n}, {c}, {pid}, {al}, {e}, {m}, {g}, {go}, {r}])''' .format(sn=self.short_name, n=self.name, \
                            c=self.count, pid=self.pid, \
                            al=self.alg_len, e=self.e_val, \
                            m=self.missmatch, g=self.gaps, \
                            go=self.gap_openings, r=self.read)

    def __str__(self):
        def preview(lst):
            if len(lst) == 0:
                return ""
            elif len(lst) <= 3:
                return ", ".join(lst)
            else:
                return ", ".join(lst[:3]) + ",..."

        return '''Name: {} \nCount: {} \n%ID-avg: {} \nAlign Len-avg: {} \nE-val-avg: {} \nMissmatch-avg: {} \nGaps-avg: {} \nGap openings-avg: {} \nReads: {}'''.format(self.short_name, self.count, mean(self.pid), mean(self.alg_len), mean(self.e_val), mean(self.missmatch), mean(self.gaps), mean(self.gap_openings), preview(self.read) )


import os
import numpy as np
import math
import cv2
from scipy import ndimage
import time

label1= np.load('/mnt/sda1/downloads/BlazePose-tensorflow-master/data_configs_aic/aic_pre_cropped_17_single.npy')
file_name = np.load('/mnt/sda1/downloads/BlazePose-tensorflow-master/data_configs_aic/files_aic_pre_cropped_17_single.npy')
p_a='/mnt/sda1/downloads/cocoapi-master/PythonAPI/aic_persons_17_single/'

def rot(im_rot,image, xy, a):
    # im_rot = ndimage.rotate(image,angle)
    org_center = (np.array(image.shape[:2][::-1])-1)/2.
    rot_center = (np.array(im_rot.shape[:2][::-1])-1)/2.
    org = xy-org_center
    # a = np.deg2rad(angle)
    new = np.array([org[0]*np.cos(a) + org[1]*np.sin(a),
            -org[0]*np.sin(a) + org[1]*np.cos(a) ])
    return new+rot_center

def align_im(img,labels):

    if labels.shape[1]>2.5:
        labels=labels[:,0:2]
    s_max=int(2*max(img.shape))
    if s_max%2==1:
        s_max=s_max+1
    filler=np.zeros((s_max,s_max,3)).astype(np.uint8)

    # translation

    mid_hip=np.array([0.5*(labels[9,0]+labels[6,0]),0.5*(labels[9,1]+labels[6,1])]).astype(int)
    mid_sh=np.array([0.5*(labels[0,0]+labels[3,0]),0.5*(labels[0,1]+labels[3,1])]).astype(int)
    stpoint=np.array([int(s_max/2-mid_hip[1]),int(s_max/2-mid_hip[0])])
    filler[stpoint[0]:stpoint[0]+img.shape[0],stpoint[1]:stpoint[1]+img.shape[1],:]=img

    for u in range(labels.shape[0]):
        labels[u,0]=labels[u,0]+stpoint[1]
        labels[u,1]=labels[u,1]+stpoint[0]
    # labels[:,0] += stpoint[1]
    # labels[:,1] += stpoint[0]

    mid_hip=np.array([0.5*(labels[9,0]+labels[6,0]),0.5*(labels[9,1]+labels[6,1])]).astype(int)
    mid_sh=np.array([0.5*(labels[0,0]+labels[3,0]),0.5*(labels[0,1]+labels[3,1])]).astype(int)
    body_vec = mid_hip-mid_sh
    img = cv2.line(img,tuple(mid_hip),tuple(mid_sh),(255,0,0),5)
    body_vec[1]=-body_vec[1]
    body_vec=-body_vec

    angle=np.arcsin(body_vec[0]/(body_vec[0] ** 2+body_vec[1]**2)**0.5)
    angle_deg=math.degrees(angle)

    filler_rot = ndimage.rotate(filler, angle_deg,reshape=False,order=0)

    # if body_vec[0]<0:
    #     angle=angle+90
    mid_hip_old=mid_hip
    for u in range(labels.shape[0]):
        labels[u,:]=rot(filler_rot,filler,labels[u,:],angle)

    mid_hip=np.array([0.5*(labels[9,0]+labels[6,0]),0.5*(labels[9,1]+labels[6,1])]).astype(int)
    mid_sh=np.array([0.5*(labels[0,0]+labels[3,0]),0.5*(labels[0,1]+labels[3,1])]).astype(int)

    diam=int(np.linalg.norm(mid_hip-mid_sh))
    final=filler_rot[mid_hip[0]-int(diam*2.2):mid_hip[0]+int(diam*2.2),mid_hip[1]-int(diam*1.5):mid_hip[1]+int(diam*1.7),:]

    for u in range(labels.shape[0]):
        # labels[u,:]=rot(filler_rot,filler,labels[u,:],angle)
        labels[u,0]=labels[u,0]-(mid_hip[1]-int(diam*1.5))
        labels[u,1]=labels[u,1]-(mid_hip[0]-int(diam*2.2))

    # labels[:,0] += (-(mid_hip[1]-int(diam*1.5)))
    # labels[:,1] += (-(mid_hip[0]-int(diam*2.2)))

    return final,labels

# label1= np.load('data_configs/cocos_mids_new_aligned_pc.npy')
# file_name = np.load('data_configs/files_mids_new_aligned_pc.npy')

new_file_name=list()
label=label1[0:file_name.shape[0],0:17,:]
new_label=np.copy(label)

tot_data=label.shape[0]

aa=time.time()
bb=time.time()

omitted_list=list()
new_labels=np.zeros((len(file_name),label1.shape[1],3))

c=0
for i in range(tot_data):
    try:
        if c<10000000:
            if i%100==0:
                print(i)
                print('just for that: {}'.format((time.time()-aa)))
                print('just for that: {}'.format((time.time()-bb)))

                aa=time.time()
            # FileName = "./dataset/lsp/images/im%04d.jpg" % (i + 1)
            FileName = p_a+file_name[i]
            # ii=cv2.imread(file_name[i])
            img = cv2.imread(FileName)
            if img.shape
            labels=np.copy(label[i,:,:])
            img1,labels2=align_im(img, np.copy(label[i,:,:]))
            # FileNames=FileName[0:45]+"aligned_"+FileName[45:]
            FileNames=FileName[0:45]+"aligned_"+FileName[45:]

            cv2.imwrite(FileNames, img1)
            for k in range(labels2.shape[0]):
                if label[i,k,2]==0:
                    labels2[k,0]=0
                    labels2[k,1]=0

            new_labels[c,:,0:2]=labels2.astype(float)
            new_labels[c,:,2]=label[i,:,2].astype(float)
            new_file_name.append(FileNames)

            c=c+1
            # new_label[i,:,2]=np.zeros((new_label.shape[1],))

    except:
        print('none')
        omitted_list.append(i)

new_labels1=new_labels[0:c]

# new_labels=np.zeros((len(new_file_name),new_label.shape[1],3))

# c=0
# for t in range(len(file_name)):
#     if t not in omitted_list:
#         new_labels[c,:,:]=new_label[t,:,:]
#         c=c+1
# print(c-len(new_file_name))

# np.save('data_configs_aic/aic_mids_croped.npy',new_labels1)
# np.save('data_configs_aic/files_aic_mids_cropped.npy',np.array(new_file_name))

# np.save('data_configs/mpii_croped.npy',new_labels1)
# np.save('data_configs/files_mpii_cropped.npy',np.array(new_file_name))


from __future__ import absolute_import
import traceback
from .utils import Eval, latexify, arguments, removeSymPy, \
    custom_implicit_transformation, synonyms, OTHER_SYMPY_FUNCTIONS, \
    close_matches
from .resultsets import find_result_set, get_card, format_by_type, \
    is_function_handled, find_learn_more_set
from sympy import latex
import sympy
from sympy.core.function import FunctionClass
from sympy.parsing.sympy_parser import stringify_expr, eval_expr, \
    standard_transformations, convert_xor, TokenError
from six.moves import map

PREEXEC = """from __future__ import division
from sympy import *
import sympy
from sympy.solvers.diophantine import diophantine
"""

def mathjax_latex(*args):
    tex_code = []
    for obj in args:
        if hasattr(obj, 'as_latex'):
            tex_code.append(obj.as_latex())
        else:
            tex_code.append(latex(obj))

    tag = '<script type="math/tex; mode=display">'
    if len(args) == 1:
        obj = args[0]
        if (isinstance(obj, sympy.Basic) and
            not obj.free_symbols and not obj.is_Integer and
            not obj.is_Float and
            obj.is_finite is not False and
            hasattr(obj, 'evalf')):
            tag = '<script type="math/tex; mode=display" data-numeric="true" ' \
                  'data-output-repr="{}" data-approximation="{}">'.format(
                      repr(obj), latex(obj.evalf(15)))

    tex_code = ''.join(tex_code)

    return ''.join([tag, tex_code, '</script>'])

class SymPyGamma(object):

    def eval(self, s):
        result = None

        try:
            result = self.eval_input(s)
        except TokenError:
            return [
                {"title": "Input", "input": s},
                {"title": "Error", "input": s, "error": "Invalid input"}
            ]
        except Exception as e:
            return self.handle_error(s, e)

        if result:
            parsed, arguments, evaluator, evaluated = result

            cards = []

            close_match = close_matches(s, sympy.__dict__)
            if close_match:
                cards.append({
                    "ambiguity": close_match,
                    "description": ""
                })

            try:
                cards.extend(self.prepare_cards(parsed, arguments, evaluator, evaluated))
            except ValueError as e:
                return self.handle_error(s, e)

            return cards

    def handle_error(self, s, e):
        if isinstance(e, SyntaxError):
            error = {
                "msg": str(e),
                "offset": e.offset
            }
            if e.text:
                error["input_start"] = e.text[:e.offset]
                error["input_end"] = e.text[e.offset:]
            return [
                {"title": "Input", "input": s},
                {"title": "Error", "input": s, "exception_info": error}
            ]
        elif isinstance(e, ValueError):
            return [
                {"title": "Input", "input": s},
                {"title": "Error", "input": s, "error": str(e)}
            ]
        else:
            trace = traceback.format_exc()
            trace = ("There was an error in Gamma.\n"
                     "For reference, the stack trace is:\n\n" + trace)
            return [
                {"title": "Input", "input": s},
                {"title": "Error", "input": s, "error": trace}
            ]

    def disambiguate(self, arguments):
        if arguments[0] == 'factor':
            if arguments.args and isinstance(arguments.args[0], sympy.Number):
                return ('factorint({})'.format(arguments.args[0]),
                        "<var>factor</var> factors polynomials, while <var>factorint</var> factors integers.")
        return None

    def eval_input(self, s):
        namespace = {}
        exec(PREEXEC, {}, namespace)

        def plot(f=None, **kwargs):
            """Plot functions. Not the same as SymPy's plot.

            This plot function is specific to Gamma. It has the following syntax::

                plot([x^2, x^3, ...])

            or::

                plot(y=x,y1=x^2,r=sin(theta),r1=cos(theta))

            ``plot`` accepts either a list of single-variable expressions to
            plot or keyword arguments indicating expressions to plot. If
            keyword arguments are used, the plot will be polar if the keyword
            argument starts with ``r`` and will be an xy graph otherwise.

            Note that Gamma will cut off plot values above and below a
            certain value, and that it will **not** warn the user if so.

            """
            pass
        namespace.update({
            'plot': plot,  # prevent textplot from printing stuff
            'help': lambda f: f
        })

        evaluator = Eval(namespace)
        # change to True to spare the user from exceptions:
        if not len(s):
            return None

        transformations = []
        transformations.append(synonyms)
        transformations.extend(standard_transformations)
        transformations.extend((convert_xor, custom_implicit_transformation))
        parsed = stringify_expr(s, {}, namespace, transformations)
        try:
            evaluated = eval_expr(parsed, {}, namespace)
        except SyntaxError:
            raise
        except Exception as e:
            raise ValueError(str(e))
        input_repr = repr(evaluated)
        namespace['input_evaluated'] = evaluated

        return parsed, arguments(parsed, evaluator), evaluator, evaluated

    def get_cards(self, arguments, evaluator, evaluated):
        first_func_name = arguments[0]
        # is the top-level function call to a function such as factorint or
        # simplify?
        is_function = False
        # is the top-level function being called?
        is_applied = arguments.args or arguments.kwargs

        first_func = evaluator.get(first_func_name)
        is_function = (
            first_func and
            not isinstance(first_func, FunctionClass) and
            not isinstance(first_func, sympy.Atom) and
            first_func_name and first_func_name[0].islower() and
            not first_func_name in OTHER_SYMPY_FUNCTIONS)

        if is_applied:
            convert_input, cards = find_result_set(arguments[0], evaluated)
        else:
            convert_input, cards = find_result_set(None, evaluated)

        components = convert_input(arguments, evaluated)
        if 'input_evaluated' in components:
            evaluated = components['input_evaluated']

        evaluator.set('input_evaluated', evaluated)

        return components, cards, evaluated, (is_function and is_applied)

    def prepare_cards(self, parsed, arguments, evaluator, evaluated):
        components, cards, evaluated, is_function = self.get_cards(arguments, evaluator, evaluated)

        if is_function:
            latex_input = ''.join(['<script type="math/tex; mode=display">',
                                   latexify(parsed, evaluator),
                                   '</script>'])
        else:
            latex_input = mathjax_latex(evaluated)

        result = []

        ambiguity = self.disambiguate(arguments)
        if ambiguity:
            result.append({
                "ambiguity": ambiguity[0],
                "description": ambiguity[1]
            })

        result.append({
            "title": "SymPy",
            "input": removeSymPy(parsed),
            "output": latex_input
        })

        if cards:
            if any(get_card(c).is_multivariate() for c in cards):
                result[-1].update({
                    "num_variables": len(components['variables']),
                    "variables": list(map(repr, components['variables'])),
                    "variable": repr(components['variable'])
                })

        # If no result cards were found, but the top-level call is to a
        # function, then add a special result card to show the result
        if not cards and not components['variable'] and is_function:
            result.append({
                'title': 'Result',
                'input': removeSymPy(parsed),
                'output': format_by_type(evaluated, arguments, mathjax_latex)
            })
        else:
            var = components['variable']

            # If the expression is something like 'lcm(2x, 3x)', display the
            # result of the function before the rest of the cards
            if is_function and not is_function_handled(arguments[0]):
                result.append(
                    {"title": "Result", "input": "",
                     "output": format_by_type(evaluated, arguments, mathjax_latex)})

            line = "simplify(input_evaluated)"
            simplified = evaluator.eval(line,
                                        use_none_for_exceptions=True,
                                        repr_expression=False)
            if (simplified != None and
                simplified != evaluated and
                arguments.args and
                len(arguments.args) > 0 and
                simplified != arguments.args[0]):
                result.append(
                    {"title": "Simplification", "input": repr(simplified),
                     "output": mathjax_latex(simplified)})
            elif arguments.function == 'simplify':
                result.append(
                    {"title": "Simplification", "input": "",
                     "output": mathjax_latex(evaluated)})

            for card_name in cards:
                card = get_card(card_name)

                if not card:
                    continue

                try:
                    result.append({
                        'card': card_name,
                        'var': repr(var),
                        'title': card.format_title(evaluated),
                        'input': card.format_input(repr(evaluated), components),
                        'pre_output': latex(
                            card.pre_output_function(evaluated, var)),
                        'parameters': card.card_info.get('parameters', [])
                    })
                except (SyntaxError, ValueError) as e:
                    pass

            if is_function:
                learn_more = find_learn_more_set(arguments[0])
                if learn_more:
                    result.append({
                        "title": "Learn More",
                        "input": '',
                        "output": learn_more
                    })
        return result

    def get_card_info(self, card_name, expression, variable):
        card = get_card(card_name)

        if not card:
            raise KeyError

        _, arguments, evaluator, evaluated = self.eval_input(expression)
        variable = sympy.Symbol(variable)
        components, cards, evaluated, _ = self.get_cards(arguments, evaluator, evaluated)
        components['variable'] = variable

        return {
            'var': repr(variable),
            'title': card.format_title(evaluated),
            'input': card.format_input(repr(evaluated), components),
            'pre_output': latex(card.pre_output_function(evaluated, variable))
        }

    def eval_card(self, card_name, expression, variable, parameters):
        card = get_card(card_name)

        if not card:
            raise KeyError

        _, arguments, evaluator, evaluated = self.eval_input(expression)
        variable = sympy.Symbol(variable)
        components, cards, evaluated, _ = self.get_cards(arguments, evaluator, evaluated)
        components['variable'] = variable
        evaluator.set(str(variable), variable)
        result = card.eval(evaluator, components, parameters)

        return {
            'value': repr(result),
            'output': card.format_output(result, mathjax_latex)
        }


import os
import cv2
import torch
import numpy as np
from scipy.io import loadmat
from types import SimpleNamespace
from typing import Union, Optional, List, Dict
from .tddfa import mobilenet_v1
from .tddfa.utils.inference import parse_roi_box_from_landmark, crop_img, predict_68pts
from .tddfa_utils import parse_param_pose, reconstruct_from_3dmm

__all__ = ['TDDFAPredictor']

class TDDFAPredictor(object):
    tri = loadmat(os.path.join(os.path.dirname(__file__), 'tddfa', 'visualize', 'tri.mat'))['tri']

    def __init__(self, device: Union[str, torch.device] = 'cuda:0', model: Optional[SimpleNamespace] = None,
                 config: Optional[SimpleNamespace] = None) -> None:
        self.device = device
        if model is None:
            model = TDDFAPredictor.get_model()
        if config is None:
            config = TDDFAPredictor.create_config()
        self.config = SimpleNamespace(**model.config.__dict__, **config.__dict__)
        self.net = getattr(mobilenet_v1, self.config.arch)(num_classes=62).to(self.device)
        pretrained_dict = torch.load(model.weights, map_location=self.device)
        if 'state_dict' in pretrained_dict.keys():
            pretrained_dict = {key.split('module.', 1)[-1] if key.startswith('module.') else key: value
                               for key, value in pretrained_dict['state_dict'].items()}
        else:
            pretrained_dict = {key.split('module.', 1)[-1] if key.startswith('module.') else key: value
                               for key, value in pretrained_dict.items()}
        self.net.load_state_dict(pretrained_dict)
        self.net.eval()
        if self.config.use_jit:
            self.net = torch.jit.trace(self.net, torch.rand(1, 3, self.config.input_size,
                                                            self.config.input_size).to(self.device))

    @staticmethod
    def get_model(name: str = 'mobilenet1') -> SimpleNamespace:
        name = name.lower()
        if name == 'mobilenet1':
            return SimpleNamespace(weights=os.path.join(os.path.dirname(mobilenet_v1.__file__),
                                                        'models', 'phase1_wpdc_vdc.pth.tar'),
                                   config=SimpleNamespace(arch='mobilenet_1', input_size=120))
        else:
            raise ValueError('name must be set to mobilenet')

    @staticmethod
    def create_config(use_jit: bool = True) -> SimpleNamespace:
        return SimpleNamespace(use_jit=use_jit)

    @torch.no_grad()
    def __call__(self, image: np.ndarray, landmarks: np.ndarray, rgb: bool = True,
                 two_steps: bool = False) -> np.ndarray:
        if landmarks.size > 0:
            # Preparation
            if rgb:
                image = image[..., ::-1]
            if landmarks.ndim == 2:
                landmarks = landmarks[np.newaxis, ...]

            # Crop the face patches
            roi_boxes = []
            face_patches = []
            for lms in landmarks:
                roi_boxes.append(parse_roi_box_from_landmark(lms.T))
                face_patches.append(cv2.resize(crop_img(image, roi_boxes[-1]),
                                               (self.config.input_size, self.config.input_size)))
            face_patches = (torch.from_numpy(np.array(face_patches).transpose(
                (0, 3, 1, 2)).astype(np.float32)).to(self.device) - 127.5) / 128.0

            # Get 3DMM parameters
            params = self.net(face_patches).cpu().numpy()
            if two_steps:
                landmarks = []
                for param, roi_box in zip(params, roi_boxes):
                    landmarks.append(predict_68pts(param, roi_box).T)
                return self.__call__(image, np.array(landmarks), rgb=False, two_steps=False)
            else:
                return np.hstack((np.array(roi_boxes, dtype=np.float32), params))
        else:
            return np.empty(shape=(0, 66), dtype=np.float32)

    @staticmethod
    def decode(tdmm_params: np.ndarray) -> List[Dict]:
        if tdmm_params.size > 0:
            if tdmm_params.ndim > 1:
                return [TDDFAPredictor.decode(x) for x in tdmm_params]
            else:
                roi_box = tdmm_params[:4]
                params = tdmm_params[4:]
                vertex, pts68, fR, T = reconstruct_from_3dmm(params)
                camera_transform = {'fR': fR, 'T': T}
                yaw, pitch, roll, t3d, f = parse_param_pose(params)
                face_pose = {'yaw': yaw, 'pitch': pitch, 'roll': roll, 't3d': t3d, 'f': f}
                return {'roi_box': roi_box, 'params': params, 'vertex': vertex, 'pts68': pts68,
                        'face_pose': face_pose, 'camera_transform': camera_transform}
        else:
            return []

    def project_vertex(self, tddfa_result: Dict, dense: bool = True) -> np.ndarray:
        vertex = (tddfa_result['camera_transform']['fR'] @
                  (tddfa_result['vertex'] if dense else tddfa_result['pts68']) +
                  tddfa_result['camera_transform']['T'])

        sx, sy, ex, ey = tddfa_result['roi_box']
        scale_x = (ex - sx) / self.config.input_size
        scale_y = (ey - sy) / self.config.input_size
        vertex[0, :] = vertex[0, :] * scale_x + sx
        vertex[1, :] = (self.config.input_size + 1 - vertex[1, :]) * scale_y + sy

        s = (scale_x + scale_y) / 2
        vertex[2, :] *= s

        return vertex.T