<reponame>naotohori/cafysis
#!/usr/bin/env python
# coding: utf-8

# This program was originally coded by <NAME> as simp.py

import scipy
import scipy.linalg
import pylab
import sys

class nya(object):
    def __init__(self, coords, pdb_Bfactors):
        self.coords = coords
        self.pdb_Bfactors = pdb_Bfactors
        self.num_atoms = len(self.coords)

    def cmpt_graph_mats(self, cutoff=10.0):
        dist = scipy.linalg.norm
        self.adj_mat = scipy.zeros((self.num_atoms, self.num_atoms))
        self.deg_mat = [0] * self.num_atoms
        for i in range(self.num_atoms - 1):
            for j in range(i + 1, self.num_atoms):
                if dist(self.coords[i, :] - self.coords[j, :]) <= cutoff:
                    self.deg_mat[i] += 1.0
                    self.deg_mat[j] += 1.0
                    self.adj_mat[i, j] = 1.0
                    self.adj_mat[j, i] = 1.0
        self.deg_mat = scipy.diag(self.deg_mat)
        self.lap_mat = self.deg_mat - self.adj_mat

    def cmpt_graph_eig(self):
        self.graph_eigval, self.graph_eigvec = scipy.linalg.eigh(self.lap_mat, self.deg_mat)

    def cmpt_hessian(self):
        self.hessian = scipy.zeros((3*self.num_atoms, 3*self.num_atoms))
        for i in range(self.num_atoms - 1):
            for j in range(i + 1, self.num_atoms):
                v_ij = self.coords[j, :] - self.coords[i, :]
                d2 = sum(v_ij * v_ij)
                for a in range(3):
                    for b in range(3):
                        self.hessian[3*i + a, 3*j + b] = -v_ij[a] * v_ij[b] / d2 * self.adj_mat[i, j]
                        self.hessian[3*j + b, 3*i + a] = self.hessian[3*i + a, 3*j + b]
        for i in range(self.num_atoms):
            for a in range(3):
                for b in range(a, 3):
                    for j in range(self.num_atoms):
                        if j != i:
                            self.hessian[3*i + a, 3*i + b] += -self.hessian[3*i + a, 3*j + b]
                    self.hessian[3*i + b, 3*i + a] = self.hessian[3*i + a, 3*i + b]

    def cmpt_en_eig(self):
        self.en_eigval, self.en_eigvec = scipy.linalg.eigh(self.hessian)

    def cmpt_inverse_hessian(self):
        self.inverse_hessian = scipy.linalg.pinv(self.hessian)

    def cmpt_Bfactors(self):
        Bfactors = [self.inverse_hessian[3*i,3*i] +
                    self.inverse_hessian[3*i+1, 3*i+1] +
                    self.inverse_hessian[3*i+2, 3*i+2]
                    for i in range(self.num_atoms)]
        k = sum(self.pdb_Bfactors) / sum(Bfactors)
        self.Bfactors = [Bfactors[i] * k for i in range(self.num_atoms)]

    def cmpt_cross_correlation(self):
        self.cross_correlation = scipy.zeros((self.num_atoms, self.num_atoms))
        self.norm_cross_correlation = scipy.zeros((self.num_atoms, self.num_atoms))
        for i in range(self.num_atoms):
            for j in range(i, self.num_atoms):
                self.cross_correlation[i, j] = (self.inverse_hessian[3*i, 3*j] +
                                                self.inverse_hessian[3*i+1, 3*j+1] +
                                                self.inverse_hessian[3*i+2, 3*j+2])
                self.cross_correlation[j, i] = self.cross_correlation[i, j]
        for i in range(self.num_atoms):
            for j in range(i, self.num_atoms):
                if i == j:
                    self.norm_cross_correlation[i, i] = 1.0
                else:
                    self.norm_cross_correlation[i, j] = (
                     self.cross_correlation[i, j] /
                     scipy.sqrt(self.cross_correlation[i, i] *
                                self.cross_correlation[j, j]))
                    self.norm_cross_correlation[j, i] = self.norm_cross_correlation[i, j]

def get_lines(filename):
    lines = []
    for line in open(filename):
        if (line[0:6] == "ATOM  " and
            line[12:16] == " CA " and
            (line[16:17] == " " or line[16:17] == "A") and
            line[21:22] == "A"):
            lines.append(line)
    return lines

def get_coords(lines):
    def ext_coords(line):
        try:
            x = float(line[30:38])
            y = float(line[38:46])
            z = float(line[46:54])
        except ValueError:
            print("Invalid format(coords)")
            print(line)
            quit()
        return (x, y, z)
    return [ext_coords(line) for line in lines]

def get_Bfactors(lines):
    def ext_Bfactors(line):
        try:
            b = float(line[60:66])
        except ValueError:
            print("Invalid format(B-factors)")
            print(line)
            quit()
        return b
    return [ext_Bfactors(line) for line in lines]

def plot_figs():
    pylab.subplot(221, aspect="equal")
    X, Y = pylab.meshgrid(list(range(ins.num_atoms)), list(range(ins.num_atoms)))
    pylab.pcolor(X, Y, ins.norm_cross_correlation)
    pylab.colorbar()
    pylab.clim(-0.15, 0.15)
    pylab.title("Cross Correlations")

    pylab.subplot(222)
    pylab.plot(pdb_Bfactors, "bo-", label="ex.")
    pylab.plot(ins.Bfactors, "ro-", label="calc.")
    pylab.legend()
    pylab.xlabel("Residue")
#    pylab.ylabel("a.u.")
    pylab.title("B factors")
    pylab.grid()

    pylab.subplot(223, aspect="equal")
    X, Y = pylab.meshgrid(list(range(ins.num_atoms)), list(range(ins.num_atoms)))
    pylab.pcolor(X, Y, ins.adj_mat)
    pylab.colorbar()
    pylab.title("Adjacency Mat.")

    pylab.subplot(224)
    pylab.plot(ins.graph_eigvec[:, 1], "go-")
    pylab.xlabel("Residue")
    pylab.grid()

    pylab.show()

if __name__ == "__main__":

    filename = sys.argv[1]

    lines = get_lines(filename)
    coords = scipy.array(get_coords(lines))
    pdb_Bfactors = get_Bfactors(lines)

    ins = nya(coords, pdb_Bfactors)
    ins.cmpt_graph_mats()
    ins.cmpt_graph_eig()
    ins.cmpt_hessian()
    ins.cmpt_en_eig()
    ins.cmpt_inverse_hessian()
    ins.cmpt_Bfactors()
    ins.cmpt_cross_correlation()

    plot_figs()


<gh_stars>0
import numpy as np
import pandas as pd
from scipy.spatial.distance import cdist, pdist, squareform
import random
from geopy import distance
import geoplotlib
from geoplotlib.utils import BoundingBox
from geoplotlib.layers import BaseLayer
from geoplotlib.core import BatchPainter

def geodesic_dist(p1, p2):
    """
    Compute geodesic distance between points
    p1 and p2 described by latitude and longitude.

    Args:
        p1 (numpy.ndarray): The first point
        p2 (numpy.ndarray): The second point
    """
    return distance.distance(p1, p2).m

def group_distance(g1, g2, method='average-linkage'):
    """
    Compute distance between the specified groups using specified method and
    metric.

    Args:
        g1 (list): The first group
        g2 (list): The second group
        method (str): The method to use (single-linkage, complete-linkage or average-linkage)

    Returns:
        (float): The evaluated distance between the groups.
    """

    # Compute distance between groups using specified metric and method.
    return np.mean(np.ravel(cdist(np.vstack(g1), np.vstack(g2), metric='cityblock')))

def get_groups(data, delta_condition):
    """
    Merge close nodes into clusters using agglomerative procedure.

    Args:
        data (np.ndarray): Data points represented as a numpy array
        delta_condition (float): Distance limit for considering nodes to
        be part of same cluster.

    Returns:
        (list): List of clusters represented as numpy arrays.
    """

    # Initialize list for storing the groups.
    groups = []

    # Go over all data-points.
    for data_idx in range(data.shape[0]):

        # Initialize list for storing indices of merged groups.
        to_remove = []

        # Consider the next point as a single group.
        group_nxt = [data[data_idx, :]]

        # Go over all existing groups.
        for idx, group in enumerate(groups):

            # Compute distance to next group.
            dist = group_distance(group_nxt, group)

            # If distance below set threshold, merge groups.
            if dist < delta_condition:
                group_nxt = group + group_nxt

                # Add index of merged group to be removed later.
                to_remove.append(idx)

        # Remove groups that were merged.
        for rem_idx in sorted(to_remove, reverse=True):
            del groups[rem_idx]
        to_remove = []

        # Append next found group to list of groups.
        groups.append(group_nxt)

    # Stack data points in groups into numpy arrays.
    return list(map(np.vstack, groups))

def get_medoids(groups):
    """
    Get medoids of found groups and stack them
    into a numpy array.

    Args:
        groups (list): List of groups

    Returns:
        (numpy.ndarray): Array of found medoids.
    """

    # Initialize list for found medoids.
    medoids = []

    # Go over groups and compute medoids.
    for group in groups:
        idx_min = np.argmin(np.sum(squareform(pdist(group, metric='cityblock')), axis=0))
        medoids.append(group[idx_min, :])

    # Stack medoids into numpy array.
    return np.vstack(medoids)

def get_grid(n_samples=10000, min_dist=10, return_sample=False):
    """
    Get grid of points using the sample and cluster process.

    Args:
        n_samples (int): Number of samples to use in the process
        min_dist (float): Distance limit for considering nodes to
        be part of same cluster.
        return_sample (bool): If true, return all the sampled nodes
        along the filtered ones as a second return value.

    Returns:
        (numpy.ndarray): Spatial points forming the grid as well as the corresponding
        sample indices.
    """

    # Parse list of latitude and longitude values and join.
    df = pd.read_csv('./data/trip_data/sampled.csv')
    lat_1 = df['Pickup_latitude'].to_numpy()
    lon_1 = df['Pickup_longitude'].to_numpy()
    lat_2 = df['Dropoff_latitude'].to_numpy()
    lon_2 = df['Dropoff_longitude'].to_numpy()
    lat_all = np.hstack((lat_1, lat_2))
    lon_all = np.hstack((lon_1, lon_2))
    data = np.vstack((lat_all, lon_all)).T

    # Sample spatial points for grid generation using specified sample size.
    sample_indices = random.sample(range(data.shape[0]), n_samples)
    node_sample = data[sample_indices, :]

    # Join nodes in clusters and find medoids.
    clusters = get_groups(node_sample, min_dist)
    nodes_filtered = get_medoids(clusters)
    return nodes_filtered if not return_sample else (nodes_filtered, node_sample)

def draw_grid(nodes, unfiltered=None):
    """
    Draw grid using computed nodes.

    Args:
        nodes (numpy.ndarray): Data points to plot
        unfiltered (numpy.ndarray): Unfiltered data points. If not None,
        plot using different color.
    """

    # Layer for plotting the nodes
    class PointsLayer(BaseLayer):

        def __init__(self, data, color, point_size):
            self.data = data
            self.color = color
            self.point_size = point_size

        def invalidate(self, proj):
            x, y = proj.lonlat_to_screen(self.data['lon'], self.data['lat'])
            self.painter = BatchPainter()
            self.painter.set_color(self.color)
            self.painter.points(x, y, point_size=self.point_size, rounded=True)

        def draw(self, proj, mouse_x, mouse_y, ui_manager):
            self.painter.batch_draw()

    # Get grid node data into dict format.
    data_grid = {
            'lat' : nodes[:, 0],
            'lon' : nodes[:, 1]
            }

    # If unfiltered nodes specified, get data into dict format.
    if unfiltered is not None:
        data_unfiltered = {
                'lat' : unfiltered[:, 0],
                'lon' : unfiltered[:, 1]
                }

    # If unfiltered nodes specified, plot on layer.
    if unfiltered is not None:
        geoplotlib.add_layer(PointsLayer(data_unfiltered, color=[255, 0, 0], point_size=4))

    # Plot grid nodes.
    geoplotlib.add_layer(PointsLayer(data_grid, color=[0, 0, 255], point_size = 7))

    # Set bounding box and show.
    geoplotlib.set_bbox(BoundingBox(north=40.897994, west=-73.199040, south=40.595581, east=-74.55040))
    geoplotlib.show()


<reponame>ClandininLab/pasna-acr-currbio2021
import itertools
import numpy as np
import scipy as sp
import seaborn as sns

def nansem(a, axis=0, ddof=1, nan_policy='omit'):
    '''
    Returns standard error of the mean, while omitting nan values.
    '''
    return sp.stats.sem(a, axis, ddof, nan_policy)

def mean_ci(data, ci=95, axis=0, bootstrap=True, n_boot=10000):
    '''
    Returns mean and 95% confidence intervals, computed by bootstrapping
    '''
    a = 1.0 * np.array(data)
    m = np.nanmean(a, axis=axis)
    if bootstrap:
        boots = sns.algorithms.bootstrap(a, n_boot=1000, func=np.nanmean, axis=axis)
        ci_lo, ci_hi = sns.utils.ci(boots, ci, axis=axis)
    else:
        se = nansem(a, axis=axis)
        h = se * sp.stats.t.ppf((1 + ci/100) / 2., len(a)-1)
        ci_lo, ci_hi = m-h, m+h
    return m, ci_lo, ci_hi

def flatten_nested_list(list_of_lists):
    '''
    Flattens a list of lists to a list
    '''
    return list(itertools.chain(*list_of_lists))

def uneven_list2d_to_np(v, fillval=np.nan):
    '''
    Given a list of uneven lists, returns a 2-dimensional numpy array in which all lists are padded with fillval
    to the length of the longest list.
    '''
    lens = np.array([len(item) for item in v])
    if len(np.unique(lens)) == 1:
        return np.asarray(v)
    mask = lens[:,None] > np.arange(lens.max())
    out = np.full(mask.shape,fillval)
    out[mask] = np.concatenate(v)
    return out

def generate_standard_timestamp(timestamps, trim=False, min_time=None, max_time=None):
    '''
    Finds mean framerate and generates a single timestamp series starting from 0 evenly spaced to the max timestamp.

    timestamps: 2d numpy array with nan padding for uneven timestamp lengths

    If trim=True, finds the largest of the leftmost timestamps and the smallest of the rightmost timestamps.
    If min_time or max_time is defined, that value is used regardless of trim.
    '''
    if not isinstance(timestamps, np.ndarray):
        timestamps = uneven_list2d_to_np(timestamps)
    mean_diff = np.nanmean(np.diff(timestamps))
    if trim:
        min_time = np.nanmax(np.nanmin(timestamps,axis=1)) if min_time is None else min_time
        max_time = np.nanmin(np.nanmax(timestamps,axis=1)) if max_time is None else max_time
    else:
        min_time = np.nanmin(timestamps) if min_time is None else min_time
        max_time = np.nanmax(timestamps) if max_time is None else max_time

    return np.arange(min_time, max_time, mean_diff)

def interpolate_to_new_timestamp(y, t, nt):
    '''
    y: 1d data, length same as t
    t: original timestamp
    nt: new timestamp to interpolate to
    Returns ny, linearly interpolated data at nt
    '''
    not_nan = ~np.isnan(y)
    return np.interp(nt, t[not_nan], y[not_nan], left=np.nan, right=np.nan)

def align_traces_to_standardized_timestamp(ts, xs, ts_standard=None, trim=False, min_time=None, max_time=None):
    '''
    Given ts and xs, 2d numpy arrays representing timestamps and corresponding values,
    returns xs_standardized, xs values interpolated to a standardized timestamp, ts_standard.
    If ts_standard is not provided, it is computed.
    '''
    if ts_standard is None:
        ts_standard = generate_standard_timestamp(ts, trim=trim, min_time=min_time, max_time=max_time)
    xs_standardized = np.array([interpolate_to_new_timestamp(xs[i], ts[i], ts_standard) for i in range(len(xs))])

    return ts_standard, xs_standardized


<filename>barry/postprocessing/bao_extractor.py
import numpy as np
from barry.postprocessing.postprocessor import PkPostProcess

class PureBAOExtractor(PkPostProcess):
    """ The pure BAO extractor detailed in Noda 2017 (1705.01475), Nishimishi 2018 (1708.00375), Noda 2019 (1901.06854)

    See https://ui.adsabs.harvard.edu/abs/2017JCAP...08..007N
    See https://ui.adsabs.harvard.edu/abs/2018JCAP...01..035N
    See https://ui.adsabs.harvard.edu/abs/2019arXiv190106854N

    Parameters
    ----------
    r_s : float
        The sound horizon distance. In units of Mpc/h
    plot : bool, optional
        Whether to output debugging plots
    delta : float, optional
        The window (in units of `r_s` to smooth)
    """

    def __init__(self, r_s, plot=False, delta=0.6):
        super().__init__()
        self.r_s = r_s
        self.plot = plot
        self.delta = delta

    def get_krange(self):
        r""" Returns $k_s \Delta$ as defined in Eq 6 of Nishimishi 2018"""
        k_s = 2 * np.pi / self.r_s  # BAO Wavenumber
        k_range = self.delta * k_s  # Range of k to sum over
        return k_range

    def postprocess(self, ks, pk, mask, return_denominator=False, plot=False):
        """ Runs the BAO Extractor method and returns the extracted BAO signal.

        Warning that this is the estimator given in Eq5 Nishimichi 2018

        As such, make sure your k values are finely sampled and linearly spaced. Alas for
        our data, this isn't always possible to do because the window function wrecks us.

        Parameters
        ----------
        ks : np.array
            The k values for the BAO power spectrum
        pk : np.array
            The power spectrum at `ks`

        Returns
        -------

        """
        k_range = self.get_krange()

        result = []
        denoms = []
        for k, p in zip(ks, pk):
            k_diff = np.abs(ks - k)
            m = k_diff < k_range
            numerator = (1 - (pk[m] / p)).sum()
            denominator = (1 - np.cos(self.r_s * (ks[m] - k))).sum()
            res = numerator / denominator
            denoms.append(denominator)
            result.append(res)
        result = np.array(result)

        # Plots for debugging purposes to make sure everything looks good
        if self.plot:
            import matplotlib.pyplot as plt

            fig, axes = plt.subplots(nrows=2, figsize=(5, 7))
            axes[0].plot(ks, pk, label="Input")
            axes[1].plot(ks, result, label="Output")
            plt.show()

        if mask is None:
            mask = np.ones(result.shape).astype(np.bool)

        # Optionally return the denominator instead
        # Used for manually verifying the correctness of the covariance
        # described in Eq7 (and Noda2019 eq 21,22,23)
        if return_denominator:
            return np.array(denoms)[mask]
        return result[mask]

class BAOExtractor(PureBAOExtractor):
    """ Implements the mix of BAO extractor and power spectrum as defined in Noda 2019, with
    index mixing taken from page 9, paragraph 1 and confirmed via private communication:

    pi_i = {1, 2, 3, 7, 15}
    rho_i = {4,5,6,8,9,10,11,12,13,14,16,17,18,19,20,21,22,23,24,25}

    """

    def __init__(self, r_s, plot=False, delta=0.6, mink=0.06, extra_ks=(0.0925, 0.1775), reorder=True, invert=False):
        super().__init__(r_s, plot=plot, delta=delta)
        self.mink = mink
        self.extra_ks = extra_ks
        self.reorder = reorder
        self.invert = invert

    def get_is_extracted(self, ks):
        # Use indexes to blend the two together
        indices = np.array(list(range(ks.size)))
        extra = None
        for e in self.extra_ks:
            ind = np.argmin(np.abs(ks - e))
            if extra is None:
                extra = indices == ind
            else:
                extra |= indices == ind
        mask_power = (ks < self.mink) | extra
        if self.invert:
            return mask_power
        else:
            return ~mask_power

    def postprocess(self, ks, pk, mask):
        """ Process the power spectrum to get a mix of extracted BAO and P(k)

        Parameters
        ----------
        ks : np.ndarray
            Wavenumbers
        pk : np.ndarray
            Power at wavenumber
        mask : np.ndarray (bool mask), optional
            Which k values to return at the end. Used to remove k values below / above certain values.
            I pass them in here because if we reorder the k values the masking cannot be done outside this function.
        """
        if mask is None:
            mask = np.ones(pk.shape).astype(np.bool)
        extracted_pk = super().postprocess(ks, pk, None)
        mask_bao = self.get_is_extracted(ks)
        if self.reorder:
            result = np.concatenate((pk[mask & ~mask_bao], extracted_pk[mask & mask_bao]))
        else:
            mask_int = mask_bao.astype(np.int)
            result = (extracted_pk * (mask_int) + pk * (1 - mask_int))[mask]
        return result

if __name__ == "__main__":
    from barry.cosmology import CambGenerator

    camb = CambGenerator(om_resolution=10, h0_resolution=1)
    ks = camb.ks
    print(ks.shape)
    r_s, pk_lin, _ = camb.get_data(0.3, 0.70)

    from scipy.interpolate import splev, splrep

    rep = splrep(ks, pk_lin)
    # ks2 = np.linspace(ks.min(), 1, 1000)
    ks2 = np.linspace(0, 0.398, 100)  # Matching the winfit_2 data binning
    pk_lin2 = splev(ks2, rep)

    print("Got pklin")
    b = BAOExtractor(r_s)
    pk_extract = b.postprocess(ks2, pk_lin2)
    print("Got pk_extract")

    import matplotlib.pyplot as plt

    fig, axes = plt.subplots(nrows=2, figsize=(5, 9), sharex=True)
    axes[0].plot(ks2, pk_lin2)
    axes[0].set_title("pk_lin")
    axes[1].plot(ks2, pk_extract)
    axes[1].set_title("Extracted BAO, using winfit_2 bins (0, 0.398, 100)")
    plt.show()

    from barry.datasets.mock_power import MockPowerSpectrum

    dataset = MockPowerSpectrum(name="Recon mean", recon=True, min_k=0.02, step_size=2, postprocess=b)
    data = dataset.get_data()
    import seaborn as sb

    sb.heatmap(data["corr"])
    plt.show()


<filename>hw4/c/spectral_clustering.py
#spectral_clustering:
#1)consruct a matrix representation of the graph;
#2)matrix decomposition:compute eigenvalue and eigenvectors of the matrix,map each point to a lower dimension
#3)clustering
import time,operator
import networkx as nx
import numpy as np
import scipy,time
from scipy.sparse.linalg import eigs
from sklearn.cluster import KMeans

#construct a grapgh of data
def parse(filename):
    G = nx.Graph()
    data = open(filename)
    n1 = []
    n2 = []
    edges=[]
    for i, rows in enumerate(data):
        if '#' in rows:
            continue
        rows = rows.strip().split('\t')
        node_a = int(rows[0])
        node_b = int(rows[1])
        G.add_edge(node_a, node_b)
    return G

#consruct a matrix representation of the graph
def getLaplacian(m):
    print time.ctime()
    d=[row.sum() for row in m]
    print time.ctime()
    D=scipy.sparse.diags(d,0)
    print D
    print time.ctime()
    L=D-m
    print time.ctime()
    return L

#matrix decomposition
def getLowerDimension(w,k):
    print w.shape
    eigValue,eigVec = eigs(w,k=100,which='LR')
    import cPickle
    print 'finished'
    print time.ctime()
    dim = len(eigValue)
    print dim
    print eigValue
    return eigVec

if __name__ == '__main__':
    print time.ctime()
    graph = parse('com-youtube.ungraph.txt')
    laplacianMat = nx.laplacian_matrix(graph)
    print time.ctime()
    print time.ctime()
    fout = open('spectral_clustering_result.txt','w')
    np_matrix = nx.to_scipy_sparse_matrix(graph,dtype=np.float32)
    print 'begin'
    lapW = getLaplacian(np_matrix)
    print 'end'
    reduced_matrix = getLowerDimension(lapW,100)
    print reduced_matrix.shape
    print time.ctime()
    #use Kmean to clustering
    kmeans_model = KMeans(n_clusters=100,init='k-means++',n_init=10).fit(reduced_matrix)
    print time.ctime()
    count=0
    for i in kmeans_model.labels_:
        count=count+1
        fout.write(str(count)+'	'+str(i)+'\n')


__author__ = 'dvgodoy'

from neurons import Input, Hidden, Output
from losses import LogLoss
from scipy.io import loadmat
import os

from activations import SigmoidActivation, ReluActivation, TanhActivation
import numpy as np
from matplotlib import pyplot as plt

class Layer(object):
    classes = ['Input', 'Hidden', 'Output']

    def __init__(self, n_units, layer_type, **kwargs):
        assert layer_type in self.classes
        self._charged = False
        self.n_units = n_units
        self.layer_type = layer_type
        neuron = globals()[layer_type]
        self.units = [neuron(**kwargs) for _ in range(n_units)]

    def connect(self, layer):
        assert isinstance(layer, Layer)
        for source in self.units:
            for destination in layer.units:
                source.connect(destination)

    def fire(self):
        mu, var = self.batch_norm()
        for unit in self.units:
            unit.fire(mu, var)

    def feedback(self):
        for unit in self.units:
            unit.feedback()

    def examples(self, X):
        assert X.shape[0] == self.n_units
        for i, unit in enumerate(self.units):
            unit.examples(X[i, np.newaxis, :])

    def responses(self, Y):
        assert Y.shape[0] == self.n_units
        for i, unit in enumerate(self.units):
            unit.responses(Y[i, np.newaxis, :])

    def batch_norm(self):
        mu, var = 0.0, 1.0
        if np.all([unit.charged for unit in self.units]):
            z = np.array([unit._z for unit in self.units])
            mu = z.mean(axis = 0)
            var = z.var(axis = 0)
        return mu, var

    @property
    def activations(self):
        activations = np.array([unit.activations for unit in self.units])
        return activations.reshape(activations.shape[0], -1)

    @property
    def weights(self):
        weights = np.array([unit.weights for unit in self.units])
        return weights.reshape(weights.shape[0], -1)

    @property
    def biases(self):
        biases = np.array([unit.biases for unit in self.units])
        return biases.reshape(biases.shape[0], -1)

    @property
    def cost(self):
        return np.array([unit.cost for unit in self.units])

if __name__ == '__main__':
    np.random.seed(13)

    name = 'var_u'
    data = loadmat(os.path.join('../data', name + '.mat'))

    sigmoid = SigmoidActivation()
    relu = ReluActivation()
    tanh = TanhActivation()

    l_i = Layer(12, 'Input')
    l_h1 = Layer(10, 'Hidden', activation_function=relu)
    l_h2 = Layer(7, 'Hidden', activation_function=relu)
    l_h3 = Layer(5, 'Hidden', activation_function=relu)
    l_h4 = Layer(4, 'Hidden', activation_function=relu)
    l_h5 = Layer(3, 'Hidden', activation_function=relu)
    l_o = Layer(1, 'Output', activation_function=sigmoid, loss_function=LogLoss(), learning_rate=0.0004)
    layers = [l_i, l_h1, l_h2, l_h3, l_h4, l_h5, l_o]

    l_i.connect(l_h1)
    l_h1.connect(l_h2)
    l_h2.connect(l_h3)
    l_h3.connect(l_h4)
    l_h4.connect(l_h5)
    l_h5.connect(l_o)

    epochs = 10000
    h1_weight_history = []
    #h2_weight_history = []
    #h3_weight_history = []
    cost_history = []
    examples = data['F'].reshape(12, -1)
    responses = data['y']
    batch_size = 4096
    for epoch in range(epochs):
        if not (epoch % 100):
            print(epoch)
        for n_batch in range(1):
            examples = data['F'].reshape(12, -1)[:, (n_batch * batch_size):(n_batch * batch_size + batch_size)]
            responses = data['y'][:, (n_batch * batch_size):(n_batch * batch_size + batch_size)]
            l_i.examples(examples)
            l_o.responses(responses)
            for layer in layers:
                layer.fire()
            cost_history.append(l_o.cost)
            h1_weight_history.append(l_h1.weights)
            #h2_weight_history.append(l_h2.weights)
            #h3_weight_history.append(l_h3.weights)
            for layer in layers[::-1]:
                layer.feedback()
        #print(l_o.activations)
        #print(l_o.cost)
    print(l_o.activations)
    predictions = l_o.activations > 0.5
    print((predictions == responses).mean())
    #h1_mean = [w.mean() for w in h1_weight_history]
    #h2_mean = [w.mean() for w in h2_weight_history]
    #h3_mean = [w.mean() for w in h3_weight_history]
    #h1_std = [w.std() for w in h1_weight_history]
    #h2_std = [w.std() for w in h2_weight_history]
    #h3_std = [w.std() for w in h3_weight_history]
    #plt.plot(h1_mean, 'k')
    #plt.plot(h2_mean, 'r')
    #plt.plot(h3_mean, 'g')
    #plt.plot(h1_std, 'k--')
    #plt.plot(h2_std, 'r--')
    #plt.plot(h3_std, 'g--')
    plt.plot(cost_history)
    plt.show()


import numpy as np
import matplotlib.pyplot as plt
import scipy.optimize

def funzione (t, A, tau, w, phi, Vbias) :
    return A * np.exp(-t/tau) * np.cos(w*t+phi) + Vbias

t, ddp = np.genfromtxt("long3.txt", unpack=True)

x = np.linspace(0, 100000, 4000)
plt.figure()
plt.plot(t, ddp, 'o')
plt.show()


<filename>RoundTrQBlackScholes.py
"""
Author : <NAME>
Date : 2019-08-04
Desc : Valuing Option Price under Transaction Cost. Re-hedging with round trip method.
"""

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from numpy.random import standard_normal, seed
from scipy.stats import norm

import sys

import datetime
import time
import bspline
import bspline.splinelab as splinelab

REG_PARAM = 1e-3

class RoundTrQBlackScholes:
    def __init__(self,
                 mu: float,
                 vol: float,
                 s0: float,
                 T: int,
                 K : float,
                 r: float,
                 num_steps: int,
                 num_paths: int,
                 risk_lambda: float,
                 tr_alpha: float):
        self.mu = mu
        self.vol = vol
        self.s0 = s0
        self.T = T
        self.K = K
        self.r = r
        self.num_steps = num_steps
        self.num_paths = num_paths
        self.risk_lambda = risk_lambda

        self.dt = T / num_steps
        self.gamma = np.exp(-r * self.dt)
        self.tr_alpha = tr_alpha

        self.s_values = np.zeros((self.num_paths, self.num_steps + 1), 'float')
        self.delta_S = None
        self.s_values[:, 0] = s0 * np.ones(self.num_paths, 'float')

        self.opt_hedge = np.zeros((self.num_paths, self.num_steps + 1), 'float')

        self.X = None
        self.data = None
        self.delta_S_hat = None
        coef = 1.0 / (2 * self.gamma * self.risk_lambda)
        self.coef = coef

        self.pi = np.zeros((self.num_paths, self.num_steps + 1), 'float')
        self.pi_hat = np.zeros((self.num_paths, self.num_steps + 1), 'float')

        self.q = np.zeros((self.num_paths, self.num_steps + 1), 'float')
        self.r = np.zeros((self.num_paths, self.num_steps + 1), 'float')

    def gen_path(self):
        # Path Generator (Black Scholes )
        seed(42)

        for i in range(1, self.num_steps + 1):
            std_norm = standard_normal(self.num_paths)
            exp_pow = (self.mu - self.vol ** 2 / 2) * self.dt \
                      + self.vol * np.sqrt(self.dt) * std_norm
            self.s_values[:, i] = self.s_values[:, i - 1] * np.exp(exp_pow)

        delta_S = (1 - self.tr_alpha) * self.s_values[:, 1:] - 1 / self.gamma * self.s_values[:, :self.num_steps]
        self.delta_S = delta_S
        self.delta_S_hat = np.apply_along_axis(lambda x: x - np.mean(x), axis=0, arr=delta_S)
        self.X = - (self.mu - 0.5 * self.vol ** 2) * np.arange(self.num_steps + 1) * self.dt + np.log(self.s_values)

        X_min = np.min(np.min(self.X))
        X_max = np.max(np.max(self.X))

        print("Shape of X : {} \n Max : {} \n Min : {}".format(self.X.shape, X_max, X_min))

        self.pi[:, -1] = np.maximum(self.s_values[:, -1] - self.K, 0)
        self.pi_hat[:, -1] = self.pi[:, -1] - np.mean(self.pi[:, -1])

        self.q[:, -1] = -self.pi[:, -1] - self.risk_lambda * np.var(self.pi[:, -1])
        self.r[:, -1] = -self.risk_lambda * np.var(self.pi[:, -1])

        p = 4
        ncolloc = 12
        tau = np.linspace(X_min, X_max, ncolloc)

        k = splinelab.aptknt(tau, p)
        basis = bspline.Bspline(k, p)

        num_basis = ncolloc
        self.data = np.zeros((self.num_steps + 1, self.num_paths, num_basis))

        t0 = time.time()
        for ix in np.arange(self.num_steps + 1):
            x = self.X[:, ix]
            self.data[ix, :, :] = np.array([basis(el) for el in x])
        t1 = time.time()
        print("\nTime for basis expansion {}".format(t1 - t0))

    def function_A_vec(self, t, reg_param=1e-3):
        """ Equation for matrix A
        self.data : T x N_MC x num_basis
        self.delta_S_hat : N_MC x T
        :param t:
        :param reg_param:
        :return:
        """
        x_data = self.data[t, :, :]
        num_basis_funcs = x_data.shape[1]
        self_dS = self.delta_S_hat[:, t]
        # hat_dS2 = (self_dS ** 2).reshape(-1, 1)
        # mat_A = np.dot(x_data.T, x_data * hat_dS2)
        x_data = x_data.T * self_dS
        mat_A = x_data @ x_data.T   # Wrong line, its result is scalar ???

        return mat_A + reg_param * np.eye(num_basis_funcs)

    def function_B_vec(self, t, pi_hat):
        x_data = self.data[t, :, :]
        this_dS = self.delta_S_hat[:, t]
        # coef = 1 / (2 * self.gamma * self.risk_lambda)
        coef = 0
        mat_B = x_data.T @ (pi_hat * this_dS + coef * self.delta_S[:, t])

        return mat_B

    def roll_backward_hedge(self):
        """
        Roll backward and get the price and optimal hedge vals
        :return:
        """
        for t in range(self.num_steps - 1, -1, -1):
            pi_next = self.pi[:, t + 1]
            pi_next_prime = pi_next + self.tr_alpha * self.opt_hedge[:, t + 1] * self.s_values[:, t + 1]
            pi_prime_hat = pi_next_prime - np.mean(pi_next_prime)

            mat_A = self.function_A_vec(t, REG_PARAM)
            vec_B = self.function_B_vec(t, pi_prime_hat)

            phi = np.linalg.inv(mat_A) @ vec_B
            self.opt_hedge[:, t] = np.dot(self.data[t, :, :], phi)
            self.pi[:, t] = self.gamma * (pi_next_prime - self.opt_hedge[:, t] * self.delta_S[:, t])
            self.r[:, t] = self.gamma * self.opt_hedge[:, t] * self.delta_S[:, t] \
                           - self.risk_lambda * np.var(self.pi[:, t])

    def function_C_vec(self, t, reg_param):
        this_data = self.data[t, :, :]
        mat_C = this_data.T @ this_data
        return mat_C + reg_param * np.eye(this_data.shape[1])

    def function_D_vec(self, t):
        this_data = self.data[t, :, :]
        this_q = self.q[:, t + 1]
        this_r = self.r[:, t]
        vec_D = this_data.T @ (this_r + self.gamma * this_q
                             - self.tr_alpha * self.opt_hedge[:, t + 1] * self.s_values[:, t + 1])
        return vec_D

    def roll_backward_q(self):
        """
        Roll backward to get q values
        :return:
        """
        start_time = time.time()
        for t in range(self.num_steps - 1, -1, -1):
            c_mat = self.function_C_vec(t, REG_PARAM)
            d_vec = self.function_D_vec(t)
            omega = np.linalg.inv(c_mat) @ d_vec

            self.q[:, t] = self.data[t, :, :] @ omega
        print("\n Time : ", time.time() - start_time)

if __name__ == "__main__":
    trMC = RoundTrQBlackScholes(0.02, 0.2, 100, 1, 0.04, 25200, 1000, 0.001, 0.001)
    trMC.gen_path()


from basic import *
from all_genes import all_genes
from scipy.cluster import hierarchy
from scipy.spatial import distance
import os
import html_colors
import parse_tsv

verbose = __name__ == '__main__'

# these helper functions used to do more work, now it's a little silly...

def get_rep( gene, organism ):
    assert gene.startswith('TR')
    return all_genes[organism][gene].rep

def get_mm1_rep( gene, organism ):
    assert gene.startswith('TR')
    return all_genes[organism][gene].mm1_rep

def get_rep_ignoring_allele( gene, organism ):
    rep = get_rep( gene, organism )
    rep = rep[:rep.index('*')]
    return rep

def get_mm1_rep_gene_for_counting( allele, organism ):
    return all_genes[organism][allele].count_rep

def countreps_from_genes( genes, organism ):
    return set( ( all_genes[organism][x].count_rep for x in genes ) )

def tree_sort( old_l, distances, return_leaves=True ): ## average linkage
    assert len(distances) == len(old_l)

    if len(old_l)==1:
        leaves = [0]
    else:
        y = distance.squareform( distances, checks=True )
        Z = hierarchy.average( y )
        #c,coph_dists = hierarchy.cophenet(Z,y)
        leaves = hierarchy.leaves_list( Z )

    new_l = [ old_l[x] for x in leaves ]

    if not return_leaves:
        return new_l
    else:
        return new_l, leaves

def get_top_genes( blast_hits_string ):
    hits = dict( [ ( x.split(':')[0], int( x.split(':')[1] ) ) for x in blast_hits_string.split(';') ] )
    top_score = max( hits.values() )
    return set( [ x for x,y in hits.iteritems() if y >= top_score ] )

def get_top_reps( blast_hits_string, organism ):
    hits = dict( [ ( x.split(':')[0], int( x.split(':')[1] ) ) for x in blast_hits_string.split(';') ] )
    top_score = max( hits.values() )
    return set( [ all_genes[organism][x].rep for x,y in hits.iteritems() if y >= top_score ] )

def reps_from_genes( genes, organism, mm1=False, trim_allele=False ):
    reps = set( ( all_genes[organism][x].mm1_rep for x in genes ) ) if mm1 else \
           set( ( all_genes[organism][x].rep     for x in genes ) )
    if trim_allele:
        reps = set( ( x[:x.index('*')] for x in reps ) )
    return reps

def readme( pngfile, text ):
    """Generate some readme text associated to an image file, that will be incorporated into the
    big html results file by run_basic_analysis.py"""

    out = open(pngfile+'.readme','w')
    cmd = ' '.join(argv)
    out.write("""
Command:
{}

Filename:
{}

Readme:
{}

""".format(cmd, pngfile, text))
    out.close()

## setup a mapping that we can use for counting when allowing mm1s and also ignoring alleles

# allele2mm1_rep_gene_for_counting = {}
# def get_mm1_rep_ignoring_allele( gene, organism ): # helper fxn
#     rep = get_mm1_rep( gene, organism )
#     rep = rep[:rep.index('*')]
#     return rep

# for organism in ['human','mouse']:
#     allele2mm1_rep_gene_for_counting[ organism ] = {}

#     for chain in 'AB':

#         ## look at gene/allele maps
#         vj_alleles = { 'V': [ id for (id,g) in all_genes[organism].iteritems() if g.chain==chain and g.region=='V'],
#                        'J': [ id for (id,g) in all_genes[organism].iteritems() if g.chain==chain and g.region=='J'] }

#         for vj, alleles in vj_alleles.iteritems():
#             gene2rep = {}
#             gene2alleles = {}
#             rep_gene2alleles = {}

#             for allele in alleles:
#                 #assert allele[2] == chain
#                 gene = allele[:allele.index('*')]
#                 rep_gene = get_mm1_rep_ignoring_allele( allele, organism )
#                 if rep_gene not in rep_gene2alleles:
#                     rep_gene2alleles[ rep_gene ] = []
#                 rep_gene2alleles[ rep_gene ].append( allele )

#                 if gene not in gene2rep:
#                     gene2rep[gene] = set()
#                     gene2alleles[gene] = []
#                 gene2rep[ gene ].add( rep_gene )
#                 gene2alleles[gene].append( allele )

#             merge_rep_genes = {}
#             for gene,reps in gene2rep.iteritems():
#                 if len(reps)>1:
#                     assert vj=='V'
#                     if verbose:
#                         print 'multireps:',organism, gene, reps
#                         for allele in gene2alleles[gene]:
#                             print ' '.join(all_genes[organism][allele].cdrs), allele, \
#                                 get_rep(allele,organism), get_mm1_rep(allele,organism)

#                     ## we are going to merge these reps
#                     ## which one should we choose?
#                     l = [ (len(rep_gene2alleles[rep]), rep ) for rep in reps ]
#                     l.sort()
#                     l.reverse()
#                     assert l[0][0] > l[1][0]
#                     toprep = l[0][1]
#                     for (count,rep) in l:
#                         if rep in merge_rep_genes:
#                             assert rep == toprep and merge_rep_genes[rep] == rep
#                         merge_rep_genes[ rep ] = toprep

#             for allele in alleles:
#                 count_rep = get_mm1_rep_ignoring_allele( allele, organism )
#                 if count_rep in merge_rep_genes:
#                     count_rep = merge_rep_genes[ count_rep ]
#                 allele2mm1_rep_gene_for_counting[ organism ][ allele] = count_rep
#                 if verbose:
#                     print 'allele2mm1_rep_gene_for_counting:',organism, allele, count_rep

def assign_label_reps_and_colors_based_on_most_common_genes_in_repertoire( tcr_infos, organism ):
    ## assumes that each element of tcr_infos is a dictionary with fields that would have come from parse_tsv_line
    ## uses the *_countreps info that was filled in by read_pair_seqs.py
    ## the _label_rep* fields get over-written if they were present
    for segtype in segtypes_lowercase:
        countreps_tag = segtype+'_countreps'
        rep_tag       = segtype+'_label_rep'
        color_tag     = segtype+'_label_rep_color' ## where we will store the rep info

        counts = {}
        for tcr_info in tcr_infos:
            reps = tcr_info[countreps_tag].split(';')
            for rep in reps:
                counts[rep] = counts.get(rep,0)+1

        newcounts = {}
        for tcr_info in tcr_infos:
            reps = tcr_info[countreps_tag].split(';')
            toprep = max( [ ( counts[x],x) for x in reps ] )[1]
            tcr_info[rep_tag] = toprep ## doesnt have allele info anymore
            newcounts[toprep] = newcounts.get(toprep,0)+1

        l = [(y,x) for x,y in newcounts.iteritems()]
        l.sort()
        l.reverse()
        rep_colors = dict( zip( [x[1] for x in l], html_colors.get_rank_colors_no_lights(len(l)) ) )
        for tcr_info in tcr_infos:
            tcr_info[ color_tag ] = rep_colors[ tcr_info[ rep_tag ] ]

    return ## we modified the elements of the tcr_infos list in place

## this is not exactly perfect, but probably OK to start with...
##
def detect_fake_chains( clones_file, Achain='A', Bchain='B' ):
    tcrs = parse_tsv.parse_tsv_file( clones_file, key_fields = [], store_fields = ['va_gene','cdr3a','vb_gene','cdr3b'] )
    fake_chains = []
    if len( set( [ (x[0],x[1]) for x in tcrs ] ) )==1:
        fake_chains.append( Achain )
    if len( set( [ (x[2],x[3]) for x in tcrs ] ) )==1:
        fake_chains.append( Bchain )
    if fake_chains:
        print 'Fake sequence data detected for chains: {}'.format( ' '.join( fake_chains ) )
    return fake_chains

# if __name__ == '__main__':
#     for organism in allele2mm1_rep_gene_for_counting:
#         for allele in allele2mm1_rep_gene_for_counting[ organism ]:
#             print 'get_mm1_rep_gene_for_counting