<gh_stars>1-10
import sys, time
import multiprocessing as mp
import numpy as np
import scipy as sp
import visualizer as vz
import mock_net as mn

class NVM:
    def __init__(self, coding, network):
        self.coding = coding
        self.network = network
        self.visualizing = False
        # Encode layer names and constants
        for symbol in self.network.get_layer_names():
            self.coding.encode(symbol)
    def __str__(self):
        pattern_list = self.network.list_patterns()
        vmstr = ''
        for (layer_name, pattern) in pattern_list:
            if self.decode(pattern)=='<?>': continue
            vmstr += '%s:%s;'%(layer_name, self.decode(pattern))
        return vmstr
    def encode(self,human_readable):
        return self.coding.encode(human_readable)
    def decode(self, machine_readable):
        return self.coding.decode(machine_readable)
    def tick(self):
        # network update
        self.network.tick()
        # answer any visualizer request
        if self.visualizing:
            if self.viz_pipe.poll():
                # flush request
                self.viz_pipe.recv()
                # respond with data
                self.send_viz_data()
    def send_viz_data(self, down_sample=2):
        """
        Protocol:
            <# layers>, <name>, <value>, <pattern>, <name>, <value>, <pattern>, ...
        """
        if not self.visualizing: return
        pattern_list = self.network.list_patterns()
        self.viz_pipe.send(len(pattern_list))
        for (layer_name, pattern) in pattern_list:
            self.viz_pipe.send(layer_name)
            self.viz_pipe.send(self.decode(pattern)) # value
            # down sample pattern
            pattern = np.concatenate((pattern, np.nan*np.ones(len(pattern) % down_sample)))
            pattern = pattern.reshape((len(pattern)/down_sample, down_sample)).mean(axis=1)
            pattern = (128*(pattern + 1.0)).astype(np.uint8).tobytes()
            self.viz_pipe.send_bytes(pattern) # bytes
    def show(self):
        self.hide() # flush any windowless viz process
        self.viz_pipe, other_end = mp.Pipe()
        self.viz_process = mp.Process(target=run_viz, args=(other_end,))
        self.viz_process.start()
        self.visualizing = True
        # send initial data for window layout
        self.send_viz_data()
    def hide(self):
        if not self.visualizing: return
        self.viz_pipe.send('shutdown')
        self.viz_process.join()
        self.viz_pipe = None
        self.viz_process = None
        self.visualizing = False
    def set_standard_input(self, message, from_human_readable=True):
        if from_human_readable:
            pattern = self.encode(message)
        else:
            pattern = np.fromstring(pattern,dtype=float)
        self.network.set_pattern('STDI', pattern)
    def get_standard_output(self, to_human_readable=True):
        pattern = self.network.get_pattern('STDO')
        if to_human_readable:
            message = self.decode(pattern)
        else:
            message = pattern.tobytes()
        return message
    def set_instruction(self, opcode, *operands):
        # clear gates
        self.network.set_pattern('A',self.network.get_pattern('A')*0)
        # set instruction
        self.network.set_pattern('OPC',self.encode(opcode))
        for op in range(len(operands)):
            self.network.set_pattern('OP%d'%(op+1), self.encode(operands[op]))
    def train(self, pattern_hash, new_pattern_hash):
        # train module with module.train
        # self.network.get_module(module_name).train(pattern_list, next_pattern_list)
        self.network.train(pattern_hash, new_pattern_hash)
    def quit(self):
        self.hide()
        sys.exit(0)

def mock_nvm(num_registers=3, layer_size=32):
    layer_names = ['IP','OPC','OP1','OP2','OP3'] # instruction
    layer_names += ['{%d}'%r for r in range(num_registers)] # registers
    layer_names += ['C1','C2','CO','N1','N2','NO'] # compare+nand
    layer_names += ['K','V'] # memory
    layer_names += ['STDI','STDO'] # io
    layer_sizes = [layer_size]*len(layer_names)
    net = mn.MockNet(layer_names, layer_sizes)
    coding = mn.MockCoding(layer_size)
    return NVM(coding, net)

def run_viz(nvm_pipe):
    viz = vz.Visualizer(nvm_pipe)
    viz.launch()

def flash_nrom(vm):
    # train vm on instruction set
    # gate_index_map = vm.network.get_module('gating').gate_index_map
    omega = np.tanh(1)
    gate_pattern = vm.network.get_pattern('A')
    # get non-gate layer names
    layer_names = vm.network.get_layer_names(omit_gates=True)
    # layer copies
    zero_gate_pattern = gate_pattern.copy()
    zero_gate_pattern[:] = 0
    for to_layer_name in layer_names:
        for from_layer_name in layer_names:
            gate_pattern[:] = 0
            gate_pattern[vm.network.get_gate_index(to_layer_name,from_layer_name)] = omega
            vm.train(
                {from_layer_name:'pattern', 'A':gate_pattern},
                {to_layer_name:'pattern','A':zero_gate_pattern})
    # set value to_layer_name
    for to_layer_name in layer_names:
        gate_pattern[:] = 0
        gate_pattern[vm.network.get_gate_index(to_layer_name,'OP1')] = omega
        vm.train({'OPC':vm.encode('set'),'OP2':vm.encode(to_layer_name)},{'A':gate_pattern})
        vm.train({'OPC':vm.encode('set'),'OP2':vm.encode(to_layer_name),'A':gate_pattern},
                {'A':zero_gate_pattern,'OPC':vm.encode('_')})
    # ccp from_layer_name to_layer_name condition_layer_name (conditional copy)
    for to_layer_name in layer_names:
        for from_layer_name in layer_names:
            for cond_layer_name in layer_names:
                gate_pattern[:] = 0
                gate_pattern[vm.network.get_gate_index(to_layer_name,from_layer_name)] = omega
                vm.train({
                    'OPC':vm.encode('ccp'),
                    'OP1':vm.encode(from_layer_name),
                    'OP2':vm.encode(to_layer_name),
                    'OP3':vm.encode(cond_layer_name),
                    cond_layer_name:vm.encode('TRUE')},
                    {'A':gate_pattern})
                vm.train({
                    'OPC':vm.encode('ccp'),
                    'OP1':vm.encode(from_layer_name),
                    'OP2':vm.encode(to_layer_name),
                    'OP3':vm.encode(cond_layer_name),
                    cond_layer_name:vm.encode('TRUE'),
                    'A':gate_pattern},
                    {'A':zero_gate_pattern,
                    'OPC':vm.encode('_')})
    # compare circuitry
    vm.train({}, {'CO':vm.encode('FALSE')}) # default FALSE behavior
    vm.train({'C1':'pattern','C2':'pattern'}, {'CO':vm.encode('TRUE')}) # unless equal
    # nand circuitry
    vm.train({}, {'NO':vm.encode('TRUE')}) # default TRUE behavior
    vm.train({'N1':vm.encode('TRUE'),'N2':vm.encode('TRUE')}, {'NO':vm.encode('FALSE')}) # unless both
    # mwr value_layer_name pointer_layer_name (memory write)
    gate_pattern[:] = 0
    key_gate_pattern = gate_pattern.copy()
    value_gate_pattern = gate_pattern.copy()
    assoc_gate_pattern = gate_pattern.copy()
    for pointer_layer_name in layer_names:
        for value_layer_name in layer_names:
            key_gate_pattern[:] = 0
            key_gate_pattern[vm.network.get_gate_index('K',pointer_layer_name)] = omega
            vm.train({
                'OPC':vm.encode('mwr'),
                'OP1':vm.encode(value_layer_name),
                'OP2':vm.encode(pointer_layer_name)},
                {'A':key_gate_pattern})
            value_gate_pattern[:] = 0
            value_gate_pattern[vm.network.get_gate_index('V',value_layer_name)] = omega
            assoc_gate_pattern[:] = 0
            assoc_gate_pattern[vm.network.get_gate_index('V','K')] = omega
            vm.train({
                'OPC':vm.encode('mwr'),
                'OP1':vm.encode(value_layer_name),
                'OP2':vm.encode(pointer_layer_name),
                'A':key_gate_pattern},
                {'A':value_gate_pattern})
            vm.train({
                'OPC':vm.encode('mwr'),
                'OP1':vm.encode(value_layer_name),
                'OP2':vm.encode(pointer_layer_name),
                'A':value_gate_pattern},
                {'W':assoc_gate_pattern})
            vm.train({
                'OPC':vm.encode('mwr'),
                'OP1':vm.encode(value_layer_name),
                'OP2':vm.encode(pointer_layer_name),
                'W':assoc_gate_pattern},
                {'OPC':vm.encode('_'),
                 'W':zero_gate_pattern})
    # mrd value_layer_name pointer_layer_name (memory read)
    gate_pattern[:] = 0
    key_gate_pattern = gate_pattern.copy()
    value_gate_pattern = gate_pattern.copy()
    for pointer_layer_name in layer_names:
        for value_layer_name in layer_names:
            key_gate_pattern[:] = 0
            key_gate_pattern[vm.network.get_gate_index('K',pointer_layer_name)] = omega
            vm.train({
                'OPC':vm.encode('mrd'),
                'OP1':vm.encode(value_layer_name),
                'OP2':vm.encode(pointer_layer_name)},
                {'A':key_gate_pattern})
            value_gate_pattern[:] = 0
            value_gate_pattern[vm.network.get_gate_index(value_layer_name,'V')] = omega
            vm.train({
                'OPC':vm.encode('mrd'),
                'OP1':vm.encode(value_layer_name),
                'OP2':vm.encode(pointer_layer_name),
                'A':key_gate_pattern},
                {'A':-key_gate_pattern}) # placeholder until hidden layer
            vm.train({
                'OPC':vm.encode('mrd'),
                'OP1':vm.encode(value_layer_name),
                'OP2':vm.encode(pointer_layer_name),
                'A':-key_gate_pattern},
                {'A':value_gate_pattern})
            vm.train({
                'OPC':vm.encode('mrd'),
                'OP1':vm.encode(value_layer_name),
                'OP2':vm.encode(pointer_layer_name),
                'A':value_gate_pattern},
                {'OPC':vm.encode('_'),
                 'A':zero_gate_pattern})

def show_tick(vm):
    period = .1
    for t in range(1):
        print('pre : %s'%vm)
        vm.tick()
        print('post: %s'%vm)
        raw_input('.')
        # time.sleep(period)

if __name__ == '__main__':

    # Should run with -i flag to allow mvm.quit()

    mvm = mock_nvm()
    # mvm.set_standard_input('NIL',from_human_readable=True)
    # print(mvm.get_standard_output(to_human_readable=True))
    flash_nrom(mvm)
    # print(mvm.network.transitions['{0}'])
    mvm.show()
    show_tick(mvm)

    # # conditional copies
    # mvm.set_instruction('set','NIL','{0}')
    # show_tick(mvm)
    # show_tick(mvm)
    # show_tick(mvm)
    # mvm.set_instruction('set','TRUE','{1}')
    # show_tick(mvm)
    # show_tick(mvm)
    # show_tick(mvm)
    # mvm.set_instruction('set','FALSE','{2}')
    # show_tick(mvm)
    # show_tick(mvm)
    # show_tick(mvm)
    # raw_input('...')
    # mvm.set_instruction('ccp','{0}','{1}','{2}')
    # show_tick(mvm)
    # mvm.set_instruction('ccp','{0}','{2}','{1}')
    # show_tick(mvm)

    # # compare/logic
    # print('set!')
    # mvm.set_instruction('set','TRUE','N1')
    # # mvm.set_instruction('set','TRUE','C1')
    # show_tick(mvm)
    # show_tick(mvm)
    # show_tick(mvm)
    # show_tick(mvm)
    # print('set!')
    # mvm.set_instruction('set','TRUE','N2')
    # # mvm.set_instruction('set','TRUE','C2')
    # show_tick(mvm)
    # show_tick(mvm)
    # show_tick(mvm)
    # show_tick(mvm)
    # print('set!')
    # mvm.set_instruction('set','NIL','N2')
    # # mvm.set_instruction('set','NIL','C2')
    # show_tick(mvm)
    # show_tick(mvm)
    # show_tick(mvm)

    # memory
    print('set!')
    mvm.set_instruction('set','TRUE','{0}')
    show_tick(mvm)
    show_tick(mvm)
    show_tick(mvm)
    show_tick(mvm)
    print('set!')
    mvm.set_instruction('set','NIL','{1}')
    show_tick(mvm)
    show_tick(mvm)
    show_tick(mvm)
    show_tick(mvm)
    print('set!')
    mvm.set_instruction('mwr','{0}','{1}')
    show_tick(mvm)
    show_tick(mvm)
    show_tick(mvm)
    show_tick(mvm)
    show_tick(mvm)
    show_tick(mvm)
    print('set!')
    mvm.set_instruction('set','_','K')
    show_tick(mvm)
    show_tick(mvm)
    show_tick(mvm)
    show_tick(mvm)
    print('set!')
    mvm.set_instruction('set','_','V')
    show_tick(mvm)
    show_tick(mvm)
    show_tick(mvm)
    show_tick(mvm)
    print('set!')
    mvm.set_instruction('mrd','{2}','{1}')
    show_tick(mvm)
    show_tick(mvm)
    show_tick(mvm)
    show_tick(mvm)
    show_tick(mvm)

    # mvm.hide()


<gh_stars>10-100

from __future__ import division, print_function
import numpy as np
import pytest
from scipy.interpolate import UnivariateSpline

from pyunfold.unfold import iterative_unfold
from pyunfold.callbacks import (Callback, CallbackList, Logger,
                                Regularizer, SplineRegularizer,
                                validate_callbacks, extract_regularizer,
                                setup_callbacks_regularizer)

@pytest.mark.parametrize('attr', ['on_unfolding_begin',
                                  'on_unfolding_end',
                                  'on_iteration_begin',
                                  'on_iteration_end'])
def test_callback_attributes(attr):
    assert hasattr(Callback(), attr)

@pytest.mark.parametrize('callbacks', [[Logger()], Logger()])
def test_logger(capsys, callbacks, example_dataset):

    # Perform iterative unfolding
    unfolded_results = iterative_unfold(data=example_dataset.data,
                                        data_err=example_dataset.data_err,
                                        response=example_dataset.response,
                                        response_err=example_dataset.response_err,
                                        efficiencies=example_dataset.efficiencies,
                                        efficiencies_err=example_dataset.efficiencies_err,
                                        return_iterations=True,
                                        callbacks=callbacks)

    # Get stdout and std err from iterative_unfold
    out, err = capsys.readouterr()

    # Build expected output
    expected_output = ''
    for row_index, row in unfolded_results.iterrows():
        row_output = ('Iteration {}: ts = {:0.4f}, ts_stopping ='
                      ' {}\n'.format(row_index + 1,
                                     row['ts_iter'],
                                     row['ts_stopping']))
        expected_output += row_output

    assert expected_output == out

def test_Logger_isinstance_Callback():

    logger = Logger()
    assert isinstance(logger, Callback)

def test_SplineRegularizer_isinstance_Regularizer():

    spline_reg = SplineRegularizer()
    assert isinstance(spline_reg, Regularizer)

def test_SplineRegularizer(example_dataset):
    degree = 3
    smooth = 20
    spline_reg = SplineRegularizer(degree=degree, smooth=smooth)

    unfolded_with_reg = iterative_unfold(data=example_dataset.data,
                                         data_err=example_dataset.data_err,
                                         response=example_dataset.response,
                                         response_err=example_dataset.response_err,
                                         efficiencies=example_dataset.efficiencies,
                                         efficiencies_err=example_dataset.efficiencies_err,
                                         return_iterations=True,
                                         callbacks=[spline_reg])

    unfolded_no_reg = iterative_unfold(data=example_dataset.data,
                                       data_err=example_dataset.data_err,
                                       response=example_dataset.response,
                                       response_err=example_dataset.response_err,
                                       efficiencies=example_dataset.efficiencies,
                                       efficiencies_err=example_dataset.efficiencies_err,
                                       return_iterations=True)

    no_reg = unfolded_no_reg.iloc[0]['unfolded']
    x = np.arange(len(no_reg), dtype=float)
    spline = UnivariateSpline(x, no_reg, k=degree, s=smooth)
    fitted_unfolded = spline(x)

    np.testing.assert_allclose(unfolded_with_reg.iloc[0]['unfolded'],
                               fitted_unfolded)

def test_SplineRegularizer_groups(example_dataset):
    degree = 3
    smooth = 20
    groups = np.empty_like(example_dataset.data)
    groups[:len(groups) // 2] = 0
    groups[len(groups) // 2:] = 1
    spline_reg = SplineRegularizer(degree=degree, smooth=smooth, groups=groups)
    unfolded_with_reg = iterative_unfold(data=example_dataset.data,
                                         data_err=example_dataset.data_err,
                                         response=example_dataset.response,
                                         response_err=example_dataset.response_err,
                                         efficiencies=example_dataset.efficiencies,
                                         efficiencies_err=example_dataset.efficiencies_err,
                                         return_iterations=True,
                                         callbacks=[spline_reg])

    unfolded_no_reg = iterative_unfold(data=example_dataset.data,
                                       data_err=example_dataset.data_err,
                                       response=example_dataset.response,
                                       response_err=example_dataset.response_err,
                                       efficiencies=example_dataset.efficiencies,
                                       efficiencies_err=example_dataset.efficiencies_err,
                                       return_iterations=True)
    # Manually regularize each group independently
    y_no_reg = unfolded_no_reg.iloc[0]['unfolded']
    x = np.arange(len(y_no_reg), dtype=float)
    fitted_unfolded_no_reg = np.empty(len(y_no_reg))
    group_ids = np.unique(groups)
    for group in group_ids:
        group_mask = groups == group
        x_group = x[group_mask]
        y_group = y_no_reg[group_mask]
        spline_group = UnivariateSpline(x_group, y_group, k=degree, s=smooth)
        fitted_unfolded_group = spline_group(x_group)
        fitted_unfolded_no_reg[group_mask] = fitted_unfolded_group

    np.testing.assert_allclose(unfolded_with_reg.iloc[0]['unfolded'],
                               fitted_unfolded_no_reg)

def test_SplineRegularizer_groups_raises(example_dataset):
    degree = 3
    smooth = 20
    groups = np.empty(len(example_dataset.data) - 1)
    groups[:len(groups) // 2] = 0
    groups[len(groups) // 2:] = 1
    spline_reg = SplineRegularizer(degree=degree, smooth=smooth, groups=groups)
    with pytest.raises(ValueError) as excinfo:
        iterative_unfold(data=example_dataset.data,
                         data_err=example_dataset.data_err,
                         response=example_dataset.response,
                         response_err=example_dataset.response_err,
                         efficiencies=example_dataset.efficiencies,
                         efficiencies_err=example_dataset.efficiencies_err,
                         return_iterations=True,
                         callbacks=[spline_reg])

    err_msg = ('Invalid groups array. There should be an entry '
               'for each cause bin. However, got len(groups)={} '
               'while there are {} cause bins.'.format(len(groups),
                                                       len(example_dataset.data)))
    assert err_msg == str(excinfo.value)

def test_validate_callbacks():
    callbacks = [Logger(), SplineRegularizer()]
    assert validate_callbacks(callbacks) == callbacks

def test_validate_empty_callbacks():
    assert validate_callbacks(None) == []

@pytest.mark.parametrize('callback', [Logger(), SplineRegularizer()])
def test_validate_callbacks_single_callback(callback):
    validate_callbacks(callback) == [callback]

def test_validate_callbacks_raises():
    callbacks = [Logger(), SplineRegularizer(), 'not a callback']
    with pytest.raises(TypeError) as excinfo:
        validate_callbacks(callbacks)

    err_msg = 'Found non-callback object in callbacks: {}'.format(['not a callback'])
    assert err_msg == str(excinfo.value)

def test_extract_regularizer_mutliple_raises():
    callbacks = [SplineRegularizer(), SplineRegularizer()]
    with pytest.raises(NotImplementedError) as excinfo:
        extract_regularizer(callbacks)

    err_msg = 'Multiple regularizer callbacks where provided.'
    assert err_msg == str(excinfo.value)

def test_extract_regularizer_no_regularizer():
    callbacks = [Logger()]
    assert extract_regularizer(callbacks) is None

@pytest.mark.parametrize('callback', [SplineRegularizer()])
def test_extract_regularizer(callback):
    callbacks = [Logger(), callback]
    assert extract_regularizer(callbacks) == callback

def test_setup_callbacks_regularizer():

    callbacks = [Logger(), SplineRegularizer()]
    c, r = setup_callbacks_regularizer(callbacks)
    assert isinstance(c, CallbackList)
    assert len(c) == 1
    assert c.callbacks[0] is callbacks[0]
    assert r is callbacks[1]

def test_callbacklist_empty():
    c = CallbackList()
    assert c.callbacks == []

def test_callbacklist_callbacks():
    logger = Logger()
    reg = SplineRegularizer()
    callbacks = [logger, reg]
    c = CallbackList(callbacks=callbacks)
    assert len(c) == len(callbacks)
    assert all(i is j for i, j in zip(c.callbacks, callbacks))

def test_callbacklist_method_calls():
    class MethodChecker(Callback):
        def __init__(self):
            super(Callback, self).__init__()
            self.called_unfolding_begin = False
            self.called_on_unfolding_end = False
            self.called_on_iteration_begin = False
            self.called_on_iteration_end = False

        def on_unfolding_begin(self, status=None):
            self.called_on_unfolding_begin = True

        def on_unfolding_end(self, status=None):
            self.called_on_unfolding_end = True

        def on_iteration_begin(self, iteration, status=None):
            self.called_on_iteration_begin = True

        def on_iteration_end(self, iteration, status=None):
            self.called_on_iteration_end = True

    method_checker = MethodChecker()
    c = CallbackList(method_checker)

    c.on_iteration_begin(1)
    assert method_checker.called_on_iteration_begin

    c.on_iteration_end(1)
    assert method_checker.called_on_iteration_end

    c.on_unfolding_begin()
    assert method_checker.called_on_unfolding_begin

    c.on_unfolding_end()
    assert method_checker.called_on_unfolding_end


<filename>main.py
# -*- coding: utf-8 -*-
import os
import sys
from statistics.basic import Statistics
from statistics.chisquare import chisquare
from optparse import OptionParser, OptionValueError

if __name__ == "__main__":
    usage = "usage: %prog [options] keyword"
    parser = OptionParser(usage)

    parser.add_option(
        "-f", "--file",
        action="store",
        type="string",
        dest="data_file",
        help="data file"
    )
    parser.add_option(
        "-b", "--bins",
        type="int",
        dest="bins",
        default=None,
        help="bins"
    )
    parser.add_option(
        "-w", "--bin_width",
        type="int",
        dest="bin_width",
        default=None,
        help="bin width"
    )
    parser.add_option(
        "-s", "--significance_level",
        type="float",
        dest="significance_level",
        default=0.05,
        help="significance level"
    )

    options, args = parser.parse_args()
    if options.data_file:
        with open(options.data_file,"r") as f:
            data = map(float, f.read().split())
    else:
        data = map(int, args)
    if not data:
        raise
    s = Statistics(
        data,
        bin_width=options.bin_width,
        bins=options.bins,
        significance_level=options.significance_level
    )
    print u"階級\t度数\t比率\t\t\tZ-Score\t\t\t累積比率\t\t期待比率\t\t期待度数"
    for c in s.classes:
        print "{:<5}\t{:<3}\t{:<18}\t{:<18}\t{:<18}\t{:<18}\t{:<18}".format(
            c,
            s.frequencies[c],
            s.proportions[c],
            s.zscores[c],
            s.cumulative_ratios[c],
            s.class_ratios[c],
            s.expectations[c]
        )
    result = s.fit_test()
    print u"カイ二乗値: X^2 =", s.chisquare
    print u"自由度:       v =", s.freedom
    print u"有意水準:     a =", s.significance_level
    print u"棄却域:     x^2 >", s.critical_region
    print(result)


import scipy
import scipy.special
import scipy.integrate
from numpy import exp,cos,sin

# A basic 1D integral:
scipy.integrate.quad(exp, 0, 1)
# (1.7182818284590453, 1.9076760487502457e-14)
scipy.integrate.quad(sin, -0.5, 0.5)
# (0.0, 2.707864644566304e-15)
scipy.integrate.quad(cos, -0.5, 0.5)
# (0.9588510772084061, 1.0645385431034061e-14)

f = lambda x : exp(-x**2)
scipy.integrate.quad(f, 0, 1)
# (0.7468241328124271, 8.291413475940725e-15)

scipy.integrate.quad(lambda x : exp(-x**2), 0, 1)
# (0.7468241328124271, 8.291413475940725e-15)

scipy.integrate.quad(lambda x : exp(-x**2), 0, inf)
# (0.8862269254527579, 7.101318390472462e-09)

scipy.integrate.quad(lambda x : exp(-x**2), -inf, 1)
# (1.6330510582651852, 3.669607414547701e-11)

scipy.integrate.quad(lambda x: scipy.special.jn(1,x),0,5)
# (1.177596771314338, 1.8083362065765924e-14)

#### Integrating Polynomials

p = np.poly1d([2, 5, 1])
p(1), p(2), p(3.5)

P = polyint(p)
q=P(5)-P(1)

#### Basic computations in linear algebra
import scipy.linalg

a = array([[-2, 3], [4, 5]])
scipy.linalg.det(a)

b = scipy.linalg.inv(a)
dot(a,b)

#### Solving systems of linear equations¶
import scipy.linalg

A = array([[2, 4, 6], [1, -3, -9], [8, 5, -7]])
b = array([4, -11, 2])

sol1 = scipy.linalg.solve(A,b)

Ainv = scipy.linalg.inv(A)
sol2 = dot(Ainv, b)
sol1==sol2


<reponame>yuanyuansjtu/chaospy<filename>chaospy/distributions/collection/binomial.py
"""Binomial probability distribution."""
from functools import wraps
import numpy
from scipy import special

from ..baseclass import SimpleDistribution
from ..operators import J

class binomial(SimpleDistribution):
    """
    Binomial probability distribution.

    Point density:
        comb(N, x) p^x (1-p)^{N-x}      x in {0, 1, ..., N}

    Examples:
        >>> distribution = chaospy.Binomial(3, 0.5)
        >>> distribution
        Binomial(3, 0.5)
        >>> xloc = numpy.arange(4)
        >>> distribution.pdf(xloc).round(4)
        array([0.125, 0.375, 0.375, 0.125])
        >>> distribution.cdf(xloc).round(4)
        array([0.125, 0.5  , 0.875, 1.   ])
        >>> distribution.fwd([-0.5, -0.49, 0, 0.49, 0.5]).round(4)
        array([0.    , 0.0013, 0.0625, 0.1238, 0.125 ])
        >>> uloc = numpy.linspace(0, 1, 8)
        >>> uloc.round(2)
        array([0.  , 0.14, 0.29, 0.43, 0.57, 0.71, 0.86, 1.  ])
        >>> distribution.inv(uloc).round(2)
        array([-0.5 ,  0.55,  0.93,  1.31,  1.69,  2.07,  2.45,  3.5 ])
        >>> distribution.sample(10)
        array([2, 1, 0, 2, 2, 2, 2, 3, 3, 0])
        >>> distribution.mom([1, 2, 3]).round(4)
        array([1.5 , 3.  , 6.75])
        >>> distribution.ttr([0, 1, 2, 3]).round(4)
        array([[1.5 , 1.5 , 1.5 , 1.5 ],
               [1.  , 0.75, 1.  , 0.75]])

    """
    interpret_as_integer = True

    def __init__(self, size, prob):
        super(binomial, self).__init__(
            parameters=dict(size=size, prob=prob),
            repr_args=[size, prob],
        )

    def _cdf(self, x_data, size, prob):
        size = numpy.round(size)
        x_data = x_data-0.5

        floor = numpy.zeros(x_data.shape)
        indices = x_data >= 0
        floor[indices] = special.bdtr(numpy.floor(x_data[indices]), size, prob)

        ceil = numpy.ones(x_data.shape)
        indices = x_data <= size
        ceil[indices] = special.bdtr(numpy.ceil(x_data[indices]), size, prob)
        ceil[numpy.isnan(ceil)] = 0  # left edge case

        offset = x_data-numpy.floor(x_data)
        out = floor*(1-offset) + ceil*offset
        return out

    def _pdf(self, x_data, size, prob):
        x_data = numpy.round(x_data)
        return special.comb(size, x_data)*prob**x_data*(1-prob)**(size-x_data)

    def _lower(self, size, prob):
        return -0.5

    def _upper(self, size, prob):
        return numpy.round(size)+0.5

    def _mom(self, k_data, size, prob):
        x_data = numpy.arange(int(size)+1, dtype=int)
        return numpy.sum(x_data**k_data*self._pdf(
            x_data, size=numpy.floor(size), prob=prob))

    def _ttr(self, k_data, size, prob):
        """Krawtchouk rule."""
        from chaospy.quadrature import discretized_stieltjes
        abscissas = numpy.arange(0, numpy.floor(size)+1)
        weights = self._pdf(abscissas, size, prob)
        (alpha, beta), _, _ = discretized_stieltjes(k_data, [abscissas], weights)
        return alpha[0, -1], beta[0, -1]

class Binomial(J):

    def __init__(self, size, prob):
        dist = binomial(size, prob)
        super(Binomial, self).__init__(dist)
        self._repr_args = [size, prob]


# -*- coding: utf-8 -*-
"""
Created on Thu Jan 26 11:12:32 2017

@author: newton
"""

import pandas as pd
import scipy as sp
import numpy as np
from sklearn.metrics import classification_report
from sklearn.metrics import precision_recall_curve, roc_curve, auc
from sklearn.ensemble import RandomForestClassifier
from sklearn import cross_validation
from sklearn import metrics
from sklearn.metrics import confusion_matrix
import csv
import matplotlib.pyplot as plt
import itertools

labels = [0,1,2,3,4,5,6,7,8,9]

#def plot_confusion_matrix(cm, title='Confusion Matrix', cmap = plt.cm.binary):
#    plt.imshow(cm, interpolation='nearest', cmap=cmap)
#    plt.title(title)
#    plt.colorbar()
#    xlocations = np.array(range(len(labels)))
#    plt.xticks(xlocations, labels, rotation=90)
#    plt.yticks(xlocations, labels)
#    plt.ylabel('True label')
#    plt.xlabel('Predicted label')

def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, cm[i, j],
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

trains = pd.read_csv("train.csv")
tests = pd.read_csv("test.csv")

Y = trains['label']

del trains['label']

X_datasarr = trains.as_matrix()
X_norm = X_datasarr > 0
X = X_norm.astype(int)

X_train,X_test,y_train,y_test = cross_validation.train_test_split(X, Y, test_size=0.3, random_state=0)

unique, counts = np.unique(y_test, return_counts=True)
print np.asarray((unique, counts)).T

X_des_datasarr = tests.as_matrix()
X_des_norm = X_des_datasarr > 0
X_des = X_des_norm.astype(int)

results = []
# 最小叶子结点的参数取值
sample_leaf_options = list(range(1, 50, 3))
# 决策树个数参数取值
n_estimators_options = list(range(1, 10, 5))
for leaf_size in sample_leaf_options:
    for n_estimators_size in n_estimators_options:

        rfc = RandomForestClassifier(min_samples_leaf=leaf_size, n_estimators=n_estimators_size, random_state=50)
        rfc.fit(X_train,y_train)

        y_pred_class = rfc.predict(X_test)
        results.append((leaf_size, n_estimators_size, (y_test == y_pred_class).mean()))

print(max(results, key=lambda x: x[2]))

rfc = RandomForestClassifier(min_samples_leaf=6, n_estimators=6, random_state=50)
rfc.fit(X_train,y_train)
cm = confusion_matrix(y_test, y_pred_class)
plot_confusion_matrix(cm,labels, title='Normalized confusion matrix')

#print metrics.accuracy_score(y_test, y_pred_class)
#print rfc.score(X_test, y_test)
#print rfc.classes_
#print y_test[0]
#print rfc.predict(X_test[0])
#print rfc.predict_proba(X_test[0])

Y_des = rfc.predict(X_des)

#Data is not binary and pos_label is not specified
#precision, recall, pr_thresholds = precision_recall_curve(y_test, y_pred_class)
#print precision,recall,pr_thresholds

headers = ['ImageId','Label']

with open('digit_submission.csv','w') as f:
    f_csv = csv.writer(f)
    f_csv.writerow(headers)
    rowid = 1
    for y in Y_des:
        row = [rowid,y]
        rowid += 1
        f_csv.writerow(row)


<filename>test/algorithms/test_bound.py

# Copyright 2021 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from nystrompca import calc_conf_bound, calculate_bound
from nystrompca.base import Kernel

import numpy as np
from numpy.testing import assert_array_almost_equal
from scipy.stats import norm

def test_calc_conf_bound1():

    L     = np.arange(1,11)[::-1] * 10
    n     = 100
    alpha = 0.5
    B     = 1

    term1 = np.sqrt(2*np.log(4)) / np.sqrt(90)
    term2 =  1 / np.sqrt(10)
    D = 0.9 * (term1 + term2)

    expected_bounds = np.cumsum(L[:-1]/10) * D**2 + D ** 3
    expected_bounds = np.r_[expected_bounds, np.nan]

    bounds = calc_conf_bound(np.diag(L), n, B, alpha)

    assert_array_almost_equal(bounds, expected_bounds)

def test_calc_conf_bound2():

    L     = np.exp(np.linspace(1,0,10))
    n     = 20
    alpha = 0.5
    B     = 10

    term1 = 10 * np.sqrt(2*np.log(4)) / np.sqrt(10)
    term2 =  100 / np.sqrt(10)
    D = 0.5 * (term1 + term2)

    expected_bounds = np.cumsum(L[:-1]/10) + D
    expected_bounds = np.r_[expected_bounds, np.nan]

    bounds = calc_conf_bound(np.diag(L), n, B, alpha)

    assert_array_almost_equal(bounds, expected_bounds)

def test_calc_conf_bound3():

    L     = np.arange(1,11)[::-1] * 10
    n     = 1000
    alpha = 0.9
    B     = 0.5

    term1 = 0.5 * np.sqrt(2*np.log(20)) / np.sqrt(990)
    term2 = 0.25 / np.sqrt(10)
    D = 0.99 * (term1 + term2)

    expected_bounds = np.cumsum(L[:-1]/10) * D**2 + D ** 3
    expected_bounds = np.r_[expected_bounds, np.nan]

    bounds = calc_conf_bound(np.diag(L), n, B, alpha)

    assert_array_almost_equal(bounds, expected_bounds)

def test_calculate_bound():

    X = np.random.randn(100,10)
    kernel = Kernel(kernel='cauchy', sigma=5)
    n = 2000
    alpha = 0.75
    K_mm = kernel.matrix(X, demean=False)
    bounds1 = calculate_bound(X, n, 'cauchy', alpha, sigma=5)
    bounds2 = calc_conf_bound(K_mm, n, 1, alpha)

    assert_array_almost_equal(bounds1, bounds2)


<filename>clef20/celebrity-profiling/pan20_celebrity_profiling_evaluator_tests.py
import unittest
from statistics import mean
import pan19_celebs_evaluator as pev

class TestRecallPrecision(unittest.TestCase):

    def test_pr_gender(self):
        truth = ["male", "female", "binary", "male", "female", "binary", "male", "female", "binary"]
        predictions = ["male", "female", "binary", "female", "binary", "male", "binary", "male", "female"]
        prec, rec = pev.mc_prec_rec(predictions, truth)
        self.assertEqual(mean(prec), 1/3)
        self.assertEqual(mean(rec), 1/3)

    def test_pr_age(self):
        truth = [1938, 1988, 2008, 1938, 1988, 2008, 1938, 1988, 2008]
        predictions = [1929, 1984, 2006, 2006, 1929, 1984, 1984, 2008, 1929]
        prec, rec = pev.mc_prec_rec(predictions, truth, hit_function=pev.age_window_hit)
        self.assertEqual(mean(prec), 2.5/3)
        self.assertEqual(mean(rec), 1/3)

    def test_pr_age2(self):
        truth = [2008, 2009, 2010, 2011, 2012]
        predictions = [2008, 2008, 2008, 2008, 2008]
        # after age_window_hit: predictions = [2008, 2009, 2008, 2008, 2008]
        prec, rec = pev.mc_prec_rec(predictions, truth, hit_function=pev.age_window_hit)
        self.assertEqual(mean([1/3, 1, 1, 0, 0]), mean(prec))
        self.assertEqual(mean([1, 1, 1, 0, 0]), mean(rec))

    def test_pr_age3(self):
        truth = [1978, 1979, 1980, 1981, 1982, 1983, 1984]
        predictions = [1978, 1978, 1978, 1978, 1978, 1978, 1978]
        # after age_window_hit: predictions = [1978, 1979, 1980, 1981, 1982, 1983, 1978]
        prec, rec = pev.mc_prec_rec(predictions, truth, hit_function=pev.age_window_hit)
        self.assertEqual(mean([1/2, 1, 1, 1, 1, 1, 0]), mean(prec))
        self.assertEqual(mean([1, 1, 1, 1, 1, 1, 0]), mean(rec))

class TestAgeWindowHit(unittest.TestCase):

    def test_bounds(self):
        self.assertTrue(pev.age_window_hit(2008, 2008))
        self.assertTrue(pev.age_window_hit(2006, 2008))
        self.assertTrue(pev.age_window_hit(2010, 2008))
        self.assertTrue(not pev.age_window_hit(2011, 2008))
        self.assertTrue(not pev.age_window_hit(2005, 2008))
        self.assertTrue(pev.age_window_hit(2008, 2010))

        self.assertTrue(pev.age_window_hit(1988, 1988))
        self.assertTrue(pev.age_window_hit(1984, 1988))
        self.assertTrue(pev.age_window_hit(1992, 1988))
        self.assertTrue(not pev.age_window_hit(1983, 1988))
        self.assertTrue(not pev.age_window_hit(1993, 1988))

        self.assertTrue(pev.age_window_hit(1929, 1938))
        self.assertTrue(pev.age_window_hit(1947, 1938))
        self.assertTrue(not pev.age_window_hit(1928, 1938))
        self.assertTrue(not pev.age_window_hit(1948, 1938))

if __name__ == "__main__":
    unittest.main()


from OpenPNM.Utilities import misc
import scipy as _sp
import numpy as _np
import os as _os
import pickle as _pickle
from xml.etree import ElementTree as _ET

class VTK():
    r"""
    Class for writing a Vtp file to be read by ParaView

    """

    _TEMPLATE = '''
    <?xml version="1.0" ?>
    <VTKFile byte_order="LittleEndian" type="PolyData" version="0.1">
        <PolyData>
            <Piece NumberOfLines="0" NumberOfPoints="0">
                <Points>
                </Points>
                <Lines>
                </Lines>
                <PointData>
                </PointData>
                <CellData>
                </CellData>
            </Piece>
        </PolyData>
    </VTKFile>
    '''.strip()

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @staticmethod
    def save(network, filename='', phases=[]):
        r"""
        Save network and phase data to a single vtp file for visualizing in
        Paraview

        Parameters
        ----------
        network : OpenPNM Network Object
            The Network containing the data to be written

        filename : string, optional
            Filename to write data.  If no name is given the file is named after
            ther network

        phases : list, optional
            A list contain OpenPNM Phase object(s) containing data to be written

        Examples
        --------
        >>> import OpenPNM
        >>> pn = OpenPNM.Network.Cubic(shape=[3,3,3])
        >>> geo = OpenPNM.Geometry.Stick_and_Ball(network=pn,
        ...                                       pores=pn.pores(),
        ...                                       throats=pn.throats())
        >>> air = OpenPNM.Phases.Air(network=pn)
        >>> phys = OpenPNM.Physics.Standard(network=pn, phase=air,
        ...                                 pores=pn.pores(), throats=pn.throats())

        >>> import OpenPNM.Utilities.IO as io
        >>> io.VTK.save(pn,'test_pn.vtp',[air])

        >>> # Delete the new file
        >>> import os
        >>> os.remove('test_pn.vtp')
        """

        if filename == '':
            filename = network.name
        filename = filename.split('.')[0] + '.vtp'

        root = _ET.fromstring(VTK._TEMPLATE)
        objs = []
        if type(phases) != list:
            phases = [phases]
        for phase in phases:
            objs.append(phase)
        objs.append(network)
        am = misc.amalgamate_data(objs=objs)
        key_list = list(sorted(am.keys()))
        points = network['pore.coords']
        pairs = network['throat.conns']

        num_points = len(points)
        num_throats = len(pairs)

        piece_node = root.find('PolyData').find('Piece')
        piece_node.set("NumberOfPoints", str(num_points))
        piece_node.set("NumberOfLines", str(num_throats))

        points_node = piece_node.find('Points')
        coords = VTK._array_to_element("coords", points.T.ravel('F'), n=3)
        points_node.append(coords)

        lines_node = piece_node.find('Lines')
        connectivity = VTK._array_to_element("connectivity", pairs)
        lines_node.append(connectivity)
        offsets = VTK._array_to_element("offsets", 2*_np.arange(len(pairs))+2)
        lines_node.append(offsets)

        point_data_node = piece_node.find('PointData')
        for key in key_list:
            array = am[key]
            if array.dtype == _np.bool:
                array = array.astype(int)
            if array.size != num_points:
                continue
            element = VTK._array_to_element(key, array)
            point_data_node.append(element)

        cell_data_node = piece_node.find('CellData')
        for key in key_list:
            array = am[key]
            if array.dtype == _np.bool:
                array = array.astype(int)
            if array.size != num_throats:
                continue
            element = VTK._array_to_element(key, array)
            cell_data_node.append(element)

        tree = _ET.ElementTree(root)
        tree.write(filename)

        # Make pretty
        with open(filename, 'r+') as f:
            string = f.read()
            string = string.replace('</DataArray>', '</DataArray>\n\t\t\t')
            f.seek(0)
            # consider adding header: '<?xml version="1.0"?>\n'+
            f.write(string)

    @staticmethod
    def load(filename):
        r"""
        Read in pore and throat data from a saved VTK file.

        Notes
        -----
        This will NOT reproduce original simulation, since all models and object
        relationships are lost.  Use IO.Save and IO.Load for that.
        """
        network = OpenPNM.Network.GenericNetwork()
        tree = _ET.parse(filename)
        piece_node = tree.find('PolyData').find('Piece')

        # Extract connectivity
        conn_element = piece_node.find('Lines').find('DataArray')
        array = VTK._element_to_array(conn_element, 2)
        network['throat.conns'] = array.T

        for element in piece_node.find('PointData').iter('DataArray'):
            key = element.get('Name')
            array = VTK._element_to_array(element)
            netname = key.split('.')[0]
            propname = key.strip(netname+'.')
            network[propname] = array

        return network

    @staticmethod
    def _array_to_element(name, array, n=1):
        dtype_map = {
            'int8': 'Int8',
            'int16': 'Int16',
            'int32': 'Int32',
            'int64': 'Int64',
            'uint8': 'UInt8',
            'uint16': 'UInt16',
            'uint32': 'UInt32',
            'uint64': 'UInt64',
            'float32': 'Float32',
            'float64': 'Float64',
            'str': 'String',
        }
        element = _ET.Element('DataArray')
        element.set("Name", name)
        element.set("NumberOfComponents", str(n))
        element.set("type", dtype_map[str(array.dtype)])
        element.text = '\t'.join(map(str, array.ravel()))
        return element

    @staticmethod
    def _element_to_array(element, n=1):
        string = element.text
        dtype = element.get("type")
        array = _np.fromstring(string, sep='\t')
        array = array.astype(dtype)
        if n is not 1:
            array = array.reshape(array.size//n, n)
        return array

class MAT():
    r"""
    Class for reading and writing OpenPNM data to a Matlab 'mat' file
    """

    @staticmethod
    def save(network, filename='', phases=[]):
        r"""
        Write Network to a Mat file for exporting to Matlab. This method will be
        enhanced in a future update, and it's functionality may change!

        Parameters
        ----------

        network : OpenPNM Network Object

        filename : string
            Desired file name, defaults to network name if not given

        phases : list of phase objects ([])
            Phases that have properties we want to write to file

        Examples
        --------
        >>> import OpenPNM
        >>> pn = OpenPNM.Network.TestNet()
        >>> geo = OpenPNM.Geometry.TestGeometry(network=pn,
        ...                                     pores=pn.pores(),
        ...                                     throats=pn.throats())
        >>> air = OpenPNM.Phases.TestPhase()
        >>> import OpenPNM.Utilities.IO as io
        >>> io.MAT.save(network=pn, filename='test_pn.mat', phases=air)

        >>> # Remove newly created file
        >>> import os
        >>> os.remove('test_pn.mat')

        """
        if filename == '':
            filename = network.name
        filename = filename.split('.')[0] + '.mat'

        pnMatlab = {}
        new = []
        old = []
        for keys in list(network.keys()):
            old.append(keys)
            new.append(keys.replace('.', '_'))

        for i in range(len(network)):
            pnMatlab[new[i]] = network[old[i]]

        if type(phases) != list:
            phases = [phases]
        if len(phases) != 0:
            for j in range(len(phases)):
                new = []
                old = []

                for keys in list(phases[j].keys()):
                    old.append(keys)
                    new.append(phases[j].name+'_'+keys.replace('.', '_'))

                for i in range(len(phases[j])):
                    pnMatlab[new[i]] = phases[j][old[i]]

        _sp.io.savemat(file_name=filename, mdict=pnMatlab)

    @staticmethod
    def load():
        r"""
        This method is not implemented yet.
        """
        raise NotImplemented()


<gh_stars>0
from scipy.optimize import curve_fit
from numpy import ndarray
from pyequalizer.optim import *
from copy import deepcopy

def find_line(front):
    """
    find_line(front): Find a rational regression line between the stress and strain values of a set of pareto fronts.
    Parameters:
    front: A list of pyequalizer.optim.Individuals that define the pareto front.
    """
    sct_x, sct_y = get_plot_pts(front)
    def ratline(x, c, e, h, eps):
        ex = deepcopy(x)
        try:
            for a in range(len(ex)):
                if ex[a] + h < 0:
                    ex[a] = -h + 0.01
        except:
            if ex + h < 0:
                ex = 0.01 - h
        ret = c*(ex+h)**(-float(e))+eps
        return ret
    params, _ = curve_fit(ratline, sct_x, sct_y, [100., 0.1, -250., 20.], maxfev=100000000)
    print (params)
    return lambda x: ratline(x, *params)

def get_error(point, reg):
    """
    Get the squared error between a specified 2-d point and
    a function, typically a regression line.

    Arguments:
        * point: A 1x2 array of numbers that define a point in space.
        * reg: A function that takes in a number and returns one other number.
               Typically defines a regression curve.
    """
    return (point[1] - reg(point[0]))**2

def get_closest(inds, reg):
    """
    Get a set of the closest individuals in a population
    to the previously generated regression line.

    Arguments:
        * inds: An array of Inds as defined in pyequalizer.optim.
        * reg: A regression line. Function taking on numeric argument and
               returning a numeric result.
    """
    ind_c = deepcopy(inds)
    pts = list(zip(*get_plot_pts(inds)))
    for i in range(len(pts)):
        cost = get_error(pts[i], reg)
        ind_c[i].fitness.append(cost)
    return isolate_pareto(ind_c)


<reponame>geopi1/Improved_USRNet
from scipy.io import loadmat
import matplotlib.pyplot as plt
import numpy as np

# kernel = np.zeros((17,17))
# kernel[6:11,6:11] = np.ones((5,5),np.float32)/25
x = loadmat('./results/soldiers_ww2_small/soldiers_ww2_small_kernel_x2.mat')['Kernel']
plt.figure()
# plt.subplot(1,2,1)
# plt.title('original kernel')
# plt.imshow(kernel,'gray')
# plt.subplot(1,2,2)
plt.title('estimated kernel')
plt.imshow(x,'gray')

plt.show()


<reponame>sequence-dev/sequence<filename>sequence/subsidence.py<gh_stars>1-10
#! /usr/bin/env python
import numpy as np
from landlab import Component
from scipy import interpolate

class SubsidenceTimeSeries(Component):

    _name = "Subsider"

    _time_units = "y"

    _info = {
        "bedrock_surface__increment_of_elevation": {
            "dtype": "float",
            "intent": "out",
            "optional": False,
            "units": "m",
            "mapping": "node",
            "doc": "Increment of elevation",
        },
        "bedrock_surface__elevation": {
            "dtype": "float",
            "intent": "inout",
            "optional": False,
            "units": "m",
            "mapping": "node",
            "doc": "Surface elevation",
        },
    }

    def __init__(self, grid, filepath=None, kind="linear"):
        """Generate subsidence rates.

        Parameters
        ----------
        grid: RasterModelGrid
            A landlab grid.
        filepath: str
            Name of csv-formatted subsidence file.
        kind: str, optional
            Kind of interpolation as a string (one of 'linear',
            'nearest', 'zero', 'slinear', 'quadratic', 'cubic').
            Default is 'linear'.
        """
        super(SubsidenceTimeSeries, self).__init__(grid)

        self._filepath = filepath
        self._kind = kind

        data = np.loadtxt(filepath, delimiter=",", comments="#")
        subsidence = SubsidenceTimeSeries._subsidence_interpolator(
            data, kind=self._kind
        )
        inc = self.grid.add_empty(
            "bedrock_surface__increment_of_elevation", at="node"
        ).reshape(self.grid.shape)
        inc[:] = subsidence(self.grid.x_of_node[self.grid.nodes_at_bottom_edge])

        self._dz = inc.copy()
        self._time = 0.0

    @staticmethod
    def _subsidence_interpolator(data, kind="linear"):
        return interpolate.interp1d(
            data[:, 0],
            data[:, 1],
            kind=kind,
            copy=True,
            assume_sorted=True,
            bounds_error=True,
        )

    @property
    def time(self):
        return self._time

    @property
    def filepath(self):
        return self._filepath

    @filepath.setter
    def filepath(self, new_path):
        self._filepath = new_path
        subsidence = SubsidenceTimeSeries._subsidence_interpolator(
            np.loadtxt(self._filepath, delimiter=",", comments="#"), kind=self._kind
        )
        inc = self.grid.at_node["bedrock_surface__increment_of_elevation"].reshape(
            self.grid.shape
        )
        inc[:] = subsidence(self.grid.x_of_node[self.grid.nodes_at_bottom_edge])
        self._dz = inc.copy()

    def run_one_step(self, dt):
        dz = self.grid.at_node["bedrock_surface__increment_of_elevation"]
        z = self.grid.at_node["bedrock_surface__elevation"]
        z_top = self.grid.at_node["topographic__elevation"]

        dz = dz.reshape(self.grid.shape)
        z = z.reshape(self.grid.shape)
        z_top = z_top.reshape(self.grid.shape)

        dz[:] = self._dz * dt
        z[:] += dz
        z_top[:] += dz

        self._time += dt


import h5py
import numpy as np
from scipy.io import loadmat
from operator import itemgetter
import math
import scipy as sp
import cv2
import matplotlib.pyplot as plt
import os, sys
import time
import multiprocessing

import random

# Generate Observation Map
def func(theta, m, I, imax, L, w, N, anglemask):
    print('*',end='')
    rotmat = np.array([[np.cos(theta), -np.sin(theta)],[np.sin(theta), np.cos(theta)]])
    p = 0.5*(L[:,0]+1)*(w-1) #x 0:w-1
    q = 0.5*(L[:,1]+1)*(w-1) #y 0:w-1
    x = [p-0.5*(w-1), q-0.5*(w-1)]
    x_ = np.dot(rotmat, x)
    p = x_[0,:]+0.5*(w-1);
    q = x_[1,:]+0.5*(w-1);
    p = np.int32(p)
    q = np.int32(q)
    light_idx = q*w + p # 0:w*w-1
    x = [N[:,0], N[:,1]]
    x_ = np.dot(rotmat, x)
    pn = x_[0,:];
    qn = x_[1,:];
    normal = [np.transpose(pn), np.transpose(qn), N[:,2]]
    normal = np.transpose(normal)
    temp = I*anglemask/np.transpose(imax)
    embed = np.zeros((m, w*w), np.float32)
    embed[:, light_idx] = temp
    embed = np.reshape(embed, (m, w, w))
    mask = np.zeros((m, w*w), np.bool_)
    mask[:, light_idx] = anglemask
    mask = np.reshape(mask, (m, w, w))
    return embed, mask, normal, rotmat

def wrapper(args):
    return func(*args)

# for multi core cpu
def light_embedding_2d_rot_invariant_multi(I, imax, L, w, N, div, isRandomThresh):

    m = I.shape[0]
    rows = w
    cols = w
    embed_rot = []
    normal_rot = []
    mask_rot = []
    rot = []

    anglemask = np.zeros((I.shape[0],I.shape[1]),np.float32)
    for k in range(I.shape[0]): # numpixel
        angle1 = 180*np.arccos(L[:,2])/np.pi
        if isRandomThresh == True:
            tgt = np.where(angle1<random.randint(20,90))
            tgtrandom = np.random.permutation(tgt[0])
            tgt = tgtrandom[:random.randint(50,np.min([1000,L.shape[0]]))]
        else:
            tgt = np.where(angle1<90)
        anglemask[k,tgt] = 1

    n = multiprocessing.cpu_count()
    p = multiprocessing.Pool(n)

    params = [(np.pi*(i*360.0/div)/180, m, I, imax, L, w, N, anglemask) for i in range(np.int32(div))]
    result = p.map(wrapper, params)
    p.close()

    embed_list = []
    mask_list = []
    nml_list = []
    rot_list = []
    for i in range(div):
        embed_list.append(result[i][0].copy())
        mask_list.append(result[i][1].copy())
        nml_list.append(result[i][2].copy())
        rot_list.append(result[i][3].copy())

    embed_list = np.array(embed_list)
    embed_list = np.transpose(embed_list, (1,0,2,3))

    mask_list = np.array(mask_list)
    mask_list = np.transpose(mask_list, (1,0,2,3))

    nml_list = np.array(nml_list)
    nml_list = np.transpose(nml_list, (1,0,2))

    del result,anglemask

    return np.array(embed_list), np.array(mask_list), np.array(nml_list), np.array(rot_list), rows, cols

# for single core cpu
def light_embedding_2d_rot_invariant(I, imax, L, w, N, div, isRandomThresh):

    m = I.shape[0]

    embed_rot = []
    normal_rot = []
    mask_rot = []
    rot = []
    count = 0

    anglemask = np.zeros((I.shape[0],I.shape[1]),np.float32)
    for k in range(I.shape[0]):

        angle1 = 180*np.arccos(L[:,2])/np.pi
        if isRandomThresh == True:
            tgt = np.where(angle1<random.randint(20,90))
            tgtrandom = np.random.permutation(tgt[0])
            tgt = tgtrandom[:random.randint(50,np.min([1000,L.shape[0]]))]
        else:
            tgt = np.where(angle1<90)
        anglemask[k,tgt] = 1

    for k in range(div):
        theta = k*360/div
        if theta < 360:
            count = count + 1
            theta = np.pi*theta/180
            rotmat = np.array([[np.cos(theta), -np.sin(theta)],[np.sin(theta), np.cos(theta)]])
            p = 0.5*(L[:,0]+1)*(w-1) #x 0:w-1
            q = 0.5*(L[:,1]+1)*(w-1) #y 0:w-1
            x = [p-0.5*(w-1), q-0.5*(w-1)]
            x_ = np.dot(rotmat, x)
            p = x_[0,:]+0.5*(w-1);
            q = x_[1,:]+0.5*(w-1);
            p = np.int32(p)
            q = np.int32(q)
            light_idx = q*w + p # 0:w*w-1

            x = [N[:,0], N[:,1]]
            x_ = np.dot(rotmat, x)
            pn = x_[0,:];
            qn = x_[1,:];
            normal = [np.transpose(pn), np.transpose(qn), N[:,2]]
            normal = np.transpose(normal)

            temp = I*anglemask/np.transpose(imax)
            embed = np.zeros((m, w*w), np.float32)
            embed[:, light_idx] = temp
            embed = np.reshape(embed, (m, w, w))
            embed_rot.append(embed.copy())
            mask = np.zeros((m, w*w), np.bool_)
            mask[:, light_idx] = anglemask
            mask = np.reshape(mask, (m, w, w))
            mask_rot.append(mask.copy())

            normal_rot.append(normal.copy())
            rot.append(rotmat.copy())

            del embed, temp, normal, mask

    rows = w
    cols = w

    embed_rot = np.array(embed_rot)
    embed_rot = np.transpose(embed_rot, (1,0,2,3))

    mask_rot = np.array(mask_rot)
    mask_rot = np.transpose(mask_rot, (1,0,2,3))

    normal_rot = np.array(normal_rot)
    normal_rot = np.transpose(normal_rot, (1,0,2))

    return np.array(embed_rot), np.array(mask_rot), np.array(normal_rot), np.array(rot), rows, cols

# main function for generating the observation map
def light_embedding_main(Iv, Nv, L, w, rotdiv, validind, isRandomThresh):

    imax = np.amax(Iv,axis=1) # for entire image
    valid = np.intersect1d(validind, np.where(imax>0))
    Iv = Iv[valid,:]
    Nv = Nv[valid,:]
    imax = imax[valid]
    if rotdiv > 1:
        embed, mask, nm, rot, rows, cols = light_embedding_2d_rot_invariant_multi(Iv, [imax], L, w, Nv, rotdiv, isRandomThresh)
    else:
        embed, mask, nm, rot, rows, cols = light_embedding_2d_rot_invariant(Iv, [imax], L, w, Nv, rotdiv, isRandomThresh)

    embed = np.reshape(embed, (embed.shape[0]*embed.shape[1],w,w))
    embed = np.reshape(embed, (embed.shape[0],1,w,w))
    mask = np.reshape(mask, (mask.shape[0]*mask.shape[1],w,w))
    mask = np.reshape(mask, (mask.shape[0],1,w,w))
    nm = np.reshape(nm, (nm.shape[0]*nm.shape[1],3))
    return embed, mask, nm

# prepare observation map for cyclesPS dataset (for training)
def prep_data_2d_from_images_cycles(dirlist, dirname, scale, w, rotdiv_in, rotdiv_on):
    S = []
    M = []
    N = []
    for d in dirlist:
        dirpath = d

        images_dir = dirpath + '/' + dirname
        normal_path = dirpath + '/' + 'gt_normal.tif'
        inboundary_path = dirpath + '/' + 'inboundary.png'
        onboundary_path = dirpath + '/' + 'onboundary.png'

        # read ground truth surface normal
        nml = np.float32(cv2.imread(normal_path,-1))/65535.0 # [-1,1]
        nml = nml[:,:,::-1]
        nml = 2*nml-1
        nml = cv2.resize(nml, None, fx = scale, fy = scale, interpolation = cv2.INTER_NEAREST)
        nShape = np.shape(nml)
        height = nShape[0]
        width = nShape[1]

        # read mask images_metallic
        inboundary = cv2.imread(inboundary_path,-1)
        inboundary = cv2.resize(inboundary, None, fx = scale, fy = scale, interpolation = cv2.INTER_NEAREST)
        inboundary = np.where(inboundary>0)
        inboundary_ind = inboundary[0]*height + inboundary[1]
        onboundary = cv2.imread(onboundary_path,-1)
        onboundary = cv2.resize(onboundary, None, fx = scale, fy = scale, interpolation = cv2.INTER_NEAREST)
        onboundary = np.where(onboundary>0)
        onboundary_ind = onboundary[0]*height + onboundary[1]

        # read light filenames
        f = open(dirpath + '/' 'light.txt')
        data = f.read()
        f.close
        lines = data.split('\n')
        numLight = len(lines)-1 # the last line is empty (how to fix it?)

        L = np.zeros((numLight,3), np.float32)
        for i,l in enumerate(lines):
            s = l.split(' ')
            if len(s) == 3:
                L[i,0] = float(s[0])
                L[i,1] = float(s[1])
                L[i,2] = float(s[2])

        # read images
        I = np.zeros((numLight, height, width), np.float32)

        for i in range(numLight):
            if i % np.floor(numLight/10) == 0:
                print('.',end='')

            image_path = images_dir + '/' + '%05d.tif' % i

            cv2_im = cv2.imread(image_path, -1)/65535.0
            cv2_im = (cv2_im[:,:,0] + cv2_im[:,:,1] + cv2_im[:,:,2])/3
            cv2_im = cv2.resize(cv2_im, (height,width), interpolation = cv2.INTER_NEAREST)
            I[i,:,:] = cv2_im

        Iv = np.reshape(I,(numLight, height*width))
        Iv = np.transpose(Iv)

        Nv = np.reshape(nml,(height*width,3))

        embed_in, mask_in, nm_in = light_embedding_main(Iv, Nv, L, w, rotdiv_in, inboundary_ind, True)
        embed_on, mask_on, nm_on = light_embedding_main(Iv, Nv, L, w, rotdiv_on, onboundary_ind, True)

        embed = []
        embed.append(embed_in.copy())
        embed.append(embed_on.copy())
        embed = np.concatenate(embed, axis=0 )

        mask = []
        mask.append(mask_in.copy())
        mask.append(mask_on.copy())
        mask = np.concatenate(mask, axis=0 )

        nm = []
        nm.append(nm_in.copy())
        nm.append(nm_on.copy())
        nm = np.concatenate(nm, axis=0 )

        S.append(embed.copy())
        M.append(mask.copy())
        N.append(nm.copy())
        print('')

        del embed_in, mask_in, nm_in
        del embed_on, mask_on, nm_on
        del embed, mask, nm, I, Iv, Nv

    S = np.concatenate(S, axis=0 )
    M = np.concatenate(M, axis=0 )
    N = np.concatenate(N, axis=0 )

    S = np.reshape(S, (S.shape[0], S.shape[2], S.shape[3], 1))
    M = np.reshape(M, (M.shape[0], M.shape[2], M.shape[3], 1))
    return np.array(S), np.array(M), np.array(N)

# prepare observation maps for test data (i.e., DiLiGenT dataset)
def prep_data_2d_from_images_test(dirlist, scale, w, rotdiv, index=-1):

    SList = []
    NList = []
    RList = []
    IDList = []
    SizeList = []
    for d in dirlist:
        print('load' + '%s' % d)
        S = []
        N = []
        dirpath = d
        images_dir = dirpath
        normal_path = dirpath + '/' + 'normal.txt'
        mask_path = dirpath + '/' + 'mask.png'

        # get image imgSize
        image_path = images_dir + '/' + '001.png'
        cv2_im = cv2.imread(image_path, -1)
        nShape = np.shape(cv2_im)
        height = nShape[0]
        width = nShape[1]

        # read ground truth surface normal
        f = open(normal_path)
        data = f.read()
        f.close
        lines = np.float32(np.array(data.split('\n')))
        nml = np.reshape(lines, (height,width,3))
        nml = cv2.resize(nml, None, fx = scale, fy = scale, interpolation = cv2.INTER_NEAREST)

        # nml = np.flipud(nml) # Uncomment when test on Harvest, the surface noraml needs to be fliped upside down

        nShape = np.shape(nml)
        height = nShape[0]
        width = nShape[1]

        # uncomment if you want to see the ground truth normal map
        # plt.figure(figsize=(16,16))
        # plt.imshow(np.uint8(127*(nml+1)))
        # plt.axis('off')
        # plt.show()

        # read mask
        mask = cv2.imread(mask_path,-1)
        mask = cv2.resize(mask, None, fx = scale, fy = scale, interpolation = cv2.INTER_NEAREST)
        validsub = np.where(mask>0)
        validind = validsub[0]*width + validsub[1]

        # read light directions
        f = open(dirpath + '/' 'light_directions.txt')
        data = f.read()
        f.close
        lines = data.split('\n')
        numLight = len(lines)-1 # the last line is empty (how to fix it?)

        L = np.zeros((numLight,3), np.float32)
        for i,l in enumerate(lines):
            s = l.split(' ')
            if len(s) == 3:
                L[i,0] = float(s[0])
                L[i,1] = float(s[1])
                L[i,2] = float(s[2])

        # read light intensities
        f = open(dirpath + '/' 'light_intensities.txt')
        data = f.read()
        f.close
        lines = data.split('\n')

        Li = np.zeros((numLight,3), np.float32)
        for i,l in enumerate(lines):
            s = l.split(' ')
            if len(s) == 3:
                Li[i,0] = float(s[0])
                Li[i,1] = float(s[1])
                Li[i,2] = float(s[2])

        if index == -1:
            setName = os.path.basename(dirpath.rstrip('/')) # if dirpath ends in '/' basename returns the empty string
            if setName == 'bearPNG':
                # the first 20 images of bearPNG have errors, see paper
                index = range(20, numLight)
            else:
                index = range(0, numLight)

        L = L[index,:]
        Li = Li[index,:]
        numLight = len(index)

        # read images
        I = np.zeros((numLight, height, width), np.float32)

        for i, idx in enumerate(index):
            if i % np.floor(numLight/10) == 0:
                print('.',end='')
            image_path = images_dir + '/' + '%03d.png' % (idx + 1)
            cv2_im = cv2.imread(image_path, -1)/65535.0
            cv2_im = (cv2_im[:,:,0]/Li[i,0] + cv2_im[:,:,1]/Li[i,1] + cv2_im[:,:,2]/Li[i,2])/3
            cv2_im = cv2.resize(cv2_im, None, fx = scale, fy = scale,interpolation = cv2.INTER_NEAREST)
            I[i,:,:] = cv2_im

        Iv = np.reshape(I,(numLight, height*width))
        Iv = np.transpose(Iv)
        Nv = np.reshape(nml,(height*width,3))

        imax = np.amax(Iv,axis=1) # for entire image
        valid = np.intersect1d(validind, np.where(imax>0))
        Iv = Iv[valid,:]
        Nv = Nv[valid,:]
        imax = imax[valid]
        embed_list = []
        embed, mask, nm, rot, rows, cols = light_embedding_2d_rot_invariant(Iv, [imax], L, w, Nv, rotdiv, False)
        SList.append(embed)
        RList.append(rot)
        NList.append(nm)
        IDList.append(valid)
        SizeList.append((height,width))

        print('')
    return np.array(SList), np.array(NList), np.array(RList), np.array(IDList), np.array(SizeList)

# Test and evaluate network
def TestNetwork(model, Sv,Nv,Rv,IDv,Szv,showFig, isTensorFlow):
    numData = len(Sv)
    for i in range(numData):
        S = Sv[i]
        N = Nv[i]
        R = Rv[i]
        ID = IDv[i]
        height = Szv[i,0]
        width  = Szv[i,1]
        rotdiv = S.shape[1]
        NestList = []
        for r in range(rotdiv):
            embed_div = S[:,r,:,:]
            if isTensorFlow:
                embed_div = np.reshape(embed_div, (embed_div.shape[0], embed_div.shape[1], embed_div.shape[2], 1))
            else:
                embed_div = np.reshape(embed_div, (embed_div.shape[0], 1, embed_div.shape[1], embed_div.shape[2]))
            # predict
            outputs=model.predict(embed_div)
            Nest = np.zeros((height*width,3), np.float32)
            error = 0
            Err = np.zeros((height*width,3), np.float32)
            rot = R[r,:,:]
            # N = np.zeros()
            for k in range(len(ID)):
                # n = outputs[k,:];
                n = np.zeros((2,1),np.float32)
                n[0] = outputs[k,0]
                n[1] = outputs[k,1]
                n = np.dot(np.linalg.inv(rot),n)
                n = [n[0,0],n[1,0],outputs[k,2]]
                n = n/np.sqrt(n[0]*n[0]+n[1]*n[1]+n[2]*n[2])
                nt = N[k,0,:];
                Nest[ID[k],:] = n
                for l in range(3):
                    Err[ID[k],l] = 180*math.acos(min(1,abs(n.dot(np.transpose(nt)))))/math.pi
                error = error + 180*math.acos(min(1,abs(n.dot(np.transpose(nt)))))/math.pi
            print('%d ' % i + '[Angle %d] Ave.Error = %.2f ' % (r,(error/len(ID))))
            NestList.append(Nest.copy())

        NestMean = np.mean(NestList,axis=0)
        Nest = np.zeros((height*width,3), np.float32)
        error = 0
        Err = np.zeros((height*width,3), np.float32)
        for k in range(len(ID)):
            # n = outputs[k,:];
            n = NestMean[ID[k],:]
            n = n/np.sqrt(n[0]*n[0]+n[1]*n[1]+n[2]*n[2])
            nt = N[k,0,:];
            Nest[ID[k],:] = n
            for l in range(3):
                Err[ID[k],l] = 180*math.acos(min(1,abs(n.dot(np.transpose(nt)))))/math.pi
            error = error + 180*math.acos(min(1,abs(n.dot(np.transpose(nt)))))/math.pi

        if rotdiv >= 2:
            print('%s ' % i + '[Mean] Ave.Error = %.2f ' % (error/len(ID)))

        Err = np.reshape(Err,(height,width,3))
        Nest = np.reshape(Nest, (height,width,3))

        if showFig == True:
            plt.figure(figsize=(16,16))
            plt.imshow(np.concatenate((np.uint8(127*(Nest+1)),5*np.uint8(Err)), axis=1))
            plt.axis('off')
            plt.show()


import sympy
from sympy.assumptions.assume import AppliedPredicate, global_assumptions
from typing import Dict, List, Union

a, b, c = sympy.symbols('a b c')
d_a, d_b, d_c = sympy.symbols('Δa Δb Δc')

class Expression:
    args: List[sympy.Symbol]
    expr: sympy.Expr

    def __init__(self, args: List[sympy.Symbol], expr: sympy.Expr):
        """Initialize an Expression instance with a sympy expression and its arguments.

        :param args: the variables in the expression
        :param expr: the mathematical expression

        >>> Expression([a, b, c], a + b + c)
        f(a, b, c) = a + b + c
        >>> Expression([a, b, c], a * b / c)
        f(a, b, c) = a*b/c
        >>> Expression([a, b, c], sympy.root(a ** b, c))
        f(a, b, c) = (a**b)**(1/c)
        """
        self.args = args
        self.expr = expr

    def __repr__(self) -> str:
        """Show this expression as a mathematical function.

        :rtype str

        >>> str(Expression([a], a * sympy.pi))
        'f(a) = pi*a'
        >>> repr(Expression([], sympy.E))
        'f() = E'
        """
        if len(self.args) == 1:
            return f"f({self.args[0]}) = {self.expr}"
        return f"f{tuple(self.args)} = {self.expr}"

    def evaluate(self, values: Dict[Union[str, sympy.Symbol], float], precision: int =3) -> sympy.Expr:
        """Evaluate the expression with the given values.

        :param values: a dictionary mapping all the sympy symbols in the args to numeric values
        :param precision: the number of digits in the results
        :return: the result of the evaluation as an sympy expression

        >>> Expression([a, b, c], a + b + c).evaluate({a: 1, b: 2, c: 3})
        6.00
        >>> Expression([a, b, c], a ** b + c).evaluate({'a': c, 'b': 1})
        2.0*c
        """
        return self.expr.subs(values).evalf(precision)

    def calculate_absolute_uncertainty(self, *assumptions: List[AppliedPredicate],
                                       refine: bool = False,
                                       delta_char: str = '\\Delta ') -> 'Expression':
        """Calculate the absolute uncertainty in the expression (IB way), assuming all args given are independent.

        :return: the absolute uncertainty of this expression
        :rtype: Expression

        >>> Expression([a], c * a).calculate_absolute_uncertainty(sympy.Q.positive(c), refine=True, delta_char='Δ')
        f(Δa) = c*Δa
        >>> Expression([a, b, c], a + b - c).calculate_absolute_uncertainty(refine=True, delta_char='Δ')
        f(Δa, Δb, Δc) = Δa + Δb + Δc
        """
        uncertainty_expr = sympy.Integer(0)  # just in case
        uncertainty_args = []
        global_assumptions.add(*assumptions)

        for var in self.args:
            d_var = sympy.Symbol(delta_char + sympy.latex(var))
            uncertainty_args.append(d_var)
            uncertainty_expr += sympy.Abs(self.expr.diff(var)) * d_var
            global_assumptions.add(sympy.Q.positive(var))
        if refine:
            uncertainty_expr = sympy.refine(uncertainty_expr)
        global_assumptions.clear()
        return Expression(uncertainty_args, uncertainty_expr)

    def calculate_fractional_uncertainty(self, *assumptions: List[AppliedPredicate],
                                         refine: bool = False,
                                         delta_char: str = '\\Delta ') -> 'Expression':
        """Calculate the absolute uncertainty in the expression (IB way), assuming all args given are independent.

        :return: the fractional uncertainty of this expression
        :rtype: Expression

        >>> Expression([a, b, c], a * b / c).calculate_fractional_uncertainty(refine=True, delta_char='Δ')
        f(Δa, Δb, Δc) = Δc/c + Δb/b + Δa/a
        >>> Expression([a], a ** b).calculate_fractional_uncertainty(sympy.Q.positive(b), refine=True, delta_char='Δ')
        f(Δa) = b*Δa/a
        """
        absolute_uncertainty = self.calculate_absolute_uncertainty(*assumptions, refine=refine, delta_char=delta_char)
        frac_uncertainty_expr = sympy.Integer(0)
        if type(absolute_uncertainty.expr) == sympy.Add:
            for addend in absolute_uncertainty.expr.args:
                frac_uncertainty_expr += addend / self.expr
        elif type(absolute_uncertainty.expr) == sympy.Mul or type(absolute_uncertainty) == sympy.Pow:
            frac_uncertainty_expr = absolute_uncertainty.expr / self.expr
        else:
            frac_uncertainty_expr = sympy.Mul(absolute_uncertainty.expr, sympy.Pow(self.expr, -1), evaluate=False)
        return Expression(absolute_uncertainty.args, frac_uncertainty_expr)

    def to_latex(self) -> str:
        r"""Get the latex form of this expression.

        :rtype: str

        >>> Expression([a, b, c], a + b + c).to_latex()
        'a + b + c'
        >>> Expression([a, b, c], a * b / c).to_latex()
        '\\frac{a b}{c}'
        >>> Expression([a, b, c], sympy.root(a ** b, c)).to_latex()
        '\\left(a^{b}\\right)^{\\frac{1}{c}}'
        """
        return sympy.latex(self.expr)

    @classmethod
    def from_string(cls, args_list: List[str], string: str, constants: Dict[str, float] = None) -> 'Expression':
        """Parse a string expression.

        :param string: expression as a string of python expressions
        :param args_list: the list of args / independent variables of the expression as strings
        :param constants: a list of local variables that are considered while parsing
        :return: an expression taking in the given args

        >>> Expression.from_string(['x'], 'sqrt(x) ^ y')
        f(x) = (sqrt(x))**y
        >>> Expression.from_string(['m'], 'm * g', constants={'g': 9.81})
        f(m) = 9.81*m
        """
        parsed_expr = sympy.sympify(string, evaluate=False, locals=constants)  # note: uses eval
        args = [symbol for symbol in parsed_expr.atoms(sympy.Symbol) if str(symbol) in args_list]
        return cls(args, parsed_expr)


import rospy
import actionlib
from move_base_msgs.msg import MoveBaseAction, MoveBaseGoal
from moveit_msgs.msg import MoveItErrorCodes
import math
from scipy.spatial.transform import Rotation as scipyR
moveit_error_dict = {}
for name in MoveItErrorCodes.__dict__.keys():
    if not name[:1] == '_':
        code = MoveItErrorCodes.__dict__[name]
        moveit_error_dict[code] = name

def euclidean_dist(p1, p2):
    dist = math.sqrt(sum([(a - b)** 2 for a, b in zip(p1, p2)]))
    return dist

def yaw_diff(quat1, quat2):
    euler1 = scipyR.from_quat(quat1).as_euler("xyz")
    euler2 = scipyR.from_quat(quat2).as_euler("xyz")
    return abs(euler1[2] - euler2[2])

class WaypointApply(object):
    class Status:
        NOT_RUNNING = "not_running"
        RUNNING = "running"
        SUCCESS = "success"
        FAIL = "fail"
    def __init__(self,
                 position, orientation,
                 action_name="navigate",
                 xy_tolerance=0.1, rot_tolerance=0.3):
        # Get an action client
        self.client = actionlib.SimpleActionClient('movo_move_base', MoveBaseAction)
        rospy.loginfo("Waiting for movo_move_base AS...")
        if not self.client.wait_for_server(rospy.Duration(20)):
            rospy.logerr("Could not connect to movo_move_base AS")
            exit()
        rospy.loginfo("Connected!")
        rospy.sleep(1.0)

        self.status = WaypointApply.Status.NOT_RUNNING
        self.action_name = action_name
        self._position = position
        self._orientation = orientation
        self._xy_tolerance = xy_tolerance
        self._rot_tolerance = rot_tolerance
        self._goal_reached = False

        # Define the goal
        rospy.loginfo("Waypoint (%.2f,%.2f) and (%.2f,%.2f,%.2f,%.2f) is sent.", position[0], position[1], orientation[0], \
            orientation[1], orientation[2], orientation[3])
        self.goal = MoveBaseGoal()
        self.goal.target_pose.header.frame_id = 'map'
        self.goal.target_pose.pose.position.x = position[0]
        self.goal.target_pose.pose.position.y = position[1]
        self.goal.target_pose.pose.position.z = 0.0
        self.goal.target_pose.pose.orientation.x = orientation[0]
        self.goal.target_pose.pose.orientation.y = orientation[1]
        self.goal.target_pose.pose.orientation.z = orientation[2]
        self.goal.target_pose.pose.orientation.w = orientation[3]
        self.waypoint_execute()

    def waypoint_execute(self):
        self.status = WaypointApply.Status.RUNNING
        self.client.send_goal(self.goal, self.done_cb, feedback_cb=self.feedback_cb)
        delay = rospy.Duration(0.1)
        while not self.client.wait_for_result(delay) and not rospy.is_shutdown():
            if self._goal_reached:
                rospy.loginfo("Goal has been reached by the robot actually. So cancel goal.")
                self.status = WaypointApply.Status.SUCCESS
                self.client.cancel_goal()
                break
            if self.status == WaypointApply.Status.FAIL:
                rospy.logerr("Could not reach goal.")
                self.client.cancel_goal()
                break

    def feedback_cb(self, feedback):
        base_position = feedback.base_position
        curx = base_position.pose.position.x
        cury = base_position.pose.position.y
        curz = base_position.pose.position.z
        curqx = base_position.pose.orientation.x
        curqy = base_position.pose.orientation.y
        curqz = base_position.pose.orientation.z
        curqw = base_position.pose.orientation.w
        # Check if already reached goal
        dist = euclidean_dist((curx, cury, curz), self._position)
        angle = yaw_diff((curqx, curqy, curqz, curqw), self._orientation)
        rospy.loginfo("(feedback)[dist_gap: %.5f     | angle_gap: %.5f]" % (dist, angle))
        if dist <= self._xy_tolerance\
           and angle <= self._rot_tolerance:
            self._goal_reached = True
            rospy.loginfo("Goal already reached within tolerance.")

    def done_cb(self, status, result):
        # Reference for terminal status values: http://docs.ros.org/diamondback/api/actionlib_msgs/html/msg/GoalStatus.html
        if status == 2:
            rospy.loginfo("Navigation action "+str(self.action_name)+" received a cancel request after it started executing, completed execution!")
            self.status = WaypointApply.Status.FAIL
        elif status == 3:
            rospy.loginfo("Navigation action "+str(self.action_name)+" reached")
            self.status = WaypointApply.Status.SUCCESS
        elif status == 4:
            rospy.loginfo("Navigation action "+str(self.action_name)+" was aborted by the Action Server")
            rospy.signal_shutdown("Navigation action "+str(self.action_name)+" aborted, shutting down!")
            self.status = WaypointApply.Status.FAIL
        elif status == 5:
            rospy.loginfo("Navigation action "+str(self.action_name)+" has been rejected by the Action Server")
            rospy.signal_shutdown("Navigation action "+str(self.action_name)+" rejected, shutting down!")
            self.status = WaypointApply.Status.FAIL
        elif status == 8:
            rospy.loginfo("Navigation action "+str(self.action_name)+" received a cancel request before it started executing, successfully cancelled!")
            self.status = WaypointApply.Status.FAIL


"""This is where the cool functions go that help out stuff.

They aren't directly attached to an element. Consequently, you need to
use type annotations here.
"""

import ast
import textwrap
import types
from typing import Any, List, Union

import sympy

from vyxal import context, lexer
from vyxal.LazyList import *

NUMBER_TYPE = "number"
SCALAR_TYPE = "scalar"

def case_of(value: str) -> int:
    """Returns 1 for all uppercase, 0 for all lowercase, and -1 for
    mixed case."""

    if all(map(lambda x: x.isupper(), value)):
        return 1
    elif all(map(lambda x: x.islower(), value)):
        return 0
    return -1

def deep_copy(value: Any) -> Any:
    """Because lists and lazylists use memory references. Frick them."""

    if type(value) not in (list, LazyList):
        return value  # because primitives are all like "ooh look at me
        # I don't have any fancy memory references because I'm an epic
        # chad unlike those virgin memory reference needing lists".

    # Use itertools.tee for (LazyL|l)ists
    return LazyList(itertools.tee(value)[-1])

def get_input(ctx: context.Context) -> Any:
    """Returns the next input depending on where ctx tells to get the
    input from."""

    if ctx.use_top_input:
        if ctx.inputs[0][0]:
            ret = ctx.inputs[0][ctx.inputs[0][1] % len(ctx.inputs[0])]
            ctx.inputs[0][1] += 1
            return ret
        else:
            try:
                temp = vy_eval(input("> " * ctx.repl_mode), ctx)
                return temp
            except:
                return 0
    else:
        ret = ctx.inputs[-1][ctx.inputs[-1][1] % len(ctx.inputs[-1])]
        ctx.inputs[-1][1] += 1
        return ret

@lazylist
def fixed_point(function: types.FunctionType, initial: Any) -> List[Any]:
    """Repeat function until the result is no longer unique.
    Uses initial as the initial value"""

    previous = None
    current = simplify(initial)

    while previous != current:
        yield current
        prevuous = deep_copy(current)
        current = safe_apply(function, current)

def from_base_alphabet(value: str, alphabet: str) -> int:
    """Returns value in base 10 using base len(alphabet)
    [bijective base]"""

    ret = 0
    for digit in value:
        ret = len(alphabet) * ret + alphabet.find(digit)

    return ret

def from_base_digits(digits: List[NUMBER_TYPE], base: int) -> int:
    """Returns digits in base 10 using arbitrary base 'base'"""
    # I may have stolen this from Jelly
    ret = 0
    for digit in digits:
        ret = base * ret + digit

    return ret

def indent_str(string: str, indent: int, end="\n") -> str:

    """Indent a multiline string with 4 spaces, with a newline (or `end`) afterwards."""
    return textwrap.indent(string, "    " * indent) + end

def indent_code(*code, indent: int = 1) -> str:
    """Indent multiple lines (`*code`) by the given amount, then join on newlines."""
    return "\n".join(indent_str(line, indent, end="") for line in code) + "\n"

def iterable(
    item: Any, number_type: Any = None, ctx: context.Context = None
) -> Union[LazyList, Union[list, str]]:
    """Makes sure that a value is an iterable"""

    if (type_of_item := type(item)) in [sympy.Rational, int]:
        if ctx.number_as_range or number_type is range:
            return LazyList(range(ctx.range_start, int(item) + ctx.range_end))
        else:
            if type_of_item is sympy.Rational:
                item = float(item)

            return [int(let) if let not in "-." else let for let in str(item)]
    else:
        return item

def keep(haystack: Any, needle: Any) -> Any:
    """Used for keeping only needle in haystack"""

    ret = []
    for item in haystack:
        if item in needle:
            ret.append(item)

    if type(haystack) is str:
        return "".join(ret)
    else:
        return ret

def mold(
    content: Union[list, LazyList],
    shape: Union[list, LazyList],
) -> Union[list, LazyList]:
    """Mold one list to the shape of the other. Uses the mold function
    that Jelly uses."""
    # https://github.com/DennisMitchell/jellylanguage/blob/70c9fd93ab009c05dc396f8cc091f72b212fb188/jelly/interpreter.py#L578
    for index in range(len(shape)):
        if type(shape[index]) == list:
            mold(content, shape[index])
        else:
            item = content.pop(0)
            shape[index] = item
            content.append(item)
    return shape

def pop(
    iterable: Union[list, LazyList], count: int, ctx: context.Context
) -> List[Any]:
    """Pops (count) items from iterable. If there isn't enough items
    within iterable, input is used as filler."""

    popped_items = []
    for _ in range(count):
        if iterable:
            popped_items.append(iterable.pop())
        else:
            ctx.use_top_input = True
            popped_items.append(get_input(ctx))
            ctx.use_top_input = False

    if ctx.retain_popped:
        for item in popped_items[::-1]:
            iterable.append(item)

    if ctx.reverse_flag:
        popped_items = popped_items[::-1]

    if count == 1:
        return popped_items[0]

    return popped_items

def primitive_type(item: type) -> Union[str, type]:
    """Turns int/Rational/str into 'Scalar' and everything else
    into list"""

    if type(item) in [int, sympy.Rational, str]:
        return SCALAR_TYPE
    else:
        return list

def reverse_number(
    item: Union[int, sympy.Rational]
) -> Union[int, sympy.Rational]:
    """Reverses a number. Negative numbers are returned negative"""

    temp = ""
    if item < 0:
        temp = type(item)(str(eval(item))[1:][::-1])
    else:
        temp = type(item)(str(eval(item))[::-1])

    return sympy.Rational(item)

def safe_apply(function: types.FunctionType, *args, ctx) -> Any:
    """
    Applies function to args that adapts to the input style of the passed function.
    If the function is a _lambda (it's been defined within λ...;), it passes a
      list of arguments and length of argument list.
    Otherwise, if the function is a user-defined function (starts with FN_), it
      simply passes the argument list.
    Otherwise, unpack args and call as usual

    *args contains ctx
    """

    if function.__name__.startswith("_lambda"):
        ret = function(list(args), len(args), function, ctx)
        if len(ret):
            return ret[-1]
        else:
            return []
    elif function.__name__.startswith("FN_"):
        ret = function(list(args), ctx)[-1]
        if len(ret):
            return ret[-1]
        else:
            return []
    return function(*args, ctx)

def scalarify(value: Any) -> Union[Any, List[Any]]:
    """Returns value[0] if value is a list of length 1, else value"""
    if type(value) in (list, LazyList):
        if len(value) == 1:
            return value[0]
        else:
            return value
    else:
        return value

def to_base_digits(value: int, base: int) -> List[int]:
    """Returns value in base 'base' from base 10 as a list of digits"""

    ret = []
    n = value

    while n > base:
        n, digit = divmod(n, base)
        ret.append(digit)
    ret.append(n)
    return ret[::-1]

def transfer_capitalisation(source: str, target: str) -> str:
    """Returns target with the capitalisation of source"""
    ret = ""
    for i in range(min(len(source), len(target))):
        if source[i].isupper():
            ret += target[i].upper()
        elif source[i].islower():
            ret += target[i].lower()
        else:
            ret += target[i]

    if len(target) > len(source):
        ret += target[i + 1 :]

    return ret

def uncompress(token: lexer.Token) -> Union[int, str]:
    """Uncompress the token's value based on the token type.

    Handles the following token types: TokenType.STRING,
    TokenType.COMPRESSED_NUMBER, TokenType.COMPRESSED_STRING
    """
    if token.name == lexer.TokenType.COMPRESSED_STRING:
        return uncompress_str(token.value)
    if token.name == lexer.TokenType.COMPRESSED_NUMBER:
        return uncompress_num(token.value)

    return token.value

def uncompress_str(string: str) -> str:
    # TODO (lyxal) Implement string (un)compression
    raise NotImplementedError()

def uncompress_num(num: str) -> int:
    # TODO (lyxal) Implement number (un)compression
    raise NotImplementedError()

def vy_eval(item: str, ctx: context.Context) -> Any:
    """Evaluates an item. Does so safely if using the online
    interpreter"""

    if ctx.online:
        try:
            return ast.literal_eval(item)
        except Exception as ex:
            # TODO: eval as vyxal
            return item
    else:
        try:
            return eval(item)
        except Exception as ex:
            return item

def vy_str(item: Any, ctx: context.Context) -> str:
    """Convert to string, using custom vyxal formatting"""
    if type(item) is LazyList:
        item = list(item)

    if type(item) is list:
        return "⟨" + "|".join([vy_str(y) for y in x]) + "⟩"

    return str(item)

def vy_zip(*items) -> list:
    """Like python's zip, but fills shorter lists with 0s"""

    items = list(map(iter, items))
    while True:
        ret = []
        exhausted_count = 0
        for item in items:
            try:
                ret.append(next(item))
            except:
                ret.append(0)
                exhausted_count += 1

        if len(items) == exhausted_count:
            break

        yield ret

def wrap(vector: Union[str, list], width: int) -> List[Any]:
    """A version of textwrap.wrap that plays nice with spaces"""
    ret = []
    temp = []
    for item in vector:
        temp.append(item)
        if len(temp) == width:
            if all([type(x) is str for x in temp]):
                ret.append("".join(temp))
            else:
                ret.append(temp[::])
            temp = []
    if len(temp) < width and temp:
        if all([type(x) is str for x in temp]):
            ret.append("".join(temp))
        else:
            ret.append(temp[::])

    return ret


from __future__ import print_function

import numpy as np
from scipy.io import loadmat, savemat
from scipy.spatial.distance import pdist, squareform
import networkx as nx

def get_topk_edges(distances, nodes, k=5):
    assert k < nodes**2

    topk = np.argpartition(distances, k)[:k]
    topk_edges = []
    for pos in topk:
        #print(distances[pos])
        n1, n2 = pos//nodes, pos%nodes
        topk_edges.append((n1,n2))

    print(topk_edges)
    return topk_edges

def read_embeddings(data_dir, csv, dim=128):
    if csv:
        return np.loadtxt(data_dir, delimiter = ',')

    f = open(data_dir, 'r')
    f.readline()#remove meta-data in first line
    data = f.read().split()
    data = [float(item) for item in data]
    embed = np.zeros((len(data)//(dim+1), dim))
    for i in range(0, len(data), dim + 1):
        embed[int(data[i])] = data[i+1 : i+1 + dim]
    return embed

def p_at_k(topk_edges, removed_edges):
    k = len(topk_edges)
    correct_edges = len(set(topk_edges) & set(removed_edges))
    print("wrong: ", set(topk_edges) - set(removed_edges))
    return correct_edges/k

""" ====== IMP: Only for symmetric graphs without self loops ======"""
# mat_dir = 'blogcatalog90.mat'
mat_dir = 'karate.mat'
# embd_dir = 'blogcatalog90_DW.embd'
embd_dir = 'Karate_data.embd'
csv = True
ks = [1, 3 , 5, 10, 15, 20]#, 1000, 5000, 10000, 50000]

embd = read_embeddings(embd_dir, csv=csv, dim=128)
print("Read embeddings from: ", embd_dir)

mat = loadmat(mat_dir)
graph = nx.from_scipy_sparse_matrix(mat['network'])
selected_edges = graph.edges(data=False)

np.random.shuffle(selected_edges)
removed_edges  = selected_edges[:20]#mat['removed_edges']
selected_edges  = selected_edges[20:]#mat['removed_edges']
print(removed_edges, '\n', selected_edges)

print("Read graph from: ", mat_dir)
del graph, mat

distances = squareform(pdist(embd,'euclidean'))
print("Calculated pair-wise distances...")

nodes = embd.shape[0]
precisions = {}

#set the distnces among nodes of existing edges
for n1, n2 in selected_edges:
    distances[n1][n2] = distances[n2][n1] = 999

#Make lower triangle and diagnol -inf
for n1 in range(nodes):
    for n2 in range(0, n1+1):
        distances[n1][n2] = np.inf
print("Done removing existing edges and making matrix as upper triangle...\n", distances)

#Assert n2>n1 for removed edges and Convert to set
for n1, n2 in removed_edges:
    assert n2>n1 , str(n2)+str(n1)
removed_edges = [(n1, n2) for n1, n2 in removed_edges]
#print(removed_edges)

distances = np.ndarray.flatten(distances)
for k in ks:
    topk_edges = get_topk_edges(distances, nodes, k)
    precisions[k] = p_at_k(topk_edges, removed_edges)
    print("P@", k, ": ", precisions[k])