import os
import torch
import torchvision.transforms as transforms
import numpy as np
import cv2
import dlib
import scipy.io as sio
import argparse
import math

from modules import mobilenet_v1
from utils import ToTensorGjz, NormalizeGjz, crop_img
from modules.morphable.morphable_model import MorphableModel
from modules import mesh

class Demo():
    def __init__(self, opt):
        self.opt = opt
        self.out_dir = opt.out_dir
        if not os.path.exists(self.out_dir):
            os.makedirs(self.out_dir)
        self.load_model()

    def load_model(self):
        # 1. load trained model
        arch = 'mobilenet_1'
        state_dict = torch.load(self.opt.checkpoint_fp, map_location=lambda storage, loc: storage)['state_dict']
        new_state_dict = {k.replace('module.', ''): state_dict[k] for k in state_dict}
        self.model = getattr(mobilenet_v1, arch)(num_classes=self.opt.num_classes)
        model_dict = self.model.load_state_dict(new_state_dict)
        self.model.cuda()
        self.model.eval()

        # 2. load dlib model for face detection and landmark used for face_cropping
        self.face_regressor = dlib.shape_predictor(self.opt.dlib_landmark_model)
        self.face_detector = dlib.get_frontal_face_detector()

        # 3. transform
        self.transform = transforms.Compose([ToTensorGjz(), NormalizeGjz(mean=127.5, std=128)])

        # 4. load morphable_model
        self.morphable_model = MorphableModel(self.opt.morphable_model, self.opt.model_auxiliary)
        self.morphable_model.model_auxiliary['std_size'] = self.opt.std_size

        #for item in self.morphable_model.model['kpt_ind']:
        #    print(item)
        #for i in range(53215):
        #    if i not in self.morphable_model.model['kpt_ind']:
        #        print(i)
        #exit()

    def predict(self, img_path):
        img_name = img_path.split('/')[-1].split('.')[0]
        img_origin = cv2.imread(img_path)
        rects = self.face_detector(img_origin, 1)

        pts_res = []
        Ps = []  # Camera matrix collection
        poses = []  # pose collection
        vertices_list = []  # store multiple face vertices
        params_list = []
        roi_box_list = []
        colors_list = []

        for ind, rect in enumerate(rects):
            if self.opt.dlib_landmark:
                # - use landmark for roi box cropping
                pts = self.face_regressor(img_origin, rect).parts()
                pts = np.array([[pt.x, pt.y] for pt in pts]).T
                roi_box = self._parse_roi_box_from_landmark(pts)
            else:
                # - use detected face bbox
                bbox = [rect.left(), rect.top(), rect.right(), rect.bottom()]
                roi_box = self._parse_roi_box_from_landmark(bbox)
            roi_box_list.append(roi_box)

            # step one
            img = crop_img(img_origin, roi_box)
            img = cv2.resize(img, dsize=(self.opt.std_size, self.opt.std_size), interpolation=cv2.INTER_LINEAR)
            img = self.transform(img).unsqueeze(0)
            with torch.no_grad():
                img = img.cuda()
                params = self.model(img)
                params = params.squeeze().cpu().numpy().flatten().astype(np.float32)

            pts68 = self.morphable_model.predict_68pts(params, roi_box)

            # two-step for more acccurate bbox to crop face
            if self.opt.bbox_init == 'two':
                roi_box = self._parse_roi_box_from_landmark(pts68)
                img_step2 = crop_img(img_origin, roi_box)
                img_step2 = cv2.resize(img_step2, dsize=(self.opt.std_size, self.opt.std_size), interpolation=cv2.INTER_LINEAR)
                _img_step2 = img_step2.copy()
                img_step2 = self.transform(img_step2).unsqueeze(0)
                with torch.no_grad():
                    img_step2 = img_step2.cuda()
                    params = self.model(img_step2)
                    params = params.squeeze().cpu().numpy().flatten().astype(np.float32)
                    pts68 = self.morphable_model.predict_68pts(params, roi_box)

            params_list.append(params)

            vertices = self.morphable_model.predict_dense(params, roi_box)
            if self.opt.dump_obj:
                path = os.path.join(self.out_dir, '{}_{}.obj'.format(img_name, ind))
                colors = mesh.transform.get_colors_from_image(img_origin, vertices) / 255.
                colors_list.append(colors)
                #tp = self.morphable_model.get_tex_params(_type='random')
                #colors = self.morphable_model.generate_colors(tp)
                #colors = np.minimum(np.maximum(colors, 0), 1)
                mesh.interact.write_obj_with_colors(path, vertices.T, self.morphable_model.model['tri'], colors)
                print(self.morphable_model.model['tri'])

                h = img_origin.shape[0]
                w = img_origin.shape[1]
                image_vertices = vertices.copy().T
                #image_vertices[:, 1] = h - image_vertices[:, 1] - 1
                fitted_image = mesh.render.render_colors(image_vertices, self.morphable_model.triangles, colors, h, w) * 255.
                print(fitted_image.shape, image_vertices.shape, self.morphable_model.triangles.shape, colors.shape)
                cv2.imwrite(path.replace('obj', 'jpg'), fitted_image.astype('uint8'))

        #self.swap(*params_list, *colors_list, *roi_box_list, h, w)

    def swap(self, params1, params2, colors1, colors2, roi_box1, roi_box2, h, w):
        params1_2 = params1.copy()
        params1_2[12: 52] = params2[12: 52]
        params2_1 = params2.copy()
        params2_1[12: 52] = params1[12: 52]

        vertices1_2 = self.morphable_model.predict_dense(params1_2, roi_box1).T
        vertices2_1 = self.morphable_model.predict_dense(params2_1, roi_box2).T
        image1_2 = mesh.render.render_colors(vertices1_2, self.morphable_model.triangles, colors2, h, w) * 255.
        image2_1 = mesh.render.render_colors(vertices2_1, self.morphable_model.triangles, colors1, h, w) * 255.

        cv2.imwrite('results/1_2.jpg', image1_2.astype('uint8'))
        cv2.imwrite('results/2_1.jpg', image2_1.astype('uint8'))

    def _parse_roi_box_from_landmark(self, pts):
        """
        Args:
            pts: (2, n). n is the number of keypoints
        Returns:
            roi_box: list. (4, ). 4->(x1, y1, x2, y2)
        """
        bbox = [min(pts[0, :]), min(pts[1, :]), max(pts[0, :]), max(pts[1, :])]
        center = [(bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2]
        radius = max(bbox[2] - bbox[0], bbox[3] - bbox[1]) / 2
        bbox = [center[0] - radius, center[1] - radius, center[0] + radius, center[1] + radius]  # (x1, y1, x2, y2)

        llength = math.sqrt((bbox[2] - bbox[0]) ** 2 + (bbox[3] - bbox[1]) ** 2)
        center_x, center_y = center[0], center[1]

        roi_box = [0] * 4
        roi_box[0] = center_x - llength / 2
        roi_box[1] = center_y - llength / 2
        roi_box[2] = center_x + llength / 2
        roi_box[3] = center_y + llength / 2

        return roi_box

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='3DDFA inference pipeline')
    parser.add_argument('--file', default='./3DDFA/samples/1.jpg', type=str)
    #parser.add_argument('--file', default='results/test2.jpg', type=str)
    parser.add_argument('--std_size', default=120, type=int)
    parser.add_argument('--bbox_init', default='two', type=str)
    parser.add_argument('--num_classes', default=62, type=int)
    parser.add_argument('--checkpoint_fp', default='./3DDFA/models/phase1_wpdc_vdc.pth.tar', type=str)
    parser.add_argument('--dlib_landmark_model', default='./3DDFA/models/shape_predictor_68_face_landmarks.dat', type=str)
    parser.add_argument('--morphable_model', default='./Face3d/Data/BFM/Out/BFM.mat', type=str)
    parser.add_argument('--model_auxiliary', default='./3DDFA/utils/model_auxiliary.mat')
    parser.add_argument('--out_dir', default='results', type=str)
    parser.add_argument('--dump_obj', default=1, type=bool)
    parser.add_argument('--dlib_landmark', default=1, type=bool)

    opt = parser.parse_args()

    demo = Demo(opt)
    demo.predict(opt.file)


import tensorflow as tf
import VNet as vn
import numpy as np

from scipy.misc.pilutil import imread
from os import  walk
from os.path import join

DATA_DIR='D:\\program\\python\\job-logging.git\\trunk\\VNet-Tensorflow-master\\清洗过 - 副本\\原图像'
LABLE_DIR='D:\\program\\python\\job-logging.git\\trunk\\VNet-Tensorflow-master\\清洗过 - 副本\\label'
# DATA_DIR = 'D:\\zhengshunjie\\workspace\\deeplab\\组织切片2018.3.29\\清洗过 - 副本\\原图像'
# LABLE_DIR= 'D:\\zhengshunjie\\workspace\\deeplab\\组织切片2018.3.29\\清洗过 - 副本\\lable'
BATCH_SIZE = 8
NUM_CLASSES = 2
IMG_HEIGHT = 616
IMG_WIDTH = 539

def read_images(data_path,lable_path):
    data_filenames = next(walk(data_path))[2]
    lable_filenames=next(walk(lable_path))[2]
    data_num_files = len(data_filenames)
    lable_num_files=len(lable_filenames)
    data_images = np.zeros((data_num_files,IMG_HEIGHT,IMG_WIDTH,3),dtype=np.float32)
    lable_images=np.zeros((lable_num_files,IMG_HEIGHT,IMG_WIDTH),dtype=np.float32)
    valuequeue=None
    valuelabel=None
    for i in range(len(data_filenames)):
        data_img=imread(join(data_path,data_filenames[i]))
        lable_img=imread(join(lable_path,lable_filenames[i]),mode='L')
        data_img = tf.cast(data_img,tf.float32)
        lable_img=tf.cast(lable_img,tf.float32)
        # data_images[i]=data_img
        # lable_images[i]=lable_img
        valuequeue=tf.train.input_producer(data_img,shuffle=False)
        valuelabel=tf.train.input_producer(lable_img,shuffle=False)
    return valuequeue,valuelabel

images,lables = read_images(DATA_DIR,LABLE_DIR)

tf_input = tf.placeholder(dtype=tf.float32, shape=(1, IMG_WIDTH, IMG_HEIGHT, 0, 3))
tf_output= tf.placeholder(dtype=tf.float32, shape=(IMG_HEIGHT, IMG_HEIGHT, 1))

logits = vn.v_net(tf_input,1.0,3)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=tf_output,logits=logits))
train_step = tf.train.ProximalGradientDescentOptimizer(0.1).minimize(loss)

init=tf.global_variables_initializer()

saver=tf.train.Saver()
file_name='saved_model/model.ckpt'
def main():
    with tf.Session() as sess:
        sess.run(init)
        for epoch in range(500):
            for batch in range(BATCH_SIZE):
                num=np.random.uniform(0,700)
                batch_xs=images.dequeue()
                batch_ys=lables.dequeue()

                sess.run(train_step,feed_dict={tf_input:batch_xs,tf_output:batch_ys})

            print("loss:",loss)
        saver.saver(sess,file_name)

if __name__=='__main__':
    main()

# import tensorflow as tf
# import VNet as vn
#
# import numpy as np
# from os.path import join
# import matplotlib.pyplot as plt
# import convert_to_tfrecords
# BATCH_SIZE = 8
#
# TRAIN_FILE = 'train.tfrecords'
# VALIDATION_FILE = 'validation.tfrecords'
#
#
# NUM_CLASSES = 2
# IMG_HEIGHT = convert_to_tfrecords.IMG_HEIGHT
# IMG_WIDTH = convert_to_tfrecords.IMG_WIDTH
# IMG_CHANNELS = convert_to_tfrecords.IMG_CHANNELS
# IMG_PIXELS = IMG_HEIGHT * IMG_WIDTH * IMG_CHANNELS
#
# NUM_TRAIN = convert_to_tfrecords.NUM_TRAIN
# NUM_VALIDARION = convert_to_tfrecords.NUM_VALIDARION
#
# def read_and_decode(filename_queue):
#
#     reader = tf.TFRecordReader()
#
#     _,serialized_example = reader.read(filename_queue)
#
#     features = tf.parse_single_example(serialized_example,features={
#         'label_raw':tf.FixedLenFeature([],tf.string),
#         'image_raw':tf.FixedLenFeature([],tf.string)
#         })
#
#     image = tf.decode_raw(features['image_raw'],tf.uint8)
#     label = tf.decode_raw(features['label_raw'],tf.uint8)
#
# #    image.set_shape([IMG_PIXELS])
#     image = tf.reshape(image,[IMG_HEIGHT,IMG_WIDTH,IMG_CHANNELS])
# #    image = tf.cast(image, tf.float32) * (1. / 255) - 0.5
#
# #    label.set_shape([IMG_HEIGHT * IMG_WIDTH * 1])
#     label = tf.reshape(label,[IMG_HEIGHT,IMG_WIDTH,1])
# #    label = tf.cast(label,tf.float32) * (1. / 255) - 0.5
#     return image,label
#
#
# def inputs(data_set,batch_size,num_epochs):
#     if not num_epochs:
#         num_epochs = None
#     if data_set == 'train':
#         file = TRAIN_FILE
#     else:
#         file = VALIDATION_FILE
#
#     with tf.name_scope('input') as scope:
#         filename_queue = tf.train.string_input_producer([file])
#     image,label = read_and_decode(filename_queue)
#
#     return image,label
# def loss_funtion(logits_mat,target_mat):
#     print()
#
#
# #loss
#
# def main():
#     with tf.Session() as sess:
#         images, labels = inputs('train', BATCH_SIZE, 1)
#         plt.imshow(images.eval())
#         plt.show()
# if __name__=='__main__':
#     main()


import gleipnir.sampled_parameter as sampled_parameter
from gleipnir.sampled_parameter import SampledParameter
from scipy.stats import norm
import numpy as np

def test_initialization():
    sp = SampledParameter('sample', norm(0.0,1.0))
    return

def test_attributes():
    sp = SampledParameter('sample', norm(0.0,1.0))
    name = sp.name
    assert sp.name == 'sample'
    prior = sp.prior_dist
    n = norm(0.0,1.0)
    assert isinstance(sp.prior_dist, type(n))

def test_func_rvs():
    sp = SampledParameter('sample', norm(0.0,1.0))
    rvs = sp.rvs(10)
    assert len(rvs) == 10

def test_func_logprior():
    sp = SampledParameter('sample', norm(0.0,1.0))
    logprior = sp.logprior(0.5)
    assert np.isclose(logprior, -1.0439385332046727)

def test_func_prior():
    sp = SampledParameter('sample', norm(0.0,1.0))
    prior = sp.prior(0.5)
    assert np.isclose(prior, 0.3520653267642995)

def test_func_invcdf():
    sp = SampledParameter('sample', norm(0.0,1.0))
    invcdf = sp.invcdf(0.5)
    assert np.isclose(invcdf, 0.0)

if __name__ == '__main__':
    test_initialization()
    test_attributes()
    test_func_rvs()
    test_func_prior()
    test_func_logprior()
    test_func_invcdf()


"""
module for describing data process.

All data structure is describing as nested combination of `dict` or `list` for `ndarray`.
Data process is a translation from data structure to another data structure or typical `ndarray`.
Data cache can be implemented based on the dynamic features of `list` and `dict`.

The full data structure is

.. code::

    {
    "particle":{
        "A":{"p":...,"m":...}
        ...
    },
    "decay":[
        {
        "A->R1+B": {
            "R1": {
            "ang":  {
                "alpha":[...],
                "beta": [...],
                "gamma": [...]
            },
            "z": [[x1,y1,z1],...],
            "x": [[x2,y2,z2],...]
            },
            "B" : {...}
        },
        "R->C+D": {
            "C": {
            ...,
            "aligned_angle":{
                "alpha":[...],
                "beta":[...],
                "gamma":[...]
            }
            },
            "D": {...}
        },
        },
        {
        "A->R2+C": {...},
        "R2->B+D": {...}
        },
        ...
    ],
    "weight": [...]
    }

"""

import random
from pprint import pprint

import numpy as np

from .config import get_config
from .tensorflow_wrapper import tf

# import tensorflow as tf
# from pysnooper import  snoop

try:
    from collections.abc import Iterable
except ImportError:  # python version < 3.7
    from collections import Iterable

def set_random_seed(seed):
    """
    set random seed for random, numpy and tensorflow
    """
    np.random.seed(seed)
    tf.random.set_seed(seed)
    random.seed(seed)

def load_dat_file(
    fnames, particles, dtype=None, split=None, order=None, _force_list=False
):
    """
    Load ``*.dat`` file(s) of 4-momenta of the final particles.

    :param fnames: String or list of strings. File names.
    :param particles: List of Particle. Final particles.
    :param dtype: Data type.
    :param split: sizes of each splited dat files
    :param order: transpose order

    :return: Dictionary of data indexed by Particle.
    """
    n = len(particles)
    if dtype is None:
        dtype = get_config("dtype")

    if isinstance(fnames, str):
        fnames = [fnames]
    elif isinstance(fnames, Iterable):
        fnames = list(fnames)
    else:
        raise TypeError("fnames must be string or list of strings")

    datas = []
    sizes = []
    for fname in fnames:
        if fname.endswith(".npz"):
            data = np.load(fname)["arr_0"]
        elif fname.endswith(".npy"):
            data = np.load(fname)
        else:
            data = np.loadtxt(fname, dtype=dtype)
        data = np.reshape(data, (-1, 4))
        sizes.append(data.shape[0])
        datas.append(data)

    if split is None:
        n_total = sum(sizes)
        if n_total % n != 0:
            raise ValueError("number of data find {}/{}".format(n_total, n))
        n_data = n_total // n
        split = [size // n_data for size in sizes]

    if order is None:
        order = (1, 0, 2)

    ret = {}
    idx = 0
    for size, data in zip(split, datas):
        data_1 = data.reshape((-1, size, 4))
        data_2 = data_1.transpose(order)
        for i in data_2:
            part = particles[idx]
            ret[part] = i
            idx += 1

    return ret

def save_data(file_name, obj, **kwargs):
    """Save structured data to files. The arguments will be passed to ``numpy.save()``."""
    return np.save(file_name, obj, **kwargs)

def save_dataz(file_name, obj, **kwargs):
    """Save compressed structured data to files. The arguments will be passed to ``numpy.save()``."""
    return np.savez(file_name, obj, **kwargs)

def load_data(file_name, **kwargs):
    """Load data file from save_data. The arguments will be passed to ``numpy.load()``."""
    if "allow_pickle" not in kwargs:
        kwargs["allow_pickle"] = True
    data = np.load(file_name, **kwargs)
    try:
        return data["arr_0"].item()
    except IndexError:
        try:
            return data.item()
        except ValueError:
            return data

def _data_split(dat, batch_size, axis=0):
    data_size = dat.shape[axis]
    if axis == 0:
        for i in range(0, data_size, batch_size):
            yield dat[i : min(i + batch_size, data_size)]
    elif axis == -1:
        for i in range(0, data_size, batch_size):
            yield dat[..., i : min(i + batch_size, data_size)]
    else:
        raise Exception("unsupported axis: {}".format(axis))

def data_generator(data, fun=_data_split, args=(), kwargs=None, MAX_ITER=1000):
    """Data generator: call ``fun`` to each ``data`` as a generator. The extra arguments will be passed to ``fun``."""
    kwargs = kwargs if kwargs is not None else {}

    def _gen(dat):
        if isinstance(dat, dict):
            if not dat:
                for i in range(MAX_ITER):
                    yield {}
            ks, vs = [], []
            for k, v in dat.items():
                ks.append(k)
                vs.append(_gen(v))
            for s_data in zip(*vs):
                yield type(dat)(zip(ks, s_data))
        elif isinstance(dat, list):
            if not dat:
                for i in range(MAX_ITER):
                    yield []
            vs = []
            for v in dat:
                vs.append(_gen(v))
            for s_data in zip(*vs):
                yield list(s_data)
        elif isinstance(dat, tuple):
            vs = []
            for v in dat:
                vs.append(_gen(v))
            for s_data in zip(*vs):
                yield s_data
        else:
            for i in fun(dat, *args, **kwargs):
                yield i

    return _gen(data)

def data_split(data, batch_size, axis=0):
    """
    Split ``data`` for ``batch_size`` each in ``axis``.

    :param data: structured data
    :param batch_size: Integer, data size for each split data
    :param axis: Integer, axis for split, [option]
    :return: a generator for split data

    >>> data = {"a": [np.array([1.0, 2.0]), np.array([3.0, 4.0])], "b": {"c": np.array([5.0, 6.0])}, "d": [], "e": {}}
    >>> for i, data_i in enumerate(data_split(data, 1)):
    ...     print(i, data_to_numpy(data_i))
    ...
    0 {'a': [array([1.]), array([3.])], 'b': {'c': array([5.])}, 'd': [], 'e': {}}
    1 {'a': [array([2.]), array([4.])], 'b': {'c': array([6.])}, 'd': [], 'e': {}}

    """
    return data_generator(
        data, fun=_data_split, args=(batch_size,), kwargs={"axis": axis}
    )

split_generator = data_split

def data_map(data, fun, args=(), kwargs=None):
    """Apply fun for each data. It returns the same structure."""
    kwargs = kwargs if kwargs is not None else {}
    if isinstance(data, dict):
        return type(data)(
            {k: data_map(v, fun, args, kwargs) for k, v in data.items()}
        )
    if isinstance(data, list):
        return [data_map(data_i, fun, args, kwargs) for data_i in data]
    if isinstance(data, tuple):
        return tuple([data_map(data_i, fun, args, kwargs) for data_i in data])
    return fun(data, *args, **kwargs)

def data_struct(data):
    """get the structure of data, keys and shape"""
    if isinstance(data, dict):
        return type(data)({k: data_struct(v) for k, v in data.items()})
    if isinstance(data, list):
        return [data_struct(data_i) for data_i in data]
    if isinstance(data, tuple):
        return tuple([data_struct(data_i) for data_i in data])
    if hasattr(data, "shape"):
        return tuple(data.shape)
    return data

def data_mask(data, select):
    """
    This function using boolean mask to select data.

    :param data: data to select
    :param select: 1-d boolean array for selection
    :return: data after selection
    """
    ret = data_map(data, tf.boolean_mask, args=(select,))
    return ret

def data_cut(data, expr, var_map=None):
    """cut data with boolean expression

    :param data: data need to cut
    :param expr: cut expression
    :param var_map: variable map between parameters in expr and data, [option]

    :return: data after being cut,
    """
    var_map = var_map if isinstance(var_map, dict) else {}
    import sympy as sym

    expr_s = sym.sympify(expr)
    params = tuple(expr_s.free_symbols)
    args = [data_index(data, var_map.get(i.name, i.name)) for i in params]
    expr_f = sym.lambdify(params, expr, "tensorflow")
    mask = expr_f(*args)
    return data_mask(data, mask)

def data_merge(*data, axis=0):
    """This function merges data with the same structure."""
    assert len(data) > 0
    if isinstance(data[0], dict):
        assert all([isinstance(i, dict) for i in data]), "not all type same"
        all_idx = [set(list(i)) for i in data]
        idx = set.intersection(*all_idx)
        return type(data[0])(
            {i: data_merge(*[data_i[i] for data_i in data]) for i in idx}
        )
    if isinstance(data[0], list):
        assert all([isinstance(i, list) for i in data]), "not all type same"
        return [data_merge(*data_i) for data_i in zip(*data)]
    if isinstance(data[0], tuple):
        assert all([isinstance(i, tuple) for i in data]), "not all type same"
        return tuple([data_merge(*data_i) for data_i in zip(*data)])
    m_data = tf.concat(data, axis=axis)
    return m_data

def data_shape(data, axis=0, all_list=False):
    """
    Get data size.

    :param data: Data array
    :param axis: Integer. ???
    :param all_list: Boolean. ???
    :return:
    """

    def flatten(dat):
        ret = []

        def data_list(dat1):
            if hasattr(dat1, "shape"):
                ret.append(dat1.shape)
            else:
                ret.append(())

        data_map(dat, data_list)
        return ret

    shapes = flatten(data)
    if all_list:
        return shapes
    return shapes[0][axis]

def data_to_numpy(dat):
    """Convert Tensor data to ``numpy.ndarray``."""

    def to_numpy(data):
        if hasattr(data, "numpy"):
            return data.numpy()
        return data

    dat = data_map(dat, to_numpy)
    return dat

def data_to_tensor(dat):
    """convert data to ``tensorflow.Tensor``."""

    def to_tensor(data):
        return tf.convert_to_tensor(data)

    dat = data_map(dat, to_tensor)
    return dat

def flatten_dict_data(data, fun="{}/{}".format):
    """Flatten data as dict with structure named as ``fun``."""

    def dict_gen(dat):
        return dat.items()

    def list_gen(dat):
        return enumerate(dat)

    if isinstance(data, (dict, list, tuple)):
        ret = {}
        gen_1 = dict_gen if isinstance(data, dict) else list_gen
        for i, data_i in gen_1(data):
            tmp = flatten_dict_data(data_i)
            if isinstance(tmp, (dict, list, tuple)):
                gen_2 = dict_gen if isinstance(tmp, dict) else list_gen
                for j, tmp_j in gen_2(tmp):
                    ret[fun(i, j)] = tmp_j
            else:
                ret[i] = tmp
        return ret
    return data

def data_index(data, key):
    """Indexing data for key or a list of keys."""

    def idx(data, i):
        if isinstance(i, int):
            return data[i]
        assert isinstance(data, dict)
        if i in data:
            return data[i]
        for k, v in data.items():
            if str(k) == str(i):
                return v
        raise ValueError("{} is not found".format(i))

    if isinstance(key, (list, tuple)):
        keys = list(key)
        if len(keys) > 1:
            return data_index(idx(data, keys[0]), keys[1:])
        return idx(data, keys[0])
    return idx(data, key)

def data_strip(data, keys):
    if isinstance(keys, str):
        keys = [keys]
    if isinstance(data, dict):
        ret = {}
        for k, v in data.items():
            if k not in keys:
                ret[k] = data_strip(v, keys)
        return ret
    if isinstance(data, list):
        return [data_strip(data_i, keys) for data_i in data]
    if isinstance(data, tuple):
        return tuple([data_strip(data_i, keys) for data_i in data])
    return data

def check_nan(data, no_raise=False):
    """check if there is nan in data"""
    head_keys = []

    def _check_nan(dat, head):
        if isinstance(dat, dict):
            return {k: _check_nan(v, head + [k]) for k, v in dat.items()}
        if isinstance(dat, list):
            return [
                _check_nan(data_i, head + [i]) for i, data_i in enumerate(dat)
            ]
        if isinstance(dat, tuple):
            return tuple(
                [
                    data_struct(data_i, head + [i])
                    for i, data_i in enumerate(dat)
                ]
            )
        if np.any(tf.math.is_nan(dat)):
            if no_raise:
                return False
            raise ValueError("nan in data[{}]".format(head))
        return True

    return _check_nan(data, head_keys)


<gh_stars>0
import re
from fractions import Fraction

from . import PenalizerInterface, InvalidPenalizerConfigError
from ..utils import ConfigDictMixin

"""Penalize a late submission."""

class LatePenalty(ConfigDictMixin):
    UNITS_REGEX = re.compile(r'^(?P<mag>[0-9/]+)\s*(?P<unit>[a-z]*)$')

    def __init__(self, after, penalty):
        self.after = self.time_to_seconds(after)
        self.penalty, penalty_unit = self.split_units(penalty)

        if penalty_unit in ('pt', 'pts'):
            self.penalty /= 100
            self.penalty_points = True
        elif penalty_unit is None:
            self.penalty_points = False
        else:
            raise InvalidPenalizerConfigError("unknown penalty unit `{}'. try "
                                              "a fraction optionally followed "
                                              "by `pt'."
                                              .format(penalty_unit))

    @classmethod
    def split_units(cls, amount_str):
        if isinstance(amount_str, (int, float)):
            return Fraction(amount_str), None

        match = cls.UNITS_REGEX.match(amount_str.lower())

        if match is None:
            raise InvalidPenalizerConfigError("unknown units format `{}'"
                                              .format())

        return Fraction(match.group('mag')), match.group('unit') or None

    @classmethod
    def time_to_seconds(cls, time_str):
        mag, unit = cls.split_units(time_str)

        unit = unit or 's'
        units = {'s': 1, 'm': 60, 'h': 60*60, 'd': 24*60*60}

        if unit not in units:
            raise InvalidPenalizerConfigError("unknown time unit `{}'. try "
                                              "one of {}"
                                              .format(unit, ', '.join(units)))
        return mag * units[unit]

    def is_late(self, submission):
        return submission.seconds_late is not None \
               and submission.seconds_late > self.after

    def adjust_grade(self, grade):
        if self.penalty_points:
            return max(0, grade - self.penalty)
        else:
            return grade * (1 - self.penalty)

class LatePenalizer(PenalizerInterface):
    """
    Penalize students for late submissions.

    Configure it like this in the assignment config file. In this
    example, after 8 hours late you get 75 points off your grade (a 85
    would go to a 10).

    penalties:
    - name: LATE
      backend: LatePenalizer
      backend-options:
        penalties:
        - after: 1h
          penalty: 25pts
        - after: 8h
          penalty: 50pts

    That is, penalties are applied in order, and they do not stop when
    there is a match. They don't necessarily have to be in increasing
    order of `after' values.
    """

    def __init__(self, penalties):
        self.penalties = [LatePenalty.from_config_dict(p) for p in penalties]

    def adjust_grade(self, submission, grade):
        for penalty in self.penalties:
            if penalty.is_late(submission):
                grade = penalty.adjust_grade(grade)

        return grade


<filename>src/modeling/_gcnn.py
from __future__ import division
import torch
import torch.nn.functional as F
import numpy as np
import scipy.sparse
import math

class SparseMM(torch.autograd.Function):
    """Redefine sparse @ dense matrix multiplication to enable backpropagation.
    The builtin matrix multiplication operation does not support backpropagation in some cases.
    """
    @staticmethod
    def forward(ctx, sparse, dense):
        ctx.req_grad = dense.requires_grad
        ctx.save_for_backward(sparse)
        return torch.matmul(sparse, dense)

    @staticmethod
    def backward(ctx, grad_output):
        grad_input = None
        sparse, = ctx.saved_tensors
        if ctx.req_grad:
            grad_input = torch.matmul(sparse.t(), grad_output)
        return None, grad_input

def spmm(sparse, dense):
    return SparseMM.apply(sparse, dense)

def gelu(x):
    """Implementation of the gelu activation function.
        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
        Also see https://arxiv.org/abs/1606.08415
    """
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

class BertLayerNorm(torch.nn.Module):
    def __init__(self, hidden_size, eps=1e-12):
        """Construct a layernorm module in the TF style (epsilon inside the square root).
        """
        super(BertLayerNorm, self).__init__()
        self.weight = torch.nn.Parameter(torch.ones(hidden_size))
        self.bias = torch.nn.Parameter(torch.zeros(hidden_size))
        self.variance_epsilon = eps

    def forward(self, x):
        u = x.mean(-1, keepdim=True)
        s = (x - u).pow(2).mean(-1, keepdim=True)
        x = (x - u) / torch.sqrt(s + self.variance_epsilon)
        return self.weight * x + self.bias

class GraphResBlock(torch.nn.Module):
    """
    Graph Residual Block similar to the Bottleneck Residual Block in ResNet
    """
    def __init__(self, in_channels, out_channels, mesh_type='body'):
        super(GraphResBlock, self).__init__()
        self.in_channels = in_channels
        self.out_channels = out_channels
        self.lin1 = GraphLinear(in_channels, out_channels // 2)
        self.conv = GraphConvolution(out_channels // 2, out_channels // 2, mesh_type)
        self.lin2 = GraphLinear(out_channels // 2, out_channels)
        self.skip_conv = GraphLinear(in_channels, out_channels)
        # print('Use BertLayerNorm in GraphResBlock')
        self.pre_norm = BertLayerNorm(in_channels)
        self.norm1 = BertLayerNorm(out_channels // 2)
        self.norm2 = BertLayerNorm(out_channels // 2)

    def forward(self, x):
        trans_y = F.relu(self.pre_norm(x)).transpose(1,2)
        y = self.lin1(trans_y).transpose(1,2)

        y = F.relu(self.norm1(y))
        y = self.conv(y)

        trans_y = F.relu(self.norm2(y)).transpose(1,2)
        y = self.lin2(trans_y).transpose(1,2)

        z = x+y

        return z

# class GraphResBlock(torch.nn.Module):
#     """
#     Graph Residual Block similar to the Bottleneck Residual Block in ResNet
#     """
#     def __init__(self, in_channels, out_channels, mesh_type='body'):
#         super(GraphResBlock, self).__init__()
#         self.in_channels = in_channels
#         self.out_channels = out_channels
#         self.conv = GraphConvolution(self.in_channels, self.out_channels, mesh_type)
#         print('Use BertLayerNorm and GeLU in GraphResBlock')
#         self.norm = BertLayerNorm(self.out_channels)
#     def forward(self, x):
#         y = self.conv(x)
#         y = self.norm(y)
#         y = gelu(y)
#         z = x+y
#         return z

class GraphLinear(torch.nn.Module):
    """
    Generalization of 1x1 convolutions on Graphs
    """
    def __init__(self, in_channels, out_channels):
        super(GraphLinear, self).__init__()
        self.in_channels = in_channels
        self.out_channels = out_channels
        self.W = torch.nn.Parameter(torch.FloatTensor(out_channels, in_channels))
        self.b = torch.nn.Parameter(torch.FloatTensor(out_channels))
        self.reset_parameters()

    def reset_parameters(self):
        w_stdv = 1 / (self.in_channels * self.out_channels)
        self.W.data.uniform_(-w_stdv, w_stdv)
        self.b.data.uniform_(-w_stdv, w_stdv)

    def forward(self, x):
        return torch.matmul(self.W[None, :], x) + self.b[None, :, None]

class GraphConvolution(torch.nn.Module):
    """Simple GCN layer, similar to https://arxiv.org/abs/1609.02907."""
    def __init__(self, in_features, out_features, mesh='body', bias=True):
        super(GraphConvolution, self).__init__()
        device=torch.device('cuda')
        self.in_features = in_features
        self.out_features = out_features

        if mesh=='body':
            adj_indices = torch.load('./src/modeling/data/smpl_431_adjmat_indices.pt')
            adj_mat_value = torch.load('./src/modeling/data/smpl_431_adjmat_values.pt')
            adj_mat_size = torch.load('./src/modeling/data/smpl_431_adjmat_size.pt')
        elif mesh=='hand':
            adj_indices = torch.load('./src/modeling/data/mano_195_adjmat_indices.pt')
            adj_mat_value = torch.load('./src/modeling/data/mano_195_adjmat_values.pt')
            adj_mat_size = torch.load('./src/modeling/data/mano_195_adjmat_size.pt')

        self.adjmat = torch.sparse_coo_tensor(adj_indices, adj_mat_value, size=adj_mat_size).to(device)

        self.weight = torch.nn.Parameter(torch.FloatTensor(in_features, out_features))
        if bias:
            self.bias = torch.nn.Parameter(torch.FloatTensor(out_features))
        else:
            self.register_parameter('bias', None)
        self.reset_parameters()

    def reset_parameters(self):
        # stdv = 1. / math.sqrt(self.weight.size(1))
        stdv = 6. / math.sqrt(self.weight.size(0) + self.weight.size(1))
        self.weight.data.uniform_(-stdv, stdv)
        if self.bias is not None:
            self.bias.data.uniform_(-stdv, stdv)

    def forward(self, x):
        if x.ndimension() == 2:
            support = torch.matmul(x, self.weight)
            output = torch.matmul(self.adjmat, support)
            if self.bias is not None:
                output = output + self.bias
            return output
        else:
            output = []
            for i in range(x.shape[0]):
                support = torch.matmul(x[i], self.weight)
                # output.append(torch.matmul(self.adjmat, support))
                output.append(spmm(self.adjmat, support))
            output = torch.stack(output, dim=0)
            if self.bias is not None:
                output = output + self.bias
            return output

    def __repr__(self):
        return self.__class__.__name__ + ' (' \
               + str(self.in_features) + ' -> ' \
               + str(self.out_features) + ')'


import sounddevice as sd
import numpy as np
import time
import eda, model, predict
from scipy.io import wavfile
from tqdm import tqdm

DURATION = 10
NUMBER_OF_SAMPLES = 5
SAMPLE_RATE = 16000

def main():
  print('Ola! Quantos locutores voce deseja cadastrar?')
  print('Cada locutor ira gravar {} amostras de {} segundos cada.'.format(NUMBER_OF_SAMPLES, DURATION))
  qtd_locutores = int(input())
  with open('dados.csv', 'w') as f:
    f.write('fname,label\n')
    for i in range(qtd_locutores):
      print('Qual o nome do ' + str(i + 1) + 'o locutor?')
      nome = input()
      for j in range(NUMBER_OF_SAMPLES):
        print('Amostra ' + str(j + 1) + ', locutor = ' + nome + '. Pressione ENTER para iniciar a gravacao.')
        enter = input()

        my_recording = sd.rec(DURATION * SAMPLE_RATE, samplerate=SAMPLE_RATE, channels=1)
        for t in tqdm(range(DURATION)):
          time.sleep(1)
        # scaled_data = np.int16(my_recording / np.max(np.abs(my_recording)) * 32767)
        print('Deseja confirmar essa gravacao? Y/N')
        option = input()
        if (option == 'Y' or option == 'y'):
          wav_dir = 'wavfiles/' + nome + str(j + 1) + '.wav' ## exemplo -> user_samples/locutor1.wav
          wavfile.write(wav_dir, SAMPLE_RATE, my_recording)
          f.write(nome + str(j + 1) + '.wav' + ',' + nome + '\n')
        else:
          j = j - 1
  # eda.main()
  # model.main()
  # predict.main()

if __name__ == '__main__':
  main()


from math import sqrt
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import scipy.stats as sct
import seaborn as sns
import statsmodels.api as sm
import statsmodels.stats as st
from sklearn.decomposition import PCA
from loguru import logger
from IPython.core.pylabtools import figsize
from sklearn.linear_model import LinearRegression
from sklearn.feature_selection import RFE

#Analise inicial
fifa = pd.read_csv("fifa.csv")
columns_to_drop = ["Unnamed: 0", "ID", "Name", "Photo", "Nationality", "Flag",
                   "Club", "Club Logo", "Value", "Wage", "Special", "Preferred Foot",
                   "International Reputation", "Weak Foot", "Skill Moves", "Work Rate",
                   "Body Type", "Real Face", "Position", "Jersey Number", "Joined",
                   "Loaned From", "Contract Valid Until", "Height", "Weight", "LS",
                   "ST", "RS", "LW", "LF", "CF", "RF", "RW", "LAM", "CAM", "RAM", "LM",
                   "LCM", "CM", "RCM", "RM", "LWB", "LDM", "CDM", "RDM", "RWB", "LB", "LCB",
                   "CB", "RCB", "RB", "Release Clause"
]

try:
    fifa.drop(columns_to_drop, axis=1, inplace=True)
except KeyError:
    logger.warning(f"Columns already dropped")

fifa.dropna(inplace= True)

"""
    Qual fração da variância consegue ser explicada pelo primeiro componente principal de fifa?
    Responda como um único float (entre 0 e 1) arredondado para três casas decimais.
"""

def q1():

    pca = PCA(n_components = 1)
    project = pca.fit(fifa)
    varianciaExplicada = project.explained_variance_ratio_[0]
    return varianciaExplicada.round(3)

q1()

"""
    Quantos componentes principais precisamos para explicar 95% da variância total?
    Responda como un único escalar inteiro.
"""

def q2():

    pca095 = PCA(n_components= 0.95)
    project = pca095.fit_transform(fifa)
    numeroComponentesPrincipais = project.shape[1]
    return numeroComponentesPrincipais

q2()

"""
    Qual são as coordenadas (primeiro e segundo componentes principais) do ponto x abaixo?
    O vetor abaixo já está centralizado. Cuidado para não centralizar o vetor novamente
    (por exemplo, invocando PCA.transform() nele).
    Responda como uma tupla de float arredondados para três casas decimais.
"""

x = [0.87747123,  -1.24990363,  -1.3191255, -36.7341814,
     -35.55091139, -37.29814417, -28.68671182, -30.90902583,
     -42.37100061, -32.17082438, -28.86315326, -22.71193348,
     -38.36945867, -20.61407566, -22.72696734, -25.50360703,
     2.16339005, -27.96657305, -33.46004736,  -5.08943224,
     -30.21994603,   3.68803348, -36.10997302, -30.86899058,
     -22.69827634, -37.95847789, -22.40090313, -30.54859849,
     -26.64827358, -19.28162344, -34.69783578, -34.6614351,
     48.38377664,  47.60840355,  45.76793876,  44.61110193,
     49.28911284
]

def q3():

    pca = PCA().fit(fifa)
    c1,c2 = pca.components_.dot(x)[0:2].round(3)
    return c1,c2

q3()

"""
    Realiza RFE com estimador de regressão linear para selecionar cinco variáveis, eliminando uma a uma.
    Quais são as variáveis selecionadas? Responda como uma lista de nomes de variáveis.
"""

def q4():

    x = fifa.drop(columns="Overall")
    y = fifa["Overall"]

    rfe = RFE(estimator= LinearRegression(), n_features_to_select= 5)
    rfe.fit(x,y)

    indexFeatureSelect = rfe.get_support(indices=True)

    featureSelect = list(x.columns[indexFeatureSelect])

    return featureSelect

q4()


<gh_stars>0
import sys
import csv
from scipy import spatial
import db_wrapper

# Get list of all classes at MIT
classes = db_wrapper.get_all_classes()
num_classes = len(classes)

# Create hash table with keys = classes and values = index in list
class_table = {k:v for k, v in zip(classes, xrange(num_classes))}

# Create nxn matrix with n = total number of classes
matrix = [[0 for x in xrange(num_classes)] for y in xrange(num_classes)]

# Create nxn similarity table
similarity_table = [[0 for x in xrange(num_classes)] for y in xrange(num_classes)]

# Implementation of item-to-item CF
print "Executing CF..."
for cls in classes:
    cls_pos = class_table[cls]
    students = db_wrapper.get_students_of_class(cls)

    for student in students:
        subjects = db_wrapper.get_student_classes_all(student)

        for subject in subjects:    # subjects are the classes that student has taken
            matrix[cls_pos][class_table[subject]] += 1     # goes down column, then across the row

    count = 0
    for c in classes:
        if count > cls_pos:
            break

        # compute similarity between cls and c
        cls_list = matrix[cls_pos]
        c_list = matrix[class_table[c]]
        similarity = 1 - spatial.distance.cosine(cls_list, c_list)  # CONSIDER PEARSON INSTEAD!
        similarity_table[cls_pos][class_table[c]] = similarity
        count += 1

# output matrix to CSV
with open("output_matrix.csv", "wb") as f:
    writer = csv.writer(f)
    writer.writerows(matrix)

# output data to CSV
with open("output_similarities.csv", "wb") as f:
    writer = csv.writer(f)
    writer.writerow(classes)
    writer.writerows(similarity_table)
sys.exit()


r"""Main module that computes the optimal policy.

"""
import numpy as np

from adapt_hypo_test.two_states import util
from adapt_hypo_test.two_states.util import (nx_to_log_odds, m_to_r, pq_to_m, x_grid, lp_grid, log_p_log_q_to_m)
from scipy.special import logsumexp

def nop_reward(log_cond_reward, m, lp):
    r"""Computes
    :math:`\mathbb{P}(\hat{S}_{k-1} = S_{k-1}|y^{k-2},\sigma^{k-2},\sigma_{k-1}=e)`

    .. math::

        \mathbb{P}(\hat{S}_{k-1} = S_{k-1}|y^{k-2},\sigma^{k-2},\sigma_{k-1}=e) = \sum_{s_{k-1}}\mathbb{P}(\hat{S}_{k-1}=s_{k-1}|s_{k-1},y^{k-2},\sigma^{k-2},\sigma_{k-1}=e)\mathbb{P}(s_{k-1}|y^{k-2}\sigma_{k-2})

    :param log_cond_reward: :math:`\log(\mathbb{P}(\hat{S}_k=s_k|s_k,y^{k-1},\sigma^{k-1}))`
        This is an array of shape ``(2, 2k-1, 2k-1)``, where the last two
        indices correspond to the grid of posterior log odds. That is, if the
        posterior log odds

        .. math::

            x(y^{k-1}) = \log(\mathbb{P}(s_k=1|y^{k-1},\sigma^{k-1})/\mathbb{P}(s_k=0|y^{k-1},\sigma^{k-1}))

        is such that :math:`x(y^{k-1}) = n_0 r_0 + n_1 r_1`, then::

            log_cond_reward[s_k, n_0+i-1, n_1+i-1]

        is the corresponding :math:`\log(\mathbb{P}(\hat{S}_k=s_k|s_k,y^{k-1},\sigma^{k-1}))`
    :param m: :math:`\log(\mathbb{P}(y|s))`
        Should be of the form::

            m = np.log(np.array([[1-p, p],[q,1-q]]))
    :param lp: Shape
        ``(2, 2(k-2)+1, 2(k-2)+1)``. Given a log odds :math:`x = r_0 n_0 + r_1 n_1`,
        we should have ``lp[s_{k-1}, n_0+(k-2), n_1-(k-2)]`` be the corresponding
        log probability :math:`\log(\mathbb{P}(s_{k-1}|y^{k-2}, \sigma^{k-2}))`.
    :return: :math:`\mathbb{P}(\hat{S}_{k-1} = S_{k-1}|y^{k-2},\sigma^{k-2},\sigma_{k-1}=e)`,
        with shape ``(2(k-2)+1, 2(k-2)+1)``.
    """
    # Get :math:`\mathbb{P}(\hat{S}_{k-1}=s_{k-1}|s_{k-1},y^{k-2},\sigma^{k-2},\sigma_{k-1}=e)`
    nop = nop_cond_reward(log_cond_reward, m)
    nop = logsumexp(lp + nop, axis=0)
    return nop

def p_reward(log_cond_reward, m, lp):
    r"""
    Computes

    .. math::
        \mathbb{P}(\hat{S}_{k-1} = S_{k-1}|y^{k-2},\sigma^{k-2},\sigma_{k-1}=\nu) = \sum_{s_{k-1}}\mathbb{P}(\hat{S}_{k-1}=s_{k-1}|s_{k-1},y^{k-2},\sigma^{k-2},\sigma_{k-1}=\nu)\mathbb{P}(s_{k-1}|y^{k-2}\sigma_{k-2})

    See :py:func:`~adapt_hypo_test.two_states.no_transitions.nop_reward` for
    details on arguments.

    :return: :math:`\mathbb{P}(\hat{S}_{k-1} = S_{k-1}|y^{k-2},\sigma^{k-2},\sigma_{k-1}=\nu)`,
        with shape ``(2(k-2)+1, 2(k-2)+1)``. The indices correspond to the grid
        of log odds at step :math:`k-2`.
    """
    # Get :math:`\mathbb{P}(\hat{S}_{k-1}=s_{k-1}|s_{k-1},y^{k-2},\sigma^{k-2},\sigma_{k-1}=\nu)`
    p = p_cond_reward(log_cond_reward, m)
    p = logsumexp(lp + p, axis=0)
    return p

def get_actions(log_cond_reward, m):
    r"""
    Makes a grid of optimal actions, given a grid of conditional rewards
    :math:`\log(\mathbb{P}(\hat{S}_k = s_k|s_k, y^{k-1}, \sigma^{k-1}))`.

    We make this determination by performing a Bellman backup. That is, we
    compare the expected reward in the case of performing a permutation to the
    case where we do not. We return the nontrivial permutation if and only if
    the expected reward is larger if we were to.

    In equations, we return the truth value of

        .. math::

            \mathbb{P}\left( \hat{S}_0=S_0| y^{n-2},\sigma^{n-2}, \sigma_{n-1} = \nu \right)
            > \mathbb{P}\left( \hat{S}_0=S_0| y^{n-2},\sigma^{n-2}, \sigma_{n-1} = e \right)

    for all possible :math:`y^{n-2}`. As usual in this module, we instead look
    for all possible posterior log odds in place of all possible sets of data,
    to make the problem computationally tractable.

    See :py:func:`~adapt_hypo_test.two_states.no_transitions.nop_reward` for
    details on arguments.

    :return: The optimal actions at the given step, encoded as an array of shape
        ``(2*i+1, 2*i+1)``. If the returned value is ``sigma``, and the current
        posterior log odds is :math:`x = n_0 r_0 + n_1 r_1`, the optimal policy
        is to perform a permutation if and only if ``sigma[n_0+i, n_1+i]`` is
        ``True``.
    """
    r = m_to_r(m)
    k = log_cond_reward.shape[-1]
    k = k//2
    lp = lp_grid(k - 1, r)

    nop = nop_reward(log_cond_reward, m, lp)
    p = p_reward(log_cond_reward, m, lp)
    sigma = np.logical_and(p > nop, np.logical_not(np.isclose(p, nop)))
    return sigma

def nop_cond_reward(log_cond_reward, m):
    r"""
    Given :math:`\log(\mathbb{P}(\hat{S}_k = s_k|s_k, y^{k-1}, \sigma^{k-1}))`,
    computes
    :math:`\log(\mathbb{P}(\hat{S}_{k-1} = s_{k-1}|s_{k-1}, y^{k-2}, \sigma^{k-2}, \sigma_{k-1}=e))`
    according to

        .. math::

            \sum_{y_k} \mathbb{P}(\hat{S}_{k}=s_{k}|s_{k}=\sigma_{k-1}(s_{k-1}),y^{k-1},\sigma^{k-1}) \mathbb{P}(y_{k-1}|s_{k-1}, \sigma_{k-1}=e)

    Defining :math:`\chi_k^{s_k}(x_k) = \mathbb{P}(\hat{S}_k = s_k|s_k, y^{k-1}, \sigma^{k-1})`,
    we compute :math:`\chi_{k-1}^{s_{k-1}}(x_{k-1})` from
    :math:`\sum_{y_{k-1}}\mathbb{P}(y_{k-1}|\sigma_{k-1}(s_{k-1}))\chi_k^{s_k=\sigma_{k-1}(s_{k-1})}((-1)^{\sigma_{k-1}}x_{k-1}+(-1)^{y_{k-1}+1}r_{y_{k-1}})`
    where for this function, :math:`\sigma_{k-1} = e` is the identity permutation,
    and :math:`(-1)^e = 1`

    See :py:func:`~adapt_hypo_test.two_states.no_transitions.nop_reward` for
    details on arguments.

    :return: If the returned value is ``nop``, and the current posterior log
        odds is :math:`x = n_0 r_0 + n_1 r_1`, ``nop[s_k, n_0+k-1, n_1+k-1]`` is the
        value of :math:`\log(\mathbb{P}(\hat{S}_{k-1} = s_{k-1}|s_{k-1}, y^{k-2}, \sigma^{k-2}, \sigma_{k-1}=e))`.
    """
    nop = np.clip(np.logaddexp(
        # For :math:`y_{k-1} = 0`, we have
        # :math:`x_k = x_{k-1} - r_0`. Therefore shift down in the 1 direction.
        # The 2 direction doesn't change, but the possible values are from
        # :math:`n_1 = -(k-1)` to :math:`n_1 = (k-1)`, so slice into the 2 axis.
        m[:, 0][:, None, None] + log_cond_reward[:, :-2, 1:-1],
        # Similarly for :math:`y_{k-1} = 1`.
        m[:, 1][:, None, None] + log_cond_reward[:, 1:-1, 2:]
    ), None, 0.)
    return nop

def p_cond_reward(log_cond_reward, m):
    r"""
    Given :math:`\mathbb{P}(\hat{S}_k = s_k|s_k, y^{k-1}, \sigma^{k-1})`,
    computes
    :math:`\mathbb{P}(\hat{S}_{k-1} = s_{k-1}|s_{k-1}, y^{k-2}, \sigma^{k-2}, \sigma_{k-1}=\nu)`
    according to
    :math:`\sum_{y_k} \mathbb{P}(\hat{S}_{k}=s_{k}|s_{k}=\sigma_{k-1}(s_{k-1}),y^{k-1},\sigma^{k-1}) \mathbb{P}(y_{k-1}|s_{k-1}, \sigma_{k-1}=\nu)`.

    Defining :math:`\chi_k^{s_k}(x_k) = \mathbb{P}(\hat{S}_k = s_k|s_k, y^{k-1}, \sigma^{k-1})`,
    we compute :math:`\chi_{k-1}^{s_{k-1}}(x_{k-1})` from
    :math:`\sum_{y_{k-1}}\mathbb{P}(y_{k-1}|\sigma_{k-1}(s_{k-1}))\chi_k^{s_k=\sigma_{k-1}(s_{k-1})}((-1)^{\sigma_{k-1}}x_{k-1}+(-1)^{y_{k-1}+1}r_{y_{k-1}})`
    where for this function, :math:`\sigma_{k-1} = \nu` is the nontrivial permutation.
    and :math:`(-1)^\nu = -1`

    See :py:func:`~adapt_hypo_test.two_states.no_transitions.nop_reward` for
    details on arguments.

    :return: If the returned value is ``p``, and the current posterior log
        odds is :math:`x = n_0 r_0 + n_1 r_1`, ``p[s_k, n_0+k-1, n_1+k-1]`` is the
        value of :math:`\log(\mathbb{P}(\hat{S}_{k-1} = s_{k-1}|s_{k-1}, y^{k-2}, \sigma^{k-2}, \sigma_{k-1}=\nu))`.
    """
    # :math:`s_{k} = \sigma_{k-1}(s_{k-1})`, so flip the 0 axis.
    # :math:`x_{k} = (-1)x_{k-1} - (-1)^{y_{k-1}}r_{y_{k-1}}, so flip the 1 and 2 axes,
    # then apply offsets below.
    flip_reward = np.flip(log_cond_reward, axis=(0, 1, 2))
    # :math:`s_{k} = \sigma_{k-1}(s_{k-1})`, so flip the 0 axis.
    flip_m = np.flip(m, axis=0)
    p = np.clip(np.logaddexp(
        # If :math:`y_{k-1} = 0`, :math:`x_k = (-1)x_{k-1} - r_0`, so that
        # :math:`x_{k-1} + r_0 = -x_k`. Therefore shift up in the 1 axis.
        # The 2 direction doesn't change, but the possible values are from
        # :math:`n_1 = -(k-1)` to :math:`n_1 = (k-1)`, so slice into the 2 axis.
        flip_m[:, 0][:, None, None] + flip_reward[:, 2:, 1:-1],
        # Similarly for :math:`y_{k-1} = 1`.
        flip_m[:, 1][:, None, None] + flip_reward[:, 1:-1, :-2]
    ), None, 0.)
    return p

def backup_cond_reward(log_cond_reward, sigma, m):
    r"""From :math:`\log(\mathbb{P}(\hat{S}_k = s_k|s_k, y^{k-1}, \sigma^{k-1}))` and
    the optimal permutations :math:`\sigma_{k-1}`, computes
    :math:`\log(\mathbb{P}(\hat{S}_{k-1} = s_{k-1}|s_{k-1}, y^{k-2}, \sigma^{k-2}))`.

    See :py:func:`~adapt_hypo_test.two_states.no_transitions.nop_reward` for
    details on other arguments.

    :param sigma: A ``(2*k-1, 2*k-1)`` array with the optimal permutations at
        step :math:`k-1` corresponding to the grid of log odds.
    :return: An array of shape ``(2*(k-2)+1, 2*(k-2)+1)`` corresponding to
        :math:`\log(\mathbb{P}(\hat{S}_{k-1} = s_{k-1}|s_{k-1}, y^{k-2}, \sigma^{k-2}))`,
        with indices corresponding to the grid of log odds.
    """
    nop = nop_cond_reward(log_cond_reward, m)
    b_reward = nop
    p = p_cond_reward(log_cond_reward, m)
    b_reward[:, sigma] = p[:, sigma]
    return b_reward

def chi_base(n, r):
    r"""The start of the computation. This initializes an array of probability
    of correct inference after all the data has been seen, conditional on the
    true final state. We work backwards from here to obtain the optimal policy.

    :param n: Number of steps to compute for.
    :param r: The parameters
        :math:`r_0 = \log((1-p)/q)` and :math:`r_1 = \log((1-q)/p)`,
        as a numpy array of shape (2,).
    :return:
        :math:`\log(\mathbb{P}(\hat{S}_{n+1} = s_{n+1}|s_{n+1}, y^n, \sigma^n))`
        The dimensions of the returned array are
        0: :math:`s_{n+1}`, 1: :math:`n_0`, 2: :math:`n_1`
    """
    nx = x_grid(n)
    lo = nx_to_log_odds(nx, r)
    ret = np.zeros((2, 2 * n + 1, 2 * n + 1))
    ret[0][lo > 0] = -float('inf')
    ret[1][lo < 0] = -float('inf')
    ret[0][lo == 0] = np.log(.5)
    ret[1][lo == 0] = np.log(.5)
    return ret

def solve(p, q, n, log=False):
    r"""Solves for the optimal permutations.

    For the system with two states and two outputs and a trivial transition
    matrix, this solves for the optimal permutations to apply. See
    :py:mod:`adapt_hypo_test` for details.

    :param p: :math:`\mathbb{P}(y = 1|s = 0)` unless ``log`` is specified,
        in which case this should be :math:`\log(\mathbb{P}(y = 1|s = 0))`.
    :param q: :math:`\mathbb{P}(y = 0|s = 1)` unless ``log`` is specified,
        in which case this should be :math:`\log(\mathbb{P}(y = 0|s = 1))`.
    :param n: Number of steps of the hypothesis testing scenario.
    :param log: Defaults to ``False``. If ``True``, the input ``p`` and ``q``
        will be interpreted as being in log space.
    :return: A 2-tuple, containing

        The optimal permutations ``sigma``, encoded as a list of 2D arrays.
        The arrays are indexed by $n_0, n_1$, so that if the posterior log odds
        after ``i`` steps is ``r_0 n_0 + r_1 n_1``, the optimal permutation to
        apply is ``sigma[i][n_0+i, n_1+i]``. The entries of the arrays are 0 or 1,
        with 0 indicating the trivial permutation, and 1 indicating the
        nontrivial permutation.

        The conditional value function as an array of shape (2, 1, 1). This is

            .. math::

                \log(\chi_i^{s_0}) = \log(\mathbb{P}(\hat{S}_{0} = s_{0}|s_{0}))

        with the 0 index being :math:`s_0`. To obtain the probability of correct
        inference of initial state, compute

            >>> from scipy.special import logsumexp
            >>> logsumexp(chi.ravel() + log_prior)

        where chi is the second value returned from this function, and
        ``log_prior`` is the prior probability of the intial states.
    """
    if log:
        m = log_p_log_q_to_m(p, q)
    else:
        m = pq_to_m(p, q)
    r = m_to_r(m)
    chi = chi_base(n, r)
    sigmas = []
    for i in range(n):
        sigma = get_actions(chi, m)
        sigmas.append(sigma)
        chi = backup_cond_reward(chi, sigma, m)
    sigmas = list(reversed(sigmas))
    return sigmas, chi

def evaluate_sigma(sigma, x):
    p = util.index_with_nx(sigma, x)
    xx = np.copy(x)
    xx[p] = -xx[p]
    return xx, p


"""The WaveBlocks Project

Script to spawn new wavepackets aposteriori to an already completed simulation.
This can be used to evaluate spawning errors and test criteria for finding the
best spawning time.

@author: <NAME>
@copyright: Copyright (C) 2010, 2011 <NAME>
@license: Modified BSD License
"""

import sys

import numpy as np
from scipy import linalg as spla

from WaveBlocks import ParameterLoader
from WaveBlocks import ParameterProvider
from WaveBlocks import IOManager
from WaveBlocks import PotentialFactory
from WaveBlocks import HagedornWavepacket
from WaveBlocks import AdiabaticSpawner

def aposteriori_spawning(fin, fout, pin, pout, bid1, bid2):
    """
    :param f: An ``IOManager`` instance providing the simulation data.
    :param datablock: The data block where the results are.
    """
    # Number of time steps we saved
    timesteps = fin.load_wavepacket_timegrid()
    nrtimesteps = timesteps.shape[0]

    params = fin.load_wavepacket_parameters()
    coeffs = fin.load_wavepacket_coefficients()

    # A data transformation needed by API specification
    coeffs = [ [ coeffs[i,j,:] for j in xrange(pin["ncomponents"]) ] for i in xrange(nrtimesteps) ]

    # Initialize a mother Hagedorn wavepacket with the data from another simulation
    HAWP = HagedornWavepacket(pin)
    HAWP.set_quadrature(None)

    # Initialize an empty wavepacket for spawning
    SWP = HagedornWavepacket(pout)
    SWP.set_quadrature(None)

    # Initialize a Spawner
    AS = AdiabaticSpawner(pout)

    # Iterate over all timesteps and spawn
    for i, step in enumerate(timesteps):
        print(" Try spawning at timestep "+str(step))

        # Configure the wave packet and project to the eigenbasis.
        HAWP.set_parameters(params[i])
        HAWP.set_coefficients(coeffs[i])
        #HAWP.project_to_eigen(Potential)

        # Try spawning a new packet
        ps = AS.estimate_parameters(HAWP, 0)

        if ps is not None:
            SWP.set_parameters(ps)
            AS.project_coefficients(HAWP, SWP)

            # Save the spawned packet
            fout.save_wavepacket_parameters(HAWP.get_parameters(), timestep=step, blockid=bid1)
            fout.save_wavepacket_coefficients(HAWP.get_coefficients(), timestep=step, blockid=bid1)

            fout.save_wavepacket_parameters(SWP.get_parameters(), timestep=step, blockid=bid2)
            fout.save_wavepacket_coefficients(SWP.get_coefficients(), timestep=step, blockid=bid2)

if __name__ == "__main__":
    # Input data manager
    iomin = IOManager()

    # Read file with simulation data
    try:
        iomin.open_file(filename=sys.argv[1])
    except IndexError:
        iomin.open_file()

    # Read a configuration file with the spawn parameters
    try:
        parametersspawn = ParameterLoader().load_from_file(sys.argv[2])
    except IndexError:
        raise IOError("No spawn configuration given!")

    parametersin = iomin.load_parameters()

    # Check if we can start a spawning simulation
    if parametersin["algorithm"] != "hagedorn":
        iomin.finalize()
        raise ValueError("Unknown propagator algorithm.")

    # Parameters for spawning simulation
    parametersout = ParameterProvider()

    # Transfer the simulation parameters
    parametersout.set_parameters(parametersin)

    # And add spawning related configurations variables
    parametersout.update_parameters(parametersspawn)

    # How much time slots do we need
    tm = parametersout.get_timemanager()
    slots = tm.compute_number_saves()

    # Second IOM for output data of the spawning simulation
    iomout = IOManager()
    iomout.create_file(parametersout, filename="simulation_results_spawn.hdf5")

    # Some data in the global data block
    iomout.add_grid(parametersout, blockid="global")
    iomout.save_grid(iomin.load_grid(blockid="global"), blockid="global")

    # Allocate all the data blocks
    gid = iomout.create_group()
    bid1 = iomout.create_block(groupid=gid)
    bid2 = iomout.create_block(groupid=gid)
    # Block for remainder / mother after spawning
    iomout.add_wavepacket(parametersin, blockid=bid1)
    # Block for spawned packet
    iomout.add_wavepacket(parametersout, blockid=bid2)

    # Really do the aposteriori spawning simulation
    aposteriori_spawning(iomin, iomout, parametersin, parametersout, bid1, bid2)

    # Close the inpout/output files
    iomin.finalize()
    iomout.finalize()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jun  2 11:17:40 2021

@author: alex
"""

import numpy as np
import scipy.signal as signal
import matplotlib
import matplotlib.pyplot as plt

from pyro.dynamic import ContinuousDynamicSystem
from pyro.dynamic import linearize

# Embed font type in PDF
matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype']  = 42

###############################################################################
class TransferFunction( ContinuousDynamicSystem ):
    """Time-invariant transfer function representation of a dynamic system

    Y(s) / U(s) =   [ num ] / [den]

    """
    ############################################
    def __init__(self, num, den):

        self.num = num
        self.den = den

        n = den.shape[0] - 1

        ContinuousDynamicSystem.__init__( self, n, 1, 1)

        self.compute_ss()
        self.compute_poleszeros()

        # Plot params
        self.figsize = (5,3)
        self.dpi = 300
        self.fontsize = 5

    #############################################
    def compute_ss(self):

        A, B, C, D = signal.tf2ss(self.num, self.den)

        self.A = A
        self.B = B
        self.C = C
        self.D = D

    #############################################
    def compute_poleszeros(self):

        tf = signal.TransferFunction(self.num, self.den)

        self.poles = tf.poles
        self.zeros = tf.zeros

    #############################################
    def f(self, x, u, t):

        dx = np.dot(self.A, x) + np.dot(self.B, u)

        return dx

    #############################################
    def h(self, x, u, t):

        y = np.dot(self.C, x) + np.dot(self.D, u)

        return y

    ############################################
    def bode_plot(self):
        """ Plot frequency response """

        tf = signal.TransferFunction(self.num, self.den)

        w, mag, phase = tf.bode()

        fig , plots = plt.subplots(2, sharex=True, figsize=self.figsize,
                                  dpi=self.dpi, frameon=True)

        plots[0].semilogx(w, mag)
        plots[1].semilogx(w, phase)

        plots[0].set_ylabel(self.output_label[0] + ' ' + self.output_units[0] +'\n-------------------\n'+ self.input_label[0] + ' ' + self.input_units[0]
                 , fontsize= self.fontsize )
        plots[1].set_ylabel( 'Phase [rad]', fontsize= self.fontsize )
        plots[1].set_xlabel( 'Freq [rad/sec]', fontsize= self.fontsize )

        for i in [0,1]:
            plots[i].grid(True)
            plots[i].tick_params( labelsize = self.fontsize )

        fig.canvas.manager.set_window_title('Bode plot of ' + self.name)

        plt.show()

    ############################################
    def pz_map(self):
        """ Plot poles and zeros """

        self.compute_poleszeros()

        fig , plot = plt.subplots(1, sharex=True, figsize=self.figsize,
                                  dpi=self.dpi, frameon=True)

        plot.plot( self.poles.real, self.poles.imag, 'x')
        plot.plot( self.zeros.real, self.zeros.imag, 'o')
        plot.grid(True)
        plot.tick_params( labelsize = self.fontsize )

        fig.canvas.manager.set_window_title('Poles and zeros of ' + self.name)

        plt.show()

#################################################################
def ss2tf( ss, u_index , y_index):
    """
    Compute the transfer function of a given input and output of a state-space
    """
    nums, den = signal.ss2tf(ss.A, ss.B, ss.C, ss.D, u_index)

    num = nums[y_index]

    tf = TransferFunction(num, den)

    tf.name = (ss.output_label[y_index] + '/' + ss.input_label[u_index] +
               ' transfer function of ' + ss.name )

    tf.output_label[0] = ss.output_label[y_index]
    tf.output_units[0] = ss.output_units[y_index]
    tf.input_label[0]  = ss.input_label[u_index]
    tf.input_units[0]  = ss.input_units[u_index]

    return tf

'''
#################################################################
##################          Main                         ########
#################################################################
'''

if __name__ == "__main__":
    """ MAIN TEST """

    num = np.array([1])
    den = np.array([1,0,1])

    #TF = signal.TransferFunction( num, den)

    sys = TransferFunction( num , den)

    #sys.bode_plot()

    from pyro.dynamic import pendulum

    non_linear_sys = pendulum.SinglePendulum()

    non_linear_sys.lc1     = 1
    non_linear_sys.m1      = 2
    non_linear_sys.I1      = 2
    non_linear_sys.d1      = 2
    non_linear_sys.gravity = 9.81

    linearized_sys = linearize( non_linear_sys )
    siso_sys       = ss2tf( linearized_sys, 0, 0)

    print('Poles',siso_sys.poles)
    print('num',siso_sys.num)
    print('den',siso_sys.den)

    siso_sys.bode_plot()


<gh_stars>0
from PIL import Image
import numpy as np
import cv2
from scipy.misc import imread
import torch
from matplotlib import pyplot as plt
def prep_im_for_blob(im, target_size):
    """Mean subtract and scale an image for use in a blob."""

    im = im.astype(np.float32, copy=False)
    # changed to use pytorch models
    im /= 255.  # Convert range to [0,1]
    # normalization for pytroch pretrained models.
    # https://pytorch.org/docs/stable/torchvision/models.html
    pixel_means = [0.485, 0.456, 0.406]
    pixel_stdens = [0.229, 0.224, 0.225]

    # normalize manual
    im -= pixel_means  # Minus mean
    im /= pixel_stdens  # divide by stddev

    # im = im[:, :, ::-1]
    im_shape = im.shape
    im_size_min = np.min(im_shape[0:2])
    im_size_max = np.max(im_shape[0:2])
    im_scale = float(target_size) / float(im_size_min)
    # Prevent the biggest axis from being more than MAX_SIZE
    # if np.round(im_scale * im_size_max) > max_size:
    #     im_scale = float(max_size) / float(im_size_max)
    # im = imresize(im, im_scale)
    im = cv2.resize(im, None, None, fx=im_scale, fy=im_scale,
                    interpolation=cv2.INTER_LINEAR)

    return im, im_scale

def crop(image, purpose, size):

    cut_image = image[int(purpose[1]):int(purpose[3]),int(purpose[0]):int(purpose[2]),:]

    height, width = cut_image.shape[0:2]

    max_hw   = max(height, width)
    cty, ctx = [height // 2, width // 2]

    cropped_image  = np.zeros((max_hw, max_hw, 3), dtype=cut_image.dtype)

    x0, x1 = max(0, ctx - max_hw // 2), min(ctx + max_hw // 2, width)
    y0, y1 = max(0, cty - max_hw // 2), min(cty + max_hw // 2, height)

    left, right = ctx - x0, x1 - ctx
    top, bottom = cty - y0, y1 - cty

    cropped_cty, cropped_ctx = max_hw // 2, max_hw // 2
    y_slice = slice(cropped_cty - top, cropped_cty + bottom)
    x_slice = slice(cropped_ctx - left, cropped_ctx + right)
    cropped_image[y_slice, x_slice, :] = cut_image[y0:y1, x0:x1, :]

    return cv2.resize(cropped_image, (size,size), interpolation=cv2.INTER_LINEAR)

if __name__ == '__main__':
    version = 'custom'
    if version == 'coco':
        im = imread('/home/yjyoo/PycharmProjects/data/coco/images/val2017/000000397133.jpg')
        query_im = imread('/home/yjyoo/PycharmProjects/data/coco/images/val2017/000000007816.jpg')

    else:
        im = imread('./scene.jpeg')
        _im = cv2.resize(im, dsize=(640, 480), interpolation=cv2.INTER_AREA)
        plt.imshow(_im)
        plt.show()
        print(np.array(im).shape)           # (480, 640, 3)
        query_im = imread('./query.jpeg')
        query_im = cv2.resize(query_im, dsize=(640, 480), interpolation=cv2.INTER_AREA)
        print(np.array(query_im).shape)     # (480, 640, 3)

    im, im_scale = prep_im_for_blob(im, target_size=600)
    im = torch.tensor(im)
    im = torch.unsqueeze(im, 0)
    im = im.transpose(1, 3)
    im = im.transpose(2, 3)

    # query_im = crop(query_im, [505.54, 53.01, 543.08, 164.09], size=128)
    query_im, query_im_scale = prep_im_for_blob(query_im, target_size=128)
    query_im = torch.tensor(query_im)
    query_im = torch.unsqueeze(query_im, 0)
    query_im = query_im.transpose(1, 3)
    query_im = query_im.transpose(2, 3)

    print(im.shape)         # torch.Size([1, 3, 600, 899])
    print(im_scale)         # 1.405152224824356
    print(query_im.shape)   # torch.Size([1, 3, 128, 128])
    print(query_im_scale)   # 1.0

# 640, 425


# -*- coding: utf-8 -*-
"""Some matrix specialization."""

import time
from pygimli.core import _pygimli_ as pg
from pygimli.utils.geostatistics import covarianceMatrix
import numpy as np

from . import _pygimli_ as pgcore
from . import (CMatrix, CSparseMapMatrix, CSparseMatrix, ElementMatrix,
               IVector, MatrixBase, R3Vector, RVector)

from .logger import critical, warn

# make core matrices (now in pgcor, later pg.core) available here for brevity
BlockMatrix = pgcore.RBlockMatrix
IdentityMatrix = pgcore.IdentityMatrix

BlockMatrix = pgcore.RBlockMatrix
SparseMapMatrix = pgcore.RSparseMapMatrix
SparseMatrix = pgcore.RSparseMatrix
Matrix = pgcore.RMatrix

class MultMatrix(pgcore.MatrixBase):
    """Base Matrix class for all matrix types holding a matrix."""
    def __init__(self, A, verbose=False):
        self._A = A
        self.ndim = self._A.ndim
        super(MultMatrix, self).__init__(verbose)

    @property
    def A(self):
        return self._A

    @A.setter
    def A(self, A):
        self._A = A

    def rows(self):
        """Return number of rows (using underlying matrix)."""
        return self.A.rows()  # this should be _A

    def cols(self):
        """Return number of columns (using underlying matrix)."""
        return self.A.cols()  # this should be _A

    def save(self, filename):
        """So it can be used in inversion with dosave flag"""
        pass

class MultLeftMatrix(MultMatrix):
    """Matrix consisting of actual RMatrix and lef-side vector."""

    def __init__(self, A, left, verbose=False):
        """Constructor saving matrix and vector."""
        if A.rows() != len(left):
            raise Exception("Matrix columns do not fit vector length!")
        super(MultLeftMatrix, self).__init__(A, verbose)

        self._l = left

    @property
    def l(self):  # better use left and right instead (pylint E743)?
        return self._l

    @l.setter
    def r(self, l):
        self._l = l

    def mult(self, x):
        """Multiplication from right-hand-side (dot product A*x)."""
        return self.A.mult(x) * self.l

    def transMult(self, x):
        """Multiplication from right-hand-side (dot product A.T * x)"""
        return self.A.transMult(x * self.l)

LMultRMatrix = MultLeftMatrix  # alias for backward compatibility

class MultRightMatrix(MultMatrix):
    """Some Matrix, multiplied with a right hand side vector r."""

    def __init__(self, A, r=None, verbose=False):
        super(MultRightMatrix, self).__init__(A, verbose)

        if r is None:
            self._r = pgcore.RVector(self.cols(), 1.0)
        else:
            self._r = r

    @property
    def r(self):
        return self._r

    @r.setter
    def r(self, r):
        self._r = r

    def mult(self, x):
        """Return M*x = A*(r*x)"""
        if hasattr(x, '__len__') and hasattr(self.r, '__len__'):
            if len(x) != len(self.r):
                # assuming A was complex
                # warn('need to double x')
                # print('mult:', self.A.rows(), " x " , self.A.cols(),
                #        'x:', len(x), 'r:', len(self.r))
                # print(self.perm)
                return self.A.mult(x[self.perm] * self.r)
                # return self.A.mult(pgcore.cat(x, x) * self.r)
        return self.A.mult(x * self.r)

    def transMult(self, x):
        """Return M.T*x=(A.T*x)*r"""
        # print('transmult', self.A.rows(), " x " , self.A.cols(), x, self.r, )
        return self.A.transMult(x) * self.r

RMultRMatrix = MultRightMatrix  # alias for backward compatibility

class MultLeftRightMatrix(MultMatrix):
    """Matrix consisting of actual RMatrix and left-hand-side vector."""

    def __init__(self, A, left, right, verbose=False):
        """Constructor saving matrix and vector."""
        if A.cols() != len(right):
            raise Exception("Matrix columns do not fit right vector length!")
        if A.rows() != len(left):
            raise Exception("Matrix rows do not fit left vector length!")

        super(MultLeftRightMatrix, self).__init__(A, verbose)
        self._r = right
        self._l = left

    @property
    def l(self):
        return self._l

    @l.setter
    def l(self, l):
        self._l = l

    @property
    def r(self):
        return self._r

    @r.setter
    def r(self, r):
        self._r = r

    def mult(self, x):
        """Multiplication from right-hand-side (dot product A*x)."""
        return self.A.mult(x * self._r) * self._l

    def transMult(self, x):
        """Multiplication from right-hand-side (dot product A.T*x)."""
        return self.A.transMult(x * self._l) * self._r

LRMultRMatrix = MultLeftRightMatrix  # alias for backward compatibility

__BlockMatrix_addMatrix__ = pgcore.RBlockMatrix.addMatrix

def __BlockMatrix_addMatrix_happy_GC__(self, M, row=None, col=None,
                                       scale=1.0, transpose=False):
    """Add an existing matrix to this block matrix and return a unique index.

    As long row and col are None, the Matrix will not be used until a matrix
    entry is has been added.

    Monkeypatched version to increase the reference counter of M to keep the
    garbage collector happy.

    TODO
    ----
        * Add numpy matrices or convertable
        * Transpose is only for 1d arrays. Needed for matrices?

    Parameters
    ----------
    M: pg.core Matrix | pg.Vector | 1d iterable
        Matrix to add to the block.
    row: long
        Starting row index.
    col: long
        Starting column index.
    scale: float[1.0]
        Scale all matrix entries.
    transpose: bool [False]
        Transpose the matrix.
    """
    if M.ndim == 1:
        if transpose is False:
            _M = SparseMapMatrix(list(range(len(M))), [0]*len(M), M)
        else:
            _M = SparseMapMatrix([0]*len(M), list(range(len(M))), M)
        M = _M
    else:
        if transpose is True:
            if isinstance(M, pgcore.RSparseMapMatrix):
                warn('Move me to core')
                v = pg.RVector()
                i = pg.IndexArray([0])
                j = pg.IndexArray([0])
                M.fillArrays(v, i, j)
                M = SparseMapMatrix(j, i, v)
            else:
                critical("don't know yet how to add transpose matrix of type",
                         type(M))

    if not hasattr(self, '__mats__'):
        self.__mats__ = []
    self.__mats__.append(M)

    matrixID = __BlockMatrix_addMatrix__(self, M)

    if row is not None and col is not None:
        self.addMatrixEntry(matrixID, row, col, scale)

    return matrixID

pgcore.RBlockMatrix.addMatrix = __BlockMatrix_addMatrix_happy_GC__
pgcore.RBlockMatrix.add = __BlockMatrix_addMatrix_happy_GC__
# pgcore.CBlockMatrix.addMatrix = __BlockMatrix_addMatrix_happy_GC__
# pgcore.CBlockMatrix.add = __BlockMatrix_addMatrix_happy_GC__

class Add2Matrix(pgcore.MatrixBase):
    """Matrix by adding two matrices."""

    def __init__(self, A, B):
        super().__init__()
        self.A = A
        self.B = B
        assert A.rows() == B.rows()
        assert A.cols() == B.cols()

    def mult(self, x):
        """Return M*x = A*(r*x)"""
        return self.A.mult(x) + self.B.mult(x)

    def transMult(self, x):
        """Return M.T*x=(A.T*x)*r"""
        return self.A.transMult(x) + self.B.transMult(x)

    def cols(self):
        """Number of columns."""
        return self.A.cols()

    def rows(self):
        """Number of rows."""
        return self.A.rows()

class Mult2Matrix(pgcore.MatrixBase):
    """Matrix  by multiplying two matrices."""

    def __init__(self, A, B):
        super().__init__()
        self.A = A
        self.B = B
        assert A.cols() == B.rows()

    def mult(self, x):
        """Return M*x = A*(r*x)"""
        return self.A.mult(self.B.mult(x))

    def transMult(self, x):
        """Return M.T*x=(A.T*x)*r"""
        return self.B.transMult(self.A.transMult(x))

    def cols(self):
        """Number of columns."""
        return self.B.cols()

    def rows(self):
        """Number of rows."""
        return self.A.rows()

class DiagonalMatrix(pgcore.MatrixBase):
    """Square matrix with a vector on the main diagonal."""

    def __init__(self, d):
        super().__init__()
        self.d = d

    def mult(self, x):
        """Return M*x = r*x (element-wise)"""
        return x * self.d

    def transMult(self, x):
        """Return M.T*x=(A.T*x)*r"""
        return x * self.d

    def cols(self):
        """Number of columns (length of diagonal)."""
        return len(self.d)

    def rows(self):
        """Number of rows (length of diagonal)."""
        return len(self.d)

class Cm05Matrix(pgcore.MatrixBase):
    """Matrix implicitly representing the inverse square-root."""

    def __init__(self, A, verbose=False):
        """Constructor saving matrix and vector.

        Parameters
        ----------
        A : ndarray
            numpy type (full) matrix
        """
        from scipy.linalg import eigh  # , get_blas_funcs

        if A.shape[0] != A.shape[1]:  # rows/cols for pgcore matrix
            raise Exception("Matrix must by square (and symmetric)!")

        self.size = A.shape[0]
        t = time.time()
        self.ew, self.EV = eigh(A)
        self.mul = np.sqrt(1./self.ew)
        if verbose:
            pgcore.info('(C) Time for eigenvalue decomposition:{:.1f}s'.format(
                time.time() - t))

        self.A = A
        super().__init__(verbose)  # only in Python 3

    def rows(self):
        """Return number of rows (using underlying matrix)."""
        return self.size

    def cols(self):
        """Return number of columns (using underlying matrix)."""
        return self.size

    def mult(self, x):
        """Multiplication from right-hand side (dot product)."""
        part1 = (np.dot(np.transpose(x), self.EV).T*self.mul).reshape(-1, 1)
        return self.EV.dot(part1).reshape(-1,)
#        return self.EV.dot((x.T.dot(self.EV)*self.mul).T)

    def transMult(self, x):
        """Multiplication from right-hand side (dot product)."""
        return self.mult(x)  # matrix is symmetric by definition

class NDMatrix(BlockMatrix):
    """Diagonal block (block-Jacobi) matrix derived from pg.matrix.BlockMatrix.

    (to be moved to a better place at a later stage)
    """

    def __init__(self, num, nrows, ncols):
        super(NDMatrix, self).__init__()  # call inherited init function
        self.Ji = []  # list of individual block matrices
        for i in range(num):
            self.Ji.append(pg.Matrix())
            self.Ji[-1].resize(nrows, ncols)
            n = self.addMatrix(self.Ji[-1])
            self.addMatrixEntry(n, nrows * i, ncols * i)

        self.recalcMatrixSize()
        print(self.rows(), self.cols())

class GeostatisticConstraintsMatrix(pg.MatrixBase):
    """Geostatistic constraints matrix

    Uses geostatistical operators described by Jordi et al. (2018),
    however corrects for the remaining non-smooth (damping) part by
    correcting for the spur of the inverse root matrix.

    <NAME>., <NAME>., <NAME>., <NAME>. & <NAME>.
    (2018): Geostatistical regularisation operators for geophysical inverse
    problems on irregular meshes. Geoph. J. Int. 213, 1374-1386,
    doi:10.1093/gji/ggy055.
    """
    def __init__(self, CM=None, mesh=None, **kwargs):
        """Initialize by computing the covariance matrix & its inverse root.

        Parameters
        ----------
        CM : pg.Matrix or pg.SparseMapMatrix
            covariance matrix, if not given, use mesh and I
        mesh : pg.Mesh
            mesh of which the cell midpoints are used for covariance
        I : float | iterable of floats
            axis correlation length (isotropic) or lengths (anisotropic)
        dip : float [0]
            angle of main axis corresponding to I[0] (2D) or I[0]&I[1] (3D)
        strike : float [0]
            angle of main axis corresponding to I[0] versus I[1] (3D)
        withRef : bool [False]
            neglect spur (reference model effect) that is otherwise corrected
        """
        super().__init__()
        if isinstance(CM, pg.Mesh):
            CM = covarianceMatrix(CM, **kwargs)
        if CM is None:
            CM = covarianceMatrix(mesh, **kwargs)

        self.nModel = CM.shape[0]
        self.CM05 = Cm05Matrix(CM)
        self.spur = self.CM05 * pg.RVector(self.nModel, 1.0)
        if kwargs.pop('withRef', False):
            self.spur *= 0.0

    def mult(self, x):
        return self.CM05.mult(x) - self.spur * x

    def transMult(self, x):
        return self.CM05.transMult(x) - self.spur * x

    def cols(self):
        return self.nModel

    def rows(self):
        return self.nModel


<filename>AMnet/preprocessing.py
import numpy
import scipy.spatial
import pkg_resources
import os
import scipy.io
import AMnet.utilities
import random

def extract_data(size):

    path_to_data = pkg_resources.resource_filename("AMnet", "data/Voxelized_GE_Files_"+size+"/")

    file_list = [f for f in os.listdir(path_to_data) if os.path.isfile(os.path.join(path_to_data, f))]
    geometry = []
    flattened_geometry = []
    mass = []
    support_material = []
    print_time =[]
    sumsum = []
    for file in file_list:
        data = scipy.io.loadmat(os.path.join(path_to_data, file))
        # print(sum(data['c'].flatten()))
        v = sum(data['c'].flatten()/pow(len(data['c']), 3))
        if v > 0:
            geometry.append(data['c'])
            flattened_geometry.append(data['c'].flatten())
            mass.append(data['mass'])
            print_time.append(data['print_time'])
            support_material.append(data['support_material'])

    N = len(geometry)
    print(N)
    G = len(geometry[0])

    numpy.savez(pkg_resources.resource_filename('AMnet', 'data/data_geometry.npz'),
                geometry=geometry,
                flattened_geometry=flattened_geometry,
                mass=mass,
                support_material=support_material,
                print_time=print_time)
    numpy.savez(pkg_resources.resource_filename('AMnet', 'data/constants.npz'), N=N, G=G)

    return True

def augment_data():
    # Load the data
    geometry, mass, support_material, print_time, _, N, G = AMnet.utilities.load_data()

    # Define some variables
    augmented_geometry = []
    augmented_flattened_geometry = []
    augmented_mass = []
    augmented_print_time = []
    augmented_support_material = []

    # Make some rotation options
    faces = []
    faces.append([1, (1, 2)])
    faces.append([2, (1, 2)])
    faces.append([3, (1, 2)])
    faces.append([4, (1, 2)])
    faces.append([1, (0, 2)])
    faces.append([3, (0, 2)])

    for i, part in enumerate(geometry):
        for face in faces:
            m = mass[i]
            sm = support_material[i]
            pt = print_time[i]
            temp = part
            temp = numpy.rot90(temp, face[0], face[1])
            for quadrant in range(4):
                temp_rotated = numpy.rot90(temp, quadrant+1, (0, 1))
                augmented_geometry.append(temp_rotated)
                augmented_flattened_geometry.append(temp_rotated.flatten())
                augmented_mass.append(m)
                augmented_print_time.append(pt)
                augmented_support_material.append(sm)

    # Shuffle the data
    x = list(range(len(augmented_mass)))
    random.shuffle(x)

    numpy.savez(pkg_resources.resource_filename('AMnet', 'data/data_geometry.npz'),
                geometry=[augmented_geometry[idx] for idx in x],
                flattened_geometry=[augmented_flattened_geometry[idx] for idx in x],
                mass=[augmented_mass[idx] for idx in x],
                support_material=[augmented_support_material[idx] for idx in x],
                print_time=[augmented_print_time[idx] for idx in x])
    numpy.savez(pkg_resources.resource_filename('AMnet', 'data/constants.npz'), N=len(augmented_mass), G=G)

    print(len(augmented_mass))

    return True


<reponame>gtpash/rom-operator-inference-Python3
# pre/test_reprojection.py
"""Tests for rom_operator_inference.pre._reprojection."""

import pytest
import numpy as np
from scipy import linalg as la

import rom_operator_inference as opinf

# Reprojection schemes ========================================================
def test_reproject_discrete(n=50, m=5, r=3):
    """Test pre._reprojection.reproject_discrete()."""
    # Construct dummy operators.
    k = 1 + r + r*(r+1)//2
    D = np.diag(1 - np.logspace(-1, -2, n))
    W = la.qr(np.random.normal(size=(n,n)))[0]
    A = W.T @ D @ W
    Ht = np.random.random((n,n,n))
    H = (Ht + Ht.T) / 20
    H = H.reshape((n, n**2))
    B = np.random.random((n,m))
    U = np.random.random((m,k))
    B1d = np.random.random(n)
    U1d = np.random.random(k)
    basis = np.eye(n)[:,:r]
    x0 = np.zeros(n)
    x0[0] = 1

    # Try with bad initial condition shape.
    with pytest.raises(ValueError) as exc:
        opinf.pre.reproject_discrete(lambda x: x, basis, x0[:-1], k)
    assert exc.value.args[0] == "basis and initial condition not aligned"

    # Linear case, no inputs.
    def f(x):
        return A @ x
    X_ = opinf.pre.reproject_discrete(f, basis, x0, k)
    assert X_.shape == (r,k)
    rom = opinf.DiscreteOpInfROM("A").fit(basis, X_)
    assert np.allclose(basis @ X_, rom.predict(X_[:,0], k))
    assert np.allclose(rom.A_.entries, basis.T @ A @ basis)

    # Linear case, 1D inputs.
    def f(x, u):
        return A @ x + B1d * u
    X_ = opinf.pre.reproject_discrete(f, basis, x0, k, U1d)
    assert X_.shape == (r,k)
    rom = opinf.DiscreteOpInfROM("AB").fit(basis, X_, inputs=U1d)
    assert np.allclose(X_, basis.T @ rom.predict(X_[:,0], k, U1d))
    assert np.allclose(rom.A_.entries, basis.T @ A @ basis)
    assert np.allclose(rom.B_.entries.flatten(), basis.T @ B1d)

    # Linear case, 2D inputs.
    def f(x, u):
        return A @ x + B @ u
    X_ = opinf.pre.reproject_discrete(f, basis, x0, k, U)
    assert X_.shape == (r,k)
    rom = opinf.DiscreteOpInfROM("AB").fit(basis, X_, inputs=U)
    assert np.allclose(X_, basis.T @ rom.predict(X_[:,0], k, U))
    assert np.allclose(rom.A_.entries, basis.T @ A @ basis)
    assert np.allclose(rom.B_.entries, basis.T @ B)

    # Quadratic case, no inputs.
    def f(x):
        return A @ x + H @ np.kron(x,x)
    X_ = opinf.pre.reproject_discrete(f, basis, x0, k)
    assert X_.shape == (r,k)
    rom = opinf.DiscreteOpInfROM("AH").fit(basis, X_)
    assert np.allclose(X_, basis.T @ rom.predict(X_[:,0], k))
    assert np.allclose(rom.A_.entries,
                       basis.T @ A @ basis, atol=1e-6, rtol=1e-6)
    H_ = basis.T @ H @ np.kron(basis, basis)
    for _ in range(10):
        x_ = np.random.random(r)
        x2_ = np.kron(x_, x_)
        assert np.allclose(rom.H_(x_), H_ @ x2_)

def test_reproject_continuous(n=100, m=20, r=10):
    """Test pre._reprojection.reproject_continuous()."""
    # Construct dummy operators.
    k = 1 + r + r*(r+1)//2
    D = np.diag(1 - np.logspace(-1, -2, n))
    W = la.qr(np.random.normal(size=(n,n)))[0]
    A = W.T @ D @ W
    Ht = np.random.random((n,n,n))
    H = (Ht + Ht.T) / 20
    H = H.reshape((n, n**2))
    B = np.random.random((n,m))
    U = np.random.random((m,k))
    B1d = np.random.random(n)
    U1d = np.random.random(k)
    basis = np.eye(n)[:,:r]
    X = np.random.random((n,k))

    # Try with bad initial condition shape.
    with pytest.raises(ValueError) as exc:
        opinf.pre.reproject_continuous(lambda x:x, basis, X[:-1,:])
    assert exc.value.args[0] == \
        f"states and basis not aligned, first dimension {n-1} != {n}"

    # Linear case, no inputs.
    def f(x):
        return A @ x
    X_, Xdot_ = opinf.pre.reproject_continuous(f, basis, X)
    assert X_.shape == (r,k)
    assert Xdot_.shape == (r,k)
    rom = opinf.ContinuousOpInfROM("A").fit(basis, X_, Xdot_)
    assert np.allclose(rom.A_.entries, basis.T @ A @ basis)

    # Linear case, 1D inputs.
    def f(x, u):
        return A @ x + B1d * u
    X_, Xdot_ = opinf.pre.reproject_continuous(f, basis, X, U1d)
    assert X_.shape == (r,k)
    assert Xdot_.shape == (r,k)
    rom = opinf.ContinuousOpInfROM("AB").fit(basis, X_, Xdot_, U1d)
    assert np.allclose(rom.A_.entries, basis.T @ A @ basis)
    assert np.allclose(rom.B_.entries.flatten(), basis.T @ B1d)

    # Linear case, 2D inputs.
    def f(x, u):
        return A @ x + B @ u
    X_, Xdot_ = opinf.pre.reproject_continuous(f, basis, X, U)
    assert X_.shape == (r,k)
    assert Xdot_.shape == (r,k)
    rom = opinf.ContinuousOpInfROM("AB").fit(basis, X_, Xdot_, U)
    assert np.allclose(rom.A_.entries, basis.T @ A @ basis)
    assert np.allclose(rom.B_.entries, basis.T @ B)

    # Quadratic case, no inputs.
    def f(x):
        return A @ x + H @ np.kron(x,x)
    X_, Xdot_ = opinf.pre.reproject_continuous(f, basis, X)
    assert X_.shape == (r,k)
    assert Xdot_.shape == (r,k)
    rom = opinf.ContinuousOpInfROM("AH").fit(basis, X_, Xdot_)
    assert np.allclose(rom.A_.entries, basis.T @ A @ basis)
    H_ = basis.T @ H @ np.kron(basis, basis)
    for _ in range(10):
        x_ = np.random.random(r)
        x2_ = np.kron(x_, x_)
        assert np.allclose(rom.H_(x_), H_ @ x2_)