import sympy
from qibo import gates, K
from qibo.config import raise_error

class HamiltonianTerm:
    """Term of a :class:`qibo.core.hamiltonians.SymbolicHamiltonian`.

    Symbolic Hamiltonians are represented by a list of
    :class:`qibo.core.terms.HamiltonianTerm` objects storred in the
    ``SymbolicHamiltonian.terms`` attribute. The mathematical expression of
    the Hamiltonian is the sum of these terms.

    Args:
        matrix (np.ndarray): Full matrix corresponding to the term representation
            in the computational basis. Has size (2^n, 2^n) where n is the
            number of target qubits of this term.
        q (list): List of target qubit ids.
    """

    def __init__(self, matrix, *q):
        for qi in q:
            if qi < 0:
                raise_error(ValueError, "Invalid qubit id {} < 0 was given "
                                        "in Hamiltonian term".format(qi))
        if not (matrix is None or isinstance(matrix, K.qnp.numeric_types) or
                isinstance(matrix, K.qnp.tensor_types)):
            raise_error(TypeError, "Invalid type {} of symbol matrix."
                                   "".format(type(matrix)))
        dim = int(matrix.shape[0])
        if 2 ** len(q) != dim:
            raise_error(ValueError, "Matrix dimension {} given in Hamiltonian "
                                    "term is not compatible with the number "
                                    "of target qubits {}."
                                    "".format(dim, len(q)))
        self.target_qubits = tuple(q)
        self._gate = None
        self.hamiltonian = None
        self._matrix = matrix

    @property
    def matrix(self):
        """Matrix representation of the term."""
        return self._matrix

    @property
    def gate(self):
        """:class:`qibo.abstractions.gates.Unitary` gate that implements the action of the term on states."""
        if self._gate is None:
            self._gate = gates.Unitary(self.matrix, *self.target_qubits)
        return self._gate

    def exp(self, x):
        """Matrix exponentiation of the term."""
        return K.qnp.expm(-1j * x * self.matrix)

    def expgate(self, x):
        """:class:`qibo.abstractions.gates.Unitary` gate implementing the action of exp(term) on states."""
        return gates.Unitary(self.exp(x), *self.target_qubits)

    def merge(self, term):
        """Creates a new term by merging the given term to the current one.

        The resulting term corresponds to the sum of the two original terms.
        The target qubits of the given term should be a subset of the target
        qubits of the current term.
        """
        if not set(term.target_qubits).issubset(set(self.target_qubits)):
            raise_error(ValueError, "Cannot merge HamiltonianTerm acting on "
                                    "qubits {} to term on qubits {}."
                                    "".format(term.target_qubits, self.target_qubits))
        matrix = K.np.kron(term.matrix, K.qnp.eye(2 ** (len(self) - len(term))))
        matrix = K.np.reshape(matrix, 2 * len(self) * (2,))
        order = []
        i = len(term)
        for qubit in self.target_qubits:
            if qubit in term.target_qubits:
                order.append(term.target_qubits.index(qubit))
            else:
                order.append(i)
                i += 1
        order.extend([x + len(order) for x in order])
        matrix = K.np.transpose(matrix, order)
        matrix = K.np.reshape(matrix, 2 * (2 ** len(self),))
        return HamiltonianTerm(self.matrix + matrix, *self.target_qubits)

    def __len__(self):
        return len(self.target_qubits)

    def __mul__(self, x):
        return HamiltonianTerm(x * self.matrix, *self.target_qubits)

    def __rmul__(self, x):
        return self.__mul__(x)

    def __call__(self, state, density_matrix=False):
        """Applies the term on a given state vector or density matrix."""
        if density_matrix:
            self.gate.density_matrix = True
            return self.gate._density_matrix_half_call(state)
        return self.gate(state) # pylint: disable=E1102

class SymbolicTerm(HamiltonianTerm):
    """:class:`qibo.core.terms.HamiltonianTerm` constructed using ``sympy`` expression.

    Example:
        ::

            from qibo.symbols import X, Y
            from qibo.core.terms import SymbolicTerm
            sham = X(0) * X(1) + 2 * Y(0) * Y(1)
            termsdict = sham.as_coefficients_dict()
            sterms = [SymbolicTerm(c, f) for f, c in termsdict.items()]

    Args:
        coefficient (complex): Complex number coefficient of the underlying
            term in the Hamiltonian.
        factors (sympy.Expr): Sympy expression for the underlying term.
        symbol_map (dict): Dictionary that maps symbols in the given ``factors``
            expression to tuples of (target qubit id, matrix).
            This is required only if the expression is not created using Qibo
            symbols and to keep compatibility with older versions where Qibo
            symbols were not available.
    """

    def __init__(self, coefficient, factors=1, symbol_map={}):
        self.coefficient = complex(coefficient)
        self._matrix = None
        self._gate = None
        self.hamiltonian = None

        # List of :class:`qibo.symbols.Symbol` that represent the term factors
        self.factors = []
        # Dictionary that maps target qubit ids to a list of matrices that act on each qubit
        self.matrix_map = {}
        if factors != 1:
            for factor in factors.as_ordered_factors():
                # check if factor has some power ``power`` so that the corresponding
                # matrix is multiplied ``pow`` times
                if isinstance(factor, sympy.Pow):
                    factor, pow = factor.args
                    assert isinstance(pow, sympy.Integer)
                    assert isinstance(factor, sympy.Symbol)
                    pow = int(pow)
                else:
                    pow = 1

                # if the user is using ``symbol_map`` instead of qibo symbols,
                # create the corresponding symbols
                if factor in symbol_map:
                    from qibo.symbols import Symbol
                    q, matrix = symbol_map.get(factor)
                    factor = Symbol(q, matrix, name=factor.name)

                if isinstance(factor, sympy.Symbol):
                    if isinstance(factor.matrix, K.qnp.tensor_types):
                        self.factors.extend(pow * [factor])
                        q = factor.target_qubit
                        # if pow > 1 the matrix should be multiplied multiple
                        # when calculating the term's total matrix so we
                        # repeat it in the corresponding list that will
                        # be used during this calculation
                        # see the ``SymbolicTerm.matrix`` property for the
                        # full matrix calculation
                        if q in self.matrix_map:
                            self.matrix_map[q].extend(pow * [factor.matrix])
                        else:
                            self.matrix_map[q] = pow * [factor.matrix]
                    else:
                        self.coefficient *= factor.matrix
                elif factor == sympy.I:
                    self.coefficient *= 1j
                elif factor.is_number:
                    self.coefficient *= complex(factor)
                else: # pragma: no cover
                    raise_error(TypeError, "Cannot parse factor {}.".format(factor))

        self.target_qubits = tuple(sorted(self.matrix_map.keys()))

    @property
    def matrix(self):
        """Calculates the full matrix corresponding to this term.

        Returns:
            Matrix as a ``np.ndarray`` of shape ``(2 ** ntargets, 2 ** ntargets)``
            where ``ntargets`` is the number of qubits included in the factors
            of this term.
        """
        if self._matrix is None:
            def matrices_product(matrices):
                """Product of matrices that act on the same tuple of qubits.

                Args:
                    matrices (list): List of matrices to multiply, as exists in
                        the values of ``SymbolicTerm.matrix_map``.
                """
                if len(matrices) == 1:
                    return matrices[0]
                matrix = K.np.copy(matrices[0])
                for m in matrices[1:]:
                    matrix = matrix @ m
                return matrix

            self._matrix = self.coefficient
            for q in self.target_qubits:
                matrix = matrices_product(self.matrix_map.get(q))
                self._matrix = K.np.kron(self._matrix, matrix)
        return self._matrix

    def copy(self):
        """Creates a shallow copy of the term with the same attributes."""
        new = self.__class__(self.coefficient)
        new.factors = self.factors
        new.matrix_map = self.matrix_map
        new.target_qubits = self.target_qubits
        return new

    def __mul__(self, x):
        """Multiplication of scalar to the Hamiltonian term."""
        new = self.copy()
        new.coefficient *= x
        if self._matrix is not None:
            new._matrix = x * self._matrix
        return new

    def __call__(self, state, density_matrix=False):
        for factor in self.factors:
            if density_matrix:
                factor.gate.density_matrix = True
                state = factor.gate._density_matrix_half_call(state)
            else:
                state = factor.gate(state)
        return self.coefficient * state

class TermGroup(list):
    """Collection of multiple :class:`qibo.core.terms.HamiltonianTerm` objects.

    Allows merging multiple terms to a single one for faster exponentiation
    during Trotterized evolution.

    Args:
        term (:class:`qibo.core.terms.HamiltonianTerm`): Parent term of the group.
            All terms appended later should target a subset of the parents'
            target qubits.
    """

    def __init__(self, term):
        super().__init__([term])
        self.target_qubits = set(term.target_qubits)
        self._term = None

    def append(self, term):
        """Appends a new :class:`qibo.core.terms.HamiltonianTerm` to the collection."""
        super().append(term)
        self.target_qubits |= set(term.target_qubits)
        self._term = None

    def can_append(self, term):
        """Checks if a term can be appended to the group based on its target qubits."""
        return set(term.target_qubits).issubset(self.target_qubits)

    @classmethod
    def from_terms(cls, terms):
        """Divides a list of terms to multiple :class:`qibo.core.terms.TermGroup`s.

        Terms that target the same qubits are grouped to the same group.

        Args:
            terms (list): List of :class:`qibo.core.terms.HamiltonianTerm` objects.

        Returns:
            List of :class:`qibo.core.terms.TermGroup` objects that contain
            all the given terms.
        """
        # split given terms according to their order
        # order = len(term.target_qubits)
        orders = {}
        for term in terms:
            if len(term) in orders:
                orders[len(term)].append(term)
            else:
                orders[len(term)] = [term]

        groups = []
        # start creating groups with the higher order terms as parents and then
        # append each term of lower order to the first compatible group
        for order in sorted(orders.keys())[::-1]:
            for child in orders[order]:
                flag = True
                for i, group in enumerate(groups):
                    if group.can_append(child):
                        group.append(child)
                        flag = False
                        break
                if flag:
                    groups.append(cls(child))
        return groups

    @property
    def term(self):
        """Returns a single :class:`qibo.core.terms.HamiltonianTerm`. after merging all terms in the group."""
        if self._term is None:
            self._term = self.to_term()
        return self._term

    def to_term(self, coefficients={}):
        """Calculates a single :class:`qibo.core.terms.HamiltonianTerm` by merging all terms in the group.

        Args:
            coefficients (dict): Optional dictionary that allows passing a different
                coefficient to each term according to its parent Hamiltonian.
                Useful for :class:`qibo.core.adiabatic.AdiabaticHamiltonian` calculations.
        """
        c = coefficients.get(self[0].hamiltonian)
        merged = self[0] * c if c is not None else self[0]
        for term in self[1:]:
            c = coefficients.get(term.hamiltonian)
            merged = merged.merge(term * c if c is not None else term)
        return merged


import os
import sys

sys.path.append(os.path.abspath(os.path.join("./libs/graph_embeddings")))
import graph_embeddings
import numpy as np
import pandas as pd
from scipy import sparse

sys.path.append(os.path.abspath(os.path.join("./libs/residual2vec")))
import residual2vec

#
# Input
#
netfile = snakemake.input["netfile"]
nodefile = snakemake.input["nodefile"] if "nodefile" in snakemake.input.keys() else None
dim = int(snakemake.params["dim"])
window_length = int(snakemake.params["window_length"])
model_name = snakemake.params["model_name"]
directed = snakemake.params["directed"] == "directed"
noselfloop = (
    snakemake.params["noselfloop"] == "True"
    if "noselfloop" in snakemake.params.keys()
    else False
)
controll_for = (
    snakemake.params["controlfor"]
    if "controlfor" in snakemake.params.keys()
    else "None"
)
backward_prob = (
    float(snakemake.params["backward_prob"])
    if "backward_prob" in snakemake.params.keys()
    else 0
)
embfile = snakemake.output["embfile"]

#
# Load
#
net = sparse.load_npz(netfile)

if nodefile is not None:
    node_table = pd.read_csv(nodefile)

#
# Preprocess
#
if directed is False:
    net = net + net.T

if noselfloop:
    net.setdiag(0)

if directed:
    eta = backward_prob / (1 - backward_prob)
    outdeg = np.array(net.sum(axis=1)).reshape(-1)
    indeg = np.array(net.sum(axis=0)).reshape(-1)
    eta_nodes = (
        outdeg * backward_prob / (indeg * (1 - backward_prob) + outdeg * backward_prob)
    )
    eta_nodes[outdeg == 0] = 1
    eta_nodes[indeg == 0] = 0
    net = sparse.diags(1 - eta_nodes) * net + sparse.diags(eta_nodes) @ net.T

#
# Load the emebdding models
#
membership = np.zeros(net.shape[0])
offset = np.zeros(net.shape[0])
if model_name == "node2vec":
    model = graph_embeddings.Node2Vec(window_length=window_length, restart_prob=0)
elif model_name == "node2vec-qhalf":
    model = graph_embeddings.Node2Vec(
        window_length=window_length, restart_prob=0, q=0.5
    )
elif model_name == "node2vec-qdouble":
    model = graph_embeddings.Node2Vec(window_length=window_length, restart_prob=0, q=2)
elif model_name == "deepwalk":
    model = graph_embeddings.DeepWalk(window_length=window_length, restart_prob=0,)
elif model_name == "glove":
    model = graph_embeddings.Glove(window_length=window_length, restart_prob=0,)
elif model_name == "fairwalk":
    if (controll_for == "None") or (node_table is None):
        model = graph_embeddings.Fairwalk(window_length=window_length)
    else:
        membership = node_table[controll_for].values
        model = graph_embeddings.Fairwalk(
            group_membership=membership, window_length=window_length,
        )
elif model_name == "residual2vec":
    if (controll_for == "None") or (node_table is None):
        model = residual2vec.residual2vec_matrix_factorization(window_length=window_length,)
    else:
        membership = node_table[controll_for].values
        model = residual2vec.residual2vec_matrix_factorization(
            group_membership=membership, window_length=window_length,
        )
elif model_name == "leigenmap":
    model = graph_embeddings.LaplacianEigenMap()
elif model_name == "netmf":
    model = graph_embeddings.NetMF(window_length=window_length)
elif model_name == "graphsage":
    model = graph_embeddings.GraphSage()
elif model_name == "gcn":
    model = graph_embeddings.GCN()
elif model_name == "graphsage-doubleK":
    model = graph_embeddings.GraphSage(num_default_features=dim * 2)
elif model_name == "gcn-doubleK":
    model = graph_embeddings.GCN(num_default_features=dim * 2)
elif model_name == "gat":
    model = graph_embeddings.GAT(layer_sizes=[64, 256])
elif model_name == "gat-doubleK":
    model = graph_embeddings.GCN(num_default_features=dim * 2)
elif model_name == "lndeg":  # fake embedding. Just to save offset
    A = sparse.csr_matrix(net)
    deg = np.array(A.sum(axis=1)).reshape(-1)
    emb = np.zeros((len(deg), dim))
    np.savez(
        embfile,
        emb=emb,
        out_emb=emb,
        membership=np.zeros_like(deg),
        offset=np.log(np.maximum(deg, 1)),
        window_length=window_length,
        dim=dim,
        directed=directed,
        model_name=model_name,
    )
    sys.exit()

#
# Embedding
#
model.fit(sparse.csr_matrix(net))
emb = model.transform(dim=dim)

try:
    offset = model.node_offset
except AttributeError:
    pass

#
# Save
#
np.savez(
    embfile,
    emb=emb,
    membership=membership,
    offset=offset,
    window_length=window_length,
    dim=dim,
    directed=directed,
    model_name=model_name,
)


<filename>evidence_inference/models/model_0.py
from os.path import join, dirname, abspath
import sys

# this monstrosity produces the module directory in an environment where this is unpacked
sys.path.insert(0, abspath(join(dirname(abspath(__file__)), '..', '..')))
import copy
import random

import numpy as np
from scipy import stats

from sklearn.metrics import accuracy_score, precision_recall_fscore_support, classification_report

from gensim.models import KeyedVectors

import torch
from torch import optim
import torch.nn as nn
from torch.nn import functional as F
from torch.autograd import Variable

USE_CUDA = True

from evidence_inference.preprocess.preprocessor import SimpleInferenceVectorizer as SimpleInferenceVectorizer
from evidence_inference.models.utils import PaddedSequence
from evidence_inference.models.attention_distributions import TokenAttention, evaluate_model_attention_distribution

class CBoWEncoder(nn.Module):
    """Bag of words encoder for Intervention (also Comparator, Outcome) token sequences.

    Note that ordering information is discarded here, and our words are represented by continuous vectors.
    """

    def __init__(self, vocab_size, embeddings: nn.Embedding=None, embedding_dim=200, use_attention=False, condition_attention=False, tokenwise_attention=False, query_dims=None):
        super(CBoWEncoder, self).__init__()

        self.vocab_size = vocab_size

        if embeddings is None:
            self.embedding = nn.Embedding(vocab_size, embedding_dim)
        else:
            self.embedding = embeddings
            self.embedding_dim = embeddings.embedding_dim

        self.use_attention = use_attention
        if self.use_attention:
            self.attention_mechanism = TokenAttention(self.embedding_dim, self.query_dims, condition_attention, tokenwise_attention)

    def forward(self, word_inputs: PaddedSequence, query_v_for_attention: torch.Tensor=None, normalize_attention_distribution=True):
        if isinstance(word_inputs, PaddedSequence):
            embedded = self.embedding(word_inputs.data)
        else:
            raise ValueError("Got an unexpected type {} for word_inputs {}".format(type(word_inputs), word_inputs))
        if self.use_attention:
            a = self.attention_mechanism(word_inputs, embedded, query_v_for_attention, normalize=normalize_attention_distribution)
            output = torch.sum(a * embedded, dim=1)
            return None, output, a
        else:
            output = torch.sum(embedded, dim=1) / word_inputs.batch_sizes.unsqueeze(-1).to(torch.float)
            return output

class GRUEncoder(nn.Module):
    """ GRU encoder for Intervention (also Comparator, Outcome) token sequences.

    Also contains attention mechanisms for use with this particular encoder
    """

    def __init__(self, vocab_size, n_layers=1, hidden_size=32, embeddings: nn.Embedding=None,
                 use_attention=False, condition_attention=False, tokenwise_attention=False, query_dims=None, bidirectional=False):
        """ Prepares a GRU encoder for the Intervention, Comparator, or outcome token sequences.

        Either initializes embedding layer from existing embeddings or creates a random one of size vocab X hidden_size.

        When using attention we either:
        * condition on a hidden unit from the encoder and some query vector of size query_dims, which passes a linear
          combination of the two through a non-linearity (Tanh) and then compresses this to a final number
        * or we use a linear function from the output of the encoder.

        In both cases, we use a softmax over the possible outputs to impose a final attention distribution.
        """
        super(GRUEncoder, self).__init__()
        if condition_attention and not use_attention:
            raise ValueError("Cannot condition attention when there is no attention mechanism! Try setting "
                             "use_attention to true or condition_attention to false, ")
        if tokenwise_attention and not use_attention:
            raise ValueError("Cannot have element-wise attention when there is no attention mechanism! Try setting "
                             "use_attention to true or condition_attention to false, ")

        self.vocab_size = vocab_size
        self.n_layers = n_layers
        self.use_attention = use_attention
        self.condition_attention = condition_attention
        self.tokenwise_attention = tokenwise_attention
        self.query_dims = query_dims
        self.bidirectional = bidirectional
        if self.bidirectional:
            self.hidden_size = hidden_size // 2
        else:
            self.hidden_size = hidden_size

        if embeddings is None:
            self.embedding = nn.Embedding(self.vocab_size, self.hidden_size)
            self.gru = nn.GRU(input_size=self.hidden_size, hidden_size=self.hidden_size, num_layers=self.n_layers, batch_first=True, bidirectional=self.bidirectional)
        else:
            self.embedding = embeddings
            self.gru = nn.GRU(input_size=embeddings.embedding_dim, hidden_size=self.hidden_size, num_layers=self.n_layers, batch_first=True, bidirectional=self.bidirectional)

        if self.use_attention:
            encoding_size = self.hidden_size + int(self.bidirectional) * self.hidden_size
            self.attention_mechanism = TokenAttention(encoding_size, self.query_dims, condition_attention, tokenwise_attention)

    def forward(self, word_inputs: PaddedSequence, init_hidden: torch.Tensor=None, query_v_for_attention: torch.Tensor=None, normalize_attention_distribution=True) -> (torch.Tensor, torch.Tensor):
        if isinstance(word_inputs, PaddedSequence):
            embedded = self.embedding(word_inputs.data)
            as_padded = word_inputs.pack_other(embedded)
            output, hidden = self.gru(as_padded, init_hidden)
            output = PaddedSequence.from_packed_sequence(output, batch_first=True)
        else:
            raise ValueError("Unknown input type {} for word_inputs: {}, try a PaddedSequence or a Tensor".format(type(word_inputs), word_inputs))

        # concatenate the hidden representations
        if self.bidirectional:
            if self.n_layers > 1:
                raise ValueError("Implement me!")
            hidden = torch.cat([hidden[0], hidden[1]], dim=1)

        if self.use_attention:
            # note that these hidden_input_states are masked to zeros (when appropriate) already when this is called.
            hidden_input_states = output
            a = self.attention_mechanism(hidden_input_states, query_v_for_attention, normalize=normalize_attention_distribution)

            # note this is an element-wise multiplication, so each of the hidden states is weighted by the attention vector
            weighted_hidden = torch.sum(a * output.data, dim=1)
            return output, weighted_hidden, a

        return output, hidden

class InferenceNet(nn.Module):
    """ Predicts the relative (statistical) benefits of a pair of medical interventions with respect to an outcome.

    The input to the model is:
    * an array of article tokens
    * an array of medical intervention tokens
    * an array of "comparator" tokens (i.e. an alternate intervention)
    * an array of outcome tokens

    The output is a distribution over whether or not the text of the particular article supports the intervention being
    statistically better (p=0.05), neutral, or worse than the comparator for the outcome.

    This model works via:
    * encoding the article via a gated recurrent unit
    * encoding the intervention, comparator, and outcome via either a gated recurrent unit or a continuous bag of words encoder
    * optionally allowing a separate attention mechanism within each of these units to either:
        * learn a distribution over article tokens
        * learn a distribution over article tokens conditioned on the intervention, comparator, and outcome encodings
    * passing the encoded result through a linear layer and then a softmax
    """

    def __init__(self, vectorizer, h_size=32,
                 init_embeddings=None,
                 init_wvs_path="embeddings/PubMed-w2v.bin",
                 weight_tying=False,
                 ICO_encoder="CBoW",
                 article_encoder="GRU",
                 attention_over_article_tokens=True,
                 condition_attention=True,
                 tokenwise_attention=False,
                 tune_embeddings=False,
                 h_dropout_rate=0.2):
        super(InferenceNet, self).__init__()
        if condition_attention and not attention_over_article_tokens:
            raise ValueError("Must have attention in order to have conditional attention!")

        self.vectorizer = vectorizer
        vocab_size = len(self.vectorizer.idx_to_str)

        if init_embeddings is None:
            print("loading pre-trained embeddings...")
            init_embedding_weights = InferenceNet.init_word_vectors(init_wvs_path, vectorizer)
            print("done.")
        else:
            print("Using provided embeddings")
            init_embedding_weights = init_embeddings

        self.ICO_encoder = ICO_encoder

        # this is the size of the concatenated <abstract, I, C, O> representations,
        # which will depend on the encoder variant being used.
        self.ICO_dims = None

        if ICO_encoder == "CBoW":
            self.intervention_encoder = CBoWEncoder(vocab_size=vocab_size, embeddings=init_embedding_weights)
            self.comparator_encoder = CBoWEncoder(vocab_size=vocab_size, embeddings=init_embedding_weights)
            self.outcome_encoder = CBoWEncoder(vocab_size=vocab_size, embeddings=init_embedding_weights)
            if article_encoder == 'CBoW':
                self.ICO_dims = init_embedding_weights.embedding_dim * 3
                MLP_input_size = self.ICO_dims + init_embedding_weights.embedding_dim
                if h_size:
                    print("Warning: ignoring the hidden size as the article encoder is CBoW and emits a fixed output")
            elif article_encoder == 'GRU' or article_encoder == 'biGRU':
                self.ICO_dims = init_embedding_weights.embedding_dim * 3
                MLP_input_size = self.ICO_dims + h_size
            else:
                raise ValueError("Unknown article_encoder type {}".format(article_encoder))
        elif ICO_encoder == "GRU" or ICO_encoder == 'biGRU':
            bidirectional = ICO_encoder == 'biGRU'
            # then use an RNN encoder for I, C, O elements.
            self.intervention_encoder = GRUEncoder(vocab_size=vocab_size, hidden_size=h_size,
                                                   embeddings=init_embedding_weights, bidirectional=bidirectional)
            self.comparator_encoder = GRUEncoder(vocab_size=vocab_size, hidden_size=h_size,
                                                 embeddings=init_embedding_weights, bidirectional=bidirectional)
            self.outcome_encoder = GRUEncoder(vocab_size=vocab_size, hidden_size=h_size,
                                              embeddings=init_embedding_weights, bidirectional=bidirectional)
            self.ICO_dims = h_size * 3
            if article_encoder == 'CBoW':
                # note that the CBoW encoder ignores the h_size here
                MLP_input_size = self.ICO_dims + init_embedding_weights.embedding_dim
            elif article_encoder == 'GRU' or article_encoder == 'biGRU':
                MLP_input_size = self.ICO_dims + h_size  # the input to the MLP is the concatentation of the ICO hidden states and the article hidden states.
            else:
                raise ValueError("Unknown article_encoder type {}".format(article_encoder))
        else:
            raise ValueError("No such encoder: {}".format(ICO_encoder))

        self.article_encoder_type = article_encoder
        if article_encoder == 'GRU' or article_encoder == 'biGRU':
            bidirectional = article_encoder == 'biGRU'
            self.article_encoder = GRUEncoder(vocab_size=vocab_size, hidden_size=h_size,
                                              embeddings=init_embedding_weights,
                                              use_attention=attention_over_article_tokens,
                                              condition_attention=condition_attention,
                                              tokenwise_attention=tokenwise_attention,
                                              query_dims=self.ICO_dims,
                                              bidirectional=bidirectional)
        elif article_encoder == 'CBoW':
            self.article_encoder = CBoWEncoder(vocab_size=vocab_size,
                                               embeddings=init_embedding_weights,
                                               use_attention=attention_over_article_tokens,
                                               condition_attention=condition_attention,
                                               tokenwise_attention=tokenwise_attention,
                                               query_dims=self.ICO_dims)
        else:
            raise ValueError("Unknown article encoder type: {}".format(article_encoder))

        if not tune_embeddings:
            print("freezing word embedding layer!")
            for layer in (
                    self.article_encoder, self.intervention_encoder, self.comparator_encoder, self.outcome_encoder):
                # note: we are relying on the fact that all encoders will have a
                # "embedding" layer (nn.Embedding).
                layer.embedding.requires_grad = False
                layer.embedding.weight.requires_grad = False

        # weight tying (optional)
        # note that this is not meaningful (or, rather, does nothing) when embeddings are
        # frozen.
        # TODO note that weights are currently tied because all the ICOEncoders use the same underlying objects.
        if weight_tying:
            print("tying word embedding layers")
            self.intervention_encoder.embedding.weight = self.comparator_encoder.embedding.weight = \
                self.outcome_encoder.embedding.weight = self.article_encoder.embedding.weight
        self.batch_first = True

        self.MLP_hidden = nn.Linear(MLP_input_size, 16)
        self.out = nn.Linear(16, 3)
        self.dropout = nn.Dropout(p=h_dropout_rate)

    def _encode(self, I_tokens, C_tokens, O_tokens):
        if self.ICO_encoder == "CBoW":
            # simpler case of a CBoW encoder.
            I_v = self.intervention_encoder(I_tokens)
            C_v = self.comparator_encoder(C_tokens)
            O_v = self.outcome_encoder(O_tokens)
        elif self.ICO_encoder == 'GRU' or self.ICO_encoder == 'biGRU':
            # then we have an RNN encoder. Hidden layers are automatically initialized
            _, I_v = self.intervention_encoder(I_tokens)
            _, C_v = self.comparator_encoder(C_tokens)
            _, O_v = self.outcome_encoder(O_tokens)
        else:
            raise ValueError("No such encoder: {}".format(self.ICO_encoder))
        return I_v, C_v, O_v

    def forward(self, article_tokens: PaddedSequence, I_tokens: PaddedSequence, C_tokens: PaddedSequence, O_tokens: PaddedSequence,
                batch_size, debug_attn=False, verbose_attn=False):
        if isinstance(article_tokens, PaddedSequence):
            assert all([isinstance(x, PaddedSequence) for x in [I_tokens, C_tokens, O_tokens]])
        elif isinstance(article_tokens, torch.Tensor):
            # TODO test this codepath
            assert all([isinstance(x, torch.Tensor) for x in [I_tokens, C_tokens, O_tokens]]) and all([x.shape[0] == 1 for x in [article_tokens, I_tokens, C_tokens, O_tokens]])
        else:
            raise ValueError("Got an unexpected type for our input tensor: {}".format(type(article_tokens)))

        ##################################################
        # First encode the I, C, O frame (the query)     #
        ##################################################
        # the output of each of these should be of shape (batch x word_embedding_size)
        I_v, C_v, O_v = self._encode(I_tokens, C_tokens, O_tokens)

        if self.article_encoder.use_attention:

            query_v = None
            if self.article_encoder.condition_attention:
                query_v = torch.cat([I_v, C_v, O_v], dim=1)

            _, a_v, attn_weights = self.article_encoder(article_tokens, query_v_for_attention=query_v)

            # @TODO return to debugging/inspecting attention
            if verbose_attn:
                attn_weights = attn_weights.data.cpu().numpy()
                for i in range(batch_size):
                    attn_weights_slice = attn_weights[i][:article_tokens.batch_sizes[i].item()].squeeze()
                    sorted_idx = np.argsort(attn_weights_slice)
                    # hack
                    if sorted_idx.size == 1:
                        continue
                    length = len(attn_weights_slice)
                    top_words = [self.vectorizer.idx_to_str[article_tokens.data[i][idx]] for idx in sorted_idx[max(-20, -1 * length):]]
                    top_words.reverse()
                    top_words_weights = [attn_weights_slice[idx] for idx in sorted_idx[max(-20, -1 * length):]]
                    top_words_weights.reverse()
                    bottom_words = [self.vectorizer.idx_to_str[article_tokens.data[i][idx]] for idx in sorted_idx[:min(20, length)]]
                    bottom_words.reverse()
                    bottom_words_weights = [attn_weights_slice[idx] for idx in sorted_idx[:min(20, length)]]
                    bottom_words_weights.reverse()

                    def tokens_to_str(tokens):
                        return ", ".join([self.vectorizer.idx_to_str[x.item()] for x in tokens])
                    print("I, C, O frame:",
                          tokens_to_str(I_tokens.data[i][:I_tokens.batch_sizes[i]]), ";",
                          tokens_to_str(C_tokens.data[i][:C_tokens.batch_sizes[i]]), ":",
                          tokens_to_str(O_tokens.data[i][:O_tokens.batch_sizes[i]]))
                    print("top words:", ", ".join(top_words))
                    print("weights:", ", ".join(str(x) for x in top_words_weights))
                    print("bottom words:", ", ".join(bottom_words))
                    print("weights:", ", ".join(str(x) for x in bottom_words_weights))

        else:
            if self.article_encoder_type == 'CBoW':
                # TODO implement attention for the CBoW model
                a_v = self.article_encoder(article_tokens)
            elif self.article_encoder_type == 'GRU' or self.article_encoder_type == 'biGRU':
                _, a_v = self.article_encoder(article_tokens)
            else:
                raise ValueError("Unknown article encoder type {}".format(self.article_encoder_type))

        # TODO document this
        if len(a_v.size()) == 3:
            a_v = a_v.squeeze(0)
        h = torch.cat([a_v, I_v, C_v, O_v], dim=1)
        h = self.dropout(h)
        raw_out = self.out(self.MLP_hidden(h))

        return F.softmax(raw_out, dim=1)

    @classmethod
    def init_word_vectors(cls, path_to_wvs, vectorizer, use_cuda=USE_CUDA) -> nn.Embedding:
        WVs = KeyedVectors.load_word2vec_format(path_to_wvs, binary=True)

        E = np.zeros((len(vectorizer.str_to_idx), WVs.vector_size))
        WV_matrix = np.matrix([WVs[v] for v in WVs.vocab.keys()])
        mean_vector = np.mean(WV_matrix, axis=0)

        for idx, token in enumerate(vectorizer.idx_to_str):
            if token in WVs:
                E[idx] = WVs[token]
            else:
                E[idx] = mean_vector
        # TODO make this cleaner
        padding_idx = int(vectorizer.str_to_idx[SimpleInferenceVectorizer.PAD])
        E[padding_idx] = torch.zeros(E.shape[1])
        embedding = nn.Embedding(E.shape[0], E.shape[1], padding_idx=padding_idx)
        embedding.weight.data.copy_(torch.from_numpy(E))
        embedding.weight.requires_grad = False
        if use_cuda:
            embedding = embedding.cuda()
        return embedding

def _get_y_vec(y_dict, as_vec=True, majority_lbl=True) -> torch.LongTensor:
    # +1 because raw labels are -1, 0, 1 -> 0, 1, 2
    # for indexing reasons that appear in the loss function
    # (cross-entropy loss wants the index of the highest value, and we index at 0)
    all_labels = [y_j[0] + 1 for y_j in y_dict]
    if majority_lbl:
        y_collapsed = int(stats.mode(all_labels)[0][0])
    else:
        y_collapsed = random.choice(all_labels)

    if as_vec:
        y_vec = np.zeros(3)
        y_vec[y_collapsed] = 1.0
        ret = torch.LongTensor(y_vec)
    else:
        ret = torch.LongTensor([y_collapsed])
    if USE_CUDA:
        ret = ret.cuda()
    return ret

def _to_torch_var(x):
    var_x = Variable(torch.LongTensor(x))
    if USE_CUDA:
        var_x = var_x.cuda()
    return var_x

def predict_for_inst(nnet, inst, verbose_attn=False):
    abstract = _to_torch_var(inst["article"]).unsqueeze(0)
    I, C, O = _to_torch_var(inst["I"]).unsqueeze(0), _to_torch_var(inst["C"]).unsqueeze(0), _to_torch_var(inst["O"]).unsqueeze(0)
    print("sizes:", abstract.size(), I.size(), C.size(), O.size())
    y_hat = nnet(abstract, I, C, O, batch_size=1, verbose_attn=verbose_attn)
    return y_hat

'''
def conf_matrix(nnet, instances):
    M = np.zeros((3,3))
    for inst in instances:
        y = _get_y_vec(inst['y'], as_vec=False)
        y_hat = np.argmax(predict_for_inst(nnet, inst))
        M[y, y_hat] += 1.0
    return M
'''

def make_preds(nnet, instances, batch_size, inference_vectorizer, verbose_attn_to_batches=False, cuda=USE_CUDA):
    # TODO consider removing the inference_vectorizer since all we need is an unk_idx from it
    y_vec = torch.cat([_get_y_vec(inst['y'], as_vec=False) for inst in instances]).squeeze()
    unk_idx = int(inference_vectorizer.str_to_idx[SimpleInferenceVectorizer.PAD])
    y_hat_vec = []
    # we batch this so the GPU doesn't run out of memory
    nnet.eval()
    for i in range(0, len(instances), batch_size):
        batch_instances = instances[i:i+batch_size]
        articles, Is, Cs, Os = [PaddedSequence.autopad([torch.LongTensor(inst[x]) for inst in batch_instances], batch_first=True, padding_value=unk_idx) for x in ['article', 'I', 'C', 'O']]
        if cuda:
            articles, Is, Cs, Os = articles.cuda(), Is.cuda(), Cs.cuda(), Os.cuda()
        verbose_attn = verbose_attn_to_batches and i in verbose_attn_to_batches
        y_hat_batch = nnet(articles, Is, Cs, Os, batch_size=len(batch_instances), verbose_attn=verbose_attn)
        y_hat_vec.append(y_hat_batch)
    nnet.train()
    return y_vec, torch.cat(y_hat_vec, dim=0)

def to_int_preds(y):
    # the cast to int is necessary as this gets passed to sklearn packages that don't understand numpy.int64, which is the default return type here.
    return [int(np.argmax(y_i)) for y_i in y.cpu()]

def _loss_for_inst(inst, nnet, criterion):
    y = _get_y_vec(inst['y'], as_vec=False).squeeze()
    y_hat = predict_for_inst(nnet, inst)
    ####
    # as per https://github.com/pytorch/pytorch/issues/5554,
    # output needs to have dims (N, C), so we add an extra
    # dim for N here (just 1).
    y_hat = torch.unsqueeze(y_hat, dim=0)
    if USE_CUDA:
        y_hat = y_hat.cuda()
        y = y.cuda()

    return criterion(y_hat, y)

def _get_majority_label(inst):
    all_lbls = [y[0] + 1 for y in inst['y']]
    return stats.mode(all_lbls)[0][0]

def train(ev_inf: InferenceNet, train_Xy, val_Xy, test_Xy, inference_vectorizer, epochs=10, batch_size=16, shuffle=True):
    # we sort these so batches all have approximately the same length (ish), which decreases the
    # average amount of padding needed, and thus total number of steps in training.
    if not shuffle:
        train_Xy.sort(key=lambda x: len(x['article']))
        val_Xy.sort(key=lambda x: len(x['article']))
        test_Xy.sort(key=lambda x: len(x['article']))
    print("Using {} training examples, {} validation examples, {} testing examples".format(len(train_Xy), len(val_Xy), len(test_Xy)))
    most_common = stats.mode([_get_majority_label(inst) for inst in train_Xy])[0][0]

    best_val_model = None
    best_val_f1 = float('-inf')
    if USE_CUDA:
        ev_inf = ev_inf.cuda()

    optimizer = optim.Adam(ev_inf.parameters())
    criterion = nn.CrossEntropyLoss(reduction='sum')  # sum (not average) of the batch losses.

    # TODO add epoch timing information here
    epochs_since_improvement = 0
    val_metrics = {
        "val_acc": [],
        "val_p": [],
        "val_r": [],
        "val_f1": [],
        "val_loss": [],
        'train_loss': [],
        'val_aucs': [],
        'train_aucs': [],
        'val_entropies': [],
        'val_evidence_token_mass': [],
        'val_evidence_token_err': [],
        'train_entropies': [],
        'train_evidence_token_mass': [],
        'train_evidence_token_err': []
    }
    for epoch in range(epochs):
        if epochs_since_improvement > 10:
            print("Exiting early due to no improvement on validation after 10 epochs.")
            break
        if shuffle:
            random.shuffle(train_Xy)

        epoch_loss = 0
        for i in range(0, len(train_Xy), batch_size):
            instances = train_Xy[i:i+batch_size]
            ys = torch.cat([_get_y_vec(inst['y'], as_vec=False) for inst in instances], dim=0)
            # TODO explain the use of padding here
            unk_idx = int(inference_vectorizer.str_to_idx[SimpleInferenceVectorizer.PAD])
            articles, Is, Cs, Os = [PaddedSequence.autopad([torch.LongTensor(inst[x]) for inst in instances], batch_first=True, padding_value=unk_idx) for x in ['article', 'I', 'C', 'O']]
            optimizer.zero_grad()
            if USE_CUDA:
                articles, Is, Cs, Os = articles.cuda(), Is.cuda(), Cs.cuda(), Os.cuda()
                ys = ys.cuda()
            verbose_attn = (epoch == epochs - 1 and i == 0) or (epoch == 0 and i == 0)
            if verbose_attn:
                print("Training attentions:")
            tags = ev_inf(articles, Is, Cs, Os, batch_size=len(instances), verbose_attn=verbose_attn)
            loss = criterion(tags, ys)
            #if loss.item() != loss.item():
            #    import pdb; pdb.set_trace()
            epoch_loss += loss.item()
            loss.backward()
            optimizer.step()
        val_metrics['train_loss'].append(epoch_loss)

        with torch.no_grad():
            verbose_attn_to_batches = set([0,1,2,3,4]) if epoch == epochs - 1 or epoch == 0 else False
            if verbose_attn_to_batches:
                print("Validation attention:")
            # make_preds runs in eval mode
            val_y, val_y_hat = make_preds(ev_inf, val_Xy, batch_size, inference_vectorizer, verbose_attn_to_batches=verbose_attn_to_batches)
            val_loss = criterion(val_y_hat, val_y.squeeze())
            y_hat = to_int_preds(val_y_hat)

            if epoch == 0:
                dummy_preds = [most_common] * len(val_y)
                dummy_acc = accuracy_score(val_y.cpu(), dummy_preds)
                val_metrics["baseline_val_acc"] = dummy_acc
                p, r, f1, _ = precision_recall_fscore_support(val_y.cpu(), dummy_preds, labels=None, beta=1, average='macro', pos_label=1, warn_for=('f-score',), sample_weight=None)
                val_metrics['p_dummy'] = p
                val_metrics['r_dummy'] = r
                val_metrics['f_dummy'] = f1

                print("val dummy accuracy: {:.3f}".format(dummy_acc))
                print("classification report for dummy on val: ")
                print(classification_report(val_y.cpu(), dummy_preds))
                print("\n\n")

            acc = accuracy_score(val_y.cpu(), y_hat)
            val_metrics["val_acc"].append(acc)
            val_loss = val_loss.cpu().item()
            val_metrics["val_loss"].append(val_loss)

            # f1 = f1_score(val_y, y_hat, average="macro")
            p, r, f1, _ = precision_recall_fscore_support(val_y.cpu(), y_hat, labels=None, beta=1, average='macro', pos_label=1, warn_for=('f-score',), sample_weight=None)
            val_metrics["val_f1"].append(f1)
            val_metrics["val_p"].append(p)
            val_metrics["val_r"].append(r)

            if ev_inf.article_encoder.use_attention:
                train_auc, train_entropies, train_evidence_token_masses, train_evidence_token_err = evaluate_model_attention_distribution(ev_inf, train_Xy, cuda=USE_CUDA, compute_attention_diagnostics=True)
                val_auc, val_entropies, val_evidence_token_masses, val_evidence_token_err = evaluate_model_attention_distribution(ev_inf, val_Xy, cuda=USE_CUDA, compute_attention_diagnostics=True)
                print("train auc: {:.3f}, entropy: {:.3f}, evidence mass: {:.3f}, err: {:.3f}".format(train_auc, train_entropies, train_evidence_token_masses, train_evidence_token_err))
                print("val auc: {:.3f}, entropy: {:.3f}, evidence mass: {:.3f}, err: {:.3f}".format(val_auc, val_entropies, val_evidence_token_masses, val_evidence_token_err))
            else:
                train_auc, train_entropies, train_evidence_token_masses, train_evidence_token_err = "", "", "", ""
                val_auc, val_entropies, val_evidence_token_masses, val_evidence_token_err = "", "", "", ""
            val_metrics['train_aucs'].append(train_auc)
            val_metrics['train_entropies'].append(train_entropies)
            val_metrics['train_evidence_token_mass'].append(train_evidence_token_masses)
            val_metrics['train_evidence_token_err'].append(train_evidence_token_err)
            val_metrics['val_aucs'].append(val_auc)
            val_metrics['val_entropies'].append(val_entropies)
            val_metrics['val_evidence_token_mass'].append(val_evidence_token_masses)
            val_metrics['val_evidence_token_err'].append(val_evidence_token_err)
            if f1 > best_val_f1:
                print("New best model at {} with val f1 {:.3f}".format(epoch, f1))
                best_val_f1 = f1
                best_val_model = copy.deepcopy(ev_inf)
                epochs_since_improvement = 0
            else:
                epochs_since_improvement += 1

            #if val_loss != val_loss or epoch_loss != epoch_loss:
            #    import pdb; pdb.set_trace()

            print("epoch {}. train loss: {}; val loss: {}; val acc: {:.3f}".format(
                epoch, epoch_loss, val_loss, acc))

            print(classification_report(val_y.cpu(), y_hat))
            print("val macro f1: {0:.3f}".format(f1))
            print("\n\n")

    val_metrics['best_val_f1'] = best_val_f1
    with torch.no_grad():
        print("Test attentions:")
        verbose_attn_to_batches = set([0,1,2,3,4])
        # make_preds runs in eval mode
        test_y, test_y_hat = make_preds(best_val_model, test_Xy, batch_size, inference_vectorizer, verbose_attn_to_batches=verbose_attn_to_batches)
        test_loss = criterion(test_y_hat, test_y.squeeze())
        y_hat = to_int_preds(test_y_hat)
        final_test_preds = zip([t['a_id'] for t in test_Xy], [t['p_id'] for t in test_Xy], y_hat)

        acc = accuracy_score(test_y.cpu(), y_hat)
        val_metrics["test_acc"] = acc
        test_loss = test_loss.cpu().item()
        val_metrics["test_loss"] = test_loss

        # f1 = f1_score(test_y, y_hat, average="macro")
        p, r, f1, _ = precision_recall_fscore_support(test_y.cpu(), y_hat, labels=None, beta=1, average='macro', pos_label=1, warn_for=('f-score',), sample_weight=None)
        val_metrics["test_f1"] = f1
        val_metrics["test_p"] = p
        val_metrics["test_r"] = r
        if ev_inf.article_encoder.use_attention:
            test_auc, test_entropies, test_evidence_token_masses, test_evidence_token_err = evaluate_model_attention_distribution(best_val_model, test_Xy, cuda=USE_CUDA, compute_attention_diagnostics=True)
            print("test auc: {:.3f}, , entropy: {:.3f}, kl_to_uniform {:.3f}".format(test_auc, test_entropies, test_evidence_token_masses))
        else:
            test_auc, test_entropies, test_evidence_token_masses, test_evidence_token_err = "", "", "", ""
        val_metrics['test_auc'] = test_auc
        val_metrics['test_entropy'] = test_entropies
        val_metrics['test_evidence_token_mass'] = test_evidence_token_masses
        val_metrics['test_evidence_token_err'] = test_evidence_token_err

        print("test loss: {}; test acc: {:.3f}".format(test_loss, acc))

        print(classification_report(test_y.cpu(), y_hat))
        print("test macro f1: {}".format(f1))
        print("\n\n")

    return best_val_model, inference_vectorizer, train_Xy, val_Xy, val_metrics, final_test_preds


<reponame>LCAV/lippmann-photography
# -*- coding: utf-8 -*-
"""
Created on Wed Jun 21 16:27:58 2017

@author: gbaechle
"""

from scipy import misc, io
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
from skimage.color import rgb2xyz, xyz2rgb
from lippmann import *
import imageio

import sys
sys.path.append("../")
import color_tools as ct

plt.close('all')

def read_image(path):

    return imageio.imread(path).astype(float)/255.

def compute_spectrum_slice(sliced, lambdas):

    #comppute the spectrum
    im_xyz   = xyz2rgb(sliced.reshape((1,-1,3))).reshape(-1, 3)
    spectrum = ct.from_xyz_to_spectrum(im_xyz, lambdas)

    return spectrum

def compute_lippmann_slice(spectrums, lambdas, depths):

    lippmann = np.zeros((len(spectrums), len(depths)))

    for i, s in enumerate(spectrums):
        print(i)
        lip, _ = lippmann_transform(lambdas, s, depths)
        lippmann[i, :] = lip

    return lippmann

def compute_end_plate(im, lambdas, vmax):

    two_k = 4 * np.pi / lambdas

    im_xyz   = xyz2rgb(im)
    spectrums = ct.from_xyz_to_spectrum(im_xyz, lambdas)

    intensity = -np.trapz(spectrums, two_k*c/2, axis=2)
    mpl.image.imsave('Figures/baseline.png', intensity, vmax=vmax, vmin=0)

    return intensity

def generate_slices(im, N=500):

    lambdas, _ = generate_wavelengths(N)
    depths = generate_depths(delta_z=2.5E-9, max_depth=2.5E-6)

    H = 883-1
    L = 883-1
    slice1 = compute_spectrum_slice(im[:H, L, :3], lambdas)
    slice2 = compute_spectrum_slice(im[H, :L, :3], lambdas)
    slice3 = compute_spectrum_slice(im[:H, 0, :3], lambdas)
    slice4 = compute_spectrum_slice(im[0, :L, :3], lambdas)

    lip1 = compute_lippmann_slice(slice1, lambdas, depths)
    lip2 = compute_lippmann_slice(slice2, lambdas, depths)
    lip3 = compute_lippmann_slice(slice3, lambdas, depths)
    lip4 = compute_lippmann_slice(slice4, lambdas, depths)

    print(np.max(lip1), np.max(lip2), np.max(lip3), np.max(lip4))
    vmax = max(np.max(lip1), np.max(lip2), np.max(lip3), np.max(lip4))

    for i in range(1,5):

        i_str = str(i)
        mpl.image.imsave('Figures/slice' + i_str + '.png', eval('lip' + i_str), vmax=vmax)

    return lambdas, vmax

if __name__ == '__main__':

#    path = '../images/original.png'
    path = '../images/lippmann_image.jpg'
    im = read_image(path)

    lambdas, vmax = generate_slices(im, N=500)

#    spectrum = compute_end_plate(im[:800, :750, :3], lambdas, vmax)
    spectrum = compute_end_plate(im[:, :, :3], lambdas, vmax)

#    misc.imsave('Figures/front.png', im[:800, :750])
    misc.imsave('Figures/front.png', im)

    plt.figure()
    plt.imshow(im)
    plt.figure()
#    plt.imshow(im[:800, :750, :3])
    plt.imshow(im[:, :, :3])


<filename>testing/gen_feature_space_samples.py
import numpy as np
import pandas as pd
import graphviz
import graphviz.backend
from numpy.distutils.system_info import f2py_info
from sklearn import tree
from sklearn.datasets import load_boston, load_iris, load_wine, load_digits, load_breast_cancer, load_diabetes, fetch_mldata
from matplotlib.figure import figaspect
import string
import re
import matplotlib.pyplot as plt
import seaborn as sns
from dtreeviz.shadow import *
from numbers import Number
import matplotlib.patches as patches
from scipy import stats
from sklearn.neighbors import KernelDensity
import inspect, sys, tempfile

from dtreeviz.trees import *

def viz_digits(features, feature_names, max_depth):
    digits = load_digits()

    # "8x8 image of integer pixels in the range 0..16."
    columns = [f'pixel[{i},{j}]' for i in range(8) for j in range(8)]

    fig, ax = plt.subplots(1, 1)
    X_train = digits.data[:,features]
    y_train = digits.target
    if len(features)==1:
        x_train = digits.data[:, features[0]]

        ctreeviz_univar(ax, x_train, y_train, max_depth=max_depth, feature_name=feature_names[0],
                        class_names=[str(i) for i in range(10)], gtype='strip', target_name='digit')
        filename = f"/tmp/digits-{feature_names[0]}-featspace-depth-{max_depth}.svg"
    else:
        ctreeviz_bivar(ax, X_train, y_train, max_depth=max_depth,
                       feature_names=feature_names, class_names=[str(i) for i in range(10)], target_name='digit')
        filename = f"/tmp/digits-{','.join(feature_names)}-featspace-depth-{max_depth}.svg"

    print(f"Create {filename}")
    plt.tight_layout()
    plt.savefig(filename, bbox_inches=0, pad_inches=0)
    plt.show()
    # plt.close()

def viz_wine(features, feature_names, max_depth):
    wine = load_wine()

    X_train = wine.data[:,features]
    y_train = wine.target
    if len(features)==1:
        figsize = (6, 2)
        fig, ax = plt.subplots(1, 1, figsize=figsize)
        x_train = wine.data[:, features[0]]

        ctreeviz_univar(ax, x_train, y_train, max_depth=max_depth, feature_name=feature_names[0],
                        class_names=list(wine.target_names), gtype='strip', target_name='wine')
        filename = f"/tmp/wine-{feature_names[0]}-featspace-depth-{max_depth}.svg"
    else:
        figsize = (6, 5)
        fig, ax = plt.subplots(1, 1, figsize=figsize)
        ctreeviz_bivar(ax, X_train, y_train, max_depth=max_depth,
                       feature_names=feature_names, class_names=list(wine.target_names), target_name='wine',show={'splits'})
        filename = f"/tmp/wine-{','.join(feature_names)}-featspace-depth-{max_depth}.svg"

    print(f"Create {filename}")
    plt.tight_layout()
    plt.savefig(filename, bbox_inches=0, pad_inches=0)
    plt.show()

def viz_knowledge(features, feature_names, max_depth):
    know = pd.read_csv("data/knowledge.csv")
    class_names = ['very_low', 'Low', 'Middle', 'High']
    know['UNS'] = know['UNS'].map({n: i for i, n in enumerate(class_names)})

    X_train = know.drop('UNS', axis=1)
    X_train = X_train.values[:,features]
    y_train = know['UNS']
    if len(features)==1:
        figsize = (6, 2)
        fig, ax = plt.subplots(1, 1, figsize=figsize)
        x_train = know.PEG

        ctreeviz_univar(ax, x_train, y_train, max_depth=max_depth, feature_name=feature_names[0],
                        class_names=class_names, gtype='strip', target_name='knowledge')
        filename = f"/tmp/knowledge-{feature_names[0]}-featspace-depth-{max_depth}.svg"
    else:
        figsize = (6, 5)
        fig, ax = plt.subplots(1, 1, figsize=figsize)
        ctreeviz_bivar(ax, X_train, y_train, max_depth=max_depth,
                       feature_names=feature_names, class_names=class_names, target_name='knowledge')
        filename = f"/tmp/knowledge-{','.join(feature_names)}-featspace-depth-{max_depth}.svg"

    print(f"Create {filename}")
    plt.tight_layout()
    plt.savefig(filename, bbox_inches=0, pad_inches=0)
    plt.show()

def viz_diabetes(features, feature_names, max_depth):
    diabetes = load_diabetes()

    X_train = diabetes.data
    X_train = X_train[:,features]
    y_train = diabetes.target
    if len(features)==1:
        figsize = (6, 2)
        fig, ax = plt.subplots(1, 1, figsize=figsize)
        x_train = diabetes.data[:, features[0]]

        rtreeviz_univar(ax, x_train, y_train, max_depth=max_depth, feature_name=feature_names[0], target_name='diabetes')
        filename = f"/tmp/diabetes-{feature_names[0]}-featspace-depth-{max_depth}.svg"
    else:
        figsize = (6, 5)
        fig, ax = plt.subplots(1, 1, figsize=figsize)
        rtreeviz_bivar_heatmap(ax, X_train, y_train, max_depth=max_depth,
                               feature_names=feature_names)
        filename = f"/tmp/diabetes-{','.join(feature_names)}-featspace-depth-{max_depth}.svg"

    print(f"Create {filename}")
    plt.tight_layout()
    plt.savefig(filename, bbox_inches=0, pad_inches=0)
    plt.show()

def viz_boston(features, feature_names, max_depth):
    boston = load_boston()

    X_train = boston.data
    X_train = X_train[:,features]
    y_train = boston.target
    if len(features)==1:
        figsize = (6, 2)
        fig, ax = plt.subplots(1, 1, figsize=figsize)
        x_train = boston.data[:, features[0]]

        rtreeviz_univar(ax, x_train, y_train, max_depth=max_depth, feature_name=feature_names[0], target_name='price')
        filename = f"/tmp/boston-{feature_names[0]}-featspace-depth-{max_depth}.svg"
    else:
        figsize = (6, 5)
        fig, ax = plt.subplots(1, 1, figsize=figsize)
        rtreeviz_bivar_heatmap(ax, X_train, y_train, max_depth=max_depth,
                               feature_names=feature_names)
        filename = f"/tmp/boston-{','.join(feature_names)}-featspace-depth-{max_depth}.svg"

    print(f"Create {filename}")
    plt.tight_layout()
    plt.savefig(filename, bbox_inches=0, pad_inches=0)
    plt.show()

viz_boston(features=[5],feature_names=['RM'], max_depth=2)
viz_boston(features=[5],feature_names=['RM'], max_depth=4)
viz_boston(features=[5,12],feature_names=['RM','LSTAT'], max_depth=2)
viz_boston(features=[5,12],feature_names=['RM','LSTAT'], max_depth=4)

viz_diabetes(features=[2],feature_names=['bmi'], max_depth=2)
viz_diabetes(features=[2],feature_names=['bmi'], max_depth=5)
viz_diabetes(features=[2,0],feature_names=['bmi','age'], max_depth=2)
viz_diabetes(features=[2,0],feature_names=['bmi','age'], max_depth=5)

viz_knowledge(features=[4],feature_names=['PEG'], max_depth=2)
viz_knowledge(features=[4],feature_names=['PEG'], max_depth=3)
viz_knowledge(features=[4,3],feature_names=['PEG','LPR'], max_depth=2)
viz_knowledge(features=[4,3],feature_names=['PEG','LPR'], max_depth=3)

viz_wine(features=[12],feature_names=['proline'], max_depth=2)
viz_wine(features=[12],feature_names=['proline'], max_depth=3)
viz_wine(features=[12,6],feature_names=['proline','flavanoids'], max_depth=1)
viz_wine(features=[12,6],feature_names=['proline','flavanoids'], max_depth=2)
viz_wine(features=[12,6],feature_names=['proline','flavanoids'], max_depth=3)
viz_digits(features=[2*8+5], feature_names=['pixel[2,5]'], max_depth=20)
viz_digits(features=[4*8+4,2*8+5], feature_names=['pixel[4,4]','pixel[2,5]'], max_depth=5)


<reponame>appleface2050/Coursera-ML
# coding:utf-8

import numpy as np
import pandas as pd
from sklearn import svm
import seaborn as sns
import scipy.io as sio
import matplotlib.pyplot as plt
from sklearn import metrics

if __name__ == '__main__':
    mat = sio.loadmat('data/ex6data3.mat')
    print(mat.keys())
    training = pd.DataFrame(mat.get('X'), columns=['X1', 'X2'])
    training['y'] = mat.get('y')

    cv = pd.DataFrame(mat.get('Xval'), columns=['X1', 'X2'])
    cv['y'] = mat.get('yval')
    print(training.shape)
    print(training.head())

    print(cv.shape)
    print(cv.head())

    candidate = [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30, 100]
    # gamma to comply with sklearn parameter name
    combination = [(C, gamma) for C in candidate for gamma in candidate]
    print(len(combination))

    search = []

    for C, gamma in combination:
        svc = svm.SVC(C=C, gamma=gamma)
        svc.fit(training[['X1', 'X2']], training['y'])
        search.append(svc.score(cv[['X1', 'X2']], cv['y']))
    print(search)
    best_score = search[np.argmax(search)]
    best_param = combination[np.argmax(search)]
    print(best_score, best_param)

    best_svc = svm.SVC(C=best_param[1], gamma=best_param[0])
    best_svc.fit(training[['X1', 'X2']], training['y'])
    ypred = best_svc.predict(cv[['X1', 'X2']])

    print(metrics.classification_report(cv['y'], ypred))


<gh_stars>1-10

import os
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from matplotlib.dates import MonthLocator
# import matplotlib.ticker
import numpy as np
from sense.canopy import OneLayer
from sense.soil import Soil
from sense import model
import scipy.stats
from scipy.optimize import minimize
import pdb

# Helper functions for statistical parameters
#--------------------------------------------
def rmse_prediction(predictions, targets):
    """ calculation of RMSE """
    return np.sqrt(np.nanmean((predictions - targets) ** 2))

def linregress(predictions, targets):
    """ Calculate a linear least-squares regression for two sets of measurements """
    slope, intercept, r_value, p_value, std_err = scipy.stats.linregress(predictions, targets)
    return slope, intercept, r_value, p_value, std_err

def read_mni_data(path, file_name, extention, field, sep=';'):
    """ read MNI campaign data """
    df = pd.io.parsers.read_csv(os.path.join(path, file_name + extension), header=[0, 1], sep=sep)
    df = df.set_index(pd.to_datetime(df[field]['date']))
    df = df.drop(df.filter(like='date'), axis=1)
    return df

def read_agrometeo(path, file_name, extentio, sep=';', decimal=','):
    """ read agro-meteorological station (hourly data) """
    df = pd.read_csv(os.path.join(path, file_name + extension), sep=sep, decimal=decimal)
    df['SUM_NN050'] = df['SUM_NN050'].str.replace(',','.')
    df['SUM_NN050'] = df['SUM_NN050'].str.replace('-','0').astype(float)

    df['date'] = df['Tag'] + ' ' + df['Stunde']

    df = df.set_index(pd.to_datetime(df['date'], format='%d.%m.%Y %H:%S'))
    return df

def filter_relativorbit(data, field, orbit1, orbit2=None, orbit3=None, orbit4=None):
    """ data filter for relativ orbits """
    output = data[[(check == orbit1 or check == orbit2 or check == orbit3 or check == orbit4) for check in data[(field,'relativeorbit')]]]
    return output

def smooth(x,window_len=11,window='hanning'):
        if x.ndim != 1:
                raise ValueError #, "smooth only accepts 1 dimension arrays."
        if x.size < window_len:
                raise ValueError #, "Input vector needs to be bigger than window size."
        if window_len<3:
                return x
        if not window in ['flat', 'hanning', 'hamming', 'bartlett', 'blackman']:
                raise ValueError #, "Window is on of 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'"
        s=np.r_[2*x[0]-x[window_len-1::-1],x,2*x[-1]-x[-1:-window_len:-1]]
        if window == 'flat': #moving average
                w=np.ones(window_len,'d')
        else:
                w=eval('np.'+window+'(window_len)')
        y=np.convolve(w/w.sum(),s,mode='same')
        return y[window_len:-window_len+1]

def read_data(path, file_name, extension, field, path_agro, file_name_agro, extension_agro):
    # Read MNI data
    df = read_mni_data(path, file_name, extension, field)

    # Read agro-meteorological station
    df_agro = read_agrometeo(path_agro, file_name_agro, extension_agro)

    # filter for field
    field_data = df.filter(like=field)

    # filter for relativorbit
    field_data_orbit = filter_relativorbit(field_data, field, 95, 168)
    # field_data = field_data_orbit

    # get rid of NaN values
    parameter_nan = 'LAI'
    field_data = field_data[~np.isnan(field_data.filter(like=parameter_nan).values)]

    # available auxiliary data
    theta_field = np.deg2rad(field_data.filter(like='theta'))
    # theta_field[:] = 45
    sm_field = field_data.filter(like='SM')
    height_field = field_data.filter(like='Height')/100
    lai_field = field_data.filter(like='LAI')
    vwc_field = field_data.filter(like='VWC')
    pol_field = field_data.filter(like='sigma_sentinel_'+pol)
    return df, df_agro, field_data, field_data_orbit, theta_field, sm_field, height_field, lai_field, vwc_field, pol_field

### Optimization ###
#-----------------------------------------------------------------
def solve_fun(VALS):

    for i in range(len(var_opt)):
        dic[var_opt[i]] = VALS[i]

    ke = dic['coef'] * np.sqrt(dic['lai'])
    # ke = dic['coef'] * np.sqrt(dic['vwc'])
    # ke=1
    dic['ke'] = ke

    # surface
    soil = Soil(mv=dic['mv'], C_hh=dic['C_hh'], C_vv=dic['C_vv'], D_hh=dic['D_hh'], D_vv=dic['D_vv'], C_hv=dic['C_hv'], D_hv=dic['D_hv'], V2=dic['V2'], s=dic['s'], clay=dic['clay'], sand=dic['sand'], f=dic['f'], bulk=dic['bulk'], l=dic['l'])

    # canopy
    can = OneLayer(canopy=dic['canopy'], ke_h=dic['ke'], ke_v=dic['ke'], d=dic['d'], ks_h = dic['omega']*dic['ke'], ks_v = dic['omega']*dic['ke'], V1=dic['V1'], V2=dic['V2'], A_hh=dic['A_hh'], B_hh=dic['B_hh'], A_vv=dic['A_vv'], B_vv=dic['B_vv'], A_hv=dic['A_hv'], B_hv=dic['B_hv'])

    S = model.RTModel(surface=soil, canopy=can, models=models, theta=dic['theta'], freq=dic['f'])
    S.sigma0()

    return S.__dict__['stot'][pol[::-1]]

def fun_opt(VALS):

    # return(10.*np.log10(np.nansum(np.square(solve_fun(VALS)-dic['pol_value']))))
    return(np.nansum(np.square(solve_fun(VALS)-dic['pol_value'])))

def data_optimized_run(n, field_data, theta_field, sm_field, height_field, lai_field, vwc_field, pol):
    n = np.int(np.floor(n/2))

    if n > 0:
        field_data = field_data.drop(field_data.index[-n:])
        field_data = field_data.drop(field_data.index[0:n])
        theta_field = theta_field.drop(theta_field.index[-n:])
        theta_field = theta_field.drop(theta_field.index[0:n])

    sm_field = field_data.filter(like='SM')
    height_field = field_data.filter(like='Height')/100
    lai_field = field_data.filter(like='LAI')
    vwc_field = field_data.filter(like='VWC')

    vv_field = field_data.filter(like='sigma_sentinel_vv')
    vh_field = field_data.filter(like='sigma_sentinel_vh')

    pol_field = field_data.filter(like='sigma_sentinel_'+pol)
    return field_data, theta_field, sm_field, height_field, lai_field, vwc_field, vv_field, vh_field, pol_field
#-----------------------------------------------------------------

### Data preparation ###
#-----------------------------------------------------------------
# storage information
path = '/media/tweiss/Daten/new_data'
file_name = 'multi10' # theta needs to be changed to for norm multi
extension = '.csv'

path_agro = '/media/nas_data/2017_MNI_campaign/field_data/meteodata/agrarmeteorological_station'
file_name_agro = 'Eichenried_01012017_31122017_hourly'
extension_agro = '.csv'

field = '508_high'
field_plot = ['508_high', '508_low', '508_med']
pol = 'vv'
# pol = 'vh'

# output path
plot_output_path = '/media/tweiss/Daten/plots/paper/'

df, df_agro, field_data, field_data_orbit, theta_field, sm_field, height_field, lai_field, vwc_field, pol_field = read_data(path, file_name, extension, field, path_agro, file_name_agro, extension_agro)

#-----------------------------------------------------------------

### Run SenSe module
#-----------------------------------------------------------------
#### Choose models
#-----------------

surface_list = ['Oh92', 'Oh04', 'Dubois95', 'WaterCloud', 'I2EM']
# surface_list = ['Oh92', 'Oh04', 'WaterCloud']
# surface_list = ['WaterCloud']
canopy_list = ['turbid_isotropic', 'water_cloud']
# canopy_list = ['water_cloud']

# surface_list = ['Oh92']
# surface_list = ['Oh04']
# surface_list = ['Dubois95']
# surface_list = ['WaterCloud']
# surface_list = ['I2EM']
# canopy_list = ['turbid_isotropic']
# canopy_list = ['water_cloud']

### option for time invariant or variant calibration of parameter
#-------------------------------
opt_mod = 'time invariant'
# opt_mod = 'time variant'
#---------------------------

### plot option: "single" or "all" modelcombination
#------------------------------
# plot = 'single'
plot = 'all'
#------------------------------

### plot option scatterplot or not
#-------------------------------
# style = 'scatterplot'
style = ''

### plot option for scatterplot single ESU
#------------------------------------
# style_2 = 'scatterplot_single_ESU'
style_2 = ''
#-----------------------------------

# Initialize plot settings
#---------------------------
if style == 'scatterplot':
    fig, ax = plt.subplots(figsize=(10, 10))
else:
    fig, ax = plt.subplots(figsize=(17, 10))
# plt.title('Winter Wheat')
plt.ylabel('Backscatter [dB]', fontsize=15)
plt.xlabel('Date', fontsize=15)
plt.tick_params(labelsize=12)

if pol == 'vv':
    ax.set_ylim([-25,-7.5])
elif pol == 'vh':
    ax.set_ylim([-30,-15])

colormaps = ['Greens', 'Purples', 'Blues', 'Oranges', 'Reds', 'Greys', 'pink', 'bone', 'Blues', 'Blues', 'Blues']
j = 0

colormap = plt.get_cmap(colormaps[j])
colors = [colormap(jj) for jj in np.linspace(0.35, 1., 3)]

for k in surface_list:

    for kk in canopy_list:
        df, df_agro, field_data, field_data_orbit, theta_field, sm_field, height_field, lai_field, vwc_field, pol_field = read_data(path, file_name, extension, field, path_agro, file_name_agro, extension_agro)
        freq = 5.405
        clay = 0.08
        sand = 0.12
        bulk = 1.5
        s = 0.0105 # vv
        s = 0.0115
        # s = 0.009 # vh ?????

        C_hh = 0
        D_hh = 0
        C_hv = -22.5
        D_hv = 3.2
        C_vv = -14.609339
        D_vv = 12.884086

        ### Canopy
        # Water Cloud (A, B, V1, V2, theta)
        # SSRT (coef, omega, theta)
        #-----------------------------------
        A_hh = 0
        B_hh = 0
        A_hv = 0.029
        B_hv = 0.0013
        A_vv = 0.0029
        B_vv = 0.13
        V1 = lai_field.values.flatten()
        V2 = V1 # initialize in surface model
        coef = 1.
        omega = 0.027 # vv
        omega = 0.015 # vh
        # IEM
        l = 0.01

        surface = k
        canopy = kk
        models = {'surface': surface, 'canopy': canopy}

        #### Optimization
        #-----------------

        if opt_mod == 'time invariant':

            dic = {"mv":sm_field.values.flatten(), "C_hh":C_hh, "C_vv":C_vv, "D_hh":D_hh, "D_vv":D_vv, "C_hv":C_hv, "D_hv":D_hv, "s":s, "clay":clay, "sand":sand, "f":freq, "bulk":bulk, "l":l, "canopy":canopy, "d":height_field.values.flatten(), "V1":V1, "V2":V2, "A_hh":A_hh, "B_hh":B_hh, "A_vv":A_vv, "B_vv":B_vv, "A_hv":A_hv, "B_hv":B_hv, "lai":lai_field.values.flatten(), "vwc":vwc_field.values.flatten(), "pol_value":pol_field.values.flatten(), "theta":theta_field.values.flatten(), "omega": omega, "coef": coef}

            if canopy == 'turbid_isotropic':
                var_opt = ['coef']
                guess = [2.]
                bounds = [(0.001,5.5)]
            elif surface == 'WaterCloud' and canopy == 'water_cloud':
                var_opt = ['A_vv', 'B_vv', 'A_hv', 'B_hv', 'C_vv', 'D_vv', 'C_hv', 'D_hv']
                guess = [A_vv, B_vv, A_hv, B_hv, C_vv, D_vv, C_hv, D_hv]
                bounds = [(0.,1), (0.,1), (0.,1), (0.,1), (-20.,-1.), (1.,20.), (-20.,-1.), (1.,20.)]
            elif canopy == 'water_cloud':
                var_opt = ['A_vv', 'B_vv', 'A_hv', 'B_hv']
                guess = [A_vv, B_vv, A_hv, B_hv]
                bounds = [(0.,1), (0.,1), (0.,1), (0.,1)]

            method = 'L-BFGS-B'

            res = minimize(fun_opt,guess,bounds=bounds, method=method)

            fun_opt(res.x)
            aaa = res.x

        if opt_mod == 'time variant':
            aaa = [[],[],[],[],[],[],[],[],[],[],[],[]]
            n=7

            for i in range(len(pol_field.values.flatten())-n+1):

                if type(coef) == float:
                    dic = {"mv":sm_field.values.flatten()[i:i+n], "C_hh":C_hh, "C_vv":C_vv, "D_hh":D_hh, "D_vv":D_vv, "C_hv":C_hv, "D_hv":D_hv, "V2":V2[i:i+n], "s":s, "clay":clay, "sand":sand, "f":freq, "bulk":bulk, "l":l, "canopy":canopy, "d":height_field.values.flatten()[i:i+n], "V1":V1[i:i+n], "A_hh":A_hh, "B_hh":B_hh, "A_vv":A_vv, "B_vv":B_vv, "A_hv":A_hv, "B_hv":B_hv, "lai":lai_field.values.flatten()[i:i+n], "vwc":vwc_field.values.flatten()[i:i+n], "pol_value":pol_field.values.flatten()[i:i+n], "theta":theta_field.values.flatten()[i:i+n], "omega": omega, "coef": coef}
                else:
                    dic = {"mv":sm_field.values.flatten()[i:i+n], "C_hh":C_hh, "C_vv":C_vv, "D_hh":D_hh, "D_vv":D_vv, "C_hv":C_hv, "D_hv":D_hv, "V2":V2[i:i+n], "s":s, "clay":clay, "sand":sand, "f":freq, "bulk":bulk, "l":l, "canopy":canopy, "d":height_field.values.flatten()[i:i+n], "V1":V1[i:i+n], "A_hh":A_hh, "B_hh":B_hh, "A_vv":A_vv, "B_vv":B_vv, "A_hv":A_hv, "B_hv":B_hv, "lai":lai_field.values.flatten()[i:i+n], "vwc":vwc_field.values.flatten()[i:i+n], "pol_value":pol_field.values.flatten()[i:i+n], "theta":theta_field.values.flatten()[i:i+n], "omega": omega, "coef": coef[i:i+n]}

                if canopy == 'turbid_isotropic' and surface == 'WaterCloud':
                    var_opt = ['coef', 'C_vv', 'D_vv', 'C_hv', 'D_hv']
                    guess = [0.01, C_vv, D_vv, C_hv, D_hv]
                    bounds = [(0.1,5.5), (-20.,-1.), (1.,20.), (-20.,-1.), (1.,20.)]
                elif canopy == 'turbid_isotropic':
                    var_opt = ['coef']
                    guess = [0.1]
                    bounds = [(0.,2)]
                elif surface == 'WaterCloud' and canopy == 'water_cloud':
                    # var_opt = ['A_vv', 'B_vv', 'A_hv', 'B_hv', 'C_vv', 'D_vv', 'C_hv', 'D_hv']
                    # guess = [A_vv, B_vv, A_hv, B_hv, C_vv, D_vv, C_hv, D_hv]
                    # bounds = [(0.,1), (guess[1]*0.55, guess[1]*1.55), (0.,1), (guess[3]*0.75, guess[3]*1.25), (-20.,-1.), (1.,20.), (-20.,-1.), (1.,20.)]
                    var_opt = ['C_vv', 'D_vv', 'C_hv', 'D_hv']
                    guess = [C_vv, D_vv, C_hv, D_hv]
                    bounds = [(-20.,-1.), (1.,20.), (-20.,-1.), (1.,20.)]
                elif canopy == 'water_cloud':
                    var_opt = ['A_vv', 'B_vv', 'A_hv', 'B_hv']
                    guess = [A_vv, B_vv, A_hv, B_hv]
                    bounds = [(0.,1), (0.,1), (0.00001,1), (0.00001,1)]

                # var_opt = ['omega']
                # guess = [0.1]
                # bounds = [(0.,5.5)]

                # var_opt = ['s', 'coef', 'omega']
                # guess = [0.01, 0.1, 0.01]
                # bounds = [(0.001,0.03),(0.,2.5),(0.001,0.1)]

                # var_opt = ['C_hv', 'D_hv']
                # guess = [-13, 14]
                # bounds = [(-200.,100.),(-200.,400.)]

                # var_opt = ['A_vv', 'B_vv']

                # try:
                #     guess = [res.x[0], res.x[1]]
                # except:
                #     guess = [0.005, 0.09]
                # # bounds = [(0.000,5.),(0.001,5.)]
                # bounds = [(guess[0]*0.75, guess[0]*1.25), (guess[1]*0.75, guess[1]*1.25)]
                # bounds = [(guess[0]*0.9, guess[0]*1.1), (guess[1]*0.75, guess[1]*1.25)]
                # var_opt = ['coef', 'omega']
                # guess = [0.1, 0.22]
                # bounds = [(0.,5.5),(0.00001,0.2)]
                method = 'L-BFGS-B'
                # method = 'trust-exact'

                res = minimize(fun_opt,guess,bounds=bounds, method=method)

                fun_opt(res.x)

                for j in range(len(res.x)):
                    aaa[j].append(res.x[j])

            field_data, theta_field, sm_field, height_field, lai_field, vwc_field, vv_field, vh_field, pol_field = data_optimized_run(n, field_data, theta_field, sm_field, height_field, lai_field, vwc_field, pol)
            V1 = lai_field.values.flatten()
            V2 = V1 # initialize in surface model

        #-----------------------------------------------------------------

        for i in range(len(res.x)):
            exec('%s = %s' % (var_opt[i],aaa[i]))

        ke = coef * np.sqrt(lai_field.values.flatten())
        # ke = smooth(ke, 11)

        soil = Soil(mv=sm_field.values.flatten(), C_hh=np.array(C_hh), C_vv=np.array(C_vv), D_hh=np.array(D_hh), D_vv=np.array(D_vv), C_hv=np.array(C_hv), D_hv=np.array(D_hv), s=s, clay=clay, sand=sand, f=freq, bulk=bulk, l=l)

        can = OneLayer(canopy=canopy, ke_h=ke, ke_v=ke, d=height_field.values.flatten(), ks_h = omega*ke, ks_v = omega*ke, V1=np.array(V1), V2=np.array(V2), A_hh=np.array(A_hh), B_hh=np.array(B_hh), A_vv=np.array(A_vv), B_vv=np.array(B_vv), A_hv=np.array(A_hv), B_hv=np.array(B_hv))

        S = model.RTModel(surface=soil, canopy=can, models=models, theta=theta_field.values.flatten(), freq=freq)
        S.sigma0()
#-----------------------------------------------------------------
        date = field_data.index

        colormap = plt.get_cmap(colormaps[j])
        colors = [colormap(jj) for jj in np.linspace(0.35, 1., 4)]

        # ax.plot(10*np.log10(pol_field), 'ks-', label='Sentinel-1 Pol: ' + pol, linewidth=3)
        # ax.plot(date, 10*np.log10(S.__dict__['s0g'][pol[::-1]]), color=colors[0], marker='s', linestyle='--', label=pol+' s0g')
        # ax.plot(date, 10*np.log10(S.__dict__['s0c'][pol[::-1]]), color=colors[1], marker='s', linestyle='--', label=pol+' s0c')
        # ax.plot(date, 10*np.log10(S.__dict__['s0cgt'][pol[::-1]]), 'ms-', label=pol+' s0cgt')
        # ax.plot(date, 10*np.log10(S.__dict__['s0gcg'][pol[::-1]]), 'ys-', label=pol+' s0gcg')

        mask = ~np.isnan(pol_field.values.flatten()) & ~np.isnan(S.__dict__['stot'][pol[::-1]])
        slope, intercept, r_value, p_value, std_err = scipy.stats.linregress((pol_field.values.flatten()[mask]), (S.__dict__['stot'][pol[::-1]][mask]))
        slope1, intercept1, r_value1, p_value1, std_err1 = scipy.stats.linregress(10*np.log10(pol_field.values.flatten())[mask], 10*np.log10(S.__dict__['stot'][pol[::-1]])[mask])
        rmse = rmse_prediction(10*np.log10(pol_field.values.flatten()), 10*np.log10(S.__dict__['stot'][pol[::-1]]))

        if k == 'Oh92':
            hm = 'Oh92'
            colors = 'blue'
        elif k == 'Oh04':
            hm = 'Oh04'
            colors = 'red'
        elif k == 'Dubois95':
            hm='Dubois95'
            colors = 'orange'
        elif k == 'WaterCloud':
            hm = 'Water Cloud'
            colors = 'purple'
        elif k == 'I2EM':
            hm = 'IEM'
            colors = 'green'

        if plot == 'all':
            if kk == 'turbid_isotropic':

                ax.plot(date, 10*np.log10(S.__dict__['stot'][pol[::-1]]), color=colors, marker='s', linestyle='dashed', label = hm+ ' + ' +  'SSRT' + '; Pol: ' + pol + '; RMSE: ' + str(rmse)[0:4] + '; $R^2$: ' + str(r_value)[0:4])
            else:
                ax.plot(date, 10*np.log10(S.__dict__['stot'][pol[::-1]]), color=colors, marker='s', label = hm+ ' + ' +  'Water Cloud' + '; Pol: ' + pol + '; RMSE: ' + str(rmse)[0:4] + '; $R^2$: ' + str(r_value)[0:4])

        if plot == 'single':
            if style == 'scatterplot':
                if pol == 'vv':
                    ax.set_xlim([-22.5,-7.5])
                elif pol == 'vh':
                    ax.set_xlim([-30,-15])

                if style_2 == 'scatterplot_single_ESU':
                    ax.plot(10*np.log10(pol_field.values.flatten()),10*np.log10(S.__dict__['stot'][pol[::-1]]), 'rs', label=field)

                    x = 10*np.log10(pol_field.values.flatten())
                    y = 10*np.log10(S.__dict__['stot'][pol[::-1]])

                    lower_position = np.nanargmin(x)
                    upper_position = np.nanargmax(x)

                    ax.plot(np.array((x[lower_position],x[upper_position])),np.array((y[lower_position],y[upper_position])), '--r')

                else:
                    aa = []
                    bb = []
                    # cc = []

                    # field_plot = ['508_high', '508_low', '508_med']
                    jj = 0
                    colors = ['ks', 'ys', 'ms', 'rs']

                    for field in field_plot:
                        df, df_agro, field_data, field_data_orbit, theta_field, sm_field, height_field, lai_field, vwc_field, pol_field = read_data(path, file_name, extension, field, path_agro, file_name_agro, extension_agro)
                        field_data, theta_field, sm_field, height_field, lai_field, vwc_field, vv_field, vh_field, pol_field = data_optimized_run(n, field_data, theta_field, sm_field, height_field, lai_field, vwc_field, pol)

                        soil = Soil(mv=sm_field.values.flatten(), C_hh=np.array(C_hh), C_vv=np.array(C_vv), D_hh=np.array(D_hh), D_vv=np.array(D_vv), C_hv=np.array(C_hv), D_hv=np.array(D_hv), s=s, clay=clay, sand=sand, f=freq, bulk=bulk, l=l)

                        can = OneLayer(canopy=canopy, ke_h=ke, ke_v=ke, d=height_field.values.flatten(), ks_h = omega*ke, ks_v = omega*ke, V1=np.array(V1), V2=np.array(V2), A_hh=np.array(A_hh), B_hh=np.array(B_hh), A_vv=np.array(A_vv), B_vv=np.array(B_vv), A_hv=np.array(A_hv), B_hv=np.array(B_hv))

                        S = model.RTModel(surface=soil, canopy=can, models=models, theta=theta_field.values.flatten(), freq=freq)
                        S.sigma0()

                        ax.plot(10*np.log10(pol_field.values.flatten()),10*np.log10(S.__dict__['stot'][pol[::-1]]), colors[jj], label=field)

                        slope, intercept, r_value, p_value, std_err = linregress(10*np.log10(pol_field.values.flatten())[~np.isnan(10*np.log10(S.__dict__['stot'][pol[::-1]]))], 10*np.log10(S.__dict__['stot'][pol[::-1]])[~np.isnan(10*np.log10(S.__dict__['stot'][pol[::-1]]))])
                        line = slope * 10*np.log10(S.__dict__['stot'][pol[::-1]]) + intercept

                        # ax.plot(10*np.log10(S.__dict__['stot'][pol[::-1]]), line)

                        lower_position = np.nanargmin(line)
                        upper_position = np.nanargmax(line)

                        ax.plot(np.array((10*np.log10(S.__dict__['stot'][pol[::-1]])[lower_position],10*np.log10(S.__dict__['stot'][pol[::-1]])[upper_position])),np.array((line[lower_position],line[upper_position])), '--'+colors[jj][0])

                        aa = np.append(aa, 10*np.log10(pol_field.values.flatten()))
                        bb = np.append(bb, 10*np.log10(S.__dict__['stot'][pol[::-1]]))
                        jj = jj+1
            else:
                ax.plot(date, 10*np.log10(S.__dict__['stot'][pol[::-1]]), color='orange', marker='s', label=S.models['surface']+ ' + ' +  S.models['canopy'] + ' Pol: ' + pol + '; RMSE: ' + str(rmse)[0:4] + '; $R^2$: ' + str(r_value)[0:4])
                ax.plot(date, 10*np.log10(S.__dict__['s0g'][pol[::-1]]), color='red', marker='s', label='Ground contribution')
                ax.plot(date, 10*np.log10(S.__dict__['s0c'][pol[::-1]]), color='green', marker='s', label='Canopy contribution')

        j = j+1

if style == 'scatterplot':
    pass
else:
    ax.plot(10*np.log10(pol_field), 'ks-', label='Sentinel-1 Pol: ' + pol, linewidth=3)
    plt.legend()
    plt.title(field)

if plot == 'all':
    # plt.show()
    plt.savefig(plot_output_path+pol+'_all_'+opt_mod)

if plot == 'single':
    if style == 'scatterplot':
        plt.ylabel(surface + ' ' + canopy + ' [dB]')
        plt.xlabel('Sentinel-1 [dB]')
        plt.legend()
        x = np.linspace(np.min(10*np.log10(pol_field.values.flatten()))-2, np.max(10*np.log10(pol_field.values.flatten()))+2, 16)
        ax.plot(x,x)
        if style_2 == 'scatterplot_single_ESU':
            www = rmse_prediction(10*np.log10(pol_field).values.flatten(), 10*np.log10(S.__dict__['stot'][pol[::-1]]))
            plt.title(pol+' ' + field + ' ' + surface + ' ' + canopy + '$R^2$='+str(r_value)+' RMSE='+str(www))
            plt.savefig(plot_output_path+'scatterplot_fertig_single_'+field+'_'+pol+'_'+file_name+'_'+S.models['surface']+'_'+S.models['canopy'])
        else:
            www = rmse_prediction(aa, bb)
            # slope, intercept, r_value, p_value, std_err = linregress(aaa[~np.isnan(bbb)], bbb[~np.isnan(bbb)])
            plt.title(pol+' ' + field + ' ' + surface + ' ' + canopy + '$R^2$='+str(r_value)+' RMSE='+str(www))
            plt.savefig(plot_output_path+'scatterplot_fertig_'+field+'_'+pol+'_'+file_name+'_'+S.models['surface']+'_'+S.models['canopy'])
    else:
        plt.savefig(plot_output_path+pol+'_single_'+opt_mod+'_'+S.models['surface']+'_'+S.models['canopy'])

pdb.set_trace()


<filename>Pandas-analyzing-dataset/code.py
# --------------
# Importing header files
import numpy as np
import pandas as pd
from scipy.stats import mode

import warnings
warnings.filterwarnings('ignore')

#Reading file
bank = pd.read_csv(path)
#Code starts here
categorical_var=bank.select_dtypes(include = 'object')
#print(categorical_var)
numerical_var=bank.select_dtypes(include = 'number')
#print(numerical_var)
print(categorical_var.shape)
print(numerical_var.shape)
bank.drop('Loan_ID',inplace=True,axis=1)
banks = pd.DataFrame(bank)
print(banks.isnull().sum())
print(banks.shape)
bank_mode = banks.mode().iloc[0]
banks.fillna(bank_mode,inplace=True)
print(banks.isnull().sum().values.sum())
avg_loan_amount = pd.pivot_table(banks,index=('Gender','Married','Self_Employed'),values='LoanAmount').agg(np.mean)
print(avg_loan_amount)
loan_approved_se = banks[(banks['Self_Employed'] == 'Yes') & (banks['Loan_Status'] == 'Y')].count()
loan_approved_nse = banks[(banks['Self_Employed'] == 'No') & (banks['Loan_Status'] == 'Y')].count()
Loan_Status = 614
percentage_se = round((loan_approved_se/Loan_Status)*100,2)
percentage_nse = round((loan_approved_nse/Loan_Status)*100,2)
print(percentage_se)
print(percentage_nse)
loan_term = banks['Loan_Amount_Term'].apply(lambda x : int(x) / 12)
big_loan_term = len(loan_term[loan_term>=25])
print(big_loan_term)
columns_to_show = ['ApplicantIncome', 'Credit_History']
loan_groupby=banks.groupby(['Loan_Status'])
loan_groupby=loan_groupby[columns_to_show]
mean_values = loan_groupby.agg(np.mean)
print(mean_values)


<reponame>ArgonneCPAC/skysim
"""
"""
import numpy as np
from scipy.stats import johnsonsb
from astropy.utils.misc import NumpyRNGContext
from halotools.empirical_models import conditional_abunmatch

def calculate_johnsonsb_params_disk(
        magr, johnsonsb_disk_table_abscissa=[-19, -21],
        johnsonsb_disk_table=[0.15, -0.15], **kwargs):
    return np.interp(magr, johnsonsb_disk_table_abscissa, johnsonsb_disk_table)

def calculate_johnsonsb_params_bulge(
        magr, johnsonsb_bulge_table_abscissa=[-19, -21],
        johnsonsb_bulge_table=[1.5, 0.6], **kwargs):
    return np.interp(magr, johnsonsb_bulge_table_abscissa, johnsonsb_bulge_table)

def monte_carlo_ellipticity_disk(magr, inclination = None, seed=None, **kwargs):
    """
    Parameters
    ----------
    magr : ndarray
        Numpy array of shape (ngals, )

    inclination : ndarray
        Numpy array of shape (ngals, )

    Returns
    -------
    ellipticity_realization : ndarray
    """

    magr = np.atleast_1d(magr)
    inclination = np.atleast_1d(inclination)

    a = calculate_johnsonsb_params_disk(magr, **kwargs)
    b = np.ones_like(a)

    with NumpyRNGContext(seed):
        ellipticity_realization = johnsonsb.rvs(a, b)

    nwin = 101
    if inclination is None:
        inclination_correlated_ellipticity = conditional_abunmatch(
            magr, inclination, magr, ellipticity_realization, nwin)
        return inclination_correlated_ellipticity
    else:
        return ellipticity_realization

def monte_carlo_ellipticity_bulge(magr, seed=None, **kwargs):
    """
    Parameters
    ----------
    magr : ndarray
        Numpy array of shape (ngals, )

    Returns
    -------
    ellipticity_realization : ndarray
    """
    magr = np.atleast_1d(magr)

    a = calculate_johnsonsb_params_bulge(magr, **kwargs)
    b = np.ones_like(a)

    with NumpyRNGContext(seed):
        ellipticity_realization = johnsonsb.rvs(a, b)
    return ellipticity_realization