<reponame>AndreasMadsen/bachelor-code

import urllib.parse as url
import http.server
import os.path as path
import ujson as json
import numpy as np
import scipy.sparse

import time

thisdir = path.dirname(path.realpath(__file__))

class npzToObj:
    def __init__(self, npz):
        """
        npz are lazyloaded, this preloads it loads
        """
        for name in npz.files:
            setattr(self, name, npz[name])

class GraphServer:
    def __init__(self, clusters, distance, connectivity, nodes, verbose=False):
        self._verbose = verbose
        if (self._verbose): print("Initializing graph server")

        self._clusters = npzToObj(clusters)
        self._distance = scipy.sparse.csr_matrix(distance)
        self._connectivity = scipy.sparse.csr_matrix(connectivity)
        self._raw_nodes = nodes
        self._nodes = [[node['title'], node['website'], node['id']] for node in nodes]

        # Create a http server
        if (self._verbose): print("\tCreating http server")
        self._server = http.server.HTTPServer(('127.0.0.1', 8000), GraphServer.Responder)
        self._server._owner = self

    def listen(self):
        if (self._verbose): print("Server listening on http://127.0.0.1:8000")
        self._server.serve_forever()

    def fetch_article(self, id):
        return self._raw_nodes[id]

    def fetch_compare(self, a, b):
        return {
            "connecitivity": bool(self._connectivity[min(a, b), max(a, b)]),
            "distance": float(self._distance[min(a, b), max(a, b)])
        }

    def _groups_from_title(self, search):
        if (self._verbose): print("\tSearching for \"%s\"" % (search))
        words = search.split()

        # Construct and execute SQL search query
        match = np.fromiter([
            np.all([(word in node[0]) for word in words])
            for node in self._nodes
        ], dtype='bool')

        # Fetch groups
        groups = set(int(group) for group in self._clusters.node_to_group[match])
        if (self._verbose): print("\tSearch complete, found %d groups" % len(groups))

        return groups

    def _fetch_single_group(self, group_id):
        if (self._verbose): print("\tFetching group %d" % group_id)

        # Create node info object
        nodes = self._clusters.group[group_id, 0:self._clusters.group_size[group_id]]
        node_info = [self._nodes[id] for id in nodes]

        # Create link info object
        if (self._verbose): print("\tBuilding link object")
        mask = np.any(self._clusters.connects_row[:, np.newaxis] == nodes, axis=1)
        if (np.sum(mask) == 0):
            link_info = []
        else:
            info = (
                self._clusters.connects_row[mask],
                self._clusters.connects_col[mask]
            )

            link_info = [
                [int(row), int(col), float(data)]
                for (row, col, data)
                in zip(info[0], info[1], self._distance[info].A1)
            ]

        # Send group info
        return (node_info, link_info)

    def fetch_graph(self, groups):
        if (self._verbose): print("Fetching groups")

        # Validate groups
        max_group_size = int(self._clusters.group.shape[0])
        for group in groups:
            if (group >= max_group_size):
                if (self._verbose): print("\tGroup with id %d do not exists" % group)
                return None

        # Initialize info array
        info = []

        # Fetch group info
        for group in groups:
            (node_info, link_info) = self._fetch_single_group(group)
            info.append({
                "group": group,
                "nodes": node_info,
                "links": link_info
            })

        # Done return result
        return info

    class Responder(http.server.BaseHTTPRequestHandler):
        def __init__(self, *args, **kwargs):
            self._owner = args[2]._owner
            self._verbose = self._owner._verbose
            http.server.BaseHTTPRequestHandler.__init__(self, *args, **kwargs)

        def do_GET(self):
            if self.path         == '/'             : self.index_page()
            elif self.path       == '/details'      : self.details_page()
            elif self.path       == '/d3.js'        : self.d3_script()
            elif self.path       == '/view.js'      : self.view_script()
            elif self.path       == '/style.css'    : self.style_script()
            elif self.path[0:11] == '/graph.json'   : self.graph_data()
            elif self.path[0:13] == '/article.json' : self.article_data()
            elif self.path[0:13] == '/compare.json' : self.compare_data()
            else                                    : self.otherwise()

        def index_page(self):
            self.send_response(200)
            self.send_header('Content-Type', 'text/html; charset=UTF-8')
            self.end_headers()

            f = open(path.join(thisdir, 'public', 'index.html'), 'rb')
            self.wfile.write(f.read())
            f.close()

        def details_page(self):
            self.send_response(200)
            self.send_header('Content-Type', 'text/html; charset=UTF-8')
            self.end_headers()

            f = open(path.join(thisdir, 'public', 'details.html'), 'rb')
            self.wfile.write(f.read())
            f.close()

        def d3_script(self):
            self.send_response(200)
            self.send_header('Content-Type', 'application/javascript; charset=UTF-8')
            self.end_headers()

            f = open(path.join(thisdir, 'public', 'd3.js'), 'rb')
            self.wfile.write(f.read())
            f.close()

        def view_script(self):
            self.send_response(200)
            self.send_header('Content-Type', 'application/javascript; charset=UTF-8')
            self.end_headers()

            f = open(path.join(thisdir, 'public', 'view.js'), 'rb')
            self.wfile.write(f.read())
            f.close()

        def style_script(self):
            self.send_response(200)
            self.send_header('Content-Type', 'text/css; charset=UTF-8')
            self.end_headers()

            f = open(path.join(thisdir, 'public', 'style.css'), 'rb')
            self.wfile.write(f.read())
            f.close()

        def article_data(self):
            self.send_response(200)
            self.send_header('Content-Type', 'application/json; charset=UTF-8')
            self.end_headers()

            query = url.parse_qs(url.urlparse(self.path).query)
            data = self._owner.fetch_article(int(query['id'][0]))
            self.wfile.write(bytes(json.dumps(data), 'ASCII'))

        def compare_data(self):
            self.send_response(200)
            self.send_header('Content-Type', 'application/json; charset=UTF-8')
            self.end_headers()

            query = url.parse_qs(url.urlparse(self.path).query)
            data = self._owner.fetch_compare(int(query['a'][0]), int(query['b'][0]))
            self.wfile.write(bytes(json.dumps(data), 'ASCII'))

        def graph_data(self):
            self.send_response(200)
            self.send_header('Content-Type', 'application/json; charset=UTF-8')
            self.end_headers()

            query = url.parse_qs(url.urlparse(self.path).query)

            # Convert input to group list
            if ('title' in query):
                groups = self._owner._groups_from_title(query['title'][0])
            elif ('groups' in query):
                groups = set(int(group) for group in query['groups'][0].split(","))
            else:
                groups = None

            # Fetch nodes and links
            if (groups is None):
                data = None
            else:
                data = self._owner.fetch_graph(groups)

            # Send data
            if (data is None):
                if (self._verbose): print("\tBad query input")
                self.wfile.write(bytes('null', 'ASCII'))
            else:
                if (self._verbose): print("\tSending result")

                self.wfile.write(bytes('[', 'ASCII'))
                for index, group in enumerate(data):
                    self.wfile.write(bytes('{"group":', 'ASCII'))
                    self.wfile.write(bytes(json.dumps(group['group']), 'ASCII'))

                    self.wfile.write(bytes(', "nodes":', 'ASCII'))
                    self.wfile.write(bytes(json.dumps(group['nodes']), 'ASCII'))

                    self.wfile.write(bytes(', "links":', 'ASCII'))
                    self.wfile.write(bytes(json.dumps(group['links']), 'ASCII'))

                    # Write } if last item otherwise write },\n
                    if (index == (len(data) - 1)):
                        self.wfile.write(bytes('}', 'ASCII'))
                    else:
                        self.wfile.write(bytes('},\n', 'ASCII'))
                self.wfile.write(bytes(']', 'ASCII'))

                if (self._verbose): print("\tResult send")

        def otherwise(self):
            self.send_response(404)
            self.send_header('Content-Type', 'text/html; charset=UTF-8')
            self.end_headers()

            self.wfile.write(bytes('
Sorry invalid path (404)
', 'UTF-8'))


<reponame>ACTCollaboration/mnms
#!/usr/bin/env python3
from pixell import enmap, utils
from mnms import utils as tnu, mpi

import numpy as np
from scipy.interpolate import RectBivariateSpline as rectinterp

# Some utilities for calculating effective mode coupling matrices in 2D Fourier space
# Currently, only supports matrices which are formed as outerproducts of two vectors
# If want general mode coupling matrices, would need to write a C extension because things
# won't separate nicely and you'll have to do a full sum

def get_outer_mask(arr1, arr2=None):
    arr1 = np.atleast_1d(arr1)
    if arr2 is None:
        arr2 = arr1
    else:
        arr2 = np.atleast_1d(arr2)
    assert arr1.ndim == 1 and arr2.ndim == 1

    return np.einsum('y,x->yx',arr1,arr2)

def get_vecs_from_outer_mask(mask):
    Ny, Nx = mask.shape
    arr1 = mask[Ny//2]
    arr2 = mask[:, Nx//2]
    return arr1, arr2

def get_1d_kernel(arr1):
    arr1 = np.atleast_1d(arr1)
    assert arr1.ndim == 1

    fnorm = np.abs(np.fft.fft(arr1) / arr1.size)**2
    kernel = np.zeros((fnorm.size, fnorm.size))
    for i in range(fnorm.size):
        for j in range(fnorm.size):
            kernel[i, j] = fnorm[i - j]
    return kernel

def get_binned_1d_kernel(kernel, bin_slices):
    assert kernel.ndim == 2
    assert kernel.shape[0] == kernel.shape[1]

    nbins = len(bin_slices)
    nperbin = kernel.shape[0] // nbins
    binned_kernel = np.zeros((nbins, nbins))
    for i in range(nbins):
        for j in range(nbins):
            binned_kernel[i, j] = kernel[bin_slices[i], bin_slices[j]].sum() / nperbin
    return binned_kernel

def get_mcm(arr1, arr2=None, bin_slices1=None, bin_slices2=None):
    arr1 = np.atleast_1d(arr1)
    if arr2 is None:
        square = True
        assert arr1.ndim == 1
    else:
        arr2 = np.atleast_1d(arr2)
        assert arr1.ndim == 1 and arr2.ndim == 1
        if np.allclose(arr1, arr2, rtol=0):
            square = True
        else:
            square = False

    kernel1 = get_1d_kernel(arr1)
    if bin_slices1 is not None:
        kernel1 = get_binned_1d_kernel(kernel1, bin_slices1)
    if square:
        kernel2 = kernel1
    else:
        kernel2 = get_1d_kernel(arr2)
        if bin_slices2 is not None:
            kernel2 = get_binned_1d_kernel(kernel2, bin_slices2)

    # kernel1 is M_yy' and kernel2 is M_xx', we want M_yxy'x'
    return np.einsum('Yy,Xx->YXyx', kernel1, kernel2)

def get_inv_mcm(arr1, arr2=None, bin_slices1=None, bin_slices2=None, verbose=False):
    M = get_mcm(arr1, arr2=arr2, bin_slices1=bin_slices1, bin_slices2=bin_slices2)
    assert M.ndim == 4
    assert M.shape[0] == M.shape[2] and M.shape[1] == M.shape[3]
    Ny = M.shape[0]
    Nx = M.shape[1]
    M = M.reshape(Ny*Nx, Ny*Nx)
    if verbose:
        print(f'Condition number of MCM is {np.round(np.linalg.cond(M), 3)}')
    M = np.linalg.inv(M)
    return M.reshape(Ny, Nx, Ny, Nx)

def get_uniform_bin_slices(arr1, nbins):
    arr1 = np.atleast_1d(arr1)
    assert arr1.ndim == 1

    npix = arr1.size
    counts, displs = mpi.mpi_distribute(npix, nbins)
    slices = tuple(slice(d, d + counts[i]) for i, d in enumerate(displs))
    return slices


<filename>1400OS_09_Codes/ceps.py
import os
import glob

import numpy as np
import scipy
import scipy.io.wavfile
from librosa.feature import mfcc

from utils import GENRE_DIR, CHART_DIR

def write_ceps(ceps, fn):
    """
    Write the MFCC to separate files to speed up processing.
    """
    base_fn, ext = os.path.splitext(fn)
    data_fn = CHART_DIR + "/" + base_fn + ".ceps"
    np.save(data_fn, ceps)
    print("Written", data_fn)

def create_ceps(fn):
    sample_rate, X = scipy.io.wavfile.read(fn)

    Y = X * 1.0

    # ceps, mspec, spec = mfcc(Y)
    ceps = mfcc(Y)
    write_ceps(ceps, fn)

def read_ceps(genre_list, base_dir=GENRE_DIR):
    X = []
    y = []
    for label, genre in enumerate(genre_list):
        for fn in glob.glob(os.path.join(base_dir, genre, "*.ceps.npy")):
            ceps = np.load(fn)
            num_ceps = len(ceps)
            X.append(np.mean(ceps[int(num_ceps / 10):int(num_ceps * 9 / 10)], axis=0))
            y.append(label)

    return np.array(X), np.array(y)

if __name__ == "__main__":
    # os.chdir(GENRE_DIR)
    glob_wav = os.path.join(".", "*.wav")
    print(glob_wav)
    for fn in glob.glob(glob_wav):
        create_ceps(fn)


<reponame>msc-acse/acse-9-independent-research-project-Wade003
from scipy import *
from pylab import *
num_detectors = 100
x = 0.5+0.25*arange(0,float(num_detectors))/float(num_detectors)
y = zeros(num_detectors) + 0.5

t = 0.
n_cycles = 1
dt = 0.1/n_cycles
tmax = 8

def vel(x,y):
    return [-(y-0.5),x-0.5]

while(t<tmax):
    t = t + dt
    [k1_x,k1_y] = vel(x,y)
    [k2_x,k2_y] = vel(x+0.5*dt*k1_x,y+0.5*dt*k1_y)
    [k3_x,k3_y] = vel(x+0.5*dt*k2_x,y+0.5*dt*k2_y)
    [k4_x,k4_y] = vel(x+dt*k3_x,y+dt*k3_y)
    x = x + dt*(k1_x/6.+k2_x/3. + k3_x/3. + k4_x/6.)
    y = y + dt*(k1_y/6.+k2_y/3. + k3_y/3. + k4_y/6.)

plot(x,y,'.')
#show()

x.tofile('Xvals.txt',sep=' ')
y.tofile('Yvals.txt',sep=' ')


"""Class for generating pronunciations from G2P models"""
from __future__ import annotations

import csv
import functools
import multiprocessing as mp
import os
import queue
import statistics
import time
from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Union

import tqdm

from montreal_forced_aligner.abc import TopLevelMfaWorker
from montreal_forced_aligner.corpus.text_corpus import TextCorpusMixin
from montreal_forced_aligner.exceptions import PyniniGenerationError
from montreal_forced_aligner.g2p.mixins import G2PTopLevelMixin
from montreal_forced_aligner.helper import comma_join, score_g2p
from montreal_forced_aligner.models import G2PModel
from montreal_forced_aligner.utils import Stopped

try:
    import pynini
    from pynini import Fst, SymbolTable, TokenType
    from pynini.lib import rewrite

    G2P_DISABLED = False
except ImportError:
    pynini = None
    TokenType = str
    Fst = None
    SymbolTable = None
    rewrite = None
    G2P_DISABLED = True

if TYPE_CHECKING:
    SpeakerCharacterType = Union[str, int]

__all__ = [
    "Rewriter",
    "RewriterWorker",
    "PyniniGenerator",
    "PyniniCorpusGenerator",
    "PyniniWordListGenerator",
]

def threshold_lattice_to_dfa(
    lattice: pynini.Fst, threshold: float = 1.0, state_multiplier: int = 2
) -> pynini.Fst:
    """Constructs a (possibly pruned) weighted DFA of output strings.
    Given an epsilon-free lattice of output strings (such as produced by
    rewrite_lattice), attempts to determinize it, pruning non-optimal paths if
    optimal_only is true. This is valid only in a semiring with the path property.
    To prevent unexpected blowup during determinization, a state threshold is
    also used and a warning is logged if this exact threshold is reached. The
    threshold is a multiplier of the size of input lattice (by default, 4), plus
    a small constant factor. This is intended by a sensible default and is not an
    inherently meaningful value in and of itself.

    Parameters
    ----------
    lattice: :class:`~pynini.Fst`
        Epsilon-free non-deterministic finite acceptor.
    threshold: float
        Threshold for weights, 1.0 is optimal only, 0 is for all paths, greater than 1
        prunes the lattice to include paths with costs less than the optimal path's score times the threshold
    state_multiplier: int
        Max ratio for the number of states in the DFA lattice to the NFA lattice; if exceeded, a warning is logged.

    Returns
    -------
    :class:`~pynini.Fst`
        Epsilon-free deterministic finite acceptor.
    """
    weight_type = lattice.weight_type()
    weight_threshold = pynini.Weight(weight_type, threshold)
    state_threshold = 256 + state_multiplier * lattice.num_states()
    lattice = pynini.determinize(lattice, nstate=state_threshold, weight=weight_threshold)
    return lattice

def optimal_rewrites(
    string: pynini.FstLike,
    rule: pynini.Fst,
    input_token_type: Optional[TokenType] = None,
    output_token_type: Optional[TokenType] = None,
    threshold: float = 1,
) -> List[str]:
    """Returns all optimal rewrites.
    Args:
    string: Input string or FST.
    rule: Input rule WFST.
    input_token_type: Optional input token type, or symbol table.
    output_token_type: Optional output token type, or symbol table.
    threshold: Threshold for weights (1 is optimal only, 0 is for all paths)
    Returns:
    A tuple of output strings.
    """
    lattice = rewrite.rewrite_lattice(string, rule, input_token_type)
    lattice = threshold_lattice_to_dfa(lattice, threshold, 4)
    return rewrite.lattice_to_strings(lattice, output_token_type)

class Rewriter:
    """
    Helper object for rewriting

    Parameters
    ----------
    fst: pynini.Fst
        G2P FST model
    input_token_type: pynini.TokenType
        Grapheme symbol table or "utf8"
    output_token_type: pynini.SymbolTable
        Phone symbol table
    num_pronunciations: int
        Number of pronunciations, default to 0.  If this is 0, thresholding is used
    threshold: float
        Threshold to use for pruning rewrite lattice, defaults to 1.5, only used if num_pronunciations is 0
    """

    def __init__(
        self,
        fst: Fst,
        input_token_type: TokenType,
        output_token_type: SymbolTable,
        num_pronunciations: int = 0,
        threshold: float = 1,
    ):
        if num_pronunciations > 0:
            self.rewrite = functools.partial(
                rewrite.top_rewrites,
                nshortest=num_pronunciations,
                rule=fst,
                input_token_type=input_token_type,
                output_token_type=output_token_type,
            )
        else:
            self.rewrite = functools.partial(
                optimal_rewrites,
                threshold=threshold,
                rule=fst,
                input_token_type=input_token_type,
                output_token_type=output_token_type,
            )

    def __call__(self, i: str) -> List[Tuple[str, ...]]:  # pragma: no cover
        """Call the rewrite function"""
        hypotheses = self.rewrite(i)
        return [x for x in hypotheses if x]

class PhonetisaurusRewriter:
    """
    Helper function for rewriting

    Parameters
    ----------
    fst: pynini.Fst
        G2P FST model
    input_token_type: pynini.SymbolTable
        Grapheme symbol table
    output_token_type: pynini.SymbolTable
    num_pronunciations: int
        Number of pronunciations, default to 0.  If this is 0, thresholding is used
    threshold: float
        Threshold to use for pruning rewrite lattice, defaults to 1.5, only used if num_pronunciations is 0
    grapheme_order: int
        Maximum number of graphemes to consider single segment
    seq_sep: str
        Separator to use between grapheme symbols
    """

    def __init__(
        self,
        fst: Fst,
        input_token_type: SymbolTable,
        output_token_type: SymbolTable,
        num_pronunciations: int = 0,
        threshold: float = 1.5,
        grapheme_order: int = 2,
        seq_sep: str = "|",
    ):
        self.fst = fst
        self.seq_sep = seq_sep
        self.input_token_type = input_token_type
        self.output_token_type = output_token_type
        self.grapheme_order = grapheme_order
        if num_pronunciations > 0:
            self.rewrite = functools.partial(
                rewrite.top_rewrites,
                nshortest=num_pronunciations,
                rule=fst,
                input_token_type=None,
                output_token_type=output_token_type,
            )
        else:
            self.rewrite = functools.partial(
                optimal_rewrites,
                threshold=threshold,
                rule=fst,
                input_token_type=None,
                output_token_type=output_token_type,
            )

    def __call__(self, graphemes: str) -> List[Tuple[str, ...]]:  # pragma: no cover
        """Call the rewrite function"""
        fst = pynini.Fst()
        one = pynini.Weight.one(fst.weight_type())
        max_state = 0
        for i in range(len(graphemes)):
            start_state = fst.add_state()
            for j in range(1, self.grapheme_order + 1):
                if i + j <= len(graphemes):
                    substring = self.seq_sep.join(graphemes[i : i + j])
                    state = self.input_token_type.find(substring)
                    if state != pynini.NO_SYMBOL:
                        fst.add_arc(start_state, pynini.Arc(state, state, one, i + j))
                    if i + j >= max_state:
                        max_state = i + j
        for _ in range(fst.num_states(), max_state + 1):
            fst.add_state()
        fst.set_start(0)
        fst.set_final(len(graphemes), one)
        fst.set_input_symbols(self.input_token_type)
        fst.set_output_symbols(self.input_token_type)
        hypotheses = self.rewrite(fst)
        hypotheses = [x.replace(self.seq_sep, " ") for x in hypotheses if x]
        return hypotheses

class RewriterWorker(mp.Process):
    """
    Rewriter process

    Parameters
    ----------
    job_queue: :class:`~multiprocessing.Queue`
        Queue to pull words from
    return_queue: :class:`~multiprocessing.Queue`
        Queue to put pronunciations
    rewriter: :class:`~montreal_forced_aligner.g2p.generator.Rewriter`
        Function to generate pronunciations of words
    stopped: :class:`~montreal_forced_aligner.utils.Stopped`
        Stop check
    """

    def __init__(
        self,
        job_queue: mp.Queue,
        return_queue: mp.Queue,
        rewriter: Rewriter,
        stopped: Stopped,
    ):
        mp.Process.__init__(self)
        self.job_queue = job_queue
        self.return_queue = return_queue
        self.rewriter = rewriter
        self.stopped = stopped
        self.finished = Stopped()

    def run(self) -> None:
        """Run the rewriting function"""
        while True:
            try:
                word = self.job_queue.get(timeout=1)
            except queue.Empty:
                break
            if self.stopped.stop_check():
                continue
            try:
                rep = self.rewriter(word)
                self.return_queue.put((word, rep))
            except rewrite.Error:
                pass
            except Exception as e:  # noqa
                self.stopped.stop()
                self.return_queue.put(e)
                raise
        self.finished.stop()
        return

def clean_up_word(word: str, graphemes: Set[str]) -> Tuple[str, Set[str]]:
    """
    Clean up word by removing graphemes not in a specified set

    Parameters
    ----------
    word : str
        Input string
    graphemes: set[str]
        Set of allowable graphemes

    Returns
    -------
    str
        Cleaned up word
    Set[str]
        Graphemes excluded
    """
    new_word = []
    missing_graphemes = set()
    for c in word:
        if c not in graphemes:
            missing_graphemes.add(c)
        else:
            new_word.append(c)
    return "".join(new_word), missing_graphemes

class OrthographyGenerator(G2PTopLevelMixin):
    """
    Abstract mixin class for generating "pronunciations" based off the orthographic word

    See Also
    --------
    :class:`~montreal_forced_aligner.g2p.mixins.G2PTopLevelMixin`
        For top level G2P generation parameters
    """

    def generate_pronunciations(self) -> Dict[str, List[str]]:
        """
        Generate pronunciations for the word set

        Returns
        -------
        dict[str, Word]
            Mapping of words to their "pronunciation"
        """
        pronunciations = {}
        for word in self.words_to_g2p:
            pronunciations[word] = [" ".join(word)]
        return pronunciations

class PyniniGenerator(G2PTopLevelMixin):
    """
    Class for generating pronunciations from a Pynini G2P model

    Parameters
    ----------
    g2p_model_path: str
        Path to G2P model
    strict_graphemes: bool
        Flag for whether to be strict with missing graphemes and skip words containing new graphemes

    See Also
    --------
    :class:`~montreal_forced_aligner.g2p.mixins.G2PTopLevelMixin`
        For top level G2P generation parameters

    Attributes
    ----------
    g2p_model: G2PModel
        G2P model
    """

    def __init__(self, g2p_model_path: str, strict_graphemes: bool = False, **kwargs):
        self.strict_graphemes = strict_graphemes
        super().__init__(**kwargs)
        self.g2p_model = G2PModel(
            g2p_model_path, root_directory=getattr(self, "workflow_directory", None)
        )

    def generate_pronunciations(self) -> Dict[str, List[str]]:
        """
        Generate pronunciations

        Returns
        -------
        dict[str, list[str]]
            Mappings of keys to their generated pronunciations
        """

        fst = pynini.Fst.read(self.g2p_model.fst_path)
        if self.g2p_model.meta["architecture"] == "phonetisaurus":
            output_token_type = pynini.SymbolTable.read_text(self.g2p_model.sym_path)
            input_token_type = pynini.SymbolTable.read_text(self.g2p_model.grapheme_sym_path)
            fst.set_input_symbols(input_token_type)
            fst.set_output_symbols(output_token_type)
            rewriter = PhonetisaurusRewriter(
                fst,
                input_token_type,
                output_token_type,
                num_pronunciations=self.num_pronunciations,
                threshold=self.g2p_threshold,
            )
        else:
            output_token_type = "utf8"
            input_token_type = "utf8"
            if self.g2p_model.sym_path is not None and os.path.exists(self.g2p_model.sym_path):
                output_token_type = pynini.SymbolTable.read_text(self.g2p_model.sym_path)
            rewriter = Rewriter(
                fst,
                input_token_type,
                output_token_type,
                num_pronunciations=self.num_pronunciations,
                threshold=self.g2p_threshold,
            )

        num_words = len(self.words_to_g2p)
        begin = time.time()
        missing_graphemes = set()
        self.log_info("Generating pronunciations...")
        to_return = {}
        skipped_words = 0
        if num_words < 30 or self.num_jobs == 1:
            with tqdm.tqdm(total=num_words, disable=getattr(self, "quiet", False)) as pbar:
                for word in self.words_to_g2p:
                    w, m = clean_up_word(word, self.g2p_model.meta["graphemes"])
                    pbar.update(1)
                    missing_graphemes = missing_graphemes | m
                    if self.strict_graphemes and m:
                        skipped_words += 1
                        continue
                    if not w:
                        skipped_words += 1
                        continue
                    try:
                        prons = rewriter(w)
                    except rewrite.Error:
                        continue
                    to_return[word] = prons
                self.log_debug(
                    f"Skipping {skipped_words} words for containing the following graphemes: "
                    f"{comma_join(sorted(missing_graphemes))}"
                )
        else:
            stopped = Stopped()
            job_queue = mp.Queue()
            for word in self.words_to_g2p:
                w, m = clean_up_word(word, self.g2p_model.meta["graphemes"])
                missing_graphemes = missing_graphemes | m
                if self.strict_graphemes and m:
                    skipped_words += 1
                    continue
                if not w:
                    skipped_words += 1
                    continue
                job_queue.put(w)
            self.log_debug(
                f"Skipping {skipped_words} words for containing the following graphemes: "
                f"{comma_join(sorted(missing_graphemes))}"
            )
            error_dict = {}
            return_queue = mp.Queue()
            procs = []
            for _ in range(self.num_jobs):
                p = RewriterWorker(
                    job_queue,
                    return_queue,
                    rewriter,
                    stopped,
                )
                procs.append(p)
                p.start()
            num_words -= skipped_words
            with tqdm.tqdm(total=num_words, disable=getattr(self, "quiet", False)) as pbar:
                while True:
                    try:
                        word, result = return_queue.get(timeout=1)
                        if stopped.stop_check():
                            continue
                    except queue.Empty:
                        for proc in procs:
                            if not proc.finished.stop_check():
                                break
                        else:
                            break
                        continue
                    pbar.update(1)
                    if isinstance(result, Exception):
                        error_dict[word] = result
                        continue
                    to_return[word] = result

            for p in procs:
                p.join()
            if error_dict:
                raise PyniniGenerationError(error_dict)
        self.log_debug(f"Processed {num_words} in {time.time() - begin} seconds")
        return to_return

class PyniniValidator(PyniniGenerator, TopLevelMfaWorker):
    """
    Class for running validation for G2P model training

    Parameters
    ----------
    word_list: list[str]
        List of words to generate pronunciations

    See Also
    --------
    :class:`~montreal_forced_aligner.g2p.generator.PyniniGenerator`
        For parameters to generate pronunciations
    """

    def __init__(self, word_list: List[str] = None, **kwargs):
        super().__init__(**kwargs)
        if word_list is None:
            word_list = []
        self.word_list = word_list

    @property
    def words_to_g2p(self) -> List[str]:
        """Words to produce pronunciations"""
        return self.word_list

    @property
    def data_source_identifier(self) -> str:
        """Dummy "validation" data source"""
        return "validation"

    @property
    def data_directory(self) -> str:
        """Data directory"""
        return self.working_directory

    @property
    def evaluation_csv_path(self) -> str:
        """Path to working directory's CSV file"""
        return os.path.join(self.working_directory, "pronunciation_evaluation.csv")

    def setup(self) -> None:
        """Set up the G2P validator"""
        if self.initialized:
            return
        self.g2p_model.validate(self.words_to_g2p)
        self.initialized = True
        self.wer = None
        self.ler = None

    def compute_validation_errors(
        self,
        gold_values: Dict[str, Set[str]],
        hypothesis_values: Dict[str, List[str]],
    ):
        """
        Computes validation errors

        Parameters
        ----------
        gold_values: dict[str, set[str]]
            Gold pronunciations
        hypothesis_values: dict[str, list[str]]
            Hypothesis pronunciations
        """
        begin = time.time()
        # Word-level measures.
        correct = 0
        incorrect = 0
        # Label-level measures.
        total_edits = 0
        total_length = 0
        # Since the edit distance algorithm is quadratic, let's do this with
        # multiprocessing.
        self.log_debug(f"Processing results for {len(hypothesis_values)} hypotheses")
        to_comp = []
        indices = []
        hyp_pron_count = 0
        gold_pron_count = 0
        output = []
        for word, gold_pronunciations in gold_values.items():
            if word not in hypothesis_values:
                incorrect += 1
                gold_length = statistics.mean(len(x.split()) for x in gold_pronunciations)
                total_edits += gold_length
                total_length += gold_length
                output.append(
                    {
                        "Word": word,
                        "Gold pronunciations": ", ".join(gold_pronunciations),
                        "Hypothesis pronunciations": "",
                        "Accuracy": 0,
                        "Error rate": 1.0,
                        "Length": gold_length,
                    }
                )
                continue
            hyp = hypothesis_values[word]
            for h in hyp:
                if h in gold_pronunciations:
                    correct += 1
                    total_length += len(h)
                    output.append(
                        {
                            "Word": word,
                            "Gold pronunciations": ", ".join(gold_pronunciations),
                            "Hypothesis pronunciations": ", ".join(hyp),
                            "Accuracy": 1,
                            "Error rate": 0.0,
                            "Length": len(h),
                        }
                    )
                    break
            else:
                incorrect += 1
                indices.append(word)
                to_comp.append((gold_pronunciations, hyp))  # Multiple hypotheses to compare
            self.log_debug(
                f"For the word {word}: gold is {gold_pronunciations}, hypothesized are: {hyp}"
            )
            hyp_pron_count += len(hyp)
            gold_pron_count += len(gold_pronunciations)
        self.log_debug(
            f"Generated an average of {hyp_pron_count /len(hypothesis_values)} variants "
            f"The gold set had an average of {gold_pron_count/len(hypothesis_values)} variants."
        )
        with mp.Pool(self.num_jobs) as pool:
            gen = pool.starmap(score_g2p, to_comp)
            for i, (edits, length) in enumerate(gen):
                word = indices[i]
                gold_pronunciations = gold_values[word]
                hyp = hypothesis_values[word]
                output.append(
                    {
                        "Word": word,
                        "Gold pronunciations": ", ".join(gold_pronunciations),
                        "Hypothesis pronunciations": ", ".join(hyp),
                        "Accuracy": 1,
                        "Error rate": edits / length,
                        "Length": length,
                    }
                )
                total_edits += edits
                total_length += length
        with open(self.evaluation_csv_path, "w", encoding="utf8", newline="") as f:
            writer = csv.DictWriter(
                f,
                fieldnames=[
                    "Word",
                    "Gold pronunciations",
                    "Hypothesis pronunciations",
                    "Accuracy",
                    "Error rate",
                    "Length",
                ],
            )
            writer.writeheader()
            for line in output:
                writer.writerow(line)
        self.wer = 100 * incorrect / (correct + incorrect)
        self.ler = 100 * total_edits / total_length
        self.log_info(f"WER:\t{self.wer:.2f}")
        self.log_info(f"LER:\t{self.ler:.2f}")
        self.log_debug(
            f"Computation of errors for {len(gold_values)} words took {time.time() - begin} seconds"
        )

    def evaluate_g2p_model(self, gold_pronunciations: Dict[str, Set[str]]) -> None:
        """
        Evaluate a G2P model on the word list

        Parameters
        ----------
        gold_pronunciations: dict[str, set[str]]
            Gold pronunciations
        """
        output = self.generate_pronunciations()
        self.compute_validation_errors(gold_pronunciations, output)

class PyniniWordListGenerator(PyniniValidator):
    """
    Top-level worker for generating pronunciations from a word list and a Pynini G2P model

    Parameters
    ----------
    word_list_path: str
        Path to word list file

    See Also
    --------
    :class:`~montreal_forced_aligner.g2p.generator.PyniniGenerator`
        For Pynini G2P generation parameters
    :class:`~montreal_forced_aligner.abc.TopLevelMfaWorker`
        For top-level parameters

    Attributes
    ----------
    word_list: list[str]
        Word list to generate pronunciations
    """

    def __init__(self, word_list_path: str, **kwargs):
        self.word_list_path = word_list_path
        super().__init__(**kwargs)

    @property
    def data_directory(self) -> str:
        """Data directory"""
        return self.working_directory

    @property
    def data_source_identifier(self) -> str:
        """Name of the word list file"""
        return os.path.splitext(os.path.basename(self.word_list_path))[0]

    def setup(self) -> None:
        """Set up the G2P generator"""
        if self.initialized:
            return
        with open(self.word_list_path, "r", encoding="utf8") as f:
            for line in f:
                self.word_list.extend(line.strip().split())
        if not self.include_bracketed:
            self.word_list = [x for x in self.word_list if not self.check_bracketed(x)]
        self.g2p_model.validate(self.words_to_g2p)
        self.initialized = True

class PyniniCorpusGenerator(PyniniGenerator, TextCorpusMixin, TopLevelMfaWorker):
    """
    Top-level worker for generating pronunciations from a corpus and a Pynini G2P model

    See Also
    --------
    :class:`~montreal_forced_aligner.g2p.generator.PyniniGenerator`
        For Pynini G2P generation parameters
    :class:`~montreal_forced_aligner.corpus.text_corpus.TextCorpusMixin`
        For corpus parsing parameters
    :class:`~montreal_forced_aligner.abc.TopLevelMfaWorker`
        For top-level parameters
    """

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    def setup(self) -> None:
        """Set up the pronunciation generator"""
        if self.initialized:
            return
        self._load_corpus()
        self.calculate_word_counts()
        self.g2p_model.validate(self.words_to_g2p)
        self.initialized = True

    @property
    def words_to_g2p(self) -> List[str]:
        """Words to produce pronunciations"""
        word_list = self.corpus_word_set
        if not self.include_bracketed:
            word_list = [x for x in word_list if not self.check_bracketed(x)]
        return word_list

class OrthographicCorpusGenerator(OrthographyGenerator, TextCorpusMixin, TopLevelMfaWorker):
    """
    Top-level class for generating "pronunciations" from the orthography of a corpus

    See Also
    --------
    :class:`~montreal_forced_aligner.g2p.generator.OrthographyGenerator`
        For orthography-based G2P generation parameters
    :class:`~montreal_forced_aligner.corpus.text_corpus.TextCorpusMixin`
        For corpus parsing parameters
    :class:`~montreal_forced_aligner.abc.TopLevelMfaWorker`
        For top-level parameters
    """

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    def setup(self) -> None:
        """Set up the pronunciation generator"""
        if self.initialized:
            return
        self._load_corpus()
        self.calculate_word_counts()
        self.initialized = True

    @property
    def words_to_g2p(self) -> List[str]:
        """Words to produce pronunciations"""
        word_list = self.corpus_word_set
        if not self.include_bracketed:
            word_list = [x for x in word_list if not self.check_bracketed(x)]
        return word_list

class OrthographicWordListGenerator(OrthographyGenerator, TopLevelMfaWorker):
    """
    Top-level class for generating "pronunciations" from the orthography of a corpus

    Parameters
    ----------
    word_list_path: str
        Path to word list file
    See Also
    --------
    :class:`~montreal_forced_aligner.g2p.generator.OrthographyGenerator`
        For orthography-based G2P generation parameters
    :class:`~montreal_forced_aligner.abc.TopLevelMfaWorker`
        For top-level parameters

    Attributes
    ----------
    word_list: list[str]
        Word list to generate pronunciations
    """

    def __init__(self, word_list_path: str, **kwargs):
        super().__init__(**kwargs)
        self.word_list_path = word_list_path
        self.word_list = []

    def setup(self) -> None:
        """Set up the pronunciation generator"""
        if self.initialized:
            return
        with open(self.word_list_path, "r", encoding="utf8") as f:
            for line in f:
                self.word_list.extend(line.strip().split())
        if not self.include_bracketed:
            self.word_list = [x for x in self.word_list if not self.check_bracketed(x)]
        self.initialized = True

    @property
    def words_to_g2p(self) -> List[str]:
        """Words to produce pronunciations"""
        return self.word_list


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import re

from Commands.Keys import Button, Direction, Hat
from Commands.PythonCommandBase import PythonCommand
# import numpy as np
from scipy.sparse.csgraph import shortest_path  # , floyd_warshall, dijkstra, bellman_ford, johnson
from scipy.sparse import csr_matrix

serial = {0: '-1',
          1: '1', 2: '2', 3: '3', 4: '4', 5: '5', 6: '6', 7: '7', 8: '8', 9: '9', 10: '0', 11: '@', 12: 'BS',
          13: 'Q', 14: 'W', 15: 'E', 16: 'R', 17: 'T', 18: 'Y', 19: 'U', 20: 'I', 21: 'O', 22: 'P', 23: '=',
          24: 'A', 25: 'S', 26: 'D', 27: 'F', 28: 'G', 29: 'H', 30: 'J', 31: 'K', 32: 'L', 33: '&', 34: ';',
          35: 'Z', 36: 'X', 37: 'C', 38: 'V', 39: 'B', 40: 'N', 41: 'M', 42: '*', 43: '#', 44: '!', 45: '?',
          46: 'SelectKeyboard', 47: 'Shift', 48: '#+=', 49: 'nl_1', 50: 'nl_2', 51: 'nl_3',
          52: 'ok_1', 53: 'ok_2', 54: 'ok_3', 55: 'blank_1', 56: 'blank_2', 57: 'blank_3', 58: 'blank_4',
          59: 'blank_5', 60: 'blank_6', 61: 'blank_7', 62: 'blank_8', }
serial_inv = {v: k for k, v in serial.items()}
serial_graph_list = [[],
                     # 1-5
                     [2, 13, 12], [1, 3, 14], [2, 4, 15], [3, 5, 16], [4, 6, 17],
                     # 6-10
                     [5, 7, 18], [6, 8, 19], [7, 9, 20], [8, 10, 21], [9, 11, 22],
                     # 11-15 @ ~ E
                     [10, 12, 23], [11, 49, 1], [1, 24, 14, 49], [2, 13, 15, 25], [3, 14, 16, 26],
                     # 16-20 R ~ I
                     [4, 15, 17, 27], [5, 16, 18, 28], [6, 17, 19, 29], [7, 18, 20, 30], [8, 19, 21, 31],
                     # 21-25 O ~ S
                     [9, 20, 22, 32], [10, 21, 23, 33], [11, 22, 34, 49], [13, 25, 35, 50], [14, 24, 26, 36],
                     # 26-30 D ~ J
                     [15, 25, 27, 37], [16, 26, 28, 38], [17, 27, 29, 39], [18, 28, 30, 40], [19, 29, 31, 41],
                     # 31-35 J ~ Z
                     [20, 30, 32, 42], [21, 31, 33, 43], [22, 32, 34, 44], [23, 33, 45, 50], [24, 46, 36, 53],
                     # 36-40 X ~ N
                     [25, 35, 37, 47], [26, 36, 38, 48], [27, 37, 39, 55], [28, 38, 40, 56], [29, 39, 41, 57],
                     # 41-45 M ~ ?
                     [30, 40, 42, 58], [31, 41, 43, 59], [32, 42, 44, 60], [33, 43, 45, 61], [34, 44, 62, 53],
                     # 46-50
                     [35, 47, 54], [36, 46, 48], [37, 47, 55], [12, 23, 13, 52], [12, 34, 24, 53],
                     # 51-56
                     [12, 34, 24, 54], [49, 45, 35], [45, 35, 50], [55, 46, 51], [38, 48, 54], [39, 48, 54],
                     # 57-62
                     [40, 48, 54], [41, 48, 54], [42, 48, 54], [43, 48, 54], [44, 48, 54], [45, 48, 54]]

class InputKeyboard(PythonCommand):
    NAME = 'シリアル入力'

    def __init__(self):
        super().__init__()
        self.s = 'F105LP98GMFCB3RA'  # 入力したい文字列
        self.now_dict = serial_graph_list
        self.now_dict_ = serial
        self.now_dict_inv = serial_inv
        self.graph = None
        self.d = None
        self.p = None
        self.n = None
        self.MakeGraph()
        self.pos = 1  # 初期位置

    def MakeGraph(self):
        self.n = len(self.now_dict)
        self.graph = [[0] * self.n for _ in range(self.n)]  # 隣接行列
        for i, g_i in enumerate(self.now_dict):
            for j in g_i:
                self.graph[i][j] = 1
        # for i in self.graph:
        # print(" ".join(list(map(str, i))))

        a = csr_matrix(self.graph)
        self.d, self.p = shortest_path(a, return_predecessors=True)

    def do(self):
        input_char = 0
        for i in self.s:
            print(self.now_dict_[self.now_dict_inv[i]])
            t = GetPath(self.pos, self.now_dict_inv[i], self.p)
            print(t)
            stick = False
            stick = self.Move(t, stick)
            if not stick:
                self.press(Button.A, wait=0.03, duration=0.05)
            input_char += 1

    def Move(self, t, stick):  # 移動のための関数
        for j in range(len(t) - 1):
            if t[j + 1] == 1 and t[j] == 12:
                self.press(Direction.RIGHT, wait=0.03, duration=0.05)
            if t[j + 1] == 12:
                if t[j] in [49, 50, 51]:
                    self.press(Direction.UP, wait=0.03, duration=0.05)
                elif t[j] == 1:
                    self.press(Direction.LEFT, wait=0.03, duration=0.05)
                elif t[j] == 11:
                    self.press(Direction.RIGHT, wait=0.03, duration=0.05)
            elif t[j + 1] == 13:  # Q
                if t[j] == 49:
                    self.press(Direction.RIGHT, wait=0.1, duration=0.05)
                elif t[j] == 1:
                    self.press(Direction.DOWN, wait=0.03, duration=0.05)
                elif t[j] == 14:
                    self.press(Direction.LEFT, wait=0.03, duration=0.05)
                elif t[j] == 24:
                    self.press(Direction.UP, wait=0.03, duration=0.05)
            elif t[j + 1] == 23:  # =
                if t[j] == 22:
                    self.press(Direction.RIGHT, wait=0.03, duration=0.05)
                elif t[j] == 11:
                    self.press(Direction.DOWN, wait=0.03, duration=0.05)
                elif t[j] == 49:
                    self.press(Direction.LEFT, wait=0.03, duration=0.05)
                elif t[j] == 34:
                    self.press(Direction.UP, wait=0.03, duration=0.05)
            elif t[j + 1] == 24:  # A
                if t[j] in [50, 51]:
                    self.press(Direction.RIGHT, wait=0.1, duration=0.05)
                elif t[j] == 13:
                    self.press(Direction.DOWN, wait=0.03, duration=0.05)
                elif t[j] == 25:
                    self.press(Direction.LEFT, wait=0.03, duration=0.05)
                elif t[j] == 35:
                    self.press(Direction.UP, wait=0.03, duration=0.05)
            elif t[j + 1] == 34:  # ;
                if t[j] == 33:
                    self.press(Direction.RIGHT, wait=0.03, duration=0.05)
                elif t[j] == 23:
                    self.press(Direction.DOWN, wait=0.03, duration=0.05)
                elif t[j] in [50, 51]:
                    self.press(Direction.LEFT, wait=0.03, duration=0.05)
                elif t[j] == 45:
                    self.press(Direction.UP, wait=0.03, duration=0.05)
            elif t[j + 1] == 35:  # Z
                if t[j] in [52, 53]:
                    self.press(Direction.RIGHT, wait=0.1, duration=0.05)
                elif t[j] == 24:
                    self.press(Direction.DOWN, wait=0.03, duration=0.05)
                elif t[j] == 36:
                    self.press(Direction.LEFT, wait=0.03, duration=0.05)
                elif t[j] == 46:
                    self.press(Direction.UP, wait=0.03, duration=0.05)
            elif t[j + 1] in [38, 39, 40, 41, 42, 43, 44] and t[j + 1] - t[j] == -17:  # Z
                self.press(Direction.UP, wait=0.03, duration=0.05)
            elif t[j + 1] == 45:  # ?
                if t[j] == 44:
                    self.press(Direction.RIGHT, wait=0.03, duration=0.05)
                elif t[j] == 34:
                    self.press(Direction.DOWN, wait=0.03, duration=0.05)
                elif t[j] in [52, 53]:
                    self.press(Direction.LEFT, wait=0.03, duration=0.05)
                elif t[j] == 62:
                    self.press(Direction.UP, wait=0.03, duration=0.05)
            elif t[j + 1] == 48 and t[j] in [55, 56, 57, 58, 59, 60, 61, 62]:
                self.press(Direction.LEFT, wait=0.03, duration=0.05)

            elif t[j + 1] == 49:
                if t[j] == 12:
                    self.press(Direction.DOWN, wait=0.03, duration=0.05)
                elif t[j] == 23:
                    self.press(Direction.RIGHT, wait=0.03, duration=0.05)
                elif t[j] == 13:
                    self.press(Direction.LEFT, wait=0.03, duration=0.05)
                elif t[j] == 52:
                    self.press(Direction.UP, wait=0.03, duration=0.05)
            elif t[j + 1] == 50:
                if t[j] == 34:
                    self.press(Direction.RIGHT, wait=0.03, duration=0.05)
                elif t[j] == 24:
                    self.press(Direction.LEFT, wait=0.03, duration=0.05)
                elif t[j] == 53:
                    self.press(Direction.UP, wait=0.03, duration=0.05)
            elif t[j + 1] == 51:
                if t[j] == 54:
                    self.press(Direction.UP, wait=0.03, duration=0.05)
            elif t[j + 1] == 52:
                if t[j] == 49:
                    self.press(Direction.DOWN, wait=0.03, duration=0.05)
            elif t[j + 1] == 53:
                if t[j] == 45:
                    self.press(Direction.RIGHT, wait=0.03, duration=0.05)
                elif t[j] == 35:
                    self.press(Direction.LEFT, wait=0.03, duration=0.05)
                elif t[j] == 50:
                    self.press(Direction.DOWN, wait=0.03, duration=0.05)
            elif t[j + 1] == 54:
                if t[j] in [55, 56, 57, 58, 59, 60, 61, 62]:
                    self.press(Direction.RIGHT, wait=0.03, duration=0.05)
                elif t[j] == 46:
                    self.press(Direction.LEFT, wait=0.03, duration=0.05)
                elif t[j] == 51:
                    self.press(Direction.DOWN, wait=0.03, duration=0.05)
            elif t[j + 1] == 55:
                if t[j] == 48:
                    self.press(Direction.RIGHT, wait=0.03, duration=0.05)
                elif t[j] == 54:
                    self.press(Direction.LEFT, wait=0.03, duration=0.05)
                elif t[j] == 38:
                    self.press(Direction.DOWN, wait=0.03, duration=0.05)
            elif t[j + 1] in [56, 57, 58, 59, 60, 61, 62] and t[j + 1] - t[j] == 17:
                self.press(Direction.DOWN, wait=0.03, duration=0.05)
            elif t[j + 1] - t[j] == 1:
                self.press(Direction.RIGHT, wait=0.03, duration=0.05)
            elif t[j + 1] - t[j] == -1:
                self.press(Direction.LEFT, wait=0.03, duration=0.05)
            elif t[j + 1] - t[j] in [11, 12]:
                self.press(Direction.DOWN, wait=0.03, duration=0.05)
            elif t[j + 1] - t[j] in [-11, -12]:
                self.press(Direction.UP, wait=0.03, duration=0.05)
            if t[j + 1] not in list(range(67, self.n)):
                self.pos = self.now_dict_inv[self.now_dict_[t[j + 1]]]
        return stick

def GetPath(start, goal, pred):
    return GetPathRow(start, goal, pred[start])

def GetPathRow(start, goal, pred_row):
    path = []
    i = goal
    while i != start and i >= 0:
        path.append(i)
        i = pred_row[i]
    if i < 0:
        return []
    path.append(i)
    return path[::-1]


<filename>filter_and_merge_reads.py
import pysam
import sys
import os
import re
import argparse
import statistics

parser=argparse.ArgumentParser()
parser.add_argument("-f","--file_in",help="Input bam file name", required=True)
parser.add_argument("-o","--file_out",help="Output sam file name", required=True)
parser.add_argument("-c","--overcovered_regions",help="File with overcovered regions")

args=parser.parse_args()

raw_bam_file_name=args.file_in
collapsed_sam_file_name=args.file_out

overcovered_starts_dict={}
overcovered_ends_dict={}
if args.overcovered_regions:
    overcovered_file=open(args.overcovered_regions,'rt')
    for line in overcovered_file:
        overcovered_chr,overcovered_start,overcovered_end=line.split()
        if overcovered_chr in overcovered_starts_dict:
            overcovered_starts_dict[overcovered_chr].append(int(overcovered_start))
            overcovered_ends_dict[overcovered_chr].append(int(overcovered_end))
        else:
            overcovered_starts_dict[overcovered_chr]=[int(overcovered_start),]
            overcovered_ends_dict[overcovered_chr]=[int(overcovered_end),]

samfile=pysam.AlignmentFile(raw_bam_file_name, "rb")
collapsed_samfile=pysam.AlignmentFile(collapsed_sam_file_name, "wb", template=samfile)
current_read_name=""
reads_with_same_name=list()

tlen_vect = []
def process_reads_with_same_name(reads_with_same_name, overcovered_starts_dict, overcovered_ends_dict):
    read1=""
    read2=""
    global tlen_vect

    for same_read in reads_with_same_name:
        #check if this is a primary read
        if not same_read.is_secondary:
            if same_read.is_read1:
                if not read1:
                    read1=same_read
                else:
                    #this read is probably split aligned
                    return("Split-aligned R1 or R2")
            elif same_read.is_read2:
                if not read2:
                    read2=same_read
                else:
                    return("Split-aligned R1 or R2")
            else:
                print("Strange: this is not read1 or read2\n")
                print(same_read.tostring())

    if not(read1 and read2):
        return("No R1 or R2")
    elif (read1.mapping_quality<60 or read2.mapping_quality<60):
        if (abs(read1.template_length)==abs(read2.template_length)):
            tlen_vect.append(abs(read1.template_length))
        return("R1 or R2 mapping quality lower than 60")
    elif (re.search('S',read1.cigarstring) or re.search('S',read2.cigarstring)):
        return("Soft-clipped bases in R1 or R2")
    elif (read1.is_reverse and read2.is_reverse):
        return("Both reads have reverse orientation")
    elif ((not read1.is_reverse) and (not read2.is_reverse)):
        return("Both reads have forward orientation")
    elif (read1.template_length>1000 or read1.template_length<-1000):
        return("Template length longer than 1000 bp")
    elif (read1.template_length==0 or read2.template_length==0):
        return("Template length 0")
    elif (not read1.is_proper_pair or not read2.is_proper_pair):
        return("R1 and R2 are not a proper pair")
    else:
        if str(read1.reference_name)==str(read2.reference_name):
            chrom=str(read1.reference_name)
            if (abs(read1.template_length)==abs(read2.template_length)):
                tlen_vect.append(abs(read1.template_length))
        else:
            return("R1 and R2 are not a proper pair")
        if read1.reference_start and read1.reference_end and read2.reference_start and read2.reference_end:
            positions_list=[read1.reference_start,read1.reference_end,read2.reference_start,read2.reference_end]
        else:
            return("R1 and R2 are not a proper pair")
        pos_start=min(positions_list)
        pos_end=max(positions_list)
        in_overcovered=0
        if chrom in overcovered_starts_dict:
            for i in range(0,len(overcovered_starts_dict[chrom])):
                if pos_start>overcovered_ends_dict[chrom][i]:
                    i+=1
                    continue
                elif pos_end<overcovered_starts_dict[chrom][i]:
                    break
                elif pos_start<overcovered_starts_dict[chrom][i] and \
                    pos_end>=overcovered_starts_dict[chrom][i]:
                    in_overcovered=1
                    break
                elif pos_start>=overcovered_starts_dict[chrom][i] and \
                    pos_start<=overcovered_ends_dict[chrom][i]:
                    in_overcovered=1
                    break
        if in_overcovered:
            return("Within overcovered region")
        elif not read1.is_reverse:
            read1.tags += [('rr', '"'+read2.tostring()+'"')]
            read1.tags += [('re', read2.reference_end)]
            collapsed_samfile.write(read1)
            return("Reads passed")
        else:
            read2.tags += [('rr', '"'+read1.tostring()+'"')]
            read2.tags += [('re', read1.reference_end)]
            collapsed_samfile.write(read2)
            return("Reads passed")

number_of_reads=0

read_dict={"Split-aligned R1 or R2":0,
           "No R1 or R2":0,
           "R1 or R2 mapping quality lower than 60":0,
           "Soft-clipped bases in R1 or R2":0,
           "Both reads have reverse orientation":0,
           "Both reads have forward orientation":0,
           "Template length longer than 1000 bp":0,
           "Template length 0":0,
           "R1 and R2 are not a proper pair":0,
           "Within overcovered region":0,
           "Reads passed":0}

for read in samfile.fetch(until_eof=True):
    if not current_read_name:
        current_read_name=read.query_name
        continue
    else:
        if read.query_name==current_read_name:
            reads_with_same_name.append(read)
            continue
        else:
            number_of_reads+=1
            res=process_reads_with_same_name(reads_with_same_name, overcovered_starts_dict, overcovered_ends_dict)
            read_dict[res]+=1
            current_read_name=read.query_name
            reads_with_same_name=[read,]
else:
    number_of_reads+=1
    res=process_reads_with_same_name(reads_with_same_name, overcovered_starts_dict, overcovered_ends_dict)
    read_dict[res]+=1

print("Overall number of reads or clusters in bam file")
print(number_of_reads)

for each_reason in read_dict.keys():
    print(each_reason,": ",read_dict[each_reason])

print("Median template length: ",statistics.median(tlen_vect))
samfile.close()
collapsed_samfile.close()


from typing import Optional

import numpy
from scipy.sparse import csr_matrix
from scipy.special import expit
from tqdm.auto import trange, tqdm

from .matrix_factorization_base import MatrixFactorizationBase

class BPR(MatrixFactorizationBase):
    def __init__(
        self,
        factors: int,
        lr: float,
        steps: int,
        reg_lambda: float = 0.0,
        random_state: Optional[int] = None,
    ):
        super().__init__(factors, random_state)
        self.__lr = lr
        self.__steps = steps
        self.__lambda = reg_lambda

    @staticmethod
    def _sample_negative(user_id: int, user_item_csr: csr_matrix) -> int:
        neg_sample = numpy.random.choice(user_item_csr.shape[1])
        while user_item_csr[user_id, neg_sample] != 0:
            neg_sample = numpy.random.choice(user_item_csr.shape[1])
        return neg_sample

    def _gradient_step(self, loss: float, user_id: int, pos_sample: int, neg_sample: int):
        # [ d ]
        du = loss * (self._V[pos_sample] - self._V[neg_sample]) + self.__lambda * self._U[user_id]
        dpi = loss * self._U[user_id] + self.__lambda * self._V[pos_sample]
        dni = loss * -self._U[user_id] + self.__lambda * self._V[neg_sample]

        self._U[user_id] -= self.__lr * du
        self._V[pos_sample] -= self.__lr * dpi
        self._V[neg_sample] -= self.__lr * dni

    def _step(self, user_id: int, pos_sample: int, user_item_csr: csr_matrix) -> float:
        neg_sample = self._sample_negative(user_id, user_item_csr)

        # [ 1 ]
        r_uij = numpy.dot(self._U[user_id], self._V[pos_sample] - self._V[neg_sample])
        sigmoid = expit(r_uij)

        self._gradient_step(sigmoid, user_id, pos_sample, neg_sample)

        return numpy.log(sigmoid)

    def fit(self, user_item_csr: csr_matrix):
        n_users, n_items = user_item_csr.shape
        self._init_matrices(n_users, n_items)

        user_item_coo = user_item_csr.tocoo()
        n_samples = user_item_csr.count_nonzero()
        assert len(user_item_coo.row) == len(user_item_coo.col) == n_samples

        epoch_range = trange(self.__steps, desc="Epoch")
        for _ in epoch_range:
            order = numpy.random.permutation(n_samples)
            log_loss = 0
            for user_id, pos_sample in tqdm(zip(user_item_coo.row[order], user_item_coo.col[order]), total=n_samples):
                log_loss += self._step(user_id, pos_sample, user_item_csr)

            epoch_range.set_postfix({"log loss": round(log_loss / n_samples, 3)})
        epoch_range.close()


#!/usr/bin/env python3
# ------------------------------------------------------------------------
# Copyright (c) 2021 megvii-model. All Rights Reserved.
# ------------------------------------------------------------------------
import os, sys
from scipy.optimize import linear_sum_assignment
import numpy as np

def mask_minimumWeightMatching(costSet, mask) -> list:

    m, n = costSet.shape
    nMax = max(m, n)

    costSet_ = np.full((nMax, nMax), np.inf)

    mask = mask > 0
    costSet_[:m, :n][mask] = costSet[mask]
    assert costSet_.shape[0] == costSet_.shape[1]

    if mask.sum():
        practicalInfinity = 2 * costSet[costSet < np.inf].max() + 10
    # except ValueError:
    else:
        practicalInfinity = 1

    # Replace infinitites with our representation.
    costSet_[costSet_ == np.inf] = practicalInfinity

    # Find a pairing of minimum total cost between matching second-level contours.
    iSet, jSet = linear_sum_assignment(costSet_)
    assert len(iSet) == len(jSet)

    # Return only pairs with finite cost.
    indices = [(iSet[k], jSet[k]) for k in range(len(iSet))
                if costSet_[iSet[k], jSet[k]] != practicalInfinity]

    rows = np.array([i for i, _ in indices])
    cols = np.array([j for _, j in indices])
    return rows, cols

def minimumWeightMatching(costSet : np.ndarray) -> list:
    '''
    Computes a minimum-weight matching in a bipartite graph
    (A union B, E).

    costSet:
    An (m x n)-matrix of real values, where costSet[i, j]
    is the cost of matching the i:th vertex in A to the j:th
    vertex of B. A value of numpy.inf is allowed, and is
    interpreted as missing the (i, j)-edge.

    returns:
    A minimum-weight matching given as a list of pairs (i, j),
    denoting that the i:th vertex of A be paired with the j:th
    vertex of B.
    '''

    m, n = costSet.shape
    nMax = max(m, n)

    # Since the choice of infinity blocks later choices for that index,
    # it is important that the cost matrix is square, so there
    # is enough space to shift the choices for infinity to the unused
    # part of the cost-matrix.
    costSet_ = np.full((nMax, nMax), np.inf)

    mask = costSet < 0
    costSet_[:m, :n][mask] = costSet[mask]
    assert costSet_.shape[0] == costSet_.shape[1]

    # We allow a cost to be infinity. Since scipy does not
    # support this, we use a workaround. We represent infinity
    # by M = 2 * maximum cost + 1. The point is to choose a distinct
    # value, greater than any other cost, so that choosing an
    # infinity-pair is the last resort. The 2 times is for large
    # values for which x + 1 == x in floating point. The plus 1
    # is for zero, for which 2 x == x.
    try:
        practicalInfinity = 2 * costSet[costSet < np.inf].max() + 10
    except ValueError:
        # This is thrown when the indexing set is empty;
        # then all elements are infinities.
        practicalInfinity = 1

    # Replace infinitites with our representation.
    costSet_[costSet_ == np.inf] = practicalInfinity

    # Find a pairing of minimum total cost between matching second-level contours.
    iSet, jSet = linear_sum_assignment(costSet_)
    assert len(iSet) == len(jSet)

    # Return only pairs with finite cost.
    indices = [(iSet[k], jSet[k])
        for k in range(len(iSet))
        if costSet_[iSet[k], jSet[k]] != practicalInfinity]

    return indices

def compute_lap(dtboxes, gtboxes, thr):

    eps = 1e-7
    n, k = dtboxes.shape[0], gtboxes.shape[0]
    if k + n < 2:
        m, n = np.array([]), np.array([])
        return m, n

    overlaps = compute_iou_matrix(dtboxes, gtboxes)

    if n < 2:
        cols = np.argmax(overlaps, axis = 1)
        rows = np.array([0])
        m, n = (rows, cols) if thr - overlaps[rows, cols] < eps else (np.array([]), np.array([]))
        return m, n

    if k < 2:

        rows = np.argmax(overlaps, axis = 0)
        cols = np.array([0])
        m,n = (rows, cols) if thr - overlaps[rows, cols] < eps else (np.array([]), np.array([]))
        return m, n

    ious = overlaps * (overlaps >= thr)

    matches = minimumWeightMatching(-ious)
    m, n = np.array([i for i, _ in matches]).astype(np.int32), np.array([i for _, i in matches]).astype(np.int32)
    indice = np.where(overlaps[m, n] < thr)[0]

    if indice.size >= m.size:
        m, n = np.array([]), np.array([])
    else:
        index = np.array(list(set(np.arange(m.size)) - set(indice))).astype(np.int)
        m, n = m[index], n[index]

    return m, n

def compute_Jaccard(dtboxes, gtboxes, bm_thr):

    rows, cols = compute_lap(dtboxes, gtboxes, bm_thr)
    return [(i, j) for i, j in (rows, cols)]

def compute_JC(dtboxes, gtboxes, bm_thr):

    rows, cols = compute_lap(dtboxes, gtboxes, bm_thr)
    return [(i, j) for i, j in zip(rows, cols)]

def compute_ioa_matrix(dboxes: np.ndarray, gboxes: np.ndarray):

    assert dboxes.shape[-1] >= 4 and gboxes.shape[-1] >= 4
    N, K = dboxes.shape[0], gboxes.shape[0]
    eps = 1e-6
    dtboxes = np.tile(np.expand_dims(dboxes, axis = 1), (1, K, 1))
    gtboxes = np.tile(np.expand_dims(gboxes, axis = 0), (N, 1, 1))

    iw = np.minimum(dtboxes[:,:,2], gtboxes[:,:,2]) - np.maximum(dtboxes[:,:,0], gtboxes[:,:,0])
    ih = np.minimum(dtboxes[:,:,3], gtboxes[:,:,3]) - np.maximum(dtboxes[:,:,1], gtboxes[:,:,1])
    inter = np.maximum(0, iw) * np.maximum(0, ih)

    dtarea = np.maximum(dtboxes[:,:,2] - dtboxes[:,:,0], 0) * np.maximum(dtboxes[:,:,3] - dtboxes[:,:,1], 0)
    ioas = inter / (dtarea + eps)
    return ioas

def compute_iou_matrix(dboxes:np.ndarray, gboxes:np.ndarray):

    assert dboxes.shape[-1] >= 4 and gboxes.shape[-1] >= 4
    eps = 1e-6
    N, K = dboxes.shape[0], gboxes.shape[0]
    dtboxes = np.tile(np.expand_dims(dboxes, axis = 1), (1, K, 1))
    gtboxes = np.tile(np.expand_dims(gboxes, axis = 0), (N, 1, 1))

    iw = np.minimum(dtboxes[:,:,2], gtboxes[:,:,2]) - np.maximum(dtboxes[:,:,0], gtboxes[:,:,0])
    ih = np.minimum(dtboxes[:,:,3], gtboxes[:,:,3]) - np.maximum(dtboxes[:,:,1], gtboxes[:,:,1])
    inter = np.maximum(0, iw) * np.maximum(0, ih)

    dtarea = (dtboxes[:,:,2] - dtboxes[:,:,0]) * (dtboxes[:,:,3] - dtboxes[:,:,1])
    gtarea = (gtboxes[:,:,2] - gtboxes[:,:,0]) * (gtboxes[:,:,3] - gtboxes[:,:,1])
    ious = inter / (dtarea + gtarea - inter + eps)
    return ious

def compute_maximal_iou(proposals:np.ndarray,gt:np.ndarray):

    ious = compute_iou_matrix(proposals, gt)
    return np.max(ious, axis = 1)


import matplotlib.pyplot as plt
import numpy as np
from scipy.stats import linregress

# Set style
plt.rcParams["font.family"] = "sans-serif"
plt.rcParams["figure.figsize"] = (10, 4)

# Data
x = np.array([0.25, 0.286, 0.333, 0.4, 0.5])
y = np.array([0.15, 0.3, 0.39, 0.54, 0.69])

x_error = np.array([0.006, 0.008, 0.011, 0.016, 0.025])
y_error = np.array([0.03, 0.01, 0.04, 0.02, 0.02])

# Linear regression
result = linregress(x, y)
x_conti = np.linspace(0.23, 0.57)
y_conti = result.intercept + result.slope * x_conti

# Plot
figure, axs = plt.subplots(1, 2)
axs[0].errorbar(x, y, yerr=y_error, xerr=x_error, fmt="o")
axs[0].plot(x_conti, y_conti)

axs[0].set_xlabel("1/a (cm^-1)")
axs[0].set_ylabel("sin(theta)")
axs[0].set_ylim(0, None)

axs[1].scatter(x, y - result.intercept - x * result.slope)
axs[1].set_xlabel("1/a (cm^-1)")
axs[1].set_ylabel("Residuals")

# Show
plt.savefig("4a.png")
plt.show()


<filename>code/preprocess/consumption/sector/tn/tn_nm.py
#! usr/bin/python3

import pandas as pd
import re
import numpy as np
import os
import sys
from collections import OrderedDict, defaultdict
import matplotlib as mpl
import matplotlib.pyplot as plt
# import seaborn as sns
from scipy import stats, integrate

# sns.set() # switch to seaborn default
# sns.set_style("whitegrid")

#load sector msncodes
tn_msncodes = pd.read_csv("data/csv/consumption/sector/tn_sector.csv", engine='c', low_memory=True)["MSN"]
#load state data
nm_data = pd.read_csv("data/csv/state_data/nm_data.csv", engine='c', low_memory=True)

nm_msn = []
nm_year = []
nm_value = []

for i in range(len(nm_data["MSN"])):
    for j in range(len(tn_msncodes)):
        if nm_data["MSN"][i] == tn_msncodes[j]:
            nm_msn.append(nm_data["MSN"][i])
            nm_year.append(nm_data["Year"][i])
            nm_value.append(nm_data["Data"][i])
        else:
            pass

nm_tn = OrderedDict()
nm_tn["MSN"] = nm_msn
nm_tn["Year"] = nm_year
nm_tn["Data"] = nm_value
nm_tn_data = pd.DataFrame(nm_tn)

nm_tn_data.to_csv("data/csv/consumption/sector/nm/nm_tn_data.csv",
                  index=False, index_label=False, sep=',')
# print(nm_tn_data)

sectors = ["TNACB", "TNCCB", "TNICB", "TNRCB"]
tnacb = OrderedDict()
tnacb["Year"] = []
tnacb["Data"] = []
tnccb = OrderedDict()
tnccb["Year"] = []
tnccb["Data"] = []
tnicb = OrderedDict()
tnicb["Year"] = []
tnicb["Data"] = []
tnrcb = OrderedDict()
tnrcb["Year"] = []
tnrcb["Data"] = []

for i in range(len(nm_tn_data["MSN"])):
    if nm_tn_data["MSN"][i] == "TNACB":
        tnacb["Year"].append(nm_tn_data["Year"][i])
        tnacb["Data"].append(nm_tn_data["Data"][i])
    elif nm_tn_data["MSN"][i] == "TNCCB":
        tnccb["Year"].append(nm_tn_data["Year"][i])
        tnccb["Data"].append(nm_tn_data["Data"][i])
    elif nm_tn_data["MSN"][i] == "TNICB":
        tnicb["Year"].append(nm_tn_data["Year"][i])
        tnicb["Data"].append(nm_tn_data["Data"][i])
    elif nm_tn_data["MSN"][i] == "TNRCB":
        tnrcb["Year"].append(nm_tn_data["Year"][i])
        tnrcb["Data"].append(nm_tn_data["Data"][i])
    else:
        pass

tnacb_data = pd.DataFrame(tnacb)
tnacb_data.to_csv("data/csv/consumption/sector/nm/tn/tnacb.csv",
                  index=False, index_label=False, sep=',')
tnccb_data = pd.DataFrame(tnccb)
tnccb_data.to_csv("data/csv/consumption/sector/nm/tn/tnccb.csv",
                  index=False, index_label=False, sep=',')
tnicb_data = pd.DataFrame(tnicb)
tnicb_data.to_csv("data/csv/consumption/sector/nm/tn/tnicb.csv",
                  index=False, index_label=False, sep=',')
tnrcb_data = pd.DataFrame(tnrcb)
tnrcb_data.to_csv("data/csv/consumption/sector/nm/tn/tnrcb.csv",
                  index=False, index_label=False, sep=',')
# print(tnacb_data)
# print(tnccb_data)
# print(tnicb_data)
# print(tnrcb_data)


r"""Solve Navier-Stokes equations for the lid driven cavity using a coupled
formulation

The equations are in strong form

.. math::

    \nu\nabla^2 u - \nabla p &= (u \cdot \nabla) u) \\
    \nabla \cdot u &= 0 \\
    i\bs{u}(x, y=1) = (1, 0) \, &\text{ or }\, \bs{u}(x, y=1) = ((1-x)^2(1+x)^2, 0) \\
    u(x, y=-1) &= 0 \\
    u(x=\pm 1, y) &= 0

In addition we require :math:`\int p d\ = 0`, which is achieved by
fixing the coefficient :math:`\hat{p}_{0, 0} = 0`.

We use a tensorproductspace with a composite Legendre for the Dirichlet space
and a regular Legendre for the pressure space.

To remove all nullspaces we use a P_{N} x P_{N-2} basis, with P_{N-2} for the
pressure.

"""
import os
import sys
import time
import numpy as np
from scipy.sparse.linalg import splu
import sympy
from shenfun import *

assert comm.Get_size() == 1, "Two non-periodic directions only have solver implemented for serial"

Re = 250.
nu = 2./Re
alfa = 0.1 # underrelaxation factor
N = 64
family = 'Chebyshev'
#family = 'Legendre'
x = sympy.symbols('x', real='True')
D0 = FunctionSpace(N, family, bc=(0, 0))
#D1 = FunctionSpace(N, family, bc=(0, 1))
D1 = FunctionSpace(N, family, bc=(0, (1-x)**2*(1+x)**2))

# Create tensor product spaces with different combination of bases
V1 = TensorProductSpace(comm, (D0, D1))
V0 = V1.get_homogeneous()
P = V1.get_orthogonal()

# To satisfy inf-sup for the Stokes problem, just pick the first N-2 items of the pressure basis
# Note that this effectively sets P_{N-1} and P_{N-2} to zero, but still the basis uses
# the same quadrature points as the Dirichlet basis, which is required for the inner products.
P.bases[0].slice = lambda: slice(0, N-2)
P.bases[1].slice = lambda: slice(0, N-2)

# Create vector space for velocity and a mixed velocity-pressure space
W1 = VectorSpace([V1, V0])
VQ = CompositeSpace([W1, P])

# Create space for nonlinearity
S1 = TensorSpace(P)

up = TrialFunction(VQ)
vq = TestFunction(VQ)

u, p = up
v, q = vq

# Assemble blocks of the complete block matrix
if family.lower() == 'legendre':
    A00 = inner(grad(v), -nu*grad(u))
    A01 = inner(div(v), p)
else:
    A00 = inner(v, nu*div(grad(u)))
    A01 = inner(v, -grad(p))

A10 = inner(q, div(u))

# Create Block matrix solver. This also takes care of boundary conditions.
sol = la.BlockMatrixSolver(A00+A01+A10)

# Create Function to hold solution
uh_hat = Function(VQ).set_boundary_dofs()
ui_hat = uh_hat[0]

# New solution (iterative)
uh_new = Function(VQ).set_boundary_dofs()
ui_new = uh_new[0]

# Create regular work arrays for right hand side.
bh_hat = Function(VQ)

# Create arrays to hold velocity vector solution
ui = Array(W1)

# Create work arrays for nonlinear part
uiuj = Array(S1.get_dealiased())
uiuj_hat = Function(S1)
BS = BlockMatrix(inner(TestFunction(W1), div(TrialFunction(S1))))

def compute_rhs(ui_hat, bh_hat):
    global uip, uiuj, uiuj_hat
    bh_hat.fill(0)
    bi_hat = bh_hat[0]
    # Get convection
    uip = ui_hat.backward(padding_factor=1.5)
    uiuj = outer(uip, uip, uiuj)
    uiuj_hat = uiuj.forward(uiuj_hat)
    #bi_hat = inner(v, div(uiuj_hat), output_array=bi_hat)
    bi_hat = BS.matvec(uiuj_hat, bi_hat) # fastest method
    #bi_hat = inner(grad(v), -uiuj_hat, output_array=bi_hat) # only Legendre
    #gradu = project(grad(ui_hat), S1)
    #bi_hat = inner(v, dot(gradu, ui_hat), output_array=bi_hat)
    return bh_hat

converged = False
count = 0
max_count = 1000
if 'pytest' in os.environ:
    max_count = 1
t0 = time.time()
while not converged:
    count += 1
    bh_hat = compute_rhs(ui_hat, bh_hat)
    uh_new = sol(bh_hat, u=uh_new, constraints=((2, 0, 0),)) # Constraint for component 2 of mixed space
    error = np.linalg.norm(ui_hat-ui_new)
    uh_hat[:] = alfa*uh_new + (1-alfa)*uh_hat
    converged = abs(error) < 1e-11 or count >= max_count
    if count % 1 == 0:
        print('Iteration %d Error %2.4e' %(count, error))

print('Time ', time.time()-t0)

# Move solution to regular Function
up = Array(VQ)
up = uh_hat.backward(up)
u_, p_ = up

if 'pytest' in os.environ: sys.exit(0)

# Postprocessing
# Solve streamfunction
r = TestFunction(V0)
s = TrialFunction(V0)
S = inner(r, div(grad(s)))
h = inner(r, -curl(ui_hat))
H = la.SolverGeneric2ND(S)
phi_h = H(h)
phi = phi_h.backward()
# Compute vorticity
P = V1.get_orthogonal()
w_h = Function(P)
w_h = project(curl(ui_hat), P, output_array=w_h)
#p0 = np.array([[0.], [0.]])
#print(w_h.eval(p0)*2)

# Find minimal streamfunction value and position
# by gradually zooming in on mesh
W = 101
converged = False
xmid, ymid = 0, 0
dx = 1
psi_old = 0
count = 0
y, x = np.meshgrid(np.linspace(ymid-dx, ymid+dx, W), np.linspace(xmid-dx, xmid+dx, W))
points = np.vstack((x.flatten(), y.flatten()))
pp = phi_h.eval(points).reshape((W, W))
while not converged:
    yr, xr = np.meshgrid(np.linspace(ymid-dx, ymid+dx, W), np.linspace(xmid-dx, xmid+dx, W))
    points = np.vstack((xr.flatten(), yr.flatten()))
    pr = phi_h.eval(points).reshape((W, W))
    xi, yi = pr.argmin()//W, pr.argmin()%W
    psi_min, xmid, ymid = pr.min()/2, xr[xi, yi], yr[xi, yi]
    err = abs(psi_min-psi_old)
    converged = err < 1e-15 or count > 10
    psi_old = psi_min
    dx = dx/4.
    print("%d %d " %(xi, yi) +("%+2.7e "*4) %(xmid, ymid, psi_min, err))
    count += 1

import matplotlib.pyplot as plt
#f = open('plot_u_y_Ghia{}.csv'.format(int(Re)))
#g = np.loadtxt(f, skiprows=1, delimiter=',')
#plt.figure()
#y = 2*(g[:, 0]-0.5)
#plt.plot(y, g[:, 1], 'r+')
X = V0.local_mesh(True)
#x = np.vstack([np.zeros(N[0]), X[1][0]])
#res = ui_hat[0].eval(x)
#plt.plot(x[1], res)
#res2 = ui_hat[0].eval(np.vstack([np.zeros(len(y)), y]))
#plt.plot(y, res2, 'bs', mfc='None')
plt.figure()
plt.contourf(X[0], X[1], p_, 100)
plt.figure()
plt.quiver(X[0], X[1], u_[0], u_[1])
plt.figure()
plt.spy(sol.mat.diags())
plt.figure()
plt.contourf(X[0], X[1], u_[0], 100)
plt.figure()
plt.contourf(X[0], X[1], u_[1], 100)
plt.figure()
plt.contourf(X[0], X[1], phi, 100)
#plt.title('Streamfunction')
#plt.show()


<gh_stars>0
import numpy as np
np.set_printoptions(precision=4)

import time
from scipy import spatial

def xy2theta(x, y):
    if (x >= 0 and y >= 0):
        theta = 180/np.pi * np.arctan(y/x);
    if (x < 0 and y >= 0):
        theta = 180 - ((180/np.pi) * np.arctan(y/(-x)));
    if (x < 0 and y < 0):
        theta = 180 + ((180/np.pi) * np.arctan(y/x));
    if ( x >= 0 and y < 0):
        theta = 360 - ((180/np.pi) * np.arctan((-y)/x));

    return theta

def pt2rs(point, gap_ring, gap_sector, num_ring, num_sector):
    x = point[0]
    y = point[1]
    # z = point[2]

    if(x == 0.0):
        x = 0.001
    if(y == 0.0):
        y = 0.001

    theta = xy2theta(x, y)
    faraway = np.sqrt(x*x + y*y)

    idx_ring = np.divmod(faraway, gap_ring)[0]
    idx_sector = np.divmod(theta, gap_sector)[0]

    if(idx_ring >= num_ring):
        idx_ring = num_ring-1 # python starts with 0 and ends with N-1

    return int(idx_ring), int(idx_sector)

def ptcloud2sc(ptcloud, sc_shape, max_length):
    num_ring = sc_shape[0]
    num_sector = sc_shape[1]

    gap_ring = max_length/num_ring
    gap_sector = 360/num_sector

    enough_large = 500
    sc_storage = np.zeros([enough_large, num_ring, num_sector])
    sc_counter = np.zeros([num_ring, num_sector])

    num_points = ptcloud.shape[0]
    for pt_idx in range(num_points):
        point = ptcloud[pt_idx, :]
        point_height = point[2] + 2.0 # for setting ground is roughly zero

        idx_ring, idx_sector = pt2rs(point, gap_ring, gap_sector, num_ring, num_sector)

        if sc_counter[idx_ring, idx_sector] >= enough_large:
            continue
        sc_storage[int(sc_counter[idx_ring, idx_sector]), idx_ring, idx_sector] = point_height
        sc_counter[idx_ring, idx_sector] = sc_counter[idx_ring, idx_sector] + 1

    sc = np.amax(sc_storage, axis=0)

    return sc

def sc2rk(sc):
    return np.mean(sc, axis=1)

def distance_sc(sc1, sc2):
    num_sectors = sc1.shape[1]

    # repeate to move 1 columns
    _one_step = 1 # const
    sim_for_each_cols = np.zeros(num_sectors)
    for i in range(num_sectors):
        # Shift
        sc1 = np.roll(sc1, _one_step, axis=1) #  columne shift

        #compare
        sum_of_cossim = 0
        num_col_engaged = 0
        for j in range(num_sectors):
            col_j_1 = sc1[:, j]
            col_j_2 = sc2[:, j]
            if (~np.any(col_j_1) or ~np.any(col_j_2)):
                # to avoid being divided by zero when calculating cosine similarity
                # - but this part is quite slow in python, you can omit it.
                continue

            cossim = np.dot(col_j_1, col_j_2) / (np.linalg.norm(col_j_1) * np.linalg.norm(col_j_2))
            sum_of_cossim = sum_of_cossim + cossim

            num_col_engaged = num_col_engaged + 1

        # save
        sim_for_each_cols[i] = sum_of_cossim / num_col_engaged

    yaw_diff = np.argmax(sim_for_each_cols) + 1 # because python starts with 0
    sim = np.max(sim_for_each_cols)
    dist = 1 - sim

    return dist, yaw_diff

class ScanContextManager:
    def __init__(self, shape=[20,60], num_candidates=10, threshold=0.15): # defualt configs are same as the original paper
        self.shape = shape
        self.num_candidates = num_candidates
        self.threshold = threshold

        self.max_length = 80 # recommended but other (e.g., 100m) is also ok.

        self.ENOUGH_LARGE = 15000 # capable of up to ENOUGH_LARGE number of nodes
        self.ptclouds = [None] * self.ENOUGH_LARGE
        self.scancontexts = [None] * self.ENOUGH_LARGE
        self.ringkeys = [None] * self.ENOUGH_LARGE

        self.curr_node_idx = 0

    def addNode(self, node_idx, ptcloud):
        sc = ptcloud2sc(ptcloud, self.shape, self.max_length)
        rk = sc2rk(sc)

        self.curr_node_idx = node_idx
        self.ptclouds[node_idx] = ptcloud
        self.scancontexts[node_idx] = sc
        self.ringkeys[node_idx] = rk

    def getPtcloud(self, node_idx):
        return self.ptclouds[node_idx]

    def detectLoop(self):
        exclude_recent_nodes = 30
        valid_recent_node_idx = self.curr_node_idx - exclude_recent_nodes

        if(valid_recent_node_idx < 1):
            return None, None, None
        else:
            # step 1
            ringkey_history = np.array(self.ringkeys[:valid_recent_node_idx])
            ringkey_tree = spatial.KDTree(ringkey_history)

            ringkey_query = self.ringkeys[self.curr_node_idx]
            _, nncandidates_idx = ringkey_tree.query(ringkey_query, k=self.num_candidates)

            # step 2
            query_sc = self.scancontexts[self.curr_node_idx]

            nn_dist = 1.0 # initialize with the largest value of distance
            nn_idx = None
            nn_yawdiff = None
            for ith in range(self.num_candidates):
                candidate_idx = nncandidates_idx[ith]
                candidate_sc = self.scancontexts[candidate_idx]
                dist, yaw_diff = distance_sc(candidate_sc, query_sc)
                if(dist < nn_dist):
                    nn_dist = dist
                    nn_yawdiff = yaw_diff
                    nn_idx = candidate_idx

            if(nn_dist < self.threshold):
                nn_yawdiff_deg = nn_yawdiff * (360/self.shape[1])
                return nn_idx, nn_dist, nn_yawdiff_deg # loop detected!
            else:
                return None, None, None