#!/usr/bin/env python
# -*- coding: utf-8 -*-
import pandas as pd
from datetime import datetime, timedelta
import numpy as np
from scipy.stats import pearsonr
import matplotlib.font_manager as fm
import netCDF4 as nc
from netCDF4 import Dataset
import os
from pyproj import Proj
import ast

#------------------------------------------------------------------------------
# Motivación codigo -----------------------------------------------------------
"""
Codigo para el reconociemto de las variables de Porfundidad optica de las nubes de lat_GOES.
Los datos son leidos desde el disco duro por si volumen.
"""

path_directory = "/media/nacorreasa/NATHALIA/Productos/CODDF/"

path_save = '/home/nacorreasa/Maestria/Datos_Tesis/GOES/GOES_COD/'
resolucion = '2 km'
fec_ini = '2020-01-01'
fec_fin = '2020-01-12'
Modo = '6'

# =*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*
# =*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*
                                        # No cambiar nada en adelante
# =*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*
# =*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*

##-----------------------------FUNCIONES PARA BUSCAR LOS ARCHIVOS---------------------------------##
import datetime as dt
fi  = dt.datetime.strptime(fec_ini, '%Y-%m-%d')
ff  = dt.datetime.strptime(fec_fin, '%Y-%m-%d')

def retorna_fecha_hora_info(ruta):
    #try:
    #    fecha = ruta[ruta.find('c2019')+1:ruta.find('c2019')+12]
    #except:
    fecha = ruta[ruta.find('c2020')+1:ruta.find('c2020')+12]
    fecha = dt.datetime.strptime(fecha, '%Y%j%H%M')
    fecha_utc = fecha.strftime('%Y/%m/%d/ %H:%M')
    fecha_local = fecha - dt.timedelta(hours=5)
    fecha_local = fecha_local.strftime('%Y/%m/%d %H:%M')
    fecha_path = fecha.strftime('%Y%m%d')
    fecha = fecha.strftime('%Y%m%d%H%M')
    return fecha_utc, fecha_path, fecha , fecha_local

def daterange(start_date, end_date):                                  ## PARA LA SELECCION DE LAS CARPETAS DE INTERES
    delta = timedelta(days=1)
    while start_date <= end_date:
        yield start_date
        start_date += delta

Folders = []
for single_date in daterange(fi, ff):
    Folders.append(single_date.date().strftime('%Y%m%d') )

def Listador(directorio, inicio, final):
    lf = []
    lista = os.listdir(directorio)
    for i in lista:
        if i.startswith(inicio) and i.endswith(final):
            lf.append(i)
    return lf

archivos = []
missing_folders = []
for folder in Folders:
    if os.path.exists(path_directory+folder) == True:
        dias =  Listador(path_directory+folder, 'CG_ABI-L2-CODDF-M'+Modo+'_G16', '.nc')
        dias.sort()
        for hora in dias:
            archivos.append(folder+'/'+hora)
    elif os.path.exists(path_directory+folder) == False:
        print ('hay q crear la carpeta: ' + folder)
        missing_folders.append(folder)
        #os.system('mkdir ' + path_directory+folder+'/')
        pass

archivos.sort()
##-----------------------------------FUNCION PARA CREAR LAS CARPETAS FALTNATES------------------------------------#
for folder in Folders:
    os.system('mkdir -p ' + path_save + folder )
##-----------------------------------FUNCIONES PARA RECORTAR------------------------------------#

def find_nearest(array, value):                              ## PARA ENCONTRAR LA CASILLA MAS CERCANA
    idx = (abs(array-value)).argmin()
    return array[idx], idx

def Posiciones(archivo, limites_zona):                        ## RELACIONAR ESAS CASILLAS CON LAS LATITUDES REQUERIDAS
    archivo = nc.Dataset(archivo)
    h = archivo.variables['goes_imager_projection'].perspective_point_height
    x = archivo.variables['x'][:]
    y = archivo.variables['y'][:]
    lon_0 = archivo.variables['goes_imager_projection'].longitude_of_projection_origin
    lat_0 = archivo.variables['goes_imager_projection'].latitude_of_projection_origin
    sat_sweep = archivo.variables['goes_imager_projection'].sweep_angle_axis
    archivo.close()
    x = x * h
    y = y * h
    p = Proj(proj='geos', h=h, lon_0=lon_0, swee=sat_sweep)
    xx, yy = np.meshgrid(x, y)
    print (xx, yy)

    lons, lats = p(xx[:, :], yy[:, :], inverse=True)
    lons = np.ma.array(lons)
    lons[lons == 1.00000000e+30] = np.ma.masked
    lats = np.ma.array(lats)
    lats[lats == 1.00000000e+30] = np.ma.masked

    mid = int(np.shape(lons)[0]/2)
    lon_O = find_nearest(lons[mid, :], limites_zona[0])[1]
    lon_E = find_nearest(lons[mid, :], limites_zona[1])[1]
    lat_N = find_nearest(lats[:, mid], limites_zona[2])[1]
    lat_S = find_nearest(lats[:, mid], limites_zona[3])[1]
    return lons, lats, lon_O, lon_E, lat_N, lat_S

def Obtiene_Lon_Lat_COD(ruta, dicc):                       ## PARA OBTENER LA INFORMACION ACOTADA A LA REGION DE INTERES
    limites_zona = dicc
    archivo = nc.Dataset(ruta)

    h = archivo.variables['goes_imager_projection'].perspective_point_height
    x = archivo.variables['x'][limites_zona[0]:limites_zona[1]]
    y = archivo.variables['y'][limites_zona[2]:limites_zona[3]]
    lon_0 = archivo.variables['goes_imager_projection'].longitude_of_projection_origin
    lat_0 = archivo.variables['goes_imager_projection'].latitude_of_projection_origin
    sat_sweep = archivo.variables['goes_imager_projection'].sweep_angle_axis
    #d = (archivo.variables['earth_sun_distance_anomaly_in_AU'][:])**2
    #esun = archivo.variables['esun'][:]

    COD = archivo.variables["COD"][limites_zona[2]:limites_zona[3], limites_zona[0]:limites_zona[1]]
    archivo.close()
    x = x * h
    y = y * h
    p = Proj(proj='geos', h=h, lon_0=lon_0, swee=sat_sweep)
    xx, yy = np.meshgrid(x,y)
    lons, lats = p(xx, yy, inverse=True)
    lons = np.ma.array(lons)
    lons[lons == 1.00000000e+30] = np.ma.masked
    lats = np.ma.array(lats)
    lats[lats == 1.00000000e+30] = np.ma.masked

    return lons, lats, COD

def LimitesZonas(resolucion, zonas, path_CH):                                   ## PARA OBTENER LA LISTA CON LOS LIMITES DE LA ZONA

    if resolucion == 0.5:
        for i in zonas.keys():
            lons, lats, lon_O, lon_E, lat_N, lat_S = Posiciones(path_CH, zonas[i])
            print (u"Posiciones dada una resolución espacial de 0.5 km")
            print ("Para la zona " + i + '\n'
                   u"La posición para la longitud occidental es : " + str(lon_O-1) + '\n' +
                   u"La posición para la longitud oriental es : " + str(lon_E) + '\n' +
                   u"La posición para la latitud norte es : " + str(lat_N-1) + '\n' +
                   u"La posición para la latitud sur es : " + str(lat_S) +'\n' +
                   u"inclúyalas así en el diccionario de posiciones:"  + '\n'
                   u"[" + str(lon_O-1) + ',' + str(lon_E) + ',' + str(lat_N-1) + ',' + str(lat_S) + "]")
        DiccionarioPos =   u"[" + str(lon_O-1) + ',' + str(lon_E) + ',' + str(lat_N-1) + ',' + str(lat_S) + "]"
        DiccionarioPos = ast.literal_eval(DiccionarioPos)
    elif resolucion == 1:
        for i in zonas.keys():
            lons, lats, lon_O, lon_E, lat_N, lat_S = Posiciones(path_CH, zonas[i])
            print (u"Posiciones dada una resolución espacial de 1 km")
            print ("Para la zona " + i + '\n'
                                         u"La posición para la longitud occidental es : " + str(lon_O - 1) + '\n' +
                   u"La posición para la longitud oriental es : " + str(lon_E) + '\n' +
                   u"La posición para la latitud norte es : " + str(lat_N - 1) + '\n' +
                   u"La posición para la latitud sur es : " + str(lat_S) + '\n' +
                   u"inclúyalas así en el diccionario de posiciones:" + '\n'
                                                                        u"[" + str(lon_O - 1) + ',' + str(
                lon_E) + ',' + str(lat_N - 1) + ',' + str(lat_S) + "]")
        DiccionarioPos =   u"[" + str(lon_O-1) + ',' + str(lon_E) + ',' + str(lat_N-1) + ',' + str(lat_S) + "]"
        DiccionarioPos = ast.literal_eval(DiccionarioPos)
    elif resolucion == 2:
        for i in zonas.keys():
            lons, lats, lon_O, lon_E, lat_N, lat_S = Posiciones(path_CH, zonas[i])
            print (u"Posiciones dada una resolución espacial de 2 km")
            print ("Para la zona " + i + '\n'
                   u"La posición para la longitud occidental es : " + str(lon_O-1) + '\n' +
                   u"La posición para la longitud oriental es : " + str(lon_E) + '\n' +
                   u"La posición para la latitud norte es : " + str(lat_N-1) + '\n' +
                   u"La posición para la latitud sur es : " + str(lat_S) +'\n' +
                   u"inclúyalas así en el diccionario de posiciones:"  + '\n'
                   u"[" + str(lon_O-1) + ',' + str(lon_E) + ',' + str(lat_N-1) + ',' + str(lat_S) + "]")
        DiccionarioPos =   u"[" + str(lon_O-1) + ',' + str(lon_E) + ',' + str(lat_N-1) + ',' + str(lat_S) + "]"
        DiccionarioPos = ast.literal_eval(DiccionarioPos)
    return DiccionarioPos

##-----------DELIMITAR LA ZONA  EN COORDENADAS CARTESIANAS Y CREAR  VARIABLES-----------#

## Coordenadas cartecianas de la zona de interés
latu = 6.5900
latd = 5.9300
lono = -75.85
lone = -75.07

zonas = {'Valle_de_Aburra': [-75.85, -75.07, 6.5900,  5.9300]}

## Ejecutar las funciones para tener la banda acotada a la región de interés y sus latitudes y longitudes

limzon = LimitesZonas(2, zonas, path_directory+archivos[0])
if resolucion == '1 km':
    limzon = [5328, 5416, 4698, 4771]
elif resolucion == '0.5 km':
    limzon = [10658, 10832, 9397, 9542]
elif resolucion == '2 km':
    limzon = [770, 815, 717, 754]

lons, lats, rad = Obtiene_Lon_Lat_COD(path_directory+ archivos[0], limzon)

## Iniciar variables
ntime    = 1
nlat     = np.shape(lats)[0]
nlon     = np.shape(lons)[1]
nrad     = np.shape(rad)

def Obtener_fecha_hora_de_Archvos (path_directory, archivos):
    local_hour = []
    local_date_str = []
    for i in range(len(archivos)):
        fecha_utc, fecha_path, fecha, fecha_local = retorna_fecha_hora_info(path_directory + archivos[i])
        local_hour.append(dt.datetime.strptime(fecha_local, '%Y/%m/%d %H:%M'))
        local_date_str.append(fecha_local)
    return local_date_str, local_hour

def retorna_scalefactor_y_addoffset_latlon(dataset,bits):         ## Nor maliza los valores de las variables
    import numpy as np
    max_o = np.max(dataset)
    min_o = np.min(dataset)
    scale_factor = (max_o - min_o) / (2**bits)
    add_offset = min_o
    return scale_factor, add_offset

fechas, time = Obtener_fecha_hora_de_Archvos (path_directory, archivos)

cdftime = 'hours since 2018-01-01 00:00:0.0'
date    = nc.date2num(time, units = cdftime)

# =*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*
# =*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*
                                        # Crear cada netCDF
# =*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*
# =*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*

for i  in range(len(archivos)):
    ## Varibales temporales
    lons_temp, lats_temp, COD_temp = Obtiene_Lon_Lat_COD(path_directory+archivos[i], limzon)

    COD = np.zeros((1, np.shape(COD_temp)[0], np.shape(COD_temp)[1]))
    COD[:, :, :] = COD_temp[:, :]

    ## Crear el nuevo archivo nc
    nw = Dataset(path_save + archivos[i], 'w', format='NETCDF4')

    ## Definir dimensiones locativas
    ncdim_lat = nw.createDimension('nlat', nlat)
    ncdim_lon = nw.createDimension('nlon', nlon)
    ncdim_time = nw.createDimension('ntime', ntime)

    # Crear variables locativas
    ncvar_lat = nw.createVariable('lat', 'f8', ('nlat', 'nlon'), zlib=True, complevel=9)
    ncvar_lon = nw.createVariable('lon', 'f8', ('nlat', 'nlon'), zlib=True, complevel=9)
    ncvar_time = nw.createVariable('time', 'f8', ('ntime',))

    ncvar_COD = nw.createVariable('COD', 'f8', ('ntime', 'nlat', 'nlon'), zlib=True, complevel=9)

    print ('netCDF variables created')

    ## Agregar unidades a las variables
    ncvar_lat.units = 'Degrees north'
    ncvar_lon.units = 'Degrees east'
    ncvar_time.units = 'Hours since 2018-01-01'

    ncvar_COD.units = 'dimensionless'

    try:
        ## Factores de escala
        ncvar_lon.scale_factor, ncvar_lon.add_offset = retorna_scalefactor_y_addoffset_latlon(lons, 11)
        ncvar_lat.scale_factor, ncvar_lat.add_offset = retorna_scalefactor_y_addoffset_latlon(lats, 11)
        ncvar_lat.scale_factor, ncvar_lat.add_offset = retorna_scalefactor_y_addoffset_latlon(COD, 11)
    except ValueError:
        pass

    ## Agregar nombres largos
    ncvar_lat.longname = 'Array of latitude values at the center of the grid box'
    ncvar_lon.longname = 'Array of longitude values at the center of the grid box'
    ncvar_time.longname = 'Hours since 2018-01-01'
    ncvar_COD.longname = 'COD'

    nw.title = archivos[i][12:]

    nw.spatial_resolution = resolucion

    nw.metadatos = "https://www.goes-r.gov/multimedia/dataAndImageryImagesGoes-16.html"

    # Agregar los datos al archivo
    ncvar_time[:] = date[i]
    ncvar_lat[:, :] = lats.data
    ncvar_lon[:, :] = lons.data

    print ('******************************************')
    print ('    writing variables in netCDF file ')
    print ('******************************************')
    ncvar_COD[:, :, :] = COD

    # Si no cierra el archivo es como dejar la BD abierta... se lo tira!
    nw.close()

print ('Hemos terminado')


<reponame>xiameng552180/Intervention
from scipy.stats import chi2_contingency
from scipy.stats import chi2

# Game on 327
#          Ctrl, V1, V2, V3
# Game       15   5   8  10
# Not Game   55  37  41  33

# Game on 544
#          Ctrl, V1, V2, V3
# Game       9   3   3  6
# Not Game   30  34  41  32
table = [[15, 5, 8, 10],
         [70-15, 42-5, 49-8, 43-10]]

table1 = [[9, 3, 3, 6], [30, 34, 41, 32]]
stat, p, dof, expected = chi2_contingency(table1)
print('* Degree of Freedom: %d' % dof)

# 解释结果
prob = 0.95
critical = chi2.ppf(prob, dof)
print('* Accept Probability = %.3f' % prob)
print('* Critical = %.3f' % critical)

print('* Interpretation as follows:')
print('- Chi^2 = %.3f' % stat)
print('- P-val = %.3f' % p)
if abs(stat) >= critical:
    print('- Dependent (reject H0)')
else:
    print('- Independent (fail to reject H0)')

alpha = 1.0 - prob
if p <= alpha:
    print('- p-value(%.2f) <= %.2f, dependent (reject H0)' % (p, alpha))
else:
    print('- p-value(%.2f) > %.2f, independent (fail to reject H0)' % (p, alpha))


<filename>dataloaders/ofos.py
from torch.utils.data import Dataset, ConcatDataset
from torch.utils.data.dataloader import DataLoader
from torch.utils.data.sampler import RandomSampler, SequentialSampler
from madmom.audio.signal import FramedSignal
import madmom.audio.spectrogram as mmspec
from madmom.io import midi
from scipy.ndimage.filters import maximum_filter1d
from copy import deepcopy
import numpy as np
import joblib
import torch
import utils
import os
import csv
import warnings

memory = joblib.memory.Memory('./joblib_cache', mmap_mode='r', verbose=1)

def get_y_from_file(midifile, n_frames, audio_options):
    with warnings.catch_warnings():
        warnings.simplefilter('ignore')

        pattern = midi.MIDIFile(midifile)
        dt = float(audio_options['hop_size']) / float(audio_options['sample_rate'])

        y_onsets = np.zeros((n_frames, 88), dtype=np.uint8)
        y_frames = np.zeros((n_frames, 88), dtype=np.uint8)
        y_offsets = np.zeros((n_frames, 88), dtype=np.uint8)

        for onset, _pitch, duration, velocity, _channel in pattern.sustained_notes:
            pitch = int(_pitch)
            label = pitch - 21

            note_start = int(np.round(onset / dt))
            note_end = int(np.round((onset + duration) / dt))

            # some of the midi-files have onsets/offsets larger
            # than n_frames. they were manually checked, and it's
            # actually not an issue at all.
            # see data-preparation/maestro-inconsistencies/* for
            # scripts that perform visual inspection!
            if note_start < n_frames:
                if note_end >= n_frames:
                    # print('weird_offset', midifile)
                    note_end = n_frames - 1

                y_onsets[note_start, label] = 1
                y_frames[note_start:note_end + 1, label] = 1
                y_offsets[note_end, label] = 1
            else:
                # print('weird_onset', midifile)
                pass

        return y_onsets, y_frames, y_offsets

def get_existing_filename(filename, suffixes):
    for suffix in suffixes:
        if os.path.exists(filename + suffix):
            return filename + suffix

    raise ValueError('provided filename "{}" does not exist with any of the endings {}'.format(filename, suffixes))

@memory.cache
def get_xy_from_file(audio_filename, midi_filename, _audio_options):
    spec_type, audio_options = utils.canonicalize_audio_options(_audio_options, mmspec)
    x = np.array(spec_type(audio_filename, **audio_options))
    y_onsets, y_frames, y_offsets = get_y_from_file(midi_filename, len(x), audio_options)

    return x, y_onsets, y_frames, y_offsets

def get_xy_from_file_subsampled(audio_filename, midi_filename, audio_options, start_end):
    x, y_onsets, y_frames, y_offsets = get_xy_from_file(
        audio_filename,
        midi_filename,
        audio_options
    )
    if start_end is None:
        return x, y_onsets, y_frames, y_offsets
    else:
        start, end = start_end
        return x[start:end], y_onsets[start:end], y_frames[start:end], y_offsets[start:end]

# maxfilter in temporal direction
def widen(x, w):
    if w % 2 == 0:
        raise RuntimeError('unsupported')

    return maximum_filter1d(x, w, axis=0, mode='constant', cval=0, origin=0)

def suppress_offets(y_onsets, y_offsets):
    # everywhere where onsets and offsets DO NOT occur simultaenously,
    # (y_onsets != y_offsets) will be False
    # only where onsets and offsets DO NOT overlap, it'll be True
    return (y_onsets != y_offsets) * 1

class OneSequenceDataset(Dataset):
    def __init__(self,
                 audio_filename,
                 midi_filename,
                 input_context,
                 target_maxfilter,
                 audio_options,
                 start_end=None,
                 offset_suppression=None):
        self.metadata = dict(
            audio_filename=audio_filename,
            midi_filename=midi_filename
        )
        self.audio_options = deepcopy(audio_options)

        x, y_onsets, y_frames, y_offsets = get_xy_from_file_subsampled(
            self.metadata['audio_filename'],
            self.metadata['midi_filename'],
            self.audio_options,
            start_end
        )

        self.y_onsets = widen(y_onsets, target_maxfilter['y_onsets'])
        self.y_frames = widen(y_frames, target_maxfilter['y_frames'])

        if offset_suppression is not None:
            # this gets passed the widened *onsets* already
            y_offsets = suppress_offets(y_onsets, y_offsets)

        # widen *after* suppression
        self.y_offsets = widen(y_offsets, target_maxfilter['y_offsets'])

        self.x = FramedSignal(
            x,
            frame_size=input_context['frame_size'],
            hop_size=input_context['hop_size'],
            origin=input_context['origin'],
        )
        if (len(self.x) != len(self.y_onsets) or
           len(self.x) != len(self.y_frames) or
           len(self.x) != len(self.y_offsets)):
            raise RuntimeError('x and y do not have the same length.')

    def __getitem__(self, index):
        _, w, h = self.x.shape
        return dict(
            x=torch.FloatTensor(self.x[index].reshape(1, w, h)),
            y_onsets=torch.FloatTensor(self.y_onsets[index]),
            y_frames=torch.FloatTensor(self.y_frames[index]),
            y_offsets=torch.FloatTensor(self.y_offsets[index])
        )

    def __len__(self):
        return len(self.x)

def get_dataset_individually(base_directory, metadata_filename, split, input_context, target_maxfilter, audio_options, start_end=None, offset_suppression=None):

    fieldnames = [
        'canonical_composer',
        'canonical_title',
        'split',
        'year',
        'midi_filename',
        'audio_filename',
        'duration'
    ]

    class magenta_dialect(csv.Dialect):
        delimiter = ','
        quotechar = '"'
        doublequote = True
        skipinitialspace = True
        lineterminator = '\n'
        quoting = csv.QUOTE_MINIMAL
    csv.register_dialect('magenta', magenta_dialect)

    sequences = []
    with open(metadata_filename, 'r') as metadata_file:
        csvreader = csv.DictReader(metadata_file, fieldnames=fieldnames, dialect='magenta')
        for row in csvreader:
            if row['split'] == split:
                sequences.append(OneSequenceDataset(
                    os.path.join(base_directory, row['audio_filename']),
                    os.path.join(base_directory, row['midi_filename']),
                    input_context,
                    target_maxfilter,
                    audio_options,
                    start_end,
                    offset_suppression
                ))
    return sequences

def get_dataset(*args, **kwargs):
    dataset = ConcatDataset(get_dataset_individually(*args, **kwargs))
    print('len(dataset)', len(dataset))
    return dataset

def get_loaders(config):
    lcs = config['modules']['dataloader']['args']
    loaders = dict()
    for key, lc in lcs.items():
        individual_files = lc.get('individual_files', False)
        if individual_files:
            sequences = get_dataset_individually(
                base_directory=lc['base_directory'],
                metadata_filename=lc['metadata_filename'],
                split=lc['split'],
                input_context=config['modules']['dataloader']['input_context'],
                target_maxfilter=config['modules']['dataloader']['target_maxfilter'],
                audio_options=config['audio_options'],
                start_end=config['modules']['dataloader'].get('start_end', None),
                offset_suppression=config['modules']['dataloader'].get('offset_suppression', None)
            )
            individual_loaders = []
            for sequence in sequences:
                if lc['sampler'] == 'RandomSampler':
                    sampler = RandomSampler(sequence)
                elif lc['sampler'] == 'SequentialSampler':
                    sampler = SequentialSampler(sequence)
                elif lc['sampler'] == 'ChunkedRandomSampler':
                    sampler = utils.ChunkedRandomSampler(sequence, lc['chunk_size'])

                individual_loader = DataLoader(
                    sequence,
                    batch_size=config['batchsize'],
                    sampler=sampler,
                    num_workers=lc.get('num_workers', 0),
                    pin_memory=lc.get('pin_memory', False)
                )
                individual_loaders.append(individual_loader)
            loaders[key] = individual_loaders
        else:
            sequences = get_dataset(
                base_directory=lc['base_directory'],
                metadata_filename=lc['metadata_filename'],
                split=lc['split'],
                input_context=config['modules']['dataloader']['input_context'],
                target_maxfilter=config['modules']['dataloader']['target_maxfilter'],
                audio_options=config['audio_options'],
                start_end=config['modules']['dataloader'].get('start_end', None),
                offset_suppression=config['modules']['dataloader'].get('offset_suppression', None)
            )

            if lc['sampler'] == 'RandomSampler':
                sampler = RandomSampler(sequences)
            elif lc['sampler'] == 'SequentialSampler':
                sampler = SequentialSampler(sequences)
            elif lc['sampler'] == 'ChunkedRandomSampler':
                sampler = utils.ChunkedRandomSampler(sequences, lc['chunk_size'])

            loader = DataLoader(
                sequences,
                batch_size=config['batchsize'],
                sampler=sampler,
                num_workers=lc.get('num_workers', 0),
                pin_memory=lc.get('pin_memory', False)
            )

            loaders[key] = loader
    return loaders

def main():
    context = dict(
        frame_size=5,
        hop_size=1,
        origin='center'
    )

    target_maxfilter = dict(
        y_onsets=3,
        y_frames=1,
        y_offsets=3
    )

    audio_options = dict(
        spectrogram_type='LogarithmicFilteredSpectrogram',
        filterbank='LogarithmicFilterbank',
        sample_rate=44100,
        num_channels=1,
        frame_size=2048,
        hop_size=1024
    )

    sequences = get_dataset(
        base_directory='./data/maestro/maestro-v1.0.0/',
        metadata_filename='./data/maestro/maestro-v1.0.0/maestro-v1.0.0.csv',
        split='train',
        input_context=context,
        target_maxfilter=target_maxfilter,
        audio_options=audio_options
    )
    print('len(sequences)', len(sequences))

    loader = DataLoader(
        sequences,
        batch_size=128,
        shuffle=False,
        sampler=SequentialSampler(sequences),
        drop_last=False
    )

    import matplotlib.pyplot as plt

    batch = next(iter(loader))
    x = batch['x'].numpy()[:, 0, 2, :]
    y_onsets = batch['y_onsets'].numpy()
    y_frames = batch['y_frames'].numpy()
    y_offsets = batch['y_offsets'].numpy()

    fig, axes = plt.subplots(nrows=4, sharex=True, sharey=True)

    axes[0].set_title('x')
    axes[1].set_title('y_onsets')
    axes[2].set_title('y_frames')
    axes[3].set_title('y_offsets')

    axes[0].imshow(x.T, origin='lower')
    axes[1].imshow(y_onsets.T, cmap='gray_r', origin='lower')
    axes[2].imshow(y_frames.T, cmap='gray_r', origin='lower')
    axes[3].imshow(y_offsets.T, cmap='gray_r', origin='lower')
    plt.show()

if __name__ == '__main__':
    main()


<reponame>rcw175/fittatoreJ
from numpy import*
from scipy import optimize, misc
from matplotlib.pyplot import *
%matplotlib inline
import pandas as pd

def fittatoreJ(fu, x, y, dx, dy, init, labels, ran=False, doppio=False):
    '''
    labels = {'x':, 'y':, 'file':}
    fu(x, *par)
    '''
    if ran==False:
        a,b = 0, x.size
    else:
        a,b = ran
    par, dev = optimize.curve_fit(fu, x[a:b], y[a:b], sigma=dy[a:b], p0=init, absolute_sigma=True)
    par = par; dev = diag(dev)
    if doppio==True:
        dyi = sqrt(dy[a:b]**2 + misc.derivative(fu, x[a:b], args=par, dx=0.0001)**2*1**2)
        par, dev = optimize.curve_fit(fu, x[a:b], y[a:b], sigma=dyi, p0=init, absolute_sigma=True)
        par = par; dev = diag(dev)
    X2 = {'obs':sum((y[a:b]-fu(x[a:b],*par))**2/dy[a:b]**2), 'exp':x[a:b].size-par.size}
    N=0
    figure(N)
    errorbar(x[a:b],y[a:b],yerr=dy[a:b],xerr=dx[a:b], fmt=',', capsize=3)
    plot(linspace(min(x[a:b]), max(x[a:b]),10000), fu(linspace(min(x[a:b]), max(x[a:b]),10000), *par), ',')
    xlabel(labels['x'])
    ylabel(labels['y'])
    savefig('figs/'+labels['file']+'.pdf')
    savefig('figs/'+labels['file']+'.png')

    figure(N+100)
    plot((y[a:b]-fu(x[a:b],*par))/dy[a:b], '.')
    plot(array([0,b-a]), array([0,0]))
    ylabel('Residui')
    savefig('figs/'+labels['file']+'_res.pdf')
    savefig('figs/'+labels['file']+'_res.png')

    return (array([par,dev]).T, X2)

def mediapesata(x,dx):
    return(sum(x/dx**2)/sum(1/dx**2), sqrt(1/sum(1/dx**2)))


<reponame>andr-kun/vireo<gh_stars>0
import itertools
import numpy as np
from scipy.stats import entropy
from scipy.special import logsumexp, digamma, betaln
from .vireo_base import normalize, loglik_amplify, beta_entropy, get_binom_coeff

__docformat__ = "restructuredtext en"

__all__ = ['Vireo', 'add_doublet_theta', 'add_doublet_GT']

class Vireo():
    """Viroe model: Variational Inference for reconstruction of ensemble origin

    The prior can be set via set_prior() before fitting the model.

    Key properties
    --------------
    beta_mu: numpy array (1, n_GT) or (n_var, n_GT)
        Beta mean parameter of theta's posterior
    beta_sum: numpy array (1, n_GT) or (n_var, n_GT), same as beta_mu
        Beta concetration parameter of theta's posterior
    ID_prob: numpy array (n_cell, n_donor)
        Posterior cell assignment probability to each donor
    GT_prob: numpy array (n_var, n_donor, n_GT)
        Posterior genotype probability per variant per donor
    """
    def __init__(self, n_cell, n_var, n_donor, n_GT=3, learn_GT=True,
        learn_theta=True, ASE_mode=False, fix_beta_sum=False,
        beta_mu_init=None, beta_sum_init=None, ID_prob_init=None,
        GT_prob_init=None):
        """Initialise Vireo model

        Note, multiple initializations are highly recomended to avoid local
        optima.

        Parameters
        ----------
        n_cell : int.
            Number of cells
        n_var : int.
            Number of variants
        n_donor : int.
            Number of donors
        n_GT : int.
            Number of genotype categories
        learn_GT: bool.
            Whether updating `GT_prob`; otherwise using the initial
        ASE_mode: bool.
            Whether setting allelic ratio `theta` to be variant specific
        fix_beta_sum: bool.
            Whether fixing the concetration parameter of theta's posterior
        beta_mu_init: numpy array (1, n_GT) or (n_var, n_GT)
            Initial value of beta_mu, the mean parameter of theta
        beta_sum_init: numpy array (1, n_GT) or (n_var, n_GT), same as beta_mu
            Initial value of beta_sum, the concetration parameter of theta
        ID_prob_init: numpy array (n_cell, n_donor)
            Initial value of ID_prob, cell assignment probability to each donor
        GT_prob_init: numpy array (n_var, n_donor, n_GT)
            Initial value of GT_prob, genotype probability per variant and donor
        """
        self.n_GT = n_GT
        self.n_var = n_var
        self.n_cell = n_cell
        self.n_donor = n_donor
        self.learn_GT = learn_GT
        self.ASE_mode = ASE_mode
        self.learn_theta = learn_theta
        self.fix_beta_sum = fix_beta_sum

        theta_len = n_var if ASE_mode else 1

        # initial key parameters
        if beta_mu_init is not None:
            self.beta_mu = beta_mu_init
        else:
            self.beta_mu = (np.ones((theta_len, n_GT)) *
                np.linspace(0.01, 0.99, n_GT).reshape(1, -1))

        if beta_sum_init is not None:
            self.beta_sum = beta_sum_init
        else:
            self.beta_sum = np.ones((theta_len, n_GT)) * 50

        if ID_prob_init is not None:
            self.ID_prob = normalize(ID_prob_init, axis=1)
        else:
            self.ID_prob = normalize(np.random.rand(n_cell, n_donor))

        if GT_prob_init is not None:
            self.GT_prob = normalize(GT_prob_init)
        else:
            self.GT_prob = normalize(np.random.rand(n_var, n_donor, n_GT))

        self.ELBO_ = np.zeros((0))
        self.set_prior()

    def set_prior(self, GT_prior=None, ID_prior=None, beta_mu_prior=None,
        beta_sum_prior=None, min_GP=0.00001):
        """Set prior for key variables: theta, GT_prob and ID_prob.
        The priors are in the same shape as its according variables.

        min_GP: float. Minimun genotype probability in GT_prior.
        """
        if beta_mu_prior is None:
            beta_mu_prior = np.expand_dims(
                np.linspace(0.01, 0.99, self.beta_mu.shape[1]), axis=0)
        if beta_sum_prior is None:
            beta_sum_prior = np.ones(beta_mu_prior.shape) * 50.0
        self.theta_s1_prior = beta_mu_prior * beta_sum_prior
        self.theta_s2_prior = (1 - beta_mu_prior) * beta_sum_prior

        if ID_prior is not None:
            if len(ID_prior.shape) == 1:
                ID_prior = np.expand_dims(ID_prior, axis=0)
            self.ID_prior = ID_prior
        else:
            self.ID_prior = normalize(np.ones(self.ID_prob.shape))

        if GT_prior is not None:
            if len(GT_prior.shape) == 2:
                GT_prior = np.expand_dims(GT_prior, axis=0)
            GT_prior[GT_prior < min_GP] = min_GP
            GT_prior[GT_prior > 1 - min_GP] = 1 - min_GP
            GT_prior = normalize(GT_prior)
            self.GT_prior = GT_prior
        else:
            self.GT_prior = normalize(np.ones(self.GT_prob.shape))

    @property
    def theta_s1(self):
        """Beta concetration1 parameter for theta posterior"""
        return self.beta_mu * self.beta_sum

    @property
    def theta_s2(self):
        """Beta concetration2 parameter for theta posterior"""
        return (1 - self.beta_mu) * self.beta_sum

    @property
    def digamma1_(self):
        """Digamma of Beta concetration1 parameter"""
        return np.expand_dims(digamma(self.theta_s1), 1)

    @property
    def digamma2_(self):
        """Digamma of Beta concetration2 parameter"""
        return np.expand_dims(digamma(self.theta_s2), 1)

    @property
    def digammas_(self):
        """Digamma of Beta concetration summary parameter"""
        return np.expand_dims(digamma(self.theta_s1 + self.theta_s2), 1)

    def update_theta_size(self, AD, DP):
        """Coordinate ascent for updating theta posterior parameters
        """
        BD = DP - AD
        S1_gt = AD * self.ID_prob  #(n_var, n_donor)
        S2_gt = BD * self.ID_prob  #(n_var, n_donor)

        _theta_s1 = np.zeros(self.beta_mu.shape)
        _theta_s2 = np.zeros(self.beta_mu.shape)
        _theta_s1 += self.theta_s1_prior.copy()
        _theta_s2 += self.theta_s2_prior.copy()
        for ig in range(self.n_GT):
            _axis = 1 if self.ASE_mode else None
            _theta_s1[:, ig:(ig+1)] += np.sum(
                S1_gt * self.GT_prob[:, :, ig], axis=_axis, keepdims=True)
            _theta_s2[:, ig:(ig+1)] += np.sum(
                S2_gt * self.GT_prob[:, :, ig], axis=_axis, keepdims=True)

        self.beta_mu = _theta_s1 / (_theta_s1 + _theta_s2)
        if self.fix_beta_sum == False:
            self.beta_sum = _theta_s1 + _theta_s2

    def update_ID_prob(self, AD, DP):
        """Coordinate ascent for updating assignment probability
        """
        BD = DP - AD
        logLik_ID = np.zeros((AD.shape[1], self.n_donor))
        for ig in range(self.n_GT):
            S1 = AD.transpose() * (self.GT_prob[:, :, ig] * self.digamma1_[:, :, ig])
            S2 = BD.transpose() * (self.GT_prob[:, :, ig] * self.digamma2_[:, :, ig])
            SS = DP.transpose() * (self.GT_prob[:, :, ig] * self.digammas_[:, :, ig])
            logLik_ID += (S1 + S2 - SS)

        self.ID_prob = normalize(np.exp(loglik_amplify(
            logLik_ID + np.log(self.ID_prior))))

        return logLik_ID

    def update_GT_prob(self, AD, DP):
        """Coordinate ascent for updating genotype probability
        """
        S1_gt = AD * self.ID_prob
        SS_gt = DP * self.ID_prob
        S2_gt = SS_gt - S1_gt

        logLik_GT = np.zeros(self.GT_prior.shape)
        for ig in range(self.n_GT):
            logLik_GT[:, :, ig] = (
                S1_gt * self.digamma1_[:, :, ig] +
                S2_gt * self.digamma2_[:, :, ig] -
                SS_gt * self.digammas_[:, :, ig])

        self.GT_prob = normalize(np.exp(loglik_amplify(
            logLik_GT + np.log(self.GT_prior))))

    def get_ELBO(self, logLik_ID, AD=None, DP=None):
        """Calculating variational evidence lower bound with current parameters

        logLik_ID: numpy array (n_cell, n_donor), the output from update_ID_prob
        """
        if logLik_ID is None:
            BD = DP - AD
            logLik_ID = np.zeros((AD.shape[1], self.n_donor))
            for ig in range(self.n_GT):
                S1 = AD.transpose() * (self.GT_prob[:, :, ig] * self.digamma1_[:, :, ig])
                S2 = BD.transpose() * (self.GT_prob[:, :, ig] * self.digamma2_[:, :, ig])
                SS = DP.transpose() * (self.GT_prob[:, :, ig] * self.digammas_[:, :, ig])
                logLik_ID += (S1 + S2 - SS)

        LB_p = np.sum(logLik_ID * self.ID_prob)
        KL_ID = -np.sum(entropy(self.ID_prob, self.ID_prior, axis=-1))
        KL_GT = -np.sum(entropy(self.GT_prob, self.GT_prior, axis=-1))
        KL_theta = -beta_entropy(
            np.append(
                np.expand_dims(self.theta_s1, 1),
                np.expand_dims(self.theta_s2, 1), axis = 1),
            np.append(
                np.expand_dims(self.theta_s1_prior, 1),
                np.expand_dims(self.theta_s2_prior, 1), axis = 1))

        # print(LB_p, KL_ID, KL_GT, KL_theta)
        return LB_p - KL_ID - KL_GT - KL_theta

    def fit(self, AD, DP, max_iter=200, min_iter=5, epsilon_conv=1e-2,
        delay_fit_theta=0, verbose=False):
        """Fit Vireo model with coordinate ascent

        Parameters
        ----------
        AD : scipy.sparse.csc_matrix (n_var, n_cell)
            Sparse count matrix for alternative allele
        DP : scipy.sparse.csc_matrix (n_var, n_cell)
            Sparse count matrix for depths, alternative + refeerence alleles
        max_iter : int
            Maximum number of iterations
        min_iter :
            Minimum number of iterations
        epsilon_conv : float
            Threshold for detecting convergence
        delay_fit_theta : int
            Number of steps to delay updating theta. This can be very useful
            for common genetics when there is good prior on allelic ratio.
        verbose : bool
            Whether print out log info
        """
        _binom_coeff = np.sum(get_binom_coeff(AD, DP))
        ELBO = np.zeros(max_iter)
        for it in range(max_iter):
            if self.learn_theta and it >= delay_fit_theta:
                self.update_theta_size(AD, DP)
            if self.learn_GT:
                self.update_GT_prob(AD, DP)

            _logLik_ID = self.update_ID_prob(AD, DP)
            ELBO[it] = self.get_ELBO(_logLik_ID) + _binom_coeff

            if it > min_iter:
                if ELBO[it] < ELBO[it - 1]:
                    if verbose:
                        print("Warning: Lower bound decreases!\n")
                elif it == max_iter - 1:
                    if verbose:
                        print("Warning: VB did not converge!\n")
                elif ELBO[it] - ELBO[it - 1] < epsilon_conv:
                    break
        self.ELBO_ = np.append(self.ELBO_, ELBO[:it])

    def predict_doublet(self, AD, DP, update_GT=True, update_ID=True,
        doublet_rate_prior=None):
        """Predict doublet with fitted Vireo model

        Parameters
        ----------
        AD : scipy.sparse.csc_matrix (n_var, n_cell)
            Sparse count matrix for alternative allele
        DP : scipy.sparse.csc_matrix (n_var, n_cell)
            Sparse count matrix for depths, alternative + refeerence alleles
        update_GT : bool
            Whether updating GT_prob after removing doublet_prob
        update_GT : bool
            Whether updating ID_prob by removing doublet_prob
        doublet_rate_prior : float
            Prior value of doublet rate

        Returns
        -------
        A tuple of two numpy arrays (doublet_prob, ID_prob)

        doublet_prob : numpy array (n_cell, n_donor * (n_donor - 1) / 2)
            Assignment probability of a cells to any doublet (donor pair)
        ID_prob : numpy array (n_cell, n_donor)
            updated ID_prob by removing doublet_prob
        """
        GT_both = add_doublet_GT(self.GT_prob)
        beta_mu_both, beta_sum_both = add_doublet_theta(self.beta_mu,
                                                        self.beta_sum)

        n_doublet_pair = GT_both.shape[1] - self.GT_prob.shape[1]
        if doublet_rate_prior is None:
            doublet_rate_prior = min(0.5, AD.shape[1] / 100000)

        ID_prior_both = np.append(
            self.ID_prior * (1 - doublet_rate_prior),
            np.ones((self.n_cell, n_doublet_pair)) / n_doublet_pair *
            doublet_rate_prior, axis=1)

        # Calculate assignment probability (same as update_ID_prob())
        BD = DP - AD
        logLik_ID = np.zeros((AD.shape[1], GT_both.shape[1]))
        _digamma1 = np.expand_dims(digamma(beta_sum_both * beta_mu_both), 1)
        _digamma2 = np.expand_dims(digamma(beta_sum_both * (1 - beta_mu_both)), 1)
        _digammas = np.expand_dims(digamma(beta_sum_both), 1)
        for ig in range(GT_both.shape[2]):
            S1 = AD.transpose() * (GT_both[:, :, ig] * _digamma1[:, :, ig])
            S2 = BD.transpose() * (GT_both[:, :, ig] * _digamma2[:, :, ig])
            SS = DP.transpose() * (GT_both[:, :, ig] * _digammas[:, :, ig])
            logLik_ID += (S1 + S2 - SS)

        ID_prob_both = normalize(np.exp(loglik_amplify(
            logLik_ID + np.log(ID_prior_both))))

        if update_ID:
            self.ID_prob = ID_prob_both[:, :self.n_donor]

        if update_GT:
            if update_ID:
                self.update_GT_prob(AD, DP)
            else:
                print("For update_GT, please turn on update_ID.")

        return ID_prob_both[:, self.n_donor:], ID_prob_both[:, :self.n_donor]

    def predit_ambient(self):
        """Predict fraction of ambient RNA contaimination.

        Not implemented yet.
        """
        print("Not implemented yet.")

def add_doublet_theta(beta_mu, beta_sum):
    """
    calculate theta for doublet genotype: GT=0&1, GT=0&2, and GT=1&2 by
    averaging thire beta paramters

    Example
    -------
    add_doublet_theta(np.array([[0.01, 0.5, 0.99]]), np.array([[30, 6, 30]]))
    """
    # TODO: support reduced GT for relatives
    combn_iter = itertools.combinations(range(beta_mu.shape[1]), 2)
    db_idx = np.array([x for x in combn_iter])

    beta_mu_db = (beta_mu[:, db_idx[:, 0]] + beta_mu[:, db_idx[:, 1]]) / 2.0
    beta_sum_db = np.sqrt(beta_sum[:, db_idx[:, 0]] * beta_sum[:, db_idx[:, 1]])

    return (np.append(beta_mu, beta_mu_db, axis=-1),
            np.append(beta_sum, beta_sum_db, axis=-1))

def add_doublet_GT(GT_prob):
    """
    Add doublet genotype by summarizing their probability:
    New GT has five categories: 0, 1, 2, 1.5, 2.5
    TODO: New GT has six categories: 0, 1, 2, 0_1, 0_2, 1_2
    """
    combn_iter = itertools.combinations(range(GT_prob.shape[2]), 2)
    gt_idx = np.array([x for x in combn_iter]) # GT combination
    g_idx1 = gt_idx[:, 0]
    g_idx2 = gt_idx[:, 1]

    combn_iter = itertools.combinations(range(GT_prob.shape[1]), 2)
    sp_idx = np.array([x for x in combn_iter]) # sample combination
    s_idx1 = sp_idx[:, 0]
    s_idx2 = sp_idx[:, 1]

    ## GT_prob has three genotypes: 0, 1, 2;
    n_gt = GT_prob.shape[2]
    GT_prob2 = np.zeros((GT_prob.shape[0], sp_idx.shape[0],
                         n_gt + gt_idx.shape[0]))

    GT_prob2[:, :, :n_gt] = (GT_prob[:, s_idx1, :] *
                             GT_prob[:, s_idx2, :])
    GT_prob2[:, :, n_gt:] = (GT_prob[:, s_idx1, :][:, :, g_idx1] *
                             GT_prob[:, s_idx2, :][:, :, g_idx2] +
                             GT_prob[:, s_idx1, :][:, :, g_idx2] *
                             GT_prob[:, s_idx2, :][:, :, g_idx1])

    GT_prob2 = normalize(GT_prob2, axis=2)
    GT_prob1 = np.append(GT_prob,
        np.zeros((GT_prob.shape[0], GT_prob.shape[1], gt_idx.shape[0])), axis=2)
    return np.append(GT_prob1, GT_prob2, axis=1)


<gh_stars>1-10
from .gen_random_input import generate_and_save_test_case, generate_input_pair
from . import run_algorithm
import multiprocessing
import numpy as np
import argparse
import csv
import random
from tqdm import tqdm
from copy import deepcopy
from pathlib import Path
import shutil
import json
import sys
import time
import shlex
from scipy.stats import binom
import bisect
from subprocess import TimeoutExpired
import math

default_bench_dir = Path(__file__).parent / "../test_cases/temp_bench"
flush_every_seconds = 5

parser = argparse.ArgumentParser(
    description="Benchmark our diff algorithm with random test cases"
)
subparsers = parser.add_subparsers(title="subcommands")
parser_prepare = subparsers.add_parser("prepare")
parser_plan_batch = subparsers.add_parser("plan-batch")
parser_run = subparsers.add_parser("run")

parser_prepare.add_argument(
    "--min-file-size",
    type=int,
    default=1,
    help="minimum length of a single generated input sequence",
)
parser_prepare.add_argument(
    "--max-file-size",
    type=int,
    default=5000,
    help="maximum length of a single generated input sequence",
)
parser_prepare.add_argument(
    "--target-file-size-steps",
    type=int,
    default=10,
    help="number of different file sizes to try while keeping all other parameters fixed",
)
parser_prepare.add_argument(
    "--max-change-strength",
    type=float,
    default=1,
    help="maximum change strength",
)
parser_prepare.add_argument(
    "--target-change-strength-steps",
    type=int,
    default=5,
    help="number of different change strengths to try while keeping all other parameters fixed",
)
parser_prepare.add_argument(
    "--only-change-strength",
    type=float,
    default=None,
    help="only change strength to benchmark with (--target-change-strength-steps must be 1 to use this option)",
)
parser_prepare.add_argument(
    "--target-chunkiness-steps",
    type=int,
    default=3,
    help="number of different chunkiness levels to try while keeping all other parameters fixed",
)
parser_prepare.add_argument(
    "--generation-strategies",
    type=str,
    default="independent,add,remove,addremove",
    help="comma separated list of strategies for generating inputs",
)
parser_prepare.add_argument(
    "--num-regens",
    type=int,
    default=3,
    help="number of times to re-generate an input using the same config",
)
parser_prepare.add_argument(
    "--output-dir",
    type=Path,
    default=default_bench_dir,
    help="path to directory where to write the generated inputs for benchmarking",
)

parser_plan_batch.add_argument(
    "--output-dir",
    type=Path,
    required=True,
    help="path to directory where to write the benchmark results as multiple CSV files",
)
parser_plan_batch.add_argument(
    "--job-start-command-format",
    default=r"bsub -n %procs% -R 'select[model==EPYC_7742]' -R 'rusage[mem=512]' -R 'span[ptile=128]' %command%",
    help=r"command for starting a single batch job. %procs% and %command% will be replaced.",
)
parser_plan_batch.add_argument(
    "--paths-relative-to",
    type=Path,
    default=Path.cwd(),
    help="refer to programs and files using paths relative to the supplied path",
)
parser_plan_batch.add_argument(
    "--mpi-procs",
    type=int,
    nargs="+",
    required=True,
    help="number of processes to run our distributed algorithm with. If multiple (space separated) numbers are supplied, every MPI program is benchmarked for each.",
)

parser_run.add_argument(
    "--output-csv",
    type=Path,
    required=True,
    help="path where to write the benchmark results as a CSV file",
)
parser_run.add_argument(
    "--no-progress-bar",
    default=False,
    action="store_true",
    help="hide the progress bar",
)
parser_run.add_argument(
    "--no-direct-mpi-procs-limit",
    default=False,
    action="store_true",
    help="don't pass -np to MPI run. Instead just assume that the number of processes is being limited to --mpi-procs somehow externally (eg. by a batch system). --mpi-procs is then effectively only written to the output CSV.",
)
parser_run.add_argument(
    "--mpi-procs",
    type=int,
    nargs="+",
    default=[None],
    help="number of processes to run our distributed algorithm with. If multiple (space separated) numbers are supplied, every MPI program is benchmarked for each.",
)

plan_batch_run_shared_args = set()

def add_plan_batch_run_shared_argument(arg_name, *args, **kwargs):
    assert arg_name.startswith("--")
    arg_name_snake = arg_name[2:].replace("-", "_")
    plan_batch_run_shared_args.add(arg_name_snake)
    for p in parser_plan_batch, parser_run:
        p.add_argument(arg_name, *args, **kwargs)

add_plan_batch_run_shared_argument(
    "--input-dir",
    type=Path,
    default=default_bench_dir,
    help="path to directory where to read the generated inputs for benchmarking",
)
add_plan_batch_run_shared_argument(
    "--limit-programs",
    type=str,
    help="comma separated list of programs to benchmark",
)
add_plan_batch_run_shared_argument(
    "--auto-repetitions",
    default=False,
    action="store_true",
    help="Determine number of repetions automatically based on the confidence interval around the median",
)
add_plan_batch_run_shared_argument(
    "--min-repetitions",
    type=int,
    default=5,
    help="minimum number of times to re-run each diff program with exactly the same input, without --auto-repetitions it is the actual number of repetitions",
)
add_plan_batch_run_shared_argument(
    "--max-repetitions",
    type=int,
    default=50,
    help="maximum number of times to re-run each diff program with exactly the same input, only active if --auto-repetitions is given",
)
add_plan_batch_run_shared_argument(
    "--max-median-error",
    type=float,
    default=0.05,
    help="maximal relative error of the median in the confidence interval, only active if --auto-repetitions is given",
)
add_plan_batch_run_shared_argument(
    "--mpi-timeout-seconds",
    type=float,
    default=60,
    help="when running an MPI algorithm, stop if it takes longer than this number of seconds for a single input",
)
add_plan_batch_run_shared_argument(
    "--skip-estimated-timeouts",
    default=False,
    action="store_true",
    help="Enable skipping of larger input sizes if a smaller size already timed out with the same diff program configuration\n"
    + "Should only be used with test cases with roughly the same number of edits relative to the file size (i.e. independent test cases)",
)
add_plan_batch_run_shared_argument(
    "--verbose",
    default=False,
    action="store_true",
    help="print detailed information when running each benchmark",
)

confidence_level = 0.95
min_repetitions_for_confidence = (
    6  # 6 for confidence level = 0.95, 8 for confidence level = 0.99
)

class CSVOutputWriter:
    def __init__(self, file):
        self._file = file
        self._writer = None

    def write_row(self, data: dict):
        if self._writer is None:
            self._writer = csv.DictWriter(self._file, data.keys())
            self._writer.writeheader()
        self._writer.writerow(data)

class NpEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, np.integer):
            return int(obj)
        elif isinstance(obj, np.floating):
            return float(obj)
        elif isinstance(obj, np.ndarray):
            return obj.tolist()
        else:
            return super(NpEncoder, self).default(obj)

class NoopProgressBar:
    def update(*args, **kwargs):
        pass

    def close(*args, **kwargs):
        pass

def get_diff_programs_for_args(args):
    diff_programs = run_algorithm.sequential_diff_programs.copy()
    for program_template in run_algorithm.mpi_diff_programs:
        for mpi_procs in args.mpi_procs:
            program = deepcopy(program_template)
            assert "extra_fields" not in "program"
            program["extra_fields"] = {
                "mpi_procs": mpi_procs,
                "timeout_seconds": args.mpi_timeout_seconds,
            }
            diff_programs.append(program)

    if args.limit_programs is not None:
        diff_programs = run_algorithm.limit_diff_programs(
            diff_programs,
            args.limit_programs,
            "unknown program names passed to --limit-programs",
        )

    return diff_programs

def get_generation_strategies_for_args(args):
    all_generation_strategies = {"independent", "add", "remove", "addremove"}
    selected_generation_strategies = {
        s.strip() for s in args.generation_strategies.strip().split(",")
    }
    unsupported_generation_strategies = (
        selected_generation_strategies - all_generation_strategies
    )
    if unsupported_generation_strategies:
        raise ValueError(
            f"unsupported values for --generation-strategies: {', '.join(sorted(unsupported_generation_strategies))}"
        )
    return sorted(selected_generation_strategies)

def prepare_benchmark(args):
    file_size_steps = np.linspace(
        args.min_file_size, args.max_file_size, args.target_file_size_steps
    )
    file_size_steps = np.unique(np.floor(file_size_steps).astype(int))

    if args.only_change_strength is not None:
        assert args.target_change_strength_steps == 1
        change_strength_steps = np.array([args.only_change_strength])
    else:
        change_strength_steps = np.linspace(
            0, args.max_change_strength, args.target_change_strength_steps + 1
        )[1:]

    chunkiness_steps = np.linspace(0, 1, args.target_chunkiness_steps)

    generation_strategies = get_generation_strategies_for_args(args)

    all_generation_configs = []
    for file_size in file_size_steps:
        for generation_strategy in generation_strategies:
            if generation_strategy == "independent":
                generation_config = {
                    "strategy": generation_strategy,
                    "length_1": file_size,
                    "change_strength": 1,
                    "chunkiness": 0,
                    "distribution": "zipf",
                }
                all_generation_configs.append(generation_config)

            else:
                for change_strength in change_strength_steps:
                    for chunkiness in chunkiness_steps:
                        generation_config = {
                            "strategy": generation_strategy,
                            "length_1": file_size,
                            "change_strength": change_strength,
                            "chunkiness": chunkiness,
                            "distribution": "zipf",
                        }
                        all_generation_configs.append(generation_config)

    print(f"{len(all_generation_configs)} unique test cases")

    shuffled_generation_configs = list(
        {"i": i, "config": c} for i, c in enumerate(all_generation_configs)
    )
    random.shuffle(shuffled_generation_configs)

    shutil.rmtree(args.output_dir, ignore_errors=True)
    args.output_dir.mkdir()
    with (args.output_dir / "index.json").open("w", encoding="utf-8") as f:
        json.dump(
            {
                "shuffled_generation_configs": shuffled_generation_configs,
                "num_regens": args.num_regens,
            },
            f,
            cls=NpEncoder,
        )

    assert args.num_regens >= 1
    gen_combination_factors = [len(all_generation_configs), args.num_regens]
    total_gen_combinations = np.prod(gen_combination_factors)
    print(
        f'{" * ".join(str(v) for v in gen_combination_factors)} = {total_gen_combinations} total input file pairs'
    )

    progress_bar = tqdm(total=total_gen_combinations)
    for entry in shuffled_generation_configs:
        for regen_i in range(args.num_regens):
            test_case_dir = args.output_dir / f"config-{entry['i']}-regen-{regen_i}"
            test_case_dir.mkdir()
            for i, values in enumerate(generate_input_pair(**entry["config"])):
                with (test_case_dir / f"in_{i + 1}.txt").open(
                    "w", encoding="utf8"
                ) as f:
                    for v in values:
                        f.write(f"{v}\n")
            progress_bar.update()

    progress_bar.close()

def plan_batch_benchmark(args):
    diff_programs = get_diff_programs_for_args(args)

    assert r"%procs%" in args.job_start_command_format
    assert r"%command%" in args.job_start_command_format

    def path_to_str(p):
        return str(p.resolve().relative_to(args.paths_relative_to.absolute()))

    def join_command(split_command):
        return " ".join(shlex.quote(arg) for arg in split_command)

    job_start_commands = []
    for program in diff_programs:
        mpi_procs = program.get("extra_fields", {}).get("mpi_procs", None)
        batch_procs = 1 if mpi_procs is None else mpi_procs
        assert isinstance(batch_procs, int)

        bench_command = [
            "python",
            "-m",
            "scripts.bench_algorithm",
            "run",
            "--limit-programs",
            program["name"],
            "--input-dir",
            path_to_str(args.input_dir),
            "--mpi-timeout-seconds",
            str(args.mpi_timeout_seconds),
            "--min-repetitions",
            str(args.min_repetitions),
            "--max-repetitions",
            str(args.max_repetitions),
            "--max-median-error",
            str(args.max_median_error),
            "--output-csv",
            path_to_str(args.output_dir / f'{program["name"]}_{batch_procs}.csv'),
        ]

        if mpi_procs is not None:
            bench_command += [
                "--mpi-procs",
                str(mpi_procs),
            ]

        if args.auto_repetitions:
            bench_command += ["--auto-repetitions"]

        if args.skip_estimated_timeouts:
            bench_command += ["--skip-estimated-timeouts"]

        if args.verbose:
            bench_command.append("--verbose")

        # use fullnodes -> make batch_procs multiple of 128
        batch_procs = -(-batch_procs // 128) * 128

        job_start_command = args.job_start_command_format.replace(
            r"%procs%",
            str(batch_procs),
        ).replace(
            r"%command%",
            shlex.quote(join_command(bench_command)),
        )

        job_start_commands.append(job_start_command)

    print(join_command(["mkdir", "-p", path_to_str(args.output_dir)]))
    for job_start_command in job_start_commands:
        print(job_start_command)

def run_benchmark(args):
    def verbose_print(*a, **kw):
        if args.verbose:
            print(*a, **kw)

    diff_programs = get_diff_programs_for_args(args)

    all_diff_program_extra_fields = sorted(
        {k for p in diff_programs for k in p.get("extra_fields", {}).keys()}
    )
    print(f"{len(diff_programs)} diff programs")

    try:
        with (args.input_dir / "index.json").open("r", encoding="utf-8") as f:
            benchmark_input_index = json.load(f)
    except Exception:
        print('Failed to load benchmark inputs. Did you run "prepare"?')
        raise
    shuffled_generation_configs = benchmark_input_index["shuffled_generation_configs"]
    num_regens = benchmark_input_index["num_regens"]
    assert num_regens >= 1

    test_combination_factors = [
        len(shuffled_generation_configs),
        num_regens,
        len(diff_programs),
        # don't count number of repetitions, because it is possibly dynamic
    ]
    total_test_combinations = np.prod(test_combination_factors)
    print(
        f'{" * ".join(str(v) for v in test_combination_factors)} = {total_test_combinations} total test combinations'
    )

    csv_output_file = open(args.output_csv, "w", newline="")
    csv_output_writer = CSVOutputWriter(csv_output_file)

    def get_extra_file_path(suffix):
        name = ".".join(args.output_csv.name.split(".")[:-1]) + suffix
        return args.output_csv.parent / name

    failed_file_path = get_extra_file_path("-FAILED.txt")
    failed_file = open(failed_file_path, "w")

    if args.auto_repetitions:
        num_repetitions = args.max_repetitions
    else:
        num_repetitions = args.min_repetitions

    if args.no_progress_bar:
        progress_bar = NoopProgressBar()
    else:
        progress_bar = tqdm(total=total_test_combinations, smoothing=0)

    if args.skip_estimated_timeouts:
        # record smallest input length with timeout per diff_program to skip larger test cases
        smallest_timeout = {}

    last_flush_time = time.monotonic()
    break_flag = False
    some_benchmarks_failed = False
    for _entry in shuffled_generation_configs:
        generation_config_i = _entry["i"]
        generation_config = _entry["config"]

        for regen_i in range(num_regens):
            verbose_print("generation_config", generation_config)
            test_case_dir = (
                args.input_dir / f"config-{generation_config_i}-regen-{regen_i}"
            )

            for diff_program in diff_programs:

                diff_prog_full_name = (
                    diff_program["name"]
                    + "_"
                    + str(diff_program.get("extra_fields", {}).get("mpi_procs", 1))
                )

                if (
                    args.skip_estimated_timeouts
                    and diff_prog_full_name in smallest_timeout
                    and generation_config["length_1"]
                    >= smallest_timeout[diff_prog_full_name]
                ):
                    some_benchmarks_failed = True
                    print(diff_prog_full_name + "\t", file=failed_file, end="")
                    print(generation_config, file=failed_file, end="")
                    print(
                        f"\t skipped due to estimated timeout, since length_1 {generation_config['length_1']} >= {smallest_timeout[diff_prog_full_name]}",
                        file=failed_file,
                    )
                    progress_bar.update()
                    continue

                # sorted list of measurements
                micros_until_len_res = []
                check_interval = min_repetitions_for_confidence - 1

                for repetition_i in range(num_repetitions):
                    if time.monotonic() - last_flush_time > flush_every_seconds:
                        csv_output_file.flush()
                        failed_file.flush()
                        last_flush_time = time.monotonic()

                    verbose_print("  diff_program", diff_program["name"])

                    extra_fields_for_output = {
                        k: diff_program.get("extra_fields", {}).get(k, "")
                        for k in all_diff_program_extra_fields
                    }

                    extra_fields_for_run = deepcopy(
                        diff_program.get("extra_fields", {})
                    )
                    if (
                        args.no_direct_mpi_procs_limit
                        and "mpi_procs" in extra_fields_for_run
                    ):
                        extra_fields_for_run["mpi_procs"] = None

                    try:
                        program_result = diff_program["run"](
                            test_case_dir / "in_1.txt",
                            test_case_dir / "in_2.txt",
                            extra_fields_for_run,
                        )
                        verbose_print(
                            "    micros_until_len", program_result.micros_until_len
                        )
                    except KeyboardInterrupt:  # exit the benchmark
                        break_flag = True
                        break
                    except TimeoutExpired as te:
                        some_benchmarks_failed = True
                        print(diff_prog_full_name + "\t", file=failed_file, end="")
                        print(generation_config, file=failed_file, end="")
                        print("\t" + repr(te), file=failed_file)
                        if args.auto_repetitions:
                            timeout_micros = te.timeout * 1e6  # seconds to microseconds
                            if (
                                repetition_i >= 5
                                and micros_until_len_res[0] == timeout_micros
                            ):
                                if args.skip_estimated_timeouts:
                                    smallest_timeout[
                                        diff_prog_full_name
                                    ] = generation_config["length_1"]
                                break  # if five iterations timed out -> assume all will timeout, don't try again
                            micros_until_len_res.append(timeout_micros)
                        else:
                            if args.skip_estimated_timeouts:
                                smallest_timeout[
                                    diff_prog_full_name
                                ] = generation_config["length_1"]
                        continue
                    except Exception as e:  # catch all
                        some_benchmarks_failed = True
                        print(diff_prog_full_name + "\t", file=failed_file, end="")
                        print(generation_config, file=failed_file, end="")
                        print("\t" + repr(e), file=failed_file)
                        break  # assumption: will always fail with these exceptions -> no need to run all repetitions

                    output_data = {
                        "generation_config_i": generation_config_i,
                        **{f"input_{k}": v for k, v in generation_config.items()},
                        "regen_i": regen_i,
                        "repetition_i": repetition_i,
                        "diff_program": diff_program["name"],
                        **extra_fields_for_output,
                        "mpi_comm_world": getattr(program_result, "mpi_comm_world", 1),
                        "micros_input": program_result.micros_input,
                        "micros_precompute": program_result.micros_precompute,
                        "micros_until_len": program_result.micros_until_len,
                        "micros_edit_script": program_result.micros_edit_script,
                        "min_edit_length": program_result.min_edit_length,
                    }

                    csv_output_writer.write_row(output_data)

                    if args.auto_repetitions:
                        bisect.insort(
                            micros_until_len_res, program_result.micros_until_len
                        )

                        if (
                            repetition_i >= args.min_repetitions
                            and repetition_i % check_interval == 0
                        ) or repetition_i == num_repetitions - 1:  # reached the last iteration
                            # check if required confidence interval is reached
                            if repetition_i % 2 == 0:  # odd number of results
                                current_median = micros_until_len_res[repetition_i // 2]
                            else:
                                current_median = (
                                    micros_until_len_res[(repetition_i - 1) // 2]
                                    + micros_until_len_res[(repetition_i + 1) // 2]
                                ) / 2

                            # check about every 20 ms = 20'000 microseconds  (overhead is about 1 ms)  => max 5% overhead
                            check_interval = math.ceil(20000 / current_median)

                            lower_idx, upper_idx = binom.interval(
                                confidence_level, repetition_i + 1, 0.5
                            )
                            # to get correct indices in python (Boudec paper Appendix A - 1)
                            lower_idx -= 1
                            # sometimes the interval is a little bit wider than in the Boudec paper, but this just means more confidence

                            if (
                                micros_until_len_res[int(lower_idx)]
                                >= (1 - args.max_median_error) * current_median
                                and micros_until_len_res[int(upper_idx)]
                                <= (1 + args.max_median_error) * current_median
                            ):
                                break

                            if repetition_i == num_repetitions - 1:
                                # failed to reach required confidence
                                some_benchmarks_failed = True
                                print(
                                    diff_prog_full_name + "\t",
                                    file=failed_file,
                                    end="",
                                )
                                print(generation_config, file=failed_file, end="")
                                print(
                                    "\t"
                                    + f"Failed to reach required confidence after {num_repetitions} repetitions; "
                                    + f"current median: {current_median}, left end of CI: {micros_until_len_res[int(lower_idx)]}, right end of CI: {micros_until_len_res[int(upper_idx)]}",
                                    file=failed_file,
                                )

                progress_bar.update()

                if break_flag:
                    break
            if break_flag:
                break
        if break_flag:
            break

    progress_bar.close()
    csv_output_file.close()
    failed_file.close()

    if not some_benchmarks_failed:
        failed_file_path.unlink()

parser_prepare.set_defaults(func=prepare_benchmark)
parser_run.set_defaults(func=run_benchmark)
parser_plan_batch.set_defaults(func=plan_batch_benchmark)

if __name__ == "__main__":
    if len(sys.argv) == 1:
        print("subcommand is required")
        parser.print_usage()
        exit(1)
    args = parser.parse_args()
    args.func(args)


<reponame>zekearneodo/ceciestunepipe<filename>ceciestunepipe/util/spikeextractors/preprocess.py
import numpy as np
import pandas as pd
import logging

from scipy.io import wavfile
from ceciestunepipe.file import bcistructure as et

from ceciestunepipe.util import sglxutil as sglu
from ceciestunepipe.util import rigutil as ru
from ceciestunepipe.util.spikeextractors.extractors.spikeglxrecordingextractor import spikeglxrecordingextractor as sglex

logger = logging.getLogger('ceciestunepipe.util.spikeextractors.preprocess')

def load_sglx_recordings(exp_struct: dict, epoch:str) -> tuple:
    # get the folders
    # make a pandas of the files
    # read into spikextractors
    # get the rig parameters
    # extract the wav channels of the nidaq signals (mics, syn)

    sgl_folders, sgl_files = sglu.sgl_file_struct(exp_struct['folders']['sglx'])
    #logger.info('loading files {}'.format(sgl_files))
    files_pd = pd.DataFrame(sgl_files)

    i_run = 0 #spikeglx epochs have the one run, this is for later
    run_meta_files = {k: v[i_run] for k, v in sgl_files.items()}
    run_recordings = {k: sglex.SpikeGLXRecordingExtractor(sglu.get_data_meta_path(v)[0]) for k, v in run_meta_files.items()}
    rig_dict = ru.get_rig_par(exp_struct)

    return run_recordings, run_meta_files, files_pd, rig_dict

def extract_nidq_channels(sess_par, run_recs_dict, rig_dict, chan_name_list, chan_type='adc') -> np.array:
    # get the channels id, numbers for the mic_list
    chan_n_list = [int(ru.lookup_signal(rig_dict, n)[1].split('-')[-1]) for n in chan_name_list]

    if chan_type=='adc':
        stream = run_recs_dict['nidq'].get_traces(channel_ids=chan_n_list)
    elif chan_type=='ttl':
        stream = run_recs_dict['nidq'].get_ttl_traces()[chan_n_list, :]
    else:
        raise NotImplementedError('dont know how to deal with {} channels'.format(chan_type))

    return stream

def save_wav(stream: np.array, s_f: np.float, wav_path: str) -> int:

    wav_s_f = int(round(s_f/1000))*1000
    logger.info('sampling rate {}'.format(wav_s_f))

    # write to wav
    logger.info('saving {}-shaped array as wav in {}'.format(stream.shape, wav_path))
    os.makedirs(os.path.split(wav_path)[0], exist_ok=True)
    wavfile.write(wav_path, wav_s_f, stream.T)

    ## also save as numpy
    npy_path = wav_path.split('.')[0] + '.npy'
    logger.info('saving {}-shaped array as npy in {}'.format(stream.shape, npy_path))
    np.save(npy_path, stream.T)
    return wav_s_f

def chans_to_wav(recording_extractor, chan_list: list, wav_path:str) -> int:
    # get the stream
    data_stream = recording_extractor.get_traces(channel_ids=chan_list)
    # make sure folder exists
    logger.info('saving {}-shaped array as wav in {}'.format(data_stream.shape, wav_path))
    os.makedirs(os.path.split(wav_path)[0], exist_ok=True)
    # write it
    s_f = int(round(recording_extractor.get_sampling_frequency()/1000))*1000
    logger.info('sampling rate {}'.format(s_f))
    wavfile.write(wav_path, s_f, data_stream.T)

    ## also save as numpy
    npy_path = wav_path.split('.')[0] + '.npy'
    logger.info('saving {}-shaped array as npy in {}'.format(data_stream.shape, npy_path))
    np.save(npy_path, data_stream.T)
    return s_f


<gh_stars>0
import numpy as np
from scipy.io import mmread

def is_pos_def(x):
    """check if a matrix is symmetric positive definite"""
    return np.all(np.linalg.eigvals(x) > 0)

def check_symmetric(x, tol=1e-8):
    return np.all(np.abs(x-x.T) < tol)

def precon(A):
    M_inv = np.diag(np.diag(A))
    for i in range(A.shape[0]):
        M_inv[i,i] = 1.0 / A[i,i]
    return M_inv

A = mmread('/home/anaxsouza/Documents/GitHub/coc757_Trabalho_05/bcsstk16.mtx').todense()
b = np.ones((A.shape[0], 1))
x = np.ones((A.shape[0], 1))
M_inv = precon(A)
i_max = 20000
epsilon = np.finfo(float).eps

i = 0
r = b - A @ x
z = M_inv @ r
delta = z.T @ r
delta_0 = delta

#if (is_pos_def(A) == False) | (check_symmetric(A) == False):
    #raise ValueError('Matrix A needs to be symmetric positive definite (SPD)')
while (i < i_max) and (delta > (epsilon ** 2)*delta_0):
    q = A @ z
    alpha = delta/(z.T @ q)
    alpha = alpha.item()
    x = x + alpha * z
    if (i % 50 == 0):
        print(i)
        r = b - A @ x
    else:
        r = r - alpha * q
    z = M_inv @ r
    delta = z.T @ r
    i += 1
print('\ni:\n',i)
print('\nx:')
print(x)
print(x.shape)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Jan 26 12:23:55 2019

@author: jxf

"""
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import datetime
#has incomplete data. 999 points are NaN
def read_file(fname):
    with open(fname, "r") as f:
        try:
            line1= f.readline().split()
        except Exception as e:
            raise Exception("problem reading first line of file %s"%fname) from e
        if line1[0] not in ("YY","#YY","YYYY"):
            raise Exception("bad header line 1 for file %s: '%s'"%(fname,line1))
        try:
            line2= f.readline().split()
        except Exception as e:
            raise Exception("problem reading second line of file %s"%fname) from e
        try:
            int(line2[0])
            has_second_header= False
            header= 0
        except ValueError:
            if line2[0]in ("#yy","#yr"):
                has_second_header=True
                header=[0,1]
            else:
                raise Exception("unexpected second header in file %s"%fname)

    #this gives it a second header
    df= pd.read_csv(fname, sep='\s+', header= header)#allows you to read the file
    if has_second_header:
        df.columns = [h1 for (h1, h2) in df.columns]
    def mkyear(v):
        v = int(v)
        if v<100:
            return 1900 + v
        else:
            return v
    if 'mm' in df.columns:
        df['timestamp']=df.apply(lambda s:datetime.datetime(mkyear(s[0]), int(s[1]), int(s[2]), int(s[3]), int(s[4])),
                                 axis=1)
    else:
        df['timestamp']=df.apply(lambda s:datetime.datetime(mkyear(s[0]), int(s[1]), int(s[2]), int(s[3]), 0),
                                 axis=1)
    df['ATMP'] = df['ATMP'].apply(lambda v:np.NaN if v>100 else v) # 999 is used to indicate no data available
    df['WTMP'] = df['WTMP'].apply(lambda v:np.NaN if v>100 else v) # 999 is used to indicate no data available
    print("%s has %d entries" % (fname, len(df)))
    return df

def build_median_df(df, base_col, year,
                    index=['01-Jan', '02-Feb', '03-Mar', '04-Apr', '05-May', '06-Jun',
                           '07-Jul', '08-Aug', '09-Sep', '10-Oct', '11-Nov', '12-Dec']):
    if 'YY' in df.columns:
        df = df[(df['YY']==year) | (df['YY']==(year-1900))].copy()
    elif '#YY' in df.columns:
        df = df[df['#YY']==year].copy()
    elif 'YYYY' in df.columns:
        df = df[df['YYYY']==year].copy()
    else:
        assert 0, "Did not find a year column in %s for %s" % (df.columns, year)
    grouped=df.groupby(pd.Grouper(key = "timestamp", freq="M")).agg({base_col:['median']})
    grouped.columns=['%s %s median'%(year, base_col)]
    grouped['month'] = index
    grouped.set_index('month', drop=True, inplace=True)
    return grouped

def load_preprocessed_file(buoyno):
        filename="../intermediate-data/processed_%s.csv.gz"%buoyno
        return pd.read_csv(filename,parse_dates=["timestamp"],
               index_col="timestamp", usecols=["ATMP","WTMP","timestamp"],header=0)

def get_monthly_averages(df):
    monthly=df.resample("M").mean()
    monthly.index = [pd.Period(str(ts)[0:7]) for ts in monthly.index]
    return monthly

MONTH_NAMES = {1:'Jan', 2:'Feb', 3:'Mar', 4:'Apr', 5:'May', 6:'Jun',
               7:'Jul', 8:'Aug', 9:'Sep', 10:'Oct', 11:'Nov', 12:'Dec'}
def monthly_computation(monthly, monthno, field):
    mask = monthly.index.map(lambda x:x.month)==monthno
    anomaly = monthly[mask][field] - (monthly[mask][field].mean())
    anomaly.index = [pd.Period(year=int(str(ym)[0:4]), freq='Y')
                     for ym in anomaly.index]
    return anomaly.rename(MONTH_NAMES[monthno])

def compute_anomalies(monthly, field):
    return pd.DataFrame([monthly_computation(monthly, m, field)
                         for m in range(1, 13)])

def plot_anomaly_graph(buoyno, temptype, anomalies):
    yearly_means = anomalies.mean()
    import scipy.stats
    slope, intercept, r_value, p_value, std_err = scipy.stats.linregress([i for (i, y) in enumerate(yearly_means.index)], yearly_means)
    fit_type = 'least squares fit'
    if np.isnan(slope):
        # If we cannot infer a straight line, just connect the endpoints
        print("Unable to fit a line")
        first_year = yearly_means.index[0]
        last_year = yearly_means.index[-1]
        print("Creating a line just using the endpoint years (%s, %s)" %
              (first_year, last_year))
        (slope, intercept) = np.polyfit([0, last_year-first_year], [yearly_means[0], yearly_means[-1]], 1)
        fit_type = 'endpoint fit'
    values = [i*slope+intercept for i in range(len(yearly_means.index))]
    linear_series = pd.Series(data=values, index=yearly_means.index, name='linear fit')
    pd.DataFrame({'yearly anomaly':yearly_means, fit_type:linear_series}).plot(figsize=(12,10));
    plt.scatter(yearly_means.index, yearly_means)
    plt.title('Yearly mean anomaly %s temperature for buoy %s (slope=%0.2f degrees/decade)' %
              (temptype, buoyno, slope*10));
    plt.ylabel('Degrees C');
    plt.savefig('../results/%s-%stemp-anomly.pdf' % (buoyno, temptype))
    plt.savefig('../results/%s-%stemp-anomly.png' % (buoyno, temptype))
    return slope*10 # the temp anomaly change per decade in degrees C


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jun 10 07:15:59 2020

 @author: <NAME>

Description: Code for testing different datasets (It is all in the noise) and ISO NE
with different multitask GP configurations from the original code of KronSum

 """

import math
import sys
import numpy as np
import numpy.matlib
import time
import scipy as SP
import os
import torch
import gpytorch
from matplotlib import pyplot as plt
import pathlib as Path
from os import listdir
import pandas as pd
#from  histogram_errors import  histogram_errors

ProjectPath = Path.Path.cwd()
utilsPath = Path.Path.joinpath(ProjectPath,"utils")
probsUtilsPath = Path.Path.joinpath(ProjectPath,"Prob-utils")
ResultsPath = Path.Path.joinpath(ProjectPath,"Results")

UTIL_DIR = utilsPath
sys.path.append(
    str(UTIL_DIR)
)

UTIL_DIR_GEN = probsUtilsPath
sys.path.append(
    str(UTIL_DIR_GEN)
)

RESULTS_DIR_GEN = ResultsPath
sys.path.append(
    str(RESULTS_DIR_GEN)
)

from EvaluateConfidenceIntervals import EvaluateConfidenceIntervals
from StandarizeData import StandarizeData
from DeStandarizeData import DeStandarizeData
from MAPE import MAPE
from GP24I_v4 import GP24I
from GPind import GPind
from GPind_ori import GPind_ori
from predGPind_ori import predGPind_ori
from GPind_lap import GPind_lap
from predGPK import predGPK
from predGPind_lap import predGPind_lap
from GPKtorch import GPKtorch
from predGPind import predGPind
from load_obj import load_obj
from save_obj import save_obj
from sklearn.metrics import r2_score
from data_to_torch import data_to_torch
from norm2laplace import norm2laplace
from EvaluateConfidenceIntervals_Laplace import EvaluateConfidenceIntervals_Laplace
from outliers_removal import outliers_removal
from load_configuration import load_configuration
from load_configuration_job_array import load_configuration_job_array
from print_configuration import print_configuration
from correcting_factor_cov import correcting_factor_cov
from correcting_factor_cov_gamma import correcting_factor_cov_gamma
from predictive_variance_white import predictive_variance_white
from  print_extra_methods import  print_extra_methods
from GP24I_v4 import GP24I
from GPMT import GPMT
from to_torch import to_torch
from predGPMT import predGPMT
from print_results_ic import print_results_ic
# #Load Power Load Data =========================================================
# #==============================================================================
method = "NMF"  # Full
methodfile = 'NMF'
kernel_type = "rbf"
forecast_method = "gpmt" # gp_ind_ori/gp_ind/gpk/gp_ind_laplace/gpmt
option_lv = "gp_ind_ori" # gp_ind_ori/gpmt

EXPERIMENT = 3  # This has to do with the verion of the NMF generated
TaskNumber = 24
Stand = True
#folder_data_name = "Exp_"+str(EXPERIMENT)
folder_data_name = "BuenosResNMF"
#LOCATIONS = ['ME','CT','NH','RI','NEMASSBOST','SEMASS','VT','WCMASS']

datapath = Path.Path.joinpath(ProjectPath,"Data",folder_data_name)
#datapath = Path.Path.joinpath(ProjectPath,"Data",folder_data_name,"NMF")
DATAPATH = str(datapath)
onlyfilesALL = [f for f in listdir(DATAPATH) if f.endswith('.pkl')]

#[onlyfiles,opt_parameters,forecast_method] = load_configuration(sys.argv,onlyfilesALL,forecast_method)
[onlyfiles,opt_parameters, forecast_method, option_lv,location,lr1,trainsize] = load_configuration_job_array(sys.argv,onlyfilesALL)
if forecast_method == "gpk":
    name_forecast_method = forecast_method +"_" +option_lv
else:
    name_forecast_method = forecast_method

gpytorch.settings.max_cg_iterations._set_value(10000)

RESULTS = {}
for archivo in range(len(onlyfiles)):
    Results = {'R224': [],'mapes':[],'mapemedio':[],'training_time':[],'test_time':[],
               'Ypred':[],'Vpred':[],'likelihood':[],'ICs':[],'ICs_lap1':[],'ICs_lap2':[],'gpk':[]}

    # LOAD DATA================================================================
    file_name = onlyfiles[archivo]
    file_path = Path.Path.joinpath(datapath,file_name)
    FILE_PATH = str(file_path)
    DATA = load_obj(FILE_PATH)
    DATA = data_to_torch(DATA)
    print(FILE_PATH)

    XTrain = DATA['X_Train_Val'].T    # N x F ### torch.from_numpy
    YTrain = DATA['Y_Train_Val']
    XTest = DATA['X_Test'].T           # N x F
    YTest = DATA['Y_Test']           # N x K
    YTest_24 = DATA['Y_Test_24']     # N x T
    YTrain_24 = DATA['Y_Train_Val_24']
    TaskNumber = np.size(DATA['Wtrain_load'],1)
    WTrain = to_torch(DATA['Wtrain_load'])
    Stds_train_load = DATA['Stds_train_load']
    Ntest = np.size(YTest_24,0)
    Ntrain = np.size(YTrain_24,0)

    #[XTrain,XTest,YTrain_24,YTest_24] = outliers_removal(XTrain,XTest,YTrain_24,YTest_24)

    # nn = 10
    # YTrain_24_std = np.divide(YTrain_24,np.matlib.repmat(Stds_train_load.T,Ntrain,1))
    # YTrain24M  = YTrain_24[0:nn,:]
    # YTrainstd24M  = YTrain_24_std[0:nn,:]
    # XTrainM = XTrain[0:nn,:]
    # YTrainM = YTrain[0:nn,:]
    # XTrain  = XTrainM
    # YTrain = YTrainM
    # YTrain_24 = YTrain24M
    # YTrain_24_std = YTrainstd24M

    # NORMATLIZATION================================================================

    if forecast_method == "gpk":
        [XTrain_S, YTrain_K_S , XTest_S, YTest_K_S,scalerX, scalerY_K]=StandarizeData(XTrain,YTrain, XTest,YTest,Standarize = Stand)
    else:
        [XTrain_S, YTrain_24_S , XTest_S, YTest_24_S,scalerX, scalerY_24]=StandarizeData(XTrain,YTrain_24, XTest,YTest_24,Standarize = Stand)

    start = time.time()
    # TRAINING================================================================
    #==========================================================================
    if forecast_method == "gp_ind_ori":
        [M,L,RESULTS,model,like] = GPind_ori(XTrain_S,YTrain_24_S,24,kernel_type,opt_parameters)
    #elif forecast_method == "gpk":
    end = time.time()
    training_time = end-start
    #=========================================================================
    if forecast_method == "gpk":
        K = YTrain.size(1)
        [M,L,RESULTS,model,like,ind_val] = GPKtorch(XTrain_S,YTrain_K_S,WTrain,K,kernel_type,option_lv,opt_parameters)

        #kernel =  C(10, (0.1, 200))*RBF(10, (10,200)) +C(1e-1, (1e-7, 10))  + WhiteKernel(noise_level=1e-3, noise_level_bounds=(1e-8, 1))
        #[YPredictedS_24gpS, VPredictedS_24gpS,model,Opt_alpha, IC1, IC2, Errors, R2s, MAPEs,VarsALL,Errors_train,R2strain,Error2Validation,ErrorValidation,NoiseParameters,YvalsOpt,Y_predValsOpt,Covpredicted_Best,training_time,val_time,test_time] = GP24I(XTrainS,YTrainS,XTestS,kernel,TaskNumber,Alphas)
    end = time.time()
    training_time = end-start
    #==========================================================================
    if forecast_method == "gpmt":
        K = YTrain.size(1)
        [M,L,RESULTS,model,like,_,_] = GPMT(XTrain,YTrain_24,24,kernel_type,opt_parameters)
    end = time.time()
    # TESTING==================================================================
    #==========================================================================
    start = time.time()
    if forecast_method == "gp_ind_ori":
        [YPredicted_24gp_S,VPredicted_24gp_S] = predGPind_ori(XTest_S,like,model)
    end = time.time()
    testing_time = end-start
    #=========================================================================
    if forecast_method == "gpk":
        [YPredictedS_KgpS,VPredicted_Kgp_S] = predGPind_ori(XTest_S,like,model)
        [_, YPredicted_24gp_K,VPredicted_24gp_K]=DeStandarizeData(YTest_K_S,YPredictedS_KgpS,scalerY_K,VPredicted_Kgp_S,Standarize = Stand)
        #YPredictedS_KgpS,VPredicted_Kgp_S] = predGPK(YPredicted_24gp_K,VPredicted_Kgp_S,WTrain,Stds_train_load = Stds_train_load)
    end = time.time()
    testing_time = end-start
    if forecast_method == "gpmt":
        [YPredicted_24gp_S,VPredicted_24gp_S] = predGPMT(XTest_S,like,model)
        [_, YPredicted_24gp_K,VPredicted_24gp_K]=DeStandarizeData(YTest_24_S,YPredicted_24gp_S,scalerY_24,VPredicted_24gp_S,Standarize = Stand)
        #YPredictedS_KgpS,VPredicted_Kgp_S] = predGPK(YPredicted_24gp_K,VPredicted_Kgp_S,WTrain,Stds_train_load = Stds_train_load)
    end = time.time()
    testing_time = end-start
    #=========================================================================

#==============================================================================
#==============================================================================
#==============================================================================
    print_configuration(file_name,name_forecast_method,kernel_type,EXPERIMENT,Stand,folder_data_name)
    if forecast_method == "gpk":
        # TRANSFORMATION====
        S2norm = torch.pow(Stds_train_load,2)
        Snorm = Stds_train_load.T.repeat(Ntest,1)
        Snorm_tr  = Stds_train_load.T.repeat(Ntrain,1)
        #ErrorValidation_std_P = torch.stack(RESULTS['ValidationPredictiveErrors'],dim =1)
        YPredicted_24gp = (YPredicted_24gp_K@WTrain.T)*Snorm
        VPredicted_24gp = torch.zeros((Ntest,24))
        # if 'ValidationErrors' in RESULTS:
        #     ErrorValidation_std = torch.stack(RESULTS['ValidationErrors'],dim =1)
        #     Nval = ErrorValidation_std.size(0)
        #     Snorm_val = Stds_train_load.T.repeat(Nval,1)
        #     NoiseEstimation_Variance3  = torch.var((ErrorValidation_std@WTrain.T)*Snorm_val,axis=0)
        #     ind_a = np.random.permutation(range(0,Ntrain))[0:100]
        #     a = correcting_factor_cov(model,WTrain,YTrain_24[ind_a,:],XTrain_S[ind_a,:],option_lv,scalerY_K,NoiseEstimation_Variance3,Stds_train_load )
        #     a_gamma = correcting_factor_cov_gamma(model,WTrain,YTrain_24[ind_a,:],XTrain_S[ind_a,:],option_lv,scalerY_K,NoiseEstimation_Variance3,Stds_train_load )
        #     for ss in range(0,Ntest):
        #         VPredicted_24gp[ss,:] = (torch.diag(WTrain@torch.diag(VPredicted_24gp_K[ss,:])@WTrain.T)*(S2norm.ravel())  +  NoiseEstimation_Variance3)*a
        #     VPredicted_24gp_white = predictive_variance_white(VPredicted_24gp_K,WTrain,NoiseEstimation_Variance3,S2norm)
        print_extra_methods(Stds_train_load,Ntest,Ntrain,WTrain,YTrain_24,YTest_24,XTrain_S,YPredicted_24gp_K,VPredicted_24gp_K,option_lv,scalerY_K,RESULTS,model,DATA)
    elif forecast_method == "gp_ind_ori":
        [_, YPredicted_24gp,VPredicted_24gp] = DeStandarizeData(YTest_24_S,YPredicted_24gp_S,scalerY_24,VPredicted_24gp_S,Standarize = Stand)
        [ICs,ICs_lap1,ICs_lap2] = print_results_ic(YPredicted_24gp,YTest_24,VPredicted_24gp,"gp_ind_ori")
    elif forecast_method == "gpmt":
        [_, YPredicted_24gp,VPredicted_24gp] = DeStandarizeData(YTest_24_S,YPredicted_24gp_S,scalerY_24,VPredicted_24gp_S,Standarize = Stand)
        [ICs,ICs_lap1,ICs_lap2] = print_results_ic(YPredicted_24gp,YTest_24,VPredicted_24gp,"gpmt")
    # METRICS==================================================================

    #histogram_errors("",YPredicted_24gp,YTest_24,VPredicted_24gp)

    mapes= MAPE(YTest_24,YPredicted_24gp)
    mapemedio = torch.mean(mapes)

    NTest = np.size(YTest_24,0)
    R2_all = np.zeros((NTest,1))

    for samp in range(0,NTest):
        R2_all[samp,0] = r2_score(YTest_24[samp,:],YPredicted_24gp[samp,:])
    r2_24gp  = np.mean(R2_all)

    # PRINT===================================================================

    print('Mape Medio 24GPs indep   ', mapemedio )
    print('R2 24GPs i:    ',r2_24gp)

    if 'ValidationErrors' in RESULTS:
        Lval = RESULTS['ValidationErrors']
        Lval_tasks = [torch.mean(x) for x in Lval]
        Lval_mean = torch.mean(torch.tensor(Lval_tasks))
        print('Mean validation loss ',Lval_mean)

    print('Training time:   ', training_time )
    print('Test time:   ',  testing_time)
    #==========================================================================
    Results['R224'] = r2_24gp
    Results['mapes'] = mapes
    Results['mapemedio'] = mapemedio
    Results['training_time'] =   training_time
    Results['test_time'] = testing_time
    Results['Ypred'] = YPredicted_24gp
    Results['Vpred'] = VPredicted_24gp
    Results['likelihood'] = like
    #Results['ICs'] = ICs
    #Results['ICs_lap1'] = ICs_lap1
    #Results['ICs_lap2'] = ICs_lap2

    if forecast_method == "gpk":
        Results['Wtrain'] = WTrain
    RESULTS[archivo] = Results

    file_name = "Exp_"+str(EXPERIMENT)+"_lr_"+str(lr1)+"+_trainsize_"+str(trainsize)+"_loc_"+location
    if 'INFO' in locals():
        file_name = "Exp_"+str(EXPERIMENT)+"_lr_"+str(lr1)+"+_trainsize_"+str(trainsize)+"_loc_"+location
    file_results = Path.Path.joinpath(ResultsPath,file_name+"_results")
    file_model = Path.Path.joinpath(ResultsPath,file_name+"_model")
    file_data = Path.Path.joinpath(ResultsPath,file_name+"_data")
    save_obj(RESULTS, file_results.as_posix())
    save_obj(model, file_model.as_posix())
    save_obj(DATA, file_data.as_posix())