<reponame>gitter-badger/mlmodels<filename>mlmodels/model_tch/vae/util.py

import os, sys
import numpy as np
import scipy as sci
import matplotlib.pyplot as plt
import pandas as pd
import cv2

"""
functionality: sine wave npz generation and image gerneration
"""

# default image shape 64x64x3
# default npz element size ([sample number], 64, 64)
data = {'resolution': 64, 'amplitude': 5, 'img_relative_folder':"/sinus_img/verbose", \
  'npz_relative_folder':"/sinus_npz", 'npz_name': "sinus.npz", \
  'img_cv_folder':"/sinus_img_cv/verbose", 'npz_cv_folder':"/sinus_npz_cv", 'npz_cv_name': "sinus_cv.npz"}

def set_resolution(resolution = 64):
  data['resolution'] = resolution

def get_resolution():
  return data['resolution']

# sinus: y = a * sin(w * t)
def generate_random_sin(n_rand_starts = 100, amplitude = 1, n_pis = 4, omega = 1, step = 0.2):
  r = np.random.randint(n_rand_starts)
  x = np.arange(r, r + n_pis*np.pi, step)
  y = amplitude * np.sin(omega * x)
  return x,y

# cosinus: y = a * cos (w * t + b) + c
def generate_random_cos(n_rand_starts = 1, a = 1, w= 1, b = 0, c = 0, x_upbound =1, x_downbound = -1, step = 0.2):
  r = np.random.randint(n_rand_starts)
  x = np.arange(x_downbound*2*np.pi+r, x_upbound*2*np.pi+r, step)
  y = a * np.cos(w * x + b) + c

  return x,y

# opencv: create wave image as numpy array
def create_sin_2d_array_cv(x, y, resoltuion = data['resolution'],amp=data['amplitude']):
  size = len(x), len(y), 3
  linewidth = int(len(y)/resoltuion + 0.5)
  vis = np.zeros(size, dtype=np.uint8)
  new_y = y.copy()
  # amplitude set here for plot
  y_max = amp
  y_min = -1*amp
  border = 16
  ratio = float((len(y)-border) /( y_max - y_min))
  for i in range(len(y)):
    new_y[i] = int(border/2+(len(y)-border)-1-(y[i]-y_min)*ratio)

  pointList = []
  for i in range(int(len(x))):
    pointList.append((i,int(new_y[i])))
  pointList = np.array(pointList)

  cv2.polylines(vis,  [pointList],  False,  (255,255,255),  linewidth)

  vis = cv2.resize(vis, (resoltuion, resoltuion), interpolation=cv2.INTER_CUBIC)
  # threshold as 50
  result = np.where(vis[:,:,0] > 50, 1, 0)
  return result

# opencv: create wave image save as images to disk
def plot_save_disk_cv(x, y, filename, xmax=data['resolution'], ymax=data['resolution'],amp=data['amplitude']):
  size = len(x), len(y), 3
  linewidth = int(len(y)/ymax + 0.5)
  vis = np.ones(size, dtype=np.uint8)
  vis = vis * 255
  new_y = y.copy()
  y_max = amp
  y_min = -1*amp
  border = 16
  ratio = float((len(y)-border) /( y_max - y_min))
  for i in range(len(y)):
    new_y[i] = int(border/2+(len(y)-border)-1-(y[i]-y_min)*ratio)
  pointList = []
  for i in range(int(len(x))):
    pointList.append((i,int(new_y[i])))
  pointList = np.array(pointList)
  cv2.polylines(vis,  [pointList],  False,  (0,0,0),  linewidth)
  vis = cv2.resize(vis, (xmax, ymax), interpolation=cv2.INTER_CUBIC)
  cv2.imwrite(filename, vis)

# matplotlit: create wave image as numpy array
def create_sin_2d_array_plt(x, y, xmax=data['resolution'], ymax=data['resolution'],amp=data['amplitude']):
  plt.rcParams['axes.facecolor']='white'
  plt.rcParams['savefig.facecolor']='white'
  fig = plt.figure(frameon=False, figsize=(xmax, ymax), dpi=1)
  ax = plt.Axes(fig, [0., 0., 1., 1.])
  # amplitude set here for plot
  ax.set_ylim([-1*amp,1*amp])
  ax.set_axis_off()
  fig.add_axes(ax)

  plt.plot(x,y, c="black", linewidth=100)
  fig.canvas.draw()
  frame = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8)
  frame = frame.reshape(fig.canvas.get_width_height()[::-1] + (3,))
  plt.clf()
  plt.close('all')
  # frame is binary
  result = np.where(frame[:,:,0] > 254, 0, 1)
  return result

# matplotlib: create wave image save as images to disk
def plot_save_disk(x, y, filename, xmax=data['resolution'], ymax=data['resolution'], amp=data['amplitude']):
  fig = plt.figure(frameon=False, figsize=(xmax, ymax), dpi=1)

  ax = plt.Axes(fig, [0., 0., 1., 1.])
  ax.set_ylim([-1*amp,1*amp])
  ax.set_axis_off()
  fig.add_axes(ax)

  plt.plot(x,y, c="black", linewidth=100)
  fig.savefig(filename)
  plt.clf()
  plt.close('all')

# matplotlib: images saves to /path/sinus_img/verbose/*.png
def generate_train_img(folder, N_type=1, amax=5, wmin=5, wmax=10, bmin=-2, bmax=2, cmin=-2 ,cmax=2 , step = 0.1, wfreq=0.5 ) :
    folder_w_subfolder = folder + data['img_relative_folder']
    os.makedirs(folder, exist_ok=True)

    os.makedirs(folder_w_subfolder, exist_ok=True)
    folder = os.path.abspath(folder_w_subfolder)

    for type_i in range(N_type):
      for amp_int in range(1, amax*2+1):
        amp_i = amp_int*0.5
        for omega_i in range(wmin, wmax, 1):
          omega_ii = wfreq * omega_i
          for b_i in range(bmin, bmax, 1):
            for c_i in range(cmin, cmax, 1):
              # use sinus gernerate:
              # x,y = generate_random_sin(N_type, amp_i, 4, omega_ii, step)
              # use cosinus gernerate:
              x,y = generate_random_cos(n_rand_starts=N_type,a=amp_i, w=omega_ii, b = b_i, c = c_i, step = step)
              filename = '{folder}/sin_{amp_i}-{omega_ii}-{b_i}-{c_i}-{type_i}'.format(folder=folder, amp_i=amp_i, omega_ii=omega_ii, b_i=b_i, c_i=c_i,type_i=type_i).replace(".","_")
              filename = filename + ".png"
              plot_save_disk(x,y, filename, xmax = data['resolution'], ymax = data['resolution'], amp = data['amplitude'])

# matplotlib: images saves to /path/sinus_npz/verbose/sinus.npz
def generate_train_npz(folder, N_type=1, amax=5, wmin=5, wmax=10, bmin=-2, bmax=2, cmin=-2 ,cmax=2 , step = 0.1, wfreq=0.5, resoltuion = data['resolution'] ) :
    folder_w_subfolder = folder + data['npz_relative_folder']
    # inital with empty numpy which is random
    generate_npy = [np.empty([resoltuion, resoltuion], dtype=int)]
    os.makedirs(folder, exist_ok=True)
    os.makedirs(folder_w_subfolder, exist_ok=True)
    folder = os.path.abspath(folder)
    is_inital = False

    for type_i in range(N_type):
      for amp_int in range(1, amax*2+1):
        amp_i = amp_int*0.5
        for omega_i in range(wmin, wmax, 1):
          omega_ii = wfreq * omega_i
          for b_i in range(bmin, bmax, 1):
            for c_i in range(cmin, cmax, 1):
              # use sinus gernerate:
              # x,y = generate_random_sin(N_type, amp_i, 4, omega_ii, step)
              # use cosinus gernerate:
              x,y = generate_random_cos(n_rand_starts=N_type,a=amp_i, w=omega_ii, b = b_i, c = c_i, step = step)
              if len(generate_npy) == 1 and is_inital == False:
                # replace the random array with first data, only do once
                generate_npy = [create_sin_2d_array_plt(x,y)]
                is_inital = True
              else:
                generate_npy=np.append(generate_npy, [create_sin_2d_array_plt(x,y)],axis=0)
    np.savez(folder_w_subfolder+ "/"+ data['npz_name'], sinus=generate_npy)
    print("npz saved")

# opencv_version: images saves to /path/sinus_img_cv/verbose/*.png
def generate_train_npz_cv(folder, N_type=1, amax=5, wmin=5, wmax=10, bmin=-2, bmax=2, cmin=-2 ,cmax=2 , step = 0.1, wfreq=0.5, resoltuion = data['resolution'] ) :
    folder_w_subfolder = folder + data['npz_cv_folder']
    # inital with empty numpy which is random
    generate_npy = [np.empty([resoltuion, resoltuion], dtype=int)]
    os.makedirs(folder, exist_ok=True)
    os.makedirs(folder_w_subfolder, exist_ok=True)
    folder = os.path.abspath(folder)
    is_inital = False

    for type_i in range(N_type):
      for amp_int in range(1, amax*2+1):
        amp_i = amp_int*0.5
        for omega_i in range(wmin, wmax, 1):
          omega_ii = wfreq * omega_i
          for b_i in range(bmin, bmax, 1):
            for c_i in range(cmin, cmax, 1):
              # use sinus gernerate:
              # x,y = generate_random_sin(N_type, amp_i, 4, omega_ii, step)
              # use cosinus gernerate:
              x,y = generate_random_cos(n_rand_starts=N_type,a=amp_i, w=omega_ii, b = b_i, c = c_i, step = step)

              if len(generate_npy) == 1 and is_inital == False:
                # replace the random array with first data, only do once
                generate_npy = [create_sin_2d_array_cv(x,y)]
                is_inital = True
              else:
                generate_npy=np.append(generate_npy, [create_sin_2d_array_cv(x,y)],axis=0)
    np.savez(folder_w_subfolder+ "/"+ data['npz_cv_name'], sinus=generate_npy)
    print("npz saved")

# opencv_version: images saves to /path/sinus_npz/verbose/sinus.npz
def generate_train_img_cv(folder, N_type=1, amax=5, wmin=5, wmax=10, bmin=-2, bmax=2, cmin=-2 ,cmax=2 , step = 0.1, wfreq=0.5) :
    folder_w_subfolder = folder + data['img_cv_folder']
    os.makedirs(folder, exist_ok=True)

    os.makedirs(folder_w_subfolder, exist_ok=True)
    folder = os.path.abspath(folder_w_subfolder)

    for type_i in range(N_type):
      for amp_int in range(1, amax*2+1):
        amp_i = amp_int*0.5
        for omega_i in range(wmin, wmax, 1):
          omega_ii = wfreq * omega_i
          for b_i in range(bmin, bmax, 1):
            for c_i in range(cmin, cmax, 1):
              # use sinus gernerate:
              # x,y = generate_random_sin(N_type, amp_i, 4, omega_ii, step)
              # use cosinus gernerate:
              x,y = generate_random_cos(n_rand_starts=N_type,a=amp_i, w=omega_ii, b = b_i, c = c_i, step = step)

              filename = '{folder}/sin_{amp_i}-{omega_ii}-{b_i}-{c_i}-{type_i}'.format(folder=folder, amp_i=amp_i, omega_ii=omega_ii, b_i=b_i, c_i=c_i,type_i=type_i).replace(".","_")
              filename = filename + ".png"
              plot_save_disk_cv(x,y, filename, xmax = data['resolution'], ymax = data['resolution'])

print("loaded")


#!/usr/bin/python

import argparse
import datetime
import numpy as np
import pickle
import os
import CLS_Clouds as CLS
import scipy.stats

parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument('--date', type=str, help='date on which the algorithm should be run')
parser.add_argument('--campaign', type=str, help='Cloudnet station on which the algorithm should be run')
parser.add_argument('--type', metavar='type_str', type=str, help='Cloud type on which the algorithm should be run')
args = parser.parse_args()

campaign=args.campaign
date=datetime.datetime.strptime(args.date, "%Y%m%d").date()
cloud_type=args.type
print("starting collector {} {} {}".format(args.campaign, args.date, args.type))

def load_object(filename):
    ifile=open(filename,"rb")
    obj = pickle.load(ifile)
    ifile.close()
    return obj

def dict_to_str(source):
    outstr=""
    keys=sorted(source.keys())
    for key in keys:
        outstr += str(source[key])+";"
    return outstr

def keys_to_str(source):
    outstr=""
    keys=sorted(source.keys())
    for key in keys:
        outstr += str(key)+";"
    return outstr

if cloud_type==None:
    cloud_type='all'

recognized_types=['pure_ice','pure_liquid','mixed-phase','liquid-based','tower','all']
assert cloud_type in recognized_types, "{} not in {}".format(cloud_type, recognized_types)

infile='../cloud_properties/cloud_properties_'+campaign+'/'+date.strftime("%Y_%m_%d")+'_clouds.dat'
outfile='../cloud_collections/'+'cloud_collection_'+campaign+'_'+cloud_type+'.csv'

if not(os.path.exists(outfile)):
    newfile=True
else:
    if os.path.getsize(outfile)==0:
        newfile=True
    else:
        newfile=False

clouds = load_object(infile)

ice_only=[4]
liquid=[1,3,5,7]
droplets_only=[1,5]
sep=6
output={}

for i in range(len(clouds)):

    #print clouds[i].cloud_type
    #print clouds[i].top_variation(),clouds[i].time_length(),clouds[i].fill_factor()

    if (clouds[i].cloud_type==cloud_type or cloud_type=='all') and clouds[i].n_profiles()>0:

       if clouds[i].top_variation()<600.0 and clouds[i].time_length()>900 and clouds[i].fill_factor()>0.7:

            begin=datetime.datetime.utcfromtimestamp(clouds[i].begin_time)
            end=datetime.datetime.utcfromtimestamp(clouds[i].end_time)
            middle=begin+datetime.timedelta(seconds=(clouds[i].end_time-clouds[i].begin_time)/2.)

            clouds[i].validate_features()

            output["A_Unique_Identifier"]=begin.strftime("%Y%m%d%H%M%S")+str(int(clouds[i].tops[0]))
            output["Cloud_Run"]=clouds[i].cloud_type
            output["Cloud_Type"]=clouds[i].most_common_type()
            output["Date_Year"]=middle.year
            output["Date_Month"]=middle.month
            output["Date_Day"]=middle.day
            output["Date_Hour"]=middle.hour
            output["Begin_Date"]=begin.strftime("%Y_%m_%d_%H_%M_%S")
            output["Begin_Date_Unix"]=clouds[i].begin_time
            output["End_Date"]=end.strftime("%Y_%m_%d_%H_%M_%S")
            output["End_Date_Unix"]=clouds[i].end_time

            output["CTH"]=np.max(clouds[i].tops)
            output["CTH_AVG"]=np.average(clouds[i].tops)
            output["CTH_STD"]=clouds[i].top_variation()
            output["CTH_DIFFSUM"]=clouds[i].top_diffsum()
            output["LLH_STD"]=clouds[i].liquid_layer_variation()
            output["CBH"]=np.min(clouds[i].bases)
            output["CBT"],output["CTT_MED"],output["CTT"]=clouds[i].temperature_range()
            output["CBP"],output["CTP"]=clouds[i].pressure_range()

            output["VEL"],output["DIR"]=clouds[i].vel_and_dir()

            output["N_Profiles"]=clouds[i].n_profiles()
            output["N_Liquid"]=clouds[i].n_profiles(["pure liquid","liquid-based layer"])
            output["N_MixedPhase"]=clouds[i].n_profiles(["mixed-phase layer"])

            output["LWC_AVG"],output["LWC_MED"],output["LWC_STD"],output["LWC_N"]=clouds[i].average("LWC",CLS.droplets_only)
            output["PATH_LWP_AVG"],output["PATH_LWP_S_AVG"],output["PATH_IWP_AVG"],output["PATH_IWP_STD"]=clouds[i].average_paths()
            output["Cloud_Thickness_AVG"],output["Cloud_Thickness_MED"],output["Cloud_Thickness_STD"], thickness_vals =clouds[i].cloud_top_thickness()

            thickness_vals_s = sorted(thickness_vals, key=lambda k: k[0])
            period_top, autocorr_top = CLS.time_analysis_from_vel(thickness_vals_s, 3)
            #
            # ey linkely wrong place
            # here the CTH_autocorr is calculated
            #
            #print(autocorr_top)
            if len(autocorr_top[1] > 0):
                i_above_thres = np.where(autocorr_top[1] > 0.8)[0][-1]
                output["CTH_autocorr_08_time"] = autocorr_top[0][i_above_thres]
            else:
                output["CTH_autocorr_08_time"] = -1
            #wh = np.where(autocorr_top[1] > 0.0)[0]
            #if len(wh) > 0:
            #    output["CTH_autocorr_08_time"] = autocorr_top[0][wh[-1]]
            #else:
            #    output["CTH_autocorr_08_time"] = -99
            # output["v_dl_period_f"] = periodogram[0].tolist()
            # output["v_dl_period_Pxx"] = periodogram[1].tolist()
            # output["v_dl_autocor_time"] = autocorr[0].tolist()
            # output["v_dl_autocor_coeff"] = autocorr[1].tolist()

            output["IWC_AVG"],output["IWC_MED"],output["IWC_STD"],output["IWC_N"]=clouds[i].average("IWC",ice_only)
            output["IWC_TOP_AVG"],output["IWC_TOP_MED"],output["IWC_TOP_STD"],output["IWC_TOP_N"]=clouds[i].separation_average("IWC",sep)
            output["ILCR_AVG"],output["ILCR_MED"],output["ILCR_N"],output["ILCR_values"]=clouds[i].ilr(sep)

            clouds[i].correct_LDR(-20,-33)
            output["LDR_AVG"],output["LDR_MED"],output["LDR_STD"],output["LDR_N"]=clouds[i].average("LDR",ice_only)
            output["LDR_values"]=list(np.histogram(clouds[i].return_values("LDR",ice_only),70,(-35.0,1))[0])
            output["LDR_TOP_AVG"],output["LDR_TOP_MED"],output["LDR_TOP_STD"],output["LDR_TOP_N"]=clouds[i].separation_average("LDR",sep)

            output["N_Melting"]=clouds[i].n_melting()
            output["N_Drizzle"]=clouds[i].n_drizzle()

            output["v_AVG"],output["v_MED"],output["v_STD"],output["v_N"]=clouds[i].average("v",ice_only)
            output["v_TOP_AVG"],output["v_TOP_MED"],output["v_TOP_STD"],output["v_TOP_N"]=clouds[i].separation_average("v",sep)
            output["v_values"]=list(np.histogram(clouds[i].return_values("v",ice_only),60,(-1.5,0.0))[0])

            #output["tfv_AVG"],output["tfv_MED"],output["tfv_STD"],output["tfv_N"]=clouds[i].average("tfv",ice_only)
            #output["tfv_TOP_AVG"],output["tfv_TOP_MED"],output["tfv_TOP_STD"],output["tfv_TOP_N"]=clouds[i].separation_average("tfv",sep)
            #output["vair_AVG"],output["vair_MED"],output["vair_STD"],output["vair_N"]=clouds[i].combined_average("vair","v",ice_only)

            #output["v_comb_tfv_AVG"],output["v_comb_tfv_MED"],output["v_comb_tfv_STD"],output["v_comb_tfv_N"]=clouds[i].combined_average("v","tfv",ice_only)

            output["Z_AVG"],output["Z_MED"],output["Z_STD"],output["Z_N"]=clouds[i].average("Z",ice_only)
            output["Z_AVG_drop"],output["Z_MED_drop"],output["Z_STD_drop"],output["Z_N_drop"]=clouds[i].average("Z",droplets_only)
            output["Z_TOP_AVG"],output["Z_TOP_MED"],output["Z_TOP_STD"],output["Z_TOP_N"]=clouds[i].separation_average("Z",sep)
            z_top_vals = clouds[i].return_values_separation("Z",sep)
            output["Z_TOP_values"]=list(np.histogram(10*np.log10(z_top_vals),90,(-70.0,20))[0])
            z_vals = clouds[i].return_values("Z",ice_only)
            #print(z_vals)
            #print(10*np.log10(z_vals))
            output["Z_values"]=list(np.histogram(10*np.log10(z_vals),90,(-70.0,20))[0])

            output["ZE_TOP_AVG"],output["ZE_TOP_MED"],output["ZE_TOP_STD"],output["ZE_TOP_N"]=clouds[i].separation_average("ratio_z_e",sep)
            print('ZE_TOP_stat ', output["ZE_TOP_AVG"],output["ZE_TOP_MED"],output["ZE_TOP_STD"],output["ZE_TOP_N"])
            output["ZE_AVG"],output["ZE_MED"],output["ZE_STD"],output["ZE_N"]=clouds[i].average("ratio_z_e",ice_only)
            print('ZE_stat ', output["ZE_AVG"],output["ZE_MED"],output["ZE_STD"],output["ZE_N"])

            output["SNR_TOP_AVG"],output["SNR_TOP_MED"],output["SNR_TOP_STD"],output["SNR_TOP_N"]=clouds[i].separation_average("SNR",sep)

            output["width_AVG"],output["width_MED"],output["width_STD"],output["width_N"]=clouds[i].average("width",ice_only)

            #output["SNR_AVG"],output["SNR_MED"],output["SNR_STD"],output["SNR_N"]=clouds[i].average("SNR",ice_only)
            #output["SNR_10pp"],output["SNR_90pp"]=clouds[i].pp90("SNR",ice_only)

            output["alpha_Hogan_AVG"],output["alpha_Hogan_MED"],output["alpha_Hogan_STD"],output["alpha_Hogan_N"]=clouds[i].average("alpha_hogan",ice_only)
            output["alpha_Hogan_TOP_AVG"],output["alpha_Hogan_TOP_MED"],output["alpha_Hogan_TOP_STD"],output["alpha_Hogan_TOP_N"]=clouds[i].separation_average("alpha_hogan",sep)
            output["beta_AVG"],output["beta_MED"],output["beta_STD"],output["beta_N"]=clouds[i].average("beta",ice_only)

            # lidar variables from pollynet
            output["qbscice_TOP_AVG"],output["qbscice_TOP_MED"],output["qbscice_TOP_STD"],output["qbscice_TOP_N"]=clouds[i].separation_average("qbsc",sep)
            output["qbsc_drop_AVG"],output["qbsc_drop_MED"],output["qbsc_drop_STD"],output["qbsc_drop_N"]=clouds[i].average("qbsc",droplets_only)
            # TODO add qiceext

            output["delta_AVG"],output["delta_MED"],output["delta_STD"],output["delta_N"]=clouds[i].average("voldepol",ice_only)
            output["voldepol_AVG"],output["voldepol_MED"],output["voldepol_STD"],output["voldepol_N"]=clouds[i].average("voldepol",ice_only)

            #output["v_lidar_AVG"],output["v_lidar_STD"],output["v_lidar_N"],output["v_lgt0_AVG"],output["v_lgt0_STD"],output["v_lgt0_N"],vv_values=clouds[i].velocities()
            #output["v_lidar_histo"]=list(np.histogram(vv_values,60,(-3.0,3.0))[0])

            # refactored vertical velocities in liquid layers
            output['v_dl_mean'], output['v_dl_std'], output['v_dl_n'], v_base, vel_locations = clouds[i].velocities()
            if len(v_base) > 0:
                output['v_dl_perc'] = np.percentile(v_base, [10,25,50,75,90]).tolist()
                output['v_dl_skew'] = scipy.stats.skew(v_base)
            else:
                output['v_dl_perc'] = [-99,-99,-99,-99,-99]
                output['v_dl_skew'] = -99

            output['v_cr_mean'], output['v_cr_std'], output['v_cr_n'], v_base, _ = clouds[i].velocities_liquid_radar('whole')
            if len(v_base) > 0:
                output['v_cr_perc'] = np.percentile(v_base, [10,25,50,75,90]).tolist()
                output['v_cr_skew'] = scipy.stats.skew(v_base)
            else:
                output['v_cr_perc'] = [-99,-99,-99,-99,-99]
                output['v_cr_skew'] = -99

            # and the timeseries analysis
            # (f, Pxx_den), (time_shifts[:500], v_autocorr[:500])
            vel_locations_s = sorted(vel_locations, key=lambda k: k[0])
            periodogram, autocorr = CLS.time_analysis_from_vel(vel_locations_s, 2)
            output["v_dl_period_f"] = periodogram[0].tolist()
            output["v_dl_period_Pxx"] = periodogram[1].tolist()
            output["v_dl_autocor_time"] = autocorr[0].tolist()
            output["v_dl_autocor_coeff"] = autocorr[1].tolist()

            #
            # now include the fixed height version
            #
            mids, mid_with_time = clouds[i].cloud_top_avg()
            output['v_dl_fix_mean'], output['v_dl_fix_std'], output['v_dl_fix_n'], v_base, vel_locations = clouds[i].velocities_fixed_height(mids)
            vel_locations_s = sorted(vel_locations, key=lambda k: k[0])
            periodogram, autocorr = CLS.time_analysis_from_vel(vel_locations_s, 2)
            output["v_dl_fix_period_f"] = periodogram[0].tolist()
            output["v_dl_fix_period_Pxx"] = periodogram[1].tolist()
            output["v_dl_fix_autocor_time"] = autocorr[0].tolist()
            output["v_dl_fix_autocor_coeff"] = autocorr[1].tolist()

            #output["pT_no_node_hist"] = clouds[i].no_node_hist()
            #output["pT_no_node_hist_cb"] = clouds[i].no_node_hist_above_cb()
            output["pT_no_node_hist_il"] = clouds[i].no_node_hist_ice_liq()

            #manually corrected LDR
            output["LDRcorr_TOP_AVG"], output["LDRcorr_TOP_MED"], output["LDRcorr_TOP_STD"], output["LDRcorr_TOP_N"] = clouds[i].separation_average("LDRcorr", sep)
            output["LDRcorr_AVG"], output["LDRcorr_MED"], output["LDRcorr_STD"], output["LDRcorr_N"] = clouds[i].average("LDRcorr", ice_only)

            #interpolation
            #il=16
            #vv_int=[]
            #if len(vv_values)>il+1:
            #    for n in range(len(vv_values)/il-il-1):
            #        vv_int.append(np.average(vv_values[il*n:il*n+il]))
            #vv_int=np.array(vv_int)
            #vv_values=vv_int

            #output["v_lidar_histo_INT"]=list(np.histogram(vv_values,60,(-3.0,3.0))[0])

            output["v_radar_AVG"],output["v_radar_STD"],output["v_radar_N"],output["v_radar_lgt0_AVG"],output["v_radar_lgt0_STD"],output["v_radar_lgt0_N"],output["v_radar_WIDTH"],vv_radar_values,output["Z_top"]=clouds[i].velocities_radar()
            output["v_radar_histo"]=list(np.histogram(vv_radar_values,60,(-3.0,3.0))[0])

            output["file_hist_class"] = clouds[i].get_class_file_history()

            #print(clouds[i].cloud_type,output["Z_top"])

            #output["adv_wind_profiler"],output["std_wind_profiler"],output["max_wind_profiler"],output["min_wind_profiler"],output["dvdh_wind_profiler"]=clouds[i].horizontal_wind(output["CTH"],4)
            #if len(clouds[i].features)>100:
            #    clouds[i].FA()
            #if output["Begin_Date"]=='2018_12_30_21_05_14':
            #    exit()

            with open(outfile, "a+") as f:
                if newfile:
                    keys=keys_to_str(output)
                    f.write(keys+"\n")
                    newfile=False

                output_str=dict_to_str(output)
                f.write(output_str+"\n")


import copy
import json
import logging
import os
import pickle
import random
import statistics
import sys
import time

import numpy as np
import torch

import yaml
from inclearn.lib import factory
from inclearn.lib import metrics, utils, results_utils
from inclearn.lib.network import FeatureGenerator
from inclearn.lib.data.samplers import NPairSampler, AuxSampler

from inclearn.utils import LOGGER as logger

def train(args):
    logger.LOGGER.setLevel(args["logging"].upper())
    autolabel = _set_up_options(args)
    if args["autolabel"]:
        args["label"] = autolabel

    if args["label"]:
        logger.LOGGER.info("Label: {}".format(args["label"]))
        try:
            os.system("echo '\ek{}\e\\'".format(args["label"]))
        except:
            pass
    if args["resume"] and not os.path.exists(args["resume"]):
        raise IOError(f"Saved model {args['resume']} doesn't exist.")

    if args["save_model"] != "never" and args["label"] is None:
        raise ValueError(f"Saving model every {args['save_model']} but no label was specified.")

    seed_list = copy.deepcopy(args["seed"])
    device = copy.deepcopy(args["device"])

    start_date = utils.get_date()
    results_folder = results_utils.get_save_folder(args["model"], start_date, args["label"])
    logger.add_file_headler(results_folder)

    orders = copy.deepcopy(args["order"])
    del args["order"]
    if orders is not None:
        assert isinstance(orders, list) and len(orders)
        assert all(isinstance(o, list) for o in orders)
        assert all([isinstance(c, int) for o in orders for c in o])
    else:
        orders = [None for _ in range(len(seed_list))]

    avg_inc_accs, last_accs, forgettings = [], [], []
    for i, seed in enumerate(seed_list):
        logger.LOGGER.warning("Launching run {}/{}".format(i + 1, len(seed_list)))
        args["seed"] = seed
        args["device"] = device

        start_time = time.time()

        for avg_inc_acc, last_acc, forgetting in _train(args, start_date, orders[i], i):
            yield avg_inc_acc, last_acc, forgetting, False

        avg_inc_accs.append(avg_inc_acc)
        last_accs.append(last_acc)
        forgettings.append(forgetting)

        logger.LOGGER.info("Training finished in {}s.".format(int(time.time() - start_time)))
        yield avg_inc_acc, last_acc, forgetting, True

    logger.LOGGER.info("Label was: {}".format(args["label"]))

    logger.LOGGER.info(
        "Results done on {} seeds: avg: {}, last: {}, forgetting: {}".format(
            len(seed_list), _aggregate_results(avg_inc_accs), _aggregate_results(last_accs),
            _aggregate_results(forgettings)
        )
    )
    logger.LOGGER.info("Individual results avg: {}".format([round(100 * acc, 2) for acc in avg_inc_accs]))
    logger.LOGGER.info("Individual results last: {}".format([round(100 * acc, 2) for acc in last_accs]))
    logger.LOGGER.info(
        "Individual results forget: {}".format([round(100 * acc, 2) for acc in forgettings])
    )

    logger.LOGGER.info(f"Command was {' '.join(sys.argv)}")

def _train(args, start_date, class_order, run_id):
    _set_global_parameters(args)
    inc_dataset, model = _set_data_model(args, class_order)
    results, results_folder = _set_results(args, start_date)

    memory, memory_val, pseudo_memory = None, None, None
    metric_logger = metrics.MetricLogger(
        inc_dataset.n_tasks, inc_dataset.n_classes, inc_dataset.increments
    )
    use_unlabeled = args.get("use_unlabeled", False)
    print(f'use_unlabeled:{use_unlabeled}')
    for task_id in range(inc_dataset.n_tasks):
        pseudo_memory_n_samples = args.get("pseudo_memory_n_samples", 2)
        task_info, train_loader, val_loader, test_loader, aux_loader, pseudo_memory_loader, pure_new_data = \
            inc_dataset.new_task(memory, pseudo_memory, memory_val, pseudo_memory_n_samples=pseudo_memory_n_samples)
        if task_info["task"] == args["max_task"]:
            break
        model.set_task_info(task_info)

        # ---------------
        # 1. Prepare Task
        # ---------------
        model.eval()
        model.before_task(train_loader, val_loader if val_loader else test_loader)

        # -------------
        # 2. Train Task
        # -------------
        pseudo_memory = _train_task(args, model, train_loader, aux_loader, pseudo_memory, pseudo_memory_loader,
                                    pure_new_data, inc_dataset, val_loader, test_loader, run_id, task_id, task_info,
                                    results_folder)

        # ----------------
        # 3. Conclude Task
        # ----------------
        model.eval()
        _after_task(args, model, inc_dataset, run_id, task_id, results_folder)

        # ------------
        # 4. Eval Task
        # ------------
        logger.LOGGER.info("Eval on {}->{}.".format(0, task_info["max_class"]))
        ypreds, ytrue = model.eval_task(test_loader)
        metric_logger.log_task(
            ypreds, ytrue, task_size=task_info["increment"], zeroshot=args.get("all_test_classes")
        )

        if args["dump_predictions"] and args["label"]:
            os.makedirs(
                os.path.join(results_folder, "predictions_{}".format(run_id)), exist_ok=True
            )
            with open(
                    os.path.join(
                        results_folder, "predictions_{}".format(run_id),
                        str(task_id).rjust(len(str(30)), "0") + ".pkl"
                    ), "wb+"
            ) as f:
                pickle.dump((ypreds, ytrue), f)

        if args["label"]:
            logger.LOGGER.info(args["label"])
        logger.LOGGER.info("Avg inc acc: {}.".format(metric_logger.last_results["incremental_accuracy"]))
        logger.LOGGER.info("Current acc: {}.".format(metric_logger.last_results["accuracy"]))
        logger.LOGGER.info(
            "Avg inc acc top5: {}.".format(metric_logger.last_results["incremental_accuracy_top5"])
        )
        logger.LOGGER.info("Current acc top5: {}.".format(metric_logger.last_results["accuracy_top5"]))
        logger.LOGGER.info("Forgetting: {}.".format(metric_logger.last_results["forgetting"]))
        logger.LOGGER.info("Cord metric: {:.2f}.".format(metric_logger.last_results["cord"]))
        if task_id > 0:
            logger.LOGGER.info(
                "Old accuracy: {:.2f}, mean: {:.2f}.".format(
                    metric_logger.last_results["old_accuracy"],
                    metric_logger.last_results["avg_old_accuracy"]
                )
            )
            logger.LOGGER.info(
                "New accuracy: {:.2f}, mean: {:.2f}.".format(
                    metric_logger.last_results["new_accuracy"],
                    metric_logger.last_results["avg_new_accuracy"]
                )
            )
        if args.get("all_test_classes"):
            logger.LOGGER.info(
                "Seen classes: {:.2f}.".format(metric_logger.last_results["seen_classes_accuracy"])
            )
            logger.LOGGER.info(
                "unSeen classes: {:.2f}.".format(
                    metric_logger.last_results["unseen_classes_accuracy"]
                )
            )

        results["results"].append(metric_logger.last_results)

        avg_inc_acc = results["results"][-1]["incremental_accuracy"]
        last_acc = results["results"][-1]["accuracy"]["total"]
        forgetting = results["results"][-1]["forgetting"]
        yield avg_inc_acc, last_acc, forgetting

        memory = model.get_memory()
        memory_val = model.get_val_memory()
    logger.LOGGER.info(
        "Average Incremental Accuracy: {}.".format(results["results"][-1]["incremental_accuracy"])
    )
    if args["label"] is not None:
        results_utils.save_results(
            results, args["label"], args["model"], start_date, run_id, args["seed"]
        )

    del model
    del inc_dataset

def get_pseudo_memory(aux_loader, model, pseudo_memory, load_folder, save_folder, run_id, task_id, re_mine=False,
                      n_classes_samples=100):
    unlabeled_data_save_path = os.path.join(save_folder, f'pseudo_memory_{task_id}_task_{run_id}.pth')
    unlabeled_data_load_path = None
    if load_folder is not None:
        unlabeled_data_load_path = os.path.join(load_folder, f'pseudo_memory_{task_id}_task_{run_id}.pth')
    if unlabeled_data_load_path is not None and os.path.exists(unlabeled_data_load_path):
        pseudo_memory = torch.load(unlabeled_data_load_path)
        logger.LOGGER.info(f'Loaded existing pseudo data form {unlabeled_data_load_path}.')
        new_data, new_label = pseudo_memory[0], pseudo_memory[1]
    else:
        if pseudo_memory is not None and not re_mine:
            existing_pseudo_mem_cls = torch.unique(pseudo_memory[1])
        else:
            existing_pseudo_mem_cls = None

        new_pseudo_memory = model.get_pseudo_memory(aux_loader, existing_cls=existing_pseudo_mem_cls,
                                                    n_classes_samples=n_classes_samples)

        if existing_pseudo_mem_cls is not None and not re_mine:
            new_data = np.concatenate((pseudo_memory[0], new_pseudo_memory[0]), axis=0)
            new_label = torch.cat((pseudo_memory[1], new_pseudo_memory[1]), dim=0).cpu()
            pseudo_memory = (new_data, new_label)
        else:
            pseudo_memory = new_pseudo_memory
            new_data = new_pseudo_memory[0]
            new_label = new_pseudo_memory[1]

        logger.LOGGER.info(f'Now unlabeled data: {len(pseudo_memory[0])}')
    if (unlabeled_data_load_path is not None and not os.path.exists(unlabeled_data_load_path)) and \
            not os.path.exists(unlabeled_data_save_path):
        # torch.save(pseudo_memory, unlabeled_data_save_path)
        logger.LOGGER.info(f' Saved pseudo memory to {unlabeled_data_save_path}.')
    return pseudo_memory, new_data, new_label

# ------------------------
# Lifelong Learning phases
# ------------------------

def _train_task(config, model, train_loader, aux_loader, pseudo_memory, pseudo_memory_loader, pure_new_data,
                inc_dataset, val_loader, test_loader, run_id, task_id, task_info, results_folder):
    pseudo_memory_valid_map_idx = config.get('pseudo_memory_valid_map_idx', 0.5)
    if config["resume"] is not None and os.path.isdir(config["resume"]) \
            and ((config["resume_first"] and task_id == 0) or not config["resume_first"]):
        model.load_parameters(config["resume"], run_id, device=config['device'][0])
        logger.LOGGER.info(
            "Skipping training phase {} because reloading pretrained model.".format(task_id)
        )
    elif config["resume"] is not None and os.path.isfile(config["resume"]) and \
            os.path.exists(config["resume"]) and task_id == 0:
        # In case we resume from a single model file, it's assumed to be from the first task.
        # model.network = config["resume"]
        model.load_parameters(config["resume"], run_id, device=config['device'][0])
        logger.LOGGER.info(
            "Skipping initial training phase {} because reloading pretrained model.".
                format(task_id)
        )
    else:
        logger.LOGGER.info("Train on {}->{}.".format(task_info["min_class"], task_info["max_class"]))
        model.train()
        logger.LOGGER.info(f'Pseudo memory feature map selection is {pseudo_memory_valid_map_idx}.')
        model.train_task(train_loader, pseudo_memory_loader, val_loader if val_loader else test_loader,
                         freeze_layers=task_id != 0)

    finetuning_config = config.get("finetuning_config")

    use_unlabeled = config.get('use_unlabeled', False)
    generator_config = config.get("generator_config", {})
    train_generator_config = generator_config.get("train_config", {})
    batch_size = config.get("labeled_batch_size", 128)
    re_mined = config.get("pseudo_re_mined", False)
    n_classes_samples = config.get("pseudo_mem_n_classes_samples", 100)
    if task_id < task_info["max_task"] - 1 and use_unlabeled:
        p = get_pseudo_memory(aux_loader, model, pseudo_memory, config["resume"], results_folder, run_id, task_id,
                              re_mine=re_mined, n_classes_samples=n_classes_samples)
        pseudo_memory = p[0]
        new_pseudo_memory = (p[1], p[2])

        pseudo_memory_n_samples = train_generator_config.get('train_generator_unlabel_n_samples', 2)
        current_pseudo_class = torch.unique(new_pseudo_memory[1])
        tmp_pseudo_memory_loader = inc_dataset.get_pseudo_memory_loader(pseudo_memory, pseudo_memory_n_samples,
                                                                        batch_size=pseudo_memory_n_samples * len(
                                                                            current_pseudo_class))
        # train the generator network
        if generator_config:
            n_class_mem = train_generator_config.get("train_generator_memory_n_samples", 12)
            n_class_new = train_generator_config.get("train_generator_new_n_samples", 12)
            model.after_task_intensive(inc_dataset, train_generator=True)
            current_memory = model.get_memory()
            mem_sampler = AuxSampler(current_memory[1], batch_size=n_class_mem * int(task_info['increment']),
                                     n_sample=n_class_mem)
            memory_loader_PK = inc_dataset.get_loader(*current_memory, memory_flags=np.zeros(current_memory[0].shape),
                                                      mode="train", sampler=mem_sampler, sampler_init=False)
            nb_class = min(len(np.unique(pure_new_data[1])), int(batch_size / n_class_new))
            train_sampler = NPairSampler(y=pure_new_data[1], n_classes=nb_class, n_samples=n_class_new)
            train_loader_PK = inc_dataset.get_loader(*pure_new_data, memory_flags=np.zeros(len(pure_new_data[1])),
                                                     mode="train", sampler=train_sampler, sampler_init=False)
            train_generator_data = {
                'labeled_loader': train_loader_PK,
                'memory_loader': memory_loader_PK,
            }
            input_dim = generator_config.get("input_dim", 64)
            latent_dim = generator_config.get("latent_dim", 64)
            num_blocks = generator_config.get("n_blocks", 2)

            for cls in range(int(task_info['min_class']), int(task_info['max_class'])):
                cls_encoder = FeatureGenerator(input_dim, latent_dim=latent_dim, num_blocks=num_blocks).to(model.device)
                lr = train_generator_config.get("lr", 0.1)
                model.create_generator_optimizer(cls_encoder.parameters(), lr=lr)
                model.class_encoders[cls] = cls_encoder
            res = False
            if task_id == 0 and config["resume"] is not None:
                generator_path = os.path.join(config["resume"], 'generators')
                if os.path.exists(generator_path):
                    res = load_generator_params(model, config, generator_path, run_id, task_info['min_class'],
                                                task_info['max_class'])
            use_generators = not config.get('softmax_ce_not_unlabeled', False)
            if not res and task_id < task_info["max_task"] - 1 and use_generators:
                model.train_task(train_loader, tmp_pseudo_memory_loader, val_loader if val_loader else test_loader,
                                 train_generator_config=train_generator_config,
                                 train_generator_data=train_generator_data)
                if task_id >= 0:
                    save_generator_path = os.path.join(results_folder, 'generator')
                    if not os.path.isdir(save_generator_path):
                        os.makedirs(save_generator_path)
                    save_generator_params(model, save_generator_path, run_id, task_info['min_class'], task_info['max_class'])

    # fine-tune
    if finetuning_config:
        model.fine_tune(pseudo_memory_loader, pseudo_memory_valid_map_idx, val_loader)
    return pseudo_memory

def save_generator_params(model, results_folder, run_id, min_class, max_class):
    for cls in range(min_class, max_class):
        e_save_path = os.path.join(results_folder, f'generators_run{run_id}_class{cls}.pth')
        torch.save({'state_dict': model.class_encoders[cls].state_dict()}, e_save_path)
        logger.LOGGER.info(f'Saved generator encoder for class {cls} to file {e_save_path}.')

def load_generator_params(model, config, results_folder, run_id, min_class, max_class):
    for cls in range(min_class, max_class):
        e_save_path = os.path.join(results_folder, f'generator_encoders_run{run_id}_class{cls}.pth')
        try:
            state_dict_saved = torch.load(e_save_path, map_location=config['device'][0])
        except Exception as e:
            logger.LOGGER.warning(f'Loading file `{e_save_path}` failed. Try to train it again.')
            return False
        model.class_encoders[cls].load_state_dict(state_dict_saved['state_dict'])
        logger.LOGGER.info(f'Loaded generator encoder for class {cls} from file {e_save_path}.')

    d_save_path = os.path.join(results_folder, f'generators_run{run_id}.pth')
    try:
        state_dict_saved = torch.load(d_save_path, map_location=config['device'][0])
    except:
        logger.LOGGER.warning(f'Loading file `{d_save_path}` failed. Try to train it again.')
        return False
    return True

def _after_task(config, model, inc_dataset, run_id, task_id, results_folder):
    if config["resume"] and os.path.isdir(config["resume"]) and not config["recompute_meta"] \
            and ((config["resume_first"] and task_id == 0) or not config["resume_first"]):
        model.load_metadata(config["resume"], run_id)
    else:
        model.after_task_intensive(inc_dataset)

    model.after_task(inc_dataset)

    if config["label"] and (
            config["save_model"] == "task" or
            (config["save_model"] == "last" and task_id == inc_dataset.n_tasks - 1) or
            (config["save_model"] == "first" and task_id == 0)
    ):
        model.save_parameters(results_folder, run_id)
        model.save_metadata(results_folder, run_id)

# ----------
# Parameters
# ----------

def _set_results(config, start_date):
    if config["label"]:
        results_folder = results_utils.get_save_folder(config["model"], start_date, config["label"])
    else:
        results_folder = None

    if config["save_model"]:
        logger.LOGGER.info("Model will be save at this rythm: {}.".format(config["save_model"]))

    results = results_utils.get_template_results(config)

    return results, results_folder

def _set_data_model(config, class_order):
    inc_dataset = factory.get_data(config, class_order)
    config["classes_order"] = inc_dataset.class_order

    model = factory.get_model(config)
    model.inc_dataset = inc_dataset

    return inc_dataset, model

def _set_global_parameters(config):
    _set_seed(config["seed"], config["threads"], config["no_benchmark"], config["detect_anomaly"])
    factory.set_device(config)

def _set_seed(seed, nb_threads, no_benchmark, detect_anomaly):
    logger.LOGGER.info("Set seed {}".format(seed))
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    if no_benchmark:
        logger.LOGGER.warning("CUDA algos are not determinists but faster!")
    else:
        logger.LOGGER.warning("CUDA algos are determinists but very slow!")
    torch.backends.cudnn.deterministic = not no_benchmark  # This will slow down training.
    torch.set_num_threads(nb_threads)
    if detect_anomaly:
        logger.LOGGER.info("Will detect autograd anomaly.")
        torch.autograd.set_detect_anomaly(detect_anomaly)

def _set_up_options(args):
    options_paths = args["options"] or []

    autolabel = []
    for option_path in options_paths:
        if not os.path.exists(option_path):
            raise IOError("Not found options file {}.".format(option_path))

        args.update(_parse_options(option_path))

        autolabel.append(os.path.splitext(os.path.basename(option_path))[0])

    return "_".join(autolabel)

def _parse_options(path):
    with open(path) as f:
        if path.endswith(".yaml") or path.endswith(".yml"):
            return yaml.load(f, Loader=yaml.FullLoader)
        elif path.endswith(".json"):
            return json.load(f)["config"]
        else:
            raise Exception("Unknown file type {}.".format(path))

# ----
# Misc
# ----

def _aggregate_results(list_results):
    res = str(round(statistics.mean(list_results) * 100, 2))
    if len(list_results) > 1:
        res = res + " +/- " + str(round(statistics.stdev(list_results) * 100, 2))
    return res


'''
viewprobe creates visualizations for a certain eval.
'''

import glob
import os
import numpy
import re
import upsample
import time
import loadseg
from scipy.misc import imread, imresize, imsave
from loadseg import normalize_label
import expdir

class NetworkProbe:
    def __init__(self, directory, blobs=None):
        self.ed = expdir.ExperimentDirectory(directory)
        # Load probe metadata
        info = self.ed.load_info()
        self.ih, self.iw = info.input_dim
        self.layers = info.blobs if blobs is None else blobs
        self.ds = loadseg.SegmentationData(info.dataset)
        self.layer = {}
        for blob in self.layers:
            self.layer[blob] = LayerProbe(self.ed, blob, self.ds)

    def score_tally_stats(self, layer, verbose=False):
        # First, score every unit
        if verbose:
            print 'Adding tallys of unit/label alignments.'
            sys.stdout.flush()
        ta, tg, ti = self.summarize_tally(layer)
        labelcat = onehot(primary_categories_per_index(self.ds))
        tc = np.count_act_with_labelcat(layer)
        # If we were doing per-category activations p, then:
        # c = numpy.dot(p, labelcat.transpose())
        epsilon = 1e-20 # avoid division-by-zero
        # If we were counting activations on non-category examples then:
        # iou = i / (a[:,numpy.newaxis] + g[numpy.newaxis,:] - i + epsilon)
        iou = ti / (tc + tg[numpy.newaxis,:] - ti + epsilon)
        # Let's tally by primary-category.
        pc = primary_categories_per_index(self.ds)
        categories = self.ds.category_names()
        ar = numpy.arange(iou.shape[1])
        # actually - let's get the top iou for every category
        pciou = numpy.array([iou * (pc[ar] == ci)[numpy.newaxis,:]
                for ci in range(len(categories))])
        # label_iou = iou.argmax(axis=1)
        label_pciou = pciou.argmax(axis=2)
        # name_iou = [self.ds.name(None, i) for i in label_iou]
        name_pciou = [
                [self.ds.name(None, j) for j in label_pciou[ci]]
                for ci in range(len(label_pciou))]
        # score_iou = iou[numpy.arange(iou.shape[0]), label_iou]
        score_pciou = pciou[
                numpy.arange(pciou.shape[0])[:,numpy.newaxis],
                numpy.arange(pciou.shape[1])[numpy.newaxis,:],
                label_pciou]
        bestcat_pciou = score_pciou.argsort(axis=0)[::-1]
        # Assign category for each label
        # cat_per_label = primary_categories_per_index(self.ds)
        # cat_iou = [categories[cat_per_label[i]] for i in label_iou]
        # Now sort units by score and visulize each one
        return bestcat_pciou, name_pciou, score_pciou, label_pciou, tc, tg, ti

    def generate_html_summary(self, layer,
            imsize=64, imcount=16, imscale=None, tally_stats=None,
            gridwidth=None, verbose=False):
        print 'Generating html summary', (
            self.ed.filename(['html', '%s.html' % expdir.fn_safe(layer)]))
        # Grab tally stats
        bestcat_pciou, name_pciou, score_pciou, _, _, _, _ = (tally_stats)
        if verbose:
            print 'Sorting units by score.'
            sys.stdout.flush()
        if imscale is None:
            imscale = imsize
        categories = self.ds.category_names()
        ordering = score_pciou.max(axis=0).argsort()[::-1]
        top = self.max_act_indexes(layer, count=imcount)
        self.ed.ensure_dir('html', 'image')
        css = ('https://maxcdn.bootstrapcdn.com/bootstrap/latest' +
               '/css/bootstrap.min.css')
        html = ['<!doctype html>', '', '',
                '<link rel="stylesheet" href="%s">' % css,
                '', '', '
']
        if gridwidth is None:
            gridname = ''
            gridwidth = imcount
            gridheight = 1
        else:
            gridname = '-%d' % gridwidth
            gridheight = (imcount + gridwidth - 1) // gridwidth
        for unit in ordering:
            if verbose:
                print 'Visualizing %s unit %d' % (layer, unit)
                sys.stdout.flush()
            tiled = numpy.full(
                ((imsize + 1) * gridheight - 1,
                 (imsize + 1) * gridwidth - 1, 3), 255, dtype='uint8')
            for x, index in enumerate(top[unit]):
                row = x // gridwidth
                col = x % gridwidth
                vis = self.activation_visualization(layer, unit, index)
                if vis.shape[:2] != (imsize, imsize):
                    vis = imresize(vis, (imsize, imsize))
                tiled[row*(imsize+1):row*(imsize+1)+imsize,
                      col*(imsize+1):col*(imsize+1)+imsize,:] = vis
            imfn = 'image/%s%s-%04d.jpg' % (
                    expdir.fn_safe(layer), gridname, unit)
            imsave(self.ed.filename(os.path.join('html', imfn)), tiled)
            labels = '; '.join(['%s (%s, %f)' %
                    (name_pciou[c][unit], categories[c], score_pciou[c, unit])
                    for c in bestcat_pciou[:,unit]])
            html.extend([
                '
%s unit %d: %s
' % (layer, unit + 1, labels),
                '' % (os.path.join('html', imfn), imscale)
                ])
        html.extend([
            '
', '', '', ''])
        with open(self.ed.filename([
                'html', '%s.html' % expdir.fn_safe(layer)]), 'w') as f:
            f.write('\n'.join(html))

    def generate_csv_summary(
            self, layer, csvfile, tally_stats, order=None, verbose=False):
        if verbose:
            print 'Generating csv summary', csvfile
            sys.stdout.flush()
        bestcat_pciou, name_pciou, score_pciou, label_pciou, tc, tg, ti = (
                tally_stats)

        # For each unit in a layer, outputs the following information:
        # - label: best interpretation
        # - object-label: top ranked interpretation for scene/object/color/etc
        # - object-truth: ground truth pixels
        # - object-activation: activating pixels
        # - object-intersect: intersecting pixels
        # - object-iou: iou score
        # - etc, for each category.
        categories = self.ds.category_names()
        csv_fields = sum([[
            '%s-label' % cat,
            '%s-truth' % cat,
            '%s-activation' % cat,
            '%s-intersect' % cat,
            '%s-iou' % cat] for cat in categories],
            ['unit', 'category', 'label', 'score'])

        if order is not None:
            csv_fields = order

        if verbose:
            print 'Sorting units by score.'
            sys.stdout.flush()
        ordering = score_pciou.max(axis=0).argsort()[::-1]
        # top = self.max_act_indexes(layer, count=imcount)

        import csv
        with open(csvfile, 'w') as f:
            writer = csv.DictWriter(open(csvfile, 'w'), csv_fields)
            writer.writeheader()

            for unit in ordering:
                # Top images are top[unit]
                bestcat = bestcat_pciou[0, unit]
                data = {
                   'unit': (unit + 1),
                   'category': categories[bestcat],
                   'label': name_pciou[bestcat][unit],
                   'score': score_pciou[bestcat][unit]
                }
                for ci, cat in enumerate(categories):
                    label = label_pciou[ci][unit]
                    data.update({
                        '%s-label' % cat: name_pciou[ci][unit],
                        '%s-truth' % cat: tg[label],
                        '%s-activation' % cat: tc[unit, label],
                        '%s-intersect' % cat: ti[unit, label],
                        '%s-iou' % cat: score_pciou[ci][unit]
                    })
                writer.writerow(data)

    def generate_quantmat(self, layer, verbose=False):
        if verbose:
            print 'Generating quantmat'
            sys.stdout.flush()
        from scipy.io import savemat
        lp = self.layer[layer]
        filename = self.ed.filename(
                'quant-%d.mat' % lp.quantdata.shape[1], blob=layer)
        savemat(filename, { 'quantile': lp.quantdata })

    def generate_imgmax(self, layer, verbose=False):
        from scipy.io import savemat
        imgmax = self.ed.open_mmap(blob=layer, part='imgmax', mode='w+',
                shape = self.layer[layer].blobdata.shape[:2])
        imgmax[...] = self.layer[layer].blobdata.max(axis=(2, 3))
        self.ed.finish_mmap(imgmax)
        # Also copy out to mat file
        filename = self.ed.filename('imgmax.mat', blob=layer)
        savemat(filename, { 'imgmax': imgmax })
        # And cache
        self.layer[layer].imgmax = imgmax

    def instance_data(self, i, normalize=True):
        record, shape = self.ds.resolve_segmentation(
                self.ds.metadata(i), categories=None)
        if normalize:
            default_shape = (1, ) + shape
            record = dict((cat, normalize_label(dat, default_shape))
                    for cat, dat in record.items())
        return record, shape

    def top_image_indexes(self, layer, unit, count=10):
        t = self.layer[layer].count_a[:,unit].argsort()[::-1]
        return t[:count]

    # Generates a mask at the "lp.level" quantile.
    def activation_mask(self, layer, unit, index, shape=None):
        if shape is None:
             record, shape = self.instance_data(index)
        sw, sh = shape
        # reduction = int(round(self.iw / float(sw)))
        lp = self.layer[layer]
        blobdata = lp.blobdata
        fieldmap = lp.fieldmap
        quantdata = lp.quantdata
        threshold = quantdata[unit, int(round(quantdata.shape[1] * lp.level))]
        up = upsample.upsampleL(
                fieldmap, blobdata[index:index+1, unit],
                shape=(self.ih, self.iw), scaleshape=(sh, sw))[0]
        mask = up > threshold
        return mask

    # Makes an iamge using the mask
    def activation_visualization(self, layer, unit, index, alpha=0.2):
        image = imread(self.ds.filename(index))
        mask = self.activation_mask(layer, unit, index, shape=image.shape[:2])
        return (mask[:, :, numpy.newaxis] * (1 - alpha) + alpha) * image

    def summarize_tally(self, layer):
        cat_count = len(self.ds.category_names())
        tally = self.layer[layer].tally
        unit_size = self.layer[layer].shape[1]
        label_size = self.ds.label_size()
        count = numpy.zeros(
                (unit_size + 1, label_size + 1 + cat_count), dtype='int64')
        for i in range(len(tally)):
            t = tally[i]
            count[t[:,0]+1, t[:,1]+1+cat_count] += t[:,2]
        # count_a.shape = (unit size,)
        count_a = count[1:,cat_count]
        # this would summarize category intersections if we tallied them
        # count_c.shape = (unit_size, cat_size)
        # count_c = count[1:,0:cat_count]
        # count_g.shape = (label_size,)
        count_g = count[0,1+cat_count:]
        # count_i.shape = (unit_size, label_size)
        count_i = count[1:,1+cat_count:]
        # return count_a, count_c, count_g, count_i
        return count_a, count_g, count_i

    def count_act_with_label(self, layer):
        # Because our dataaset is sparse, instead of using count_a to count
        # all activations, we can compute count_act_with_label to count
        # activations only within those images which contain an instance
        # of a given label.
        tally = self.layer[layer].tally
        unit_size = self.layer[layer].shape[1]
        label_size = self.ds.label_size()
        count = numpy.zeros((unit_size, label_size), dtype='int64')
        for i in range(len(tally)):
            c1 = numpy.zeros((unit_size + 1, label_size + 1), dtype='int64')
            t = tally[i]
            c1[t[:,0]+1, t[:,1]+1] = t[:,2]
            count += c1[1:,0][:,numpy.newaxis] * (c1[0,1:][numpy.newaxis] > 0)
        return count

    def count_act_with_labelcat(self, layer):
        # Because our dataaset is sparse, instead of using count_a to count
        # all activations, we can compute count_act_with_labelcat to count
        # activations only within those images which contain an instance
        # of a given label category.
        labelcat = onehot(primary_categories_per_index(self.ds))
        # Be sure to zero out the background label - it belongs to no category.
        labelcat[0,:] = 0
        tally = self.layer[layer].tally
        unit_size = self.layer[layer].shape[1]
        label_size = self.ds.label_size()
        count = numpy.zeros((unit_size, labelcat.shape[1]), dtype='int64')
        for i in range(len(tally)):
            c1 = numpy.zeros((unit_size + 1, label_size + 1), dtype='int64')
            t = tally[i]
            c1[t[:,0]+1, t[:,1]+1] = t[:,2]
            count += c1[1:,0][:,numpy.newaxis] * (
                    numpy.dot(c1[0,1:], labelcat) > 0)
        # retval: (unit_size, label_size)
        return numpy.dot(count, numpy.transpose(labelcat))

    def max_act_indexes(self, layer, count=10):
        max_per_image = self.layer[layer].imgmax
        return max_per_image.argsort(axis=0)[:-1-count:-1,:].transpose()

    def top_act_indexes(self, layer, count=10):
        tally = self.layer[layer].tally
        unit_size = self.layer[layer].shape[1]
        label_size = self.ds.label_size()
        all_acts = numpy.zeros((len(tally), unit_size), dtype='int64')
        for i in range(len(tally)):
            acts = numpy.zeros((unit_size + 1, 2), dtype='int32')
            t = tally[i]
            acts[t[:,0] + 1, (t[:,1] != -1).astype('int')] = t[:,2]
            all_acts[i] = acts[1:,0]
        return all_acts.argsort(axis=0)[:-1-count:-1,:].transpose()

class LayerProbe:
    def __init__(self, ed, blob, ds):
        info = ed.load_info(blob=blob)
        self.shape = info.shape
        self.fieldmap = info.fieldmap
        # Load the raw activation data
        if ed.has_mmap(blob=blob):
            self.blobdata = ed.open_mmap(blob=blob, shape=self.shape, mode='r')
        # Load the blob quantile data and grab thresholds
        if ed.has_mmap(blob=blob, part='quant-*'):
            self.quantdata = ed.open_mmap(blob=blob, part='quant-*',
                                shape=(self.shape[1], -1), mode='r')
        # Load tally too; tally_depth is inferred from file size.
        self.tally = ed.open_mmap(blob=blob, part='tally-*', decimal=True,
                shape=(ds.size(), -1, 3), dtype='int32', mode='r')
        # And load imgmax
        if ed.has_mmap(blob=blob, part='imgmax'):
            self.imgmax = ed.open_mmap(blob=blob, part='imgmax',
                    shape=(ds.size(), self.shape[1]), mode='r')
        # Figure out tally level that was used.
        self.level = ed.glob_number(
                'tally-*.mmap', blob=blob, decimal=True)

def primary_categories_per_index(ds, categories=None):
    '''
    Returns an array of primary category numbers for each label, where the
    first category listed in ds.category_names is given category number 0.
    '''
    if categories is None:
        categories = ds.category_names()
    catmap = {}
    for cat in categories:
        imap = ds.category_index_map(cat)
        if len(imap) < ds.label_size(None):
            imap = numpy.concatenate((imap, numpy.zeros(
                ds.label_size(None) - len(imap), dtype=imap.dtype)))
        catmap[cat] = imap
    result = []
    for i in range(ds.label_size(None)):
        maxcov, maxcat = max(
                (ds.coverage(cat, catmap[cat][i]) if catmap[cat][i] else 0, ic)
                for ic, cat in enumerate(categories))
        result.append(maxcat)
    return numpy.array(result)

def onehot(arr, minlength=None):
    '''
    Expands an array of integers in one-hot encoding by adding a new last
    dimension, leaving zeros everywhere except for the nth dimension, where
    the original array contained the integer n.  The minlength parameter is
    used to indcate the minimum size of the new dimension.
    '''
    length = numpy.amax(arr) + 1
    if minlength is not None:
        length = max(minlength, length)
    result = numpy.zeros(arr.shape + (length,))
    result[list(numpy.indices(arr.shape)) + [arr]] = 1
    return result

if __name__ == '__main__':
    import argparse
    import sys
    import traceback

    try:
        parser = argparse.ArgumentParser(
            description='Generate visualization for probed activation data.')
        parser.add_argument(
                '--directory',
                default='.',
                help='output directory for the net probe')
        parser.add_argument(
                '--format',
                default='html',
                help='html or csv or both')
        parser.add_argument(
                '--csvorder',
                help='csv header order')
        parser.add_argument(
                '--blobs',
                nargs='*',
                help='network blob names to visualize')
        parser.add_argument(
                '--gridwidth',
                type=int, default=None,
                help='width of visualization grid')
        parser.add_argument(
                '--imsize',
                type=int, default=72,
                help='thumbnail dimensions')
        parser.add_argument(
                '--imscale',
                type=int, default=None,
                help='thumbnail dimensions')
        parser.add_argument(
                '--imcount',
                type=int, default=16,
                help='number of thumbnails to include')
        args = parser.parse_args()
        np = NetworkProbe(args.directory, blobs=args.blobs)
        for blob in args.blobs:
            formats = args.format.split(',')
            if 'imgmax' in formats:
                np.generate_imgmax(blob)
            if 'html' in formats or 'csv' in formats:
                tally_stats = np.score_tally_stats(blob, verbose=True)
            if 'html' in formats:
                np.generate_html_summary(blob,
                        imsize=args.imsize, imscale=args.imscale,
                        imcount=args.imcount, tally_stats=tally_stats,
                        gridwidth=args.gridwidth,
                        verbose=True)
            if 'csv' in formats:
                filename = os.path.join(args.directory,
                        '%s-result.csv' % expdir.fn_safe(blob))
                np.generate_csv_summary(blob, filename, tally_stats,
                        order=(args.csvorder.split(',')
                            if args.csvorder else None), verbose=True)
            if 'quantmat' in formats:
                np.generate_quantmat(blob, verbose=True)
    except:
        traceback.print_exc(file=sys.stdout)
        sys.exit(1)


#============================================================================
# Copyright (c) 2018 <NAME>. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#============================================================================
# Author: <NAME>
# E-mail: <EMAIL>
# Description: Improvements of stripe artifact removal methods:
# [1] <NAME>, <NAME>, and <NAME>, "Superior techniques
#     for eliminating ring artifacts in X-ray micro-tomography," Optics
#     Express 26, 28396-28412 (2018). https://doi.org/10.1364/OE.26.028396.
# [2] <NAME>, <NAME>, and <NAME>,"Preprocessing techniques
#     for removing artifacts in synchrotron-based tomographic images,"
#     Proc. SPIE 11113, Developments in X-Ray Tomography XII.
#     https://doi.org/10.1117/12.2530324.
# Publication date: 09th October 2019
#============================================================================

"""
Module for stripe removal methods proposed in:
https://doi.org/10.1117/12.2530324
"""

import numpy as np
from scipy import interpolate
from scipy.signal.windows import gaussian
from scipy.ndimage import median_filter
from scipy.ndimage import binary_dilation
# import scipy.fftpack as fft
import pyfftw.interfaces.scipy_fftpack as fft

from sarepy.prep.stripe_removal_original import remove_stripe_based_sorting
from sarepy.prep.stripe_removal_original import remove_stripe_based_fitting
from sarepy.prep.stripe_removal_original import apply_gaussian_filter
from sarepy.prep.stripe_removal_original import detect_stripe

def remove_stripe_based_filtering_sorting(sinogram, sigma, size, dim=1):
    """
    Removing stripes using the filtering and sorting technique, combination of
    algorithm 2 and algorithm 3 in Ref.[1]. Angular direction is along the axis 0.

    Parameters
    ----------
    sinogram : array_like
        2D array. Sinogram image.
    sigma : int
        Sigma of the Gaussian window used to separate the low-pass and
        high-pass components of the intensity profile of each column.
    size : int
        Window size of the median filter.
    dim : {1, 2}, optional
        Dimension of the window.

    Returns
    -------
    ndarray
        2D array. Stripe-removed sinogram.

    References
    ----------
    .. [1] https://doi.org/10.1364/OE.26.028396
    """
    pad = min(150, int(0.1 * sinogram.shape[0]))
    sinogram = np.transpose(sinogram)
    sino_pad = np.pad(sinogram, ((0, 0), (pad, pad)), mode='reflect')
    (_, ncol) = sino_pad.shape
    window = gaussian(ncol, std=sigma)
    list_sign = np.power(-1.0, np.arange(ncol))
    sino_smooth = np.copy(sinogram)
    for i, sino_1d in enumerate(sino_pad):
        sino_smooth[i] = np.real(
            fft.ifft(fft.fft(sino_1d * list_sign) * window) * list_sign)[pad:ncol - pad]
    sino_sharp = sinogram - sino_smooth
    sino_smooth_cor = np.transpose(
        remove_stripe_based_sorting(np.transpose(sino_smooth), size, dim))
    return np.transpose(sino_smooth_cor + sino_sharp)

def remove_stripe_based_sorting_fitting(sinogram, order, sigmax, sigmay):
    """
    Remove stripes using the sorting and fitting technique, combination of
    algorithm 2 and 1 in Ref. [1]. Angular direction is along the axis 0.

    Parameters
    ----------
    sinogram : array_like
        2D array. Sinogram image.
    order : int
        Polynomial fit order.
    sigmax : int
        Sigma of the Gaussian window in the x-direction.
    sigmay : int
        Sigma of the Gaussian window in the y-direction.

    Returns
    -------
    ndarray
        2D array. Stripe-removed sinogram.

    References
    ----------
    .. [1] https://doi.org/10.1364/OE.26.028396
    """
    sinogram = np.transpose(sinogram)
    (nrow, ncol) = sinogram.shape
    list_index = np.arange(0.0, ncol, 1.0)
    mat_index = np.tile(list_index, (nrow, 1))
    mat_comb = np.asarray(np.dstack((mat_index, sinogram)))
    mat_sort = np.asarray(
        [row[row[:, 1].argsort()] for row in mat_comb])
    sino_sort = mat_sort[:, :, 1]
    sino_cor = np.transpose(
        remove_stripe_based_fitting(np.transpose(sino_sort), order, sigmax, sigmay))
    mat_sort[:, :, 1] = sino_cor
    mat_sort_back = np.asarray(
        [row[row[:, 0].argsort()] for row in mat_sort])
    return np.transpose(mat_sort_back[:, :, 1])

def remove_stripe_based_2d_filtering_sorting(sinogram, sigma, size, dim=1):
    """
    Remove stripes using a 2D low-pass filter and the sorting-based technique,
    algorithm in section 3.3.4 in Ref. [1]. Angular direction is along the axis 0.

    Parameters
    ---------
    sinogram : array_like
        2D array. Sinogram image.
    sigma : int
        Sigma of the Gaussian window.
    size : int
        Window size of the median filter.
    dim : {1, 2}, optional
        Dimension of the window.

    Returns
    -------
    ndarray
         2D array. Stripe-removed sinogram.

    References
    ----------
    .. [1] https://doi.org/10.1117/12.2530324
    """
    pad = min(150, int(0.1 * sinogram.shape[0]))
    sino_smooth = apply_gaussian_filter(sinogram, sigma, sigma, pad)
    sino_sharp = sinogram - sino_smooth
    sino_cor = remove_stripe_based_sorting(sino_sharp, size, dim)
    return sino_smooth + sino_cor

def remove_stripe_based_interpolation(sinogram, snr, size, drop_ratio=0.1, norm=True):
    """
    Combination of algorithm 4, 5, and 6 in Ref. [1].
    Remove stripes using a detection technique and an interpolation method.
    Angular direction is along the axis 0.

    Parameters
    ----------
    sinogram : array_like
        2D array. Sinogram image
    snr : float
        Ratio used to segment between useful information and noise.
    size : int
        Window size of the median filter used to detect stripes.
    drop_ratio : float, optional
        Ratio of pixels to be dropped, which is used to to reduce
        the possibility of the false detection of stripes.
    norm : bool, optional
        Apply normalization if True.

    Returns
    -------
    ndarray
        2D array. Stripe-removed sinogram.

    References
    ----------
    .. [1] https://doi.org/10.1364/OE.26.028396
    """
    drop_ratio = np.clip(drop_ratio, 0.0, 0.8)
    sinogram = np.copy(sinogram)
    (nrow, ncol) = sinogram.shape
    ndrop = int(0.5 * drop_ratio * nrow)
    sino_sort = np.sort(sinogram, axis=0)
    sino_smooth = median_filter(sino_sort, (1, size))
    list1 = np.mean(sino_sort[ndrop:nrow - ndrop], axis=0)
    list2 = np.mean(sino_smooth[ndrop:nrow - ndrop], axis=0)
    list_fact = np.divide(list1, list2,
                          out=np.ones_like(list1), where=list2 != 0)
    list_mask = detect_stripe(list_fact, snr)
    list_mask = np.float32(binary_dilation(list_mask, iterations=1))
    mat_fact = np.tile(list_fact, (nrow, 1))
    if norm is True:
        sinogram = sinogram / mat_fact
    list_mask[0:2] = 0.0
    list_mask[-2:] = 0.0
    listx = np.where(list_mask < 1.0)[0]
    listy = np.arange(nrow)
    matz = sinogram[:, listx]
    finter = interpolate.interp2d(listx, listy, matz, kind='linear')
    listx_miss = np.where(list_mask > 0.0)[0]
    if len(listx_miss) > 0:
        sinogram[:, listx_miss] = finter(listx_miss, listy)
    return sinogram


<reponame>uluturki/Mathematics-of-Epidemics-on-Networks<filename>docs/examples/fig1p2.py
import EoN
import networkx as nx
from matplotlib import rc
import matplotlib.pylab as plt

import scipy
import random

colors = ['#5AB3E6','#FF2000','#009A80','#E69A00', '#CD9AB3', '#0073B3',
        '#F0E442']

#commands to make legend be in LaTeX font
#rc('font', **{'family': 'serif', 'serif': ['Computer Modern']})
rc('text', usetex=True)

rho = 0.025
target_k = 6
#Ns=[10000]#, 1000000]
N=10000
tau = 0.5
gamma = 1.
ts = scipy.arange(0,40,0.05)
count = 50 #number of simulations to run for each

def generate_network(Pk, N, ntries = 100):
    r'''Generates an N-node random network whose degree distribution is given by Pk'''
    counter = 0
    while counter< ntries:
        counter += 1
        ks = []
        for ctr in range(N):
            ks.append(Pk())
        if sum(ks)%2 == 0:
            break
    if sum(ks)%2 ==1:
        raise EoN.EoNError("cannot generate even degree sum")
    G = nx.configuration_model(ks)
    return G

#An erdos-renyi network has a Poisson degree distribution.
def PkPoisson():
    return scipy.random.poisson(target_k)
def PsiPoisson(x):
    return scipy.exp(-target_k*(1-x))
def DPsiPoisson(x):
    return target_k*scipy.exp(-target_k*(1-x))

#a regular (homogeneous) network has a simple generating function.

def PkHomogeneous():
    return target_k
def PsiHomogeneous(x):
    return x**target_k
def DPsiHomogeneous(x):
    return target_k*x**(target_k-1)

#The following 30 - 40 lines or so are devoted to defining the degree distribution
#and the generating function of the truncated power law network.

#defining the power law degree distribution here:
assert(target_k==6) #if you've changed target_k, then you'll
                   #want to update the range 1..61 and/or
                   #the exponent 1.5.

PlPk = {}
exponent = 1.5
kave = 0
for k in range(1,61):
    PlPk[k]=k**(-exponent)
    kave += k*PlPk[k]

normfactor= sum(PlPk.values())
for k in PlPk:
    PlPk[k] /= normfactor

def PkPowLaw():
    r = random.random()
    for k in PlPk:
        r -= PlPk[k]
        if r<0:
            return k

def PsiPowLaw(x):
    #print PlPk
    rval = 0
    for k in PlPk:
        rval += PlPk[k]*x**k
    return rval

def DPsiPowLaw(x):
    rval = 0
    for k in PlPk:
        rval += k*PlPk[k]*x**(k-1)
    return rval
#End of power law network properties.

def process_degree_distribution(N, Pk, color, Psi, DPsi, symbol, label, count):
    report_times = scipy.linspace(0,30,3000)
    sums = 0*report_times
    for cnt in range(count):
        G = generate_network(Pk, N)
        t, S, I, R = EoN.fast_SIR(G, tau, gamma, rho=rho)
        plt.plot(t, I*1./N, '-', color = color,
                                alpha = 0.1, linewidth=1)
        subsampled_I = EoN.subsample(report_times, t, I)
        sums += subsampled_I*1./N
    ave = sums/count
    plt.plot(report_times, ave, color = 'k')

    #Do EBCM
    N= G.order()#N is arbitrary, but included because our implementation of EBCM assumes N is given.
    t, S, I, R = EoN.EBCM_uniform_introduction(N, Psi, DPsi, tau, gamma, rho, tmin=0, tmax=10, tcount = 41)
    plt.plot(t, I/N, symbol, color = color, markeredgecolor='k', label=label)

    for cnt in range(3):  #do 3 highlighted simulations
        G = generate_network(Pk, N)
        t, S, I, R = EoN.fast_SIR(G, tau, gamma, rho=rho)
        plt.plot(t, I*1./N, '-', color = 'k', linewidth=0.1)

plt.figure(figsize=(8,4))

#Powerlaw
process_degree_distribution(N, PkPowLaw, colors[3], PsiPowLaw, DPsiPowLaw, 'd', r'Truncated Power Law', count)

#Poisson
process_degree_distribution(N, PkPoisson, colors[0], PsiPoisson, DPsiPoisson, '^', r'Erd\H{o}s--R\'{e}nyi', count)

#Homogeneous
process_degree_distribution(N, PkHomogeneous, colors[2], PsiHomogeneous, DPsiHomogeneous, 's', r'Homogeneous', count)

plt.xlabel(r'$t$', fontsize=12)
plt.ylabel(r'Proportion infected', fontsize=12)
plt.legend(loc = 'upper right', numpoints = 1)

plt.axis(xmax=10, xmin=0, ymin=0)
plt.savefig('fig1p2.png')


<gh_stars>0
from sympy import Rational as frac
from sympy import sqrt

from ..helpers import article, untangle
from ._helpers import HexahedronScheme, fs_r00, fs_rr0, fs_rrs

citation = article(
    authors=["<NAME>"],
    title="A Formula for Approximate Computation of a Triple Integral",
    journal="The American Mathematical Monthly",
    volume="47",
    number="8",
    month="oct",
    year="1940",
    pages="539-543",
    url="https://doi.org/10.2307/2303834",
)

def sadowsky():
    data = [
        (frac(91, 450), fs_r00(1)),
        (frac(-20, 225), fs_rr0(1)),
        (frac(8, 225), fs_rrs(sqrt(frac(5, 8)), 1)),
    ]
    points, weights = untangle(data)
    weights *= 8
    return HexahedronScheme("Sadowsky", weights, points, 5, citation)