<reponame>zischg/targetedheuristics<filename>targetedheuristics_vulnerabilityfunctions.py<gh_stars>0
#***************************************************************
#script for plotting flood vulnerability functions including targeted heuristics method
#<NAME>, 09.11.2019
#***************************************************************
import math
import numpy
import pandas as pd
from scipy.interpolate import interp1d
import matplotlib.pyplot as plt

#**************************************************************************
#functions block
#**************************************************************************
def definebuildingcategory(G_KATS_NWG,G_BAUP, buildingvalue):
    # buildingvalue in Swiss Francs
    #catlist=["EA", "EN", "MA", "MN", "GG", "GK", "SH", "SS"]
    if G_BAUP>8018 and G_KATS_NWG==1021:
        buildingcategory="EN"#detached house new
    elif G_BAUP<=8018 and G_KATS_NWG==1021:
        buildingcategory="EA"#detached house old
    elif G_BAUP>8018 and G_KATS_NWG in [1025,2046]:
        buildingcategory="MN"#apartment building new
    elif G_BAUP<=8018 and G_KATS_NWG in [1025,2046]:
        buildingcategory="MA"#apartment building old
    else:
        if buildingvalue>850000:
            buildingcategory="GG"
        else:
            buildingcategory="GK"
    return buildingcategory
def vulnerabilityTotschnig(flowdepth):
    # vulnerability function (Totschnig et al. 2011)
    # <NAME>., <NAME>., and <NAME>.: A quantitative vulnerability function for fluvial sediment transport
    # Nat Hazards, 58, 681-703, doi:10.1007/s11069-010-9623-5, 2011.
    # requires math
    dol = 0.0
    if float(flowdepth) >=0.0:
        u = (float(flowdepth)+1.442)/1.442-1
        v = math.pow(u, 2.233)
        w = -0.443*v
        x = 1-math.exp(w)
        if x >=0.0 and x <=1.0 :
            dol = x
        elif x>1.0:
            dol = 1.0
        elif x<0.0:
            dol = 0.0
    else:
        dol = 0.0
    if dol>1:
        dol=1
    elif dol<0:
        dol=0
    return dol
def vulnerabilityFuchs(magnitude):
    #<NAME>., <NAME>., <NAME>., <NAME>., Papathoma - <NAME>., and <NAME>.: Short communication: A model to predict flood loss in mountain areas,
    #Environ.Modell.Softw., doi:10.1016/j.envsoft.2019.03.026, 2019
    dol = 0.0
    if magnitude <= 0:
        dol = 0.0
    else:
        dol = (pow(math.e,-7.40+2.56*magnitude)+(pow(math.e,-3.27+1.67*magnitude)/(1+pow(math.e,-3.27+1.67*magnitude))))/(1+pow(math.e,-9.49*magnitude)+pow(math.e,-7.40+2.56*magnitude))
    if dol>1:
        dol=1.0
    elif dol<0:
        dol=0.0
    return dol
def vulnerabilityPapathoma(magnitude):
    # vulnerability function (Papathoma)
    # Papathoma-<NAME>., <NAME>., <NAME>., <NAME>., and <NAME>.:
    # Loss estimation for landslides in mountain areas - An integrated toolbox for vulnerability assessment
    # and damage documentation, Environmental Modelling & Software, 63, 156-169, doi:10.1016/j.envsoft.2014.10.003, 2015.
    # requires math
    dol = 0.0
    a=-1.671
    b=3.189
    c=1.746
    if magnitude <= 0:
        dol = 0.0
    else:
        dol = 1-pow(math.e,a*pow(((magnitude+b)/b-1),c))
    if dol>1:
        dol=1.0
    elif dol<0:
        dol=0.0
    return dol
def vulnerabilityHydrotec(magnitude):
    # vulnerability function HYDROTEC (2001)
    # <NAME>., <NAME>., <NAME>., <NAME>. (2009): Flood risk
    # analyses - how detailed do we need to be?.- Natural Hazards, 49, 1, 79-98
    # DOI: 10.1007/s11069-008-9277-8
    # Squareroot function
    # requires math
    dol=0.0
    if magnitude <= 0:
        dol = 0.0
    else:
        dol = (27*math.sqrt(magnitude))/100
    if dol>1:
        dol=1.0
    elif dol<0:
        dol=0.0
    return dol
def vulnerabilityKaltenrieder(flowdepth):
    # *******************************************************************************
    # <NAME> (2017, Master thesis at GIUB):
    # "Physische Vulnerabilitaet von Gebaeuden gegenueber Hochwasserereignissen".
    # Model 1:
    # 1 explaining Variable (flow depth), Tab 24, p 54
    # Power 2 function#
    # *******************************************************************************
    dol=0.0
    if flowdepth <= 0:
        dol=0.0
    else:
        dol=math.pow((0.18846 + 0.17152*flowdepth),2)
    if dol > 1:
        dol = 1.0
    elif dol < 0:
        dol == 0.0
    return dol
def vulnerabilityJonkman(flowdepth, arr):
    # ***********************************************************************************************************************
    # <NAME>., <NAME>., <NAME>., and <NAME>.: Integrated hydrodynamic and economic modelling of flood
    # damage in the Netherlands, Ecological Economics, 66, 77-90, https://doi.org/10.1016/j.ecolecon.2007.12.022, 2008.
    # ***********************************************************************************************************************
    # requires math
    dol=0.0
    depthmin=min(arr[:,0])
    depthmax=max(arr[:,0])
    dolmax=max(arr[:,1])
    x=arr[:,0]
    y=arr[:,1]
    f=interp1d(x, y)
    if flowdepth < depthmin:
        dol = 0.0
    elif flowdepth > depthmax:
        dol = dolmax
    else:
        dol = f(float(flowdepth))
    if dol>1:
        dol=1.0
    elif dol<0:
        dol=0.0
    return dol
def vulnerabilityDutta(flowdepth, arr):
    # *******************************************************************************
    # <NAME>, <NAME>, <NAME>. A mathematical model for
    # flood loss estimation. Journal of Hydrology, Volume 277 (1-2), 2003.
    # *******************************************************************************
    # requires math
    dol=0.0
    depthmin=min(arr[:,0])
    depthmax=max(arr[:,0])
    dolmax=max(arr[:,1])
    x=arr[:,0]
    y=arr[:,1]
    f=interp1d(x, y)
    if flowdepth < depthmin:
        dol = 0.0
    elif flowdepth > depthmax:
        dol = dolmax
    else:
        dol = f(float(flowdepth))
    if dol>1:
        dol=1.0
    elif dol<0:
        dol=0.0
    return dol
def vulnerabilityKaragiorgos(flowdepth):
    # vulnerability function (Karagiorgos et al. 2016)
    # Karagiorgos, Konstantinos; <NAME>; <NAME>; <NAME>; <NAME> (2016): Integrated flash flood vulnerability assessment. Insights from East Attica, Greece. In Journal of Hydrology 541, pp.553-562. DOI: 10.1016/j.jhydrol.2016.02.052.
    if float(flowdepth) >=0.0:
        u = (float(flowdepth)+1.76)/1.76-1
        v = math.pow(u, 0.283)
        w = -0.032*v
        x = 1-math.exp(w)
        if x >=0.0 and x <=1.0 :
            dol = x
        elif x>1.0:
            dol = 1.0
        elif x<0.0:
            dol = 0.0
    else:
        dol = 0.0
    if dol>1:
        dol=1
    elif dol<0:
        dol=0
    return dol
def vulnerabilityEconoMe(flowdepth, velocity, buildingcategory):
    # *******************************************************************************
    # <NAME> (Ed.) 2009. Risikokonzept für Naturgefahren - Leitfaden.
    # Nationale Plattform für Naturgefahren PLANAT, Bern. 420 S.
    # s. Chapter 4 (Richtwerte EconoMe), p. 158
    # resid_units = 1: single-family house (default)
    # resid_units > 1: multi-family house
    # resid_units = 0: non-residential buildings
    # *******************************************************************************
    dol = 0.0
    vh = velocity * flowdepth
    if buildingcategory in ["EN","EA"]:#G_KATS_NWG = 1021
        #single family house
        if flowdepth > 0 and flowdepth < 0.5:
          dol = 0.02
        elif (flowdepth >= 0.5 and flowdepth < 2) or (vh >= 0.5 and vh < 2):
          dol = 0.2
        elif flowdepth >= 2 or vh >= 2:
          dol = 0.3
    elif buildingcategory in ["MN","MA"]:#G_KATS_NWG == 1025 or G_KATS_NWG == 2046
    # Einheitlicher Gebäudetyp Mehrfamilienhaus oder Ein/Mehrfamilienhaus gemischt
        if flowdepth > 0 and flowdepth < 0.5:
            dol = 0.006
        elif (flowdepth >= 0.5 and flowdepth < 2) or (vh >= 0.5 and vh < 2):
            dol = 0.15
        elif flowdepth >= 2 or vh >= 2:
            dol = 0.25
    else:
    # all other building types
        if flowdepth > 0 and flowdepth < 0.5:
          dol = 0.1
        elif (flowdepth >= 0.5 and flowdepth < 2) or (vh >= 0.5 and vh < 2):
          dol = 0.3
        elif flowdepth >= 2 or vh >= 2:
          dol = 0.4
    return dol
def mortalityJonkmanFastRise(magnitude):
    # mortality function Jonkman (2007) - dam breaks
    mue=1.46
    sigma=0.28
    if magnitude <= 0.0:
        dol = 0.0
    else:
        dol = (math.log(magnitude,math.e)-mue)/sigma
    if dol>1:
        dol=1.0
    elif dol<0:
        dol=0.0
    return dol
def mortalityJonkmanSlowRise(magnitude):
    #mortality function Jonkman (2007) - slow rise
    mue=7.60
    sigma=2.75
    if magnitude <= 0.0:
        dol = 0.0
    else:
        dol = (math.log(magnitude,math.e)-mue)/sigma
    if dol>1:
        dol=1.0
    elif dol<0:
        dol=0.0
    return dol
def vulnerabilityMURL(flowdepth):
    if numpy.float(flowdepth) >=0.0 and numpy.float(flowdepth) <=5.0:
        x = 0.02*flowdepth
        if x >=0.0 and x <=1.0 :
            dol = x
        elif x>1.0:
            dol = 1.0
        elif x<0.0:
            dol = 0.0
    elif numpy.float(flowdepth) >=5.0:
        dol=0.10
    else:
        dol = 0.0
    if dol>1:
        dol=1
    elif dol<0:
        dol=0
    return dol
def vulnerabilityICPR(flowdepth):
    if numpy.float(flowdepth) >=0.0:
        x = (2*flowdepth*flowdepth+2*flowdepth)/100.0
        if x >=0.0 and x <=1.0 :
            dol = x
        elif x>1.0:
            dol = 1.0
        elif x<0.0:
            dol = 0.0
    else:
        dol = 0.0
    if dol>1:
        dol=1
    elif dol<0:
        dol=0
    return dol
def vulnerabilityTargetedHeuristics(magnitude, buildingcategory):
    dol=0.0
    if buildingcategory=="EA":
        #old detached house
        if magnitude>0.0 and magnitude <=0.25:
            dol = 0.008
        elif magnitude>0.25 and magnitude <=0.5:
            dol = 0.152
        elif magnitude>0.5 and magnitude <=1.0:
            dol = 0.366
        elif magnitude>1.0:
            dol = 0.464
    elif buildingcategory=="EN":
        #new detached house
        if magnitude>0.0 and magnitude <=0.25:
            dol = 0.148
        elif magnitude>0.25:
            dol = 0.319
    elif buildingcategory=="MA":
        #old apartment building
        if magnitude>0.0 and magnitude <=0.5:
            dol = 0.238
        elif magnitude>0.5 and magnitude <=0.75:
            dol = 0.320
        elif magnitude>0.75 and magnitude <=1.0:
            dol = 0.328
        elif magnitude>1.0:
            dol = 0.393
    elif buildingcategory=="MN":
        #new apartment building
        if magnitude>0.0 and magnitude <=0.25:
            dol = 0.083
        elif magnitude>0.25 and magnitude <=0.5:
            dol = 0.222
        elif magnitude>0.5 and magnitude <=0.75:
            dol = 0.233
        elif magnitude>0.75:
            dol = 0.363
    elif buildingcategory=="GG":
        #commercial property, large
        if magnitude>0.0 and magnitude <=1.0:
            dol = 0.041
        elif magnitude>1.0:
            dol = 0.070
    elif buildingcategory=="GK":
        #commercial property, small
        if magnitude>0.0 and magnitude <=0.25:
            dol = 0.088
        elif magnitude>0.25 and magnitude <=0.5:
            dol = 0.111
        elif magnitude>0.5 and magnitude <=0.75:
            dol = 0.139
        elif magnitude>0.75:
            dol = 0.194
    elif buildingcategory=="SH":
        #hotel
        if magnitude>0.0 and magnitude <=0.75:
            dol = 0.020
        if magnitude>0.75 and magnitude <=1.5:
            dol = 0.051
        elif magnitude>1.5:
            dol = 0.076
    else:
        # all others, same as GK
        if magnitude > 0.0 and magnitude <=0.25:
            dol = 0.088
        elif magnitude > 0.25 and magnitude <=0.5:
            dol = 0.111
        elif magnitude > 0.5 and magnitude <=0.75:
            dol = 0.139
        elif magnitude > 0.75:
            dol = 0.194
    return dol
def flowvelocity(flowdepth, slope, strickler):
    velocity=strickler*math.pow(flowdepth,(2.0/3.0))*math.pow(slope, 0.5)
    return velocity
#**************************************************************************
#end of functions block
#**************************************************************************

#**************************************************************************
#prepare x and y vectors for plotting the vulnerability functions
xlist=numpy.arange(0.0,5.1,0.05).tolist()
ytot=[]
for item in xlist:
    ytot.append(vulnerabilityTotschnig(item))
yfuchs=[]
for item in xlist:
    yfuchs.append(vulnerabilityFuchs(item))
ykar=[]
for item in xlist:
    ykar.append(vulnerabilityKaragiorgos(item))
ypap=[]
for item in xlist:
    ypap.append(vulnerabilityPapathoma(item))
yhyd=[]
for item in xlist:
    yhyd.append(vulnerabilityHydrotec(item))
ykal=[]
for item in xlist:
    ykal.append(vulnerabilityKaltenrieder(item))
yjon=[]
for item in xlist:
    yjon.append(vulnerabilityJonkman(item, jonkmanarr))
ydut=[]
for item in xlist:
    ydut.append(vulnerabilityDutta(item, duttaarr))
yecoEFH=[]
for item in xlist:
    yecoEFH.append(vulnerabilityEconoMe(item, 0, "EN"))
yecoMFH=[]
for item in xlist:
    yecoMFH.append(vulnerabilityEconoMe(item, 0, "MN"))
yecorest=[]
for item in xlist:
    yecorest.append(vulnerabilityEconoMe(item, 0, "GG"))
yheurEN=[]
for item in xlist:
    yheurEN.append(vulnerabilityTargetedHeuristics(item, "EN"))
yheurEA=[]
for item in xlist:
    yheurEA.append(vulnerabilityTargetedHeuristics(item, "EA"))
yheurMN=[]
for item in xlist:
    yheurMN.append(vulnerabilityTargetedHeuristics(item, "MN"))
yheurMA=[]
for item in xlist:
    yheurMA.append(vulnerabilityTargetedHeuristics(item, "MA"))
yheurGG=[]
for item in xlist:
    yheurGG.append(vulnerabilityTargetedHeuristics(item, "GG"))
yheurGK=[]
for item in xlist:
    yheurGK.append(vulnerabilityTargetedHeuristics(item, "GK"))

#plot combined figure vulnerability functions
fig, axes =plt.subplots(nrows=2, ncols=2, figsize = (10,10), sharey=True, sharex=True)
xlabels=[0,1,2,3,4]
x = numpy.arange(len(xlabels))
#targeted heuristics
axes[0][0].plot(xlist,yheurEN,label="new detached building", color="blue")
axes[0][0].plot(xlist,yheurEA,label="old detached building", color="blue", dashes=[6,2])
axes[0][0].plot(xlist,yheurMN,label="new apartment building", color="red")
axes[0][0].plot(xlist,yheurMA,label="old apartment building", color="red", dashes=[6,2])
axes[0][0].plot(xlist,yheurGG,label="commercial building, high value", color="grey")
axes[0][0].plot(xlist,yheurGK,label="commercial building", color="grey", dashes=[6,2])
axes[0][0].set_title('a) targeted heuristics')
#axes[0][0].set_xlabel("flow depth [m]", color="black")
axes[0][0].set_xticks(x)
axes[0][0].set_xticklabels(xlabels)
axes[0][0].legend(loc="upper left", frameon=False)
axes[0][0].set_ylim((0,1))
axes[0][0].set_xlim((0,4))
axes[0][0].set_ylabel("degree of damage [-]")
#multi-model enesemble
axes[0][1].plot(xlist,ydut,label="Dutta et al.")
axes[0][1].plot(xlist,yjon,label="Jonkman et al.")
axes[0][1].plot(xlist,ykar,label="Karagiorgos et al.")
axes[0][1].plot(xlist,yfuchs,label="Fuchs et al.")
axes[0][1].set_title('b) multi-model ensemble')
#axes[0][0].set_xlabel("flow depth [m]", color="black")
axes[0][1].set_xticks(x)
axes[0][1].set_xticklabels(xlabels)
axes[0][1].legend(loc="upper left", frameon=False)
axes[0][1].set_ylim((0,1))
axes[0][1].set_xlim((0,4))
#Kaltenrieder
axes[1][0].plot(xlist,ykal)
axes[1][0].set_title('c) regional calibration')
#axes[0][1].set_ylabel("degree of loss [-]")
#axes[0][1].set_xlabel("flow depth [m]", color="black")
axes[1][0].set_xticks(x)
axes[1][0].set_xticklabels(xlabels)
axes[1][0].set_ylim((0,1))
axes[1][0].set_xlim((0,4))
axes[1][0].set_ylabel("degree of damage [-]")
axes[1][0].set_xticks(x)
axes[1][0].set_xticklabels(xlabels)
axes[1][0].set_xlabel("flow depth [m]", color="black")
#econome
axes[1][1].plot(xlist, yecoEFH, label="detached building", color="blue")
axes[1][1].plot(xlist, yecoMFH, label="apartment building", color="red")
axes[1][1].plot(xlist, yecorest, label="commercial building", color="grey")
axes[1][1].set_title('d) econoMe')
axes[1][1].set_xlabel("flow depth [m]", color="black")
axes[1][1].set_xticks(x)
axes[1][1].set_xticklabels(xlabels)
axes[1][1].legend(loc="upper left", frameon=False)
axes[1][1].set_ylim((0,1))
axes[1][1].set_xlim((0,4))
#axes[1][1].set_ylabel("degree of damage [-]")
axes[1][1].set_xlabel("flow depth [m]", color="black")
axes[1][1].set_xticks(x)
axes[1][1].set_xticklabels(xlabels)
axes[1][1].set_ylim((0,1))
axes[1][1].set_xlim((0,4))
#tight layout
fig.subplots_adjust(hspace=0.4)
fig.tight_layout()
plt.show()


<filename>src/core/zone.py
"""
    Zone Module - reference storage
        - loaded from database.
        - imported from '../data/houses/subfolder/zones.csv'
        - holds zone area and CPE means.
        - holds runtime sampled CPE per zone.
        - calculates Cpe pressure load from wind pressure.
"""
import scipy.stats
import numpy
import math
from sqlalchemy import Integer, String, Float, Column, ForeignKey

import database

# hackerama to get scipy seeded
def seed_scipy(seed=42):
    myrs = numpy.random.RandomState(seed)

    def mysample(size=1):
        return myrs.uniform(size=size)

    numpy.random.sample = mysample

def getZoneLocFromGrid(gridCol, gridRow):
    """
    Create a string location (eg 'A10') from zero based grid refs (col=0,
    row=11)
    """
    locX = chr(ord('A') + gridCol)
    locY = str(gridRow + 1)
    return locX + locY

def getGridFromZoneLoc(loc):
    """
    Extract 0 based grid refs from string location (eg 'A10' to 0, 11)
    """
    locCol = loc[0]
    locRow = int(loc[1:])
    gridCol = ord(locCol) - ord('A')
    gridRow = locRow - 1
    return gridCol, gridRow

dirs = ['S', 'SW', 'W', 'NW', 'N', 'NE', 'E', 'SE']

class Zone(database.Base):
    __tablename__ = 'zones'
    id = Column(Integer, primary_key=True)
    zone_name = Column(String)
    zone_area = Column(Float)
    coeff_N = Column(Float)
    coeff_NE = Column(Float)
    coeff_E = Column(Float)
    coeff_SE = Column(Float)
    coeff_S = Column(Float)
    coeff_SW = Column(Float)
    coeff_W = Column(Float)
    coeff_NW = Column(Float)
    struct_coeff_N = Column(Float)
    struct_coeff_NE = Column(Float)
    struct_coeff_E = Column(Float)
    struct_coeff_SE = Column(Float)
    struct_coeff_S = Column(Float)
    struct_coeff_SW = Column(Float)
    struct_coeff_W = Column(Float)
    struct_coeff_NW = Column(Float)
    eaves_coeff_N = Column(Float)
    eaves_coeff_NE = Column(Float)
    eaves_coeff_E = Column(Float)
    eaves_coeff_SE = Column(Float)
    eaves_coeff_S = Column(Float)
    eaves_coeff_SW = Column(Float)
    eaves_coeff_W = Column(Float)
    eaves_coeff_NW = Column(Float)
    leading_roof_N = Column(Integer)
    leading_roof_NE = Column(Integer)
    leading_roof_E = Column(Integer)
    leading_roof_SE = Column(Integer)
    leading_roof_S = Column(Integer)
    leading_roof_SW = Column(Integer)
    leading_roof_W = Column(Integer)
    leading_roof_NW = Column(Integer)
    cpi_alpha = Column(Float)
    wall_dir = Column(Integer)
    house_id = Column(Integer, ForeignKey('houses.id'))

    def getCpeMeanForDir(self, dir_index):
        return getattr(self, 'coeff_%s' % dirs[dir_index])

    def getCpeStructMeanForDir(self, dir_index):
        return getattr(self, 'struct_coeff_%s' % dirs[dir_index])

    def getCpeEavesMeanForDir(self, dir_index):
        return getattr(self, 'eaves_coeff_%s' % dirs[dir_index])

    def getIsLeadingRoofEdgeForDir(self, dir_index):
        return getattr(self, 'leading_roof_%s' % dirs[dir_index])

    def getIsWallZone(self):
        if len(self.zone_name) > 3 and self.zone_name[0] == 'W':
            return True
        return False

    def __repr__(self):
        return "('%s', '%f', '%f')" % (
        self.zone_name, self.zone_area, self.cpi_alpha)

def calc_A(cpe_k):
    return (1.0 / cpe_k) * (1.0 - math.gamma(1.0 + cpe_k))

def calc_B(cpe_k):
    base_ = math.pow(1.0 / cpe_k, 2) * (
        math.gamma(1.0 + 2 * cpe_k) - math.pow(math.gamma(1.0 + cpe_k), 2))
    return math.pow(base_, 0.5)

def calc_a_u(mean, cpe_V, A, B):
    if mean >= 0:
        a = (mean * cpe_V) / B
        u = mean - a * A
    else:
        mean = abs(mean)
        a = (mean * cpe_V) / B
        u = mean - a * A
    return a, u

def sample_gev(mean, A, B, cpe_V, cpe_k):
    a, u = calc_a_u(mean, cpe_V, A, B)
    if mean >= 0:
        return float(scipy.stats.genextreme.rvs(cpe_k, loc=u, scale=a, size=1))
    else:
        return float(-scipy.stats.genextreme.rvs(cpe_k, loc=u, scale=a, size=1))

def sample_zone_pressures(zones, wind_dir_index, cpe_V, cpe_k, cpe_struct_V):
    """
    Sample external Zone Pressures for sheeting, structure and eaves Cpe,
    based on TypeIII General Extreme Value distribution. Prepare effective
    zone areas for load calculations.
    """
    A = calc_A(cpe_k)
    B = calc_B(cpe_k)
    for z in zones:
        z.result_effective_area = float(z.zone_area)
        z.sampled_cpe = sample_gev(z.getCpeMeanForDir(wind_dir_index), A, B,
                                   cpe_V, cpe_k)
        z.sampled_cpe_struct = sample_gev(
            z.getCpeStructMeanForDir(wind_dir_index), A, B, cpe_struct_V, cpe_k)
        z.sampled_cpe_eaves = sample_gev(
            z.getCpeEavesMeanForDir(wind_dir_index), A, B, cpe_struct_V, cpe_k)

def calc_zone_pressures(zones, wind_dir_index, cpi, qz, Ms, building_spacing,
                        diff_shielding):
    """
    Determine wind pressure loads on each zone (to be distributed onto
    connections)
    """
    for z in zones:
        # optionally apply differential shielding
        diff_shielding = 1.0
        if building_spacing > 0 and diff_shielding:
            front_facing = z.getIsLeadingRoofEdgeForDir(wind_dir_index)
            Ms2 = math.pow(Ms, 2)
            dsn = 1.0
            dsd = 1.0
            if building_spacing == 40 and Ms >= 1.0 and front_facing == 0:
                dsd = Ms2
            elif building_spacing == 20 and front_facing == 1:
                dsd = Ms2
                if Ms <= 0.85:
                    dsn = math.pow(0.7, 2)
                else:
                    dsn = math.pow(0.8, 2)
            diff_shielding = (dsn / dsd)

            # calculate zone pressure
        z.result_pz = qz * (
        z.sampled_cpe - (z.cpi_alpha * cpi)) * diff_shielding

        # calculate zone structure pressure
        z.result_pz_struct = qz * (z.sampled_cpe_struct - (
        z.cpi_alpha * cpi) - z.sampled_cpe_eaves) * diff_shielding

    # unit tests

if __name__ == '__main__':
    import unittest

    class MyTestCase(unittest.TestCase):
        # def test_breaks(self):
        #     self.assertEquals(12, 134)

        def test_zonegrid(self):
            loc = 'N12'
            gridCol, gridRow = getGridFromZoneLoc(loc)
            self.assertEquals(gridRow, 11)
            self.assertEquals(gridCol, 13)
            self.assertEquals(getZoneLocFromGrid(gridCol, gridRow), loc)

        def test_gev_calc(self):
            A = calc_A(0.1)
            B = calc_B(0.1)
            a, u = calc_a_u(0.95, 0.07, A, B)
            self.assertAlmostEqual(a, 0.058, 2)
            self.assertAlmostEqual(u, 0.922, 2)
            self.assertAlmostEqual(A, 0.4865, 3)
            self.assertAlmostEqual(B, 1.1446, 3)

    suite = unittest.TestLoader().loadTestsFromTestCase(MyTestCase)
    unittest.TextTestRunner(verbosity=2).run(suite)


<gh_stars>0
import statistics

class Solution:
  def minMoves2(self, nums: List[int]) -> int:
    median = int(statistics.median(nums))
    return sum(abs(num - median) for num in nums)


"""
Unittesting for pystokes
"""
import sys
import pystokes
import unittest
import inspect
import numpy as np
import scipy as sp
from pystokes.unbounded import Rbm
from pystokes.unbounded import Flow

class UnboundedTest(unittest.TestCase):

    def test_translation(self):
        r = np.array([0,0,0.])
        F = np.array([0,0,1.])

        a, Np, eta = 1, 1, 1
        mu = 1/(6*np.pi*a*eta)

        V1 = mu*F
        V2 = 0*mu*F

        uRbm = pystokes.unbounded.Rbm(a, Np, eta)
        uRbm.mobilityTT(V2, r, F)

        diff = V1[2] - V2[2]
        self.assertTrue((np.asarray(diff) < 0.001).all(),
                       "Stokes law for translation is not satisfied")

    def test_rotation(self):
        r = np.array([0,0,0.])
        T = np.array([0,0,1.])

        a, Np, eta = 1, 1, 1
        mu = 1/(8*np.pi*a**3*eta)

        W1 = mu*T
        W2 = 0*mu*T

        uRbm = pystokes.unbounded.Rbm(a, Np, eta)
        uRbm.mobilityRR(W2, r, T)

        diff = W1[2] - W2[2]
        self.assertTrue((np.asarray(diff) < 0.001).all(),
                       "Stokes law for rotation is not satisfied")

class WallBoundedTest(unittest.TestCase):

    def test_parallelTranslation(self):
        r = np.array([0,0,1.])
        F = np.array([0,1,0.])

        a, Np, eta = 1, 1, 1
        mu = 1/(6*np.pi*a*eta)
        mu = mu*(1- 9./16 + 1/8)  # add the standard wall-correction

        V1 = mu*F
        V2 = 0*mu*F

        uRbm = pystokes.wallBounded.Rbm(a, Np, eta)
        uRbm.mobilityTT(V2, r, F)

        diff = V1[1] - V2[1]
        self.assertTrue((np.asarray(diff) < 0.001).all(),
                       "Stokes law for translation || to wall is not satisfied")

    def test_perpTranslation(self):
        r = np.array([0,0,1.])
        F = np.array([0,0,1.])

        a, Np, eta = 1, 1, 1
        mu = 1/(6*np.pi*a*eta)
        mu = mu*(1 - 9/8. + 1./2)  # add the standard wall-correction

        V1 = mu*F
        V2 = 0*mu*F

        uRbm = pystokes.wallBounded.Rbm(a, Np, eta)
        uRbm.mobilityTT(V2, r, F)

        diff = V1[2] - V2[2]
        self.assertTrue((np.asarray(diff) < 0.001).all(),
                       "Stokes law for translation perp to wall is not satisfied")

class InterfaceTest(unittest.TestCase):

    def test_parallelTranslation(self):
        r = np.array([0,0,1.])
        F = np.array([0,1,0.])

        a, Np, eta = 1, 1, 1
        mu = 1/(6*np.pi*a*eta)
        mu = mu*(1 + 3./8 + 1/16)  # add the standard wall-correction

        V1 = mu*F
        V2 = 0*mu*F

        uRbm = pystokes.interface.Rbm(a, Np, eta)
        uRbm.mobilityTT(V2, r, F)

        diff = V1[1] - V2[1]
        self.assertTrue((np.asarray(diff) < 0.001).all(),
                       "Stokes law for translation || to wall is not satisfied")

    def test_perpTranslation(self):
        r = np.array([0,0,1.])
        F = np.array([0,0,1.])

        a, Np, eta = 1, 1, 1
        mu = 1/(6*np.pi*a*eta)
        mu = mu*(1 - 3/4. + 1./8)  # add the standard wall-correction

        V1 = mu*F
        V2 = 0*mu*F

        uRbm = pystokes.interface.Rbm(a, Np, eta)
        uRbm.mobilityTT(V2, r, F)

        diff = V1[2] - V2[2]
        self.assertTrue((np.asarray(diff) < 0.001).all(),
                       "Stokes law for translation perp to wall is not satisfied")

class PeriodicTest(unittest.TestCase):

    def test_effectiveMobility(self):
        a, eta, Np = 1.0, 1.0/6, 1
        v = np.zeros(3*Np)
        r = np.zeros(3*Np)
        F = np.zeros(3*Np); F[2]=-1

        ll = ((4*np.pi/3)**(1.0/3))/0.3   # length of simulation box
        pRbm = pystokes.periodic.Rbm(a, Np, eta, ll)

        pRbm.mobilityTT(v, r, F)

        mu=1.0/(6*np.pi*eta*a)
        diff = -v[2]/mu - 0.498
        self.assertTrue((np.asarray(diff) < 0.002).all(),
                       "Effective mobility does not match Zick & Homsy (1982)")

if __name__ == '__main__':
    unittest.main()


<reponame>Imperial-iGEM/synbio<filename>synbio/protocols/clone.py
"""Clone fragments together with digestion and ligation."""

from statistics import mean
from typing import Dict, List

from Bio.Restriction.Restriction import RestrictionType
from Bio.SeqRecord import SeqRecord

from ..assembly import clone_many_combinatorial
from ..containers import Container, Well
from ..designs import Design
from ..instructions import Temperature
from ..mix import Mix
from ..protocol import Protocol
from ..reagents import Reagent
from ..steps import Setup, Pipette, ThermoCycle, HeatShock

CLONING_MIX = Mix(
    {Reagent("10X NEBuffer"): 5.0, SeqRecord: 1.0, RestrictionType: 1.0},
    fill_with=Reagent("water"),
    fill_to=50.0,
)

class Clone(Protocol):
    """Clone SeqRecords together using BioPython enzymes.

    Digest the SeqRecords with all the Enzymes provided, find valid circularized
    assemblies, and create a protocol for preparing and ligating the fragments.

    This protocol is based on NEB's clone guide:
    https://www.neb.com/tools-and-resources/usage-guidelines/clone-guide

    Keyword Args:
        enzymes: the list of enzymes to digest the SeqRecords with
        include: include only plasmids with a feature matching something
            in the include list use in backbone selection (default: {None})
        mix: the assembly mix to use when mixing the assemblies with enzymes
        min_count: the minimum number of SeqRecords in an assembly for it to
            be considered valid. smaller assemblies are ignored
    """

    def __init__(
        self,
        name: str = "",
        design: Design = Design(),
        enzymes: List[RestrictionType] = None,
        mix: Mix = CLONING_MIX,
        include: List[str] = None,
        min_count: int = -1,
        separate_reagents: bool = False,
    ):
        super().__init__(name=name, design=design, separate_reagents=separate_reagents)

        self.enzymes = enzymes or []
        self.include = include
        self.mix = mix
        self.min_count = min_count
        self.wells_to_construct: Dict[Container, Container] = {}

    def run(self):
        """Filter designs to those that will form valid and new plasmids.

        Run each Clone step on the protocol. See:
        https://www.neb.com/protocols/2018/10/02/golden-gate-assembly-protocol-for-using-neb-golden-gate-assembly-mix-e1601
        """

        # get all the unique contents and set them up in their own wells
        mixed_wells = self._create_mixed_wells()

        # get the mean incubation temperature from the enzymes
        incubate_temp = mean([e.opt_temp for e in self.enzymes])

        for step in [
            Setup(target=mixed_wells),
            Pipette(
                name="Mix DNA with the enzymes, NEBuffer, and water", target=mixed_wells
            ),
            ThermoCycle(
                [
                    Temperature(temp=incubate_temp, time=3600),
                    Temperature(temp=65, time=300),
                    Temperature(temp=4, time=-1),  # hold at 4 degrees
                ],
                mutate=self.mutate,  # set the SeqRecords
            ),
        ] + HeatShock:
            step(self)

    def _create_mixed_wells(self) -> List[Container]:
        """Return the valid circularizable assemblies.

        Also build up the dictionary for `self.wells_to_construct`, a map from sorted
        Fragment IDs to the SeqRecord that they will form after digestion and ligation.

        Returns:
            a list of wells to mix fragments for Clone
        """

        if not self.enzymes:
            raise ValueError("Clone protocol lacks list of BioPython Enzymes")

        mixed_wells: List[Container] = []
        for plasmids, fragments in clone_many_combinatorial(
            self.design,
            enzymes=self.enzymes,
            include=self.include,
            min_count=self.min_count,
            linear=self.design.linear,
        ):
            # add reaction mix and water
            well_contents, well_volumes = self.mix(fragments + self.enzymes)

            # create a well that mixes the assembly mix, plasmids, and reagents
            well = Well(contents=well_contents, volumes=well_volumes)

            # used in self.mutate
            self.wells_to_construct[well] = Well(plasmids, [sum(well_volumes)])
            mixed_wells.append(well)

        if not mixed_wells:
            raise RuntimeError(f"Failed to create any Clone assemblies")

        return sorted(mixed_wells)

    def mutate(self, well: Container) -> Container:
        """Given the contents of a well, return single SeqRecord after digest/ligation."""

        if well in self.wells_to_construct:
            return self.wells_to_construct[well]

        raise KeyError(f"{well} not recognized as a Clone assembly")


<gh_stars>1-10
import os
import dill
import shutil
import numpy as np
from glob import glob
from subprocess import call
from subprocess import Popen
from subprocess import PIPE
from time import sleep
from tqdm import tqdm
from scipy.interpolate import LinearNDInterpolator
from saenopy import Solver
import saenopy
from saenopy.materials import SemiAffineFiberMaterial
import pandas as pd

def read_meshfile(meshfile, r_inner=None, r_outer=None):
    # open mesh file
    with open(meshfile, 'r') as f:
        lines = f.readlines()

    # if r_inner or r_outer are defined in the mesh-file, ignore user input
    meshinfo = {}
    try:
        paragraph = lines[lines.index('$Jointforces\n') + 1:lines.index('$EndJointforces\n')]
        for line in paragraph:
            key, value = line.strip().split('=')
            try:
                meshinfo[key] = float(value)
            except ValueError:
                meshinfo[key] = value

        r_inner = meshinfo['r_inner']
        r_outer = meshinfo['r_outer']

        print('Geometry for spherical contraction is defined in mesh file (r_inner={:.2f}, r_outer={:.2f}).'.format(r_inner, r_outer))
        if (r_inner is not None) or (r_outer is not None):
            print('Will ignore user-defined values of r_inner and r_outer.')
    except ValueError:
        if r_inner is None:
            raise ValueError('r_inner not defined')
        if r_outer is None:
            raise ValueError('r_outer not defined')

    # scale radii to meter
    r_inner *= 10**-6
    r_outer *= 10**-6

    # transform nodes and connection in SAENO format
    # nodes
    index_nodes = lines.index('$Nodes\n')
    n_nodes = int(lines[index_nodes + 1])

    coords = np.zeros((n_nodes, 3))
    for i in range(n_nodes):
        coords[i] = np.array([np.float(x) for x in lines[i + index_nodes + 2].split()[1:]])

    # connections
    index_elements = lines.index('$Elements\n')
    n_elements = int(lines[index_elements + 1])

    tets = np.zeros((n_elements, 4))
    for i in range(n_elements):
        tets[i] = lines[i + index_elements + 2].split()[-4:]

    # to start with 0 and not 1
    tets -= 1

    return coords, tets, r_inner, r_outer

def spherical_contraction_solver(meshfile, outfolder, pressure, material, r_inner=None, r_outer=None, logfile=False, initial_displacenemts=None, max_iter = 600, step = 0.0033, conv_crit = 0.01):

    coords, tets, r_inner, r_outer = read_meshfile(meshfile, r_inner, r_outer)

    # read in material parameters
    K_0 = material['K_0']
    D_0 = material['D_0']
    L_S = material['L_S']
    D_S = material['D_S']

    # create output folder if it does not exist, print warning otherwise
    if not os.path.exists(outfolder):
        os.makedirs(outfolder)
    else:
        print('WARNING: Output folder already exists! ({})'.format(outfolder))

    # Initialize saenopy solver opbject
    M = Solver()
    material_saenopy = SemiAffineFiberMaterial(K_0, D_0, L_S, D_S)
    M.setMaterialModel(material_saenopy)
    M.setNodes(coords)
    M.setTetrahedra(tets)

    # define boundary conditions
    distance = np.sqrt(np.sum(coords ** 2., axis=1))
    mask_inner = distance < r_inner * 1.001
    mask_outer = distance > r_outer * 0.999

    # Save Node Density at inner and outer sphere
    # Area per inner node
    A_node_inner = (np.pi*4*(r_inner)**2)/np.sum(mask_inner)
    # simple sqrt as spacing
    inner_spacing = np.sqrt(A_node_inner)

    # Area per outer node
    A_node_outer = (np.pi*4*(r_outer)**2)/np.sum(mask_outer)
    # simple sqrt as spacing
    outer_spacing = np.sqrt(A_node_outer)

    print ('Inner node spacing: '+str(inner_spacing*1e6)+'µm')
    print ('Outer node spacing: '+str(outer_spacing*1e6)+'µm')

    # displacements are everywhere NaN
    bcond_displacement = np.zeros((len(coords), 3))*np.nan
    # except of a the outer border
    bcond_displacement[mask_outer] = 0

    # forces are everywhere 0
    bcond_forces = np.zeros((len(coords), 3))
    # except at the outer boder there they are NaN
    bcond_forces[mask_outer] = np.nan
    # and at the innter border they depend on the pressure
    bcond_forces[mask_inner] = coords[mask_inner]
    bcond_forces[mask_inner] /= distance[mask_inner, None]
    A_inner = 4 * np.pi * r_inner ** 2.
    force_per_node = pressure * A_inner / np.sum(mask_inner)
    bcond_forces[mask_inner, :3] *= force_per_node

    # give the boundary conditions to the solver
    M.setBoundaryCondition(bcond_displacement, bcond_forces)

    if initial_displacenemts is not None:
        M.setInitialDisplacements(initial_displacenemts)

    # create info file with all relevant parameters of the simulation
    parameters = r"""K_0 = {}
D_0 = {}
L_S = {}
D_S = {}
PRESSURE = {}
FORCE_PER_SURFACE_NODE = {}
INNER_RADIUS = {} µm
OUTER_RADIUS = {} µm
INNER_NODE_SPACING = {} µm
OUTER_NODE_SPACING = {} µm
SURFACE_NODES = {}
TOTAL_NODES = {}""".format(K_0, D_0, L_S, D_S, pressure, force_per_node, r_inner*1e6, r_outer*1e6, inner_spacing*1e6,
                           outer_spacing*1e6, np.sum(mask_inner), len(coords))

    with open(outfolder + "/parameters.txt", "w") as f:
        f.write(parameters)

    # solve the boundary problem
    M.solve_boundarycondition(stepper=step, i_max=max_iter, rel_conv_crit=conv_crit, relrecname=outfolder + "/relrec.txt") #, verbose=True
    M.save(outfolder + "/solver.npz")

def distribute_solver(func, const_args, var_arg='pressure', start=0.1, end=1000, n=120, log_scaling=True, n_cores=None, get_initial=True, max_iter = 600, step = 0.0033, conv_crit = 0.01, callback=None):
    # get_intial = True takes the deformationfield from previous simulation as start values for the next simulations, which reduces computation time

    # by default use spherical contraction as function
    func = spherical_contraction_solver

    if n_cores is None:
        n_cores = os.cpu_count()

    if log_scaling:
        values = np.logspace(np.log10(start), np.log10(end), n, endpoint=True)
    else:
        values = np.linspace(np.log10(start), np.log10(end), n, endpoint=True)

    outfolder = const_args['outfolder']
    del const_args['outfolder']

    if not os.path.exists(outfolder):
        os.makedirs(outfolder)

    np.savetxt(outfolder+'/'+var_arg+'-values.txt', values)

    values = list(values)

    if n_cores == 1:
        U = None
        for index, v in enumerate(values):
            func(const_args["meshfile"], outfolder + '/simulation' + str(index).zfill(6),
                    v, const_args["material"], None, None, False, U, max_iter, step, conv_crit)
            # get the last displacement
            name = outfolder + '/simulation' + str(index).zfill(6) + "/solver.npz"
            if os.path.exists(name):
                M = saenopy.load(name)
                U = M.U
            # optionally call the callback
            if callback is not None:
                callback(index, len(values))
    else:
        index = 0
        processes = []
        import multiprocessing
        while True:
            processes = [p for p in processes if p.is_alive()]

            if len(processes) < n_cores and index < len(values):
                v = values[index]
                U = None
                if get_initial==True:
                    for i in range(index-3, -1, -1):
                        name = outfolder+'/simulation'+str(i).zfill(6) + "/solver.npz"
                        if os.path.exists(name):
                            M = saenopy.load(name)
                            U = M.U
                            break
                p = multiprocessing.Process(target=func,
                                            args=(const_args["meshfile"],  outfolder+'/simulation'+str(index).zfill(6),
                                                  v, const_args["material"], None, None, False, U, max_iter, step, conv_crit))

                p.start()
                processes.append(p)
                if callback is not None:
                    callback(index, len(values))
                index += 1
            sleep(1.)

            if len(processes) == 0:
                break
    if callback is not None:
        callback(index, len(values))
    return

def extract_deformation_curve_solver(folder, x):
    # get simulation parameters
    with open(folder+'/parameters.txt', 'r') as f:
        lines = f.readlines()

    parameters = {}
    for line in lines:
        try:
            key, value = line.split('= ')
            value = value.split(' ')[0]
            parameters[key.strip()] = float(value.strip())
        except:
            pass

    # load coordinates
    M = saenopy.load(folder + "/solver.npz")
    coords = M.R
    # load displacements
    displ = M.U

    # compute binned normed displacements and normed coordinates
    u = np.sqrt(np.sum(coords ** 2., axis=1)) / (parameters['INNER_RADIUS']*10**-6)
    v = np.sqrt(np.sum(displ ** 2., axis=1)) / (parameters['INNER_RADIUS']*10**-6)

    y = np.array([np.nanmedian(v[(u >= x[i]) & (u < x[i + 1])]) for i in range(len(x) - 1)])

    # save results
    results = {'pressure': parameters['PRESSURE'], 'bins': x, 'displacements': y}
    return results

def create_lookup_table_solver(folder, x0=1, x1=50, n=100):
    subfolders = glob(folder+'/*/')

    x = np.logspace(np.log10(x0), np.log10(x1), n+1, endpoint=True)
    x_center = 10**(0.5*(np.log10(x[1:]) + np.log10(x[:-1])))

    pressure_values = []
    displacement_curves = []

    for subfolder in tqdm(subfolders):
        res = extract_deformation_curve_solver(subfolder, x)
        pressure_values.append(res['pressure'])
        displacement_curves.append(res['displacements'])

    pressure_values = np.array(pressure_values)
    displacement_curves = np.array(displacement_curves)

    return {'pressure': pressure_values, 'x': x_center, 'y': displacement_curves}

"""
Command_line_version
"""

def spherical_contraction(meshfile, outfolder, pressure, material, r_inner=None, r_outer=None, logfile=False,  max_iter = 600, step = 0.0033, conv_crit = 0.01):
    # open mesh file
    with open(meshfile, 'r') as f:
        lines = f.readlines()

    # if r_inner or r_outer are defined in the mesh-file, ignore user input
    meshinfo = {}
    try:
        paragraph = lines[lines.index('$Jointforces\n') + 1:lines.index('$EndJointforces\n')]
        for line in paragraph:
            key, value = line.strip().split('=')
            try:
                meshinfo[key] = float(value)
            except ValueError:
                meshinfo[key] = value

        r_inner = meshinfo['r_inner']
        r_outer = meshinfo['r_outer']

        print('Geometry for spherical contraction is defined in mesh file (r_inner={:.2f}, r_outer={:.2f}).'.format(r_inner, r_outer))
        if (r_inner is not None) or (r_outer is not None):
            print('Will ignore user-defined values of r_inner and r_outer.')
    except ValueError:
        if r_inner is None:
            raise ValueError('r_inner not defined')
        if r_outer is None:
            raise ValueError('r_outer not defined')

    # scale radii to meter
    r_inner *= 10**-6
    r_outer *= 10**-6

    # read in material parameters
    K_0 = material['K_0']
    D_0 = material['D_0']
    L_S = material['L_S']
    D_S = material['D_S']

    # create output folder if it does not exist, print warning otherwise
    if not os.path.exists(outfolder):
        os.makedirs(outfolder)
    else:
        print('WARNING: Output folder already exists! ({})'.format(outfolder))

    # transform nodes and connection in SAENO format
    # nodes
    index_nodes = lines.index('$Nodes\n')
    n_nodes = int(lines[index_nodes + 1])

    coords = np.zeros((n_nodes, 3))
    for i in range(n_nodes):
        coords[i] = np.array([np.float(x) for x in lines[i + index_nodes + 2].split()[1:]])
    np.savetxt(outfolder + '/coords.dat', coords)

    # connections
    index_elements = lines.index('$Elements\n')
    n_elements = int(lines[index_elements + 1])

    tets = np.zeros((n_elements, 4))
    for i in range(n_elements):
        tets[i] = lines[i + index_elements + 2].split()[-4:]
    np.savetxt(outfolder + '/tets.dat', tets, fmt='%i')

    # define boundary conditions
    distance = np.sqrt(np.sum(coords ** 2., axis=1))
    mask_inner = distance < r_inner * 1.001
    mask_outer = distance > r_outer * 0.999

    # Save Node Density at inner and outer sphere
    # Area per inner node
    A_node_inner = (np.pi*4*(r_inner)**2)/np.sum(mask_inner)
    # simple sqrt as spacing
    inner_spacing = np.sqrt(A_node_inner)

    # Area per outer node
    A_node_outer = (np.pi*4*(r_outer)**2)/np.sum(mask_outer)
    # simple sqrt as spacing
    outer_spacing = np.sqrt(A_node_outer)

    print ('Inner node spacing: '+str(inner_spacing*1e6)+'µm')
    print ('Outer node spacing: '+str(outer_spacing*1e6)+'µm')

    bcond = np.zeros((len(coords), 4))
    bcond[:, 3] = 1.

    # fixed displacements for outer boundary
    bcond[mask_outer, 3] = 0

    # fixed non-zero force at spheroid surface
    bcond[mask_inner, :3] = coords[mask_inner, :3]
    bcond[mask_inner, :3] /= distance[mask_inner, None]

    A_inner = 4 * np.pi * r_inner ** 2.
    force_per_node = pressure * A_inner / np.sum(mask_inner)
    bcond[mask_inner, :3] *= force_per_node

    np.savetxt(outfolder + '/bcond.dat', bcond)

    # define initial configuration
    iconf = np.zeros((len(coords), 3))
    np.savetxt(outfolder + '/iconf.dat', iconf)

    # create config file for SAENO
    config = r"""MODE = relaxation
BOXMESH = 0
FIBERPATTERNMATCHING = 0
REL_CONV_CRIT = {}
REL_ITERATIONS = {}
REL_SOLVER_STEP = {}
K_0 = {}
D_0 = {}
L_S = {}
D_S = {}
CONFIG = {}\config.txt
DATAOUT = {}""".format(conv_crit, max_iter, step, K_0, D_0, L_S, D_S, os.path.abspath(outfolder), os.path.abspath(outfolder))

    with open(outfolder + "/config.txt", "w") as f:
        f.write(config)

    # create info file with all relevant parameters of the simulation
    parameters = r"""K_0 = {}
D_0 = {}
L_S = {}
D_S = {}
PRESSURE = {}
FORCE_PER_SURFACE_NODE = {}
INNER_RADIUS = {} µm
OUTER_RADIUS = {} µm
INNER_NODE_SPACING = {} µm
OUTER_NODE_SPACING = {} µm
SURFACE_NODES = {}
TOTAL_NODES = {}""".format(K_0, D_0, L_S, D_S, pressure, force_per_node, r_inner*1e6, r_outer*1e6, inner_spacing*1e6,
                           outer_spacing*1e6, np.sum(mask_inner), len(coords))

    with open(outfolder + "/parameters.txt", "w") as f:
        f.write(parameters)

    # Create log file if activated
    if logfile:

        # create log file with system output
        logfile = open(outfolder + "/saeno_log.txt", 'w')
        cmd = Popen(["saenopy","CONFIG","{}//config.txt".format(os.path.abspath(outfolder))], stdout=PIPE,
                    universal_newlines=True, shell=False)
        # print and save a reduced version of saeno log
        for line in cmd.stdout:
            if not '%' in line:
                print (line, end='')
                logfile.write(str(line))
        # close again to avoid loops
        cmd.stdout.close()

    # if false just show the non reduced system output
    else:
        cmd = call(["saenopy","CONFIG", "{}//config.txt".format(os.path.abspath(outfolder))])

    # copy result files from "*_py2" folder
    for filename in glob(outfolder+'_py2/*.*'):
        shutil.copy(filename, outfolder)

    # remove "*_py2" folder
    shutil.rmtree(outfolder+'_py2')

def distribute(func, const_args, var_arg='pressure', start=0.1, end=1000, n=120, log_scaling=True, n_cores=None):
    if n_cores is None:
        n_cores = os.cpu_count()

    if log_scaling:
        values = np.logspace(np.log10(start), np.log10(end), n, endpoint=True)
    else:
        values = np.linspace(np.log10(start), np.log10(end), n, endpoint=True)

    outfolder = const_args['outfolder']
    del const_args['outfolder']

    if not os.path.exists(outfolder):
        os.makedirs(outfolder)

    np.savetxt(outfolder+'/'+var_arg+'-values.txt', values)

    values = list(values)
    index = list(np.arange(len(values)))
    processes = []

    while True:
        if len(processes) < n_cores and len(values) > 0:
            command = '''python -c "import jointforces; import jointforces as jf; {}('''.format(func)
            for key in const_args:
                if isinstance(const_args[key], str):
                    command += '''{}='{}','''.format(key, const_args[key])
                else:
                    command += '''{}={},'''.format(key, const_args[key])
            command += '''outfolder='{}', {}={})"'''.format(outfolder+'/simulation'+str(index[0]).zfill(6), var_arg, values[0])
            print ("Simulations in the queue: "+str(len(values)))
            processes.append(Popen(command, shell='True'))
            del values[0]
            del index[0]

        sleep(1.)

        processes = [p for p in processes if p.poll() is None]

        if len(processes) == 0:
            break

def extract_deformation_curve(folder, x):
    # get simulation parameters
    with open(folder+'/parameters.txt', 'r') as f:
        lines = f.readlines()

    parameters = {}
    for line in lines:
        try:
            key, value = line.split('= ')
            value = value.split(' ')[0]
            parameters[key.strip()] = float(value.strip())
        except:
            pass

    # load coordinates
    coords = np.genfromtxt(folder + '/R.dat')

    # load displacements
    displ = np.genfromtxt(folder + '/U.dat')

    # compute binned normed displacements and normed coordinates
    u = np.sqrt(np.sum(coords ** 2., axis=1)) / (parameters['INNER_RADIUS']*10**-6)
    v = np.sqrt(np.sum(displ ** 2., axis=1)) / (parameters['INNER_RADIUS']*10**-6)

    y = np.array([np.nanmedian(v[(u >= x[i]) & (u < x[i + 1])]) for i in range(len(x) - 1)])

    # save results
    results = {'pressure': parameters['PRESSURE'], 'bins': x, 'displacements': y}
    return results

def create_lookup_table(folder, x0=1, x1=50, n=100):
    subfolders = glob(folder+'/*/')

    x = np.logspace(np.log10(x0), np.log10(x1), n+1, endpoint=True)
    x_center = 10**(0.5*(np.log10(x[1:]) + np.log10(x[:-1])))

    pressure_values = []
    displacement_curves = []

    for subfolder in tqdm(subfolders):
        res = extract_deformation_curve(subfolder, x)
        pressure_values.append(res['pressure'])
        displacement_curves.append(res['displacements'])

    pressure_values = np.array(pressure_values)
    displacement_curves = np.array(displacement_curves)

    return {'pressure': pressure_values, 'x': x_center, 'y': displacement_curves}

def create_lookup_functions(lookup_table):
    pressure = lookup_table['pressure']
    distance = lookup_table['x']
    displacement = lookup_table['y']

    log_pressure = np.log(pressure)
    log_distance = np.log(distance)

    x, y = np.meshgrid(log_distance, log_pressure)

    mask = ~(np.isnan(x) | np.isnan(y) | np.isnan(displacement))

    x = x[mask]
    y = y[mask]
    displacement = displacement[mask]

    f = LinearNDInterpolator(np.array([x, y]).T, displacement)

    def get_displacement(distance, pressure):
        return f(np.log(distance), np.log(pressure))

    f_inv = LinearNDInterpolator(np.array([x, displacement]).T, y)

    def get_pressure(distance, displacement):
        return np.exp(f_inv(np.log(distance), displacement))

    return get_displacement, get_pressure

def save_lookup_functions(get_displacement, get_pressure, outfile):
    with open(outfile, 'wb') as f:
        dill.dump((get_displacement, get_pressure), f)

def load_lookup_functions(file):
    with open(file, 'rb') as f:
        get_displacement, get_pressure = dill.load(f)
    return get_displacement, get_pressure

def linear_lookup_interpolator(emodulus, output_newtable="new-lin-lookup.pkl", reference_folder=None):
    """
    Create individual lookup-tables for linear materials by shifting a reference lookuptable for a linear fiber material.

    For linear fiber materials the following relation is used:  k0 = 6 * E_Modulus  (for a possion ration of 0.25, see Steinwachs[2015])

    Original simulation reached up to 10 000 Pa for a simulated k0 = 2364 (emodul ~394 Pa) - interpolation should be usefull for a wide range of
     emoduli - however keep in mind that there might be limits for extreme Emoduli-pressures combination due to the range of the original simulations
     (in such a case a constant maximal or minimal-pressure value will be returned since there are no better fitting simulations)

    emodulus:ArithmeticError Desired Youngs modulus for which the linear lookup table is created
    reference_folder: Folder containing the reference lookup functions and reference interpolators to create
    the new look up table; By default (None) will search for the reference files automatically
    output_newtable: name for the new reference table (needs .pkl ending)
    """

    # if not specified differently we find the correct reference files automatically
    if not reference_folder:
        import jointforces
        reference_folder = os.path.join(jointforces.__file__,"..","..","docs","data","linear_reference_table")

    # load in  reference lookuptable to for a simulated k2364 (emodul ~394 Pa) up to 10 000 Pa
    get_displacement_ref, get_pressure_ref = load_lookup_functions(os.path.join(reference_folder,'linear-ref-functions-k2364.pkl'))

    # load in in reference interpolators
    f_ref,f_inv_ref = load_lookup_functions(os.path.join(reference_folder,'linear-interp-f-finv-k2364.pkl'))

    # shift the table accordingly
    def get_displacement_new(distance, pressure):
        return f_ref(np.log(distance), np.log(pressure)) * 2364 /(emodulus*6)

    def get_pressure_new(distance, displacement):
        return np.exp(f_inv_ref(np.log(distance), displacement)) * (emodulus*6)/2364

    # save the new lookup functions
    save_lookup_functions(get_displacement_new, get_pressure_new, output_newtable)

    return get_displacement_new, get_pressure_new

def plot_lookup_table(lookup_table, pressure=[0,10000], log_scale = True, distance=[2,50], linewidth=2, n_lines = 1000, save_plot = None,
                      fig_size=(5,4), figure=None, show=True):
    """
    Create a figure of your (.pkl) material lookuptable

    lookup_table: path to .pkl file
    pressure: pressure range which will be plotted as [pressure_min,pressure_max] in pascal (use the range that was used to calculate the lookuptable)
    log_scale: plot logarithmically or linear if set to False
    distance: The distance to the spheroid which is plotted on the x-axis; Unit is spheroid radii
    linewidth: linewidth for the individual plots
    n_lines: number of lines plotted between the minimal and maximal pressure
    save_plot: saves the plot as png file if a path is given
    """

    import matplotlib.cm as cm
    import matplotlib.colors as colors
    import matplotlib.pyplot as plt

    # load lookup table
    get_displacement, get_pressure = load_lookup_functions(lookup_table)

    ## to avoid scaling issue for log(0)
    if pressure[0] == 0:
        pressure[0] = 0.01

    # define pressure range
    if log_scale:
        pressure_list = np.logspace(np.log(pressure[0])/np.log(10), np.log(pressure[1])/np.log(10),  num=n_lines, base=10)
    else:
        pressure_list = np.arange(pressure[0],pressure[1], step = (pressure[1]-pressure[0])/n_lines )

    # make cmaps
    mycmap = cm.get_cmap('viridis')
    mynorm = colors.LogNorm(vmin=np.min(pressure_list),vmax=np.max(pressure_list))
    c = mycmap(mynorm(pressure_list))

    # make a colorbar
    sm = plt.cm.ScalarMappable(cmap=mycmap, norm= mynorm)
    sm.set_array(c)

    # create distance list
    distance_list = np.arange(distance[0],distance[1], step =(distance[1]-distance[0]) / 1000 )

    # get displacements for pressures list
    displacement_list = [get_displacement(distance_list,i) for i in pressure_list]

    # create a figure if no figure is specified
    if figure is None:
        figure = plt.figure(figsize=fig_size)

    for i in range(len(displacement_list)):
        plt.plot( distance_list , displacement_list[i], c= c[i],linewidth=linewidth,alpha=0.5)

    # set x,y limit - go a bit above minimum and maximum
    plt.ylim(np.nanmin(displacement_list)-(0.1*np.nanmin(displacement_list)),
             np.nanmax(displacement_list)+(0.1*np.nanmax(displacement_list)))

    # set log if activated and go even more above minimum/maximum in log scale
    if log_scale:
        plt.loglog()
        plt.ylim(np.nanmin(displacement_list)-(0.8*np.nanmin(displacement_list)),
             np.nanmax(displacement_list)+(0.8*np.nanmax(displacement_list)))

    plt.grid(False)
    plt.ylabel('Normalized matrix deformations (u/r₀)', fontsize=10)           # plt.ylabel('Normed Deformation', fontsize=14)
    plt.xlabel('Normalized distance from organoid center (r/r₀)', fontsize=10) # plt.xlabel('Normed Distance', fontsize=14)

    # make a colorbar
    cbar= plt.colorbar(sm, )
    cbar.ax.get_yaxis().labelpad = 15
    cbar.ax.set_ylabel('Pressure (Pa)', rotation=270 , fontsize=14)
    plt.tight_layout()
    # save plot if specified
    if save_plot is not None:
        plt.savefig(save_plot,dpi=500)
    # show figure if specified
    if show==True:
        plt.show()
    return figure

def plot_lookup_data(lookup_table, data_folder, timesteps=[10,30,60], distance=[2,50], linewidth=2,
              color_line="k", color_raw="r", scatter_raw_data = True, marker_size_scatter=0.1,
              marker_size_mean=10,angle_filter=20, color_list = None,  plot_means = True,
              label_list = None, timesteps_scatter=None , result_file_name ="result.xlsx"):
    """
    plot the pressure for certain timesteps into the lookup table as a grey line;
    scatter_raw_data option to vuisualize all deformtation-distance data at that timestep

    Use the function after calling "plot_lookup_table" to visualize a certain pressure within the created lookuptable

    lookup_table: path to .pkl filer
    data_folder: path to the evaluated folder containing result.xlsx and the dis*.npy & seg*.npy data
    timesteps: list of the timesteps to plot into the lookup function
    scatter_raw_data: option to scatter the individual deformations
    timesteps_scatter: might be used to scatter only several timesteps - if none identical to timesteps
    angle_filter: use same value as in evaluation
    if color_list & label_list is provided the given colors and labels are used when plotting the raw data
    e.g color_list=["C0","C1","C2"], label_list=["1h","3h","12h"],

    timesteps_scatter: might be used to scatter only several timesteps - if timesteps_scatter=None identical to timesteps;
    keep the same length as timesteps to asure same coloring :
    e.g timesteps=[1,2,3] --> timesteps_scatter= [None,2,None]  to show only second element
    """
    import matplotlib.pyplot as plt
    from glob import glob
    from natsort import natsorted
    from .simulation import load_lookup_functions
    from .utils import load
    from .force import infer_pressure
    from tqdm import tqdm

    # scatter same timesteps if not spcified
    if not timesteps_scatter:
        timesteps_scatter = timesteps

    # read in the data
    pressure = pd.read_excel(os.path.join(data_folder,result_file_name))["Mean Pressure (Pa)"]

    # load lookup table
    get_displacement, get_pressure = load_lookup_functions(lookup_table)

    # define pressure range  # -1 since pressure list is 1 element shorter
    # compared to image list (differences are taken)
    pressure_list = [pressure[i-1] for i in timesteps]
    print (pressure_list)

    # create distance list
    distance_list = np.arange(distance[0],distance[1], step =(distance[1]-distance[0]) / 1000 )

    # get displacements for pressures list
    displacement_list = [get_displacement(distance_list,i) for i in pressure_list]

    # draw simulations in uniform color
    for i in range(len(displacement_list)):
       plt.plot( distance_list , displacement_list[i], c= color_line, #linestyle="--",
                linewidth=linewidth,alpha=0.5,zorder=30)#, label="Simulation")
       if i==0: # plot label once
           plt.plot( [] ,[] , c= color_line, #linestyle="--",
                    linewidth=linewidth,alpha=0.5,zorder=30, label="Simulations")

    # read in segemntation
    seg_files = natsorted(glob(data_folder+'/seg*.npy'))[:np.nanmax(timesteps)]

    # load deformations
    # look for accumulated deformation files (new standard)
    d_accumulated_files = natsorted(glob(data_folder+'/def*.npy'))[:np.nanmax(timesteps)]  # do not calcualte more then the necessary time steps
    # look also for not-accummulated deformations (old standard)
    d_notaccumulated_files = natsorted(glob(data_folder+'/dis*.npy'))[:np.nanmax(timesteps)]  # do not calcualte more then the necessary time steps
    # if not-accumulated deformations are found chose different mode
    if len(d_notaccumulated_files) > len(d_accumulated_files):
        accumulated = False
        dis_files = d_notaccumulated_files
        print("Found not-accumulated deformation files (old standard) and will conduct calculations accordingly.")
    # else do the calcualtion with accumulated d eformations already
    else:
        accumulated = True
        dis_files = d_accumulated_files

    # initial spheroid radius and surface (used for force estimation)
    r0 = load(seg_files[0])['radius']

    u_sum = None
    v_sum = None

    distance_list_raw = []
    displacement_list_raw = []
    pressure_list_raw = []

    # loop over series of PIV results
    for (dis_file, seg_file) in tqdm(zip(dis_files, seg_files)):
        dis = load(dis_file)
        seg = load(seg_file)

        x_rav = np.ravel(dis['x'])
        y_rav = np.ravel(dis['y'])
        #print("data_points:"+str(len(x_rav)))

        # get deformations
        # sum up if we have not-accummulated deformations (old standard)
        if accumulated == False:
            try:
                u_sum += np.ravel(dis['u'])
                v_sum += np.ravel(dis['v'])
            except:
                u_sum = np.ravel(dis['u'])
                v_sum = np.ravel(dis['v'])
        # else read in accummulated deformations directly (new standard)
        else:
            u_sum = np.ravel(dis['u'])
            v_sum = np.ravel(dis['v'])

        cx, cy = seg['centroid']
        distance_raw, displacement_raw, angle_raw, pressure_raw = infer_pressure(x_rav, y_rav, u_sum, v_sum, cx, cy, r0, get_pressure , angle_filter=angle_filter)
        #print (len(distance_raw)) # length of valid datapoints after angle filter

        # create list with accumulated deformations
        distance_list_raw.append(distance_raw)
        displacement_list_raw.append(displacement_raw)
        pressure_list_raw.append(pressure_raw)

    # SCATTERED RAW DATA
    #now plot the accumulated raw data at the corresponding timepoints if activated;
    # deformation and image data have a length difference of 1
    if scatter_raw_data:
        # in case colors and labels are defined;
        if color_list and label_list:
          for ci,t in enumerate(timesteps_scatter):
            if t == None: #do not plot None elements
                  continue
            plt.scatter(distance_list_raw[t-1],displacement_list_raw[t-1],s=marker_size_scatter,zorder=20,c = color_list[ci])

            ## calculate CoV of pressure values for 72h and add to plot
            # if t == 3*70-2: #3*72:
            #     mask = (distance_list_raw[t-1]>=5)&(distance_list_raw[t-1]<=10)

            #     data_masked = pressure_list_raw[t-1][mask]
            #     CoV = np.std(data_masked)/np.nanmean(data_masked)
            #     print (len(data_masked))
            #     print (len(distance_list_raw[t-1]))
            #     print (CoV)

            #     plt.text(0.05,0.93,s=f"CoV (72h; 5-10r): {np.around(CoV,3)}",
            #              transform=plt.gca().transAxes, zorder=200,
            #              fontsize=11,c="darkred")

        # else same color for all here
        else:
           for ci,t in enumerate(timesteps_scatter):
            if t == None: #do not plot None elements
                  continue
            plt.scatter(distance_list_raw[t-1],displacement_list_raw[t-1],s=marker_size_scatter,zorder=20,c = color_raw) # deformation and image data have a length difference of 1

    # PLOT THE MEAN RAW DATA
    if color_list and label_list:
        for ci,t in enumerate(timesteps):
            if plot_means == False:
                continue
            # calculate the mean in distance windows for timesteps
            mean_distance = []
            mean_displacement = []
            for i in range(distance[0],int(np.max(distance_list_raw[t-1]))):
                mean_distance.append(i+0.5)
                mean_disp = np.nanmean(displacement_list_raw[t-1][(distance_list_raw[t-1]>=i) & (distance_list_raw[t-1]<i+1)])
                mean_displacement.append(mean_disp)
            plt.plot(mean_distance,mean_displacement,"o-",ms=marker_size_mean, zorder=2000,
                     linewidth=linewidth,  markerfacecolor="w",  markeredgecolor=color_list[ci],
                     markeredgewidth=2 , label=label_list[ci], color="w" )
        ax = plt.legend( markerscale=0.4, fontsize=6.5,loc="upper right")
        ax.set_zorder(2000)
    else:
         for ci,t in enumerate(timesteps):
            if plot_means == False:
              continue
            # calculate the mean in distance windows for timesteps
            mean_distance = []
            mean_displacement = []
            for i in range(distance[0],int(np.max(distance_list_raw[t-1]))):
                mean_distance.append(i+0.5)
                mean_disp = np.nanmean(displacement_list_raw[t-1][(distance_list_raw[t-1]>=i) & (distance_list_raw[t-1]<i+1)])
                mean_displacement.append(mean_disp)
            plt.plot(mean_distance,mean_displacement,"o-",ms=marker_size_mean,zorder=2000,
                     linewidth=linewidth, markerfacecolor="w",color ="w",
                     markeredgecolor=color_raw, markeredgewidth=2)
        #plt.xlim(2,20);plt.ylim(1e-4,3)

    return


import pandas as pd
import matplotlib.pyplot as plt
import numpy as np

housePrice = pd.read_csv('metroMelbHousePrices.csv',encoding = 'ISO-8859-1')
commute = pd.read_csv('metroMelbCommuteDistance.csv',encoding = 'ISO-8859-1')
distance = pd.read_csv('distanceToCBD.csv',encoding = 'ISO-8859-1')

df = pd.merge(housePrice,commute,on='postcode')
df = df.sort_values('medPrice')
plt.scatter(df['medCommute'], df['medPrice'])
plt.xlabel('medCommute (km)')
plt.ylabel('medPrice ($)')
plt.grid(True)

x = np.array(df['medCommute'])
y = np.array(df['medPrice'])
m,b = np.polyfit(x, y, 1)
plt.plot(x, m*x + b)
plt.savefig('withOutliers.png')

##Outlier Detection using IQR
# Keep only values inside the IQR
Q1 = df['medPrice'].quantile(0.25)
Q3 = df['medPrice'].quantile(0.75)

Q1b = df['medCommute'].quantile(0.25)
Q3b = df['medCommute'].quantile(0.75)

df1 = df.loc[df['medPrice'] > Q1]
df1 = df1.loc[df1['medCommute'] > Q1b]
df2 = df1.loc[df1['medPrice'] < Q3]
df2 = df2.loc[df2['medCommute'] < Q3b]

# re-plot
plt.clf()
plt.scatter(df2['medCommute'], df2['medPrice'])
plt.xlabel('medCommute (km)')
plt.ylabel('medPrice ($)')
plt.grid(True)
x = np.array(df2['medCommute'])
y = np.array(df2['medPrice'])
m,b = np.polyfit(x, y, 1)
plt.plot(x, m*x + b)
plt.savefig('noOutliersIQR.png')

# z-score outlier detection #########################################################################
from scipy import stats

df['zPrice'] = np.abs(stats.zscore(df['medPrice']))
df['zCommute'] = np.abs(stats.zscore(df['medCommute']))
df1 = df.iloc[np.where(df['zPrice'] < 2)]
df2 = df.iloc[np.where(df1['zCommute'] < 2)]

# re-plot
plt.clf()
plt.scatter(df2['medCommute'], df2['medPrice'])
plt.xlabel('medCommute (km)')
plt.ylabel('medPrice ($)')
plt.grid(True)
x = np.array(df2['medCommute'])
y = np.array(df2['medPrice'])
m,b = np.polyfit(x, y, 1)
plt.plot(x, m*x + b)
plt.savefig('noOutliersZSCORE2STD.png')

df1 = df.iloc[np.where(df['zPrice'] < 1)]
df2 = df.iloc[np.where(df1['zCommute'] < 1)]
# re-plot
plt.clf()
plt.scatter(df2['medCommute'], df2['medPrice'])
plt.xlabel('medCommute (km)')
plt.ylabel('medPrice ($)')
plt.grid(True)
x = np.array(df2['medCommute'])
y = np.array(df2['medPrice'])
m,b = np.polyfit(x, y, 1)
plt.plot(x, m*x + b)
plt.savefig('noOutliersZSCORE1STD.png')

################################################################################
plt.clf()
df = pd.merge(housePrice,distance,on='postcode')
df = df.sort_values('medPrice')
plt.scatter(df['distance'], df['medPrice'])
plt.xlabel('distance (km)')
plt.ylabel('medPrice ($)')
plt.grid(True)

df['zPrice'] = np.abs(stats.zscore(df['medPrice']))
df['zDistance'] = np.abs(stats.zscore(df['distance']))
df1 = df.iloc[np.where(df['zPrice'] < 2)]
df2 = df.iloc[np.where(df1['zDistance'] < 2)]

# re-plot
plt.clf()
plt.scatter(df2['distance'], df2['medPrice'])
plt.xlabel('distance (km)')
plt.ylabel('medPrice ($)')
plt.grid(True)
x = np.array(df2['distance'])
y = np.array(df2['medPrice'])
m,b = np.polyfit(x, y, 1)
plt.plot(x, m*x + b)
plt.savefig('DISTANCEnoOutliersZSCORE2STD.png')


<reponame>jernelv/SpecAnalysis

import numpy as np
import scipy

def Der(x,y):
	"""Function for finding first derivative of spectral data. Uses finite differences."""
	n=len(x)
	x2=np.zeros(n-1)
	y2=np.zeros(n-1)
	for i in range(n-1):
		x2[i]=0.5*(x[i]+x[i+1])
		y2[i]=(y[i+1]-y[i])/(x[i+1]-x[i])
	return(x2,y2)

def Der2(x,y):
	"""Function for finding second derivative of spectral data. Uses finite differences."""
	n=len(x)
	x2=np.zeros(n-2)
	y2=np.zeros(n-2)
	dx2=(x[1]-x[0])**2 # assumed constant
	for i in range(n-2):
		x2[i]=x[i+1]
		y2[i]=(y[i]-2*y[i+1]+y[i+2])/dx2
	return(x2,y2)

def mlr(x,y,order):
	"""Multiple linear regression fit of the columns of matrix x
	(dependent variables) to constituent vector y (independent variables)
	order -     order of a smoothing polynomial, which can be included
	in the set of independent variables. If order is
	not specified, no background will be included.
	b -         fit coeffs
	f -         fit result (m x 1 column vector)
	r -         residual   (m x 1 column vector)
	"""
	if order > 0:
		s=scipy.ones((len(y),1))
		for j in range(order):
			s=scipy.concatenate((s,(scipy.arange(0,1+(1.0/(len(y)-1))-0.5/(len(y)-1),1.0/(len(y)-1))**j)[:,nA]),1)
		X=scipy.concatenate((x, s),1)
	else:
		X = x
	b = scipy.dot(scipy.dot(scipy.linalg.pinv(scipy.dot(scipy.transpose(X),X)),scipy.transpose(X)),y)
	f = scipy.dot(X,b)
	r = y - f
	return b,f,r

def emsc(case, order, fit=None):
	"""Extended multiplicative scatter correction
	case -   spectral data for background correction
	order -     order of polynomial
	fit -       if None then use average spectrum, otherwise provide a spectrum
				as a column vector to which all others fitted
	corr -      EMSC corrected data
	mx -        fitting spectrum
	"""
	if not type(fit)==type(None):
		mx = fit
	else:
		mx = scipy.mean(case,axis=0)[:,nA]
	corr = scipy.zeros(case.shape)
	for i in range(len(case)):
		b,f,r = mlr(mx, case[i,:][:,nA], order)
		corr[i,:] = scipy.reshape((r/b[0,0]) + mx, (corr.shape[1],))
	corr=np.nan_to_num(corr)
	return corr

def baseline_corr(case):
	"""Baseline correction that sets the first independent variable of each
	spectrum to zero."""
	size = case.shape
	subtract = scipy.transpose(scipy.resize(scipy.transpose(case[:,0]),(size[1],size[0])))
	return (case-subtract)

def baseline_avg(case):
	"""Baseline correction that subtracts an average of the first and last
	independent variable from each variable."""
	size = case.shape
	subtract = scipy.transpose(scipy.resize(scipy.transpose((case[:,0]+case[:size[1]-1])/2),(size[1],size[0])))
	return (case-subtract)

def baseline_linear(case):
	"""Baseline correction that subtracts a linearly increasing baseline between
	the first and last independent variable."""
	size, t = case.shape, 0
	subtract = scipy.zeros((size[0],size[1]), 'd')
	while t < size[0]:
		a = case[t,0]
		b = case[t,size[1]-1]
		div = (b-a)/size[1]
		if div == 0:
			div = 1
		arr = scipy.arrange(a,b,div,'d')
		subtract[t,:] = scipy.resize(arr,(size[1],))
		t = t+1
	return case-subtract


# test hamiltonian

import numpy as np
import matplotlib.pyplot as plt
from scipy.linalg import eig_banded

Vq = np.array([-0.5, -0.25])*10
nbasis = 20
max_pos_freq = len(Vq)
hamiton_mat = np.zeros((max_pos_freq, nbasis), dtype=np.complex64)
for i in range(max_pos_freq):
    for j in range(i, nbasis):
        if i==0:
            hamiton_mat[max_pos_freq-1, j] = Vq[0]
        else:
            hamiton_mat[max_pos_freq-i-1, j] = Vq[i]

h = np.zeros((nbasis, nbasis))
for i in range(nbasis):
    h[i, i] = Vq[0]
    for j in range(i+1, nbasis):
        if j-i>1:
            break
        else:
            h[i, j] = h[j, i] = Vq[1]

En_k, Uq_k = eig_banded(hamiton_mat, overwrite_a_band=True, select='a')
w, v = np.linalg.eigh(h)

print(w - En_k)
plt.matshow(v.real - Uq_k.real)
plt.colorbar()
plt.show()


<gh_stars>1-10
#!/usr/bin/env python
# -*- coding: utf-8 -*-

""" layer module

A layer is a collection of bases (see base module) that have the same bottom-up
and top-down input from adjacent layers.

"""

import scipy as sp
from scipy.misc import toimage

import utils
import config
from base import *

import pycuda_convolution
#conv = scipy.signal.convolve
conv = pycuda_convolution.convolve_gpu

# -----------------------------------------------------------------------------
class Layer:
    def __init__(self, model, label, prv_layer=None):
        """ Constructor

        Input:
            model -- a dict containing the structure and parameters of the layer
            label -- label assigned to the layer (e.g. "Layer 1")
            prv_layer -- pointer to the previous layer in the network

        """

        # -- copy parameter values to class data members
        self.model = model
        self.label = label
        self.prv_layer = prv_layer

        # -- copy values from model to class data members, to improve code
        # readability
        self.block_shape = model['block_shape']
        self.btmup_window_shape = model['btmup_window_shape']
        print "LAYER:BUWS = ", self.btmup_window_shape
        self.epsilon = model['epsilon']
        self.regL2 = model['regL2']
        self.pbias = model['pbias']
        self.pbias_lambda = model['pbias_lambda']
        self.init_bias = model['init_bias']
        self.vbias = model['vbias']
        self.std_gaussian = model['sigma_start']
        self.CD_steps = model['CD_steps']

        # amount of change made in the bias of the visible layer while updating
        self.vbias_inc = 0

        # a list to hold the errors at each training cycle of the epoch
        self.epoch_err = []

        # -- determine the shape of the input vector, depending on whether this
        # is the first layer in the network.
        # -- If this is not the first layer
        if (prv_layer):
            self.num_channels = len(prv_layer.bases)
            inshape = prv_layer.output_shape
        # -- for the first layer
        else:
            self.num_channels = model['num_channels']
            inshape = model['input_data_shape']

        wshape = self.btmup_window_shape  # convolution window shape
        pshape = self.block_shape  # pooling shape

        # trim the output of the previous layer to be used for maxpooling
        self.input_shape = utils.trim_array_maxpool(arr_shape=inshape, \
                                                     conv_window_shape=wshape, \
                                                     pooling_shape=pshape)

        # -- shape of hidden units in each base of the layer
        h = self.input_shape[0] - self.btmup_window_shape[0] + 1
        w = self.input_shape[1] - self.btmup_window_shape[1] + 1
        self.hidden_shape = (h, w)

        # -- shape of output (pooling) units in each base of the layer
        h = self.hidden_shape[0] / self.block_shape[0]
        w = self.hidden_shape[0] / self.block_shape[1]
        self.output_shape = (h,w)

        # negative data, i.e. network's belief
        self.neg_data = np.zeros((self.input_shape[0], \
                                    self.input_shape[1], self.num_channels))

        # positive data - input from previus layer (raw input if first layer)
        self.pos_data = np.zeros((self.input_shape[0],
                                    self.input_shape[1], self.num_channels))

        # list of bases in the layer
        self.bases = []
        # -- create the bases
        for i in range(model['num_bases']):
            self.bases.append(Base(self))

#-----------------------------------------------------------------------------
    def init_weights_using_prvlayer(self):
        """ Weight Initialization

        Initilizes the weights of the layer using input from the prv_layer.
        Using this sort of initialization is expected to speed up the
        convergence. However, it is not used in this current version since
        using random initial weights is a better verification test for the
        network.

        """

        # make sure there is a previous layer
        if config.DEBUG_MODE:
            assert self.prv_layer != None

        # for each base in the layer, grab a random patch of the previous
        # layer's output and assign the base's weights to the output values
        for base in self.bases:
            x_start = np.random.randint(0, self.prv_layer.output_shape[0] - base.Wb.shape[0])
            y_start = np.random.randint(0, self.prv_layer.output_shape[1] - base.Wb.shape[1])

            x_end = x_start + base.Wb.shape[0]
            y_end = y_start + base.Wb.shape[1]

            for i in range(self.num_channels):
                c = 10e-5
                base.Wb[:, :, i] = c * self.prv_layer.bases[i].pooling_units[x_start:x_end, y_start:y_end]

#-----------------------------------------------------------------------------
    def update(self, learn = False):
        """ Update Layer

        Performs Gibbs sampling of the layer's state variables (given the
        previous layer), and then updates the parameters and weights
        accordingly. Here is the steps performed:
            1) Sample each base given

        Input:
            layer_to_learn -- index of the layer to be learned

        """

        # ------------------------- Prositive Phase --------------------------
        print "\nPositive phase for ", self.label, "..."

        #timer = utils.Timer('positive phase')
        #with timer:  # measures the time
        #print "self.pos_data:", self.pos_data
        for base in self.bases:
            base.pos_sample()

        # ------------------------- Negative Phase ---------------------------
        # -- computes P(v|h) : Equation at the end of Section 2.3 in the paper
        print "Negative phase for", self.label, "..."
        #timer = utils.Timer('negative phase')
        #with timer:  # measures the time
        # perform the following Gibbs sampling steps, CD_steps times
        for step_idx in xrange(self.CD_steps):
            # -- compute the negative data given the hidden layer
            self.neg_data[:, :, :] = 0
            for base in self.bases:
                for channel in range(base.num_channels):
                    w =  base.Wb[:, :, channel]
                    self.neg_data[:, :, channel] += conv(base.pos_states, w,
                                                                        'full')
            self.neg_data += self.vbias

            # -- debugging assertion
            if config.DEBUG_MODE:
                assert ~np.isnan(self.neg_data).any()

            for base in self.bases:
                base.neg_sample()

        # -- compute the error as Euclidean distance between positive and
        # negative data
        err = np.mean( (utils.normalize_image(self.pos_data, 0, 1) -
                                    utils.normalize_image(self.neg_data, 0, 1))**2)
        self.epoch_err.append(err)
        print "Mean error so far: %.3f" % np.mean(self.epoch_err)

        # -- update the bases only if this layer is being currently
        # learned
        for base in self.bases:
            # -- reset some book keeping values
            base.bias_inc = 0
            base.Wb_inc = 0
            if (learn == True):
                base.update()

        self.vbias_inc = 0
        if (learn == True):
            # -- update the visible layer (prevous layer) bias
            print "Update phase for", self.label, "..."
            # Gradient Descent change
            dV_GD = np.mean(self.pos_data - self.neg_data)
            self.vbias_inc = self.epsilon * dV_GD
            self.vbias += self.vbias_inc

            # print the current state variables of the layer
            self.print_statistics()

#-----------------------------------------------------------------------------
    def print_statistics(self):
        """ Print Statistics

        Prints the current state variables of the network, including sparsity
        of units' activation, length and change of the weight vector, hidden
        and visible baises, as well as length of the positive and negative data
        vectors.

        """

        W_sum = 0
        Winc_sum = 0
        Hb_sum = 0
        Hbinc_sum = 0
        S_sum = 0
        # -- update the bases only if this layer is being currently
        # learned
        for base in self.bases:
            W_sum += np.sum(base.Wb) ** 2
            Winc_sum += np.sum(base.Wb_inc) ** 2
            S_sum += np.sum(base.pos_states)
            Hb_sum += base.bias ** 2
            Hbinc_sum += base.bias_inc ** 2

        num_units = len(self.bases) * self.hidden_shape[0] *self.hidden_shape[1]
        print self.label, ": Sparsity measure: %.2f percent" % (100 * float(S_sum)/num_units)
        print self.label, ": ||W|| = %.2f ||dW|| = %.5f" % (np.sqrt(W_sum), np.sqrt(Winc_sum))
        print self.label, ": ||Hb|| = %.2f  ||dHb|| = %.5f" % (np.sqrt(Hb_sum), np.sqrt(Hbinc_sum))
        print self.label, ": ||Vb|| = %.5f  ||dVb|| = %.6f" % (abs(self.vbias), abs(self.vbias_inc))

#-----------------------------------------------------------------------------
    def biases_for_visualization(self, tile_shape):
        """ Visualize Biases

        Prepares a visualization array for the hidden biases of the bases in
        the layer

        Input:
            tile_shape -- shape used to arrange the values for different bases

        Output:
             ret_array -- 2D array containing visualization of biases for
                          each base in the shape tile_shape

        """

        all_biases = []
        for base in self.bases:
            all_biases.append(base.bias)
        ret_array = np.array(all_biases).reshape(tile_shape)
        return ret_array

#-----------------------------------------------------------------------------
    def simple_activations_for_visualization(self, tile_shape):
        """ Visualize Activations

        Prepares a visualization array for the hidden activations of the bases
        in the layer

        Input:
            tile_shape -- shape used to arrange the values for different bases

        Output:
             ret_array -- 2D array containing visualization of activations
                          for each base in the shape tile_shape

        """

        all_acts = []
        for base in self.bases:
            all_acts.append(base.pos_activation)

        ret_array = np.array(all_acts).reshape(tile_shape)
        return ret_array

    def avg_filters_for_visualization(self, tile_shape, dir_path="./", save=False):
        """ Visualize Filters

        Prepares a visualization array for the filters of the bases in the
        layer. Filter for each base is computed as a weighted linear
        combination of the filters in the previous layer, where the weight to
        each filter in the previous layer is proportional to the sum of
        weights (weight vector of the network) originated from that base.

        Inputs:
            tile_shape -- shape used to arrange the values for different bases
            dir_path -- path to save the result image
            save -- whether to save the image to a file as well

        Output:
            all_weights -- 2D array containing visualization of filters
                            for each base in the shape tile_shape

        """

        # not implemeted for layer 1 yet
        if self.prv_layer == None:
            return

        filt_height = self.prv_layer.btmup_window_shape[0] + self.btmup_window_shape[0] - 1
        filt_width = self.prv_layer.btmup_window_shape[1] + self.btmup_window_shape[1] - 1

        filt_size = filt_height * filt_width
        all_filters = np.zeros((len(self.bases), filt_size))

        for base_idx in range(len(self.bases)):
            base_filter = np.zeros((filt_height, filt_width))
            for cnl_idx in range(len(self.prv_layer.bases)):
                # TODO: should be prv_layers's filter
                reg = 1
                if np.isnan((self.bases[base_idx].Wb.any())):
                    print "NaN weights while visualizing filters!"
                    exit(1)
                base_filter += conv(self.bases[base_idx].Wb[:, :, cnl_idx]**reg, self.prv_layer.bases[cnl_idx].Wb[:, :, 0], 'full')

            all_filters[base_idx, :] = np.reshape(base_filter, filt_size)

        img_shape = (filt_height, filt_width)
        all_filters = utils.tile_raster_images(all_filters, img_shape, tile_shape, tile_spacing = (1,1))
        all_filters = utils.normalize_image(all_filters)

        if save:
            # -- save the visualization array to a PNG file
            filename = dir_path + "/" + self.label + "-filters.jpg"
            #img = toimage(all_filters)
            #img.save(filename)
            scipy.misc.imsave(filename, all_filters)

            #if config.DEBUG_MODE:
                #img.show()

            print "Filters of", self.label, "were saved to", filename

        return all_filters

#-----------------------------------------------------------------------------
    def weights_for_visualization(self, channel, tile_shape, dir_path="./", save=False):
        """ Visualize Weights

        Prepares a visualization array for the bottom-up weights of the bases
        in the layer

        Input:
            channel -- index of the channel to whose corresponding weights will
                        be shown
            tile_shape -- shape used to arrange the values for different bases

        Output:
             all_weights -- 2D array containing visualization of weights for
                            the specified channel of each base in the shape
                            tile_shape

        """

        w_size = self.bases[0].Wb.shape[0]*self.bases[0].Wb.shape[1]
        all_weights = np.zeros((len(self.bases), w_size))

        for i in range(all_weights.shape[0]):
            if channel == None:
                for cnl in range(self.num_channels):
                    all_weights[i] += np.reshape(self.bases[i].Wb[:, :, cnl], w_size)
            else:
                all_weights[i] = np.reshape(self.bases[i].Wb[:, :, channel], w_size)

        img_shape = (self.bases[0].Wb.shape[0], self.bases[0].Wb.shape[1])
        all_weights = utils.tile_raster_images(all_weights, img_shape, tile_shape, tile_spacing = (1,1))
        all_weights = utils.normalize_image(all_weights)

        if save:
            # -- save the visualization array to a PNG file
            filename = dir_path + "/" + self.label + "-cnl_" + str(channel) + "-weights.jpg"
            #img = toimage(all_weights)
            #img.save(filename)
            scipy.misc.imsave(filename, all_weights)

            #if config.DEBUG_MODE:
                #img.show()

            print "Weights of", self.label, "were saved to", filename

        return all_weights

#-----------------------------------------------------------------------------
    def posstates_for_visualization(self, tile_shape):
        """ Visualize Positive States

        Prepares a visualization array for the states of the hidden units
        of the bases in the layer inferred by positive data

        Input:
            tile_shape -- shape used to arrange the values for different bases

        Output:
             all_states -- 2D array containing visualization of activations
                           for each base in the shape tile_shape

        """

        s_size = self.bases[0].pos_states.shape[0]*self.bases[0].pos_states.shape[1]
        all_states = np.zeros((len(self.bases), s_size))

        for i in xrange(all_states.shape[0]):
            all_states[i] = np.reshape(self.bases[i].pos_states, s_size)

        img_shape = self.bases[0].pos_states.shape
        all_states = utils.tile_raster_images(all_states, img_shape, tile_shape, tile_spacing = (1,1))
        all_states = utils.normalize_image(all_states)
        return all_states

#-----------------------------------------------------------------------------
    def output_for_visualization(self, tile_shape, tile_spacing):
        """ Visualize Outputs

        Prepares a visualization array for the output of the bases in the
        layer (taken from the pooling units)

        Input:
            tile_shape -- shape used to arrange the values for different bases
            tile_spacing -- number of space to put in between tiles of the
                            output array to make neighboring tiles
                            distinguishable

        Output:
             ret_array -- 2D array containing visualization of outputs
                          for each base in the shape tile_shape

        """

        size = self.bases[0].pooling_units.size
        all_outputs = np.zeros((len(self.bases), size))

        for i in xrange(all_outputs.shape[0]):
            all_outputs[i] = np.reshape(self.bases[i].pooling_units, size)

        img_shape = self.bases[0].pooling_units.shape
        all_outputs = utils.tile_raster_images(all_outputs, img_shape, tile_shape, tile_spacing)
        all_outputs = utils.normalize_image(all_outputs)
        return all_outputs

#-----------------------------------------------------------------------------
    def visualize_to_files(self, tile_shape, dir_path):
        """

        Saves the weight vector, and filters to files as images. More images
        can easily be added if needed.

        """

        for cnl in range(self.num_channels):
            print "Saving to file", cnl
            self.weights_for_visualization(cnl, tile_shape, dir_path, save=True)

        self.avg_filters_for_visualization(tile_shape, dir_path, save=True)

        # -- visualize the positive data
        # -- visualize the negagive data
        for cnl in range(self.num_channels):
            img = toimage(self.pos_data[:, :, cnl])
            filename = dir_path + "/" + self.label + "-cnl_" + str(cnl) + "-pos_data.png"
            img.save(filename)

            img = toimage(self.neg_data[:, :, cnl])
            filename = dir_path + "/" + self.label + "-cnl_" + str(cnl) + "-neg_data.png"
            img.save(filename)

        filename = dir_path + "/" + self.label + "-poooling.png"
        all_outputs = self.output_for_visualization(tile_shape, tile_spacing = (1,1))
        scipy.misc.imsave(filename, all_outputs)

#-----------------------------------------------------------------------------
    def visualize_bu_weights(self, label):

        all_weights = self.weights_for_visualization()
        img = Image.frombuffer('L', (all_weights.shape[1], all_weights.shape[0]), all_weights, "raw", 'L', 0, 1)
        img = img.resize((img.size[0]*5, img.size[1]*5), Image.NEAREST)
        #img.show()
        #img.save(label + ".tif")