function updateT0!(allvar::AllVar)
    @extract allvar : alg pbf
    @extract pbf : L
    @extract alg : verbose nprint epsconv maxiter mindec μext μint
    en = Inf
    assign = zeros(Int,L,2)
    ndec = 0
    initialize_msg0(allvar)

    for it in 1:maxiter
        onesweepT0!(allvar)
        ndec, assign = assignment!(allvar,ndec,assign)
        en = compute_en(allvar)
        if mod(it, nprint) == 0 && verbose
            @printf("it = %d -- en %.3f μext %.2f μint %.2f ndec %d\n", it, en,μext, μint,ndec)
        end
        if ndec >= mindec
            if verbose
                @printf("it = %d -- en %.3f μext %.2f μint %.2f ndec %d\n", it, en,μext, μint,ndec)
            end
            return it, :converged, en
        end
    end
    if verbose
        @printf("it = %d -- en %.3f μext %.2f μint %.2f ndec %d\n", maxiter, en,μext, μint,ndec)
    end
    return maxiter, :unconverged, en
end

function initialize_msg0(allvar::AllVar)
    @extract allvar : pbf
    @extract pbf : L N P B F newP newB newF

    for marg in (newP, newB, newF)
        marg[1,0] = -Inf
        marg[1,N+1] = -Inf
        value, idx = findmax(marg)
        marg .= marg .- marg[idx]
    end
    for i in 1:L
        for marg in (P[i], F[i], B[i])
            marg[1,0] = - Inf
            marg[1,N+1] = -Inf
            if i == 1
                for n = 1:N+1
                    marg[0,n] = -Inf
                end
            end
            if i == L
                for n = 0:N
                    marg[0,n] = -Inf
                end
            end
            value, idx = findmax(marg)
            marg .= marg .- marg[idx]
        end
    end

end

function onesweepT0!(allvar::AllVar)
    @extract allvar : pbf alg
    @extract pbf : L P B F newP newB newF
    @extract alg : damp

    v = randperm(L)
    @inbounds for idx in 1:L # i-th marginal 1 ≤ i ≤ L
        i = v[idx]
        newPBFiT0!(allvar, i::Int)
        p,f,b = P[i], F[i], B[i]
        p .= newP; f .= newF; b .= newB;
        #p .= damp .* p .+ (1.0 - damp) .* newP # inplace sum
        #f .= damp .* f .+ (1.0 - damp) .* newF
        #b .= damp .* b .+ (1.0 - damp) .* newB
    end
end

function newPBFiT0!(allvar::AllVar, i::Int)
    @extract allvar : pbf
    @extract pbf : L N forward central bckward newP newB newF

    for fcb in (forward,central,bckward)
        fill!(fcb, 0.0)
    end

    centralT0!(central, allvar, i)
    i > 1 &&  forwardT0!(forward, allvar, i)
    i < L && backwardT0!(bckward, allvar, i)

    if i==1 # case site = 1 first site
        newF .= central
        newP .= central .+ bckward
        newB .= newP
    elseif 1 < i < L #internal sites
        newF .= forward .+ central
        newP .= newF .+ bckward
        newB .= central .+ bckward
    elseif i == L# case site = L
        newF .= forward .+ central
        newB .= central
        newP .= newF
    else
        error("the end of the world has come ...")
    end
    newP[1,0] = -Inf
    newB[1,0] = -Inf
    newF[1,0] = -Inf
    newP[1,N+1] = -Inf
    newB[1,N+1] = -Inf
    newF[1,N+1] = -Inf
    if i == 1
        for n in 1:N+1
            newP[0,n] = -Inf
            newB[0,n] = -Inf
            newF[0,n] = -Inf
        end
    end
    if i == L
        for n in 0:N
            newP[0,n] = -Inf
            newB[0,n] = -Inf
            newF[0,n] = -Inf
        end
    end
    for marg in (newP,newB,newF)
        v, idx = findmax(marg)
        marg .= marg .- marg[idx]
    end
end

# function myargmax(Pj,nbeg,nend)
#     #argm = -Inf
#     val = (-1,-1,-Inf)
#     @inbounds for n = nbeg:nend
#         for x = 0:1
#             val = ifelse(Pj[x, n]>val[3],(x,n,Pj[x,n]),val)
#         end
#     end
#     @assert val[1] > -1
#     @assert val[2] > -1
#     return val[1],val[2]
# end

function myargmax(Pj,nbeg,nend)
    argm = -Inf
    xj = -1
    nj = -1
    @inbounds for n = nbeg:nend
        for x = 0:1
            if Pj[x, n] > argm
                argm = Pj[x, n]
                xj = x
                nj = n
            end
        end
    end
    @assert(xj > -1)
    @assert(nj > -1)
    return xj,nj
end

function centralT0!(central::Marginal, allvar::AllVar, i::Int)
    @extract allvar:pbf jh seq alg
    @extract pbf:N P L q
    @extract jh:J h
    @extract seq:intseq
    @extract alg:μext μint

    A0 = q
    # case xᵢ = 1 (match)
    @inbounds for ni = 1:N
        Ani = intseq[ni]
        scra = h[Ani, i]
        for j = 1:i-2
            xj, nj = myargmax(P[j], 0, ni - 1)
            Anj = (nj == 0 || xj == 0) ? A0 : intseq[nj]
            scra += J[Ani, Anj, i, j]
        end
        for j = i+2:L
            xj, nj = myargmax(P[j], ni + 1, N + 1)
            Anj = (xj == 0 || nj == N + 1) ? A0 : intseq[nj]
            scra += J[Ani, Anj, i, j]
        end
        central[1, ni] = scra
    end
    # case xᵢ = 0 (gap)
    @inbounds for ni = 0:N+1
        scra = 0
        Ani = A0
        for j = 1:i-2
            xj, nj = myargmax(P[j], 0, ni)
            Anj = (nj == 0 || nj == N + 1 || xj == 0) ? A0 : intseq[nj]
            scra += J[Ani, Anj, i, j]
        end
        for j = i+2:L
            xj, nj = myargmax(P[j], ni, N + 1)
            Anj = (nj == 0 || nj == N + 1 || xj == 0) ? A0 : intseq[nj]
            scra += J[Ani, Anj, i, j]
        end
        central[0, ni] = scra
    end
    @inbounds for ni = 0:N+1
        central[0, ni] =
            (ni == 0 || ni == N + 1) ? central[0, ni] - μext + h[A0, i] :
            central[0, ni] - μint + h[A0, i]
    end
    #cannot match n = 0 pointer
    #central[1,0] = -Inf
    #mcentral = maximum(central)
    #central = central .- mcentral
end

function forwardT0!(forward::Marginal,allvar::AllVar,i::Int)
    @extract allvar : pbf jh seq alg
    @extract pbf : N P F q
    @extract jh : J
    @extract seq : intseq
    @extract alg : λo λe
    fwdm1 = F[i-1]
    A0 = q
    # case xᵢ = 1 (match)
    @inbounds for ni in 1:N
        Ani = intseq[ni]
        scra = -Inf
        @simd for nim in 0:ni-1
           Anim = nim == 0 ? A0 : intseq[nim]
           Δn = ni - nim - 1
           pen = (Δn > 0)*(nim > 0)*(- λo[i] - λe[i]*(Δn - 1))
           scra = max(scra, fwdm1[0,nim] + J[Ani, A0,   i, i-1] + pen)
           scra = max(scra, fwdm1[1,nim] + J[Ani, Anim, i, i-1] + pen)
        end
        forward[1, ni] = scra
    end
    # case xᵢ = 0 gap
    @inbounds @simd for ni in 1:N
        Ani = intseq[ni]
        forward[0,ni] = max(fwdm1[1,ni]+J[A0,Ani,i,i-1],fwdm1[0,ni]+J[A0,A0,i,i-1])
    end
    forward[0,0] = fwdm1[0,0] + J[A0,A0,i,i-1]
    scra = -Inf
    @inbounds @simd for nim = 1:N
        Anim = intseq[nim]
        scra = max(scra, fwdm1[1,nim] + J[A0,Anim,i,i-1])
    end
    forward[0,N+1] = max(scra, fwdm1[0,N+1] + J[A0,A0,i,i-1])
end

function backwardT0!(backward::Marginal,allvar::AllVar,i::Int)
    @extract allvar : pbf jh seq alg
    @extract pbf : N P B q
    @extract jh : J
    @extract seq : intseq
    @extract alg : λo λe

    bckp1 = B[i+1]
    A0 = q

    # case xᵢ = 1 (match)
    @inbounds for ni in 1:N
        # case xᵢp = 0 (gap)
        Ani = intseq[ni]
        scra = bckp1[0,ni] + J[A0,Ani,i+1,i]
        scra = max(scra, bckp1[0,N+1] + J[A0,Ani,i+1,i])
        for nip in ni+1:N
            Anip = intseq[nip]
            Δn = nip - ni- 1
	        pen = (Δn > 0)*(- λo[i+1] - λe[i+1]*(Δn - 1))
            scra = max(scra, bckp1[1,nip] + J[Anip, Ani, i+1,i] + pen)
        end
        backward[1,ni] = scra
    end
    # case xᵢ = 0 (gap)
    @inbounds for ni in 0:N+1
        scra = bckp1[0,ni] + J[A0,A0,i+1,i]
        for nip in ni+1:N
            Anip = intseq[nip]
            Δn = nip - ni - 1
	        pen = (ni > 0)*(Δn > 0)*(-λo[i+1] - λe[i+1]*(Δn - 1))
            scra = max(scra, bckp1[1,nip], J[Anip,A0,i+1,i] + pen)
        end
        backward[0,ni] = scra
    end
    #mbackward = maximum(backward)
    #backward = backward .- mbackward
end

function assignment!(allvar::AllVar, ndec::Int, assign::Matrix{Int})

    @extract allvar : pbf
    @extract pbf : L N P
    count = 0
    @inbounds for i in 1:L
        Pi = P[i]
        #v, idx = findmax(Pi)
        new_a = argmax(Pi)
        #display(new_a)
        #count how many nodes change state
        if (new_a[1] != assign[i,1]) || (new_a[2] != assign[i,2])
            count += 1
        end
        assign[i,1] = new_a[1]
        assign[i,2] = new_a[2]
    end

    if count == 0
        ndec += 1
    else
        ndec = 0
    end

    return ndec, assign
end


#	MIT License
#
#  Copyright © 2020 Michael J Simms. All rights reserved.
#
#	Permission is hereby granted, free of charge, to any person obtaining a copy
#	of this software and associated documentation files (the "Software"), to deal
#	in the Software without restriction, including without limitation the rights
#	to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#	copies of the Software, and to permit persons to whom the Software is
#	furnished to do so, subject to the following conditions:
#
#	The above copyright notice and this permission notice shall be included in all
#	copies or substantial portions of the Software.
#
#	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#	AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#	OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
#	SOFTWARE.

module IsolationForest

# Tree node, used internally.
mutable struct Node
    featureName::String
    splitValue::Float64
    left::Any
    right::Any
end

# This struct represents a sample. Each sample has a name and list of features.
mutable struct Sample
    name::String
    features::Dict{String,Float64}
end

# Isolation Forest.
mutable struct Forest
    numTrees::UInt64
    subSamplingSize::UInt64
    featureValues::Dict{String,Array} # Dictionary that maps feature names to a sorted array of feature values
    trees::Array
end

# Adds the features to the specified sample.
function add_features_to_sample(sample::Sample, features::Dict{String,Float64})
    merge!(sample.features, features)
end

# Adds each of the sample's features to the list of known features with the corresponding set of unique values.
function add_sample_to_forest(forest::Forest, sample::Sample)

    # We don't store the sample directly, just the features.
    for feature_name in keys(sample.features)
        feature_value = sample.features[feature_name] # Value to insert

        try
            existing_feature_values = forest.featureValues[feature_name]
            push!(existing_feature_values, feature_value)
            sort(existing_feature_values)
            forest.featureValues[feature_name] = existing_feature_values
        catch error
            if isa(error, KeyError)
                existing_feature_values = []
                push!(existing_feature_values, feature_value)
                forest.featureValues[feature_name] = existing_feature_values
            end
        end
    end
end

# Creates and returns a single tree. As this is a recursive function, depth indicates the current depth of the recursion.
function create_tree(forest::Forest, feature_values::Dict{String,Array}, depth::UInt64)

    # Sanity check
    feature_values_len = length(feature_values)
    if feature_values_len <= 1
        return Nothing
    end

    # If we've exceeded the maximum desired depth, then stop.
    if (forest.subSamplingSize > 0) && (depth >= forest.subSamplingSize)
        return Nothing
    end

    # Randomly select a feature.
    selected_feature_index = rand(1:feature_values_len)
    all_feature_names = collect(keys(feature_values))
    selected_feature_name = all_feature_names[selected_feature_index]

    # Randomly select a split value.
    feature_value_set = feature_values[selected_feature_name]
    feature_value_set_len = length(feature_value_set)
    if feature_value_set_len <= 1
        return Nothing
    end
    split_value_index = rand(1:feature_value_set_len)
    split_value = feature_value_set[split_value_index]

    # Create a tree node to hold the split value.
    tree = Node(selected_feature_name, split_value, Nothing, Nothing)

    # Create two versions of the feature value set that we just used,
    # one for the left side of the tree and one for the right.
    temp_feature_values = feature_values

    # Create the left subtree.
    left_features = feature_value_set[1:split_value_index]
    temp_feature_values[selected_feature_name] = left_features
    tree.left = IsolationForest.create_tree(forest, temp_feature_values, depth + 1)

    # Create the right subtree.
    if split_value_index + 1 < feature_value_set_len
        right_features = feature_value_set[split_value_index + 1:feature_value_set_len]
        temp_feature_values[selected_feature_name] = right_features
        tree.right = IsolationForest.create_tree(forest, temp_feature_values, depth + 1)
    end

    return tree
end

# Creates a forest containing the number of trees specified to the constructor.
function create_forest(forest::Forest)
    for i = 1:forest.numTrees
        temp_feature_values = deepcopy(forest.featureValues)
        tree = create_tree(forest, temp_feature_values, UInt64(0))
        if tree != Nothing
            push!(forest.trees, tree)
        end
    end
end

# Scores the sample against the specified tree.
function score_sample_against_tree(tree::Node, sample::Sample)
    depth = 0.0
    current_node = tree

    while current_node != Nothing
        found_feature = false

        # Find the next feature in the sample.
        for current_feature in sample.features
            current_feature_name = current_feature.first

            # If the current node has the feature in question.
            if current_feature_name == current_node.featureName
                current_feature_value = current_feature.second
                if current_feature_value < current_node.splitValue
                    current_node = current_node.left
                else
                    current_node = current_node.right
                end

                depth = depth + 1.0
                found_feature = true
                break
            end
        end

        # If the tree contained a feature not in the sample then take
        # both sides of the tree and average the scores together.
        if found_feature == false
            left_depth = depth + score_sample_against_tree(sample, current_node.left)
            right_depth = depth + score_sample_against_tree(sample, current_node.right)
            return (left_depth + right_depth) / 2.0
        end
    end

    return depth
end

# Scores the sample against the entire forest of trees. Result is the average path length.
function score_sample_against_forest(forest::Forest, sample::Sample)
    num_trees = 0
    avg_path_len = 0.0

    for tree in forest.trees
        path_len = score_sample_against_tree(tree, sample)
        if path_len > 0
            avg_path_len = avg_path_len + path_len
            num_trees = num_trees + 1
        end
    end

    if num_trees > 0
        avg_path_len = avg_path_len / num_trees
    end

    return avg_path_len
end

# Scores the sample against the entire forest of trees. Result is normalized so that values
# close to 1 indicate anomalies and values close to zero indicate normal values.
function H(i)
    return log(i) + 0.5772156649
end
function C(n)
    return 2 * H(n - 1) - (2 * (n - 1) / n)
end
function score_sample_against_forest_normalized(forest::Forest, sample::Sample)

    # Compute the average path length for all valid trees.
    num_trees = 0
    avg_path_len = 0.0

    for tree in forest.trees
        path_len = score_sample_against_tree(tree, sample)
        if path_len > 0
            avg_path_len = avg_path_len + path_len
            num_trees = num_trees + 1
        end
    end

    if num_trees > 0
        avg_path_len = avg_path_len / num_trees
    end

    # Normalize, per the original paper.
    score = 0.0
    if num_trees > 1.0
        exponent = -1.0 * (avg_path_len / C(num_trees))
        score = 2 ^ exponent
    end

    return score
end

# Destroys a single tree.
function destroy_tree(tree::Node)
end

# Destroys the entire forest of trees.
function destroy()
    for tree in self.trees
        self.destroy_tree(tree)
    end
end

# Returns the specified node as a dictionary object.
function dump_node(node::Node)
    data = Dict()
    if node != Nothing
        data["Feature Name"] = node.featureName
        data["Split Value"] = node.splitValue
        if node.left == Nothing
            data["Left"] = Dict()
        else
            data["Left"] = dump_node(node.left)
        end
        if node.right == Nothing
            data["Right"] = Dict()
        else
            data["Right"] = dump_node(node.right)
        end
    end
    return data
end

# Loads/creates a node from a JSON object.
function load_node(data::Dict)
    try
        node = Node(data["Feature Name"], data["Split Value"])
        node.left = load_node(data["Left"])
        node.right = load_node(data["Right"])
        return node
    catch error
    end

    return Nothing
end

# Returns the specified tree as a dictionary object.
function dump_tree(tree::Node)
    data = dump_node(tree)
    return data
end

# Loads/creates a tree from a dictionary object.
function load_tree(forest::Forest, data::Dict)
    tree = load_node(data)
    if tree != Nothing
        push!(forest.trees, tree)
    end
end

# Returns the forest as a JSON object.
function dump(forest::Forest)
    data = Dict()
    data["Sub Sampling Size"] = forest.subSamplingSize
    data["Feature Values"] = forest.featureValues
    tree_data = []
    for tree in forest.trees
        push!(tree_data, dump_tree(tree))
    end
    data["Trees"] = tree_data
    return data
end

# Loads the forest from a JSON object.
function load(data::Dict)
    forest = Forest(0, 0, Dict(), [])
    forest.subSamplingSize = data["Sub Sampling Size"]
    forest.featureValues = data["Feature Values"]
    for tree_data in data["Trees"]
        load_tree(forest, tree_data)
    end
    forest.numTrees = length(forest.trees)
    return forest
end

end


################################################################################
###################### DEFINE MASTER PARAMETERS FOR THIS FILE ##################
################################################################################

using Mimi
using Distributions
using CSVFiles
using DataFrames
using CSV
using Random
using StatsBase
using Statistics

include("main_model_growth.jl")

# define the model regions in the order that Mimi returns stuff
myregions = ["EU", "USA", "Other OECD","Former USSR","China","Southeast Asia","Africa","Latin America"]
myyears = [2020, 2030, 2040, 2050, 2075, 2100, 2150, 2200, 2250, 2300]

# define the output directory
dir_output = joinpath(@__DIR__, "../../output/")

# define number of Monte Carlo runs
if !@isdefined samplesize
    samplesize = 50000
end

# define the seed
masterseed = 22081994

# define the pulse size
scc_pulse_size = 75000.

################################################################################
################### MC MODEL RUNS    ###########################################
################################################################################

### PART I: results for fixed persistence values

# create a DataFrame where results will be stored
df_sccMC_singleGE = DataFrame(damagePAGE09=false, permafr=false, seaice=false, ge=-999., scen="-999", pulse_size=-999.,
                                 civvalue=-999.,
                                 ge_adapt = -999.,
                                  mean=-999., median=-999., min=-999., max=-999., perc25=-999.,
                                  perc75=-999., sd=-999., varcoeff=-999., perc05=-999., perc95=-999.,
                                  perc10=-999., perc90=-999.,
                                  share_zeroSCC = -999.)

# loop through all relevant parameter combinations and save out results and SCC summary stats
for jj_page09damages in [false] # whether or not to use the market damage function from PAGE09 instead of the new PAGE-ICE function using Burke et al. (2015)
    for jj_permafr in [true] # whether or not to include the permafrost carbon feedback, see Yumashev et al. (2019)
        for jj_seaice in [true] # whether or not to include the surface albedo feedback, see Yumashev et al. (2019)
            for jj_scen in ["RCP4.5 & SSP2"]
                for jj_civvalue in [1., 10.0^20] # whether or not to hold the civilization value damage cap at its default value

                    # skip undesired combinations of parameters
                    if jj_page09damages != false && (jj_permafr != true  || jj_seaice != true || jj_civvalue != 1.)
                        continue
                    elseif jj_permafr != true && (jj_seaice != true || jj_civvalue != 1.)
                        continue
                    elseif jj_permafr != jj_seaice
                        continue
                    end

                    # print the parameter combination to track progress
                    print(string("page09_", jj_page09damages, " Permafr_", jj_permafr, " Scen_", jj_scen, " Seaice_", jj_seaice
                                     ))

                    # loop through the different values for persistence and its decay rate (= adaptation)
                    for jj_ge in [0.0:0.1:1.0;]
                        for jj_geadapt in [0.0:0.005:0.04;]

                            # skip redundant iterations with non-zero adaptation and zero persistence
                            if jj_ge == 0. && jj_geadapt > 0.
                                continue
                            elseif jj_geadapt != 0. && jj_civvalue != 1.
                                continue
                            end

                            # print out the growth effects to track progress
                            print(string("Rho = ", jj_ge, " ge_adapt = ", jj_geadapt))

                            # define the output directory for the Monte Carlo files
                            dir_MCoutput = string(dir_output, "montecarlo-singleGE/scen", jj_scen, "_permafr", jj_permafr, "_seaice", jj_seaice,
                                                            "_page09", jj_page09damages, "/", "ge", jj_ge,
                                                            "_geadapt", jj_geadapt,
                                                            "_civ", jj_civvalue, "/")

                            # fix the seed and calculate the SCC using a triangular distribution collapsing to a single value
                            Random.seed!(masterseed)
                            global scc_mcs_object = get_scc_mcs(samplesize, 2020, dir_MCoutput,
                                                                    scenario=jj_scen,
                                                                    pulse_size=scc_pulse_size,
                                                                    use_permafrost=jj_permafr,
                                                                    use_seaice=jj_seaice,
                                                                    use_page09damages=jj_page09damages,
                                                                    ge_minimum=jj_ge,
                                                                    ge_maximum=jj_ge + 10^(-30),
                                                                    ge_mode=jj_ge,
                                                                    civvalue_multiplier=jj_civvalue,
                                                                    geadrate = jj_geadapt)

                            # write out the full distribution
                            writedlm(string(dir_MCoutput, "SCC_MC.csv"), scc_mcs_object, ",")

                            # write results into the data frame
                            push!(df_sccMC_singleGE, [jj_page09damages, jj_permafr, jj_seaice, jj_ge,
                                                          jj_scen, scc_pulse_size, jj_civvalue,
                                                          jj_geadapt,
                                                          mean(scc_mcs_object[:, 1]),
                                                          median(scc_mcs_object[:, 1]),
                                                          minimum(scc_mcs_object[:, 1]),
                                                          maximum(scc_mcs_object[:, 1]),
                                                          StatsBase.percentile(scc_mcs_object[:, 1], 25),
                                                          StatsBase.percentile(scc_mcs_object[:, 1], 75),
                                                          Statistics.std(scc_mcs_object[:, 1]),
                                                          Statistics.std(scc_mcs_object[:, 1]) / mean(scc_mcs_object[:, 1]),
                                                          StatsBase.percentile(scc_mcs_object[:, 1], 5),
                                                          StatsBase.percentile(scc_mcs_object[:, 1], 95),
                                                          StatsBase.percentile(scc_mcs_object[:, 1], 10),
                                                          StatsBase.percentile(scc_mcs_object[:, 1], 90),
                                                          count(scc_mcs_object .== 0.) / samplesize])

                          # clean out the MCS objects
                            scc_mcs_object = nothing

                        end
                    end
                end
            end
        end
    end
end

# remove the first placeholder row
df_sccMC_singleGE = df_sccMC_singleGE[df_sccMC_singleGE[!, :scen] .!= "-999", :]

# export the results
CSV.write(string(dir_output, "MimiPageGrowthEffectsResults_SCC_fixedGE.csv"), df_sccMC_singleGE)

### PART II: results for the EMPIRICAL persistence distribution

# create a DataFrame where results will be stored
df_sccMC = DataFrame(permafr=false, seaice=false, ge_string="-999", scen="-999",
                                  convergence=-999.,  bound=-999., eqwshare=-999.,
                                  civvalue=-999., pulse=-999., ge_adapt = -999.,
                                  emfeed = -999., switch = -999.,
                                  mean=-999., median=-999., min=-999., max=-999., perc25=-999.,
                                  perc75=-999., sd=-999., varcoeff=-999.,
                                  perc05=-999., perc95=-999., perc10=-999., perc90=-999.,
                                  share_zeroSCC = -999., count_NaN = -999)

# loop through all relevant parameter/scenario combinations and save out results and SCC summary stats
for jj_scen in ["RCP4.5 & SSP2", "RCP2.6 & SSP1", "RCP8.5 & SSP5", "RCP1.9 & SSP1"]
    for jj_gestring in ["EMPIRICAL"] # which persistence distribution to use (use "EMPIRICAL+" for a version where negative persistence draws are trimmed to zero)
        for jj_permafr in [true]
            for jj_seaice in [true]
                for jj_civvalue in [1., 10.0^20]
                    for jj_cbabs in [740.65] # the level of the subsistence boundary for per capita consumption
                        for jj_eqwshare in [0.99, 0.95, 0.999] # the share of per capita consumption until which damages are equity-weighted
                            for jj_convergence in [1., 0.] # whether or not to use the convergence system for the subsistence and equity-weighting thresholds
                                for jj_pulse in [scc_pulse_size] # the size of the CO2 pulse used for SCC calculations
                                    for jj_geadapt in [0.0:0.01:0.05;] # the decay rate of persistence over time
                                        for jj_emfeed in [1., 0.] # whether or not persistence-related GDP reductions cause proportional reductions in emissions
                                            for jj_regionswitch in [0., 1.] # whether or not persistence is only limited to Global South regions

                                        # overwrite jj_geadapt with nothing if its zero (which will triger the deterministic default value of zero)
                                        if jj_geadapt == 0.
                                            jj_geadapt = nothing
                                        end

                                        # skip undesired or infeasible combinations of parameters/scenarios
                                        if jj_scen != "RCP4.5 & SSP2" && (jj_gestring != "EMPIRICAL" || jj_permafr != true ||
                                                                            jj_seaice != true || jj_civvalue != 1. || jj_cbabs != 740.65 ||
                                                                            jj_eqwshare != 0.99 || jj_convergence != 1. || jj_pulse != scc_pulse_size ||
                                                                            jj_geadapt != nothing || jj_emfeed != 1. || jj_regionswitch != 0.)
                                            continue
                                        elseif jj_permafr != jj_seaice
                                            continue
                                        elseif jj_gestring != "EMPIRICAL" && (jj_cbabs != 740.65 || jj_eqwshare != 0.99 || jj_convergence != 1. ||
                                                                                jj_pulse != scc_pulse_size || jj_geadapt != nothing  || jj_emfeed != 1. || jj_regionswitch != 0.)
                                            continue
                                        elseif jj_civvalue != 1. && (jj_cbabs != 740.65 || jj_eqwshare != 0.99 || jj_convergence != 1. ||
                                                                    jj_pulse != scc_pulse_size || jj_geadapt != nothing || jj_emfeed != 1. || jj_regionswitch != 0.)
                                            continue
                                        elseif jj_cbabs != 740.65 && (jj_civvalue != 1. || jj_gestring != "EMPIRICAL" || jj_eqwshare != 0.99 ||
                                                                    jj_convergence != 1. || jj_pulse != scc_pulse_size || jj_geadapt != nothing || jj_emfeed != 1. || jj_regionswitch != 0.)
                                            continue
                                        elseif jj_eqwshare != 0.99 && (jj_civvalue != 1. || jj_gestring != "EMPIRICAL" || jj_convergence != 1. ||
                                                                        jj_pulse != scc_pulse_size || jj_geadapt != nothing || jj_emfeed != 1. || jj_regionswitch != 0.)
                                            continue
                                        elseif jj_convergence != 1. && (jj_civvalue != 1. || jj_gestring != "EMPIRICAL" || jj_pulse != scc_pulse_size ||
                                                                        jj_geadapt != nothing  || jj_emfeed != 1. || jj_regionswitch != 0.)
                                            continue
                                        elseif jj_pulse != scc_pulse_size && (jj_geadapt != nothing  || jj_emfeed != 1. || jj_regionswitch != 0.)
                                            continue
                                        elseif jj_geadapt != nothing  && (jj_emfeed != 1. || jj_regionswitch != 0.)
                                            continue
                                        elseif jj_emfeed != 1. && jj_regionswitch != 0.
                                            continue
                                        end

                                        # set the ge_string parameters
                                        if jj_gestring == "MILD"
                                            ge_string_min = 0.
                                            ge_string_mode = 0.
                                            ge_string_max = 0.5280681 # value based on converging ratio of marginal impacts from Burke et al regression for 1Lag (temperature and precipitation)
                                            ge_use_empirical = 0.
                                        elseif jj_gestring == "MEDIUM"
                                            ge_string_min = 0.
                                            ge_string_mode = 0.5280681
                                            ge_string_max = 1.
                                            ge_use_empirical = 0.
                                        elseif jj_gestring == "EMPIRICAL" # note: for EMPIRICAL and EMPIRICAL+, the triangular will be overwritten by draws from the empirical distribution
                                            ge_string_min = 0.
                                            ge_string_mode = 0.
                                            ge_string_max = 0.0001
                                            ge_use_empirical = 1.
                                        elseif jj_gestring == "EMPIRICAL+"
                                            ge_string_min = 0.
                                            ge_string_mode = 0.
                                            ge_string_max = 0.0001
                                            ge_use_empirical = 2.
                                        end

                                        # print out the parameters to track progress
                                        print(string("Scen_", jj_scen, " Ge-mode_", jj_gestring,
                                                     " Permafr_", jj_permafr, " Seaice_", jj_seaice, " Civvalue_", jj_civvalue,
                                                     " cbabs_", jj_cbabs, " eqwshare_", jj_eqwshare,
                                                     " convergence", jj_convergence, "pulse_", jj_pulse,
                                                     " geadapt", jj_geadapt, " emfeed", jj_emfeed,
                                                     " switch", jj_regionswitch))

                                        # define the output directory for the Monte Carlo files
                                        dir_MCoutput = string(dir_output, "mc_diGE/ge", jj_gestring,
                                                                        "_scen", jj_scen,
                                                                        "_pf", jj_permafr,
                                                                        "_se", jj_seaice,
                                                                        "_co", jj_convergence, "_bd", jj_cbabs, "_eq", jj_eqwshare,
                                                                        "_ci", jj_civvalue,
                                                                        "_p", jj_pulse,
                                                                        "_gead", jj_geadapt,
                                                                        "_emfe", jj_emfeed,
                                                                        "_sw", jj_regionswitch,
                                                                         "/")

                                        # calculate the MC SCC
                                        Random.seed!(masterseed)
                                        global scc_mcs_object = get_scc_mcs(samplesize, 2020, dir_MCoutput,
                                                                            scenario=jj_scen,
                                                                            pulse_size=jj_pulse,
                                                                            use_permafrost=jj_permafr,
                                                                            use_seaice=jj_seaice,
                                                                            use_page09damages=false,
                                                                            ge_minimum=ge_string_min,
                                                                            ge_maximum=ge_string_max,
                                                                            ge_mode=ge_string_mode,
                                                                            ge_use_empirical=ge_use_empirical,
                                                                            civvalue_multiplier=jj_civvalue,
                                                                            use_convergence=jj_convergence,
                                                                            cbabs=jj_cbabs,
                                                                            eqwbound=jj_eqwshare,
                                                                            geadrate = jj_geadapt,
                                                                            emfeedback = jj_emfeed,
                                                                            ge_use_switch = jj_regionswitch
                                                                            )

                                        # write out the full distribution
                                        writedlm(string(dir_MCoutput, "SCC_MC.csv"),
                                                                scc_mcs_object, ",")

                                        # create a NaN-free version of the SCC distribution
                                        scc_mcs_object = filter(!isnan, scc_mcs_object)

                                        # push the results into the data frame
                                        push!(df_sccMC, [jj_permafr, jj_seaice, jj_gestring,  jj_scen,
                                                        jj_convergence,  jj_cbabs, jj_eqwshare,
                                                        jj_civvalue, jj_pulse,
                                                        ifelse(jj_geadapt == nothing, 0., jj_geadapt),
                                                        jj_emfeed, jj_regionswitch,
                                                        mean(scc_mcs_object[:, 1]),
                                                        median(scc_mcs_object[:, 1]),
                                                        minimum(scc_mcs_object[:, 1]),
                                                        maximum(scc_mcs_object[:, 1]),
                                                        StatsBase.percentile(scc_mcs_object[:, 1], 25),
                                                        StatsBase.percentile(scc_mcs_object[:, 1], 75),
                                                        Statistics.std(scc_mcs_object[:, 1]),
                                                        Statistics.std(scc_mcs_object[:, 1]) / mean(scc_mcs_object[:, 1]),
                                                        StatsBase.percentile(scc_mcs_object[:, 1], 5),
                                                        StatsBase.percentile(scc_mcs_object[:, 1], 95),
                                                        StatsBase.percentile(scc_mcs_object[:, 1], 10),
                                                        StatsBase.percentile(scc_mcs_object[:, 1], 90),
                                                        count(scc_mcs_object .== 0.) / length(scc_mcs_object),
                                                        samplesize - length(scc_mcs_object)])

                                        # clean out the scc_mcs_object
                                        global scc_mcs_object = nothing
                                        ge_string_min = nothing
                                        ge_string_max = nothing
                                        ge_string_mode = nothing

                                            end
                                        end
                                    end
                                end
                            end
                        end
                    end
                end
            end
        end
    end
end

# remove the first placeholder row
df_sccMC = df_sccMC[df_sccMC[!, :scen] .!= "-999", :]

# export the results
CSV.write(string(dir_output, "MimiPageGrowthEffectsResultsMonteCarlo.csv"), df_sccMC)

### PART III: run the default scenario/parameter settings with an empirical persistence distribution derived from low-pass filtering climate vars

# rerun the main settings for an empirical distribution that is derived from low-pass filtering climate variables
include("main_model_growth_lowpass.jl")

# set the output directory for MC results
dir_MCoutput = string(dir_output, "lowpassfiltered_persistence/")

# calculate the stochastic mean SCC using the default parameter & scenario settings
Random.seed!(masterseed)
global scc_mcs_object = get_scc_mcs(samplesize, 2020, dir_MCoutput,
                                    scenario="RCP4.5 & SSP2",
                                    pulse_size=scc_pulse_size,
                                    use_permafrost=true,
                                    use_seaice=true,
                                    use_page09damages=false,
                                    ge_minimum=0.,
                                    ge_maximum=0.0001,
                                    ge_mode=0.,
                                    ge_use_empirical=1.,
                                    civvalue_multiplier=1.,
                                    use_convergence= 1.,
                                    cbabs=740.65,
                                    eqwbound=0.99,
                                    geadrate = 0.,
                                    emfeedback = 1.,
                                    ge_use_switch = 0.
                                    )

# # write out the full distribution
writedlm(string(dir_MCoutput, "SCC_MC.csv"),
                        scc_mcs_object, ",")

# clean out
global scc_mcs_object = nothing


# Filters using Nullable types to represent missing measurements

function update{T}(kf::KalmanFilter,y::Nullable{Vector{T}})
    if isnull(y.y)
        return kf
    else
        return update(kf,Observation(get(y)))
    end
end

function update!{T}(kf::KalmanFilter,y::Nullable{Vector{T}})
    if isnull(y)
        return kf
    else
        update!(kf,Observation(get(y)))
        return kf
    end
end

function predictupdate{T}(kf::BasicKalmanFilter,y::Nullable{Vector{T}})
    update(predict(kf),y)
end

function predictupdate!{T}(kf::BasicKalmanFilter,y::Nullable{Vector{T}})
    update!(predict!(kf),y)
end