export get_pha!

"""
    T = get_pha(Δ::Float64, z::Float64)

Command-line interface to IRIS online travel time calculator, which calls TauP [1-2].
Returns a matrix of strings.

Specify Δ in decimal degrees, z in km treating down as +.

Standard keywords: pha, to, v

Additional keywords:
* model: velocity model ("iasp91")

### References
[1] TauP manual: http://www.seis.sc.edu/downloads/TauP/taup.pdf
[2] Crotwell, H. P., Owens, T. J., & Ritsema, J. (1999). The TauP Toolkit:
Flexible seismic travel-time and ray-path utilities, SRL 70(2), 154-160.
"""
function get_pha!(Ev::SeisEvent;
                  pha::String   = KW.pha,
                  model::String = "iasp91",
                  to::Int64     = KW.to,
                  v::Int64      = KW.v
                  )

  # Check that distaz has been done
  TD = getfield(Ev, :data)
  N = getfield(TD, :n)
  z = zeros(Float64, N)
  if (TD.az == z) && (TD.baz == z) && (TD.dist == z)
    v > 0 && println(stdout, "az, baz, and dist are unset; calling distaz!...")
    distaz!(Ev)
  end

  # Generate URL and do web query
  src_dep = getfield(getfield(getfield(Ev, :hdr), :loc), :dep)
  if isempty(pha) || pha == "all"
    pq = "&phases=ttall"
  else
    pq = string("&phases=", pha)
  end
  url_tail = string("&evdepth=", src_dep, pq, "&model=", model, "&mintimeonly=true&noheader=true")

  # Loop begins
  dist = getfield(TD, :dist)
  PC = getfield(TD, :pha)
  for i = 1:N
    Δ = getindex(dist, i)
    pcat = getindex(PC, i)

    url = string("http://service.iris.edu/irisws/traveltime/1/query?", "distdeg=", Δ, url_tail)
    v > 1 && println(stdout, "url = ", url)

    req_info_str = string("\nIRIS travel time request:\nΔ = ", Δ, "\nDepth = ", z, "\nPhases = ", pq, "\nmodel = ", model)
    (R, parsable) = get_http_req(url, req_info_str, to)

    # Parse results
    if parsable
      req = String(take!(copy(IOBuffer(R))))
      pdat = split(req, '\n')
      deleteat!(pdat, findall(isempty, pdat))   # can have trailing blank line
      npha = length(pdat)
      for j = 1:npha
        pha = split(pdat[j], keepempty=false)
        pcat[pha[10]] = SeisPha(0.0,                              # a
                                parse(Float64, pha[8]),           # d
                                parse(Float64, pha[7]),           # ia
                                0.0,                              # res
                                parse(Float64, pha[5]),           # rp
                                parse(Float64, pha[6]),           # ta
                                parse(Float64, pha[4]),           # tt
                                0.0,                              # unc
                                ' ',                              # pol
                                ' ',                              # qual
                                )
      end
    end
  end
  return nothing
end
# "Model: iasp91"
# "Distance   Depth   Phase        Travel    Ray Param  Takeoff  Incident  Purist    Purist    "
# "  (deg)     (km)   Name         Time (s)  p (s/deg)   (deg)    (deg)   Distance   Name      "
# "--------------------------------------------------------------------------------------------"
# "   66.83    19.7   P             650.37     6.375     19.49    19.42    66.83   = P"
# "   66.83    19.7   pP            656.77     6.385    160.48    19.45    66.83   = pP"
# "   66.83    19.7   sP            659.32     6.383    168.84    19.45    66.83   = sP"
# "   66.83    19.7   PcP           678.82     4.167     12.59    12.55    66.83   = PcP"
# "   66.83    19.7   PP            797.49     8.699     27.08    26.98    66.83   = PP"
# "   66.83    19.7   PKiKP        1038.70     1.352      4.06     4.04    66.83   = PKiKP"
# "   66.83    19.7   pPKiKP       1045.47     1.352    175.94     4.04    66.83   = pPKiKP"
# "   66.83    19.7   sPKiKP       1047.95     1.352    177.65     4.04    66.83   = sPKiKP"
# "   66.83    19.7   S            1182.56    12.085     21.49    21.42    66.83   = S"
# "   66.83    19.7   pS           1190.65    12.109    140.68    21.46    66.83   = pS"
# "   66.83    19.7   sS           1193.48    12.101    158.48    21.45    66.83   = sS"
# "   66.83    19.7   SP           1201.90    13.645     24.43    45.38    66.83   = SP"
# "   66.83    19.7   PS           1204.86    13.644     45.55    24.35    66.83   = PS"
# "   66.83    19.7   SKS          1246.03     7.572     13.27    13.23    66.83   = SKS"
# "   66.83    19.7   SKKS         1246.05     7.585     13.29    13.25    66.83   = SKKS"
# "   66.83    19.7   ScS          1246.40     7.761     13.61    13.56    66.83   = ScS"
# "   66.83    19.7   SKiKP        1250.70     1.408      2.45     4.21    66.83   = SKiKP"
# "   66.83    19.7   pSKS         1254.86     7.573    156.66    13.23    66.83   = pSKS"
# "   66.83    19.7   sSKS         1257.44     7.573    166.73    13.23    66.83   = sSKS"
# "   66.83    19.7   SS           1441.81    15.478     27.98    27.88    66.83   = SS"
# "   66.83    19.7   PKIKKIKP     1860.33     1.411      4.23     4.22   293.17   = PKIKKIKP"
# "   66.83    19.7   SKIKKIKP     2072.34     1.352      2.35     4.05   293.17   = SKIKKIKP"
# "   66.83    19.7   PKIKKIKS     2074.81     1.352      4.06     2.34   293.17   = PKIKKIKS"
# "   66.83    19.7   SKIKKIKS     2286.62     1.297      2.25     2.25   293.17   = SKIKKIKS"
# "   66.83    19.7   PKIKPPKIKP   2359.03     1.666      5.00     4.98   293.17   = PKIKPPKIKP"
# "   66.83    19.7   PKPPKP       2361.68     2.976      8.96     8.93   293.17   = PKPPKP"
# "   66.83    19.7   PKPPKP       2364.24     3.928     11.86    11.82   293.17   = PKPPKP"
# "   66.83    19.7   SKIKSSKIKS   3216.94     1.426      2.48     2.47   293.17   = SKIKSSKIKS"


#=
New DA structure that allows minimal manipulation
=#

struct EdgeTreeRates2
    tree::Tree                          # the grafted tree
    stem_rate::Array{Float64,1}         # λ_i : every other rates are given divided by λ_i
    tip_rate::Float64                   # the tip rate
    tip_id::Int64                       # tip_id
    tip_rates::Array{Float64,1}         # rates at t_i (for internal branches)
    effective_bl::Float64               # t_j * λ_j / λ_i
    tip_number::Int64                   # extant tips
    scale::Float64
    #relative_rates::Array{Float64,1}
    depth::Float64
    stem_depth::Float64
    parent_edge::Int64
end

function EdgeTreeRates2(tree::Tree, stem_rate::Float64, tip_rate::Float64,
    tip_id::Int64, tip_rates::Array{Float64,1}, tip_number::Int64, depth::Float64,stem_depth::Float64, parent_edge::Int64)
    effective_bl = sum(extract_branch_lengths(tree) .* extract_rates(tree)) / stem_rate
    return EdgeTreeRates2(tree, [stem_rate], tip_rate, tip_id, tip_rates,
        effective_bl, tip_number, depth, stem_depth,parent_edge)
end

function EdgeTreeRates2(tree::Tree, stem_rate::Float64, tip_rate::Float64,
    tip_id::Int64, tip_rates::Array{Float64,1},
    effective_bl::Float64, tip_number::Int64,depth::Float64,stem_depth::Float64,
    parent_edge::Int64)
    return EdgeTreeRates2(tree, [stem_rate], tip_rate, tip_id, tip_rates, effective_bl, tip_number, depth,stem_depth, parent_edge)
end

function EdgeTreeRates2(tree::Tree, stem_rate::Array{Float64,1}, tip_rate::Float64,
    tip_id::Int64, tip_rates::Array{Float64,1},
    effective_bl::Float64, tip_number::Int64,
    depth::Float64,stem_depth::Float64,parent_edge::Int64)

    return EdgeTreeRates2(tree, stem_rate, tip_rate, tip_id, tip_rates, effective_bl,
        tip_number, stem_rate[1], depth,stem_depth, parent_edge)
end

#=
A few functions to manipulate it
=#

function change_bl(tree::Tree, scale::Float64)
    function aux(sub_tree)
        if sub_tree.n_nodes < 2
            return Tree(sub_tree.offsprings,
                sub_tree.branch_length / scale,
                sub_tree.attributes.*scale,
                sub_tree.n_nodes,
                sub_tree.extant)
        else
            left = aux(sub_tree.offsprings[1])
            right = aux(sub_tree.offsprings[2])
            return Tree([left,right],
                sub_tree.branch_length / scale,
                sub_tree.attributes.*scale,
                sub_tree.n_nodes,
                sub_tree.extant)
        end
    end

    aux(tree)
end

function change_bl(tree::Tree, scale::Float64, stem_rate::Float64)
    function aux(sub_tree)
        if sub_tree.n_nodes < 2
            return Tree(sub_tree.offsprings,
                sub_tree.branch_length / scale,
                sub_tree.attributes.*stem_rate,
                sub_tree.n_nodes,
                sub_tree.extant)
        else
            left = aux(sub_tree.offsprings[1])
            right = aux(sub_tree.offsprings[2])
            return Tree([left,right],
                sub_tree.branch_length / scale,
                sub_tree.attributes.*stem_rate,
                sub_tree.n_nodes,
                sub_tree.extant)
        end
    end

    aux(tree)
end

function graft_edge_trees(tree, edge_trees::Array{EdgeTreeRates2,1})

    function aux(subtree, sub_edge_trees)
        if subtree.n_nodes < 2
            return change_bl(sub_edge_trees[1].tree, sub_edge_trees[1].scale, sub_edge_trees[1].stem_rate[1])
        else
            n_edges_left = subtree.offsprings[1].n_nodes + 1
            sub_left = deepcopy(sub_edge_trees[2:n_edges_left])
            sub_right = deepcopy(sub_edge_trees[(n_edges_left+1):subtree.n_nodes])
            subtree_left = aux(subtree.offsprings[1], sub_left)
            subtree_right = aux(subtree.offsprings[2], sub_right)
            return graft_to_edge(Tree([subtree_left, subtree_right], 0., subtree.attributes),
                change_bl(sub_edge_trees[1].tree, sub_edge_trees[1].scale, sub_edge_trees[1].stem_rate[1]), 0, sub_edge_trees[1].tip_id)
        end
    end

    n_edges_left = tree.offsprings[1].n_nodes
    left = deepcopy(edge_trees[1:n_edges_left])
    right = deepcopy(edge_trees[(n_edges_left+1):(tree.n_nodes - 1)])
    tree_left = aux(tree.offsprings[1], left)
    tree_right = aux(tree.offsprings[2], right)
    return Tree([tree_left, tree_right], tree.branch_length, tree.attributes)
end

function node_depths_base(tree::Tree, edge_trees::Array{EdgeTreeRates2,1})
    function aux(subtree, node_times, current_depth, att)
        if subtree.n_nodes < 2
            pushfirst!(node_times, current_depth)
        else
            aux(subtree.offsprings[2],node_times, current_depth + subtree.branch_length/att, att)
            aux(subtree.offsprings[1],node_times, current_depth + subtree.branch_length/att, att)
            pushfirst!(node_times,current_depth)
        end
    end

    node_times = node_depths_base(tree)[2:end]
    node_times_full = Array{Float64,1}(undef,0)
    for i in length(edge_trees):-1:1
        aux(edge_trees[i].tree, node_times_full, node_times[i], edge_trees[i].scale)
    end
    #pushfirst!(node_times,0.)
    return node_times_full
end

function node_depths(tree::Tree, edge_trees::Array{EdgeTreeRates2,1})
    function aux(subtree, node_times, current_depth, sa)
        if subtree.n_nodes < 2
            pushfirst!(node_times, current_depth + subtree.branch_length/sa)
        else
            aux(subtree.offsprings[2],node_times, current_depth + subtree.branch_length/sa, sa)
            aux(subtree.offsprings[1],node_times, current_depth + subtree.branch_length/sa, sa)
            pushfirst!(node_times,current_depth+ subtree.branch_length/sa)
        end
    end

    node_times = node_depths_base(tree)[2:end]
    node_times_full = Array{Float64,1}(undef,0)
    for i in length(edge_trees):-1:1
        aux(edge_trees[i].tree, node_times_full, node_times[i], edge_trees[i].scale)
    end
    #pushfirst!(node_times,0.)
    return node_times_full
end

function node_depths_base(tree::Tree)
    function aux(subtree, node_times, current_depth)
        if subtree.n_nodes < 2
            pushfirst!(node_times, current_depth)
        else
            aux(subtree.offsprings[2],node_times, current_depth + subtree.branch_length)
            aux(subtree.offsprings[1],node_times, current_depth + subtree.branch_length)
            pushfirst!(node_times,current_depth)
        end
    end

    node_times = Array{Float64,1}(undef,0)
    aux(tree, node_times, 0.)
    #pushfirst!(node_times,0.)
    return node_times
end

function node_depths(tree::Tree)
    function aux(subtree, node_times, current_depth)
        if subtree.n_nodes < 2
            pushfirst!(node_times, current_depth + subtree.branch_length)
        else
            aux(subtree.offsprings[2],node_times, current_depth + subtree.branch_length)
            aux(subtree.offsprings[1],node_times, current_depth + subtree.branch_length)
            pushfirst!(node_times,current_depth+ subtree.branch_length)
        end
    end

    node_times = Array{Float64,1}(undef,0)
    aux(tree, node_times, 0.)
    #pushfirst!(node_times,0.)
    return node_times
end

function graft_to_tip(tree,subtree, tip_id)

    function aux(t1, t2, tip)
        if t1.n_nodes == 1
            return Tree(t2.offsprings, t2.branch_length + t1.branch_length, t1.attributes, t2.n_nodes, t2.extant)
        else
            t11 = t1.offsprings[1]
            t12 = t1.offsprings[2]
            t11_tips = (t11.n_nodes + 1) / 2
            if t11_tips < tip
                t12 = aux(t12, t2, tip - t11_tips)
                return Tree([t11, t12], t1.branch_length, t1.attributes)
            else
                t11 = aux(t11, t2, tip)
                return Tree([t11, t12], t1.branch_length, t1.attributes)
            end
        end
    end

    aux(tree,subtree, tip_id)
end

function graft_to_edge(tree, sub_tree, edge, tip)

    if sub_tree.n_nodes < 2
        return Tree(tree.offsprings, tree.branch_length + sub_tree.branch_length, tree.attributes, tree.n_nodes, sub_tree.extant)
    end

    function aux(t1, t2, edge_id)
        if t1.n_nodes == 1
            return t2
        elseif edge_id == 1
            new_t1 = Tree(t1.offsprings, 0., t1.attributes, t1.n_nodes, t1.extant)
            return graft_to_tip(t2, new_t1, tip)
        else
            t11 = t1.offsprings[1]
            t12 = t1.offsprings[2]
            if t11.n_nodes < (edge_id-1)
                t12 = aux(t12, t2, edge_id-1-t11.n_nodes)
                return Tree([t11, t12], t1.branch_length, t1.attributes)
            else
                t11 = aux(t11, t2, edge_id-1)
                return Tree([t11, t12], t1.branch_length, t1.attributes)
            end
        end
    end

    return aux(tree, sub_tree, edge+1)
end

function graft_to_tips(tree, subtree_list, tips)
    if length(tips) == 0
        return tree
    end

    ntips = Int64((tree.n_nodes + 1) / 2)
    subtrees = fill(Tree(),ntips)
    #println("$tips, $ntips")

    for i in 1:ntips
        for j in 1:length(tips)
            if i==tips[j]
                subtrees[i] = subtree_list[j]
                break
            end
        end
    end

    function aux(t1, t2)
        if t1.n_nodes <= 1
            #=if ! t2[1].extant
                println("is it ok ? $(t2[1]) ; $(t1.extant)")
            end=#
            return Tree(t2[1].offsprings, t2[1].branch_length + t1.branch_length, t1.attributes, t2[1].extant)
        else
            t11 = t1.offsprings[1]
            t12 = t1.offsprings[2]
            t11_tips = Int64((t11.n_nodes + 1) / 2)

            t11 = aux(t11, t2[1:t11_tips])
            t12 = aux(t12, t2[(t11_tips+1):end])

            return Tree([t11, t12], t1.branch_length, t1.attributes)
        end
    end

    new_tree = aux(tree,subtrees)
    return new_tree
end

function init_edge_tree_rates2(tree, rates)
    bl = extract_branch_lengths(tree)
    edge_trees = Array{EdgeTreeRates2,1}(undef,0)
    lefts = n_left(tree)
    for i in 2:tree.n_nodes
        nd = get_node_depth(tree, i-1)
        parent_edge = get_parent_edge(tree, i-1)

        ne = 0
        if lefts[i]==0
            ne = 1
        end

        push!(edge_trees,
            EdgeTreeRates2(Tree(Array{Tree,1}(undef,0), bl[i] * rates[i], 1.),       # tree
                rates[i],                                                           # stem_rate
                1.,                                                                 # tip_rate
                1,                                                                  # tip_id
                [1.],                                                               # tip_rates
                bl[i],                                                              # effective_bl
                ne,                                                                  # tip_number
                nd,
                nd + bl[i],
                parent_edge))
    end
    return edge_trees
end

function sim_ClaDS2_time(root_age,σ,α,ε,λ0,u,sf,lf ; return_if_extinct = true, max_node_number = Inf, make_tree = true, prune_extinct = false, return_if_max = true)
    # accesory functions that will be called latter
    bernouilli=Bernoulli(ε/(1+ε))
    function dead()
        return (rand(bernouilli)[1] == 1)
    end

    lambda_law = LogNormal(log(α),σ)
    function new_rates(λ)
        λ * rand(lambda_law, 2)
    end
    #=function new_rates(λ::Float64)
        X = (rand(lambda_law, 2))
        λ * X#* sort2!(X)
    end=#
    #println()
    function aux_ext(time, rate, n_max, s)
        #println(n_max)
        if n_max == 0
            #println("max")
            return Tree(Array{Tree,1}(undef,0), -1., [-1], -1, false), 0, -Inf
        end
        node_time = randexp()/(rate*(1+ε))
        if node_time <= 0
            return Tree(Array{Tree,1}(undef,0), -1., [-1], -1, false), 0, -Inf
        end
        if node_time > time
            s += lf
            #print(" $s $u ;")
            if u<s
                return Tree(Array{Tree,1}(undef,0), time, [rate], 1, true), n_max + 1, s
            else
                #print(" !")
                return Tree(Array{Tree,1}(undef,0), -1., [-1], -1, false), 0, s
            end
        else
                is_dead = dead()
                left_time = time - node_time
                if time == left_time
                    return Tree(Array{Tree,1}(undef,0), -1., [-1], -1, false), 0
                end
                if is_dead
                    #print("dead")
                    return Tree(Array{Tree,1}(undef,0), node_time, [rate], 1, false), n_max + 1, s
                else
                    offspring_rates = new_rates(rate)
                    if offspring_rates[1] > offspring_rates[2]
                        offspring_rates = [offspring_rates[2],offspring_rates[1]]
                    end
                    #print("$depth ;")
                    left_tree = aux_ext(left_time, offspring_rates[1], n_max - 1, s)
                    #print(left_tree[2])
                    if left_tree[1].n_nodes == -1
                        return left_tree
                    end
                    right_tree = aux_ext(left_time, offspring_rates[2], left_tree[2], left_tree[3])
                    if right_tree[1].n_nodes == -1
                        return right_tree
                    end
                    return Tree([left_tree[1], right_tree[1]], node_time, [rate]), right_tree[2], right_tree[3]
                end
        end
    end
    tree = aux_ext(root_age, λ0, max_node_number, sf)

    if prune_extinct
        if (tree[1].n_nodes == -1) & return_if_max
            return Tree(Array{Tree,1}(undef,0), -1., [-1], -1, false), -Inf
        else
            return prune_extinct_lineages(tree[1]), tree[3]
        end
    else
        if (tree[1].n_nodes == -1) & return_if_max
            return Tree(Array{Tree,1}(undef,0), -1., [-1], -1, false), -Inf
        else
            return tree[1], tree[3]
        end
    end
end

function sim_ClaDS2_time_rates(root_age,σ,α,ε,λ0 ; return_if_extinct = true, max_node_number = 1_000,
    make_tree = true, prune_extinct = false, return_if_max = true, return_na = false)

    # accesory functions that will be called latter
    bernouilli=Bernoulli(ε/(1+ε))
    function dead()
        return (rand(bernouilli)[1] == 1)
    end

    lambda_law = LogNormal(log(α),σ)
    function new_rates(λ)
        λ * rand(lambda_law, 2)
    end
    #=function new_rates(λ::Float64)
        X = (rand(lambda_law, 2))
        λ * X#* sort2!(X)
    end=#
    if return_na
        function aux_na(time, rate, n_max, rates, live)
            #println(n_max)
            if n_max == 0
                return Tree(Array{Tree,1}(undef,0), -1., [-1], -1, false), 0
            end
            node_time = randexp()/(rate*(1+ε))
            if node_time <= 0
                return Tree(Array{Tree,1}(undef,0), -1., [-1], -1, false), 0
            end
            if node_time > time
                #offsprings, branch_length, attributes, n_nodes
                push!(rates, rate)
                push!(live, true)
                return Tree(Array{Tree,1}(undef,0), time, [rate], 1, true), n_max
            else
                is_dead = dead()
                if is_dead
                    #print("dead")
                    push!(live, false)
                    return Tree(Array{Tree,1}(undef,0), node_time, [rate], 1, false), n_max + 1
                else
                    offspring_rates = new_rates(rate)
                    left_time = time - node_time
                    if time == left_time
                        return Tree(Array{Tree,1}(undef,0), -1., [-1], -1, false), 0
                    end
                    #print("$depth ;")
                    left_tree = aux_na(left_time, offspring_rates[1], n_max - 1, rates, live)
                    #print(left_tree[2])
                    if left_tree[1].n_nodes == -1
                        return left_tree
                    end
                    right_tree = aux_na(left_time, offspring_rates[2], left_tree[2], rates, live)
                    if right_tree[1].n_nodes == -1
                        return right_tree
                    end
                    return Tree([left_tree[1], right_tree[1]], node_time, [rate]), right_tree[2]
                end
            end
        end

        rates = Array{Float64,1}(undef,0)
        alive = Array{Bool,1}(undef,0)

        tree = aux_na(root_age, λ0, max_node_number, rates, alive)[1]
        if (tree.n_nodes == -1) & return_if_max
            return Tree(Array{Tree,1}(undef,0), -1., [-1], -1, false), Array{Float64,1}(undef,0), Array{Bool,1}(undef,0)
        else
            return tree, rates, alive
        end
    else
        function aux_ext(time, rate, n_max, rates)
            #println(n_max)
            if n_max == 0
                return Tree(Array{Tree,1}(undef,0), -1., [-1], -1, false), 0
            end
            node_time = randexp()/(rate*(1+ε))
            if node_time > time
                #offsprings, branch_length, attributes, n_nodes
                push!(rates, rate)
                return Tree(Array{Tree,1}(undef,0), time, [rate], 1, true), n_max
            else
                is_dead = dead()
                if is_dead
                    #print("dead")
                    return Tree(Array{Tree,1}(undef,0), node_time, [rate], 1, false), n_max + 1
                else
                    offspring_rates = new_rates(rate)
                    left_time = time - node_time
                    #print("$depth ;")
                    left_tree = aux_ext(left_time, offspring_rates[1], n_max - 1, rates)
                    #print(left_tree[2])
                    if left_tree[1].n_nodes == -1
                        return left_tree
                    end
                    right_tree = aux_ext(left_time, offspring_rates[2], left_tree[2], rates)
                    if right_tree[1].n_nodes == -1
                        return right_tree
                    end
                    return Tree([left_tree[1], right_tree[1]], node_time, [rate]), right_tree[2]
                end
            end
        end

        rates = Array{Float64,1}(undef,0)

        tree = aux_ext(root_age, λ0, max_node_number, rates)[1]
        if (tree.n_nodes == -1) & return_if_max
            return Tree(Array{Tree,1}(undef,0), -1., [-1], -1, false), Array{Float64,1}(undef,0)
        else
            return tree, rates
        end
    end

end

function sim_ClaDS2_time_unsampled_rates(root_age,σ,α,ε,λ0, u,fl ;
    sampling_proba = 1., return_if_extinct = true, max_node_number = 1_000,
    make_tree = false, prune_extinct = false)

    while true
        tree, rates, s = sim_ClaDS2_time_rates_tip(root_age,σ,α,ε,λ0,u,sampling_proba,fl,
            return_if_extinct = true, max_node_number = max_node_number,
            prune_extinct = false)
        if tree.n_nodes == -1
            return Tree(Array{Tree,1}(undef,0), -1., [-1], -1, false), false, Inf, rates
        else
            n_alive = length(rates)
            if s < u
                return tree, false, n_alive, rates
            else
                return tree, true, n_alive, rates
            end
        end
    end
end

function sim_ClaDS2_time_rates_tip(root_age,σ,α,ε,λ0,u,sf,fl ; return_if_extinct = true, max_node_number = 1_000,
    make_tree = true, prune_extinct = false, return_if_max = true, return_na = false)

    # accesory functions that will be called latter
    #print(" $(ε/(1+ε))")
    bernouilli=Bernoulli(ε/(1+ε))
    function dead()
        return (rand(bernouilli)[1] == 1)
    end

    lambda_law = LogNormal(log(α),σ)
    function new_rates(λ)
        λ * rand(lambda_law, 2)
    end
    #=function new_rates(λ::Float64)
        X = (rand(lambda_law, 2))
        λ * X#* sort2!(X)
    end=#
    if return_na
    else
        function aux_ext(time, rate, n_max, rates, s ,n)
            if n_max == 0
                return Tree(Array{Tree,1}(undef,0), -1., [-1], -1, false), 0,0,0
            end
            node_time = randexp()/(rate*(1+ε))
            #println(node_time)
            if node_time <= 0
                return Tree(Array{Tree,1}(undef,0), -1., [-1], -1, false), 0,0,0
            end#print(" $s $n $u;")# $time $node_time $rate;")
            if node_time > time
                #print("zzzz")
                new_s = s
                #offsprings, branch_length, attributes, n_nodes
                push!(rates, rate)
                if n>0
                    new_s = s + log(1-sf)+log(n+1)-log(n)
                else
                    new_s = s + log(sf)
                end
                if new_s >= s
                    #print("a")
                    return Tree(Array{Tree,1}(undef,0), time, [rate], 1, true), n_max, new_s , n+1
                elseif u < new_s
                    #print("b")
                    return Tree(Array{Tree,1}(undef,0), time, [rate], 1, true), n_max, new_s , n+1
                else
                    #print("c")
                    return Tree(Array{Tree,1}(undef,0), -1., [-1], -1, false), 0,new_s ,n+1
                end
            else
                is_dead = dead()
                if is_dead
                    return Tree(Array{Tree,1}(undef,0), node_time, [rate], 1, false), n_max + 1, s, n
                else
                    offspring_rates = new_rates(rate)
                    left_time = time - node_time
                    if time == left_time
                        return Tree(Array{Tree,1}(undef,0), -1., [-1], -1, false), 0,0,0
                    end
                    #print("$depth ;")
                    left_tree = aux_ext(left_time, offspring_rates[1], n_max - 1, rates, s, n)
                    #print(left_tree[2])
                    if left_tree[1].n_nodes == -1
                        return left_tree
                    end
                    right_tree = aux_ext(left_time, offspring_rates[2], left_tree[2], rates, left_tree[3], left_tree[4])
                    if right_tree[1].n_nodes == -1
                        return right_tree
                    end
                    return Tree([left_tree[1], right_tree[1]], node_time, [rate]), right_tree[2], right_tree[3], right_tree[4]
                end
            end
        end

        rates = Array{Float64,1}(undef,0)

        tree = aux_ext(root_age, λ0, max_node_number, rates,fl,0)
        #if ((tree[1].n_nodes == -1) & return_if_max) | (tree[4] == 0) | (u > tree[3])
        #    return Tree(Array{Tree,1}(undef,0), -1., [-1], -1, false), Array{Float64,1}(undef,0)
        #else
        return tree[1], rates, tree[3]
        #end
    end

end

function update_edges_ETR2!(tree::Tree, edge_trees::Array{EdgeTreeRates2,1}, σ::Float64, α::Float64, ε::Float64, rates::Array{Float64,1},
    lefts::Array{Int64,1}; it_rates = 1, prior_ε = "lognormal", logε0 = 0., sd = 0.5)

    if prior_ε == "uniform"
        ε = draw_ε_crown_priorUnif(tree, edge_trees, lefts)
    elseif prior_ε == "uniformInf"
        ε = draw_ε_crown(tree, edge_trees, lefts)
    elseif prior_ε == "ClaDS0"
        ε = 0.
    else
        ε = draw_ε_crown_priorln(tree, edge_trees, lefts, logε0 = logε0, sd = sd)
    end

    for j in 1:it_rates

        draw_λi_quad!(rates, edge_trees, σ, α, ε, tree, lefts)
        relative_rates = extract_relative_rates(tree, edge_trees, rates)

        σ = draw_σ(relative_rates, α, β0 = 0.05, α0 = 0.5)
        α = draw_α(relative_rates, σ,  α_0 = -0.05, σ_0 = 0.1)

        if prior_ε == "uniform"
            ε = draw_ε_crown_priorUnif(tree, edge_trees, lefts)
        elseif prior_ε == "uniformInf"
            ε = draw_ε_crown(tree, edge_trees, lefts)
        elseif prior_ε == "ClaDS0"
            ε = 0.
        else
            ε = draw_ε_crown_priorln(tree, edge_trees, lefts, logε0 = logε0, sd = sd)
        end
    end

    return ε, σ, α
end


module XInts

export XInt

using Base.GMP: Limb, BITS_PER_LIMB, SLimbMax, ULimbMax, ClongMax, CulongMax, CdoubleMax
import Base.GMP: isneg, ispos
import Base.GMP.MPZ
import Base.MPFR
using Base.GC: @preserve
using Base: BitInteger
import Base: +, -, *, ^, &, |, ==, /, ~, <<, >>, >>>, <, <=,
             string, widen, hastypemax, tryparse_internal,
             unsafe_trunc, trunc, mod, rem, iseven, isodd, gcd, lcm, xor, div, fld, cld,
             invmod, count_ones, trailing_zeros, trailing_ones, cmp, isqrt,
             flipsign, powermod, gcdx, promote_rule, factorial, binomial,
             digits!, ndigits0zpb, signbit, sign, iszero, isone,
             AbstractFloat, BigFloat, float, _prevpow2, _nextpow2,
             hash, hash_integer, sum, divrem

mutable struct Wrap
    b::BigInt
    d::Ptr{Limb}

    function Wrap(isoutput::Bool=false)
        b = BigInt()
        @assert b.alloc == 1
        @assert b.size == 0
        wrap = new(b, b.d)
        if !isoutput
            # init! was called
            finalizer(wrap) do wrap
                wrap.b.d = wrap.d
                wrap.b.alloc = 1
                wrap.b.size = 0
            end
        end
        wrap
    end
end

if BITS_PER_LIMB == 32
    const SLimb = Int32
    const SLimbW = Int64
    const LimbW = UInt64

    @inline _divLimb(n) = n >>> 5
    @inline _modLimb(n) = n & 31
    @inline _divByteLimb(n) = n >>> 2
    @inline _modByteLimb(n) = n & 3
elseif BITS_PER_LIMB == 64
    const SLimb = Int64
    const SLimbW = Int128
    const LimbW = UInt128

    @inline _divLimb(n) = n >>> 6
    @inline _modLimb(n) = n & 63
    @inline _divByteLimb(n) = n >>> 3
    @inline _modByteLimb(n) = n & 7
else
    error()
end

const slimbmin = typemin(SLimb)
const slimbmax = typemax(SLimb)
# when there is only one limb in a vector, this limb must always be >= limb1min
# reciprocally, a direct integer must have its absolute value < limb1min
const limb1min = slimbmin % Limb # typically 0x8000000000000000

const Short = Union{Limb,SLimb}
const ShortMax = Union{ULimbMax,SLimbMax}

abstract type XSigned <: Signed end

# T must be a "bit integer", assumed for efficiency, and also sizeof
struct Static{T<:Integer} <: XSigned
    x::T
end

# we could re-use Static for this, but Static <: Number already has getindex defined,
# so we clearly separate the two concepts
struct SVec{U<:Unsigned}
    u::U
end

is_short(s::Static) = sizeof(s.x) < sizeof(Limb) ? true : fits(s.x)
short(s::Static) = s.x % SLimb

vec(s::Static{<:Signed}) = SVec(unsigned(abs(s.x)))
vec(s::Static{<:Unsigned}) = SVec(s.x)

function lenvec(s::Static)
    # @assert !is_short(s)
    v = vec(s)
    lz = leading_zeros(v.u) >>> 3 # count leading full zero bytes
    usedbytes = sizeof(v.u)-lz
    _divByteLimb(usedbytes - 1) + 1, v
end

len(s::Static) = lenvec(s)[1]
vec(s::Static) = lenvec(s)[2]

iszero(s::Static) = iszero(s.x)
ispos(s::Static) = s.x > 0
signbit(s::Static) = s.x < 0
_sign(s::Static) = ispos(s) ? SLimb(1) : SLimb(-1)

function Base.getindex(s::SVec, i::Integer)
    # getindex is called only when sizeof(x) >= sizeof(Limb)
    if sizeof(s.u) == sizeof(Limb)
        # we always assume @inbounds
        s.u % Limb
    else
        (s.u >> ((i-1)*BITS_PER_LIMB)) % Limb
    end
end

Base.show(io::IO, s::Static) =
    if is_short(s)
        print(io, "Static($(s.x))")
    else
        sl, sv = lenvec(s)
        print(io, "Static(", [sv[i] for i=1:sl], ")")
    end

struct XInt <: XSigned
    x::SLimb # immediate integer or sign+length
    v::Union{Nothing,Vector{Limb}}

    @inline XInt(x::SLimb) =
        x === typemin(SLimb) ?
            new(-one(SLimb), [limb1min]) :
            new(x, nothing)

    global _XInt

    # unsafe version which doesn't check for typemin(SLimb)
    _XInt(x::SLimb, ::Nothing=nothing) = new(x, nothing)

    function _XInt(x::SLimb, v::Vector{Limb}, reduce::Bool=false)
        if reduce
            # we still assume length(v) >= abs(x); reduce here means to not store
            # a too small integer in a vector representation
            xl = abs(x)
            if xl <= 1
                if xl == 0
                    return new(SLimb(0), nothing)
                else # xl == 1
                    limb = @inbounds v[1]
                    limb < limb1min &&
                        return new(flipsign(limb % SLimb, x), nothing)
                end
            end
        end
        new(x, v)
    end
end

XInt(x::XInt) = x
XInt(x::Static) = XInt(x.x)

# reducing version of XInt(::XInt)
_XInt(x::XInt) =
    if x.v === nothing
        x
    else
        _XInt(x.x, vec(x), true)
    end

_XInt(u::Limb) = _XInt(u % SLimb) # no check done that u fits

is_short(x::XInt) = x.v === nothing
is_short(x::XInt, y::XInt) = x.v === nothing === y.v

short(x::XInt) = x.x
len(x::XInt) = abs(x.x)

include("MPN.jl")
include("mutants.jl")

MPN.ptr(x::XInt) = pointer(x.v::Vector{Limb})
MPN.len(x::XInt) = abs(x.x) % MPN.Size

function _XInt(z::BigInt, extra::SLimb=0)
    # @assert extra >= 0
    len = abs(z.size)
    if len == 0
        XInt(zero(SLimb))
    elseif len == 1 && (x = unsafe_load(z.d)) < limb1min
        XInt(flipsign(x % SLimb, z.size))
    else
        x = _XInt(z.size % SLimb, Vector{Limb}(undef, len+extra))
        @preserve x z unsafe_copyto!(pointer(x.v), z.d, len)
        x
    end
end

XInt(z::BigInt) = _XInt(z)

fits(z::Limb) = !Core.is_top_bit_set(z) # z < limb1min
fits(z::SLimb) = z !== slimbmin
fits(z::Signed) = slimbmin < z <= slimbmax
fits(z::Unsigned) = z <= slimbmax

XInt(z::Limb) = XInt!(nothing, z)

# only SLimb and Limb in this Union need to be validated, but are taken care of by
# other more specific constructors
XInt(z::Union{ULimbMax,SLimbMax}) = _XInt(z % SLimb)

XInt(z::Union{SLimbW,LimbW}) = XInt!(nothing, z)
_XInt(z::Union{SLimbW,LimbW}, extra::SLimb=0) = XInt!(nothing, z, extra)

# TODO: this can be made quite faster for "bit integers": for some reason, allocating
# limbs according to sizeof(z) (and breaking the loop when iszero(zz))
# is more efficient than via ndigits (although ndigits seems to be fast)
# NOTE: also, it allocates more than expected for e.g. Int256, and this disappears
# if the function is re-evaled (then we get 1 allocation as expected)
function _XInt(z::Integer, extra::SLimb=0)::XInt
    slimbmin < z <= slimbmax &&
        return _XInt((z % SLimb)::SLimb)
    nd = ndigits(z, base=2)
    xl = _divLimb(nd)
    xl += !iszero(_modLimb(nd))
    limbs = _vec(xl+extra)
    zz = abs(z)
    for i=1:xl
        @inbounds limbs[i] = zz % Limb
        zz >>>= BITS_PER_LIMB
    end
    _XInt(flipsign(xl, z), limbs)
end

@inline XInt(z::Integer) = _XInt(z)

function XInt(x::Float64)
    isinteger(x) || throw(InexactError(:XInt, XInt, x))
    unsafe_trunc(XInt, x)
end

# no Union, to disambiguate
XInt(x::Float16) = XInt(Float64(x))
XInt(x::Float32) = XInt(Float64(x))

function init!(z::Wrap, x::XInt)
    b = z.b
    if is_short(x)
        b.alloc = 1
        b.size = sign(x.x)
        b.d = z.d
        unsafe_store!(z.d, abs(x.x) % Limb)
    else
        len = x.x
        b.alloc = abs(len)
        b.size = len
        b.d = pointer(x.v)
    end
    z.b::BigInt
end

struct Wraps
    # read-write temporary storage (output arguments)
    U::Wrap
    V::Wrap
    W::Wrap
    # read-only shallow wrappers of BigInt objects ("const" arguments in libgmp)
    X::Wrap
    Y::Wrap
    Z::Wrap

    Wraps() = new(Wrap(true), Wrap(true), Wrap(true), Wrap(), Wrap(), Wrap())
end

const tWraps = Wraps[]

mutable struct Buffer
    vec::Vector{Limb}
    # TODO: determine whether `used` is really needed; it's to try to work
    # around the problem mentioned in Julia#17015, that affects also usage
    # of the @bigint macro, related to GC
    used::Bool

    Buffer() = new(sizehint!(Limb[], 20), false)
end

const tBuffer = Buffer[]

function tmpvec(n::Integer)
    buf = tBuffer[Threads.threadid()]
    if buf.used
        # if a previous call failed to be followed by freetmpvec,
        # or maybe finalizers triggered by GC call tmpvec while the
        # resource is already in use
        _vec(n)
    else
        v = vec!(buf.vec, n)
        buf.used = true # after possible allocation above, which might fail
        v
    end
end

function freetmpvec(v::Vector)
    buf = tBuffer[Threads.threadid()]
    if v === buf.vec
        @assert buf.used
        buf.used = false
    end
    nothing
end

function __init__()
    copy!(tWraps, [Wraps() for _=1:Threads.nthreads()])
    copy!(tBuffer, [Buffer() for _=1:Threads.nthreads()])
end

macro bigint(args...)
    @assert length(args) >= 2
    out = args[1]
    xs = esc.(args[2:end-1])
    body = args[end]

    lets = :(let
                 $(esc(body))
             end)
    vars = lets.args[1].args

    @gensym mpzs
    push!(vars, :($(esc(mpzs)) = tWraps[Threads.threadid()]))
    if !(out isa Expr)
        out = Expr(:tuple, out)
    end
    for (i, outi) in enumerate(out.args)
        push!(vars, :($(esc(outi)) = getfield($(esc(mpzs)), $i).b))
    end
    for (i, xi) in enumerate(xs)
        push!(vars, :($xi = init!(getfield($(esc(mpzs)), $i+3), $xi)))
    end
    Expr(:gc_preserve, lets, xs...)
end

function BigInt(x::XInt)
    if is_short(x)
        BigInt(x.x)
    else
        xl, xv = lenvec(x)
        nbits = xl*8*sizeof(Limb)
        b = VERSION >= v"1.3" ? BigInt(; nbits=nbits) :
                                MPZ.realloc2(nbits)
        @preserve b xv unsafe_copyto!(b.d, pointer(xv), xl)
        b.size = x.x
        b
    end
end

Base.show(io::IO, x::XInt) = print(io, string(x))

function string(n::XInt; base::Integer = 10, pad::Integer = 1)
    is_short(n) && return string(n.x; base=base, pad=pad)
    base < 0 && return Base._base(Int(base), n, pad, (base>0) & isneg(n))
    2 <= base <= 62 || throw(ArgumentError("base must be 2 ≤ base ≤ 62, got $base"))
    nd1 = ndigits(n, base=base)
    nd  = max(nd1, pad)
    sv  = Base.StringVector(nd + isneg(n))
    xl, xv = lenvec(n)
    if !ispow2(base)
        # MPN.get_str might clobber the input
        xv = copy(xv)
    end
    str_size = @preserve sv Int(MPN.get_str(pointer(sv) + nd - nd1 + isneg(n), base, xv, xl))
    @assert str_size == nd1
    digits = base <= 36 ? Base.base36digits : Base.base62digits
    @inbounds for i = (1:nd-nd1) .+ isneg(n)
        sv[i] = '0' % UInt8
    end
    @inbounds for j = nd-nd1+isneg(n)+1:lastindex(sv)
        sv[j] = digits[1+sv[j]]
    end
    isneg(n) && (sv[1] = '-' % UInt8)
    String(sv)
end

widen(::Type{XInt}) = XInt

hastypemax(::Type{XInt}) = false

promote_rule(::Type{XInt}, ::Type{<:Integer}) = XInt
promote_rule(::Type{XInt}, ::Type{BigInt}) = XInt
promote_rule(::Type{BigInt}, ::Type{XInt}) = XInt
promote_rule(::Type{XInt}, ::Type{<:AbstractFloat}) = BigFloat

tryparse_internal(::Type{XInt}, s::AbstractString, startpos::Int, endpos::Int,
                  base_::Integer, raise::Bool) =
                      XInt(tryparse_internal(BigInt, s, startpos, endpos, base_, raise))

unsafe_trunc(::Type{XInt}, x::Union{Float32,Float64}) = XInt(unsafe_trunc(BigInt, x))

function trunc(::Type{XInt}, x::Union{Float32,Float64})
    isfinite(x) || throw(InexactError(:trunc, XInt, x))
    unsafe_trunc(XInt, x)
end

rem(x::XInt, ::Type{Bool}) =
    is_short(x) ? x.x % Bool :
                  (@inbounds x.v[1]) % Bool

rem(x::XInt, ::Type{T}) where T<:Union{SLimbMax,ULimbMax} =
    is_short(x) ? x.x % T : flipsign((@inbounds x.v[1] % T), x.x)

# TODO: extend to "AbstractBitInteger", when available in Base
function rem(x::XInt, ::Type{T}) where T<:Union{Base.BitUnsigned,Base.BitSigned}
    is_short(x) && return x.x % T
    u = zero(T)
    # TODO: this is missing an optimization that BigInt has; it would be faster here
    # to have a chain of if/elseif based on the value of the min below
    for l = 1:min(abs(x.x), cld(sizeof(T), sizeof(Limb)))
        u += (@inbounds x.v[l] % T) << ((sizeof(Limb)<<3)*(l-1))
    end
    flipsign(u, x.x)
end

rem(x::Integer, ::Type{XInt}) = XInt(x)

isodd(x::XInt) = is_short(x) ? isodd(x.x) : isodd(@inbounds x.v[1])
iseven(x::XInt) = !isodd(x)

function (::Type{T})(x::XInt) where T<:Base.BitUnsigned
    if is_short(x)
        convert(T, x.x)
    elseif sizeof(T) < sizeof(Limb)
        convert(T, convert(Limb, x))
    else
        0 <= x.x <= cld(sizeof(T), sizeof(Limb)) || throw(InexactError(nameof(T), T, x))
        x % T
    end
end

ispos(x::XInt) = x.x > 0
ispos(x::XInt, y::XInt) = x.x > 0 && y.x > 0
isneg(x::XInt) = x.x < 0
signbit(x::XInt) = x.x < 0
sign(x::XInt) = _XInt(sign(x.x))
_sign(x::XInt) = sign(x.x)
iszero(x::XInt) = iszero(x.x)
isone(x::XInt) = is_short(x) ? isone(x.x) : isone(@inbounds x.v[1])
# TODO: remove is_short check when we guarantee that short integers are always stored in x.x

function (::Type{T})(x::XInt) where T<:Base.BitSigned
    is_short(x) && return T(x.x)
    n = abs(x.x)
    if sizeof(T) < sizeof(Limb)
        # @assert SLimb == typeof(Signed(one(Limb)))
        convert(T, convert(SLimb, x))
    else
        0 <= n <= cld(sizeof(T), sizeof(Limb)) || throw(InexactError(nameof(T), T, x))
        y = x % T
        ispos(x) ⊻ (y > 0) && throw(InexactError(nameof(T), T, x)) # catch overflow
        y
    end
end

function Float64(x::XInt, ::RoundingMode{:ToZero})
    if is_short(x)
        Float64(x.x, RoundToZero)
    else
        MPN.get_d(x, len(x), x.x)
    end
end

function (::Type{T})(x::XInt, ::RoundingMode{:ToZero}) where T<:Union{Float16,Float32}
    T(Float64(x, RoundToZero), RoundToZero)
end

function (::Type{T})(n::XInt, ::RoundingMode{:Down}) where T<:CdoubleMax
    x = T(n, RoundToZero)
    x > n ? prevfloat(x) : x
end

function (::Type{T})(n::XInt, ::RoundingMode{:Up}) where T<:CdoubleMax
    x = T(n, RoundToZero)
    x < n ? nextfloat(x) : x
end

for T = [Float16, Float32, Float64]
    @eval (::Type{$T})(x::XInt, r::RoundingMode{:Nearest}=RoundNearest) =
        is_short(x) ? $T(x.x, r) : @bigint () x $T(x, r)
end

AbstractFloat(x::XInt) = BigFloat(x)

BigFloat(x::XInt, r::MPFR.MPFRRoundingMode=MPFR.ROUNDING_MODE[];
         precision::Integer=MPFR.DEFAULT_PRECISION[]) =
             is_short(x) ? BigFloat(x.x, r; precision=precision) :
                           @bigint () x BigFloat(x, r; precision=precision)

float(::Type{XInt}) = BigFloat

# Binary ops
for (fJ, fC) in ((:mod, :fdiv_r), (:rem, :tdiv_r),
                 (:gcd, :gcd), (:lcm, :lcm))
    fC! = Symbol(fC, :!)
    @eval begin
        ($fJ)(x::XInt, y::XInt) =
            is_short(x, y) ? XInt(($fJ)(widen(x.x), widen(y.x))) :
                             @bigint z x y XInt(MPZ.$fC!(z, x, y))
    end
end

# TODO: 3+ args specializations for some ops, like in gmp.jl
# (already done for +)

# we put @inline, as it seems these methods don't by themselves,
# maybe because add! is already inline and too big
@inline +(x::XInt, y::XInt)       = add!(nothing, x, y)
@inline +(x::XInt, y::BitInteger) = add!(nothing, x, y)
@inline +(x::XInt, y::Integer)    = add!(_XInt(y, 1), x)
@inline +(x::Integer, y::XInt)    = y + x

@inline -(x::XInt, y::XInt)       = sub!(nothing, x, y)
@inline -(x::BitInteger, y::XInt) = sub!(nothing, x, y)
@inline -(x::Integer, y::XInt)    = sub!(_XInt(x, 1), y)
@inline -(x::XInt, y::Integer)    = neg!(y-x)

@inline *(x::XInt, y::XInt)     = mul!(nothing, x, y)
@inline *(x::XInt, c::ShortMax) = mul!(nothing, x, c)
@inline *(x::XInt, y::Integer)  = mul!(_XInt(y, safe_len(x)), x)
@inline *(x::Integer, y::XInt)  = y * x

@inline (&)(x::XInt, y::XInt)       = and!(nothing, x, y)
@inline (&)(x::XInt, y::BitInteger) = and!(nothing, x, y)
@inline (&)(x::XInt, y::Integer)    = and!(XInt(y), x)
@inline (&)(x::Integer, y::XInt)    = y & x

@inline (|)(x::XInt, y::XInt)       = ior!(nothing, x, y)
@inline (|)(x::XInt, y::BitInteger) = ior!(nothing, x, y)
@inline (|)(x::XInt, y::Integer)    = ior!(XInt(y), x)
@inline (|)(x::Integer, y::XInt)    = y | x

@inline xor(x::XInt, y::XInt)       = xor!(nothing, x, y)
@inline xor(x::XInt, y::BitInteger) = xor!(nothing, x, y)
@inline xor(x::XInt, y::Integer)    = xor!(XInt(y), x)
@inline xor(x::Integer, y::XInt)    = y ⊻ x

# it doesn't seem worthwhile to define 4+ args specific + methods,
# the vararg one is better
@inline +(x::XInt, y::XInt, z::XInt) = add!((x+y), z)
+(x::XInt, y::XInt, z::XInt, r::XInt...) = sum(tuple(x, y, z, r...))

function sum(arr::Union{AbstractArray{XInt},Tuple{Vararg{XInt}}})
    sp = _XInt(0) # > 0
    sn = _XInt(0) # < 0
    s0 = zero(Int128)

    for x = arr
        if is_short(x)
            s0 += short(x)
        elseif ispos(x)
            sp = addbig_positive!(sp, x)
        else
            sn = addbig_positive!(sn, x)
        end
    end
    if !iszero(sp)
        if !iszero(sn)
            s = addbig!(sp, sp, neg!(sn))
            add!(s, s0)
        else
            add!(sp, s0)
        end
    elseif !iszero(sn)
        add!(neg!(sn), s0)
    else
        XInt(s0)
    end
end

for (r, f!) in ((RoundToZero, :tdiv_q!),
               (RoundDown, :fdiv_q!),
               (RoundUp, :cdiv_q!))
    @eval div(x::XInt, y::XInt, ::typeof($r)) =
        is_short(x, y) ? XInt(div(widen(x.x), widen(y.x), $r)) :
                         @bigint z x y XInt(MPZ.$f!(z, x, y))
end

# For compat only. Remove in 2.0.
div(x::XInt, y::XInt) = div(x, y, RoundToZero)
fld(x::XInt, y::XInt) = div(x, y, RoundDown)
cld(x::XInt, y::XInt) = div(x, y, RoundUp)

divrem(x::XInt, y::XInt) =
    is_short(x, y) ? XInt.(divrem(x.x, y.x)) :
                     @bigint (u, v) x y XInt.(MPZ.tdiv_qr!(u, v, x, y))

/(x::XInt, y::XInt) = float(x)/float(y)

invmod(x::XInt, y::XInt) =
    is_short(x, y) ? XInt(invmod(widen(x.x), widen(y.x))) :
                     @bigint () x y XInt(invmod(x, y))

/(x::XInt, y::Union{ClongMax,CulongMax}) = float(x)/y
/(x::Union{ClongMax,CulongMax}, y::XInt) = x/float(y)

# unary ops
(-)(x::XInt) = neg!(x)
(~)(x::XInt) = com!(nothing, x)

<<(x::XInt, c::UInt) = lshift!(nothing, x, c)
>>(x::XInt, c::UInt) = rshift!(nothing, x, c)
>>>(x::XInt, c::UInt) = x >> c

trailing_zeros(x::XInt) =
    if is_short(x)
        iszero(x.x) && throw(DomainError(x)) # InexactError for BigInt
        trailing_zeros(x.x)
    else
        @bigint () x MPZ.scan1(x, 0)
    end

trailing_ones(x::XInt) =
    if is_short(x)
        x.x == -1 && throw(DomainError(x))
        trailing_ones(x.x)
    else
        @bigint () x MPZ.scan0(x, 0)
    end

count_ones(x::XInt) =
    if is_short(x)
        x.x < 0 && throw(DomainError(x))
        count_ones(x.x)
    else
        @bigint () x MPZ.popcount(x)
    end

cmp(x::XInt, y::XInt) = is_short(x, y) ? cmp(x.x, y.x) : @bigint () x y cmp(x, y)
cmp(x::XInt, y::Integer) = is_short(x) ? cmp(x.x, y) : @bigint () x cmp(x, y)
cmp(x::XInt, y::BigInt)  = is_short(x) ? cmp(x.x, y) : @bigint () x cmp(x, y) # disambiguation
cmp(y::Integer, x::XInt) = -cmp(x, y)
cmp(y::BigInt, x::XInt)  = -cmp(x, y) # disambiguation
cmp(x::XInt, y::CdoubleMax) = is_short(x) ? cmp(x.x, y) : @bigint () x cmp(x, y)
cmp(y::CdoubleMax, x::XInt) = -cmp(x, y)

function ==(x::XInt, y::XInt)
    if is_short(x, y)
        x.x == y.x
    else
        @bigint () x y x == y
    end
end

==(x::XInt, y::Integer) =
    if is_short(x)
        x.x == y
    else
        @bigint () x x == y
    end

==(x::Integer, y::XInt) = y == x

# disambiguation
==(x::XInt, y::BigInt) = invoke(==, Tuple{XInt, Integer}, x, y)
==(x::BigInt, y::XInt) = y == x

==(x::XInt, f::CdoubleMax) = isnan(f) ? false : cmp(x, f) == 0
==(f::CdoubleMax, x::XInt) = isnan(f) ? false : cmp(x, f) == 0

<=(x::XInt, y::XInt) = cmp(x,y) <= 0
<=(x::XInt, i::Integer) = cmp(x,i) <= 0
<=(x::XInt, i::BigInt)  = cmp(x,i) <= 0 # disambiguation
<=(i::Integer, x::XInt) = cmp(x,i) >= 0
<=(i::BigInt, x::XInt)  = cmp(x,i) >= 0 # disambiguation
<=(x::XInt, f::CdoubleMax) = isnan(f) ? false : cmp(x,f) <= 0
<=(f::CdoubleMax, x::XInt) = isnan(f) ? false : cmp(x,f) >= 0

<(x::XInt, y::XInt) = cmp(x,y) < 0
<(x::XInt, i::Integer) = cmp(x,i) < 0
<(x::XInt, i::BigInt)  = cmp(x,i) < 0 # disambiguation
<(i::Integer, x::XInt) = cmp(x,i) > 0
<(i::BigInt, x::XInt)  = cmp(x,i) > 0 # disambiguation

<(x::XInt, f::CdoubleMax) = isnan(f) ? false : cmp(x,f) < 0
<(f::CdoubleMax, x::XInt) = isnan(f) ? false : cmp(x,f) > 0

isqrt(x::XInt) = is_short(x) ? XInt(isqrt(x.x)) : @bigint z x XInt(MPZ.sqrt!(z, x))

^(x::XInt, y::Culong) = @bigint z x XInt(MPZ.pow_ui!(z, x, y))

function xint_pow(x::XInt, y::Integer)
    if y<0; throw(DomainError(y, "`y` cannot be negative.")); end
    @noinline throw1(y) =
        throw(OverflowError("exponent $y is too large and computation will overflow"))
    if x== 1; return x; end
    if x==-1; return isodd(y) ? x : -x; end
    if y>typemax(Culong)
       x==0 && return x

       #At this point, x is not 1, 0 or -1 and it is not possible to use
       #gmpz_pow_ui to compute the answer. Note that the magnitude of the
       #answer is:
       #- at least 2^(2^32-1) ≈ 10^(1.3e9) (if Culong === UInt32).
       #- at least 2^(2^64-1) ≈ 10^(5.5e18) (if Culong === UInt64).
       #
       #Assume that the answer will definitely overflow.

       throw1(y)
    end
    return x^convert(Culong, y)
end

^(x::XInt   , y::XInt   ) = xint_pow(x, y)
^(x::XInt   , y::BigInt ) = xint_pow(x, y)
^(x::BigInt , y::XInt   ) = Base.GMP.bigint_pow(x, y)
^(x::XInt   , y::Bool   ) = y ? x : one(x)
^(x::XInt   , y::Integer) = xint_pow(x, y)
^(x::Integer, y::XInt   ) = xint_pow(XInt(x), y)
^(x::Bool   , y::XInt   ) = Base.power_by_squaring(x, y)

powermod(x::XInt, p::XInt, m::XInt) =
    is_short(x, p) && is_short(m) ? XInt(powermod(x.x, p.x, m.x)) :
                                    @bigint () x p m XInt(powermod(x, p, m))

powermod(x::Integer, p::Integer, m::XInt) = powermod(XInt(x), XInt(p), m)

function gcdx(a::XInt, b::XInt)
    is_short(a, b) &&
        return XInt.(gcdx(widen(a.x), widen(b.x)))
        # widen above necessary at least for (a, b) == (typemin(Int), 0)
    if iszero(b) # shortcut this to ensure consistent results with gcdx(a,b)
        return a < 0 ? (-a,-one(XInt),b) : (a,one(XInt),b)
    end
    g, s, t = XInt.(@bigint (x, y, z) a b MPZ.gcdext!(x, y, z, a, b))
    if t == 0
        # work around a difference in some versions of GMP
        if a == b
            return g, t, s
        elseif abs(a)==abs(b)
            return g, t, -s
        end
    end
    g, s, t
end

flipsign(x::XInt, y::Integer) = signbit(y) ? -x : x
flipsign(x::XInt, y::XInt)  = signbit(y) ? -x : x

factorial(x::XInt) = signbit(x) ? throw(DomainError(x)) :
                                  @bigint z x XInt(MPZ.fac_ui!(z, x))

binomial(x::XInt, k::UInt) = @bigint z x XInt(MPZ.bin_ui!(z, x, k))
binomial(x::XInt, k::Integer) = k < 0 ? XInt(0) : binomial(x, UInt(k))

function digits!(a::AbstractVector{T}, n::XInt; base::Integer = 10) where {T<:Integer}
    if 2 ≤ base ≤ 62
        s = codeunits(string(n; base=base))
        i, j = firstindex(a)-1, length(s)+1
        lasti = min(lastindex(a), firstindex(a) + length(s)-1 - isneg(n))
        while i < lasti
            # base ≤ 36: 0-9, plus a-z for 10-35
            # base > 36: 0-9, plus A-Z for 10-35 and a-z for 36..61
            x = s[j -= 1]
            a[i += 1] = base ≤ 36 ? (x>0x39 ? x-0x57 : x-0x30) : (x>0x39 ? (x>0x60 ? x-0x3d : x-0x37) : x-0x30)
        end
        lasti = lastindex(a)
        while i < lasti
            a[i+=1] = zero(T)
        end
        isneg(n) ? map!(-,a,a) : a
    else
        invoke(digits!, Tuple{typeof(a), Integer}, a, n; base=base) # slow generic fallback
    end
end

ndigits0zpb(x::XInt, b::Integer) = is_short(x) ? ndigits0zpb(x.x, b) :
                                                 @bigint () x ndigits0zpb(x, b)

function _prevpow2(x::XInt)
    if is_short(x)
        XInt(_prevpow2(x.x))
    else
        len = abs(x.x)
        @assert length(x.v) >= len
        high = @inbounds x.v[len]
        @assert !iszero(high) # like for BigInt/mpz_t
        shift = BITS_PER_LIMB - leading_zeros(high) - 1
        v = fill(zero(Limb), len)
        @inbounds v[len] = one(Limb) << shift
        _XInt(x.x, v)
    end
end

function _nextpow2(x::XInt)
    if is_short(x)
        y = _nextpow2(x.x)
        if sign(x) != sign(y)
            @assert y == typemin(Int)
            _XInt(1, Limb[typemin(Int) % Limb])
        else
            XInt(y)
        end
    else
        len = abs(x.x)
        @assert length(x.v) >= len
        popcount = GC.@preserve x MPZ.mpn_popcount(pointer(x.v), len)
        if popcount <= 1
            @assert popcount == 1
            return x
        end
        high = @inbounds x.v[len]
        @assert !iszero(high) # like for BigInt/mpz_t
        shift = (BITS_PER_LIMB - leading_zeros(high)) & (BITS_PER_LIMB-1)
        newlen = len + iszero(shift)
        v = fill(zero(Limb), newlen)
        @inbounds v[newlen] = one(Limb) << shift
        _XInt(flipsign(newlen, x.x), v)
    end
end

Base.checked_abs(x::XInt) = abs(x)
Base.checked_neg(x::XInt) = -x
Base.checked_add(a::XInt, b::XInt) = a + b
Base.checked_sub(a::XInt, b::XInt) = a - b
Base.checked_mul(a::XInt, b::XInt) = a * b
Base.checked_div(a::XInt, b::XInt) = div(a, b)
Base.checked_rem(a::XInt, b::XInt) = rem(a, b)
Base.checked_fld(a::XInt, b::XInt) = fld(a, b)
Base.checked_mod(a::XInt, b::XInt) = mod(a, b)
Base.checked_cld(a::XInt, b::XInt) = cld(a, b)
Base.add_with_overflow(a::XInt, b::XInt) = a + b, false
Base.sub_with_overflow(a::XInt, b::XInt) = a - b, false
Base.mul_with_overflow(a::XInt, b::XInt) = a * b, false

_copy(x::XInt) = _XInt(x.x, _copy(x.v))
_copy(v::Vector) = copy(v)
_copy(::Nothing) = nothing

Base.deepcopy_internal(x::XInt, d::IdDict) = get!(() -> _copy(x), d, x)

if Limb === UInt
    using Base: hash_uint

    function hash_integer(x::XInt, h::UInt)
        is_short(x) && return hash_integer(x.x, h)
        sz = x.x
        xv = x.v
        b = @inbounds xv[1]
        h ⊻= hash_uint(ifelse(sz < 0, -b, b) ⊻ h)
        for k = 2:abs(sz)
            h ⊻= hash_uint(@inbounds xv[k] ⊻ h)
        end
        h
    end

    function hash(x::XInt, h::UInt)
        GC.@preserve x begin
            is_short(x) && return hash(x.x, h)
            sz = x.x
            xv = x.v
            if sz == 1
                return hash(@inbounds(xv[1]), h)
            elseif sz == -1
                limb = @inbounds xv[1]
                limb <= typemin(Int) % UInt && return hash(-(limb % Int), h)
            end
            pow = trailing_zeros(x)
            nd = Base.ndigits0z(x, 2)
            idx = _divLimb(pow) + 1
            shift = _modLimb(pow) % UInt
            upshift = BITS_PER_LIMB - shift
            asz = abs(sz)
            if shift == 0
                limb = @inbounds xv[idx]
            else
                limb1 = @inbounds xv[idx]
                limb2 = idx < asz ? @inbounds(xv[idx+1]) : UInt(0)
                limb = limb2 << upshift | limb1 >> shift
            end
            if nd <= 1024 && nd - pow <= 53
                # TODO: add test for this branch
                return hash(ldexp(flipsign(Float64(limb), sz), pow), h)
            end
            h = hash_integer(1, h)
            h = hash_integer(pow, h)
            h ⊻= hash_uint(flipsign(limb, sz) ⊻ h)
            for idx = idx+1:asz
                if shift == 0
                    limb = @inbounds xv[idx]
                else
                    limb1 = limb2
                    if idx == asz
                        limb = limb1 >> shift
                        limb == 0 && break # don't hash leading zeros
                    else
                        limb2 = @inbounds xv[idx+1]
                        limb = limb2 << upshift | limb1 >> shift
                    end
                end
                h ⊻= hash_uint(limb ⊻ h)
            end
            return h
        end
    end
end

## Random

import Random: Sampler, rand, rand!
using Random: Random, Repetition, AbstractRNG

abstract type SamplerXInt <: Sampler{XInt} end

struct SamplerXIntBig{SP<:Sampler{Limb}} <: SamplerXInt
    a::XInt           # first
    m::XInt           # range length - 1 (generate z ∈ [0, m])
    nlimbsmax::SLimb  # max number of limbs for z+a
    highsp::SP        # sampler for the highest limb
end

safe_len(x::XInt) = is_short(x) ? SLimb(!iszero(x)) : abs(x.x)

@noinline function SamplerXIntBig(::Type{RNG}, r1::XInt, r2::XInt) where {RNG<:AbstractRNG}
    m = r2-r1
    isneg(m) && throw(ArgumentError("range must be non-empty"))
    if is_short(m)
        mvl = m.x % Limb
        nlimbsmax = 0
    else
        nlimbs, mv = lenvec(m)
        if nlimbs == 1
            mvl = @inbounds mv[1]
            nlimbsmax = 0
        else
            nlimbsmax = max(nlimbs, safe_len(r2), safe_len(r1))
            nlimbsmax += ispos(r1) # add! will request one more limb just in case
            mvl = @inbounds mv[nlimbs]
        end
    end
    highsp = Sampler(RNG, zero(Limb):mvl)
    SamplerXIntBig(r1, m, nlimbsmax, highsp)
end

struct SamplerXIntShort{SP<:Sampler{SLimb}} <: SamplerXInt
    sp::SP
end

@inline function Sampler(::Type{RNG}, r::AbstractUnitRange{XInt}, N::Repetition
                 ) where {RNG<:AbstractRNG}
    r1, r2 = first(r), last(r)
    if is_short(r1, r2)
        SamplerXIntShort(Sampler(RNG, r1.x:r2.x, N))
    else
        SamplerXIntBig(RNG, r1, r2)
    end
end

rand(rng::AbstractRNG, sp::SamplerXInt) = rand!(rng, _XInt(0), sp)

function rand!(rng::AbstractRNG, x::XInt, sp::SamplerXIntBig)
    iszero(sp.nlimbsmax) &&
        return add!(x, sp.a, _XInt(rand(rng, sp.highsp)))
    nlimbs, mv = lenvec(sp.m)
    limbs = vec!(x, sp.nlimbsmax)
    mvl = @inbounds mv[nlimbs]
    while true
        @inbounds rand!(rng, @view(limbs[1:nlimbs-1]))
        @inbounds h = limbs[nlimbs] = rand(rng, sp.highsp)
        h < mvl && break
        MPN.cmp(limbs, sp.m, nlimbs) <= 0 && break
    end
    len = normalize(limbs, nlimbs)
    add!(_XInt(len, limbs, false), _XInt(len, limbs, true), sp.a)
end

rand!(rng::AbstractRNG, ::XInt, sp::SamplerXIntShort) = _XInt(rand(rng, sp.sp))

end # module


module DownloadArgo

using Printf, Dates, YAML, NetCDF, NCDatasets, CSV, DataFrames, Dierckx

"""
    GDAC_FTP(b::String,y::Int,m::Int)

Download Argo data files for one regional domain (b), year (y), and
month (m) from the `GDAC` FTP server (`ftp://ftp.ifremer.fr/ifremer/argo`
or, equivalently, `ftp://usgodae.org/pub/outgoing/argo`).

```
b="atlantic"; yy=2009:2009; mm=8:12;
for y=yy, m=mm;
    println("\$b/\$y/\$m"); DownloadArgo.GDAC_FTP(b,y,m)
end
```
"""
function GDAC_FTP(b::String,y::Int,m::Int)
    yy = @sprintf "%04d" y
    mm = @sprintf "%02d" m
    c=`wget --quiet -r ftp://ftp.ifremer.fr/ifremer/argo/geo/"$b"_ocean/$yy/$mm`
    run(c)
end

"""
    mitprof_interp_setup(fil::String)

Get parameters etc from yaml file (`fil`).
"""
function mitprof_interp_setup(fil::String)

    meta=YAML.load(open(fil))

    #1. file list

    d=meta["dirIn"]
    b=meta["subset"]["basin"]
    y=meta["subset"]["year"]

    list0=Array{Array,1}(undef,12)
    for m=1:12
        sd="$b"*"_ocean/$y/"*Printf.@sprintf("%02d/",m)
        tmp=readdir(d*sd)
        list0[m]=[sd*tmp[i] for i=1:length(tmp)]
    end

    nf=sum(length.(list0))
    list1=Array{String,1}(undef,nf)
    f=0
    for m=1:12
        for ff=1:length(list0[m])
            f+=1
            list1[f]=list0[m][ff]
        end
    end

    meta["fileInList"]=list1;

    #2. coordinate

    z_std=meta["depthLevels"]
    if length(z_std)>1
        tmp1=(z_std[2:end]+z_std[1:end-1])/2
        z_top=[z_std[1]-(z_std[2]-z_std[1])/2;tmp1]
        z_bot=[tmp1;z_std[end]+(z_std[end]-z_std[end-1])/2]
    else
        z_top=0.9*z_std
        z_bot=1.1*z_std
    end

    meta["z_std"]=z_std
    meta["z_top"]=z_top
    meta["z_bot"]=z_bot

    #3. various other specs

    meta["inclZ"] = false
    meta["inclT"] = true
    meta["inclS"] = true
    meta["inclU"] = false
    meta["inclV"] = false
    meta["inclPTR"] = false
    meta["inclSSH"] = false
    meta["TPOTfromTINSITU"] = true

    meta["doInterp"] = true
    meta["addGrid"] = true
    meta["outputMore"] = false
    meta["method"] = "interp"
    meta["fillval"] = -9999.0
    meta["buffer_size"] = 10000

    meta["fileOut"]=meta["name"]*"_"*meta["subset"]["basin"]*"_"*string(meta["subset"]["year"])*".nc"
    meta["var_out"]=meta["variables"]

    return meta
end

"""
    GetOneProfile(m)

Get one profile from netcdf file.
"""
function GetOneProfile(ds,m)

    #
    t=ds["JULD"][m]
    ymd=Dates.year(t)*1e4+Dates.month(t)*1e2+Dates.day(t)
    hms=Dates.hour(t)*1e4+Dates.minute(t)*1e2+Dates.second(t)

    lat=ds["LATITUDE"][m]
    lon=ds["LONGITUDE"][m]
    lon < 0.0 ? lon=lon+360.0 : nothing

    direction=ds["DIRECTION"][m]
    direc=0
    direction=='A' ? direc=1 : nothing
    direction=='D' ? direc=2 : nothing

    #
    pnum_txt=ds["PLATFORM_NUMBER"][:,m]
    #pnum_txt=pnum_txt[findall((!ismissing).(pnum_txt))]
    ii=findall(skipmissing(in.(pnum_txt,"0123456789")))
    ~isempty(ii) ? pnum_txt=String(vec(Char.(pnum_txt[ii]))) : pnum_txt="9999"
    pnum=parse(Int,pnum_txt)

    #
    p=ds["PRES_ADJUSTED"][:,m]
    p_QC=ds["PRES_ADJUSTED_QC"][:,m]
    if sum((!ismissing).(p))==0
        p=ds["PRES"][:,m]
        p_QC=ds["PRES_QC"][:,m]
    end

    #set qc to 5 if missing
    p_QC[findall(ismissing.(p))].='5'
    #avoid potential duplicates
    for n=1:length(p)-1
        if ~ismissing(p[n])
            tmp1=( (!ismissing).(p[n+1:end]) ).&( p[n+1:end].==p[n] )
            tmp1=findall(tmp1)
            p[n.+tmp1].=missing
            p_QC[n.+tmp1].='5'
        end
    end

    #position and date
    isBAD=0
    ~in(ds["POSITION_QC"][m],"1258") ? isBAD=1 : nothing
    ~in(ds["JULD_QC"][m],"1258") ? isBAD=1 : nothing

    #pressure
    tmp1=findall( (!in).(p_QC,"1258") )
    if (length(tmp1)<=5)&&(length(tmp1)>0)
        #omit these few bad points but keep the profile
        p[tmp1].=missing
    elseif length(tmp1)>5
        #flag the profile (will be omitted later)
        #but keep the bad points (for potential inspection)
        isBAD=1
    end

    #temperature
    t=ds["TEMP_ADJUSTED"][:,m]
    t_QC=ds["TEMP_ADJUSTED_QC"][:,m]
    t_ERR=ds["TEMP_ADJUSTED_ERROR"][:,m]
    t_ERR[findall( (ismissing).(t_ERR) )].=0.0

    if sum((!ismissing).(t))==0
        t=ds["TEMP"][:,m]
        t_QC=ds["TEMP_QC"][:,m]
    end

    #salinity
    if haskey(ds,"PSAL")
        s=ds["PSAL_ADJUSTED"][:,m]
        s_QC=ds["PSAL_ADJUSTED_QC"][:,m]
        s_ERR=ds["PSAL_ADJUSTED_ERROR"][:,m]
        s_ERR[findall( (ismissing).(t_ERR) )].=0.0
        if sum((!ismissing).(t))==0
            s=ds["PSAL"][:,m]
            s_QC=ds["PSAL_QC"][:,m]
        end
    else
        s=fill(missing,size(t))
        s_QC=Char.(32*ones(size(t_QC)))
    end

    if ismissing(t[1]) #this file does not contain temperature data...
        t=fill(missing,size(t))
        t_ERR=fill(0.0,size(t))
    else #apply QC
        tmp1=findall(skipmissing((!in).(t_QC,"1258")))
        t[tmp1].=missing
    end

    if ismissing(s[1]) #this file does not contain salinity data...
        s=fill(missing,size(s))
        s_ERR=fill(0.0,size(s))
    else #apply QC
        tmp1=findall(skipmissing((!in).(s_QC,"1258")))
        s[tmp1].=missing
    end

    prof=Dict()
    prof["pnum_txt"]=pnum_txt
    prof["ymd"]=convert(Union{Int,Missing},ymd)
    prof["hms"]=convert(Union{Int,Missing},hms)
    prof["lat"]=convert(Union{Float64,Missing},lat)
    prof["lon"]=convert(Union{Float64,Missing},lon)
    prof["direc"]=convert(Union{Int,Missing},direc)
    prof["T"]=convert(Array{Union{Float64,Missing}},t)
    prof["S"]=convert(Array{Union{Float64,Missing}},s)
    prof["p"]=convert(Array{Union{Float64,Missing}},p)
    prof["T_ERR"]=convert(Array{Union{Float64,Missing}},t_ERR)
    prof["S_ERR"]=convert(Array{Union{Float64,Missing}},s_ERR)
    prof["isBAD"]=isBAD
    prof["DATA_MODE"]=ds["DATA_MODE"][m]

    return prof
end

"""
    prof_PtoZ!(prof,meta)

Convert prof["p"] to prof["depth"]
"""
function prof_PtoZ!(prof,meta)
    l=prof["lat"]
    v=meta["var_out"][1]
    prof[v]=similar(prof["p"],Union{Missing,Float64})
    prof[v].=missing
    k=findall((!ismissing).(prof["p"]))
    prof[v][k]=[DownloadArgo.sw_dpth(Float64(prof["p"][kk]),Float64(l)) for kk in k]
end

"""
    prof_TtoΘ!(prof,meta)

Convert prof["T"] to potential temperature
"""
function prof_TtoΘ!(prof,meta)
    T=prof[meta["var_out"][2]]
    P=0.981*1.027*prof[meta["var_out"][1]]
    S=35.0*ones(size(T))
    k=findall( (!ismissing).(T) )
    T[k]=[DownloadArgo.sw_ptmp(Float64(S[kk]),Float64(T[kk]),Float64(P[kk])) for kk in k]
end

"""
    prof_interp!(prof,meta)

Interpolate from prof["depth"] to meta["z_std"]
"""
function prof_interp!(prof,meta)
    for ii=2:length(meta["var_out"])
        v=meta["var_out"][ii]
        v_e=v*"_ERR"

        z_std=meta["z_std"]
        t_std=similar(z_std,Union{Missing,Float64})
        e_std=similar(z_std,Union{Missing,Float64})

        z=prof["depth"]
        t=prof[v]
        do_e=haskey(prof,v_e)
        do_e ? e=prof[v_e] : e=[]

        kk=findall((!ismissing).(z.*t))
        if (meta["doInterp"])&&(length(kk)>1)
            z_in=z[kk]; t_in=t[kk]
            do_e ? e_in=e[kk] : nothing
            k=sort(1:length(kk),by= i -> z_in[i])
            z_in=z_in[k]; t_in=t_in[k]
            do_e ? e_in=e_in[k] : nothing
            #omit values outside observed range:
            D0=minimum(skipmissing(z_in))
            D1=maximum(skipmissing(z_in))
            msk1=findall( (z_std.<D0).|(z_std.>D1) )
            #avoid duplicates:
            msk2=findall( ([false;(z_in[1:end-1]-z_in[2:end]).==0.0]).==true )
            if length(kk)>5
                spl = Spline1D(z_in, t_in)
                t_std[:] = spl(z_std)
                t_std[msk1].=missing
                t_std[msk2].=missing
                if do_e
                    spl = Spline1D(z_in, e_in)
                    e_std[:] = spl(z_std)
                    e_std[msk1].=missing
                    e_std[msk2].=missing
                end
            else
                t_std = []
                e_std = []
            end
            prof[v]=t_std
        end
    end
end

"""
    sw_dpth(P,LAT)

Calculate depth in meters from pressure (P; in decibars) and
latitude (LAT; in °N)

```
d = sw_dpth(100.0,20.0)
```
"""
function sw_dpth(P,LAT)
    # Original author:  Phil Morgan 92-04-06  (morgan@ml.csiro.au)
    # Reference: Unesco 1983. Algorithms for computation of fundamental properties of
    # seawater, 1983. _Unesco Tech. Pap. in Mar. Sci._, No. 44, 53 pp. Eqn 25, p26

    c1 = 9.72659
    c2 = -2.2512E-5
    c3 = 2.279E-10
    c4 = -1.82E-15
    gam_dash = 2.184e-6

    LAT = abs.(LAT)
    X   = sin.(deg2rad.(LAT))
    X   = X.*X
    bot_line = 9.780318*(1.0+(5.2788E-3+2.36E-5*X).*X) + gam_dash*0.5*P
    top_line = (((c4*P+c3).*P+c2).*P+c1).*P
    DEPTHM   = top_line./bot_line

    return DEPTHM
end

"""
    sw_ptmp(S,T,P,PR)

Calculate potential temperature as per UNESCO 1983 report from salinity (S;
in psu), in situ temperature (T; in °C), and pressure (P; in decibar)
relative to PR (in decibar; 0 by default).

```
ptmp = sw_ptmp(S,T,P,PR=missing)
```
"""
function sw_ptmp(S,T,P,PR=0.0)
# Original author:  Phil Morgan
# Reference: Fofonoff, P. and Millard, R.C. Jr
#    Unesco 1983. Algorithms for computation of fundamental properties of
#    seawater. _Unesco Tech. Pap. in Mar. Sci._, No. 44. Eqn.(31) p.39
#    Bryden, H. 1973.
#    "New Polynomials for thermal expansion, adiabatic temperature gradient
#    and potential temperature of sea water." DEEP-SEA RES., 1973, Vol 20

# theta1
del_P  = PR - P
del_th = del_P.*sw_adtg(S,T,P);
th     = T + 0.5*del_th;
q      = del_th;

# theta2
del_th = del_P.*sw_adtg(S,th,P+0.5*del_P);
th     = th + (1 - 1/sqrt(2))*(del_th - q);
q      = (2-sqrt(2))*del_th + (-2+3/sqrt(2))*q;

# theta3
del_th = del_P.*sw_adtg(S,th,P+0.5*del_P);
th     = th + (1 + 1/sqrt(2))*(del_th - q);
q      = (2 + sqrt(2))*del_th + (-2-3/sqrt(2))*q;

# theta4
del_th = del_P.*sw_adtg(S,th,P+del_P);
PT     = th + (del_th - 2*q)/6;

return PT
end

"""
    sw_adtg(S,T,P)

Calculate adiabatic temperature gradient as per UNESCO 1983 routines from salinity
(S; in psu), in situ temperature (T; in °C), and pressure (P; in decibar)
```
adtg = sw_adtg(S,T,P)
```
"""
function sw_adtg(S,T,P)
    # Original author:  Phil Morgan
    # Reference: Fofonoff, P. and Millard, R.C. Jr
    #    Unesco 1983. Algorithms for computation of fundamental properties of
    #    seawater. _Unesco Tech. Pap. in Mar. Sci._, No. 44. Eqn.(31) p.39
    #    Bryden, H. 1973.
    #    "New Polynomials for thermal expansion, adiabatic temperature gradient
    #    and potential temperature of sea water." DEEP-SEA RES., 1973, Vol 20

    a0 =  3.5803E-5;
    a1 =  8.5258E-6;
    a2 = -6.836E-8;
    a3 =  6.6228E-10;

    b0 =  1.8932E-6;
    b1 = -4.2393E-8;

    c0 =  1.8741E-8;
    c1 = -6.7795E-10;
    c2 =  8.733E-12;
    c3 = -5.4481E-14;

    d0 = -1.1351E-10;
    d1 =  2.7759E-12;

    e0 = -4.6206E-13;
    e1 =  1.8676E-14;
    e2 = -2.1687E-16;

    ADTG =      a0 + (a1 + (a2 + a3.*T).*T).*T + (b0 + b1.*T).*(S-35.0) +
    ( (c0 + (c1 + (c2 + c3.*T).*T).*T) + (d0 + d1.*T).*(S-35.0) ).*P +
    (  e0 + (e1 + e2.*T).*T ).*P.*P

    return ADTG
end

end


#######################################################################################################################################################################################################
#
# Changes made to this function
# General
#     2021-Oct-01: rename the function to leaf_spectra! as the function updates not only fluorescence but also reflectance, transmittance, and absorption spectra
#     2021-Oct-22: add another method to prescribe leaf spectra such as transmittance and reflectance from broadband method
#
#######################################################################################################################################################################################################
"""
This function updates leaf level reflectance, transmittance, and fluorescence spectra related parameters. Supported methods are

$(METHODLIST)

"""
function leaf_spectra! end

#######################################################################################################################################################################################################
#
# Changes made to this method
# General
#     2020-Mar-30: account for carotenoid absorption as APAR as well as chlorophyll
#     2020-Mar-31: use 40° rather than 59° for _τ_α calculation (following PROSPECT-D)
#     2021-Aug-07: replace function `expint` with that from SpecialFunctions
#     2021-Oct-21: add α to input parameters so that one can roll back to 50° for _τ_α calculation
#     2021-Nov-29: separate HyperspectralAbsorption as constant struct
#     2022-Jan-13: use LeafBiophysics directly in the function rather than Leaf
#     2022-Feb-02: fix documentation
# Bug fix
#     2021-Aug-06: If bio.CBC and bio.PRO are not zero, they are accounted for twice in bio.LMA, thus the spectrum from LMA need to subtract the contribution from CBC and PRO
# To do
#     TODO: make brown pigment and absoption curve more general using realistic units
#     TODO: add References for this methods
#     TODO: speed up this function by preallocate memories using a cache structure
#
#######################################################################################################################################################################################################
"""

    leaf_spectra!(bio::LeafBiophysics{FT}, wls::WaveLengthSet{FT}, lha::HyperspectralAbsorption{FT}; APAR_car::Bool = true, α::FT=FT(40)) where {FT<:AbstractFloat}

Update leaf reflectance and transmittance spectra, and fluorescence spectrum matrices, given
- `bio` `ClimaCache.LeafBiophysics` type struct that contains leaf biophysical parameters
- `wls` `ClimaCache.WaveLengthSet` type struct that contain wave length bins
- `lha` `ClimaCache.HyperspectralAbsorption` type struct that contains absorption characteristic curves
- `APAR_car` If true, carotenoid absorption is accounted for in APAR, default is `true`
- `α` Optimum angle of incidence (default is 40° as in PROSPECT-D, SCOPE uses 59°)

# Examples
```julia
wls = WaveLengthSet{Float64}();
bio = LeafBiophysics{Float64}(wls);
lha = HyperspectralAbsorption{Float64}(wls);
leaf_spectra!(bio, wls, lha);
leaf_spectra!(bio, wls, lha; APAR_car=false);
leaf_spectra!(bio, wls, lha; APAR_car=false, α=59.0);
```
"""
leaf_spectra!(bio::LeafBiophysics{FT}, wls::WaveLengthSet{FT}, lha::HyperspectralAbsorption{FT}; APAR_car::Bool = true, α::FT=FT(40)) where {FT<:AbstractFloat} = (
    @unpack MESOPHYLL_N, NDUB = bio;
    @unpack K_ANT, K_BROWN, K_CAB, K_CAR_V, K_CAR_Z, K_CBC, K_H₂O, K_LMA, K_PRO, K_PS, NR = lha;
    @unpack IΛ_SIF, IΛ_SIFE, Λ_SIF, Λ_SIFE = wls;

    # calculate the average absorption feature and relative Cab and Car partitions
    bio.k_all    .= (K_CAB   .* bio.cab .+                      # chlorophyll absorption
                     K_CAR_V .* bio.car .* (1 - bio.f_zeax) .+  # violaxanthin carotenoid absorption
                     K_CAR_Z .* bio.car .* bio.f_zeax .+        # zeaxanthin carotenoid absorption
                     K_ANT   .* bio.ant .+                      # anthocynanin absorption absorption
                     K_BROWN .* bio.brown .+                    # TODO: needs to be a concentration
                     K_H₂O   .* bio.l_H₂O .+                    # water absorption
                     K_CBC   .* bio.cbc .+                      # carbon-based constituents absorption
                     K_PRO   .* bio.pro .+                      # protein absorption
                     K_LMA   .* (bio.lma - bio.cbc - bio.pro)   # dry mass absorption (if some remained)
                    ) ./ MESOPHYLL_N;
    bio.α_cab    .= (K_CAB .* bio.cab) ./ bio.k_all ./ MESOPHYLL_N;
    bio.α_cabcar .= (K_CAB .* bio.cab .+ K_CAR_V .* bio.car .* (1 - bio.f_zeax) .+ K_CAR_Z .* bio.car .* bio.f_zeax) ./ bio.k_all ./ MESOPHYLL_N;

    # calculate the reflectance and transmittance at the interfaces of one layer
    _τ   = (1 .- bio.k_all) .* exp.(-bio.k_all) .+ bio.k_all .^ 2 .* expint.(bio.k_all .+ eps(FT));
    _τ_α = average_transmittance.(α, NR);
    _ρ_α = 1 .- _τ_α;
    _τ₁₂ = average_transmittance.(FT(90), NR);
    _ρ₁₂ = 1 .- _τ₁₂;
    _τ₂₁ = _τ₁₂ ./ (NR .^ 2);
    _ρ₂₁ = 1 .- _τ₂₁;

    # top surface side
    _denom = 1 .- (_τ .* _ρ₂₁) .^ 2;
    _τ_top = _τ_α .* _τ .* _τ₂₁ ./ _denom;
    _ρ_top = _ρ_α .+ _ρ₂₁ .* _τ .* _τ_top;

    # bottom surface side
    _τ_bottom = _τ₁₂ .* _τ .* _τ₂₁ ./ _denom;
    _ρ_bottom = _ρ₁₂ .+ _ρ₂₁ .* _τ .* _τ_bottom;

    # calculate the reflectance and transmittance at the interfaces of N layer
    _d     = sqrt.((1 .+ _ρ_bottom .+ _τ_bottom) .* (1 .+ _ρ_bottom .- _τ_bottom) .* (1 .- _ρ_bottom .+ _τ_bottom) .* (1 .- _ρ_bottom .- _τ_bottom));
    _ρ²    = _ρ_bottom .^ 2;
    _τ²    = _τ_bottom .^ 2;
    _a     = (1 .+ _ρ² .- _τ² .+ _d) ./ (2 .* _ρ_bottom);
    _b     = (1 .- _ρ² .+ _τ² .+ _d) ./ (2 .* _τ_bottom);
    _bⁿ⁻¹  = _b .^ (MESOPHYLL_N - 1);
    _b²ⁿ⁻² = _bⁿ⁻¹ .^ 2;
    _a²    = _a .^ 2;
    _denom = _a² .* _b²ⁿ⁻² .- 1;
    _ρ_sub = _a .* (_b²ⁿ⁻² .- 1) ./ _denom;
    _τ_sub = _bⁿ⁻¹ .* (_a² .- 1) ./ _denom;

    # avoid case of zero absorption
    _j = findall(_ρ_bottom .+ _τ_bottom .>= 1);
    _τ_sub[_j] = _τ_bottom[_j] ./ (_τ_bottom[_j] + (1 .- _τ_bottom[_j]) * (MESOPHYLL_N - 1));
    _ρ_sub[_j] = 1 .- _τ_sub[_j];

    # reflectance & transmittance of the leaf: combine top layer with next N-1 layers
    _denom   = 1 .- _ρ_sub .* _ρ_bottom;
    bio.τ_SW = _τ_top .* _τ_sub ./ _denom;
    bio.ρ_SW = _ρ_top .+ _τ_top .* _ρ_sub .* _τ_bottom ./ _denom;
    bio.α_SW = 1 .- bio.τ_SW .- bio.ρ_SW;

    # Doubling method used to calculate fluoresence is now only applied to the part of the leaf where absorption takes place, that is, the part exclusive of the leaf-air interfaces.
    # The reflectance (rho) and transmittance (tau) of this part of the leaf are now determined by "subtracting" the interfaces.
    # CF Note: All of the below takes about 10 times more time than the RT above. Need to rething speed and accuracy. (10nm is bringing it down a lot!)
    _ρ_b = (bio.ρ_SW .- _ρ_α) ./ (_τ_α .* _τ₂₁ .+ (bio.ρ_SW - _ρ_α) .* _ρ₂₁);
    _tt1 = _τ_α .* _τ₂₁;
    _tt2 = bio.τ_SW .* (1 .- _ρ_b .* _ρ₂₁);
    _z   = _tt2 ./ _tt1;
    _tt1 = _ρ_b - _ρ₂₁ .* _z .^ 2;
    _tt2 = 1 .- (_ρ₂₁.* _z) .^ 2;
    _ρ   = max.(0, _tt1 ./ _tt2);
    _tt1 = 1 .- _ρ_b .* _ρ₂₁;
    _τ   = _tt1 ./ _tt2 .* _z;

    # Derive Kubelka-Munk s and k
    _i      = findall((_ρ .+ _τ) .< 1);
    _j      = findall((_ρ .+ _τ) .> 1);
    _d[_i] .= sqrt.((1 .+ _ρ[_i] .+ _τ[_i]) .* (1 .+ _ρ[_i] .- _τ[_i]) .* (1 .- _ρ[_i] .+ _τ[_i]) .*  (1 .- _ρ[_i] .- _τ[_i]));
    _a[_i] .= (1 .+ _ρ[_i] .^ 2 .- _τ[_i] .^ 2 .+ _d[_i]) ./ (2 .* _ρ[_i]);
    _b[_i] .= (1 .- _ρ[_i] .^ 2 .+ _τ[_i] .^ 2 .+ _d[_i]) ./ (2 .* _τ[_i]);
    _a[_j] .= 1;
    _b[_j] .= 1;

    _i      = findall((_a .> 1) .& (_a .!= Inf));
    _s      = _ρ ./ _τ;
    _k      = log.(_b);
    _s[_i] .= 2 .* _a[_i] ./ (_a[_i] .^ 2 .- 1) .* log.(_b[_i]);
    _k[_i] .= (_a[_i] .- 1) ./ (_a[_i] .+ 1) .* log.(_b[_i]);
    _k_chl  = (APAR_car ? bio.α_cabcar : bio.α_cab) .* _k;

    # indices of WLE and WLF within wlp
    _ϵ       = FT(2) ^ -NDUB;
    _τ_e     = 1 .- (_k[IΛ_SIFE] .+ _s[IΛ_SIFE]) * _ϵ;
    _τ_f     = 1 .- (_k[IΛ_SIF] .+ _s[IΛ_SIF]) * _ϵ;
    _ρ_e     = _s[IΛ_SIFE] * _ϵ;
    _ρ_f     = _s[IΛ_SIF] * _ϵ;
    _sigmoid = 1 ./ (1 .+ exp.(-Λ_SIF ./ 10) .* exp.(Λ_SIFE' ./ 10));
    _mat_f   = K_PS[IΛ_SIF] .* _ϵ ./ 2 .* _k_chl[IΛ_SIFE]' .* _sigmoid;
    _mat_b   = K_PS[IΛ_SIF] .* _ϵ ./ 2 .* _k_chl[IΛ_SIFE]' .* _sigmoid;

    # Doubling adding routine
    _1_h = ones(FT, 1, length(_τ_e));
    _1_v = ones(FT, length(_τ_f), 1);
    for i in 1:NDUB
        _x_e     = _τ_e ./ (1 .- _ρ_e .^ 2);
        _x_f     = _τ_f ./ (1 .- _ρ_f .^ 2);
        _τ_e_n   = _τ_e .* _x_e;
        _τ_f_n   = _τ_f .* _x_f;
        _ρ_e_n   = _ρ_e .* (1 .+ _τ_e_n);
        _ρ_f_n   = _ρ_f .* (1 .+ _τ_f_n);
        _a₁₁     = _x_f * _1_h .+ _1_v * _x_e';
        _a₁₂     = (_x_f * _x_e') .* (_ρ_f * _1_h .+ _1_v * _ρ_e');
        _a₂₁     = 1 .+ (_x_f * _x_e') .* (1 .+ _ρ_f * _ρ_e');
        _a₂₂     = (_x_f .* _ρ_f) * _1_h .+ _1_v * (_x_e.*_ρ_e)';
        _mat_f_n = _mat_f .* _a₁₁ .+ _mat_b .* _a₁₂;
        _mat_b_n = _mat_b .* _a₂₁ .+ _mat_f .* _a₂₂;
        _τ_e     = _τ_e_n;
        _ρ_e     = _ρ_e_n;
        _τ_f     = _τ_f_n;
        _ρ_f     = _ρ_f_n;
        _mat_f   = _mat_f_n;
        _mat_b   = _mat_b_n;
    end;

    # This reduced red SIF quite a bit in backscatter, not sure why.
    _ρ_b = _ρ .+ _τ .^ 2 .* _ρ₂₁ ./ (1 .- _ρ .* _ρ₂₁);
    _x_e = _1_v * (_τ_α[IΛ_SIFE] ./ (1 .- _ρ₂₁[IΛ_SIFE] .* _ρ_b[IΛ_SIFE]))';
    _x_f = _τ₂₁[IΛ_SIF] ./ (1 .- _ρ₂₁[IΛ_SIF] .* _ρ_b[IΛ_SIF]) * _1_h;
    _y_e = _1_v * (_τ[IΛ_SIFE] .* _ρ₂₁[IΛ_SIFE] ./ (1 .- _ρ[IΛ_SIFE] .* _ρ₂₁[IΛ_SIFE]))';
    _y_f = _τ[IΛ_SIF] .* _ρ₂₁[IΛ_SIF] ./ (1 .- _ρ[IΛ_SIF] .* _ρ₂₁[IΛ_SIF]) * _1_h;
    _a   = _x_e .* (1 .+ _y_e .* _y_f) .* _x_f;
    _b   = _x_e .* (_y_e .+ _y_f) .* _x_f;

    bio.mat_b = _a .* _mat_b + _b .* _mat_f;
    bio.mat_f = _a .* _mat_f + _b .* _mat_b;

    return nothing
);

#######################################################################################################################################################################################################
#
# Changes made to this method
# General
#     2021-Oct-22: add another method to prescribe leaf spectra such as transmittance and reflectance from broadband method
#     2021-Nov-29: separate HyperspectralAbsorption as constant struct
#     2022-Jan-13: use LeafBiophysics directly in the function rather than Leaf
#     2022-Feb-02: fix documentation
#
#######################################################################################################################################################################################################
"""

    leaf_spectra!(bio::LeafBiophysics{FT}, wls::WaveLengthSet{FT}, ρ_par::FT, ρ_nir::FT, τ_par::FT, τ_nir::FT) where {FT<:AbstractFloat}

Update leaf reflectance and transmittance (e.g., prescribe broadband PAR and NIR values), given
- `bio` `ClimaCache.LeafBiophysics` type struct that contains leaf biophysical parameters
- `wls` `ClimaCache.WaveLengthSet` type struct that contain wave length bins
- `ρ_par` Reflectance at PAR region
- `ρ_nir` Reflectance at NIR region
- `τ_par` Transmittance at PAR region
- `τ_nir` Transmittance at NIR region

# Examples
```julia
wls = WaveLengthSet{Float64}();
bio = LeafBiophysics{Float64}(wls);
leaf_spectra!(bio, wls, 0.1, 0.45, 0.05, 0.25);
```
"""
leaf_spectra!(bio::LeafBiophysics{FT}, wls::WaveLengthSet{FT}, ρ_par::FT, ρ_nir::FT, τ_par::FT, τ_nir::FT) where {FT<:AbstractFloat} = (
    @unpack IΛ_NIR, IΛ_PAR = wls;

    bio.ρ_SW[IΛ_PAR] .= ρ_par;
    bio.ρ_SW[IΛ_NIR] .= ρ_nir;
    bio.τ_SW[IΛ_PAR] .= τ_par;
    bio.τ_SW[IΛ_NIR] .= τ_nir;

    bio.α_SW = 1 .- bio.τ_SW .- bio.ρ_SW;

    return nothing
);


"""
Abstract Experimental Data types
"""
abstract type ExperimentalData end
"""
Base for data in the form of samples
"""
abstract type SampleData <: ExperimentalData end
"""
Base for data in the form of a distribution
"""
abstract type HistogramData <: ExperimentalData end

abstract type AbstractRNAData{hType} <: HistogramData end

"""
Data structures

Do not use underscore "_" in name

arguments:
name: label for the data set
gene: gene name (case sensitive)
nRNA: length of histogram
histRNA: RNA histograms
bins: number of live cell recording time bins
OFF:  OFF time probability density
ON:: ON time probability density
"""
struct RNAData{nType,hType} <: AbstractRNAData{hType}
    name::String
    gene::String
    nRNA::nType
    histRNA::hType
end
struct TransientRNAData{nType,tType,hType} <: AbstractRNAData{hType}
    name::String
    gene::String
    nRNA::nType
    time::tType
    histRNA::hType
end
struct RNAMixedData{hType} <: AbstractRNAData{hType}
    name::String
    gene::String
    nRNA::Array
    fish::Array{Bool,1}
    histRNA::hType
end
struct LiveCellData <: HistogramData
    name::String
    gene::String
    bins::Array
    OFF::Array
    ON::Array
end
struct RNALiveCellData <: HistogramData
    name::String
    gene::String
    nRNA::Int
    histRNA::Array
    bins::Array
    OFF::Array
    ON::Array

end
struct MultiRNALiveCellData <: HistogramData
    name::String
    gene::String
    nRNA::Array
    histRNA::Tuple
    bins::Tuple
    OFF::Tuple
    ON::Tuple
end

"""
Abstract model types
"""
abstract type Model end
abstract type StochasticGRmodel <: Model end
abstract type AbstractGMmodel <: StochasticGRmodel end
abstract type AbstractGRMmodel <: StochasticGRmodel end
abstract type AbstractGMlossmodel <: AbstractGMmodel end

"""
Model structures

arguments:
G: number of G steps
R: number of R steps
nalleles: number of alleles producing RNA
type: type of model (e.g. splice, premature RNA death, etc.)
rates: transition rates
rateprior: prior for rates
proposal: MCMC proposal distribution
fittedparam: indices of rates to be fitted
    randomeffects: indices of rates that are fixed to each other, in the form of a 2tuple of vectors
    with index 1 the tied index vector and 2 the corresponding fitted index vector
method: numerical method for solving Master equation
"""
struct GMmodel{RateType,PriorType,ProposalType,ParamType,MethodType} <: AbstractGMmodel
    G::Int
    nalleles::Int
    rates::RateType
    rateprior::PriorType
    proposal::ProposalType
    fittedparam::ParamType
    method::MethodType
end

struct GMrescaledmodel{RateType,PriorType,ProposalType,ParamType,MethodType} <: AbstractGMmodel
    G::Int
    nalleles::Int
    rates::RateType
    rateprior::PriorType
    proposal::ProposalType
    fittedparam::ParamType
    method::MethodType
end

struct GMmultimodel{RateType,PriorType,ProposalType,ParamType,MethodType} <: AbstractGMmodel
    G::Int
    nalleles::Int
    rates::RateType
    rateprior::PriorType
    proposal::ProposalType
    fittedparam::ParamType
    method::MethodType
end

struct GMtransientmodel{RateType,PriorType,ProposalType,ParamType,MethodType} <: AbstractGMmodel
    G::Int
    nalleles::Int
    rates::RateType
    rateprior::PriorType
    proposal::ProposalType
    fittedparam::ParamType
    method::MethodType
end

struct GMdelaymodel{RateType,PriorType,ProposalType,ParamType,MethodType} <: AbstractGMmodel
    G::Int
    nalleles::Int
    rates::RateType
    rateprior::PriorType
    proposal::ProposalType
    fittedparam::ParamType
    method::MethodType
end

struct GMlossmodel{RateType,PriorType,ProposalType,ParamType,MethodType} <: AbstractGMlossmodel
    G::Int
    nalleles::Int
    rates::RateType
    rateprior::PriorType
    proposal::ProposalType
    fittedparam::ParamType
    method::MethodType
end

struct GMfixedeffectslossmodel{RateType,PriorType,ProposalType,ParamType,MethodType} <: AbstractGMlossmodel
    G::Int
    nalleles::Int
    rates::RateType
    rateprior::PriorType
    proposal::ProposalType
    fittedparam::ParamType
    fixedeffects::Tuple
    method::MethodType
end

struct GMmixedmodel{RateType,PriorType,ProposalType,ParamType,MethodType} <: AbstractGMmodel
    G::Int
    nalleles::Int
    rates::RateType
    rateprior::PriorType
    proposal::ProposalType
    fittedparam::ParamType
    method::MethodType
end

struct GRMmodel{RateType,PriorType,ProposalType,ParamType,MethodType} <: AbstractGRMmodel
    G::Int
    R::Int
    nalleles::Int
    rates::RateType
    rateprior::PriorType
    proposal::ProposalType
    fittedparam::ParamType
    method::MethodType
end

struct GRSMmodel{RateType,PriorType,ProposalType,ParamType,MethodType} <: AbstractGRMmodel
    G::Int
    R::Int
    nalleles::Int
    type::String
    rates::RateType
    rateprior::PriorType
    proposal::ProposalType
    fittedparam::ParamType
    method::MethodType
end

function write_model(model)

end

"""
Abstract Option types for fitting methods
"""
abstract type Options end
abstract type Results end

# Model and Data dependent functions

"""
datahistogram(data)
Return the RNA histogram data as one vector
"""
# datahistogram(data::RNAData) = data.histRNA
function datahistogram(data::AbstractRNAData{Array{Array,1}})
# function datahistogram(data::TransientRNAData)
    v = data.histRNA[1]
    for i in 2:length(data.histRNA)
        v = vcat(v,data.histRNA[i])
    end
    return v
end
datahistogram(data::AbstractRNAData{Array{Float64,1}}) = data.histRNA
datahistogram(data::RNALiveCellData) = [data.OFF;data.ON;data.histRNA]

datapdf(data::AbstractRNAData{Array{Float64,1}}) = normalize_histogram(data.histRNA)
datapdf(data::RNALiveCellData) = [normalize_histogram(data.OFF);normalize_histogram(data.ON);normalize_histogram(data.histRNA)]
# datapdf(data::AbstractRNAData{Array{Array,1}}) = normalize_histogram(datahistogram(data))
function datapdf(data::AbstractRNAData{Array{Array,1}})
# function datahistogram(data::TransientRNAData)
    v = normalize_histogram(data.histRNA[1])
    for i in 2:length(data.histRNA)
        v = vcat(v,normalize_histogram(data.histRNA[i]))
    end
    return v
end

#### Model likelihoods   ####

"""
likelihoodfn(param,data,model)
model likelihoodfn called by loglikelihood
"""
function likelihoodfn(param,data::RNAData,model::GMmodel)
    r = get_rates(param,model)
    n = model.G-1
    steady_state(r[1:2*n+2],n,data.nRNA,model.nalleles)
end
function likelihoodfn(param,data::RNAData,model::AbstractGMlossmodel)
    r = get_rates(param,model)
    yieldfactor = r[end]
    n = model.G-1
    steady_state(r[1:2*n+2],yieldfactor,n,data.nRNA,model.nalleles)
end
function likelihoodfn(param,data::RNAData{T1,T2},model::AbstractGMlossmodel) where {T1 <: Array, T2 <: Array}
    h = likelihoodarray(get_rates(param,model),data,model)
    hconcat = Array{Float64,1}(undef,0)
    for h in h
        hconcat = vcat(hconcat,h)
    end
    return hconcat
end
function likelihoodfn(param::Vector,data::RNALiveCellData,model::GRSMmodel)
    modelOFF, modelON, histF = likelihoodtuple(get_rates(param,model),data,model)
    return [modelOFF;modelON;histF]
end

"""
likelihoodarray(r,data,model)

Compute likelihoods for multiple distributions

r = full rate vector including yield if it exists
data = data structure
model = model structure

For time dependent model likelihoods
first set of parameters gives the initial histogram
2nd set gives the new parameters at subsequent times
data.histRNA holds array of histograms for time points given by data.time
transient computes the time evolution of the histogram
model.method=1 specifies finite difference solution otherwise use eigendecomposition solution,
"""
function likelihoodarray(r,data::TransientRNAData,model::AbstractGMmodel)
    h=likelihoodarray(r,data,model,maximum(data.nRNA))
    trim(h,data.nRNA)
end
function likelihoodarray(r,data::TransientRNAData,model::AbstractGMlossmodel)
    yieldfactor = r[end]
    nh = nhist_loss(maximum(data.nRNA),yieldfactor)
    h = likelihoodarray(r[1:end-1],data::TransientRNAData,model,yieldfactor,nh)
    # technical_loss!(h,yieldfactor)
    trim(h,data.nRNA)
end
function likelihoodarray(r,data::RNAData{T1,T2},model::AbstractGMlossmodel) where {T1 <: Array, T2 <: Array}
    yieldfactor = r[end]
    n = model.G-1
    h = Array{Array{Float64,1},1}(undef,length(data.nRNA))
    for i in eachindex(data.nRNA)
        h[i] =steady_state(r[(i-1)*2*model.G+1 : i*2*model.G],yieldfactor,n,data.nRNA[i],model.nalleles)
    end
    trim(h,data.nRNA)
end
function likelihoodarray(r,data::TransientRNAData,model::AbstractGMlossmodel,yieldfactor,maxdata)
    G = model.G
    h0 = steady_state_full(r[1:2*G],G-1,maxdata)
    transient(data.time,r[2*G+1:4*G],yieldfactor,G-1,model.nalleles,h0,model.method)
    # transient(t,r,yieldfactor,n,nalleles,P0::Vector,method)
end
function likelihoodarray(r,data::RNAData,model::GMmultimodel)
    G = model.G
    h = Array{Array{Float64,1},1}(undef,length(data.nRNA))
    for i in eachindex(data.nRNA)
        g = steady_state(r[1:2*G],G-1,data.nRNA[i],model.nalleles)
        h[i] = threshold_noise(g,r[2*G+1],r[2*G+1+i],data.nRNA[i])
    end
    return h
end
function likelihoodarray(r,data::RNAMixedData,model::AbstractGMmodel)
    G = model.G
    h = Array{Array{Float64,1},1}(undef,length(data.nRNA))
    j = 1
    for i in eachindex(data.fish)
        g = steady_state(r[1:2*G],G-1,maximum(data.nRNA),model.nalleles)
        if data.fish[i]
            h[i] = threshold_noise(g,r[2*G+j],r[2*G+j+1],data.nRNA[i])
            j += 2
        else
            h[i] = technical_loss(g,r[2*G+j],data.nRNA[i])
            j += 1
        end
    end
    return h
end

function likelihoodtuple(r,data::RNALiveCellData,model::GRSMmodel)
    if model.type == "offdecay"
        modelOFF, modelON = offonPDF(data.bins,r,model.G-1,model.R,model.method)
        histF = steady_state_offpath(r,model.G-1,model.R,data.nRNA,model.nalleles)
    elseif model.type == "offeject"
        modelOFF, modelON = offonPDF_offeject(data.bins,r,model.G-1,model.R,model.method)
        histF = steady_state_offeject(r,model.G-1,model.R,data.nRNA,model.nalleles)
    else
        modelOFF, modelON = offonPDF(data.bins,r,model.G-1,model.R,model.method)
        histF = steady_state(r,model.G-1,model.R,data.nRNA,model.nalleles)
    end
    return modelOFF, modelON, histF
end

"""
get_rates(param,model)
replace fitted rates with new values and return
"""
function get_rates(param,model::AbstractGMmodel)
    r = get_r(model)
    r[model.fittedparam] = param
    return r
end
function get_rates(param,model::GRSMmodel)
    r = get_r(model)
    r[model.fittedparam] = param
    setr(r,model)
end
get_rates(param,model::GMtransientmodel) = param[1:2*model.G]

"""
get_rates(param,model::GMrescaledmodel)

gammas are scaled by nu
"""
function get_rates(param,model::GMrescaledmodel)
    r = get_r(model)
    n = get_n(model)
    nu = n in model.fittedparam ? param[findfirst(model.fittedparam .== n)] : r[n]
    r[1:n-1] /= r[n]
    r[model.fittedparam] = param
    r[1:n-1] *= nu
    if r[2*model.G + 3] > 1
        r[2*model.G + 3] = 1
    end
    return r
end

function get_rates(param,model::GMfixedeffectslossmodel)
    r = get_r(model)
    n = get_n(model)
    r[model.fittedparam] = param
    r[model.fixedeffects[1]] = r[model.fixedeffects[2]]
    return r
end

get_r(model) = copy(model.rates)
get_n(model) = 2*model.G - 1

"""
get_param(model)

get fitted parameters from model
"""
get_param(model::StochasticGRmodel) = model.rates[model.fittedparam]

function get_param(model::GMrescaledmodel)
    r = copy(model.rates)
    n = 2*model.G - 1
    r[1:n-1] /= r[n]
    r[model.fittedparam]
end

"""
setr(r,model)

"""
function setr(r,model::GRSMmodel)
    n = model.G-1
    nr = model.R
    eta = geteta(r,n,nr)
    r[2*n + 1 + nr + 1:2*n + 1 + nr + nr] = eta
    r
end
"""
logprior(param,model::AbstractGMmodel)

compute log of the prior
"""
function logprior(param,model::AbstractGMmodel)
    d = model.rateprior
    p=0
    for i in eachindex(d)
        p -= logpdf(d[i],param[i])
    end
    return p
end

"""
logprior(x,model::GRSModel)
Compute log prior using distributions in Model.rateprior
called by mhstep() in metropolis_hastings.jl
"""
function logprior(param,model::GRSMmodel)
    r = get_rates(param,model)
    d = model.rateprior
    G = model.G
    R = model.R
    p=0
    j = 1
    #  G rates
    for i in Grange(G)
        p -= logpdf(d[j],r[i])
        j += 1
    end
    # initiation rate
    i = initiation(G)
    p -= logpdf(d[j],r[i])
    j += 1
    # sum of R Steps rates are bounded by length of insertion site to end of gene, i.e sum of reciprocal rates is bounded
    t = sum(1 ./ r[Rrange(G,R)])
    p -= logpdf(d[j],1/t)
    j += 1
    # priors for splice rates
    rs = 0
    for i in Srange(G,R)
        rs += r[i]
        p -= logpdf(d[j],rs)
        j += 1
    end
    return p
end