const pyjlnumbertype = pynew()
const pyjlcomplextype = pynew()
const pyjlrealtype = pynew()
const pyjlrationaltype = pynew()
const pyjlintegertype = pynew()

struct pyjlnumber_op{OP}
    op :: OP
end
(op::pyjlnumber_op)(self) = Py(op.op(self))
function (op::pyjlnumber_op)(self, other_::Py)
    if pyisjl(other_)
        other = pyjlvalue(other_)
    else
        other = @pyconvert(Number, other_, return Py(pybuiltins.NotImplemented))
    end
    Py(op.op(self, other))
end
function (op::pyjlnumber_op)(self, other_::Py, other2_::Py)
    if pyisjl(other_)
        other = pyjlvalue(other_)
    else
        other = @pyconvert(Number, other_, return Py(pybuiltins.NotImplemented))
    end
    if pyisjl(other2_)
        other2 = pyjlvalue(other2_)
    else
        other2 = @pyconvert(Number, other2_, return Py(pybuiltins.NotImplemented))
    end
    Py(op.op(self, other, other2))
end
pyjl_handle_error_type(op::pyjlnumber_op, self, exc) = exc isa MethodError && exc.f === op.op ? pybuiltins.TypeError : PyNULL

struct pyjlnumber_rev_op{OP}
    op :: OP
end
function (op::pyjlnumber_rev_op)(self, other_::Py)
    if pyisjl(other_)
        other = pyjlvalue(other_)
    else
        other = @pyconvert(Number, other_, return Py(pybuiltins.NotImplemented))
    end
    Py(op.op(other, self))
end
function (op::pyjlnumber_rev_op)(self, other_::Py, other2_::Py)
    if pyisjl(other_)
        other = pyjlvalue(other_)
    else
        other = @pyconvert(Number, other_, return Py(pybuiltins.NotImplemented))
    end
    if pyisjl(other2_)
        other2 = pyjlvalue(other2_)
    else
        other2 = @pyconvert(Number, other2_, return Py(pybuiltins.NotImplemented))
    end
    Py(op.op(other, self, other2))
end
pyjl_handle_error_type(op::pyjlnumber_rev_op, self, exc) = exc isa MethodError && exc.f === op.op ? pybuiltins.TypeError : PyNULL

pyjlreal_trunc(self::Real) = Py(trunc(Integer, self))
pyjl_handle_error_type(::typeof(pyjlreal_trunc), self, exc::MethodError) = exc.f === trunc ? pybuiltins.TypeError : PyNULL

pyjlreal_floor(self::Real) = Py(floor(Integer, self))
pyjl_handle_error_type(::typeof(pyjlreal_floor), self, exc::MethodError) = exc.f === floor ? pybuiltins.TypeError : PyNULL

pyjlreal_ceil(self::Real) = Py(ceil(Integer, self))
pyjl_handle_error_type(::typeof(pyjlreal_ceil), self, exc::MethodError) = exc.f === ceil ? pybuiltins.TypeError : PyNULL

function pyjlreal_round(self::Real, ndigits_::Py)
    ndigits = pyconvertarg(Union{Int,Nothing}, ndigits_, "ndigits")
    if ndigits === nothing
        Py(round(Integer, self))
    else
        Py(round(self; digits = ndigits))
    end
end
pyjl_handle_error_type(::typeof(pyjlreal_round), self, exc::MethodError) = exc.f === round ? pybuiltins.TypeError : PyNULL

function init_jlwrap_number()
    jl = pyjuliacallmodule
    filename = "$(@__FILE__):$(1+@__LINE__)"
    pybuiltins.exec(pybuiltins.compile("""
    class NumberValue(AnyValue):
        __slots__ = ()
        __module__ = "juliacall"
        def __bool__(self):
            return not self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(iszero))))
        def __add__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(+))), other)
        def __sub__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(-))), other)
        def __mul__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(*))), other)
        def __truediv__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(/))), other)
        def __floordiv__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(÷))), other)
        def __mod__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(%))), other)
        def __pow__(self, other, modulo=None):
            if modulo is None:
                return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(^))), other)
            else:
                return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(powermod))), other, modulo)
        def __lshift__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(<<))), other)
        def __rshift__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(>>))), other)
        def __and__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(&))), other)
        def __xor__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(⊻))), other)
        def __or__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(|))), other)
        def __radd__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(+))), other)
        def __rsub__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(-))), other)
        def __rmul__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(*))), other)
        def __rtruediv__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(/))), other)
        def __rfloordiv__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(÷))), other)
        def __rmod__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(%))), other)
        def __rpow__(self, other, modulo=None):
            if modulo is None:
                return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(^))), other)
            else:
                return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(powermod))), other, modulo)
        def __rlshift__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(<<))), other)
        def __rrshift__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(>>))), other)
        def __rand__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(&))), other)
        def __rxor__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(⊻))), other)
        def __ror__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(|))), other)
        def __eq__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(==))), other)
        def __ne__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(!=))), other)
        def __le__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(≤))), other)
        def __lt__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(<))), other)
        def __ge__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(≥))), other)
        def __gt__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(>))), other)
    class ComplexValue(NumberValue):
        __slots__ = ()
        __module__ = "juliacall"
        def __complex__(self):
            return self._jl_callmethod($(pyjl_methodnum(pycomplex)))
        @property
        def real(self):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(real))))
        @property
        def imag(self):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(imag))))
        def conjugate(self):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(conj))))
    class RealValue(ComplexValue):
        __slots__ = ()
        __module__ = "juliacall"
        def __float__(self):
            return self._jl_callmethod($(pyjl_methodnum(pyfloat)))
        @property
        def real(self):
            return self
        @property
        def imag(self):
            return 0
        def conjugate(self):
            return self
        def __complex__(self):
            return complex(float(self))
        def __trunc__(self):
            return self._jl_callmethod($(pyjl_methodnum(pyjlreal_trunc)))
        def __floor__(self):
            return self._jl_callmethod($(pyjl_methodnum(pyjlreal_floor)))
        def __ceil__(self):
            return self._jl_callmethod($(pyjl_methodnum(pyjlreal_ceil)))
        def __round__(self, ndigits=None):
            return self._jl_callmethod($(pyjl_methodnum(pyjlreal_round)), ndigits)
    class RationalValue(RealValue):
        __slots__ = ()
        __module__ = "juliacall"
        @property
        def numerator(self):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(numerator))))
        @property
        def denominator(self):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(denominator))))
    class IntegerValue(RationalValue):
        __slots__ = ()
        __module__ = "juliacall"
        def __int__(self):
            return self._jl_callmethod($(pyjl_methodnum(pyint)))
        def __index__(self):
            return self.__int__()
        @property
        def numerator(self):
            return self
        @property
        def denominator(self):
            return 1
    import numbers
    numbers.Number.register(NumberValue)
    numbers.Complex.register(ComplexValue)
    numbers.Real.register(RealValue)
    numbers.Rational.register(RationalValue)
    numbers.Integral.register(IntegerValue)
    del numbers
    """, filename, "exec"), jl.__dict__)
    pycopy!(pyjlnumbertype, jl.NumberValue)
    pycopy!(pyjlcomplextype, jl.ComplexValue)
    pycopy!(pyjlrealtype, jl.RealValue)
    pycopy!(pyjlrationaltype, jl.RationalValue)
    pycopy!(pyjlintegertype, jl.IntegerValue)
end

pyjl(v::Number) = pyjl(pyjlnumbertype, v)
pyjl(v::Complex) = pyjl(pyjlcomplextype, v)
pyjl(v::Real) = pyjl(pyjlrealtype, v)
pyjl(v::Rational) = pyjl(pyjlrationaltype, v)
pyjl(v::Integer) = pyjl(pyjlintegertype, v)


module Samplers

export GibbsSampler, SequentialMonteCarlo, SequentialImportanceSampler, MetropolisHastingsSampler
export MetropolisHastingsSampler, MetropolisWithinGibbsSampler

abstract type Sampler end

abstract type MetropolisWithinGibbsSampler <: Sampler  end

struct GibbsSampler <: MetropolisWithinGibbsSampler
    num_iterations::Int64
    num_burn_in::Int64
    skip::Int64
    assignment_types::Vector{String}
    function GibbsSampler(;num_iterations::Int64=1000, num_burn_in::Int64=0, skip::Int64=1,
                          assignment_types::Vector{String}=["random"])
        if num_iterations < 1
            error("Number of iterations should be positive.")
        end
        if num_burn_in < 0
            error("Number of burn in iterations should be positive.")
        end
        if !(all(map(x -> x in ["random", "all same cluster", "all different clusters"], assignment_types)))
            error("Initial assignment types can only be one of 'random', 'all same cluster' or 'all different clusters'")
        end
        return new(num_iterations, num_burn_in, skip, assignment_types)
    end
end

mutable struct MetropolisHastingsSampler <: MetropolisWithinGibbsSampler
    num_iterations::Int64
    num_burn_in::Int64
    proposal_radius::Int64
    skip::Int64
    adaptive::Bool
    assignment_types::Vector{String}
    function MetropolisHastingsSampler(;num_iterations=1000, num_burn_in=1000, proposal_radius=5, skip=1,
                                       adaptive=false, assignment_types::Vector{String}=["random"])
        if num_iterations < 1
            error("Number of iterations should be positive.")
        end
        if num_burn_in < 0
            error("Number of burn in iterations should be positive.")
        end
        if !(all(map(x -> x in ["random", "all same cluster", "all different clusters"], assignment_types)))
            error("Initial assignment types can only be one of 'random', 'all same cluster' or 'all different clusters'")
        end
        return new(num_iterations, num_burn_in, proposal_radius, skip, adaptive, assignment_types)
    end
end

struct SequentialMonteCarlo <: Sampler
    num_particles::Int64
    ess_threshold::Float64
    function SequentialMonteCarlo(;num_particles::Int64=1000, ess_threshold::T=0.5) where
                                  {T <: Real}
        if num_particles < 1
            error("Number of particles should be positive.")
        end
        if ess_threshold < 0
            error("ESS Threshold should be non-negative.")
        end
        if typeof(ess_threshold) <: AbstractFloat && ess_threshold > 1
            error("ESS Threshold should be either a non-negative integer or a fraction")
        end
        if ess_threshold < 1
            ess_threshold = num_particles*ess_threshold
        end
        return new(num_particles, ess_threshold)
    end
end

struct SequentialImportanceSampler <: Sampler
    num_particles::Int64
    function SequentialImportanceSampler(;num_particles=1000)
        return new(num_particles)
    end
end

end


export iauUtctai
"""
Time scale transformation:  Coordinated Universal Time, UTC, to
International Atomic Time, TAI.

This function is part of the International Astronomical Union's
SOFA (Standards of Fundamental Astronomy) software collection.

Status:  canonical.

Given:
   utc1,utc2  double   UTC as a 2-part quasi Julian Date (Notes 1-4)

Returned:
   tai1,tai2  double   TAI as a 2-part Julian Date (Note 5)

Returned (function value):
               int      status: +1 = dubious year (Note 3)
                                 0 = OK
                              -1 = unacceptable date

Notes:

   1. utc1+utc2 is quasi Julian Date (see Note 2), apportioned in any
   convenient way between the two arguments, for example where utc1
   is the Julian Day Number and utc2 is the fraction of a day.

   2. JD cannot unambiguously represent UTC during a leap second unless
   special measures are taken.  The convention in the present
   function is that the JD day represents UTC days whether the
   length is 86399, 86400 or 86401 SI seconds.  In the 1960-1972 era
   there were smaller jumps (in either direction) each time the
   linear UTC(TAI) expression was changed, and these "mini-leaps"
   are also included in the SOFA convention.

   3. The warning status "dubious year" flags UTCs that predate the
   introduction of the time scale or that are too far in the future
   to be trusted.  See iauDat for further details.

   4. The function iauDtf2d converts from calendar date and time of day
   into 2-part Julian Date, and in the case of UTC implements the
   leap-second-ambiguity convention described above.

   5. The returned TAI1,TAI2 are such that their sum is the TAI Julian
   Date.

Called:
   iauJd2cal    JD to Gregorian calendar
   iauDat       delta(AT) = TAI-UTC
   iauCal2jd    Gregorian calendar to JD

References:

   McCarthy, D. D., Petit, G. (eds.), IERS Conventions (2003),
   IERS Technical Note No. 32, BKG (2004)

   Explanatory Supplement to the Astronomical Almanac,
   P. Kenneth Seidelmann (ed), University Science Books (1992)

This revision:  2013 July 26

SOFA release 2018-01-30

Copyright (C) 2018 IAU SOFA Board.  See notes at end.
"""

# int iauUtctai(double utc1, double utc2, double *tai1, double *tai2)

function iauUtctai(utc1::Real, utc2::Real)
   ref_tai1 = Ref{Float64}(0.0)
   ref_tai2 = Ref{Float64}(0.0)

   status = ccall((:iauUtctai, libsofa_c), Cint,
                   (Cdouble, Cdouble, Ref{Cdouble}, Ref{Cdouble}),
                   convert(Float64, utc1), convert(Float64, utc2),
                   ref_tai1, ref_tai2)

   return status, ref_tai1[], ref_tai2[]
end


using DataFrames  # readtable function
# using Requests
type Bus
   nodeID::Int
   root::Int
   Pd::Float64
   Qd::Float64
   Vmax::Float64
   Vmin::Float64
   B::Float64
   R::Float64
   X::Float64
   children::Vector{Int}
   ancestor::Vector{Int}
   genids::Int
   function Bus(nodeID, root, Pd, Qd, B, R, X, Vmax, Vmin)
      b = new(nodeID, root, Pd, Qd)
      b.Vmax = Vmax
      b.Vmin = Vmin
      b.R = R
      b.X = X
      b.B = B
      b.children = Int[]
      b.ancestor = Int[]
      # b.genids = 0
      return b
   end
end
#################################################################
type Generator
   genID::Int
   busidx::Int
   Pgmax::Float64
   Pgmin::Float64
   Qgmax::Float64
   Qgmin::Float64
   cost::Float64
   function Generator( busidx, Pgmax, Pgmin, Qgmax, Qgmin, cost)
      g = new(busidx)
      g.busidx = busidx
      g.cost = cost
      g.Pgmax = Pgmax
      g.Pgmin = Pgmin
      g.Qgmax = Qgmax
      g.Qgmin = Qgmin
      return g
   end
end

##################################################################
type Line
   arcID::Int
   tail::Int # the "to" node
   head::Int # the "from" node
   r::Float64 # the resistance value
   x::Float64 # the reactance value
   u::Float64 # the capacity of the line
   function Line(arcID, tail, head, r, x, u)
      line = new(arcID, tail, head, r, x)
      line.u = u
      return line
   end
end
#########################################
## storage data
type Storage
   ID::Int
   busidx::Int
   Pkmax::Float64
   Ekmax::Float64
   Aleph::Float64
   Co::Float64
   Cp::Float64
   Ce::Float64
   function Storage(ID, busidx, Pkmax, Ekmax, Aleph, Co, Cp, Ce)
      s = new(ID)
      s.ID = ID
      s.busidx = busidx
      s.Pkmax = Pkmax
      s.Ekmax = Ekmax
      s.Aleph = Aleph
      s.Co = Co
      s.Cp = Cp
      s.Ce = Ce
      return s
   end
end
#########################################
## station data
type Station
   ID::Int
   busidx::Int
   Cf::Float64
   function Station(ID, busidx, Cf)
      st = new(ID)
      st.ID = ID
      st.busidx = busidx
      st.Cf = Cf
      return st
   end
end
######################################################################################################

function DataImport(filename_Node, filename_Generator, filename_Line, filename_Storage, filename_Station, filename_SMP)

   ######################################################################################################
   # Bus/Node Data
   # busmat = readtable(filename_Node)
   busmat = readcsv(filename_Node, header=true)[1]

   buses = Bus[]
   busIDmap = Dict()
   for i in 1:size(busmat,1)

      nodeID = i
      busIDmap[busmat[i,1]] = i

      if i==1
         root = busIDmap[busmat[i,1]]
      else
         root = 0
      end

      Pd = busmat[i,2]
      Qd = busmat[i,3]
      R = busmat[i,6]
      X = busmat[i,7]

      Vmax = busmat[i,4]
      Vmin = busmat[i,5]
      B = busmat[i,8]

      b = Bus(nodeID, root, Pd, Qd, B, R, X, Vmax, Vmin)
      push!(buses, b)
   end

   #######################################################################################################
   ## generator data
   generatorlist = Int[]
   generators = Generator[]
   # genmat = readtable(filename_Generator)
   genmat = readcsv(filename_Generator, header=true)[1]

   for i in 1:size(genmat,1)
      busidx = genmat[i,1]
      Pgmax = genmat[i,2]
      Pgmin = genmat[i,3]
      Qgmax = genmat[i,4]
      Qgmin = genmat[i,5]
      cost = genmat[i,6]

      g = Generator(busidx, Pgmax, Pgmin, Qgmax, Qgmin, cost)
      push!(generators, g)
      # setg(buses[busidx], i)
   end
   #for g in 1:length(generators)
   #   generators[g].cost = genmat[g,4]
   #end
   #################################################################
   ## branch data
   # branchmat = readtable(filename_Line)
   branchmat = readcsv(filename_Line, header=true)[1]

   lines = Line[]
   for i in 1:size(branchmat,1)
      fbus = busIDmap[branchmat[i,2]]
      tbus = busIDmap[branchmat[i,1]]
      abus = busIDmap[branchmat[i,1]]
      x = branchmat[i,4]
      r = branchmat[i,3]
      u = branchmat[i,6]  # flow limit
      push!(buses[tbus].children, fbus)#children
      push!(buses[fbus].ancestor, abus)#ancestor
      l = Line(i, tbus, fbus, r, x, u)
      push!(lines,l)
   end
   #########################################
   ## storage data
   # storagemat = readtable(filename_Storage)
   storagemat = readcsv(filename_Storage, header=true)[1]

   storages = Storage[]
   for i in 1:size(storagemat,1)
      storageID = i
      busidx = 1
      Pkmax = storagemat[i,2]
      Ekmax = storagemat[i,3]
      Aleph = storagemat[i,4]
      Co = storagemat[i,5]
      Cp = storagemat[i,6]
      Ce = storagemat[i,7]

      temp = Storage(storageID, busidx, Pkmax, Ekmax, Aleph, Co, Cp, Ce)
      push!(storages,temp)
   end

   #########################################
   ## charging/discharing station data
   # stationmat = readtable(filename_Station)
   stationmat = readcsv(filename_Station, header=true)[1]
   stations = Station[]
   for i in 1:size(stationmat,1)
      stationID = i
      busidx = stationmat[i,2]
      Cf = stationmat[i,3]

      temp = Station(stationID, busidx, Cf)
      push!(stations,temp)
   end

   #########################################
   SMP_raw = readcsv(filename_SMP, header=true)[1]
   SMP = SMP_raw[3:end]

   return buses, generators,lines, storages, stations, SMP
end


using Base
using Interpolations

# While the StateID struct and its helper functions may seem a little over-engineered, they
# primarily exist to document the iSLP and iLV quantities used by TOPBase

"""
    StateID(iSLP, iLV)

`StateID` represents a given state (electron configuration) of an ion species. The arguments are
taken from the TOPBase.

# Arguments
- `iSLP::UInt16`: An integer representing the quantum atomic numbers SLπ (total spin, orbital
  angular momentum, and parity). The value is given by `100*(2*S+1) + 10*L + π`. Note that TOPBase
  documentation refers to the parity quantum number as P or π, but for consistency with how TOPBase
  presents the data, we prefer P in our implementation code.
- `iLV::UInt8`: Distinguishes between configurations that have the same values of `iSLP`. TOPBase
  seems to index the configuration based on the energy level of the configuration (with `iLV = 1`
  corresponding to the lowest energy).

# Notes
The combination of this struct and an ion species (or neutral element) should be enough to fully
specify a given electron configuration. For example, consider He I:
- `iSLP = 100, iLV = 1` has the electron configuration: 1s²
- `iSLP = 100, iLV = 2` has the electron configuration: 1s 2s
- `iSLP = 100, iLV = 3` has the electron configuration: 1s 3s
- `iSLP = 111, iLV = 1` has the electron configuration: 1s 2p
- `iSLP = 111, iLV = 2` has the electron configuration: 1s 3p
"""
struct StateID
    iSLP::UInt16
    iLV::UInt8
    function StateID(iSLP, iLV)
        @assert 99 < iSLP < 992 # this upper limit is set based on the data source
        @assert (iSLP % 10) < 2 # trailing digit must be 0 or 1
        @assert 0 < iLV # the upper limit is set based on the data source
        new(iSLP, iLV)
    end
end

Base.:(<)(a::StateID, b::StateID) = (a.iSLP < b.iSLP) || ((a.iSLP == b.iSLP) && (a.iLV < b.iLV))
Base.show(io::IO, obj::StateID) = print(io, "StateID{iSLP = ", obj.iSLP, ", iLV = ", obj.iLV, "}")

_parity_vals = ("even", "odd") # predefine these to avoid allocations

function _quantum_prop_type(state_id::StateID)
    S_code = div(state_id.iSLP, 100) # hundreds place of state_id.iSLP
    twice_S = S_code - 1
    L = div(state_id.iSLP - S_code*100, 10) # tens place of state_id.iSLP
    parity = _parity_vals[(state_id.iSLP % 2) + 1] # 0 or 1 in ones place of state_id.iSLP
    (Int32(twice_S), Int32(L), parity)
end
get_S_quantum_num(state_id::StateID) = _quantum_prop_type(state_id)[1] / 2.0f0
get_L_quantum_num(state_id::StateID) = _quantum_prop_type(state_id)[2]
get_parity(state_id::StateID) = _quantum_prop_type(state_id)[3]

function get_statistical_weight(state_id::StateID)
    twice_S, L, _ = _quantum_prop_type(state_id)
    (twice_S+1)*(2*L+1)
end

"""
    StateProp(ion_energy_ryd, excitation_potential_ryd, statistical_weight, electron_config)

`StateProp` holds data associated with different electron configurations. It's used to hold
data read from the TOPBase cross-section queries and to compute the statistical weight.
"""
struct StateProp
    ion_energy_ryd::Float64 # energy with respect to ionization potential in Rydbergs
    excitation_potential_ryd::Float64 # energy above ground state in Rydbergs
    statistical_weight::UInt16
    electron_config::String
end

# define some generally useful functions related to parsing
function _parse_species_stateid(str::AbstractString, start_Z::Integer,
                                start_numElectrons::Integer, start_iSLP::Integer,
                                start_iLV::Integer, last_iLV::Integer)
    # parses the atomic_num, number of electrons, iSLP, and iLV entries of the table
    atomic_num = parse(UInt8, str[start_Z:start_numElectrons-1])
    num_electrons = parse(UInt8, str[start_numElectrons:start_iSLP-1])
    iSLP = parse(UInt16, str[start_iSLP:start_iLV-1])
    iLV = parse(UInt16, str[start_iLV:last_iLV])
    (Species(Formula(atomic_num), atomic_num - num_electrons), StateID(iSLP, iLV))
end

# Functionallity for reading photoionization cross sections. Since these tables can be massive, we
# use an iterator to do this. This is based on the design used in the Julia standard library for
# implementing the ``EachLine`` iterable object. That code used the following license:
#
# Copyright (c) 2009-2021: Jeff Bezanson, Stefan Karpinski, Viral B. Shah,
# and other contributors:
#
# https://github.com/JuliaLang/julia/contributors
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

const _TABLE_BREAK = " ================================================"
const _HEADER_LINE_LENGTH = length(_TABLE_BREAK)

struct EachPhotoIonSubtable{IOT <: IO}
    stream::IOT
    ondone::Function
end

"""
    each_photo_ion_subtable(filename::AbstractString)

Create an iterable `EachPhotoIonSubtable` object for parsing photoionization cross-sections.

The input file is organized as a series of subtables. Each subtable provides the photoionization
energies (make sure this isn't the energy of the photoionized electron) in Ryd, and the associated
cross section (in megabarns) for a different state.

Each entry in the `EachPhotoIonSubtable` iterable provides a named tuple with the keys:
- name: holds the species name
- state_id: holds a `StateID` instance
- photon_energy_ryd: the photoionization values (check), in Ryd, stored in an array of Float32 vals
- cross_section_MegaBarn: the associated photoionization cross-sections. This array should have the
  same shape and datatype as photon_energy_ryd

The underlying file is only closed after the `EachPhotoIonSubtable` is garbage collected.

#Notes
This implementation is not robust enough to deal with blank lines at the start or end of the file.

The table should have been produced by a
[TOPbase Photoionisation Cross Section query](http://cdsweb.u-strasbg.fr/topbase/xsections.html)
that uses the following query options:
- Only a single atomic number and a single number of electrons is included in the query (other
  choices may yield too many results, which causes TOPBase to truncate output).
- There isn't any limitation on the number the quantum numbers, the energy or the levels
- The query request levels ordered in the "Level order in each series"
"""
function each_photo_ion_subtable(filename::AbstractString)
    stream = open(filename, "r")
    # confirm that standard file header is present and move the position of the
    # stream past this point
    @assert readline(stream) == _TABLE_BREAK
    @assert readline(stream) == "       I  NZ  NE  ISLP  ILV        E(RYD)      NP"
    @assert readline(stream) == _TABLE_BREAK
    EachPhotoIonSubtable(stream, ()->close(stream))::EachPhotoIonSubtable
end

function Base.iterate(itr::EachPhotoIonSubtable, state=nothing)
    next_line = readline(itr.stream)

    if next_line == _TABLE_BREAK
        return (itr.ondone(); nothing)
    end

    # first parse the subtable's header
    species, state_id = _parse_species_stateid(next_line, 9, 13, 17, 23, 27)
    ion_energy_ryd = parse(Float32, next_line[28:41])

    num_entries = parse(UInt32, next_line[42:49])

    # this could probably be 32-bit.
    photon_energy_ryd = Array{Float32,1}(undef, num_entries)
    cross_section_MegaBarn = Array{Float32,1}(undef, num_entries)
    for i = 1:num_entries # this properly handles the case when num_entries == 0
        cur_line = readline(itr.stream)
        photon_energy_ryd[i] = parse(Float32, cur_line[1:14])
        cross_section_MegaBarn[i] = parse(Float32, cur_line[15:24])
    end

    # parse the species, and the level
    ((species = species, state_id=state_id, ion_energy_ryd = ion_energy_ryd,
      photon_energy_ryd_arr = photon_energy_ryd,
      cross_section_MegaBarn_arr = cross_section_MegaBarn), nothing)
end

# this and the following definition implement Julia's informal iterator interface
Base.eltype(::Type{<:EachPhotoIonSubtable}) =
    Tuple{String, StateID, Float32, Array{Float32,1}, Array{Float32,1}}
Base.IteratorSize(::Type{<:EachPhotoIonSubtable}) = SizeUnknown()

# In a couple tables, there's a place where the photon energy is listed out of order. This seems to
# be okay given that this is in a section of the table where the cross-section is zero.
const _species_with_problematic_subtables = (Species("He_I"), Species("C_I"), Species("C_II"),
                                             Species("Al_II"), Species("O_II"))

function _interpolator_from_subtable(table_photon_energy_ryd::Vector{Float32},
                                     table_cross_section_MBarn::Vector{Float32},
                                     species::Union{Species,Nothing} = nothing,
                                     extrapolation_bc = 0.0)
    if species in _species_with_problematic_subtables

        for j in 1:length(table_photon_energy_ryd)-1
            if table_photon_energy_ryd[j+1] < table_photon_energy_ryd[j]
                @assert j > 1
                @assert (j + 2) <= length(table_photon_energy_ryd)
                @assert table_photon_energy_ryd[j+2] >= table_photon_energy_ryd[j+1]
                @assert table_photon_energy_ryd[j+2] >= table_photon_energy_ryd[j]
                @assert table_photon_energy_ryd[j+1] >= table_photon_energy_ryd[j-1]
                @assert table_photon_energy_ryd[j] >= table_photon_energy_ryd[j-1]

                @assert table_cross_section_MBarn[j+2] == 0.0
                @assert table_cross_section_MBarn[j+1] == 0.0
                @assert table_cross_section_MBarn[j] == 0.0
                @assert table_cross_section_MBarn[j-1] == 0.0

                # swap the order of the 2 values
                temp = table_photon_energy_ryd[j]
                table_photon_energy_ryd[j] = table_photon_energy_ryd[j+1]
                table_photon_energy_ryd[j+1] = temp
            end
        end
    end
    LinearInterpolation(table_photon_energy_ryd, table_cross_section_MBarn,
                        extrapolation_bc=extrapolation_bc)
end

function _try_env_prefix_path(prefix_env_var, fname)
    prefix = get(ENV, prefix_env_var, nothing)
    if isnothing(prefix)
        e = ErrorException(string("The \"", prefix_env_var, "\" environment variable was not set"))
        throw(e);
    end
    path = joinpath(prefix, fname)
    if !isfile(path) # this properly handles the case where path is a symlink to a file
        throw(ErrorException(string("There is no file at ", path)));
    end
    path
end

# return the minimum iterator over cross_section subtables and the ionization energy of the ground
# state (according to the table)
function _get_cross_sec_data(species::Species,
                             cross_sec_file::Union{AbstractString,Nothing} = nothing)

    # get the photo-ionization subtable iterator:
    my_cross_sec_file = if isnothing(cross_sec_file)
        # determine the string representation of Species
        # we don't directly use repr(species) since the representation could change in the future
        # to not use roman numeral (e.g. if support is added for negatively charged species)
        formula_str = repr("text/plain", species.formula)
        roman_numeral = get_roman_numeral(species)
        fname = string(formula_str, "_", roman_numeral, ".txt")

        println("cross_sec_file isn't specified. Searching for file in the directory specified ",
                "by the KORG_OP_DIR environment variable.")
        _try_env_prefix_path("KORG_OP_DIR", fname)
    else
        cross_sec_file
    end

    # determine the ionization energy of the ground state (from the subtables)
    gs_ion_energy_ryd = begin # this could be optimized significantly!
        cur_min = floatmax(Float32)
        for (cur_species, _, ion_energy_ryd, _, _) in each_photo_ion_subtable(my_cross_sec_file)
            @assert (cur_species == species) # this will catch mistakes in the query that generated
                                             # the cross_section data file
            cur_min = min(ion_energy_ryd, cur_min)
        end
        cur_min
    end

    subtable_iter = each_photo_ion_subtable(my_cross_sec_file)
    subtable_iter, gs_ion_energy_ryd
end

@doc raw"""
    weighted_bf_cross_section_TOPBase(λ_vals, T_vals, species; extrapolation_bc = 0.0
                                      cross_sec_file = nothing,
                                      partition_func = partition_funcs[species])

Calculate the combined bound-free cross section (in cm²) of an ion species using data from the
Opacity Project. The result is the mean of the the bound-free cross section of each electron state,
weighted by their Boltzmann probabilities. It includes the LTE correction for stimulated emission.
The result is stored in a 2D array with a size given by `(length(λ_vals), length(T_vals))`.

# Arguments
- `λ_vals::AbstractVector`: wavelengths (in cm) to compute the cross sections at
- `T_vals::AbstractVector`: temperatures (in K) to compute the cross sections at
- `species::Species`: the starting atom or atomic ion of the bound-free reaction.

# Keyword Arguments
- `extrapolation_bc`: Specifies handling of extrapolation during linear interpolation (this is
  passed to `Interpolations.LinearInterpolation`).
- `cross_sec_file`: Optional keyword that specifies the path to the text file that holds the
  cross-section subtables. When `nothing` (the default), this function tries to load the file
  from `$KORG_OP_DIR/<atomic-symbol>_<ion-state>.txt`. `<atomic-symbol>` is replaced with the
  atomic symbol of the species (e.g. `H`, `He`, `Al`) and `<ion-state>` is replaced with the
  capital Roman numeral(s) specifying the ionization state.
- `partition_func`: Specifies the partition function for the current species. This should be a
  callable that accepts temperature as an argument. When this is `nothing`, it falls back to the
  default partition functions.

# Explanation
In more mathematical rigor, this function basically evaluates the following equation:

``\overline{\sigma_{\rm bf}}(\lambda,T) = \sum_{{\bf n}\in{\rm states}} \frac{ g_{\bf n}}{U(T)} e^{-E_{\bf n}/(k_B T)}\left(1 - e^{-h c/(\lambda k_B T)}\right) \sigma_{{\rm bf}, {\bf n}}(\lambda)``

In this equation:
- ``g_{\bf n}`` is the statistical weight of state ``{\bf n}``.
- ``U`` is the partition function.
- ``E_{\bf n}`` is the excitation potential of state ``{\bf n}``. In other words, its the energy
  difference between state ``{\bf n}`` and the ground state.
- ``\sigma_{\lambda,{\rm bf}, {\bf n}}`` is the bound-free cross-section (a.k.a. photo-ionization
  cross section) of state ``{\bf n}``.

The linear aborption coefficient for bound-free absorption is simply:
``\alpha_{\lambda,{\rm bf}} = n_{\rm tot} \overline{\sigma_{\rm bf}}(\lambda,T)``.

# Notes
In the future, it needs to be possible to pass an argument that adjusts the energy levels of
different electron configurations.
"""
function weighted_bf_cross_section_TOPBase(λ_vals::AbstractVector, T_vals::AbstractVector,
                                           species::Species; extrapolation_bc=0.0,
                                           cross_sec_file::Union{AbstractString,Nothing} = nothing,
                                           partition_func = partition_funcs[species])
    # Note: it would not take a lot of work to make this work with 32-bit floats (since the tables
    # only store 32-bit floats). If doing that, it might be wise to accept λ in units of ångstroms
    # (for precision purposes)

    # TODO: make it possible to pass in a dict mapping dict names to the various energy levels
    #       (to allow for overwriting the values in the table with empirical values)

    @assert !ismolecule(species) # species must be an atom or atomic ion

    # precompute Temperature-dependent constant
    inv_partition_func_val = 1.0./partition_func.(T_vals)

    β_Ryd = RydbergH_eV./(kboltz_eV .* T_vals)

    # convert λ_vals to photon energies
    photon_energies = (hplanck_eV * c_cgs / RydbergH_eV) ./ λ_vals

    # prepare the output array where results will be accumulated
    weighted_average = zeros(eltype(photon_energies), (length(photon_energies),length(T_vals)))

    cross_section_subtables, gs_ion_energy_ryd = _get_cross_sec_data(species, cross_sec_file)

    for subtable in cross_section_subtables
        if length(subtable.photon_energy_ryd_arr) == 0
            continue
        end

        cross_section_interp = _interpolator_from_subtable(subtable.photon_energy_ryd_arr,
                                                           subtable.cross_section_MegaBarn_arr,
                                                           species, extrapolation_bc)
        statistical_weight = get_statistical_weight(subtable.state_id)
        excitation_potential_ryd = subtable.ion_energy_ryd - gs_ion_energy_ryd

        # now iterate over Temperatures
        for j in 1:length(T_vals)

            # now compute the weighting of the current state under LTE
            weight = inv_partition_func_val[j] * statistical_weight * exp(-excitation_potential_ryd
                                                                          * β_Ryd[j])

            current_cross_section = ( cross_section_interp.(photon_energies) .*
                                      (1.0 .- exp.(-photon_energies.*β_Ryd[j])) )
            view(weighted_average, :, j) .+= (current_cross_section .* weight)
        end
    end

    # convert weighted_average units from megabarn to cm²
    weighted_average .*= 1e-18
    weighted_average
end

"""
    absorption_coef_bf_TOPBase(λ, T, ndens_species, species; kwargs...)

Computes the bound-free linear absorption coefficient, α, (in units of cm⁻¹) using data from the
Opacity Project (downloaded from TOPBase) and assuming LTE. The result is stored in a 2D array with
a size given by `(length(λ), length(T))`.

# Arguments
- `λ::AbstractVector`: wavelengths (in cm) to compute α at.
- `T::AbstractVector`: temperatures (in K) to compute α at.
- `ndens_species::AbstractVector`: number density (in cm⁻³) of the species at each temperature
  (this should have the same length as T).
- `species::Species`: the starting atom or atomic ion of the bound-free reaction.

# Keyword Arguments
This function accepts all the same keyword arguments as `weighted_bf_cross_section_TOPBase`.
"""
function absorption_coef_bf_TOPBase(λ::AbstractVector, T::AbstractVector,
                                    ndens_species::AbstractVector, species::Species; kwargs...)
    @assert size(T) == size(ndens_species)

    arr = weighted_bf_cross_section_TOPBase(λ, T, species; kwargs...)

    # to save time and memory, let's modify the values of arr in-place
    for (i,n) in enumerate(ndens_species)
        view(arr,:,i) .*= n
    end

    arr
end


using Revise
using Plots
using Pkg
# Pkg.activate(".")
using RadiativeTransfer
using RadiativeTransfer.Architectures
using RadiativeTransfer.Absorption
using RadiativeTransfer.Scattering
using RadiativeTransfer.vSmartMOM
using RadiativeTransfer.SolarModel
using InstrumentOperator
using Interpolations
using Polynomials
using ForwardDiff
using Distributions
using NCDatasets

## Atmospheric Radiative Transfer

# Load parameters from file
parameters = vSmartMOM.parameters_from_yaml("RadiativeTransfer/test/helper/O2Parameters.yaml")
FT = Float64

# oco2_L1bScND_18688a_180105_B8100r_180206190633

oco_file = "/net/fluo/data1/group/oco2/L1bSc/oco2_L1bScGL_15258a_170515_B10003r_200214061601.h5"
oco_met_file = "/net/fluo/data1/group/oco2/L2Met/oco2_L2MetGL_26777a_190715_B10003r_200429213029.h5"
ils_file = "/home/rjeyaram/RadiativeTransfer/src/vSmartMOM/ils_oco2.json"

fp = 5
iOrbit = 4400
band = 1

sza_ = 32.4436
vza_ = [0.072]

function conv_spectra_local(m::VariableKernelInstrument, ν, spectrum; stride=1)
    # FT = eltype(m.ν_out)
    # Define grid where to perform convolution:

    # Padding at both sides required:
    off = ceil(Int, size(m.kernel, 1) / 2)
    ind = off:stride:(length(ν) - off)

    # knots where convolution will be applied to
    knots = view(ν, ind)
    te = LinearInterpolation(m.ν_out, Float32.(m.ind_out))
    spec_out = zeros(Real, length(knots));
    for i in eachindex(knots)
        # Simple first, nearest neighbor ILS
        ind_fraction = round(Int, te(knots[i]));
        kernel = view(m.kernel, :, ind_fraction)
        for j in eachindex(kernel)
            spec_out[i] += kernel[j] * spectrum[ind[i] + j]
        end
    end
    # Change this later to only perform conv around output grid!
    fin = LinearInterpolation(ν[ind], spec_out; extrapolation_bc=Interpolations.Flat())
    return fin(m.ν_out)
end;

# Runner is used to set AD fields as duals
function runner!(y, x, parameters=parameters, oco_file=oco_file,
                                              oco_met_file=oco_met_file,
                                              ils_file=ils_file, fp=fp, iOrbit=iOrbit,
                                              sza_=sza_, vza_=vza_)

    # Set parameters fields as the dual numbers
    parameters.brdf = [LambertianSurfaceScalar(x[1])]

    parameters.scattering_params.rt_aerosols[1].τ_ref = x[2];

    # parameters.scattering_params.rt_aerosols[1].aerosol.size_distribution = LogNormal(log(x[3]), log(x[4]), check_args=false)

    # parameters.scattering_params.rt_aerosols[1].aerosol.nᵣ = x[5];
    # parameters.scattering_params.rt_aerosols[1].aerosol.nᵢ = x[6];

    # parameters.scattering_params.rt_aerosols[1].p₀ = x[7];
    # parameters.scattering_params.rt_aerosols[1].σp = x[8];

    # Set profiles properly
    met = Dataset(oco_met_file);
    T_met = met.group["Meteorology"]["temperature_profile_met"][:,fp,iOrbit];
    ak = met.group["MeteorologyDiagnostics"]["ak"][:,fp,iOrbit];
    bk = met.group["MeteorologyDiagnostics"]["bk"][:,fp,iOrbit];
    p_surf = met.group["Meteorology"]["surface_pressure_met"][fp,iOrbit];
    p_half = (ak + bk * p_surf);
    p_half = vcat(p_half, p_surf);
    q = met.group["Meteorology"]["specific_humidity_profile_met"][:,fp,iOrbit];
    parameters.p = p_half / 100
    parameters.q = q # zeros(size(q))
    parameters.T = T_met
    parameters.sza = sza_
    parameters.vza = vza_

    @show parameters.sza, parameters.vza

    # @show parameters.sza
    # plot!(parameters.p)
    # return

    model = model_from_parameters(parameters);

    # Run the model to obtain reflectance matrix
    R = vSmartMOM.rt_run(model, i_band=1);

    # Produce black-body in wavenumber range
    T = 5777
    λ_grid = reverse(1e4 ./ parameters.spec_bands[1]) #collect(757:0.01:777)
    black_body = planck_spectrum_wl(T, λ_grid) * 2.1629e-05 * pi
    black_body = SolarModel.watts_to_photons(λ_grid, black_body)

    # Get solar transmittance spectrum
    solar_transmission = solar_transmission_from_file("/home/rjeyaram/RadiativeTransfer/src/solar_merged_20160127_600_26316_100.out", parameters.spec_bands[1])

    # Get outgoing solar radiation
    sun_out = reverse(solar_transmission) .* black_body

    # Apply Earth reflectance matrix
    earth_out = sun_out .* reverse(R[:])

    # Set up InstrumentOperator
    ils_Δ, ils_in, dispersion = InstrumentOperator.read_ils_table(oco_file, ils_file);

    # Define model grid:
    res = 0.001

    # Just consider the ILS within ± 0.35nm
    grid_x = -0.35e-3:res*1e-3:0.35e-3

    extended_dims = [fp,band] # Footprint, band

    # Re-interpolate I from ν_grid to new grid/resolution
    interp_I = LinearInterpolation(λ_grid, earth_out);
    wl = 757.5:res:771.0
    I_wl = interp_I(wl/1000)

    # Pixels to be used
    ind_out = collect(0:1015);

    # Eventual grid of OCO-2 for Band 1, FP 5:
    dispPoly = Polynomial(view(dispersion, :, extended_dims...))
    ν = Float32.(dispPoly.(1:1016))

    # Prepare ILS table:
    ils_pixel   = InstrumentOperator.prepare_ils_table(grid_x, ils_in, ils_Δ,extended_dims)
    oco2_kernel = VariableKernelInstrument(ils_pixel, ν, ind_out)

    # Convolve input spectrum with variable kernel
    I_conv = conv_spectra_local(oco2_kernel, wl./1e3, I_wl)

    y[:] = I_conv[:]

end

directory = "/net/fluo/data1/group/oco2/L1bSc/"
files_list = filter(x->endswith(x, ".h5") && startswith(x, "oco2_L1bScGL_26"), readdir(directory))

met_files_list = filter(x->endswith(x, ".h5"), readdir("/net/fluo/data1/group/oco2/L2Met/"))

I_convs_all = zeros(1016, length(files_list))
oco2_Abands_all = zeros(1016, length(files_list))

albedos = zeros(length(files_list))
cloud_levels = zeros(length(files_list))

optimal_x = zeros(8, length(files_list))

for i in 1:length(files_list)

    @show i

    oco_file = files_list[i]

    # try
        ocoData = Dataset(directory * oco_file)

        oco_met_file = "/net/fluo/data1/group/oco2/L2Met/" * filter(x->contains(x, oco_file[14:19]), met_files_list)[1]

        oco_file = directory * oco_file
        # @show oco_file
        # @show oco_met_file

        met = Dataset(oco_met_file)
        cloud_levels[i] = met.group["Meteorology"]["cloud_liquid_water_path_met"][fp,iOrbit]

        # println()

        # continue

        ## Getting an OCO spectrum for fun to fit:
        o2AbandSpectra = ocoData.group["SoundingMeasurements"]["radiance_o2"]
        SoundingGeometry = ocoData.group["SoundingGeometry"]
        vza = SoundingGeometry["sounding_zenith"]
        sza = SoundingGeometry["sounding_solar_zenith"]
        lat = SoundingGeometry["sounding_latitude"]
        lon = SoundingGeometry["sounding_longitude"]
        o2AbandSpectra = ocoData.group["SoundingMeasurements"]["radiance_o2"]
        oco2_Aband = o2AbandSpectra[:,fp,iOrbit]

        println("$(round(Float64(lat[fp,iOrbit]), digits=2)) $(round(Float64(lon[fp,iOrbit]), digits=4))")
        # continue

        sza_ = sza[fp,iOrbit]
        vza_ = [vza[fp,iOrbit]]

        x = FT[0.05,
        0.05]#,
        # 1.3,
        # 2.0,
        # 1.3,
        # 0.00000001,
        # 90000,
        # 5000.0]

        # Run FW model:
        # @time runner(x);
        I_conv = zeros(1016)
        dfdx = ForwardDiff.jacobian(runner!, I_conv, x);

        dfdx[:,2] .= 0;

        @show size(dfdx)

        y = oco2_Aband;
        Fx = I_conv;
        K = dfdx;

        # Regular least squares:
        dx = K \ (y-Fx)

        I_convs_all[:,i] = I_conv
        oco2_Abands_all[:,i] = oco2_Aband

        # optimal_x[:,i] = x+dx
        albedos[i] = x[1] + dx[1]

        # @show dx, x+dx

    # catch
    #     println("Error on " * oco_file)
    #     continue
    # end

end


# simulate.jl: functions to simulate the model
# include("setup.jl")
function evolve!( pop::Population, generations::Int64 = 1 )
    # from ReputationSets https://github.com/tkessinger/reputation_sets
    # the main function to evolve the population
    # generations specifies the total number of generations for which to run the simulation
    for i::Int64 in 1:generations
        if pop.verbose println("\ninitiating generation $(pop.generation)") end

        # first, update payoffs
        if pop.verbose println("updating actions and payoffs") end
        update_payoffs_using_matrix!(pop)  # method that stores payoffs in a matrix

        # then, select a learner and a role model and perform update
        if pop.verbose println("") end
        if pop.verbose println("evolving, generation $(pop.generation)") end
        update_strategies_and_opinions_db!(pop)

        if pop.verbose println("") end
        if pop.verbose println("ending generation $(pop.generation)") end
        pop.generation += 1
    end
end

function update_payoffs_using_matrix!( pop::Population )
    # update individual payoffs by computing pairwise payoffs and storing them in a matrix

    # to make the code run faster, we only update payoffs in a given generation
    # if strategies or sets have changed. sets_changed and strategies_changed are
    # initialized as true so that payoffs are computed in the first round
    if pop.sets_changed
        # if (previous) learner's sets have changed, re-compute all payoffs
        if pop.verbose println("updating all payoffs because set_changed = $(pop.sets_changed)") end
        pop.payoffs_mat  .= zero(Float64) # reset all payoffs_mat
        pop.prev_actions .= zero(Int64)   # reset all prev_actions

        # then, recompute all payoffs
        for k in 1:pop.sets.M  # for each set
            for (i, j) in pop.sets.set_pairs[k]   # for each pair within each set
                compute_payoffs!( pop, i, j, k )  # compute payoffs for i & j when intereacting in set k
                                                  # this also updates pop.prev_actions accordingly
            end
        end

    elseif pop.strategies_changed
        # if (previous) learner's strategy has changed, recompute payoffs involving that individual
        if pop.verbose println("updating payoffs involving $(pop.prev_learner) because strategies_changed = $(pop.strategies_changed)") end
        pop.payoffs_mat[pop.prev_learner,:]    .= zero(Float64) # first, reset payoffs involving prev_learner
        pop.payoffs_mat[:,pop.prev_learner]    .= zero(Float64)
        pop.prev_actions[:,pop.prev_learner,:] .= zero(Int64)
        pop.prev_actions[:,:,pop.prev_learner] .= zero(Int64)

        # then, recompute payoffs involving prev_learner
        for k in 1:pop.sets.M  # for each set
            for (i, j) in pop.sets.set_pairs[k]  # for each pair within each set
                if i == pop.prev_learner || j == pop.prev_learner
                    compute_payoffs!( pop, i, j, k )  # compute payoffs for i & j when intereacting in set k
                                                      # this also updates pop.prev_actions accordingly
                end
            end
        end
    else
        if pop.verbose println("not updating payoffs b/c sets_changed = $(pop.sets_changed) and strategies_changed = $(pop.strategies_changed)") end
        return # no need to update prev_interactions or payoffs
    end

    # sum rows to get total individual payoffs
    pop.payoffs = [round( x, digits=13 ) for x in vec( sum(pop.payoffs_mat, dims=2) )]  # rounding avoids precision errors
    pop.prev_interactions = 2 * sum([length(x) for x in pop.sets.set_pairs])

    if pop.verbose println( "\npayoff matrix is now $(pop.payoffs_mat)" ) end
    if pop.verbose println( "payoffs are now $(pop.payoffs)" ) end
    if pop.verbose println( "there were $(pop.prev_interactions) interactions, $(sum(pop.prev_actions)) of which were C" ) end
end

function compute_payoffs!( pop::Population, i::Int64, j::Int64, k::Int64 )
    # function to compute payoffs for individuals i and j when they interact in set k
    pop.sim.i_action, pop.sim.j_action = -1, -1  # set to -1 to catch any errors in action assignment

    # actions depend on whether i and j agree on issue k or not
    if pop.sets.h[i,k] == 0 || pop.sets.h[j,k] == 0
        # if either i or j does not belong to set k, something's wrong
        # leave pop.sim.i_action, pop.sim.j_action as -1 so that the compute_payoffs! throws an error when assigning payoffs
        @warn("$i and $j should not be interacting in set $k"); return
    elseif pop.sets.h[i,k] == pop.sets.h[j,k]
        pop.sim.i_action, pop.sim.j_action = pop.strategies[i,1], pop.strategies[j,1] # if they agree, play s_agree
    else
        pop.sim.i_action, pop.sim.j_action = pop.strategies[i,2], pop.strategies[j,2] # if they disagree, play s_disagree
    end

    if pop.verbose
        println("\n\tupdating actions of $i and $j in set $k; agree on issue $k? $(pop.sets.h[i,k] == pop.sets.h[j,k])")
        println("\t\t$i has strats $(pop.strategies[i,:]); $i plays strat $(pop.sim.i_action), earns a payoff of $(pop.game.A[pop.sim.i_action+1, pop.sim.j_action+1])")
        println("\t\t$j has strats $(pop.strategies[j,:]); $j plays strat $(pop.sim.j_action), earns a payoff of $(pop.game.A[pop.sim.j_action+1, pop.sim.i_action+1])")
    end

    # add i and j's payoffs to the payoff matrix according to their strategies and the game matrix
    pop.payoffs_mat[i,j] += pop.game.A[pop.sim.i_action+1, pop.sim.j_action+1] # +1 because julia is 1-indexed
    pop.payoffs_mat[j,i] += pop.game.A[pop.sim.j_action+1, pop.sim.i_action+1]

    # track actions
    pop.prev_actions[k,i,j] = pop.sim.i_action
    pop.prev_actions[k,j,i] = pop.sim.j_action
end

function update_strategies_and_opinions_db!( pop::Population )
    # function to perform death-birth update
    if pop.verbose println("Strategies before updating:\t $(pop.strategies)") end
    if pop.verbose println("Set memberships before updating: $(pop.sets.h)") end

    # randomly choose a learner and store a list of non_learners
    pop.sim.learner = sample( 1:pop.sets.N )

    # compute the fitness of every individual in the population
    for i in 1:pop.sets.N pop.sim.fitnesses[i] = 1.0 + pop.game.β * pop.payoffs[i] end
    if any(x->x<0.0, pop.sim.fitnesses) throw("!!! negative fitness detected, aborting !!!"); end

    # choose a role model from the rest of the population, weighted by fitness
    # OLD: pop.sim.role_model = sample( pop.sim.non_learners, Weights(pop.sim.fitnesses) )
    pop.sim.role_model = sample( 1:pop.sets.N, Weights(pop.sim.fitnesses) )

    # learner updates
    if pop.verbose println("\n\trandomly chosen learner is $(pop.sim.learner), role_model is $(pop.sim.role_model)") end
    if pop.verbose println("\t\tall payoffs are $(pop.payoffs)") end
    if pop.verbose println("\t\tfitnesses are $(pop.sim.fitnesses)") end

    # compute probability of imitation: 1 if same party, 1-p if different parties
    pop.sets.affils[pop.sim.learner] == pop.sets.affils[pop.sim.role_model] ? imit_prob = 1.0 : imit_prob = 1-pop.game.ps[pop.sets.affils[pop.sim.learner]]

    if pop.verbose
        println("\t\tlearner and role model in the same party? $(pop.sets.affils[ pop.sim.learner ] == pop.sets.affils[ pop.sim.role_model ]), imitate with prob $(imit_prob)")
    end

    # update strategies and sets: imitation / mutation occurs with probability imit_prob
    if rand() < imit_prob
        if pop.verbose println("\twill consider imitating") end
        update_strategies!( pop ) # includes mutation
        update_opinions!( pop )   # includes mutation
        pop.num_imitating += 1
    else
        if pop.verbose println("\tnah won't imitate") end
        no_strategy_or_opinion_update!( pop )
        pop.num_not_imitating += 1
    end

    # track updates
    pop.prev_learner = pop.sim.learner
    if pop.sets_changed == true
        # h is updated above; need to update h_bar, h_base10, h_bar_base10, set_members, set_pairs
        pop.sets.set_members, pop.sets.set_pairs = set_members_and_pairs(pop.sets.h)  # update set_members and set_pairs
        pop.sets.h_bar        = [abs(x) for x in pop.sets.h]
        pop.sets.h_base10     = [vectobase3(pop.sets.h[i,:]) for i in 1:pop.sets.N]
        pop.sets.h_bar_base10 = [vectobase3(pop.sets.h_bar[i,:]) for i in 1:pop.sets.N]
        pop.num_sets_changed += 1
    end
    if pop.strategies_changed == true
        # strategies are updated above; need to update strategies_base10
        pop.strategies_base10 = [2 * pop.strategies[i,1] + pop.strategies[i,2] for i in 1:pop.sets.N]
        pop.num_strategies_changed += 1
    end
    if pop.strategies_changed && pop.sets_changed
        pop.num_both_changed += 1
    end

    if pop.verbose println("\nStrategies after updating:\t $(pop.strategies)") end
    if pop.verbose println("Set memberships after updating:\t $(pop.sets.h)") end
    if pop.verbose
        println("tracking: prev_learner = $(pop.prev_learner), strategies_changed = $(pop.strategies_changed), sets_changed = $(pop.sets_changed)")
    end
end

function update_strategies!( pop::Population )
    # function to update strategy of learner based on role_model
    pop.sim.strategy_copy = copy(pop.strategies[pop.sim.learner,:])  # save a copy to track changes
    # copy suffices here because the assignments below replace elements of pop.sets.h with Int64 (immutable)

    if rand() < pop.game.u
        # strategy mutation occurs
        pop.strategies[pop.sim.learner,:] = rand(collect(0:1), 1, 2) # conditional case
        pop.num_random_strategies += 1
        pop.num_random_sa_flipped += (pop.sim.strategy_copy[1] != pop.strategies[pop.sim.learner,1])
        pop.num_random_sd_flipped += (pop.sim.strategy_copy[2] != pop.strategies[pop.sim.learner,2])
        if pop.verbose println("\t\tmutating $(pop.sim.learner) to strategy $(pop.strategies[pop.sim.learner,:])") end
    else
        # learner imitates the strategy of role model
        pop.strategies[pop.sim.learner,:] = pop.strategies[pop.sim.role_model,:]
        if pop.verbose println("\t\tadopting $(pop.sim.role_model)'s strategy $(pop.strategies[pop.sim.role_model,:])") end
    end
    pop.strategies_changed = (pop.sim.strategy_copy != pop.strategies[pop.sim.learner,:]) # track changes
end

function update_opinions!( pop::Population )
    # function to update set memberships / opinions of learner based on role_model
    pop.sim.sets_copy = copy(pop.sets.h[pop.sim.learner,:])     # save a copy to track changes
    # copy suffices here because the assignments below replace elements of pop.sets.h with Int64 (immutable)

    # with probability v, set mutation occurs
    if rand() < pop.game.v
        if rand() < 1 - pop.game.ϵ * pop.game.ps[pop.sets.affils[pop.sim.learner]]
            # with probability 1-p*ϵ, select opinions without bias
            # select a random arrangement of K opinions
            pop.sets.h[pop.sim.learner,:] = shuffle( vcat( zeros(Int64, pop.sets.M-pop.sets.K), rand([-1,1], pop.sets.K) ) )

            pop.num_random_opinions += 1
            pop.num_random_opinions_flipped += (pop.sim.sets_copy != pop.sets.h[ pop.sim.learner,: ]) # new
            if pop.verbose println("\t\tmutating $(pop.sim.learner) to random opinions $(pop.sets.h[pop.sim.learner,:])") end
        else
            # with probability p*ϵ, select opinions with bias
            # select a random set of K issues, select opinions corresponding to party
            if pop.sets.affils[pop.sim.learner] == 1
                pop.sets.h[pop.sim.learner,:] = shuffle( vcat( zeros(Int64, pop.sets.M-pop.sets.K), -ones(Int64, pop.sets.K) ) )
            elseif pop.sets.affils[pop.sim.learner] == 2
                pop.sets.h[pop.sim.learner,:] = shuffle( vcat( zeros(Int64, pop.sets.M-pop.sets.K), ones(Int64, pop.sets.K) ) )
            end
            pop.num_biased_opinions += 1
            pop.num_biased_opinions_flipped += (pop.sim.sets_copy != pop.sets.h[ pop.sim.learner,: ]) # new
            if pop.verbose
                println("\t\t$(pop.sim.learner) is in party $(pop.sets.affils[ pop.sim.learner ]), mutating to biased opinions $(pop.sets.h[ pop.sim.learner,: ])")
            end
        end
    else
        # with probability 1-v, learner imitates the strategy of role model
        pop.sets.h[ pop.sim.learner,: ] = pop.sets.h[ pop.sim.role_model,: ]
        if pop.verbose println("\t\tadopting $( pop.sim.role_model )'s set membership $(pop.sets.h[ pop.sim.role_model,: ])") end
    end
    # h is updated above; need to update h_bar, sets_changed, set_members, set_pairs
    # pop.sets.set_members, pop.sets.set_pairs = set_members_and_pairs(pop.sets.h)  # update set_members and set_pairs # moved up
    pop.sets_changed = (pop.sim.sets_copy != pop.sets.h[ pop.sim.learner,: ]) # track changes
end

function no_strategy_or_opinion_update!( pop::Population )
    # function to keep track of changes when no imitation/mutation occurs
    pop.strategies_changed = false  # track changes
    pop.sets_changed       = false
end


##### Get prognostic variable list

import ..BalanceLaws: prognostic_vars

prognostic_vars(::DryModel) = ()
prognostic_vars(::EquilMoist) = (TotalMoisture(),)
prognostic_vars(::NonEquilMoist) =
    (TotalMoisture(), LiquidMoisture(), IceMoisture())

prognostic_vars(::NoPrecipitation) = ()
prognostic_vars(::RainModel) = (Rain(),)
prognostic_vars(::RainSnowModel) = (Rain(), Snow())

prognostic_vars(::NoTracers) = ()
prognostic_vars(::NTracers{N}) where {N} = (Tracers{N}(),)

prognostic_vars(m::AtmosModel) = (
    Mass(),
    Momentum(),
    Energy(),
    prognostic_vars(m.moisture)...,
    prognostic_vars(m.precipitation)...,
    prognostic_vars(m.tracers)...,
    prognostic_vars(m.turbconv)...,
)


#
# Powell.jl --
#
# Mike Powell's derivative free optimization methods for Julia.
#
# ----------------------------------------------------------------------------
#
# This file is part of OptimPack.jl which is licensed under the MIT
# "Expat" License:
#
# Copyright (C) 2015, Éric Thiébaut.
#
# ----------------------------------------------------------------------------

# Possible status values returned by COBYLA.
const COBYLA_INITIAL_ITERATE       = convert(Cint,  2)
const COBYLA_ITERATE               = convert(Cint,  1)
const COBYLA_SUCCESS               = convert(Cint,  0)
const COBYLA_ROUNDING_ERRORS       = convert(Cint, -1)
const COBYLA_TOO_MANY_EVALUATIONS  = convert(Cint, -2)
const COBYLA_BAD_ADDRESS           = convert(Cint, -3)
const COBYLA_CORRUPTED             = convert(Cint, -4)

# Get a textual explanation of the status returned by COBYLA.
function cobyla_reason(status::Integer)
    ptr = ccall((:cobyla_reason, opklib), Ptr{UInt8}, (Cint,), status)
    if ptr == C_NULL
        error("unknown COBYLA status: ", status)
    end
    bytestring(ptr)
end

# Wrapper for the objective function in COBYLA, the actual objective
# function is provided by the client data.
function cobyla_objfun(n::Cptrdiff_t, m::Cptrdiff_t, x_::Ptr{Cdouble},
                       c_::Ptr{Cdouble}, f_::Ptr{Any})
    x = pointer_to_array(x_, n)
    f = unsafe_pointer_to_objref(f_)
    fx::Cdouble = (m > 0 ? f(x, pointer_to_array(c_, m)) : f(x))
    return fx
end
const cobyla_objfun_c = cfunction(cobyla_objfun, Cdouble,
                                  (Cptrdiff_t, Cptrdiff_t, Ptr{Cdouble},
                                   Ptr{Cdouble}, Ptr{Any}))

function cobyla_check(n::Integer, m::Integer, rhobeg::Real, rhoend::Real)
    if n < 2
        "bad number of variables"
    elseif m < 0
        "bad number of constraints"
    elseif rhoend < 0 || rhoend > rhobeg
        "bad trust region radius settings"
    end
end

function cobyla!(f::Function, x::Vector{Cdouble},
                 m::Integer, rhobeg::Real, rhoend::Real;
                 verbose::Integer=0, maxeval::Integer=500)
    n = length(x)
    reason = cobyla_check(n, m, rhobeg, rhoend)
    reason == nothing || error(reason)
    w = Array(Cdouble, n*(3*n + 2*m + 11) + 4*m + 6)
    iact = Array(Cptrdiff_t, m + 1)
    maxfun = Cptrdiff_t[maxeval]
    status = ccall((:cobyla, opklib), Cint,
                   (Cptrdiff_t, Cptrdiff_t, Ptr{Void}, Ptr{Any},
                    Ptr{Cdouble}, Cdouble, Cdouble, Cptrdiff_t,
                    Ptr{Cptrdiff_t}, Ptr{Cdouble}, Ptr{Cptrdiff_t}),
                   n, m, cobyla_objfun_c, pointer_from_objref(f),
                   x, rhobeg, rhoend, verbose, maxfun, w, iact)
    if status != COBYLA_SUCCESS
        error(cobyla_reason(status))
    end
    #return w[1]
end

# Context for reverse communication variant of COBYLA.
type CobylaContext
    ptr::Ptr{Void}
    n::Int
    m::Int
    rhobeg::Cdouble
    rhoend::Cdouble
    verbose::Int
    maxeval::Int
end

# Create a new reverse communication workspace for COBYLA algorithm.
# A typical usage is:
# ```
# x = Array(Cdouble, n)
# c = Array(Cdouble, m)
# x[...] = ... # initial solution
# ctx = cobyla_create(n, m, rhobeg, rhoend, verbose=1, maxeval=500)
# status = cobyla_get_status(ctx)
# while status == COBYLA_ITERATE
#     fx = ...       # compute function value at X
#     c[...] = ...   # compute constraints at X
#     status = cobyla_iterate(ctx, fx, x, c)
# end
# if status != COBYLA_SUCCESS
#     println("Something wrong occured in COBYLA: ", cobyla_reason(status))
# end
# ```
function cobyla_create(n::Integer, m::Integer,
                       rhobeg::Real, rhoend::Real;
                       verbose::Integer=0, maxeval::Integer=500)
    reason = cobyla_check(n, m, rhobeg, rhoend)
    reason == nothing || error(reason)
    ptr = ccall((:cobyla_create, opklib), Ptr{Void},
                (Cptrdiff_t, Cptrdiff_t, Cdouble, Cdouble,
                 Cptrdiff_t, Cptrdiff_t),
                n, m, rhobeg, rhoend, verbose, maxeval)
    if ptr == C_NULL
        reason = (errno() == Base.Errno.ENOMEM
                  ? "insufficient memory"
                  : "unexpected error")
        error(reason)
    end
    ctx = CobylaContext(ptr, n, m, rhobeg, rhoend, verbose, maxeval)
    finalizer(ctx, ctx -> ccall((:cobyla_delete, opklib), Void,
                                (Ptr{Void},), ctx.ptr))
    return ctx
end

# Perform the next iteration of the reverse communication variant of the
# COBYLA algorithm.  On entry, the workspace status must be
# `COBYLA_ITERATE`, `f` and `c` are the function value and the constraints
# at `x`.  On exit, the returned value (the new workspace status) is:
# `COBYLA_ITERATE` if a new trial point has been stored in `x` and if user
# is requested to compute the function value and the constraints on the new
# point; `COBYLA_SUCCESS` if algorithm has converged and `x` has been set
# with the variables at the solution (the corresponding function value can
# be retrieved with `cobyla_get_last_f`); anything else indicates an error
# (see `cobyla_reason` for an explanatory message).
function cobyla_iterate(ctx::CobylaContext, f::Real, x::Vector{Cdouble},
                        c::Vector{Cdouble})
    length(x) == ctx.n || error("bad number of variables")
    length(c) == ctx.m || error("bad number of constraints")
    ccall((:cobyla_iterate, opklib), Cint,
          (Ptr{Void}, Cdouble, Ptr{Cdouble}, Ptr{Cdouble}),
          ctx.ptr, f, x, c)
end
# The same but without constraints.
function cobyla_iterate(ctx::CobylaContext, f::Real, x::Vector{Cdouble})
    length(x) == ctx.n || error("bad number of variables")
    ctx.m == 0 || error("bad number of constraints")
    ccall((:cobyla_iterate, opklib), Cint,
          (Ptr{Void}, Cdouble, Ptr{Cdouble}, Ptr{Void}),
          ctx.ptr, f, x, C_NULL)
end

# Restart COBYLA algorithm using the same parameters.  The return value is
# the new status of the algorithm, see `cobyla_get_status` for details.
function cobyla_restart(ctx::CobylaContext)
    ccall((:cobyla_restart, opklib), Cint, (Ptr{Void},), ctx.ptr)
end

# Get the current status of the algorithm.  Result is: `COBYLA_ITERATE` if
# user is requested to compute F(X) and C(X); `COBYLA_SUCCESS` if algorithm
# has converged; anything else indicates an error (see `cobyla_reason` for
# an explanatory message).
function cobyla_get_status(ctx::CobylaContext)
    ccall((:cobyla_get_status, opklib), Cint, (Ptr{Void},), ctx.ptr)
end

# Get the current number of function evaluations.  Result is -1 if
# something is wrong (e.g. CTX is NULL), nonnegative otherwise.
function cobyla_get_nevals(ctx::CobylaContext)
    ccall((:cobyla_get_nevals, opklib), Cptrdiff_t, (Ptr{Void},), ctx.ptr)
end

# Get the current size of the trust region.  Result is 0 if algorithm not
# yet started (before first iteration), -1 if something is wrong (e.g. CTX
# is NULL), strictly positive otherwise.
function cobyla_get_rho(ctx::CobylaContext)
    ccall((:cobyla_get_rho, opklib), Cdouble, (Ptr{Void},), ctx.ptr)
end

# Get the last function value.  Upon convergence of `cobyla_iterate`
# (i.e. return with status `COBYLA_SUCCESS`), this value corresponds to the
# function at the solution; otherwise, this value corresponds to the
# previous set of variables.
function cobyla_get_last_f(ctx::CobylaContext)
    ccall((:cobyla_get_last_f, opklib), Cdouble, (Ptr{Void},), ctx.ptr)
end

# Get a textual explanation of the current status.
cobyla_get_reason(ctx::CobylaContext) = cobyla_reason(cobyla_get_status(ctx))

function cobyla_test(revcom::Bool=false)
    # Beware that order of operations may affect the result (whithin
    # rounding errors).  I have tried to keep the same ordering as F2C
    # which takes care of that, in particular when converting expressions
    # involving powers.
    prt(s) = println("\n       "*s)
    for nprob in 1:10
        if nprob == 1
            # Minimization of a simple quadratic function of two variables.
            prt("Output from test problem 1 (Simple quadratic)")
            n = 2
            m = 0
            xopt = Array(Cdouble, n)
            xopt[1] = -1.0
            xopt[2] = 0.0
            function ftest(x::Vector{Cdouble})
                r1 = x[1] + 1.0
                r2 = x[2]
                fc = 10.0*(r1*r1) + (r2*r2)
                return fc
            end
        elseif nprob == 2
            # Easy two dimensional minimization in unit circle.
            prt("Output from test problem 2 (2D unit circle calculation)")
            n = 2
            m = 1
            xopt = Array(Cdouble, n)
            xopt[1] = sqrt(0.5)
            xopt[2] = -xopt[1]
            function ftest(x::Vector{Cdouble}, con::Vector{Cdouble})
                fc = x[1]*x[2]
                con[1] = 1.0 - x[1]*x[1] - x[2]*x[2]
                return fc
            end
        elseif nprob == 3
            # Easy three dimensional minimization in ellipsoid.
            prt("Output from test problem 3 (3D ellipsoid calculation)")
            n = 3
            m = 1
            xopt = Array(Cdouble, n)
            xopt[1] = 1.0/sqrt(3.0)
            xopt[2] = 1.0/sqrt(6.0)
            xopt[3] = -0.33333333333333331
            function ftest(x::Vector{Cdouble}, con::Vector{Cdouble})
                fc = x[1]*x[2]*x[3]
                con[1] = 1.0 - (x[1]*x[1]) - 2.0*(x[2]*x[2]) - 3.0*(x[3]*x[3])
                return fc
            end
        elseif nprob == 4
            # Weak version of Rosenbrock's problem.
            prt("Output from test problem 4 (Weak Rosenbrock)")
            n = 2
            m = 0
            xopt = Array(Cdouble, n)
            xopt[1] = -1.0
            xopt[2] = 1.0
            function ftest(x::Vector{Cdouble})
                r2 = x[1]
                r1 = r2*r2 - x[2]
                r3 = x[1] + 1.0
                fc = r1*r1 + r3*r3
                return fc
            end
        elseif nprob == 5
            # Intermediate version of Rosenbrock's problem.
            prt("Output from test problem 5 (Intermediate Rosenbrock)")
            n = 2
            m = 0
            xopt = Array(Cdouble, n)
            xopt[1] = -1.0
            xopt[2] = 1.0
            function ftest(x::Vector{Cdouble})
                r2 = x[1]
                r1 = r2*r2 - x[2]
                r3 = x[1] + 1.0
                fc = r1*r1*10.0 + r3*r3
                return fc
            end
        elseif nprob == 6
            # This problem is taken from Fletcher's book Practical Methods
            # of Optimization and has the equation number (9.1.15).
            prt("Output from test problem 6 (Equation (9.1.15) in Fletcher)")
            n = 2
            m = 2
            xopt = Array(Cdouble, n)
            xopt[1] = sqrt(0.5)
            xopt[2] = xopt[1]
            function ftest(x::Vector{Cdouble}, con::Vector{Cdouble})
                fc = -x[1] - x[2]
                r1 = x[1]
                con[1] = x[2] - r1*r1
                r1 = x[1]
                r2 = x[2]
                con[2] = 1.0 - r1*r1 - r2*r2
                return fc
            end
        elseif nprob == 7
            # This problem is taken from Fletcher's book Practical Methods
            # of Optimization and has the equation number (14.4.2).
            prt("Output from test problem 7 (Equation (14.4.2) in Fletcher)")
            n = 3
            m = 3
            xopt = Array(Cdouble, n)
            xopt[1] = 0.0
            xopt[2] = -3.0
            xopt[3] = -3.0
            function ftest(x::Vector{Cdouble}, con::Vector{Cdouble})
                fc = x[3]
                con[1] = x[1]*5.0 - x[2] + x[3]
                r1 = x[1]
                r2 = x[2]
                con[2] = x[3] - r1*r1 - r2*r2 - x[2]*4.0
                con[3] = x[3] - x[1]*5.0 - x[2]
                return fc
            end
        elseif nprob == 8
            # This problem is taken from page 66 of Hock and Schittkowski's
            # book Test Examples for Nonlinear Programming Codes. It is
            # their test problem Number 43, and has the name Rosen-Suzuki.
            prt("Output from test problem 8 (Rosen-Suzuki)")
            n = 4
            m = 3
            xopt = Array(Cdouble, n)
            xopt[1] = 0.0
            xopt[2] = 1.0
            xopt[3] = 2.0
            xopt[4] = -1.0
            function ftest(x::Vector{Cdouble}, con::Vector{Cdouble})
                r1 = x[1]
                r2 = x[2]
                r3 = x[3]
                r4 = x[4]
                fc = (r1*r1 + r2*r2 + r3*r3*2.0 + r4*r4 - x[1]*5.0
                      - x[2]*5.0 - x[3]*21.0 + x[4]*7.0)
                r1 = x[1]
                r2 = x[2]
                r3 = x[3]
                r4 = x[4]
                con[1] = (8.0 - r1*r1 - r2*r2 - r3*r3 - r4*r4 - x[1]
                          + x[2] - x[3] + x[4])
                r1 = x[1]
                r2 = x[2]
                r3 = x[3]
                r4 = x[4]
                con[2] = (10.0 - r1*r1 - r2*r2*2.0 - r3*r3 - r4*r4*2.0
                          + x[1] + x[4])
                r1 = x[1]
                r2 = x[2]
                r3 = x[3]
                con[3] = (5.0 - r1*r1*2.0 - r2*r2 - r3*r3 - x[1]*2.0
                          + x[2] + x[4])
                return fc
            end
        elseif nprob == 9
            # This problem is taken from page 111 of Hock and
            # Schittkowski's book Test Examples for Nonlinear Programming
            # Codes. It is their test problem Number 100.
            prt("Output from test problem 9 (Hock and Schittkowski 100)")
            n = 7
            m = 4
            xopt = Array(Cdouble, n)
            xopt[1] =  2.330499
            xopt[2] =  1.951372
            xopt[3] = -0.4775414
            xopt[4] =  4.365726
            xopt[5] = -0.624487
            xopt[6] =  1.038131
            xopt[7] =  1.594227
            function ftest(x::Vector{Cdouble}, con::Vector{Cdouble})
                r1 = x[1] - 10.0
                r2 = x[2] - 12.0
                r3 = x[3]
                r3 *= r3
                r4 = x[4] - 11.0
                r5 = x[5]
                r5 *= r5
                r6 = x[6]
                r7 = x[7]
                r7 *= r7
                fc = (r1*r1 + r2*r2*5.0 + r3*r3 + r4*r4*3.0
                      + r5*(r5*r5)*10.0 + r6*r6*7.0 + r7*r7
                      - x[6]*4.0*x[7] - x[6]*10.0 - x[7]*8.0)
                r1 = x[1]
                r2 = x[2]
                r2 *= r2
                r3 = x[4]
                con[1] = (127.0 - r1*r1*2.0 - r2*r2*3.0 - x[3]
                          - r3*r3*4.0 - x[5]*5.0)
                r1 = x[3]
                con[2] = (282.0 - x[1]*7.0 - x[2]*3.0 - r1*r1*10.0
                          - x[4] + x[5])
                r1 = x[2]
                r2 = x[6]
                con[3] = (196.0 - x[1]*23.0 - r1*r1 - r2*r2*6.0
                          + x[7]*8.0)
                r1 = x[1]
                r2 = x[2]
                r3 = x[3]
                con[4] = (r1*r1*-4.0 - r2*r2 + x[1]*3.0*x[2]
                          - r3*r3*2.0 - x[6]*5.0 + x[7]*11.0)
                return fc
            end
        elseif nprob == 10
            # This problem is taken from page 415 of Luenberger's book
            # Applied Nonlinear Programming. It is to maximize the area of
            # a hexagon of unit diameter.
            prt("Output from test problem 10 (Hexagon area)")
            n = 9
            m = 14
            xopt = Array(Cdouble, n)
            function ftest(x::Vector{Cdouble}, con::Vector{Cdouble})
                fc = -0.5*(x[1]*x[4] - x[2]*x[3] + x[3]*x[9] - x[5]*x[9]
                           + x[5]*x[8] - x[6]*x[7])
                r1 = x[3]
                r2 = x[4]
                con[1] = 1.0 - r1*r1 - r2*r2
                r1 = x[9]
                con[2] = 1.0 - r1*r1
                r1 = x[5]
                r2 = x[6]
                con[3] = 1.0 - r1*r1 - r2*r2
                r1 = x[1]
                r2 = x[2] - x[9]
                con[4] = 1.0 - r1*r1 - r2*r2
                r1 = x[1] - x[5]
                r2 = x[2] - x[6]
                con[5] = 1.0 - r1*r1 - r2*r2
                r1 = x[1] - x[7]
                r2 = x[2] - x[8]
                con[6] = 1.0 - r1*r1 - r2*r2
                r1 = x[3] - x[5]
                r2 = x[4] - x[6]
                con[7] = 1.0 - r1*r1 - r2*r2
                r1 = x[3] - x[7]
                r2 = x[4] - x[8]
                con[8] = 1.0 - r1*r1 - r2*r2
                r1 = x[7]
                r2 = x[8] - x[9]
                con[9] = 1.0 - r1*r1 - r2*r2
                con[10] = x[1]*x[4] - x[2]*x[3]
                con[11] = x[3]*x[9]
                con[12] = -x[5]*x[9]
                con[13] = x[5]*x[8] - x[6]*x[7]
                con[14] = x[9]
                return fc
            end
        else
            error("bad problem number ($nprob)")
        end

        x = Array(Cdouble, n)
        for icase in 1:2
            for i in 1:n
                x[i] = 1.0
            end
            rhobeg = 0.5
            rhoend = (icase == 2 ? 1e-4 : 0.001)
            if revcom
                # Test the reverse communication variant.
                c = (m > 0 ? Array(Cdouble, m) : nothing)
                ctx = cobyla_create(n, m, rhobeg, rhoend, verbose=1, maxeval=2000)
                status = cobyla_get_status(ctx)
                while status == COBYLA_ITERATE
                    if m > 0
                        # Some constraints.
                        fx = ftest(x, c)
                        status = cobyla_iterate(ctx, fx, x, c)
                    else
                        # No constraints.
                        fx = ftest(x)
                        status = cobyla_iterate(ctx, fx, x)
                    end
                end
                if status != COBYLA_SUCCESS
                    println("Something wrong occured in COBYLA: ", cobyla_reason(status))
                end
            else
                cobyla!(ftest, x, m, rhobeg, rhoend, verbose=1, maxeval=2000)
            end
            if nprob == 10
                tempa = x[1] + x[3] + x[5] + x[7]
                tempb = x[2] + x[4] + x[6] + x[8]
                tempc = 0.5/sqrt(tempa*tempa + tempb*tempb)
                tempd = tempc*sqrt(3.0)
                xopt[1] = tempd*tempa + tempc*tempb
                xopt[2] = tempd*tempb - tempc*tempa
                xopt[3] = tempd*tempa - tempc*tempb
                xopt[4] = tempd*tempb + tempc*tempa
                for i in 1:4
                    xopt[i + 4] = xopt[i]
                end
            end
            temp = 0.0
            for i in 1:n
                r1 = x[i] - xopt[i]
                temp += r1*r1
            end
            @printf("\n     Least squares error in variables =%16.6E\n", sqrt(temp))
        end
        @printf("  ------------------------------------------------------------------\n")
    end
end

# ----------------------------------------------------------------------------

# Possible status values returned by NEWUOA.
const NEWUOA_INITIAL_ITERATE      = convert(Cint,  2)
const NEWUOA_ITERATE              = convert(Cint,  1)
const NEWUOA_SUCCESS              = convert(Cint,  0)
const NEWUOA_BAD_NPT              = convert(Cint, -1)
const NEWUOA_ROUNDING_ERRORS      = convert(Cint, -2)
const NEWUOA_TOO_MANY_EVALUATIONS = convert(Cint, -3)
const NEWUOA_STEP_FAILED          = convert(Cint, -4)
const NEWUOA_BAD_ADDRESS          = convert(Cint, -5)
const NEWUOA_CORRUPTED            = convert(Cint, -6)

# Get a textual explanation of the status returned by NEWUOA.
function newuoa_reason(status::Integer)
    ptr = ccall((:newuoa_reason, opklib), Ptr{UInt8}, (Cint,), status)
    if ptr == C_NULL
        error("unknown NEWUOA status: ", status)
    end
    bytestring(ptr)
end

# Wrapper for the objective function in NEWUOA, the actual objective
# function is provided by the client data.
function newuoa_objfun(n::Cptrdiff_t, x_::Ptr{Cdouble}, f_::Ptr{Any})
    x = pointer_to_array(x_, n)
    f = unsafe_pointer_to_objref(f_)
    fx::Cdouble = f(x)
    return fx
end
const newuoa_objfun_c = cfunction(newuoa_objfun, Cdouble,
                                  (Cptrdiff_t, Ptr{Cdouble}, Ptr{Any}))

function newuoa_check(n::Integer, npt::Integer, rhobeg::Real, rhoend::Real)
    if n < 2
        "bad number of variables"
    elseif npt < n + 2 || npt > div((n + 2)*(n + 1),2)
        "NPT is not in the required interval"
    elseif rhoend < 0 || rhoend > rhobeg
        "bad trust region radius settings"
    end
end

function newuoa!(f::Function, x::Vector{Cdouble},
                 npt::Integer, rhobeg::Real, rhoend::Real;
                 verbose::Integer=0, maxeval::Integer=500)
    n = length(x)
    reason = newuoa_check(n, npt, rhobeg, rhoend)
    reason == nothing || error(reason)
    w = Array(Cdouble, (npt + 13)*(npt + n) + div(3*n*(n + 3),2))
    status = ccall((:newuoa, opklib), Cint,
                   (Cptrdiff_t, Cptrdiff_t, Ptr{Void}, Ptr{Any},
                    Ptr{Cdouble}, Cdouble, Cdouble, Cptrdiff_t,
                    Cptrdiff_t, Ptr{Cdouble}),
                   n, npt, newuoa_objfun_c, pointer_from_objref(f),
                   x, rhobeg, rhoend, verbose, maxeval, w)
    if status != NEWUOA_SUCCESS
        error(newuoa_reason(status))
    end
    return nothing
end

# Context for reverse communication variant of NEWUOA.
type NewuoaContext
    ptr::Ptr{Void}
    n::Int
    npt::Int
    rhobeg::Cdouble
    rhoend::Cdouble
    verbose::Int
    maxeval::Int
end

# Create a new reverse communication workspace for NEWUOA algorithm.
# A typical usage is:
# ```
# x = Array(Cdouble, n)
# x[...] = ... # initial solution
# ctx = newuoa_create(n, npt, rhobeg, rhoend, verbose=0, maxeval=500)
# status = newuoa_get_status(ctx)
# while status == NEWUOA_ITERATE
#   fx = ... # compute function value at X
#   status = newuoa_iterate(ctx, fx, x)
# end
# if status != NEWUOA_SUCCESS
#   println("Something wrong occured in NEWUOA: ", newuoa_reason(status))
# end
# ```
function newuoa_create(n::Integer, npt::Integer,
                       rhobeg::Real, rhoend::Real;
                       verbose::Integer=0, maxeval::Integer=500)
    reason = newuoa_check(n, npt, rhobeg, rhoend)
    reason == nothing || error(reason)
    ptr = ccall((:newuoa_create, opklib), Ptr{Void},
                (Cptrdiff_t, Cptrdiff_t, Cdouble, Cdouble,
                 Cptrdiff_t, Cptrdiff_t),
                n, npt, rhobeg, rhoend, verbose, maxeval)
    if ptr == C_NULL
        reason = (errno() == Base.Errno.ENOMEM
                  ? "insufficient memory"
                  : "unexpected error")
        error(reason)
    end
    ctx = NewuoaContext(ptr, n, npt, rhobeg, rhoend, verbose, maxeval)
    finalizer(ctx, ctx -> ccall((:newuoa_delete, opklib), Void,
                                (Ptr{Void},), ctx.ptr))
    return ctx
end

# Perform the next iteration of the reverse communication version of the
# NEWUOA algorithm.  On entry, the wokspace status must be
# `NEWUOA_ITERATE`, `f` is the function value at `x`.  On exit, the
# returned value (the new wokspace status) is: `NEWUOA_ITERATE` if a new
# trial point has been stored in `x` and if user is requested to compute
# the function value for the new point; `NEWUOA_SUCCESS` if algorithm has
# converged; anything else indicates an error (see `newuoa_reason` for an
# explanatory message).
function newuoa_iterate(ctx::NewuoaContext, f::Real, x::Vector{Cdouble})
    length(x) == ctx.n || error("bad number of variables")
    ccall((:newuoa_iterate, opklib), Cint,
          (Ptr{Void}, Cdouble, Ptr{Cdouble}),
          ctx.ptr, f, x)
end

# Restart NEWUOA algorithm using the same parameters.  The return value is the
# new status of the algorithm, see `newuoa_get_status` for details.
function newuoa_restart(ctx::NewuoaContext)
    ccall((:newuoa_restart, opklib), Cint, (Ptr{Void},), ctx.ptr)
end

# Get the current status of the algorithm.  Result is: `NEWUOA_ITERATE` if
# user is requested to compute F(X); `NEWUOA_SUCCESS` if algorithm has
# converged; anything else indicates an error (see `newuoa_reason` for an
# explanatory message).
function newuoa_get_status(ctx::NewuoaContext)
    ccall((:newuoa_get_status, opklib), Cint, (Ptr{Void},), ctx.ptr)
end

# Get the current number of function evaluations.  Result is -1 if
# something is wrong (e.g. CTX is NULL), nonnegative otherwise.
function newuoa_get_nevals(ctx::NewuoaContext)
    ccall((:newuoa_get_nevals, opklib), Cptrdiff_t, (Ptr{Void},), ctx.ptr)
end

# Get the current size of the trust region.  Result is 0 if algorithm not
# yet started (before first iteration), -1 if something is wrong (e.g. CTX
# is NULL), strictly positive otherwise.
function newuoa_get_rho(ctx::NewuoaContext)
    ccall((:newuoa_get_rho, opklib), Cdouble, (Ptr{Void},), ctx.ptr)
end

function newuoa_test(revcom::Bool=false)
    # The Chebyquad test problem (Fletcher, 1965) for N = 2,4,6 and 8, with
    # NPT = 2N+1.
    function ftest(x::Vector{Cdouble})
        n = length(x)
        np = n + 1
        y = Array(Cdouble, np, n)
        for j in 1:n
            y[1,j] = 1.0
            y[2,j] = x[j]*2.0 - 1.0
        end
        for i in 2:n
            for j in 1:n
                y[i+1,j] = y[2,j]*2.0*y[i,j] - y[i-1,j]
            end
        end
        f = 0.0
        iw = 1
        for i in 1:np
            sum = 0.0
            for j in 1:n
                sum += y[i,j]
            end
            sum /= n
            if iw > 0
                sum += 1.0/(i*i - 2*i)
            end
            iw = -iw
            f += sum*sum
        end
        return f
    end

    # Run the tests.
    rhoend = 1e-6
    for n = 2:2:8
        npt = 2*n + 1
        x = Array(Cdouble, n)
        for i in 1:n
            x[i] = i/(n + 1)
        end
        rhobeg = x[1]*0.2
        @printf("\n\n    Results with N =%2d and NPT =%3d\n", n, npt)
        if revcom
            # Test the reverse communication variant.
            ctx = newuoa_create(n, npt, rhobeg, rhoend, verbose=2, maxeval=5000)
            status = newuoa_get_status(ctx)
            while status == NEWUOA_ITERATE
                fx = ftest(x)
                status = newuoa_iterate(ctx, fx, x)
            end
            if status != NEWUOA_SUCCESS
                println("Something wrong occured in NEWUOA: ", newuoa_reason(status))
            end
        else
            newuoa!(ftest, x, npt, rhobeg, rhoend, verbose=2, maxeval=5000)
        end
    end
end

# ----------------------------------------------------------------------------

const BOBYQA_SUCCESS              = convert(Cint,  0)
const BOBYQA_BAD_NPT              = convert(Cint, -1)
const BOBYQA_TOO_CLOSE            = convert(Cint, -2)
const BOBYQA_ROUNDING_ERRORS      = convert(Cint, -3)
const BOBYQA_TOO_MANY_EVALUATIONS = convert(Cint, -4)
const BOBYQA_STEP_FAILED          = convert(Cint, -5)

function bobyqa_reason(status::Integer)
    if status == BOBYQA_SUCCESS
        return "algorithm converged"
    elseif status == BOBYQA_BAD_NPT
        return "NPT is not in the required interval"
    elseif status == BOBYQA_TOO_CLOSE
        return "insufficient space between the bounds"
    elseif status == BOBYQA_ROUNDING_ERRORS
        return "too much cancellation in a denominator"
    elseif status == BOBYQA_TOO_MANY_EVALUATIONS
        return "maximum number of function evaluations exceeded"
    elseif status == BOBYQA_STEP_FAILED
        return "a trust region step has failed to reduce quadratic approximation"
    else
        return "unknown BOBYQA status"
    end
end

# Wrapper for the objective function in BOBYQA, the actual objective
# function is provided by the client data.
function bobyqa_objfun(n::Cptrdiff_t, x_::Ptr{Cdouble}, f_::Ptr{Any})
    x = pointer_to_array(x_, n)
    f = unsafe_pointer_to_objref(f_)
    fx::Cdouble = f(x)
    return fx
end
const bobyqa_objfun_c = cfunction(bobyqa_objfun, Cdouble,
                                  (Cptrdiff_t, Ptr{Cdouble}, Ptr{Any}))

function bobyqa!(f::Function, x::Vector{Cdouble},
                 xl::Vector{Cdouble}, xu::Vector{Cdouble},
                 rhobeg::Real, rhoend::Real;
                 npt::Union{Integer,Void}=nothing,
                 verbose::Integer=0, maxeval::Integer=500)
    n = length(x)
    length(xl) == n || error("bad length for inferior bound")
    length(xu) == n || error("bad length for superior bound")
    if npt == nothing
        npt = 2*n + 1
    end
    w = Array(Cdouble, (npt + 5)*(npt + n) + div(3*n*(n + 5),2))
    status = ccall((:bobyqa, opklib), Cint,
                (Cptrdiff_t, Cptrdiff_t, Ptr{Void}, Ptr{Any},
                 Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cdouble},
                 Cdouble, Cdouble, Cptrdiff_t, Cptrdiff_t,
                 Ptr{Cdouble}),
                   n, npt, bobyqa_objfun_c, pointer_from_objref(f),
                   x, xl, xu, rhobeg, rhoend, verbose, maxeval, w)
    if status != BOBYQA_SUCCESS
        error(bobyqa_reason(status))
    end
    return w[1]
end

function bobyqa_test()
    # The test function.
    function f(x::Vector{Cdouble})
        fx = 0.0
        n = length(x)
        for i in 4:2:n
            for j in 2:2:i-2
                tempa = x[i - 1] - x[j - 1]
                tempb = x[i] - x[j]
                temp = tempa*tempa + tempb*tempb
                temp = max(temp,1e-6)
                fx += 1.0/sqrt(temp)
            end
        end
        return fx
    end

    # Run the tests.
    bdl = -1.0
    bdu =  1.0
    rhobeg = 0.1
    rhoend = 1e-6
    for m in (5,10)
        q = 2.0*pi/m
        n = 2*m
        x = Array(Cdouble, n)
        xl = Array(Cdouble, n)
        xu = Array(Cdouble, n)
        for i in 1:n
            xl[i] = bdl
            xu[i] = bdu
        end
        for jcase in 1:2
            if jcase == 2
                npt = 2*n + 1
            else
                npt = n + 6
            end
            @printf("\n\n     2D output with M =%4ld,  N =%4ld  and  NPT =%4ld\n",
                    m, n, npt)
            for j in 1:m
                temp = q*j
                x[2*j - 1] = cos(temp)
                x[2*j]     = sin(temp)
            end
            fx = bobyqa!(f, x, xl, xu, rhobeg, rhoend, npt=npt, verbose=2, maxeval=500000)
            @printf("\n***** least function value: %.15e\n", fx)
        end
    end
end


using Parameters, Random
import Distributions: logpdf, loglikelihood

struct Paired{T1,T2} <: ContinuousUnivariateDistribution
    d::T1
    parms::T2
end

Paired(;d, parms) = Paired(d, parms)

loglikelihood(d::Paired, data::Array{<:NamedTuple,1}) = logpdf(d, data)

function logpdf(d::Paired, data::Array{<:NamedTuple,1})
    LL = computeLL(data, d.parms; d=d.d)
    return LL
end

function sample_stimuli(stimuli, N)
    return stimuli[1:N]
end

function simulate(all_stimuli, fixed_parms, n_blocks, n_trials, deadline=5.0, isi=5.0; d)
    # sample unique stimului for the simulation
    stimuli = sample_stimuli(all_stimuli, n_trials)
    # initialize declarative memory
    chunks = [Chunk(word=:_,number=1)] |> empty!
    memory = Declarative(;memory=chunks)
    # create act-r object
    actr = ACTR(;declarative=memory, fixed_parms..., d)
    # preallocate data
    data = Array{NamedTuple,1}()
    # iterate through all blocks
    for block in 1:n_blocks
        # randomize stimulus order
        shuffle!(stimuli)
        temp = simulate_block(actr, stimuli, block, deadline, isi)
        push!(data, temp...)
    end
    return vcat(data...)
end

function simulate_block(actr, stimuli, block, deadline=5.0, isi=5.0)
    # initialize data for the current block
    data = Array{NamedTuple,1}()
    start_time = get_time(actr)
    for stimulus in stimuli
        temp = simulate_trial(actr, stimulus, block, deadline)
        push!(data, temp)
        # 5 second response deadline + additional 5 seconds
        start_time += (deadline + isi)
        set_time!(actr, start_time)
    end
    return data
end

function simulate_trial(actr, stimulus, block, deadline=5.0)
    N=0;L=0.0;recent=Float64[];retrieved=:failed;time_created=0.0
    trial_start = get_time(actr)
    # encode and conflict resolution time
    e_time = 0.085 + 0.05 + 0.05
    add_time!(actr, e_time)
    # retrieve paired associate based on word
    chunk = retrieve(actr; word=stimulus.word)
    # retrieval time
    r_time = compute_RT(actr, chunk)
    # add encoding time and retrieval time to reaction time
    rt = e_time + r_time + .05 + 0.300
    # interrupt retrieval if exceeds deadline
    if rt > deadline
        # set to truncated value
        rt = deadline
        # advance current time to deadline
        set_time!(actr, trial_start + deadline)
        retrieved = :truncated
        # retrieval info if no chunk is retrieved
        chunk = get_chunks(actr; stimulus...)
        if !isempty(chunk)
            # get chunk state at time of retrieval
            @unpack N,L,recent,time_created = chunk[1]
            recent = copy(recent)
        end
    elseif !isempty(chunk)
        # add retreival time to current time
        add_time!(actr, r_time)
        # next conflict resolution
        add_time!(actr, .05)
        retrieved = :retrieved
        # get chunk state at time of retrieval
        @unpack N,L,recent,time_created = chunk[1]
        recent = copy(recent)
        # add chunk if new, or update N and time stamps
        add_chunk!(actr; stimulus...)
    else
        # add retreival failure time to current time
        add_time!(actr, r_time)
        # next conflict resolution
        add_time!(actr, .05)
        # retrieval info if no chunk is retrieved
        chunk = get_chunks(actr; stimulus...)
        if !isempty(chunk)
            # get chunk state at time of retrieval
            @unpack N,L,recent,time_created = chunk[1]
            recent = copy(recent)
        end
    end
    # feedback is always presented after the deadline
    set_time!(actr, trial_start + deadline)
    # encode feedback
    add_time!(actr, 0.05 + 0.05 + 0.085)
    # add chunk: create new chunk if does not exist, otherwise increment N and add time stamp
    add_chunk!(actr; stimulus...)
    data = (stimulus=stimulus,N=N,L=L,block=block,time_created=time_created,
        recent=recent,rt=rt,retrieved=retrieved)
    return data
end

function computeLL(data, fixed_parms; d)
    chunk = Chunk(act=zero(d))
    # initialize declarative memory
    memory = Declarative(;memory=[chunk])
    # create ACTR object with declarative memory and parameters
    actr = ACTR(;declarative=memory, fixed_parms..., d)
    # extract parameters
    @unpack s,lf,τ,ter=actr.parms
    # do not add noise activation values
    actr.parms.noise = false
    σ = s * pi / sqrt(3)
    # initialize log likelihood
    LL = 0.0
    for k in data
        cur_time = k.L + k.time_created
        if k.retrieved == :retrieved
            # reproduce the memory state right before retrieval
            modify!(chunk; N=k.N, recent=k.recent, time_created=k.time_created)
            # compute activation
            compute_activation!(actr, chunk, cur_time)
            # adjust activation values for latency factor
            v = [chunk.act,τ] .- log(lf)
            # log normal race distribution object
            dist = LNR(-v, σ, ter)
            # compute log likelihood of retrieving chunk at time k.rt
            LL += logpdf(dist, 1, k.rt)
        elseif k.retrieved == :truncated
            # special case in which chunk has not been created yet
            if k.N == 0
                v = τ .- log(lf)
                # compute log likelihood of failing to respond within deadline
                LL += logccdf(LogNormal(-v, σ), 5)
            else
                # reproduce the memory state right before retrieval
                modify!(chunk; N=k.N, recent=k.recent, time_created=k.time_created)
                # compute activation
                compute_activation!(actr, chunk, cur_time)
                # adjust activation values for latency factor
                v = [chunk.act,τ] .- log(lf)
                # compute log likelihood of failing to respond within deadline
                LL += logccdf(LogNormal(-v[1], σ), 5) + logccdf(LogNormal(-v[2], σ), 5)
            end
        else
            # retrieval failure for novel pair
            if k.N == 0
                dist = LNR(-[τ-log(lf)], σ, ter)
                LL += logpdf(dist, 1, k.rt)
            else
                # reproduce the memory state immediately prior to retrieval
                modify!(chunk; N=k.N, recent=k.recent, time_created=k.time_created)
                # compute activation
                compute_activation!(actr, chunk, cur_time)
                # mean activation values
                v = [chunk.act,τ] .- log(lf)
                # log normal race distribution object
                dist = LNR(-v, σ, ter)
                # compute log likelihood of failing to retrieve chunk at time k.rt
                LL += logpdf(dist, 2, k.rt)
            end
        end
    end
    return LL
end

function show_learning(all_stimuli, parms, n_blocks=8, n_trials=20; d)
    data = simulate(all_stimuli, parms, n_blocks, n_trials; d)
    df = DataFrame(data)
    groups = groupby(df, [:block,:retrieved])
    return combine(groups, :rt=>mean=>:rt)
end