module Atmos

export AtmosModel,
       AtmosAcousticLinearModel, AtmosAcousticGravityLinearModel,
       RemainderModel,
       AtmosLESConfiguration,
       AtmosGCMConfiguration

using LinearAlgebra, StaticArrays
using ..VariableTemplates
using ..MoistThermodynamics
using ..PlanetParameters
import ..MoistThermodynamics: internal_energy
using ..SubgridScaleParameters
using GPUifyLoops
using ..MPIStateArrays: MPIStateArray
using ..Mesh.Grids: VerticalDirection, HorizontalDirection, min_node_distance

import CLIMA.DGmethods: BalanceLaw, vars_aux, vars_state, vars_gradient,
                        vars_diffusive, flux_nondiffusive!,
                        flux_diffusive!, source!, wavespeed, boundary_state!,
                        gradvariables!, diffusive!, init_aux!, init_state!,
                        update_aux!, LocalGeometry, lengthscale,
                        resolutionmetric, DGModel, calculate_dt,
                        nodal_update_aux!, num_state,
                        num_integrals,
                        vars_integrals, vars_reverse_integrals,
                        indefinite_stack_integral!,
                        reverse_indefinite_stack_integral!,
                        integral_load_aux!, integral_set_aux!,
                        reverse_integral_load_aux!,
                        reverse_integral_set_aux!
import ..DGmethods.NumericalFluxes: boundary_state!,
                                    boundary_flux_diffusive!,
                                    NumericalFluxNonDiffusive,
                                    NumericalFluxGradient,
                                    NumericalFluxDiffusive

"""
    AtmosModel <: BalanceLaw

A `BalanceLaw` for atmosphere modeling.

# Usage

    AtmosModel(orientation, ref_state, turbulence, moisture, radiation, source,
               boundarycondition, init_state)

"""
struct AtmosModel{FT,O,RS,T,M,P,R,S,BC,IS} <: BalanceLaw
  orientation::O
  ref_state::RS
  turbulence::T
  moisture::M
  precipitation::P
  radiation::R
  source::S
  # TODO: Probably want to have different bc for state and diffusion...
  boundarycondition::BC
  init_state::IS
end

function calculate_dt(grid, model::AtmosModel, Courant_number)
    T = 290.0
    return Courant_number * min_node_distance(grid, VerticalDirection()) / soundspeed_air(T)
end

abstract type AtmosConfiguration end
struct AtmosLESConfiguration <: AtmosConfiguration end
struct AtmosGCMConfiguration <: AtmosConfiguration end

function AtmosModel{FT}(::Type{AtmosLESConfiguration};
                         orientation::O=FlatOrientation(),
                         ref_state::RS=HydrostaticState(LinearTemperatureProfile(FT(200),
                                                                                FT(280),
                                                                                FT(grav) / FT(cp_d)),
                                                                                FT(0)),
                         turbulence::T=SmagorinskyLilly{FT}(0.21),
                         moisture::M=EquilMoist(),
                         precipitation::P=NoPrecipitation(),
                         radiation::R=NoRadiation(),
                         source::S=( Gravity(),
                                     Coriolis(),
                                     GeostrophicForcing{FT}(7.62e-5, 0, 0)),
                         # TODO: Probably want to have different bc for state and diffusion...
                         boundarycondition::BC=NoFluxBC(),
                         init_state::IS=nothing) where {FT<:AbstractFloat,O,RS,T,M,P,R,S,BC,IS}
  @assert init_state ≠ nothing

  atmos = (
        orientation,
        ref_state,
        turbulence,
        moisture,
        precipitation,
        radiation,
        source,
        boundarycondition,
        init_state,
       )

  return AtmosModel{FT,typeof.(atmos)...}(atmos...)
end
function AtmosModel{FT}(::Type{AtmosGCMConfiguration};
                         orientation::O        = SphericalOrientation(),
                         ref_state::RS         = HydrostaticState(
                                                   LinearTemperatureProfile(
                                                    FT(200),
                                                    FT(280),
                                                    FT(grav) / FT(cp_d)),
                                                  FT(0)),
                         turbulence::T         = SmagorinskyLilly{FT}(0.21),
                         moisture::M           = EquilMoist(),
                         precipitation::P      = NoPrecipitation(),
                         radiation::R          = NoRadiation(),
                         source::S             = (Gravity(), Coriolis()),
                         boundarycondition::BC = NoFluxBC(),
                         init_state::IS=nothing) where {FT<:AbstractFloat,O,RS,T,M,P,R,SU,S,BC,IS}
  @assert init_state ≠ nothing
  atmos = (
        orientation,
        ref_state,
        turbulence,
        moisture,
        precipitation,
        radiation,
        source,
        boundarycondition,
        init_state,
       )

  return AtmosModel{FT,typeof.(atmos)...}(atmos...)
end

function vars_state(m::AtmosModel, FT)
  @vars begin
    ρ::FT
    ρu::SVector{3,FT}
    ρe::FT
    turbulence::vars_state(m.turbulence, FT)
    moisture::vars_state(m.moisture, FT)
    radiation::vars_state(m.radiation, FT)
  end
end
function vars_gradient(m::AtmosModel, FT)
  @vars begin
    u::SVector{3,FT}
    h_tot::FT
    turbulence::vars_gradient(m.turbulence,FT)
    moisture::vars_gradient(m.moisture,FT)
  end
end
function vars_diffusive(m::AtmosModel, FT)
  @vars begin
    ∇h_tot::SVector{3,FT}
    turbulence::vars_diffusive(m.turbulence,FT)
    moisture::vars_diffusive(m.moisture,FT)
  end
end

function vars_aux(m::AtmosModel, FT)
  @vars begin
    ∫dz::vars_integrals(m, FT)
    ∫dnz::vars_reverse_integrals(m, FT)
    coord::SVector{3,FT}
    orientation::vars_aux(m.orientation, FT)
    ref_state::vars_aux(m.ref_state,FT)
    turbulence::vars_aux(m.turbulence,FT)
    moisture::vars_aux(m.moisture,FT)
    radiation::vars_aux(m.radiation,FT)
  end
end
function vars_integrals(m::AtmosModel,FT)
  @vars begin
    radiation::vars_integrals(m.radiation,FT)
  end
end
function vars_reverse_integrals(m::AtmosModel,FT)
  @vars begin
    radiation::vars_reverse_integrals(m.radiation,FT)
  end
end

include("orientation.jl")
include("ref_state.jl")
include("turbulence.jl")
include("moisture.jl")
include("precipitation.jl")
include("radiation.jl")
include("source.jl")
include("boundaryconditions.jl")
include("linear.jl")
include("remainder.jl")

"""
    flux_nondiffusive!(m::AtmosModel, flux::Grad, state::Vars, aux::Vars,
                       t::Real)

Computes flux non-diffusive flux portion of `F` in:

```
∂Y
-- = - ∇ • (F_{adv} + F_{press} + F_{nondiff} + F_{diff}) + S(Y)
∂t
```
Where

 - `F_{adv}`      Advective flux             ; see [`flux_advective!`]@ref()
 - `F_{press}`    Pressure flux              ; see [`flux_pressure!`]@ref()
 - `F_{diff}`     Fluxes that state gradients; see [`flux_diffusive!`]@ref()
"""
@inline function flux_nondiffusive!(m::AtmosModel, flux::Grad, state::Vars,
                                    aux::Vars, t::Real)
  ρ = state.ρ
  ρinv = 1/ρ
  ρu = state.ρu
  u = ρinv * ρu

  # advective terms
  flux.ρ   = ρ * u
  flux.ρu  = ρ * u .* u'
  flux.ρe  = u * state.ρe

  # pressure terms
  p = pressure(m.moisture, m.orientation, state, aux)

  if m.ref_state isa HydrostaticState
    flux.ρu += (p-aux.ref_state.p)*I
  else
    flux.ρu += p*I
  end
  flux.ρe += u*p
  flux_radiation!(m.radiation, m, flux, state, aux, t)
  flux_moisture!(m.moisture, m, flux, state, aux, t)
end

function gradvariables!(atmos::AtmosModel, transform::Vars, state::Vars, aux::Vars, t::Real)
  ρinv = 1/state.ρ
  transform.u = ρinv * state.ρu
  transform.h_tot = total_specific_enthalpy(atmos.moisture, atmos.orientation, state, aux)

  gradvariables!(atmos.moisture, transform, state, aux, t)
  gradvariables!(atmos.turbulence, transform, state, aux, t)
end

function diffusive!(atmos::AtmosModel, diffusive::Vars, ∇transform::Grad, state::Vars, aux::Vars, t::Real)
  diffusive.∇h_tot = ∇transform.h_tot

  # diffusion terms required for SGS turbulence computations
  diffusive!(atmos.turbulence, atmos.orientation, diffusive, ∇transform, state, aux, t)
  # diffusivity of moisture components
  diffusive!(atmos.moisture, diffusive, ∇transform, state, aux, t)
end

@inline function flux_diffusive!(atmos::AtmosModel, flux::Grad, state::Vars,
                                 diffusive::Vars, aux::Vars, t::Real)
  ν, τ = turbulence_tensors(atmos.turbulence, state, diffusive, aux, t)
  D_t = (ν isa Real ? ν : diag(ν)) * inv_Pr_turb
  d_h_tot = -D_t .* diffusive.∇h_tot
  flux_diffusive!(atmos, flux, state, τ, d_h_tot)
  flux_diffusive!(atmos.moisture, flux, state, diffusive, aux, t, D_t)
end

#TODO: Consider whether to not pass ρ and ρu (not state), foc BCs reasons
@inline function flux_diffusive!(atmos::AtmosModel, flux::Grad, state::Vars,
                                 τ, d_h_tot)
  flux.ρu += τ * state.ρ
  flux.ρe += τ * state.ρu
  flux.ρe += d_h_tot * state.ρ
end

@inline function wavespeed(m::AtmosModel, nM, state::Vars, aux::Vars, t::Real)
  ρinv = 1/state.ρ
  u = ρinv * state.ρu
  return abs(dot(nM, u)) + soundspeed(m.moisture, m.orientation, state, aux)
end

function update_aux!(dg::DGModel, m::AtmosModel, Q::MPIStateArray, t::Real)
  FT = eltype(Q)
  auxstate = dg.auxstate

  if num_integrals(m, FT) > 0
    indefinite_stack_integral!(dg, m, Q, auxstate, t)
    reverse_indefinite_stack_integral!(dg, m, Q, auxstate, t)
  end

  nodal_update_aux!(atmos_nodal_update_aux!, dg, m, Q, t)

  return true
end

function atmos_nodal_update_aux!(m::AtmosModel, state::Vars, aux::Vars,
                                 t::Real)
  atmos_nodal_update_aux!(m.moisture, m, state, aux, t)
  atmos_nodal_update_aux!(m.radiation, m, state, aux, t)
  atmos_nodal_update_aux!(m.turbulence, m, state, aux, t)
end

function integral_load_aux!(m::AtmosModel, integ::Vars, state::Vars, aux::Vars)
  integral_load_aux!(m.radiation, integ, state, aux)
end

function integral_set_aux!(m::AtmosModel, aux::Vars, integ::Vars)
  integral_set_aux!(m.radiation, aux, integ)
end

function reverse_integral_load_aux!(m::AtmosModel, integ::Vars, state::Vars, aux::Vars)
  reverse_integral_load_aux!(m.radiation, integ, state, aux)
end

function reverse_integral_set_aux!(m::AtmosModel, aux::Vars, integ::Vars)
  reverse_integral_set_aux!(m.radiation, aux, integ)
end

# TODO: figure out a nice way to handle this
function init_aux!(m::AtmosModel, aux::Vars, geom::LocalGeometry)
  aux.coord = geom.coord
  atmos_init_aux!(m.orientation, m, aux, geom)
  atmos_init_aux!(m.ref_state, m, aux, geom)
  atmos_init_aux!(m.turbulence, m, aux, geom)
end

"""
    source!(m::AtmosModel, source::Vars, state::Vars, diffusive::Vars, aux::Vars, t::Real)
Computes flux `S(Y)` in:
```
∂Y
-- = - ∇ • F + S(Y)
∂t
```
"""
function source!(m::AtmosModel, source::Vars, state::Vars, diffusive::Vars, aux::Vars, t::Real)
  atmos_source!(m.source, m, source, state, diffusive, aux, t)
end

boundary_state!(nf, m::AtmosModel, x...) =
  atmos_boundary_state!(nf, m.boundarycondition, m, x...)

function init_state!(m::AtmosModel, state::Vars, aux::Vars, coords, t, args...)
  m.init_state(m, state, aux, coords, t, args...)
end

boundary_flux_diffusive!(nf::NumericalFluxDiffusive,
                         atmos::AtmosModel,
                         F,
                         state⁺, diff⁺, aux⁺, n⁻,
                         state⁻, diff⁻, aux⁻,
                         bctype, t,
                         state1⁻, diff1⁻, aux1⁻) =
  atmos_boundary_flux_diffusive!(nf, atmos.boundarycondition, atmos,
                                 F,
                                 state⁺, diff⁺, aux⁺, n⁻,
                                 state⁻, diff⁻, aux⁻,
                                 bctype, t,
                                 state1⁻, diff1⁻, aux1⁻)

end # module


"""
`module  JAC.AutoIonization`
    ... a submodel of JAC that contains all methods for computing Auger properties between some initial and final-state
        multiplets.
"""
module AutoIonization

    using  Printf, ..AngularMomentum, ..Basics, ..Continuum, ..Defaults, ..InteractionStrength, ..ManyElectron, ..Nuclear,
                   ..PlasmaShift, ..Radial, ..SpinAngular, ..TableStrings

    """
    `struct  Settings`  ... defines a type for the details and parameters of computing Auger lines.

        + calcAnisotropy          ::Bool                         ... True, if the intrinsic alpha_2,4 angular parameters are to be calculated,
                                                                     and false otherwise.
        + printBefore             ::Bool                         ... True, if all energies and lines are printed before their evaluation.
        + lineSelection           ::LineSelection                ... Specifies the selected levels, if any.
        + minAugerEnergy          ::Float64                      ... Minimum energy of free (Auger) electrons to be included.
        + maxAugerEnergy          ::Float64                      ... Maximum energy of free (Auger) electrons to be included.
        + maxKappa                ::Int64                        ... Maximum kappa value of partial waves to be included.
        + operator                ::AbstractEeInteraction        ... Auger operator that is to be used for evaluating the Auger amplitudes:
                                                                     allowed values are: CoulombInteraction(), BreitInteraction(), ...
    """
    struct Settings
        calcAnisotropy            ::Bool
        printBefore               ::Bool
        lineSelection             ::LineSelection
        minAugerEnergy            ::Float64
        maxAugerEnergy            ::Float64
        maxKappa                  ::Int64
        operator                  ::AbstractEeInteraction
    end

    """
    `AutoIonization.Settings()`  ... constructor for the default values of AutoIonization line computations
    """
    function Settings()
        Settings(false, false, LineSelection(), 0., 10e5, 100, CoulombInteraction())
    end

    """
    `AutoIonization.Settings(set::AutoIonization.Settings;`

            calcAnisotropy=..,      printBefore=..,
            minAugerEnergy=..,      maxAugerEnergy=..,          maxKappa=..,            operator=..)

        ... constructor for modifying the given AutoIonization.Settings by 'overwriting' the previously selected parameters.
    """
    function Settings(set::AutoIonization.Settings;
        calcAnisotropy::Union{Nothing,Bool}=nothing,            printBefore::Union{Nothing,Bool}=nothing,
        lineSelection::Union{Nothing,LineSelection}=nothing,
        minAugerEnergy::Union{Nothing,Float64}=nothing,         maxAugerEnergy::Union{Nothing,Float64}=nothing,
        maxKappa::Union{Nothing,Int64}=nothing,                 operator::Union{Nothing,String}=nothing)

        if  calcAnisotropy  == nothing   calcAnisotropyx  = set.calcAnisotropy    else  calcAnisotropyx  = calcAnisotropy   end
        if  printBefore     == nothing   printBeforex     = set.printBefore       else  printBeforex     = printBefore      end
        if  lineSelection   == nothing   lineSelectionx   = set.lineSelection     else  lineSelectionx   = lineSelection    end
        if  minAugerEnergy  == nothing   minAugerEnergyx  = set.minAugerEnergy    else  minAugerEnergyx  = minAugerEnergy   end
        if  maxAugerEnergy  == nothing   maxAugerEnergyx  = set.maxAugerEnergy    else  maxAugerEnergyx  = maxAugerEnergy   end
        if  maxKappa        == nothing   maxKappax        = set.maxKappa          else  maxKappax        = maxKappa         end
        if  operator        == nothing   operatorx        = set.operator          else  operatorx        = operator         end

        Settings( calcAnisotropyx, printBeforex, lineSelectionx, minAugerEnergyx, maxAugerEnergyx, maxKappax, operatorx)
    end

    # `Base.show(io::IO, settings::AutoIonization.Settings)`  ... prepares a proper printout of the variable settings::AutoIonization.Settings.
    function Base.show(io::IO, settings::AutoIonization.Settings)
        println(io, "calcAnisotropy:                $(settings.calcAnisotropy)  ")
        println(io, "printBefore:                   $(settings.printBefore)  ")
        println(io, "lineSelection:                 $(settings.lineSelection)  ")
        println(io, "minAugerEnergy:                $(settings.minAugerEnergy)  ")
        println(io, "maxAugerEnergy:                $(settings.maxAugerEnergy)  ")
        println(io, "maxKappa:                      $(settings.maxKappa)  ")
        println(io, "operator:                      $(settings.operator)  ")
    end

    """
    `struct  Channel`
        ... defines a type for a AutoIonization channel to help characterize a scattering (continuum) state of many
            electron-states with a single free electron.

        + kappa          ::Int64                ... partial-wave of the free electron
        + symmetry       ::LevelSymmetry        ... total angular momentum and parity of the scattering state
        + phase          ::Float64              ... phase of the partial wave
        + amplitude      ::Complex{Float64}     ... Auger amplitude associated with the given channel.
    """
    struct  Channel
        kappa            ::Int64
        symmetry         ::LevelSymmetry
        phase            ::Float64
        amplitude        ::Complex{Float64}
    end

    """
    `struct  Line`
        ... defines a type for a AutoIonization line that may include the definition of sublines and their
            corresponding amplitudes.

        + initialLevel   ::Level           ... initial-(state) level
        + finalLevel     ::Level           ... final-(state) level
        + electronEnergy ::Float64         ... Energy of the (incoming free) electron.
        + totalRate      ::Float64         ... Total rate of this line.
        + angularAlpha   ::Float64         ... Angular alpha_2 coefficient.
        + hasChannels    ::Bool
            ... Determines whether the individual scattering (sub-) channels are defined in terms of their free-electron energy, kappa
                and the total angular momentum/parity as well as the amplitude, or not.
        + channels       ::Array{AutoIonization.Channel,1}  ... List of AutoIonization channels of this line.
    """
    struct  Line
        initialLevel     ::Level
        finalLevel       ::Level
        electronEnergy   ::Float64
        totalRate        ::Float64
        angularAlpha     ::Float64
        hasChannels      ::Bool
        channels         ::Array{AutoIonization.Channel,1}
    end

    """
    `AutoIonization.Line(initialLevel::Level, finalLevel::Level, totalRate::Float64)`
        ... constructor for an AutoIonization line between a specified initial and final level.
    """
    function Line(initialLevel::Level, finalLevel::Level, totalRate::Float64)
        Line(initialLevel, finalLevel, 0., totalRate, 0., false, AutoIonization.Channel[])
    end

    # `Base.show(io::IO, line::AutoIonization.Line)`  ... prepares a proper printout of the variable line::AutoIonization.Line.
    function Base.show(io::IO, line::AutoIonization.Line)
        println(io, "initialLevel:           $(line.initialLevel)  ")
        println(io, "finalLevel:             $(line.finalLevel)  ")
        println(io, "electronEnergy:         $(line.electronEnergy)  ")
        println(io, "totalRate:              $(line.totalRate)  ")
        println(io, "angularAlpha:           $(line.angularAlpha)  ")
        println(io, "hasChannels:            $(line.hasChannels)  ")
        println(io, "channels:               $(line.channels)  ")
    end

    """
    `AutoIonization.amplitude(kind::String, channel::AutoIonization.Channel, continuumLevel::Level, initialLevel::Level,
                              grid::Radial.Grid; printout::Bool=true)`
        ... to compute the kind in  CoulombInteraction(), BreitInteraction(), CoulombBreit()   Auger amplitude
            <(alpha_f J_f, kappa) J_i || O^(Auger, kind) || alpha_i J_i>  due to the interelectronic interaction for the given
            final and initial level. A value::ComplexF64 is returned.
    """
    function amplitude(kind::AbstractEeInteraction, channel::AutoIonization.Channel, continuumLevel::Level, initialLevel::Level, grid::Radial.Grid;
                       printout::Bool=true)
        nt = length(continuumLevel.basis.csfs);    ni = length(initialLevel.basis.csfs);    partial = Subshell(9,channel.kappa)
        if  printout  printstyled("Compute ($kind) Auger matrix of dimension $nt x $ni in the continuum- and initial-state bases " *
                                  "for the transition [$(initialLevel.index)- ...] and for partial wave $(string(partial)[2:end]) ... ",
                                  color=:light_green)    end
        matrix = zeros(ComplexF64, nt, ni)
        #
        if      kind in [ CoulombInteraction(), BreitInteraction(), CoulombBreit()]        ## pure V^Coulomb interaction
        #--------------------------------------------------------------------------
            for  r = 1:nt
                for  s = 1:ni
                    if  initialLevel.basis.csfs[s].J != initialLevel.J  ||  initialLevel.basis.csfs[s].parity != initialLevel.parity      continue    end
                    ##x wa = compute("angular coefficients: e-e, Ratip2013", continuumLevel.basis.csfs[r], initialLevel.basis.csfs[s])
                    # Calculate the spin-angular coefficients
                    if  Defaults.saRatip()
                        waR = compute("angular coefficients: e-e, Ratip2013", continuumLevel.basis.csfs[r], initialLevel.basis.csfs[s])
                        wa  = waR
                    end
                    if  Defaults.saGG()
                        subshellList = continuumLevel.basis.subshells
                        opa  = SpinAngular.TwoParticleOperator(0, plus, true)
                        waG2 = SpinAngular.computeCoefficients(opa, continuumLevel.basis.csfs[r], initialLevel.basis.csfs[s], subshellList)
                        wa   = [1.0, waG2]
                    end
                    if  Defaults.saRatip() && Defaults.saGG() && true
                        if  length(waR[2]) != 0     println("\n>> Angular coeffients from Ratip2013   = $(waR[2]) ")    end
                        if  length(waG2)   != 0     println(  ">> Angular coeffients from SpinAngular = $waG2 ")        end
                    end
                    #
                    me = 0.
                    for  coeff in wa[2]
                        if   kind in [ CoulombInteraction(), CoulombBreit()]
                            me = me + coeff.V * InteractionStrength.XL_Coulomb(coeff.nu,
                                                    continuumLevel.basis.orbitals[coeff.a], continuumLevel.basis.orbitals[coeff.b],
                                                    initialLevel.basis.orbitals[coeff.c],   initialLevel.basis.orbitals[coeff.d], grid)   end
                        if   kind in [ BreitInteraction(), CoulombBreit()]
                            me = me + coeff.V * InteractionStrength.XL_Breit(coeff.nu,
                                                    continuumLevel.basis.orbitals[coeff.a], continuumLevel.basis.orbitals[coeff.b],
                                                    initialLevel.basis.orbitals[coeff.c],   initialLevel.basis.orbitals[coeff.d], grid)   end
                    end
                    matrix[r,s] = me
                end
            end
            if  printout  printstyled("done. \n", color=:light_green)    end
            amplitude = transpose(continuumLevel.mc) * matrix * initialLevel.mc
            amplitude = im^Basics.subshell_l(Subshell(101, channel.kappa)) * exp( -im*channel.phase ) * amplitude
            #
            #
         elseif  kind == "H-E"
        #--------------------
            iLevel = finalLevel;   fLevel = initialLevel
            amplitude = 0.;    error("stop a")
        else    error("stop b")
        end

        return( amplitude )
    end

    """
    `AutoIonization.channelAmplitude(kind::String, channel::AutoIonization.Channel, energy::Float64, finalLevel::Level,
                                     initialLevel::Level, grid::Radial.Grid)`
        ... to compute the kind = (CoulombInteraction(), BreitInteraction(), CoulombBreit())   Auger amplitude
            <(alpha_f J_f, kappa) J_i || O^(Auger, kind) || alpha_i J_i>  due to the interelectronic interaction for the given final and
            initial level. A newChannel::AutoIonization.Channel is returned.
    """
    function channelAmplitude(kind::String, channel::AutoIonization.Channel, energy::Float64, finalLevel::Level,
                              initialLevel::Level, grid::Radial.Grid)
        newiLevel = Basics.generateLevelWithSymmetryReducedBasis(initialLevel)
        newiLevel = Basics.generateLevelWithExtraSubshell(Subshell(101, channel.kappa), newiLevel)
        newfLevel = Basics.generateLevelWithSymmetryReducedBasis(finalLevel)
        cOrbital, phase  = Continuum.generateOrbital(energy, Subshell(101, channel.kappa), newfLevel, grid, contSettings)
        newcLevel  = Basics.generateLevelWithExtraElectron(cOrbital, channel.symmetry, newfLevel)
        newChannel = AutoIonization.Channel(channel.kappa, channel.symmetry, phase, 0.)
        amplitude = AutoIonization.amplitude(kind, channel, newcLevel, newiLevel, grid)

        newChannel = AutoIonization.Channel(newChannel.kappa, newChannel.symmetry, newChannel.phase, amplitude)
        return( newChannel )
    end

    """
    `AutoIonization.computeAmplitudesProperties(line::AutoIonization.Line, nm::Nuclear.Model, grid::Radial.Grid, nrContinuum::Int64,
                                                settings::AutoIonization.Settings; printout::Bool=true)`
        ... to compute all amplitudes and properties of the given line; a line::AutoIonization.Line is returned for which the amplitudes
            and properties are now evaluated.
    """
    function computeAmplitudesProperties(line::AutoIonization.Line, nm::Nuclear.Model, grid::Radial.Grid, nrContinuum::Int64, settings::AutoIonization.Settings;
                                         printout::Bool=true)
        newChannels = AutoIonization.Channel[];   contSettings = Continuum.Settings(false, nrContinuum);   rate = 0.
        for channel in line.channels
            newiLevel = Basics.generateLevelWithSymmetryReducedBasis(line.initialLevel, line.initialLevel.basis.subshells)
            newfLevel = Basics.generateLevelWithSymmetryReducedBasis(line.finalLevel, newiLevel.basis.subshells)
            newiLevel = Basics.generateLevelWithExtraSubshell(Subshell(101, channel.kappa), newiLevel)
            cOrbital, phase  = Continuum.generateOrbitalForLevel(line.electronEnergy, Subshell(101, channel.kappa), newfLevel, nm, grid, contSettings)
            newcLevel  = Basics.generateLevelWithExtraElectron(cOrbital, channel.symmetry, newfLevel)
            newChannel = AutoIonization.Channel(channel.kappa, channel.symmetry, phase, 0.)
            amplitude  = AutoIonization.amplitude(settings.operator, newChannel, newcLevel, newiLevel, grid, printout=printout)
            rate       = rate + conj(amplitude) * amplitude
            push!( newChannels, AutoIonization.Channel(newChannel.kappa, newChannel.symmetry, newChannel.phase, amplitude) )
        end
        totalRate = 2pi* rate;   angularAlpha = 0.
        newLine   = AutoIonization.Line(line.initialLevel, line.finalLevel, line.electronEnergy, totalRate, angularAlpha, true, newChannels)
        #
        if  settings.calcAnisotropy    angularAlpha = AutoIonization.computeIntrinsicAlpha(2, newLine)
            newLine   = AutoIonization.Line(line.initialLevel, line.finalLevel, line.electronEnergy, totalRate, real(angularAlpha), true, newChannels)
        end

        return( newLine )
    end

    """
    `AutoIonization.computeAmplitudesPropertiesPlasma(line::AutoIonization.Line, nm::Nuclear.Model, grid::Radial.Grid, nrContinuum::Int64,
                                                      settings::PlasmaShift.AugerSettings; printout::Bool=true)`
        ... to compute all amplitudes and properties of the given line but for the given plasma model; a line::AutoIonization.Line is returned
            for which the amplitudes and properties are now evaluated.
    """
    function computeAmplitudesPropertiesPlasma(line::AutoIonization.Line, nm::Nuclear.Model, grid::Radial.Grid, nrContinuum::Int64,
                                               settings::PlasmaShift.AugerSettings; printout::Bool=true)
        newChannels = AutoIonization.Channel[];   contSettings = Continuum.Settings(false, nrContinuum);   rate = 0.
        for channel in line.channels
            newiLevel = Basics.generateLevelWithSymmetryReducedBasis(line.initialLevel)
            newiLevel = Basics.generateLevelWithExtraSubshell(Subshell(101, channel.kappa), newiLevel)
            newfLevel = Basics.generateLevelWithSymmetryReducedBasis(line.finalLevel)
            @warn "Adapt a proper continuum orbital for the plasma potential"
            cOrbital, phase  = Continuum.generateOrbitalForLevel(line.electronEnergy, Subshell(101, channel.kappa), newfLevel, nm, grid, contSettings)
            newcLevel  = Basics.generateLevelWithExtraElectron(cOrbital, channel.symmetry, newfLevel)
            newChannel = AutoIonization.Channel(channel.kappa, channel.symmetry, phase, 0.)
            @warn "Adapt a proper Auger amplitude for the plasma e-e interaction"
            amplitude = 1.0
            # amplitude  = AutoIonization.amplitude(settings.operator, newChannel, newcLevel, newiLevel, grid)
            rate       = rate + conj(amplitude) * amplitude
            push!( newChannels, AutoIonization.Channel(newChannel.kappa, newChannel.symmetry, newChannel.phase, amplitude) )
        end
        totalRate = 2pi* rate;   angularAlpha = 0.
        newLine   = AutoIonization.Line(line.initialLevel, line.finalLevel, line.electronEnergy, totalRate, angularAlpha, true, newChannels)

        return( newLine )
    end

    """
    `AutoIonization.computeIntrinsicAlpha(k::Int64, line::AutoIonization.Line)`
        ... to compute the intrinsic alpha_k anisotropy parameter for the given line. A value::Float64 is returned.
    """
    function  computeIntrinsicAlpha(k::Int64, line::AutoIonization.Line)
        if  !line.hasChannels   error("No channels are defined for the given AutoIonization.line.")                   end
        wn = 0.;    for  channel in line.channels    wn = wn + conj(channel.amplitude) * channel.amplitude   end
        wa = 0.;    Ji = line.initialLevel.J;    Jf = line.finalLevel.J;
        for  cha  in line.channels
            j = AngularMomentum.kappa_j(cha.kappa);    l = AngularMomentum.kappa_l(cha.kappa)
            for  chp  in line.channels
                jp = AngularMomentum.kappa_j(chp.kappa);    lp = AngularMomentum.kappa_l(chp.kappa)
                wa = wa + sqrt( AngularMomentum.bracket([l, lp, j, jp]) ) *
                          AngularMomentum.ClebschGordan(l, AngularM64(0), lp, AngularM64(0), AngularJ64(k), AngularM64(0)) *
                          AngularMomentum.Wigner_6j(Ji, j, Jf, jp, Ji, AngularJ64(k)) *
                          AngularMomentum.Wigner_6j(l,  j, AngularJ64(1//2), jp, lp, AngularJ64(k)) *
                          cha.amplitude * conj(chp.amplitude)
            end
        end
        value = AngularMomentum.phaseFactor([Ji, +1, Jf, +1, AngularJ64(k), -1, AngularJ64(1//2)]) *
                sqrt(Basics.twice(Ji) + 1) * wa / wn

        if  false
            # Calculate the value given by M.H. Chen, PRA 47 (1993) 3733; this does not agree exactly so far.
            wa = 0.
            for  cha  in line.channels
                j = AngularMomentum.kappa_j(cha.kappa);    l = AngularMomentum.kappa_l(cha.kappa);    phase = cha.phase
                for  chp  in line.channels
                    jp = AngularMomentum.kappa_j(chp.kappa);    lp = AngularMomentum.kappa_l(chp.kappa);    phasep = chp.phase
                    wa = wa + AngularMomentum.phaseFactor([Ji, +1, Jf, -1, AngularJ64(1//2)]) * im^(Float64(l) - Float64(lp)) * cos(phase - phasep) *
                              sqrt( AngularMomentum.bracket([l, lp, j, jp, AngularJ64(k), Ji]) ) *
                              AngularMomentum.Wigner_3j(lp, l, AngularJ64(k), AngularJ64(0), AngularJ64(0), AngularJ64(0)) *
                              AngularMomentum.Wigner_6j(j, jp, AngularJ64(k), lp, l, AngularJ64(1//2)) *
                              AngularMomentum.Wigner_6j(Ji, Ji, AngularJ64(k), jp, j, Jf) *
                              cha.amplitude * conj(chp.amplitude)
                end
            end
            println("*** Comparison value = $value    Chen-value = $(wa/wn)")
        end

        return( value )
    end

    """
    `AutoIonization.computeLines(finalMultiplet::Multiplet, initialMultiplet::Multiplet, nm::Nuclear.Model, grid::Radial.Grid,
                                 settings::AutoIonization.Settings; output=true, printout::Bool=true)`
        ... to compute the Auger transition amplitudes and all properties as requested by the given settings. A list of
            lines::Array{AutoIonization.Lines} is returned.
    """
    function  computeLines(finalMultiplet::Multiplet, initialMultiplet::Multiplet, nm::Nuclear.Model, grid::Radial.Grid,
                           settings::AutoIonization.Settings; output=true, printout::Bool=true)
        println("")
        printstyled("AutoIonization.computeLines(): The computation of Auger rates and properties starts now ... \n", color=:light_green)
        printstyled("------------------------------------------------------------------------------------------- \n", color=:light_green)
        println("")
        lines = AutoIonization.determineLines(finalMultiplet, initialMultiplet, settings)
        # Display all selected lines before the computations start
        if  settings.printBefore    AutoIonization.displayLines(lines)    end
        # Determine maximum energy and check for consistency of the grid
        maxEnergy = 0.;   for  line in lines   maxEnergy = max(maxEnergy, line.electronEnergy)   end
        nrContinuum = Continuum.gridConsistency(maxEnergy, grid)
        # Calculate all amplitudes and requested properties
        newLines = AutoIonization.Line[]
        for  line in lines
            newLine = AutoIonization.computeAmplitudesProperties(line, nm, grid, nrContinuum, settings)
            push!( newLines, newLine)
        end
        # Print all results to screen
        AutoIonization.displayRates(stdout, newLines, settings)
        AutoIonization.displayLifetimes(stdout, newLines)
        printSummary, iostream = Defaults.getDefaults("summary flag/stream")
        if  printSummary   AutoIonization.displayRates(iostream, newLines, settings);   AutoIonization.displayLifetimes(iostream, newLines)     end
        #
        if    output    return( lines )
        else            return( nothing )
        end
    end

    """
    `AutoIonization.computeLinesCascade(finalMultiplet::Multiplet, initialMultiplet::Multiplet, nm::Nuclear.Model, grid::Radial.Grid,
                                        settings::AutoIonization.Settings; output::Bool=true, printout::Bool=true)`
        ... to compute the Auger transition amplitudes and all properties as requested by the given settings. The computations
            and printout is adapted for large cascade computations by including only lines with at least one channel and by sending
            all printout to a summary file only. A list of lines::Array{AutoIonization.Lines} is returned.
    """
    function  computeLinesCascade(finalMultiplet::Multiplet, initialMultiplet::Multiplet, nm::Nuclear.Model, grid::Radial.Grid,
                                  settings::AutoIonization.Settings; output::Bool=true, printout::Bool=true)

        lines = AutoIonization.determineLines(finalMultiplet, initialMultiplet, settings)
        # Display all selected lines before the computations start
        # if  settings.printBefore    AutoIonization.displayLines(lines)    end
        # Determine maximum energy and check for consistency of the grid
        maxEnergy = 0.;   for  line in lines   maxEnergy = max(maxEnergy, line.electronEnergy)   end
        nrContinuum = Continuum.gridConsistency(maxEnergy, grid)
        # Calculate all amplitudes and requested properties
        newLines = AutoIonization.Line[]
        for  (i,line)  in  enumerate(lines)
            if  rem(i,10) == 0    println("> Auger line $i:  ... calculated ")    end
            newLine = AutoIonization.computeAmplitudesProperties(line, nm, grid, nrContinuum, settings, printout=printout)
            push!( newLines, newLine)
        end
        # Print all results to a summary file, if requested
        printSummary, iostream = Defaults.getDefaults("summary flag/stream")
        if  printSummary   AutoIonization.displayRates(iostream, newLines, settings)     end
        #
        if    output    return( newLines )
        else            return( nothing )
        end
    end

    """
    `AutoIonization.computeLinesFromOrbitals(finalMultiplet::Multiplet, initialMultiplet::Multiplet, nm::Nuclear.Model, grid::Radial.Grid,
                                             settings::AutoIonization.Settings, contOrbitals::Dict{Subshell, Orbital}; output::Bool=true, printout::Bool=true)`
        ... to compute the Auger transition amplitudes and all properties as requested by the given settings but by using the given set of
            continuum orbitals. The computations and printout is adapted for large cascade computations by including only lines with at least
            one channel and by sending all printout to a summary file only. A list of lines::Array{AutoIonization.Lines} is returned.
    """
    function  computeLinesFromOrbitals(finalMultiplet::Multiplet, initialMultiplet::Multiplet, nm::Nuclear.Model, grid::Radial.Grid,
                                       settings::AutoIonization.Settings, contOrbitals::Dict{Subshell, Orbital}; output::Bool=true, printout::Bool=true)

        lines = AutoIonization.determineLines(finalMultiplet, initialMultiplet, settings)
        # Calculate all amplitudes and requested properties
        newLines = AutoIonization.Line[]
        for  (i,line)  in  enumerate(lines)
            if  rem(i,500) == 0    println("> Auger line $i:  ... calculated ")    end
            # Calculate the individual channels with the given orbitals
            newChannels = AutoIonization.Channel[];   rate = 0.
            for channel in line.channels
                newiLevel  = Basics.generateLevelWithSymmetryReducedBasis(line.initialLevel, line.initialLevel.basis.subshells)
                newfLevel  = Basics.generateLevelWithSymmetryReducedBasis(line.finalLevel, newiLevel.basis.subshells)
                sh         = Subshell(101, channel.kappa)
                if haskey(contOrbitals, sh)   cOrbital = contOrbitals[sh]      else    println(">>> skip Auger channel for $sh");   continue    end
                newiLevel  = Basics.generateLevelWithExtraSubshell(sh, newiLevel)
                newcLevel  = Basics.generateLevelWithExtraElectron(cOrbital, channel.symmetry, newfLevel)
                newChannel = AutoIonization.Channel(channel.kappa, channel.symmetry, 0., 0.)
                amplitude  = AutoIonization.amplitude(settings.operator, newChannel, newcLevel, newiLevel, grid, printout=printout)
                rate       = rate + conj(amplitude) * amplitude
                push!( newChannels, AutoIonization.Channel(newChannel.kappa, newChannel.symmetry, newChannel.phase, amplitude) )
            end
            totalRate = 2pi* rate;   angularAlpha = 0.
            newLine   = AutoIonization.Line(line.initialLevel, line.finalLevel, line.electronEnergy, totalRate, angularAlpha, true, newChannels)
            ##
            ## if  settings.calcAnisotropy    angularAlpha = AutoIonization.computeIntrinsicAlpha(2, newLine)
            ##  newLine   = AutoIonization.Line(line.initialLevel, line.finalLevel, line.electronEnergy, totalRate, real(angularAlpha), true, newChannels)
            ## end
            push!( newLines, newLine)
        end
        # Print all results to a summary file, if requested
        printSummary, iostream = Defaults.getDefaults("summary flag/stream")
        if  printSummary   AutoIonization.displayRates(iostream, newLines, settings)     end
        #
        if    output    return( newLines )
        else            return( nothing )
        end
    end

    """
    `AutoIonization.computeLinesPlasma(finalMultiplet::Multiplet, initialMultiplet::Multiplet, nm::Nuclear.Model, grid::Radial.Grid,
                                       settings::PlasmaShift.AugerSettings; output=true)`
        ... to compute the Auger transition amplitudes and all properties as requested by the given settings. A list of
            lines::Array{AutoIonization.Lines} is returned.
    """
    function  computeLinesPlasma(finalMultiplet::Multiplet, initialMultiplet::Multiplet, nm::Nuclear.Model, grid::Radial.Grid,
                                 settings::PlasmaShift.AugerSettings; output=true)
        println("")
        printstyled("AutoIonization.computeLinesPlasma(): The computation of Auger rates starts now ... \n", color=:light_green)
        printstyled("---------------------------------------------------------------------------------- \n", color=:light_green)
        println("")
        augerSettings = AutoIonization.Settings(false, settings.printBefore, settings.selectLines, settings.selectedLines, 0., 1.0e6, 100, CoulombInteraction())
        lines = AutoIonization.determineLines(finalMultiplet, initialMultiplet, augerSettings)
        # Display all selected lines before the computations start
        if  settings.printBefore    AutoIonization.displayLines(lines)    end
        # Determine maximum energy and check for consistency of the grid
        maxEnergy = 0.;   for  line in lines   maxEnergy = max(maxEnergy, line.electronEnergy)   end
        nrContinuum = Continuum.gridConsistency(maxEnergy, grid)
        # Calculate all amplitudes and requested properties
        newLines = AutoIonization.Line[]
        for  line in lines
            newLine = AutoIonization.computeAmplitudesPropertiesPlasma(line, nm, grid, nrContinuum, settings)
            push!( newLines, newLine)
        end
        # Print all results to screen
        AutoIonization.displayRates(stdout, newLines, augerSettings)
        AutoIonization.displayLifetimes(stdout, newLines)
        printSummary, iostream = Defaults.getDefaults("summary flag/stream")
        if  printSummary   AutoIonization.displayRates(iostream, newLines, augerSettings);   AutoIonization.displayLifetimes(iostream, newLines)     end
        #
        if    output    return( lines )
        else            return( nothing )
        end
    end

    """
    `AutoIonization.determineChannels(finalLevel::Level, initialLevel::Level, settings::AutoIonization.Settings)`
        ... to determine a list of Auger Channel for a transitions from the initial to final level and by taking into account the particular
            settings of for this computation; an Array{AutoIonization.Channel,1} is returned.
    """
    function determineChannels(finalLevel::Level, initialLevel::Level, settings::AutoIonization.Settings)
        channels  = AutoIonization.Channel[];
        symi      = LevelSymmetry(initialLevel.J, initialLevel.parity);    symf = LevelSymmetry(finalLevel.J, finalLevel.parity)
        kappaList = AngularMomentum.allowedKappaSymmetries(symi, symf)
        for  kappa in kappaList
            if  abs(kappa) > settings.maxKappa      continue    end
            push!(channels, AutoIonization.Channel(kappa, symi, 0., Complex(0.)) )
        end
        return( channels )
    end

    """
    `AutoIonization.determineLines(finalMultiplet::Multiplet, initialMultiplet::Multiplet, settings::AutoIonization.Settings)`
        ... to determine a list of AutoIonization.Line's for transitions between levels from the initial- and final-state multiplets, and
            by taking into account the particular selections and settings for this computation; an Array{AutoIonization.Line,1} is returned.
            Apart from the level specification, all physical properties are set to zero during the initialization process.
    """
    function  determineLines(finalMultiplet::Multiplet, initialMultiplet::Multiplet, settings::AutoIonization.Settings)
        lines = AutoIonization.Line[]
        for  iLevel  in  initialMultiplet.levels
            for  fLevel  in  finalMultiplet.levels
                if  Basics.selectLevelPair(iLevel, fLevel, settings.lineSelection)
                    energy = iLevel.energy - fLevel.energy
                    if   energy < 0.01                                                             continue   end
                    if   energy < settings.minAugerEnergy  ||  energy > settings.maxAugerEnergy    continue   end
                    channels = AutoIonization.determineChannels(fLevel, iLevel, settings)
                    push!( lines, AutoIonization.Line(iLevel, fLevel, energy, 0., 0., true, channels) )
                end
            end
        end
        return( lines )
    end

    """
    `AutoIonization.displayLines(lines::Array{AutoIonization.Line,1})`
        ... to display a list of lines and channels that have been selected due to the prior settings. A neat table of all selected
            transitions and energies is printed but nothing is returned otherwise.
    """
    function  displayLines(lines::Array{AutoIonization.Line,1})
        nx = 150
        println(" ")
        println("  Selected Auger lines:")
        println(" ")
        println("  ", TableStrings.hLine(nx))
        sa = "  ";   sb = "  "
        sa = sa * TableStrings.center(18, "i-level-f"; na=2);                                sb = sb * TableStrings.hBlank(20)
        sa = sa * TableStrings.center(18, "i--J^P--f"; na=4);                                sb = sb * TableStrings.hBlank(22)
        sa = sa * TableStrings.center(14, "Energy"; na=4);
        sb = sb * TableStrings.center(14, TableStrings.inUnits("energy"); na=4)
        sa = sa * TableStrings.center(14, "Energy e_A"; na=4);
        sb = sb * TableStrings.center(14, TableStrings.inUnits("energy"); na=4)
        sa = sa * TableStrings.flushleft(37, "List of kappas and total symmetries"; na=4)
        sb = sb * TableStrings.flushleft(37, "partial (total J^P)                "; na=4)
        println(sa);    println(sb);    println("  ", TableStrings.hLine(nx))
        #
        for  line in lines
            sa  = "  ";    isym = LevelSymmetry( line.initialLevel.J, line.initialLevel.parity)
                           fsym = LevelSymmetry( line.finalLevel.J,   line.finalLevel.parity)
            sa = sa * TableStrings.center(18, TableStrings.levels_if(line.initialLevel.index, line.finalLevel.index); na=2)
            sa = sa * TableStrings.center(18, TableStrings.symmetries_if(isym, fsym); na=4)
            sa = sa * @sprintf("%.8e", Defaults.convertUnits("energy: from atomic", line.initialLevel.energy))  * "    "
            sa = sa * @sprintf("%.8e", Defaults.convertUnits("energy: from atomic", line.electronEnergy))       * "   "
            kappaSymmetryList = Tuple{Int64,LevelSymmetry}[]
            for  i in 1:length(line.channels)
                push!( kappaSymmetryList, (line.channels[i].kappa, line.channels[i].symmetry) )
            end
            sa = sa * TableStrings.kappaSymmetryTupels(80, kappaSymmetryList)
            println( sa )
        end
        println("  ", TableStrings.hLine(nx), "\n")
        #
        return( nothing )
    end

    """
    `AutoIonization.displayLifetimes(stream::IO, lines::Array{AutoIonization.Line,1})`
        ... to list all lifetimes as associated with the selected lines. A neat table is printed but nothing is returned otherwise.
    """
    function  displayLifetimes(stream::IO, lines::Array{AutoIonization.Line,1})
        nx = 104
        println(stream, " ")
        println(stream, "  Auger lifetimes, total rates and widths:")
        println(stream, " ")
        println(stream, "  ", TableStrings.hLine(nx))
        sa = "  ";   sb = "  "
        sa = sa * TableStrings.center(10, "Level";    na=2);                           sb = sb * TableStrings.hBlank(12)
        sa = sa * TableStrings.center( 8, "J^P";      na=4);                           sb = sb * TableStrings.hBlank(12)
        sa = sa * TableStrings.center(12, "Lifetime"; na=4);
        sb = sb * TableStrings.center(12,TableStrings.inUnits("time"); na=4)
        sa = sa * TableStrings.center(14, "Total rate"; na=6);
        sb = sb * TableStrings.center(14,TableStrings.inUnits("rate"); na=4)
        sa = sa * TableStrings.center(42, "Widths"; na=2);
        sb = sb * TableStrings.center(42, "  Hartrees         Kaysers           eV"; na=2)
        println(stream, sa);    println(stream, sb);    println(stream, "  ", TableStrings.hLine(nx))
        #
        notYetDone = trues(1000)
        for  line in lines
            if  notYetDone[line.initialLevel.index]
                notYetDone[line.initialLevel.index] = false
                totalRate = 0.
                for  ln in lines
                    if  ln.initialLevel.index == line.initialLevel.index   totalRate = totalRate + ln.totalRate    end
                end
                sa  = "  ";    isym = LevelSymmetry( line.initialLevel.J, line.initialLevel.parity)
                sa = sa * TableStrings.center(10, TableStrings.level(line.initialLevel.index); na=2)
                sa = sa * TableStrings.center( 8, string(isym); na=4)
                sa = sa * @sprintf("%.6e", Defaults.convertUnits("time: from atomic",  1/totalRate))            * "     "
                sa = sa * @sprintf("%.6e", Defaults.convertUnits("rate: from atomic",    totalRate))            * "      "
                sa = sa * @sprintf("%.6e", totalRate)                                                           * "    "
                sa = sa * @sprintf("%.6e", Defaults.convertUnits("energy: from atomic to Kayser",  totalRate))  * "    "
                sa = sa * @sprintf("%.6e", Defaults.convertUnits("energy: from atomic to eV",      totalRate))  * "    "
                println(stream, sa)
            end
        end
        println(stream, "  ", TableStrings.hLine(nx))
        #
        return( nothing )
    end

    """
    `AutoIonization.displayRates(stream::IO, lines::Array{AutoIonization.Line,1}, settings::AutoIonization.Settings)`
        ... to list all results, energies, rates, etc. of the selected lines. A neat table is printed but nothing is returned
            otherwise.
    """
    function  displayRates(stream::IO, lines::Array{AutoIonization.Line,1}, settings::AutoIonization.Settings)
        nx = 106
        println(stream, " ")
        if  settings.calcAnisotropy    println(stream, "  Auger rates and intrinsic angular parameters: \n")
        else                           println(stream, "  Auger rates (without angular parameters): \n")        end
        println(stream, "  ", TableStrings.hLine(nx))
        sa = "  ";   sb = "  "
        sa = sa * TableStrings.center(18, "i-level-f"; na=2);                         sb = sb * TableStrings.hBlank(20)
        sa = sa * TableStrings.center(18, "i--J^P--f"; na=4);                         sb = sb * TableStrings.hBlank(22)
        sa = sa * TableStrings.center(12, "Energy"   ; na=2);
        sb = sb * TableStrings.center(12,TableStrings.inUnits("energy"); na=4)
        sa = sa * TableStrings.center(14, "Electron energy"   ; na=2);
        sb = sb * TableStrings.center(14,TableStrings.inUnits("energy"); na=2)
        sa = sa * TableStrings.center(14, "Auger rate"; na=2);
        sb = sb * TableStrings.center(14, TableStrings.inUnits("rate"); na=2)
        sa = sa * TableStrings.center(15, "alpha_2"; na=2);                           sb = sb * TableStrings.hBlank(18)
        println(stream, sa);    println(stream, sb);    println(stream, "  ", TableStrings.hLine(nx))
        #
        for  line in lines
            sa  = "  ";    isym = LevelSymmetry( line.initialLevel.J, line.initialLevel.parity)
                           fsym = LevelSymmetry( line.finalLevel.J,   line.finalLevel.parity)
            sa = sa * TableStrings.center(18, TableStrings.levels_if(line.initialLevel.index, line.finalLevel.index); na=2)
            sa = sa * TableStrings.center(18, TableStrings.symmetries_if(isym, fsym); na=4)
            sa = sa * @sprintf("%.6e", Defaults.convertUnits("energy: from atomic", line.initialLevel.energy))  * "    "
            sa = sa * @sprintf("%.6e", Defaults.convertUnits("energy: from atomic", line.electronEnergy))       * "    "
            sa = sa * @sprintf("%.6e", Defaults.convertUnits("rate: from atomic", line.totalRate))              * "    "
            sa = sa * TableStrings.flushright(13, @sprintf("%.4e", line.angularAlpha))            * "    "
            println(stream, sa)
        end
        println(stream, "  ", TableStrings.hLine(nx))
        #
        return( nothing )
    end

end # module


module VolumeRHS

using BenchmarkTools
using CUDA
using StableRNGs
using StaticArrays

function loopinfo(name, expr, nodes...)
    if expr.head != :for
        error("Syntax error: pragma $name needs a for loop")
    end
    push!(expr.args[2].args, Expr(:loopinfo, nodes...))
    return expr
end

macro unroll(expr)
    expr = loopinfo("@unroll", expr, (Symbol("llvm.loop.unroll.full"),))
    return esc(expr)
end

# HACK: module-local versions of core arithmetic; needed to get FMA
for (jlf, f) in zip((:+, :*, :-), (:add, :mul, :sub))
    for (T, llvmT) in ((Float32, "float"), (Float64, "double"))
        ir = """
            %x = f$f contract nsz $llvmT %0, %1
            ret $llvmT %x
        """
        @eval begin
            # the @pure is necessary so that we can constant propagate.
            @inline Base.@pure function $jlf(a::$T, b::$T)
                Base.llvmcall($ir, $T, Tuple{$T, $T}, a, b)
            end
        end
    end
    @eval function $jlf(args...)
        Base.$jlf(args...)
    end
end

let (jlf, f) = (:div_arcp, :div)
    for (T, llvmT) in ((Float32, "float"), (Float64, "double"))
        ir = """
            %x = f$f fast $llvmT %0, %1
            ret $llvmT %x
        """
        @eval begin
            # the @pure is necessary so that we can constant propagate.
            @inline Base.@pure function $jlf(a::$T, b::$T)
                Base.llvmcall($ir, $T, Tuple{$T, $T}, a, b)
            end
        end
    end
    @eval function $jlf(args...)
        Base.$jlf(args...)
    end
end
rcp(x) = div_arcp(one(x), x) # still leads to rcp.rn which is also a function call

# div_fast(x::Float32, y::Float32) = ccall("extern __nv_fast_fdividef", llvmcall, Cfloat, (Cfloat, Cfloat), x, y)
# rcp(x) = div_fast(one(x), x)

# note the order of the fields below is also assumed in the code.
const _nstate = 5
const _ρ, _U, _V, _W, _E = 1:_nstate
const stateid = (ρ = _ρ, U = _U, V = _V, W = _W, E = _E)

const _nvgeo = 14
const _ξx, _ηx, _ζx, _ξy, _ηy, _ζy, _ξz, _ηz, _ζz, _MJ, _MJI,
_x, _y, _z = 1:_nvgeo
const vgeoid = (ξx = _ξx, ηx = _ηx, ζx = _ζx,
                ξy = _ξy, ηy = _ηy, ζy = _ζy,
                ξz = _ξz, ηz = _ηz, ζz = _ζz,
                MJ = _MJ, MJI = _MJI,
                x = _x,   y = _y,   z = _z)

const N = 4
const nmoist = 0
const ntrace = 0

Base.@irrational grav 9.81 BigFloat(9.81)
Base.@irrational gdm1 0.4 BigFloat(0.4)

function volumerhs!(rhs, Q, vgeo, gravity, D, nelem)
    Q = Base.Experimental.Const(Q)
    vgeo = Base.Experimental.Const(vgeo)
    D = Base.Experimental.Const(D)

    nvar = _nstate + nmoist + ntrace

    Nq = N + 1

    s_D = CuStaticSharedArray(eltype(D), (Nq, Nq))
    s_F = CuStaticSharedArray(eltype(Q), (Nq, Nq, _nstate))
    s_G = CuStaticSharedArray(eltype(Q), (Nq, Nq, _nstate))

    r_rhsρ = MArray{Tuple{Nq}, eltype(rhs)}(undef)
    r_rhsU = MArray{Tuple{Nq}, eltype(rhs)}(undef)
    r_rhsV = MArray{Tuple{Nq}, eltype(rhs)}(undef)
    r_rhsW = MArray{Tuple{Nq}, eltype(rhs)}(undef)
    r_rhsE = MArray{Tuple{Nq}, eltype(rhs)}(undef)

    e = blockIdx().x
    j = threadIdx().y
    i = threadIdx().x

    @inbounds begin
        for k in 1:Nq
            r_rhsρ[k] = zero(eltype(rhs))
            r_rhsU[k] = zero(eltype(rhs))
            r_rhsV[k] = zero(eltype(rhs))
            r_rhsW[k] = zero(eltype(rhs))
            r_rhsE[k] = zero(eltype(rhs))
        end

        # fetch D into shared
        s_D[i, j] = D[i, j]
        @unroll for k in 1:Nq
            sync_threads()

            # Load values will need into registers
            MJ = vgeo[i, j, k, _MJ, e]
            ξx, ξy, ξz = vgeo[i,j,k,_ξx,e], vgeo[i,j,k,_ξy,e], vgeo[i,j,k,_ξz,e]
            ηx, ηy, ηz = vgeo[i,j,k,_ηx,e], vgeo[i,j,k,_ηy,e], vgeo[i,j,k,_ηz,e]
            ζx, ζy, ζz = vgeo[i,j,k,_ζx,e], vgeo[i,j,k,_ζy,e], vgeo[i,j,k,_ζz,e]
            z = vgeo[i,j,k,_z,e]

            U, V, W = Q[i, j, k, _U, e], Q[i, j, k, _V, e], Q[i, j, k, _W, e]
            ρ, E = Q[i, j, k, _ρ, e], Q[i, j, k, _E, e]

            # GPU performance trick
            # Allow optimizations to use the reciprocal of an argument rather than perform division.
            # IEEE floating-point division is implemented as a function call
            ρinv = rcp(ρ)
            ρ2inv = rcp(2ρ)
            # ρ2inv = 0.5f0 * pinv

            P = gdm1*(E - (U^2 + V^2 + W^2)*ρ2inv - ρ*gravity*z)

            fluxρ_x = U
            fluxU_x = ρinv * U * U + P
            fluxV_x = ρinv * U * V
            fluxW_x = ρinv * U * W
            fluxE_x = ρinv * U * (E + P)

            fluxρ_y = V
            fluxU_y = ρinv * V * U
            fluxV_y = ρinv * V * V + P
            fluxW_y = ρinv * V * W
            fluxE_y = ρinv * V * (E + P)

            fluxρ_z = W
            fluxU_z = ρinv * W * U
            fluxV_z = ρinv * W * V
            fluxW_z = ρinv * W * W + P
            fluxE_z = ρinv * W * (E + P)

            s_F[i, j,  _ρ] = MJ * (ξx * fluxρ_x + ξy * fluxρ_y + ξz * fluxρ_z)
            s_F[i, j,  _U] = MJ * (ξx * fluxU_x + ξy * fluxU_y + ξz * fluxU_z)
            s_F[i, j,  _V] = MJ * (ξx * fluxV_x + ξy * fluxV_y + ξz * fluxV_z)
            s_F[i, j,  _W] = MJ * (ξx * fluxW_x + ξy * fluxW_y + ξz * fluxW_z)
            s_F[i, j,  _E] = MJ * (ξx * fluxE_x + ξy * fluxE_y + ξz * fluxE_z)

            s_G[i, j,  _ρ] = MJ * (ηx * fluxρ_x + ηy * fluxρ_y + ηz * fluxρ_z)
            s_G[i, j,  _U] = MJ * (ηx * fluxU_x + ηy * fluxU_y + ηz * fluxU_z)
            s_G[i, j,  _V] = MJ * (ηx * fluxV_x + ηy * fluxV_y + ηz * fluxV_z)
            s_G[i, j,  _W] = MJ * (ηx * fluxW_x + ηy * fluxW_y + ηz * fluxW_z)
            s_G[i, j,  _E] = MJ * (ηx * fluxE_x + ηy * fluxE_y + ηz * fluxE_z)

            r_Hρ = MJ * (ζx * fluxρ_x + ζy * fluxρ_y + ζz * fluxρ_z)
            r_HU = MJ * (ζx * fluxU_x + ζy * fluxU_y + ζz * fluxU_z)
            r_HV = MJ * (ζx * fluxV_x + ζy * fluxV_y + ζz * fluxV_z)
            r_HW = MJ * (ζx * fluxW_x + ζy * fluxW_y + ζz * fluxW_z)
            r_HE = MJ * (ζx * fluxE_x + ζy * fluxE_y + ζz * fluxE_z)

            # one shared access per 10 flops
            for n = 1:Nq
                Dkn = s_D[k, n]

                r_rhsρ[n] += Dkn * r_Hρ
                r_rhsU[n] += Dkn * r_HU
                r_rhsV[n] += Dkn * r_HV
                r_rhsW[n] += Dkn * r_HW
                r_rhsE[n] += Dkn * r_HE
            end

            r_rhsW[k] -= MJ * ρ * gravity

            sync_threads()

            # loop of ξ-grid lines
            @unroll for n = 1:Nq
                Dni = s_D[n, i]
                Dnj = s_D[n, j]

                r_rhsρ[k] += Dni * s_F[n, j, _ρ]
                r_rhsρ[k] += Dnj * s_G[i, n, _ρ]

                r_rhsU[k] += Dni * s_F[n, j, _U]
                r_rhsU[k] += Dnj * s_G[i, n, _U]

                r_rhsV[k] += Dni * s_F[n, j, _V]
                r_rhsV[k] += Dnj * s_G[i, n, _V]

                r_rhsW[k] += Dni * s_F[n, j, _W]
                r_rhsW[k] += Dnj * s_G[i, n, _W]

                r_rhsE[k] += Dni * s_F[n, j, _E]
                r_rhsE[k] += Dnj * s_G[i, n, _E]
            end
        end # k

        @unroll for k in 1:Nq
            MJI = vgeo[i, j, k, _MJI, e]

            # Updates are a performance bottleneck
            # primary source of stall_long_sb
            rhs[i, j, k, _U, e] += MJI*r_rhsU[k]
            rhs[i, j, k, _V, e] += MJI*r_rhsV[k]
            rhs[i, j, k, _W, e] += MJI*r_rhsW[k]
            rhs[i, j, k, _ρ, e] += MJI*r_rhsρ[k]
            rhs[i, j, k, _E, e] += MJI*r_rhsE[k]
        end
    end
    return
end

function main()
    DFloat = Float32
    nelem = 240_000

    rng = StableRNG(123)

    Nq = N + 1
    nvar = _nstate + nmoist + ntrace

    Q = 1 .+ CuArray(rand(rng, DFloat, Nq, Nq, Nq, nvar, nelem))
    Q[:, :, :, _E, :] .+= 20
    vgeo = CuArray(rand(rng, DFloat, Nq, Nq, Nq, _nvgeo, nelem))

    # make sure the entries of the mass matrix satisfy the inverse relation
    vgeo[:, :, :, _MJ, :] .+= 3
    vgeo[:, :, :, _MJI, :] .= 1 ./ vgeo[:, :, :, _MJ, :]

    D = CuArray(rand(rng, DFloat, Nq, Nq))

    rhs = CuArray(zeros(DFloat, Nq, Nq, Nq, nvar, nelem))

    threads=(N+1, N+1)

    kernel = @cuda launch=false volumerhs!(rhs, Q, vgeo, DFloat(grav), D, nelem)
    # XXX: should we print these for all kernels? maybe upload them to Codespeed?
    @info """volumerhs! details:
              - $(CUDA.registers(kernel)) registers, max $(CUDA.maxthreads(kernel)) threads
              - $(Base.format_bytes(CUDA.memory(kernel).local)) local memory,
                $(Base.format_bytes(CUDA.memory(kernel).shared)) shared memory,
                $(Base.format_bytes(CUDA.memory(kernel).constant)) constant memory"""
    results = @benchmark begin
        CUDA.@sync $kernel($rhs, $Q, $vgeo, $(DFloat(grav)), $D, $nelem;
                           threads=$threads, blocks=$nelem)
    end

    # BenchmarkTools captures inputs, JuliaCI/BenchmarkTools.jl#127, so forcibly free them
    CUDA.unsafe_free!(rhs)
    CUDA.unsafe_free!(Q)
    CUDA.unsafe_free!(vgeo)
    CUDA.unsafe_free!(D)

    results
end

end

VolumeRHS.main()


using DFTK
using MPI
using JSON
using JLD2
using Printf
using Plots
using Statistics
using LinearAlgebra
using Dates
using Logging
include("quantum_espresso.jl")

function build_magnetic_moments(atoms::AbstractArray; magmoms...)
    magmoms = Dict(magmoms)
    map(atoms) do (element, positions)
        element => fill(magmoms[element.symbol], length(positions))
    end
end

function attach_pseudos(atoms::AbstractArray; pseudomap...)
    pseudomap = Dict(pseudomap)

    map(atoms) do (element, position)
        pspfile = get(pseudomap, element.symbol, nothing)
        ElementPsp(element.symbol, psp=load_psp(pspfile)) => position
    end
end

function standard_cases()
    cases = Dict{String, Any}(
        "adaptive"    => (damping=DFTK.AdaptiveDamping(), ),  # Parameter-free adaptive damping
        "adaptive_05" => (damping=DFTK.AdaptiveDamping(0.5), ),
    )
    for α in 1:9
        cases["standard_0$α"] = (damping=DFTK.FixedDamping(α / 10), )
    end
    cases["standard_10"] = (damping=DFTK.FixedDamping(1.0), )
    cases
end

function random_logfile(prefix)
    logfile = "$(prefix)_$(@sprintf "%05d" rand(UInt16)).log"
    while isfile(logfile)
        logfile = "$(prefix)_$(@sprintf "%05d" rand(UInt16)).log"
    end
    logfile
end

function run_step_on_cases(payload, cases)
    spay = string(payload)
    header_printed = false

    anymissing = false
    for dir in sort(collect(keys(cases)))
        config = cases[dir]

        MPI.Barrier(MPI.COMM_WORLD)
        if isfile(joinpath(config.directory, string(payload) * ".done"))
            continue
        end

        for i in 1:10  # To avoid infinite loop
            MPI.Barrier(MPI.COMM_WORLD)
            res = nothing
            logfile = nothing
            if mpi_master()
                if !header_printed
                    println("#####" * "#"^length(spay) * "#####")
                    println("#--  $(string(payload))  --#")
                    println("#####" * "#"^length(spay) * "#####")
                    header_printed = true
                end

                logfile = random_logfile(joinpath(config.directory, string(payload)))
                println(basename(dir), "  ->  ", basename(dir) * "/" * basename(logfile))
                res = open(logfile, "w") do fp
                    redirect_stdout(fp) do
                        with_logger(SimpleLogger(stdout, Logging.Debug)) do
                            payload(config)
                        end
                    end
                end
                println("    ... $res\n")
            else
                res = payload(config)
            end

            if res == :exhausted
                if mpi_master()
                    rm(logfile)
                    anymissing || touch(joinpath(config.directory, string(payload) * ".done"))
                end
                break
            elseif res == :missing
                anymissing = true
                mpi_master() && rm(logfile)
            end
        end
    end

    MPI.Barrier(MPI.COMM_WORLD)
    nothing
end

function has_results(fileprefix)
    resultsfile = fileprefix * ".scfres.json"
    isfile(resultsfile) && open(JSON.parse, resultsfile, "r")["done"]
end

function basis_from_config(config)
    model = model_PBE(config.lattice, config.atoms;
                      magnetic_moments=config.magnetic_moments, config.kwargs_model...)
    fft_size = compute_fft_size(model, config.Ecut, supersampling=config.supersampling)
    if !haskey(config, :kgrid)
        kgrid = kgrid_size_from_minimal_spacing(config.lattice, 2π * 0.022)
    else
        kgrid=config.kgrid
    end
    PlaneWaveBasis(model, config.Ecut, kgrid=kgrid, kshift=[0, 0, 0], fft_size=fft_size)
end

function scf(config)
    basis = basis_from_config(config)

    scfkey = nothing
    if mpi_master()
        for scf in sort(collect(keys(config.cases)))
            lockfile = joinpath(config.directory, scf * ".running")
            fileprefix = joinpath(config.directory, scf)
            (isfile(lockfile) || has_results(fileprefix))  && continue
            write(lockfile, read(`hostname`, String))
            scfkey = scf
            break
        end
    end
    scfkey = MPI.bcast(scfkey, 0, MPI.COMM_WORLD)

    scfkey === nothing && return :exhausted
    fileprefix  = joinpath(config.directory, scfkey)
    lockfile    = joinpath(config.directory, scfkey * ".running")
    resultsfile = joinpath(config.directory, scfkey * ".scfres.json")
    guessfile   = joinpath(config.directory, "init.jld2")

    !isfile(guessfile) && return :missing
    ρ, ψ0, V = load_guess(basis, guessfile)

    try
        if mpi_master()
            println("#\n#-- SCF $(basename(config.directory)) -- case=$scfkey\n#")
            println("    keywords: $((; config.cases[scfkey]..., config.kwargs_scf...))")
        end

        MPI.Barrier(MPI.COMM_WORLD)
        mpi_master() && DFTK.reset_timer!(DFTK.timer)
        if haskey(config.cases[scfkey], :type) && config.cases[scfkey].type == :qe
            caseargs = Dict(k => v for (k, v) in pairs(config.cases[scfkey]) if k != :type)
            run_qe(basis, fileprefix, resultsfile;
                   magnetic_moments=config.magnetic_moments, caseargs...)
        else
            run_dftk(basis, resultsfile; ρ=ρ, ψ0=ψ0, V=V,
                     mixing=config.mixing, config.cases[scfkey]..., config.kwargs_scf...)
        end
        MPI.Barrier(MPI.COMM_WORLD)
        mpi_master() && println(DFTK.timer)
    catch
        mpi_master() && rm(joinpath(config.directory, "scf.done"), force=true)
        rethrow()
    finally
        mpi_master() && rm(lockfile)
    end
    mpi_master() && plot_convergence_summary(config)

    :done
end

function dump_guess(config)
    basis     = basis_from_config(config)
    guessfile = joinpath(config.directory, "init.jld2")
    isfile(guessfile) && return :exhausted

    ρ = guess_density(basis, config.magnetic_moments)
    _, ham = energy_hamiltonian(basis, nothing, nothing; ρ=ρ)

    V = nothing
    isnothing(config.potential) || (V = config.potential(basis))
    if !isnothing(V)
        ham = DFTK.hamiltonian_with_total_potential(ham, V)
    end

    n_bands = (haskey(config, :n_bands) ? (n_bands = config.n_bands)
                                        : DFTK.default_n_bands(basis.model))
    res_ψ0 = DFTK.next_density(ham; n_bands=DFTK.default_n_bands(basis.model),
                               n_ep_extra=3, miniter=2, tol=0.0001)
    diagiter = DFTK.mpi_mean(mean(res_ψ0.diagonalization.iterations), basis.comm_kpts)
    mpi_master() && println("guess:    diag = $diagiter  diagtol = 0.0001")
    isnothing(V) && (V = DFTK.total_local_potential(ham))

    # Gather ψ array on master
    ψ = DFTK.gather_kpts(res_ψ0.ψ, basis)
    if mpi_master()
        JLD2.jldopen(guessfile, "w") do jld
            jld["ρ"] = ρ
            jld["V"] = V
            jld["ψ"] = ψ
        end
    end

    :done
end

function load_guess(basis, guessfile)
    JLD2.jldopen(guessfile, "r") do jld
        ψ = jld["ψ"][basis.krange_thisproc]
        jld["ρ"], ψ, jld["V"]
    end
end

function run_dftk(basis, resultsfile;
                  ρ, ψ0, V, mixing=SimpleMixing(),
                  damping, tol=1e-11, type=:potmix, diagtol_max=1e-3, kwargs...)

    n_bands = (haskey(config, :n_bands) ? (n_bands = config.n_bands)
                                        : DFTK.default_n_bands(basis.model))
    if mpi_master()
        println()
        println("hostname        = $(read(`hostname`, String))")
        println("started on      = $(Dates.now())")
        println("julia threads   = $(Threads.nthreads())")
        println("BLAS threads    = $(BLAS.get_num_threads())")
        println("MPI procs       = $(mpi_nprocs())")
        println()
        println("temperature     = $(basis.model.temperature)")
        println("smearing        = $(basis.model.smearing)")
        println("lattice         = $(round.(basis.model.lattice, sigdigits=4))")
        println("Ecut            = $(basis.Ecut)")
        println("fft_size        = $(basis.fft_size)")
        println("kgrid           = $(basis.kgrid)")
        println("kshift          = $(basis.kshift)")
        println("n_irreducible_k = $(sum(length, basis.krange_allprocs))")
        println("n_bands         = $(n_bands)")
        println("n_electrons     = $(basis.model.n_electrons)")
        println("type            = $type")
        println("mixing          = $mixing")
        println("damping         = $damping")
        flush(stdout)
    end

    # Collection of data to dump later
    save_dict = Dict(
        "temperature"  => basis.model.temperature,
        "smearing"     => string(basis.model.smearing),
        "lattice"      => basis.model.lattice,
        "Ecut"         => basis.Ecut,
        "fft_size"     => basis.fft_size,
        "kgrid"        => basis.kgrid,
        "n_kirred"     => sum(length, basis.krange_allprocs),
        "n_bands"      => n_bands,
        "n_electr"     => basis.model.n_electrons,
        "mixing"       => string(mixing),
        "damping"      => string(damping),
        "type"         => string(type),
        "kwargs"       => string(kwargs),
        "done"         => false,
        #
        "energies"     => Float64[],
        "residuals"    => Float64[],
        "ndiag"        => Int[],
        "α"            => Float64[],
    )

    # Empty existing results
    if mpi_master()
        open(fp -> JSON.print(fp, save_dict), resultsfile, "w")
    end

    function ExtractCallback()
        function callback(info)
            if info.stage != :finalize
                push!(save_dict["energies"],  info.energies.total)
                push!(save_dict["ndiag"],     length(info.diagonalization))
                push!(save_dict["α"], isnan(info.α) ? DFTK.trial_damping(damping) : info.α)

                if type == :potmix
                    # Note: This misses a factor sqrt(info.basis.dvol)), which is taken into
                    # account in the analyse.jl script
                    push!(save_dict["residuals"], norm(info.Vout - info.Vin))
                elseif type == :densmix
                    # Note: This misses a factor sqrt(info.basis.dvol)), which is taken into
                    # account in the analyse.jl script
                    push!(save_dict["residuals"], norm(info.ρout - info.ρin))
                end
            end

            if mpi_master()
                open(fp -> JSON.print(fp, save_dict), resultsfile, "w")
            end
        end
    end

    determine_diagtol = DFTK.ScfDiagtol(; ratio_ρdiff=0.01, diagtol_max, diagtol_min=1e-12)
    callback = ExtractCallback() ∘ DFTK.ScfDefaultCallback()

    if type == :potmix
        run_scf = ((damping isa DFTK.AdaptiveDamping) ? DFTK.scf_potential_mixing_adaptive
                                                      : DFTK.scf_potential_mixing)
        scfres = run_scf(basis; tol=tol, ψ=ψ0, determine_diagtol=determine_diagtol,
                         mixing=mixing, callback=callback, damping=damping, ρ=ρ, V=V,
                         diag_miniter=2, kwargs...)
    elseif type == :densmix
        @assert damping isa Number
        scfres = self_consistent_field(basis; tol=tol, ψ=ψ0,
                                       determine_diagtol=determine_diagtol,
                                       mixing=mixing, callback=callback,
                                       damping=damping, ρ=ρ, kwargs...)
    else
        error("Unsupported SCF type: $type")
    end

    if mpi_master()
        println()
        println(scfres.energies)
        flush(stdout)
        save_dict["done"] = true
        open(fp -> JSON.print(fp, save_dict), resultsfile, "w")
    end

    scfres
end

function run_qe(basis, qe_directory, resultsfile; magnetic_moments,
                mixing_beta=0.7, kwargs...)
    if !mpi_master()
        MPI.Barrier(MPI.COMM_WORLD)
        return nothing
    end
    # This only runs on MPI master

    println()
    println("hostname        = $(read(`hostname`, String))")
    println("started on      = $(Dates.now())")
    println()
    flush(stdout)

    model = basis.model
    outputfile = joinpath(qe_directory, "espresso.pwo")
    if !isfile(outputfile)
        qeres = scf_quantum_espresso(model.lattice, model.atoms;
                                     Ecut=basis.Ecut,
                                     fileprefix=joinpath(qe_directory, "espresso"),
                                     temperature=model.temperature,
                                     mixing_beta=mixing_beta,
                                     kgrid=basis.kgrid,
                                     magnetic_moments=magnetic_moments,
                                     kwargs...)
    @assert qeres.outputfile == outputfile
    end

    # Copy output file to stdout
    println.(readlines(outputfile))
    flush(stdout)

    # Extract results
    save_dict = Dict(
        "done"         => true,
        "energies"     => Float64[],
        "residuals"    => Float64[],
        "ndiag"        => Int[],
        "α"            => Float64[],
    )
    for iteration in parse_quantum_espresso_pwo(outputfile)
        push!(save_dict["energies"],  iteration.total_energy)
        push!(save_dict["residuals"], iteration.accuracy)
        push!(save_dict["ndiag"],     1)
        push!(save_dict["α"],         mixing_beta)
    end
    open(fp -> JSON.print(fp, save_dict), resultsfile, "w")

    MPI.Barrier(MPI.COMM_WORLD)
    nothing
end

function collect_minimal_total_energy(config::NamedTuple; kwargs...)
    collect_minimal_total_energy(config.directory, config.cases; kwargs...)
end
function collect_minimal_total_energy(directory::AbstractString, cases::AbstractDict;
                                      kwargs...)
    collect_minimal_total_energy(directory, collect(keys(cases)); kwargs...)
end
function collect_minimal_total_energy(directory::AbstractString, cases::AbstractVector;
                                      tol_resid=0.05, tol_energy=1e-4)
    mintotals = Float64[]
    for scfkey in sort(cases)
        startswith(scfkey, "espresso_") && continue
        resultfile = joinpath(directory, scfkey * ".scfres.json")
        isfile(resultfile) || continue

        residuals = open(JSON.parse, resultfile, "r")["residuals"]
        energies  = open(JSON.parse, resultfile, "r")["energies"]
        energies === nothing && continue
        isempty(energies) && continue

        minidx = argmin(energies[max(end-4, 1):end]) + max(length(energies) - 4, 1) - 1
        residuals[minidx] > tol_resid && continue

        min_matching = findfirst(e -> abs(e - energies[minidx]) < tol_energy, mintotals)
        if min_matching === nothing
            push!(mintotals, energies[minidx])
        else
            mintotals[min_matching] = min(mintotals[min_matching], energies[minidx])
        end
    end

    mintotals
end

function parse_scfkey_label(scfkey)
    if scfkey == "adaptive"
        ("adaptive", 0.2)
    elseif startswith(scfkey, "adaptive_") && all(isnumeric, scfkey[10:end])
        digits = length(scfkey[10:end]) - 1
        alpha  = parse(Int, scfkey[10:end]) / 10^digits
        ("adaptive", alpha)
    elseif startswith(scfkey, "adaptive_")
        ("adaptive", scfkey[10:end])
    elseif startswith(scfkey, "standard_") && all(isnumeric, scfkey[10:end])
        digits = length(scfkey[10:end]) - 1
        alpha  = parse(Int, scfkey[10:end]) / 10^digits
        ("fixed", alpha)
    elseif startswith(scfkey, "espresso_") && all(isnumeric, scfkey[10:end])
        digits = length(scfkey[10:end]) - 1
        alpha  = parse(Int, scfkey[10:end]) / 10^digits
        ("espresso", alpha)
    elseif startswith(scfkey, "highprec_") && all(isnumeric, scfkey[10:end])
        digits = length(scfkey[10:end]) - 1
        alpha  = parse(Int, scfkey[10:end]) / 10^digits
        ("highprec", alpha)
    else
        (scfkey, nothing)
    end
end

function plot_convergence_summary(config; iteration=Inf, tol=1e-10, damping=true,
                                  outputfile=joinpath(config.directory, "summary.svg"),
                                  scfkeys=nothing, mintotals=nothing, kwargs...)
    function deduce_label(scfkey)
        method, alpha = parse_scfkey_label(scfkey)
        method * (isnothing(alpha) ? "" : " (α = $alpha)")
    end

    !mpi_master() && error("Plotting only on MPI master supported.")
    if isnothing(mintotals)
        mintotals = collect_minimal_total_energy(config)
    end
    isempty(mintotals) && return plot()  # Return on no data

    if scfkeys === nothing
        scfkeys = collect(keys(config.cases))
    end

    errors   = Dict{String, Vector{Float64}}()
    cumdiags = Dict{String, Vector{Int}}()
    dampings = Dict{String, Vector{Float64}}()
    whichmin = Dict{String, Int}()
    for scfkey in sort(scfkeys)
        method, _ = parse_scfkey_label(scfkey)
        resultfile = joinpath(config.directory, scfkey * ".scfres.json")
        isfile(resultfile) || continue

        energies  = open(JSON.parse, resultfile, "r")["energies"]
        ndiags    = open(JSON.parse, resultfile, "r")["ndiag"]
        damps     = open(JSON.parse, resultfile, "r")["α"]

        energies === nothing && continue
        if method == "espresso"
            values = open(JSON.parse, resultfile, "r")["residuals"]
            mintrial  = minimum(energies[max(end-4, 1):end])
            bestmatch = sortperm(abs.(mintrial .- mintotals))[1]
        else
            isempty(energies) && continue
            # Find best matching minimum
            mintrial  = minimum(energies[max(end-4, 1):end])
            bestmatch = sortperm(abs.(mintrial .- mintotals))[1]
            values = abs.(energies .- mintotals[bestmatch])
            idx = findfirst(values .< 1e-16)
            if !isnothing(idx)
                values = values[1:idx-1]
                ndiags = ndiags[1:idx-1]
                damps  = damps[1:idx-1]
            end
        end
        isempty(values) && continue

        whichmin[scfkey] = bestmatch
        errors[scfkey]   = values
        dampings[scfkey] = damps
        cumdiags[scfkey] = cumsum(ndiags)
    end

    # If no data return
    isempty(errors) && return plot()

    any_adaptive = false
    ymax = 0
    maxiter = 0
    colors = Dict{Any, Int}()
    p = plot(; yaxis=:log, legend=:topright, title=basename(config.directory), kwargs...)
    for (i, scfkey) in enumerate(sort(collect(keys(errors))))
        label = deduce_label(scfkey)
        maxiter = max(maxiter, length(errors[scfkey]))
        ymax = max(ymax, maximum(errors[scfkey]))

        if length(mintotals) > 1
            label *= " (min $(whichmin[scfkey]))"
        end

        method, alpha = parse_scfkey_label(scfkey)
        if isnothing(alpha)
            color = i
        elseif alpha in keys(colors)
            color = colors[alpha]
        else
            colors[alpha] = i
            color = colors[alpha]
        end

        mark = :x
        if method == "adaptive"
            mark = :o
            any_adaptive = true
        elseif method == "espresso"
            mark = :rtriangle
        elseif method == "highprec"
            mark = :ltriangle
        end
        plot!(p, cumdiags[scfkey], errors[scfkey];
              label=label, color=color, mark=mark, lw=1.3, markersize=4)
    end
    ylims!(p, (tol, 2 * ymax))
    ylabel!(p, "Total energy absolute error")

    if damping && any_adaptive
        q = twinx()
        for scfkey in sort(collect(keys(errors)))
            method, alpha = parse_scfkey_label(scfkey)
            method == "adaptive" || continue
            (alpha in keys(colors)) || continue
            plot!(q, cumdiags[scfkey], abs.(dampings[scfkey]), mark=:+,
                  color=colors[alpha], label="", linestyle=:dot)
        end
        ylims!(q, (0.0, 1.0))
        ylabel!(q, "Damping parameter")
    end
    xlims!(p, (0, min(iteration, maxiter)))
    xlabel!(p, "Iteration / Hamiltonian diagonalization")

    if isnothing(outputfile)
        p
    else
        savefig(p, outputfile)
    end
end


# This file is a part of AstroLib.jl. License is MIT "Expat".
# Copyright (C) 2016 Mosè Giordano.

"""
List of locations of North Magnetic Pole since 1590.

This is provided by World Magnetic Model
(https://www.ngdc.noaa.gov/geomag/data/poles/NP.xy).
"""
const POLELATLONG =
    try
        let
            local polelatlong, rows, floattype, temp
            polelatlong = readdlm(joinpath(dirname(@__FILE__),
                                           "..", "deps", "NP.xy"))
            rows = size(polelatlong, 1)
            floattype = typeof(polelatlong[1])
            temp = Dict{floattype, Tuple{floattype, floattype}}()
            for i = 1:rows
                merge!(temp, Dict(polelatlong[2rows + i]=>
                                  (polelatlong[rows + i],
                                   polelatlong[i])))
            end
            temp
        end
    catch
        warn("""Could not find file NP.xy, you will not be able to use
"geo2mag" and "mag2geo" functions.  Build again the package with
    Pkg.build("AstroLib")
then restart Julia session in order to fix this problem.""")
    end

const AU = 1.495978707e11 # Astronomical unit in meters
const J2000 = jdcnv(2000, 01, 01, 12) # 2451545
const JULIANYEAR = 365.25 # days in Julian year

# Constant used in ct2lst function, see Meeus, p.84.
const ct2lst_c  = (280.46061837, 360.98564736629, 0.000387933, 38710000.0)

# Used in "bprecess" and "jprecess".
const A_precess  = 1e-6*[-1.62557, -0.31919, -0.13843] # In radians
const A_dot_precess = 0.001*[1.244 , -1.579, -0.660] # In arc seconds per century

"""
List of observing sites.  The observatories have `Observatory` type.
"""
const observatories =
    Dict("mgio"=>Observatory("Mount Graham International Observatory",
                             (32,42,04.69), (-109,53,31.25), 3191.0, -7),
         "whitin"=>Observatory("Whitin Observatory, Wellesley College",
                               42.295, -71.305833, 32, -5),
         "fmo"=>Observatory("Fan Mountain Observatory",
                            (37,52,41), (-78,41,34), 556, -5),
         "lmo"=>Observatory("Leander McCormick Observatory",
                            (38,02,00), (-78,31,24), 264, -5),
         "holi"=>Observatory("Observatorium Hoher List (Universitaet Bonn) - Germany",
                             50.16276, 6.85, 541, 1),
         "ca"=>Observatory("Calar Alto Observatory",
                           (37,13,25), (-2,32,46.5), 2168, 1),
         "bgsuo"=>Observatory("Bowling Green State Univ Observatory",
                              (41,22,42), (-83,39,33), 225., -5),
         "irtf"=>Observatory("NASA Infrared Telescope Facility",
                             19.826218, -155.471999, 4168, -10),
         "rozhen"=>Observatory("National Astronomical Observatory Rozhen - Bulgaria",
                               (41,41,35), (24,44,38), 1759, 2),
         "bosque"=>Observatory("Estacion Astrofisica Bosque Alegre, Cordoba",
                               (-31,35,54), (-64,32,45), 1250, -3),
         "casleo"=>Observatory("Complejo Astronomico El Leoncito, San Juan",
                               (-31,47,57), (-69,18,00), 2552, -3),
         "saao"=>Observatory("South African Astronomical Observatory",
                             (-32,22,46), (20,48,38.5), 1798., 2),
         "lna"=>Observatory("Laboratorio Nacional de Astrofisica - Brazil",
                            (-22,32,04), -45.5825, 1864., -3),
         "oro"=>Observatory("Oak Ridge Observatory",
                            (42,30,18.94), (-71,33,29.32), 184., -5),
         "flwo"=>Observatory("Whipple Observatory",
                             (31,40,51.4), (-110,52,39), 2320., -7),
         "vbo"=>Observatory("Vainu Bappu Observatory",
                            12.57666, 78.8266, 725., 5.5),
         "lowell"=>Observatory("Lowell Observatory",
                               (35,05.8), (-111,32.1), 2198., -7),
         "apo"=>Observatory("Apache Point Observatory",
                            (32,46.8), (-105,49.2), 2798., -7),
         "loiano"=>Observatory("Bologna Astronomical Observatory, Loiano - Italy",
                               (44,15,33), (11,20,2), 785., 1),
         "ekar"=>Observatory("Mt. Ekar 182 cm. Telescope",
                             (45,50,54.92), (11,34,52.08), 1413.69, 1),
         "keck"=>Observatory("W. M. Keck Observatory",
                             (19,49.7), (-155,28.7), 4160., -10),
         "bao"=>Observatory("Beijing XingLong Observatory",
                            (40,23.6), (117,34.5), 950., 8),
         "bmo"=>Observatory("Black Moshannon Observatory",
                            (40,55.3), (-78,00.3), 738., -5),
         "nov"=>Observatory("National Observatory of Venezuela",
                            (8,47.4), (-70,52.0), 3610, -4),
         "mdm"=>Observatory("Michigan-Dartmouth-MIT Observatory",
                            (31,57.0), (-111,37.0), 1938.5, -7),
         "palomar"=>Observatory("The Hale Telescope",
                                (33,21,21.6), (-116,51,46.80), 1706., -8),
         # https://en.wikipedia.org/wiki/Tonantzintla_Observatory
         "tona"=>Observatory("Observatorio Astronomico Nacional, Tonantzintla",
                             (19,01,58), (-98,18,50), 2166., -6),
         "spm"=>Observatory("Observatorio Astronomico Nacional, San Pedro Martir",
                            (31,01,45), (-115,29,13), 2830., -7),
         "dao"=>Observatory("Dominion Astrophysical Observatory",
                            (48,31.3), (-123,25.0), 229., -8),
         "mtbigelow"=>Observatory("Catalina Observatory: 61 inch telescope",
                                  (32,25.0), (-110,43.9), 2510., -7),
         "lco"=>Observatory("Las Campanas Observatory",
                            (-29,0.2), (-70,42.1), 2282, -4),
         "mcdonald"=>Observatory("McDonald Observatory",
                                 30.6716667, -104.0216667, 2075, -6),
         "aao"=>Observatory("Anglo-Australian Observatory",
                            (-31,16,37.34), (149,3,57.91), 1164, 10),
         "sso"=>Observatory("Siding Spring Observatory",
                            (-31,16,24.10), (149,3,40.3), 1149, 10),
         "mso"=>Observatory("Mt. Stromlo Observatory",
                            (-35,19,14.34), (149,1,27.6), 767, 10),
         "lapalma"=>Observatory("Roque de los Muchachos, La Palma",
                                (28,45.5), (-17,52.8), 2327, 0),
         "cfht"=>Observatory("Canada-France-Hawaii Telescope",
                             (19,49.6), (-155,28.3), 4215., -10),
         "mmto"=>Observatory("MMT Observatory",
                             (31,41.3), (-110,53.1), 2600., -7),
         "lick"=>Observatory("Lick Observatory",
                             (37,20.6), (-121,38.2), 1290., -8),
         "eso"=>Observatory("European Southern Observatory",
                            (-29,15.4), (-70,43.8), 2347., -4),
         "ctio"=>Observatory("Cerro Tololo Interamerican Observatory",
                             -30.16527778, -70.815, 2215., -4),
         "kpno"=>Observatory("Kitt Peak National Observatory",
                             (31,57.8), (-111,36.0), 2120., -7))

"""
List of planets of the Solar System, from Mercury to Pluto.  The elements of the
list have `Planet` type.

Reference for most quantities is the Planetary Fact Sheet:
http://nssdc.gsfc.nasa.gov/planetary/factsheet/index.html
"""
const planets =
    Dict("mercury"=>Planet("mercury", 2439.7e3, 2439.7e3, 2439.7e3, 3.3011e23,
                           0.20563069, 57909050e3, 87.9691*86400),
         "venus"=>Planet("venus", 6051.8e3, 6051.8e3, 6051.8e3, 4.8675e24,
                         0.00677323, 108208000e3, 224.701*86400),
         "earth"=>Planet("earth", 6371e3, 6378.137e3,  6356.752e3, 5.97237e24,
                         0.0167086, 149598023e3, 365.256363004*86400),
         "mars"=>Planet("mars", 3389.5e3, 3396.2e3, 3376.2e3, 3.3011e23,
                        0.09341233, 57909050e3, 87.9691*86400),
         "jupiter"=>Planet("jupiter", 69911e3, 71492e3, 66854e3, 1898.19e24,
                           0.04839266, 5.20336301*AU, 4332.589*86400),
         "saturn"=>Planet("saturn", 58232e3, 60268e3, 54364e3, 568.34e23,
                          0.05415060, 9.53707032*AU, 10759.22*86400),
         "uranus"=>Planet("uranus", 25362e3, 25559e3, 24973e3, 86.813e24,
                          0.04716771, 19.19126393*AU, 30685.4*86400),
         "neptune"=>Planet("neptune", 24622e3, 24764e3, 24341e3, 102.413e24,
                           0.00858587, 30.06896348*AU, 60189*86400),
         "pluto"=>Planet("pluto", 1187e3, 1187e3, 1187e3, 0.01303e24,
                         0.24880766, 39.48168677*AU, 90560*86400))