# Snow crab --- Areal unit modelling of habitat  -- no reliance upon stmv fields

###------------------------------------------------
#OS Compatability
###------------------------------------------------

#The use of the BYM2 model precludes using Windows, must be run in Linux
#Virtual box install of Ubuntu or Debian is likely easiest option
#BZ- July 2020- There may be an option to allow Windows to run the CAR steps as found here:
#http://brettklamer.com/diversions/statistical/faster-blas-in-r/
#Not yet tested

#Choose one- likely latter
{
  year.assessment=lubridate::year(Sys.Date())      # year.assessment
  year.assessment=lubridate::year(Sys.Date()) -1   # or year previous to current
}

#To add a title to any carstm_plot, please see below example
#carstm_plot( p=p, res=res, vn=vn, main=list(label="my plot title", cex=2) )

# -------------------------------------------------
# Part 1 -- construct basic parameter list defining the main characteristics of the study
require(aegis)

 p = bio.snowcrab::snowcrab_carstm( DS="parameters", assessment.years=1999:year.assessment )

  # misc run params adjustments here:
  p$inla_num.threads = 6
  p$inla_blas.num.threads = 6

  plot.dir=file.path(p$modeldir,"prediction.plots", year.assessment)
  dir.create(plot.dir)
# ------------------------------------------------
# Part 2 -- polygon structure
  sppoly = areal_units( p=p )  # to reload
  plot(sppoly)
  spplot( sppoly, "au_sa_km2", main="AUID", sp.layout=p$coastLayout )
  if (0) {
    # create if not yet made
    MS = snowcrab.db( p=p, DS="biological_data"  )  # domain is  sse     # the underlying observations/data
    # ensure if polys exist and create if required
    for (au in c("cfanorth", "cfasouth", "cfa4x", "cfaall" )) plot(polygons_managementarea( species="snowcrab", au))
    sppoly = areal_units( p=p, areal_units_constraint=MS[, c("lon", "lat")], redo=TRUE )  # create constrained polygons with neighbourhood as an attribute
    coastLayout = aegis.coastline::coastline_layout( p=p, redo=TRUE )
    MS = NULL
  }

# -------------------------------------------------
# Part 3 -- create covariate field for bathymetry
# bathymetry -- ensure the data assimilation in bathymetry is first completed :: 01.bathymetry_data.R
# about 2 hrs to redo; 15 configs @ 0.5 hrs each

  pB = bathymetry_carstm( p=p, DS="parameters_override" )
  M = bathymetry.db( p=pB, DS="aggregated_data", redo=TRUE )
  M = bathymetry_carstm( p=pB, DS="carstm_inputs", redo=TRUE  ) # will redo if not found
  M = NULL; gc()
  res = carstm_model( p=pB, M='bathymetry_carstm( p=pB, DS="carstm_inputs" )', DS="redo", carstm_model_label="production"  ) # run model and obtain predictions

  if (0) {
    # to use a saved instance
    res = carstm_model( p=pB, DS="carstm_modelled", carstm_model_label="production" ) # run model and obtain predictions
    fit = carstm_model( p=pB, DS="carstm_modelled_fit", carstm_model_label="production" )  # extract currently saved model fit
    # maps of some of the results
    vn = paste(pB$variabletomodel, "predicted", sep=".")
    zplot=sc_carstm_plot( p=pB, res=res, vn=vn )

    #to save map of predicted mean

    vn = paste(pB$variabletomodel, "predicted_se", sep=".")
    zplot=sc_carstm_plot( p=pB, res=res, vn=vn )

    vn = paste(pB$variabletomodel, "random_auid_nonspatial", sep=".")
    sc_carstm_plot( p=pB, res=res, vn=vn )

    vn = paste(pB$variabletomodel, "random_auid_spatial", sep=".")
    sc_carstm_plot( p=pB, res=res, vn=vn )

    plot(fit, plot.prior=TRUE, plot.hyperparameters=TRUE, plot.fixed.effects=FALSE, single=TRUE )

    #Framework Run (to 2018)

    # Time used:
    #     Pre = 2.71, Running = 4366, Post = 3.26, Total = 4372
    # Fixed effects:
    #             mean sd 0.025quant 0.5quant 0.975quant  mode kld
    # (Intercept) 7.872  0      7.871    7.872      7.873 7.872   0

    # Random effects:
    #   Name	  Model
    #     auid BYM2 model

    # Model hyperparameters:
    #                                             mean     sd 0.025quant 0.5quant 0.975quant     mode
    # Precision for the lognormal observations 4786.388  21.24   4744.647 4786.372   4828.230 4786.400
    # Precision for auid                       1412.186 227.70   1091.296 1368.558   1964.898 1254.958
    # Phi for auid                                0.979   0.02      0.924    0.986      0.999    0.996

    # Expected number of effective parameters(stdev): 3270.84(12.81)
    # Number of equivalent replicates : 33.03

    # Deviance Information Criterion (DIC) ...............: 1094943.11
    # Deviance Information Criterion (DIC, saturated) ....: 111315.40
    # Effective number of parameters .....................: 3271.74

    # Marginal log-Likelihood:  -547965.21
    # Posterior marginals for the linear predictor and
    # the fitted values are computed

  }

# -------------------------------------------------
# Part 4 -- create covariate field for  substrate
# ensure the data assimilation in substrate is first completed :: 01.substrate_data.R
# 25 configs @ 5 min each, total time 2 hrs
  pS = substrate_carstm(p=p, DS="parameters_override" )
  M = substrate.db( p=pS, DS="aggregated_data", redo=TRUE )  # used for data matching/lookup in other aegis projects that use substrate
  M = substrate_carstm( p=pS, DS="carstm_inputs", redo=TRUE )  # will redo if not found
  M = NULL; gc()
  res = carstm_model( p=pS, M='substrate_carstm( p=pS, DS="carstm_inputs")', DS="redo", carstm_model_label="production"  )  # run model and obtain predictions

  if(0) {
    res = carstm_model( p=pS, DS="carstm_modelled", carstm_model_label="production"   ) # run model and obtain predictions
    fit = carstm_model( p=pS, DS="carstm_modelled_fit", carstm_model_label="production" )  # extract currently saved model fit
    summary(fit)

# Fixed effects:
#               mean    sd 0.025quant 0.5quant 0.975quant   mode kld
# (Intercept) -1.038 0.024     -1.086   -1.038     -0.991 -1.038   0

# Random effects:
#   Name	  Model
#     inla.group(z, method = "quantile", n = 13) RW2 model
#    auid BYM2 model

# Model hyperparameters:
#                                                           mean    sd 0.025quant 0.5quant 0.975quant  mode
# Precision for the lognormal observations                 1.519 0.007      1.506    1.519      1.533 1.519
# Precision for inla.group(z, method = "quantile", n = 13) 5.833 2.643      2.047    5.401     12.175 4.490
# Precision for auid                                       0.903 0.110      0.714    0.893      1.144 0.871
# Phi for auid                                             0.962 0.034      0.871    0.972      0.996 0.989

# Expected number of effective parameters(stdev): 195.14(0.203)
# Number of equivalent replicates : 493.07

# Deviance Information Criterion (DIC) ...............: 34384.88
# Deviance Information Criterion (DIC, saturated) ....: 96394.93
# Effective number of parameters .....................: 196.18

# Marginal log-Likelihood:  -17603.13

    vn = paste(pS$variabletomodel, "predicted", sep=".")
    ssplot=sc_carstm_plot( p=pS, res=res, vn=vn, main="Substrate Grain Size" ) # maps of some of the results
    ssplot

    #to save map of predicted mean
    fn=paste("substrate.predicted", year.assessment,"pdf", sep="." )
    pdf(file=paste(plot.dir, fn, sep="/"))
    ssplot
    dev.off()

    vn = paste(pS$variabletomodel, "predicted_se", sep=".")
    sc_carstm_plot( p=pS, res=res, vn=vn )

    vn = paste(pS$variabletomodel, "random_auid_nonspatial", sep=".")
    sc_carstm_plot( p=pS, res=res, vn=vn )

    vn = paste(pS$variabletomodel, "random_auid_spatial", sep=".")
    sc_carstm_plot( p=pS, res=res, vn=vn )

    plot(fit, plot.prior=TRUE, plot.hyperparameters=TRUE, plot.fixed.effects=FALSE, single=TRUE )

    #------------
    # 6567.727 sec = 1.8hrs

    pS$carstm_model_label = "production_inla.group_quantile_25"
    # DIC:
    #   Mean of Deviance ................. 33840.1
    #   Deviance at Mean ................. 33636.8
    #   Effective number of parameters ... 203.318
    #   DIC .............................. 34043.4
    # DIC (Saturated):
    #   Mean of Deviance ................. 95897.9
    #   Deviance at Mean ................. 95694.6
    #   Effective number of parameters ... 203.318
    #   DIC .............................. 96101.3

    pS$carstm_model_label = "production_inla.group_quantile_20"
    # DIC:
    # 	Mean of Deviance ................. 33995.4
    # 	Deviance at Mean ................. 33794.6
    # 	Effective number of parameters ... 200.881
    # 	DIC .............................. 34196.3
    # DIC (Saturated):
    # 	Mean of Deviance ................. 96187.3
    # 	Deviance at Mean ................. 95986.4
    # 	Effective number of parameters ... 200.881
    # 	DIC .............................. 96388.2

    pS$carstm_modelcall = paste('
      inla(
        formula =', pS$variabletomodel, ' ~ 1
          + f( inla.group(z, method="quantile", n=13) ,  model="rw2", scale.model=TRUE, hyper=H$rw2)
          + f(auid, model="bym2", graph=sppoly@nb, scale.model=TRUE, constr=TRUE, hyper=H$bym2),
        family = "lognormal",
        data= M,
        control.compute=list(dic=TRUE, config=TRUE),
        control.results=list(return.marginals.random=TRUE, return.marginals.predictor=TRUE ),
        control.predictor=list(compute=FALSE, link=1 ),
        control.fixed=H$fixed,  # priors for fixed effects, generic is ok
        verbose=TRUE
      ) ' )
    res = carstm_model( p=pS, M='substrate_carstm( p=pS, DS="carstm_inputs")', DS="redo", carstm_model_label=pS$carstm_model_label  )  # run model and obtain predictions
    fit = carstm_model( p=pS, DS="carstm_modelled_fit", carstm_model_label=pS$carstm_model_label )  # extract currently saved model fit

  }

# -------------------------------------------------
# Part 5 -- create covariate field for temperature
# ensure the data assimilation in temperature is first completed :: 01.temperature_data.R
# total: 30 min, 80 configs .. fast
# -------------------------------------------------
# Part 1 -- construct basic parameter list defining the main characteristics of the study

  pT = temperature_carstm(p=p, DS="parameters_override" )
  M = temperature.db( p=pT, DS="aggregated_data", redo=TRUE )  #  used for data matching/lookup in other aegis projects that use temperature
  M = temperature_carstm( p=pT, DS="carstm_inputs", redo=TRUE )  # will redo if not found
  M = NULL; gc()
  res = carstm_model( p=pT, M='temperature_carstm( p=pT, DS="carstm_inputs")', DS="redo", carstm_model_label="production"  ) # run model and obtain predictions
  if (0) {
    res = carstm_model( p=pT, DS="carstm_modelled", carstm_model_label="production" ) # run model and obtain predictions
    fit = carstm_model(  p=pT, DS="carstm_modelled_fit", carstm_model_label="production" )  # extract currently saved model fit
    summary(fit)

#Framework Model Summary (to 2018)
#-------------------------------
# Time used:
#     Pre = 2.99, Running = 612, Post = 7.01, Total = 622
# Fixed effects:
#             mean    sd 0.025quant 0.5quant 0.975quant  mode kld
# (Intercept) 4.52 0.285      3.937    4.522      5.089 4.524   0

# Random effects:
#   Name	  Model
#     dyri AR1 model
#    inla.group(z, method = "quantile", n = 13) RW2 model
#    auid BYM2 model

# Model hyperparameters:
#                                                           mean    sd 0.025quant 0.5quant 0.975quant  mode
# Precision for the Gaussian observations                  0.504 0.005      0.495    0.504      0.514 0.504
# Precision for dyri                                       6.302 2.551      2.482    5.935     12.346 5.158
# Rho for dyri                                             0.725 0.121      0.447    0.740      0.913 0.780
# Precision for inla.group(z, method = "quantile", n = 13) 0.413 0.111      0.243    0.396      0.676 0.365
# Precision for auid                                       0.427 0.024      0.383    0.426      0.476 0.423
# Phi for auid                                             0.427 0.034      0.355    0.429      0.488 0.439
# GroupRho for auid                                        0.841 0.011      0.817    0.842      0.862 0.843

# Expected number of effective parameters(stdev): 1860.76(37.35)
# Number of equivalent replicates : 14.44

# Deviance Information Criterion (DIC) ...............: 96494.96
# Deviance Information Criterion (DIC, saturated) ....: 28729.33
# Effective number of parameters .....................: 1862.94

# Marginal log-Likelihood:  -47560.07
# Posterior marginals for the linear predictor and

     vn = paste(pT$variabletomodel, "predicted", sep=".")

     sc_carstm_plot( p=pT, res=res, vn=vn, time_match=list(year="2000", dyear="0.85", main="Temperature" ) )       # maps of some of the results

     vn = paste(pT$variabletomodel, "predicted_se", sep=".")
     sc_carstm_plot( p=pT, res=res, vn=vn, time_match=list(year="2000", dyear="0.85" ) )       # maps of some of the results

     vn = paste(pT$variabletomodel, "random_auid_nonspatial", sep=".")
     sc_carstm_plot( p=pT, res=res, vn=vn, time_match=list(year="2000"  ) )       # maps of some of the results

     vn = paste(pT$variabletomodel, "random_auid_spatial", sep=".")
     sc_carstm_plot( p=pT, res=res, vn=vn, time_match=list(year="2000"  ) )       # maps of some of the results

     plot(fit, plot.prior=TRUE, plot.hyperparameters=TRUE, plot.fixed.effects=FALSE, single=TRUE )

   }

 #to plot predicted temperature maps for last six years

   recent=as.character((year.assessment-6): year.assessment)
   vn = paste(pT$variabletomodel, "predicted", sep=".")

  for (x in recent){
    fn=paste(x,"t",  "pdf", sep=".")
    outfile=paste(plot.dir, fn, sep="/")
    each.plot=   sc_carstm_plot( p=pT, res=res, vn=vn, time_match=list(year=x, dyear="0.85" ), main= )
    pdf(outfile)
    print(each.plot)
    dev.off()
  }

# -------------------------------------------------
# Part 6 -- create covariate field for species composition 1
# ensure that survey data is assimilated : bio.snowcrab::01snowcb_data.R, aegis.survey::01.surveys.data.R , etc.
# 30 min, 150 configs
  require(aegis.speciescomposition)
  require(carstm)

  pPC1 = speciescomposition_carstm(p=p, DS="parameters_override", varnametomodel="pca1" )
  M = speciescomposition_carstm( p=pPC1, DS="carstm_inputs", varnametomodel="pca1", redo=TRUE )  # will redo if not found
  M = NULL; gc()
  res = carstm_model( p=pPC1, M='speciescomposition_carstm( p=p, DS="carstm_inputs", varnametomodel="pca1" )', DS="redo", carstm_model_label="production"   ) # run model and obtain predictions
  if (0) {
    res = carstm_model( p=pPC1, DS="carstm_modelled", carstm_model_label="production"  ) # run model and obtain predictions
    fit = carstm_model( p=pPC1, DS="carstm_modelled_fit", carstm_model_label="production" )  # extract currently saved model fit
    summary(fit)

    plot( fit, plot.prior=TRUE, plot.hyperparameters=TRUE, plot.fixed.effects=FALSE )

    vn = paste(pPC1$variabletomodel, "predicted", sep=".")
    sc_carstm_plot( p=pPC1, res=res, vn=vn, time_match=list(year="2017" ), dyear="0.85" ) # maps of some of the results

    vn = paste(pPC1$variabletomodel, "predicted_se", sep=".")
    sc_carstm_plot( p=pPC1, res=res, vn=vn, time_match=list(year="2017" ), dyear="0.85" ) # maps of some of the results

    vn = paste(pPC1$variabletomodel, "random_auid_nonspatial", sep=".")
    sc_carstm_plot( p=pPC1, res=res, vn=vn, time_match=list(year="2017" ), dyear="0.85" )       # maps of some of the results , dyear="0.85"

    vn = paste(pPC1$variabletomodel, "random_auid_spatial", sep=".")
    sc_carstm_plot( p=pPC1, res=res, vn=vn, time_match=list(year="2017" ), dyear="0.85" )       # maps of some of the results , dyear="0.85"

#From Framework Run(up to 2018)
# Time used:
#     Pre = 2.67, Running = 667, Post = 4.98, Total = 675
# Fixed effects:
#              mean    sd 0.025quant 0.5quant 0.975quant  mode kld
# (Intercept) 0.143 0.015       0.11    0.143      0.173 0.144   0

# Random effects:
#   Name	  Model
#     dyri AR1 model
#    inla.group(t, method = "quantile", n = 13) RW2 model
#    inla.group(z, method = "quantile", n = 13) RW2 model
#    inla.group(substrate.grainsize, method = "quantile", n = 13) RW2 model
#    auid BYM2 model

# Model hyperparameters:
#                                                                                mean       sd 0.025quant 0.5quant
# Precision for the Gaussian observations                                    1.91e+02 2.88e+00    185.644 1.91e+02
# Precision for dyri                                                         1.03e+03 6.60e+02    273.266 8.62e+02
# Rho for dyri                                                               5.50e-02 2.47e-01     -0.430 5.90e-02
# Precision for inla.group(t, method = "quantile", n = 13)                   1.70e+04 1.45e+04   2813.038 1.30e+04
# Precision for inla.group(z, method = "quantile", n = 13)                   1.61e+02 7.50e+01     53.252 1.49e+02
# Precision for inla.group(substrate.grainsize, method = "quantile", n = 13) 7.29e+02 2.72e+03      9.795 1.95e+02
# Precision for auid                                                         1.71e+02 1.21e+01    147.609 1.71e+02
# Phi for auid                                                               5.19e-01 3.80e-02      0.442 5.20e-01
# GroupRho for auid                                                          9.19e-01 7.00e-03      0.905 9.19e-01
#                                                                            0.975quant     mode
# Precision for the Gaussian observations                                      1.97e+02  191.036
# Precision for dyri                                                           2.75e+03  612.450
# Rho for dyri                                                                 5.19e-01    0.069
# Precision for inla.group(t, method = "quantile", n = 13)                     5.54e+04 7245.080
# Precision for inla.group(z, method = "quantile", n = 13)                     3.41e+02  121.730
# Precision for inla.group(substrate.grainsize, method = "quantile", n = 13)   4.73e+03   19.393
# Precision for auid                                                           1.95e+02  170.771
# Phi for auid                                                                 5.92e-01    0.523
# GroupRho for auid                                                            9.33e-01    0.919

# Expected number of effective parameters(stdev): 1129.49(30.74)
# Number of equivalent replicates : 9.58

# Deviance Information Criterion (DIC) ...............: -24998.14
# Deviance Information Criterion (DIC, saturated) ....: 11953.32
# Effective number of parameters .....................: 1132.66

# Marginal log-Likelihood:  13670.07

  }

# -------------------------------------------------
# Part 7 -- create covariate field for species composition 2
# ensure that survey data is assimilated : bio.snowcrab::01snowcb_data.R, aegis.survey::01.surveys.data.R ,
# etc.30 min, 30 min

  pPC2 = speciescomposition_carstm(p=p, DS="parameters_override", varnametomodel="pca2" )
  M = speciescomposition_carstm( p=pPC2, DS="carstm_inputs", redo=TRUE )  # will redo if not found
  M = NULL; gc()
  res = carstm_model( p=pPC2, M='speciescomposition_carstm( p=p, DS="carstm_inputs", varnametomodel="pca2" )', DS="redo" , carstm_model_label="production"  ) # run model and obtain predictions
  if (0) {
    res = carstm_model( p=pPC2, DS="carstm_modelled", carstm_model_label="production"   ) # run model and obtain predictions
    fit = carstm_model( p=pPC2, DS="carstm_modelled_fit", carstm_model_label="production"  )  # extract currently saved model fit
    summary(fit)
    plot( fit, plot.prior=TRUE, plot.hyperparameters=TRUE, plot.fixed.effects=FALSE )

    vn = paste(pPC2$variabletomodel, "predicted", sep=".")
    sc_carstm_plot( p=pPC2, res=res, vn=vn, time_match=list(year="2017" ), dyear="0.85" )       # maps of some of the results

    vn = paste(pPC2$variabletomodel, "predicted_se", sep=".")
    sc_carstm_plot( p=pPC2, res=res, vn=vn, time_match=list(year="2017" ), dyear="0.85" ) # maps of some of the results

    vn = paste(pPC2$variabletomodel, "random_auid_nonspatial", sep=".")
    sc_carstm_plot( p=pPC2, res=res, vn=vn, time_match=list(year="2017" ), dyear="0.85" )       # maps of some of the results , dyear="0.85"

    vn = paste(pPC2$variabletomodel, "random_auid_spatial", sep=".")
    carstm_plot( p=pPC2, res=res, vn=vn, time_match=list(year="2017" ), dyear="0.85" )       # maps of some of the results , dyear="0.85"

  #   Time used:
  #     Pre = 4.86, Running = 1768, Post = 11.8, Total = 1785
  # Fixed effects:
    #             mean    sd 0.025quant 0.5quant 0.975quant mode kld
    # (Intercept) 0.02 0.021     -0.024     0.02      0.065 0.02   0

    # Random effects:
    #   Name	  Model
    #     dyri AR1 model
    #   inla.group(t, method = "quantile", n = 13) RW2 model
    #   inla.group(z, method = "quantile", n = 13) RW2 model
    #   inla.group(substrate.grainsize, method = "quantile", n = 13) RW2 model
    #   auid BYM2 model

    # Model hyperparameters:
    #                                                                               mean       sd 0.025quant 0.5quant
    # Precision for the Gaussian observations                                     255.139 3.86e+00    247.336  255.239
    # Precision for dyri                                                          972.573 5.21e+02    267.742  875.460
    # Rho for dyri                                                                  0.601 2.19e-01      0.123    0.627
    # Precision for inla.group(t, method = "quantile", n = 13)                   7071.547 1.04e+04    757.266 4032.628
    # Precision for inla.group(z, method = "quantile", n = 13)                    340.448 2.10e+02    109.684  285.481
    # Precision for inla.group(substrate.grainsize, method = "quantile", n = 13) 1267.298 1.65e+03     97.368  769.744
    # Precision for auid                                                          398.917 3.31e+01    335.265  398.669
    # Phi for auid                                                                  0.538 7.00e-02      0.383    0.546
    # GroupRho for auid                                                             0.914 8.00e-03      0.897    0.915
    #                                                                           0.975quant     mode
    # Precision for the Gaussian observations                                      2.63e+02  255.632
    # Precision for dyri                                                           2.25e+03  661.698
    # Rho for dyri                                                                 9.32e-01    0.750
    # Precision for inla.group(t, method = "quantile", n = 13)                     3.22e+04 1784.791
    # Precision for inla.group(z, method = "quantile", n = 13)                     8.90e+02  210.519
    # Precision for inla.group(substrate.grainsize, method = "quantile", n = 13)   5.51e+03  259.117
    # Precision for auid                                                           4.65e+02  399.531
    # Phi for auid                                                                 6.53e-01    0.578
    # GroupRho for auid                                                            9.30e-01    0.915

    # Expected number of effective parameters(stdev): 921.94(40.80)
    # Number of equivalent replicates : 11.74

    # Deviance Information Criterion (DIC) ...............: -28335.04
    # Deviance Information Criterion (DIC, saturated) ....: 11769.79
    # Effective number of parameters .....................: 928.42

    # Marginal log-Likelihood:  15491.21

  }

# finished covariates ... move onto abundance index estimation

# -------------------------------------------------
# Part 8 -- Snow crab anbundance -- main mode used for production
# 9 hrs 281 configs

  M = snowcrab_carstm( p=p, DS="carstm_inputs", redo=TRUE )  # will redo if not found
  #To compare values of M, run the following line:
  #load(paste(p$modeldir, "M.summary.rdata", sep="/"))

   M = NULL; gc()
  res = carstm_model( p=p, M='snowcrab_carstm( p=p, DS="carstm_inputs" )' ) # 151 configs and long optim .. 19 hrs

  # m = get("inla.models", INLA:::inla.get.inlaEnv())
	# m$latent$rw2$min.diff = NULL
	# assign("inla.models", m, INLA:::inla.get.inlaEnv())

    # extract results
    res = carstm_model( p=p, DS="carstm_modelled" ) # to load currently saved res
    fit = carstm_model( p=p, DS="carstm_modelled_fit" )  # extract currently saved model fit
    summary(fit)

# Time used:
#     Pre = 2.67, Running = 17238, Post = 3.15, Total = 17243
# Fixed effects:
#              mean    sd 0.025quant 0.5quant 0.975quant mode kld
# (Intercept) 5.483 0.117      5.244    5.486       5.71 5.49   0

# Random effects:
#   Name	  Model
#     dyri AR1 model
#    inla.group(t, method = "quantile", n = 13) RW2 model
#    inla.group(z, method = "quantile", n = 13) RW2 model
#    inla.group(substrate.grainsize, method = "quantile", n = 13) RW2 model
#    inla.group(pca1, method = "quantile", n = 13) RW2 model
#    inla.group(pca2, method = "quantile", n = 13) RW2 model
#    auid BYM2 model

# Model hyperparameters:
#                                                                              mean     sd 0.025quant 0.5quant 0.975quant
# Precision for dyri                                                         24.644 13.295      7.198   21.987     57.758
# Rho for dyri                                                               -0.010  0.132     -0.273   -0.008      0.245
# Precision for inla.group(t, method = "quantile", n = 13)                    6.078  2.716      2.367    5.559     12.835
# Precision for inla.group(z, method = "quantile", n = 13)                    4.846  2.067      2.060    4.431     10.025
# Precision for inla.group(substrate.grainsize, method = "quantile", n = 13)  0.243  0.081      0.125    0.229      0.439
# Precision for inla.group(pca1, method = "quantile", n = 13)                 7.210  3.278      2.842    6.549     15.399
# Precision for inla.group(pca2, method = "quantile", n = 13)                20.402  9.195      8.012   18.592     43.366
# Precision for auid                                                          0.430  0.035      0.364    0.429      0.502
# Phi for auid                                                                0.688  0.056      0.571    0.691      0.790
# GroupRho for auid                                                           0.817  0.013      0.790    0.818      0.843
#                                                                              mode
# Precision for dyri                                                         16.790
# Rho for dyri                                                               -0.001
# Precision for inla.group(t, method = "quantile", n = 13)                    4.642
# Precision for inla.group(z, method = "quantile", n = 13)                    3.734
# Precision for inla.group(substrate.grainsize, method = "quantile", n = 13)  0.204
# Precision for inla.group(pca1, method = "quantile", n = 13)                 5.425
# Precision for inla.group(pca2, method = "quantile", n = 13)                15.463
# Precision for auid                                                          0.427
# Phi for auid                                                                0.696
# GroupRho for auid                                                           0.818

# Expected number of effective parameters(stdev): 1920.79(15.10)
# Number of equivalent replicates : 3.96

# Deviance Information Criterion (DIC) ...............: 47009.33
# Deviance Information Criterion (DIC, saturated) ....: 28943.24
# Effective number of parameters .....................: 1894.56

# Marginal log-Likelihood:  -23611.26
# Posterior marginals for the linear predictor and
#  the fitted values are computed

    vn = paste(p$variabletomodel, "predicted", sep=".")
    carstm_plot( p=p, res=res, vn=vn, time_match=list(year="2000" ) )     # maps of some of the results

  plot(fit)
    plot(fit, plot.prior=TRUE, plot.hyperparameters=TRUE, plot.fixed.effects=FALSE, single=TRUE )
    s = summary(fit)
    s$dic$dic
    s$dic$p.eff

    # maps of some of the results
    vn = paste(p$variabletomodel, "random_sample_iid", sep=".")
    if (exists(vn, res)) carstm_plot( p=p, res=res, vn=vn, time_match=list(year="1950", dyear="0.05") )

    vn = paste(p$variabletomodel, "random_auid_nonspatial", sep=".")
    if (exists(vn, res)) {
      res_dim = dim( res[[vn]] )
      if (res_dim == 1 ) time_match = NULL
      if (res_dim == 2 ) time_match = list(year="2000")
      if (res_dim == 3 ) time_match = list(year="2000", dyear="0.85" )
      carstm_plot( p=p, res=res, vn=vn, time_match=time_match )
    }

    vn = paste(p$variabletomodel, "random_auid_spatial", sep=".")
    if (exists(vn, res)) {
      res_dim = dim( res[[vn]] )
      if (res_dim == 1 ) time_match = NULL
      if (res_dim == 2 ) time_match = list(year="2000")
      if (res_dim == 3 ) time_match = list(year="2000", dyear="0.85" )
      sc_carstm_plot( p=p, res=res, vn=vn, time_match=time_match )
    }

    sppoly = areal_units( p=p )  # to reload
    M = snowcrab_carstm( p=p, DS="carstm_inputs" )
    M$yr = M$year  # req for meanweights

    # mean weight by auidxyear
    wgts = meanweights_by_arealunit(
      set=M[M$tag=="observations",],
      AUID=as.character( sppoly$AUID ),
      yrs=p$yrs,
      fillall=TRUE,
      annual_breakdown=TRUE,
      robustify_quantiles=c(0, 0.99)  # high upper bounds are more dangerous
    )

    carstm_summary( p=p, operation="compute", carstm_model_label=p$carstm_model_label, wgts=wgts )

    RES = carstm_summary(p=p, operation="load_timeseries", carstm_model_label=p$carstm_model_label  )

    bio = carstm_summary(p=p, operation="load_spacetime_biomass", carstm_model_label=p$carstm_model_label  )
    num = carstm_summary(p=p, operation="load_spacetime_number", carstm_model_label=p$carstm_model_label  )

    plot( cfaall ~ yrs, data=RES, lty=1, lwd=2.5, col="red", type="b", ylab="Biomass (kt)", xlab="")
    plot( cfasouth ~ yrs, data=RES, lty=1, lwd=2.5, col="red", type="b", ylab="Biomass (kt)", xlab="")
    plot( cfanorth ~ yrs, data=RES, lty=1, lwd=2.5, col="red", type="b", ylab="Biomass (kt)", xlab="")
    plot( cfa4x ~ yrs, data=RES, lty=1, lwd=2.5, col="red", type="b", ylab="Biomass (kt)", xlab="")

    p$boundingbox = list( xlim=p$corners$lon, ylim=p$corners$lat) # bounding box for plots using spplot

    p$coastLayout = aegis.coastline::coastline_layout(p=p)
    p$mypalette=RColorBrewer::brewer.pal(9, "YlOrRd")
      sppoly = areal_units( p=p )  # to reload

    # map it ..mean density
    vn = "pred"
    sppoly@data[,vn] = bio[,"2017"]
    brks = interval_break(X= sppoly[[vn]], n=length(p$mypalette), style="quantile")
    spplot( sppoly, vn, col.regions=p$mypalette, main=vn, at=brks, sp.layout=p$coastLayout, col="transparent" )

    #to save plots of the last six years:
    #Needs to be run in Windows or outside Linux Rstudio for now
    recent=as.character((year.assessment-6): year.assessment)

    for (x in recent){
      sppoly@data[,vn] = bio[,x]
      brks = interval_break(X= sppoly[[vn]], n=length(p$mypalette), style="quantile")
      each.plot=spplot( sppoly, vn, col.regions=p$mypalette, main=x, at=brks, sp.layout=p$coastLayout, col="transparent" )
      fn=paste(x,"biomass",  "pdf", sep=".")
      outfile=paste(plot.dir, fn, sep="/")
      pdf(outfile)
      print(each.plot)
      dev.off()
    }

    plot( fit, plot.prior=TRUE, plot.hyperparameters=TRUE, plot.fixed.effects=FALSE )
    plot( fit$marginals.hyperpar$"Phi for auid", type="l")  # posterior distribution of phi nonspatial dominates
    plot( fit$marginals.hyperpar$"Precision for auid", type="l")
    plot( fit$marginals.hyperpar$"Precision for setno", type="l")

# end


# compare the ant host tree to each of the blochmannia gene trees

library(ape)
library(phytools)
library(seqinr)
options(scipen=999)

# read in species tree
species_tree <- read.nexus("camp_summed.tre")
species_tree <- midpoint.root(species_tree)
# remove single quotes from tip labels
species_tree$tip.label <- gsub("'", "", species_tree$tip.label)
# remove furthest outgroup
species_tree  <- drop.tip(species_tree, "SRX5650044")
# rename tips to coincide with blochmannia trees
species_tree$tip.label[species_tree$tip.label == "C-028"] <- "28_vicinus"
species_tree$tip.label[species_tree$tip.label == "C-005"] <- "5_vicinus"
species_tree$tip.label[species_tree$tip.label == "C-029"] <- "29_vicinusunk"
species_tree$tip.label[species_tree$tip.label == "C-024"] <- "24_vicinusunk"
species_tree$tip.label[species_tree$tip.label == "C-056"] <- "56_herculeanus"
species_tree$tip.label[species_tree$tip.label == "C-049"] <- "49_herculeanus"
species_tree$tip.label[species_tree$tip.label == "C-050"] <- "50_herculeanus"
species_tree$tip.label[species_tree$tip.label == "C-019"] <- "19_modoc"
species_tree$tip.label[species_tree$tip.label == "C-002"] <- "2_modoc"
species_tree$tip.label[species_tree$tip.label == "C-036"] <- "36_modoc"
species_tree$tip.label[species_tree$tip.label == "C-010"] <- "10_pennunk"
species_tree$tip.label[species_tree$tip.label == "C-016"] <- "16_pennunk"
species_tree$tip.label[species_tree$tip.label == "C-018"] <- "18_pennunk"
species_tree$tip.label[species_tree$tip.label == "C-003"] <- "3_laevigatus"
species_tree$tip.label[species_tree$tip.label == "C-046"] <- "46_laevigatus"
species_tree$tip.label[species_tree$tip.label == "SRX022802"] <- "floridanus"
species_tree$tip.label[species_tree$tip.label == "C-006"] <- "6_ocreatus"
species_tree$tip.label[species_tree$tip.label == "C-039"] <- "39_vicinus"
plot(species_tree)

# identify the trees to read in and then parse out their names
all_trees <- list.files("bloch_fasta_trees", pattern="RAxML*", full.names=T)
tree_names <- substr(all_trees, 55, nchar(all_trees))
tree_names <- sapply(strsplit(tree_names, "\\.1_"), "[[", 2)
tree_names <- sapply(strsplit(tree_names, "\\.tre"), "[[", 1)

# calculate the Kuhner and Felsenstein (1994) branch length score differences between blochmannia
# genes and the ant species tree
kf94 <- c()
for(a in 1:length(all_trees)) {
	a_rep <- read.tree(all_trees[a])
	a_rep <- midpoint.root(a_rep)
	# measure the Kuhner and Felsenstein (1994) branch length score
	kf94 <- c(kf94, dist.topo(species_tree, a_rep, method="score")[1])
}

# define the species for measuring phylogenetic distance
species_names <- c("tanaemyrmex", "camponotus")
groups <- list()
groups[[1]] <- c("28_vicinus", "5_vicinus", "39_vicinus", "29_vicinusunk", "24_vicinusunk")
groups[[2]] <- c("3_laevigatus", "46_laevigatus", "56_herculeanus", "49_herculeanus", "50_herculeanus", "19_modoc", "2_modoc", "36_modoc", "10_pennunk", "16_pennunk", "18_pennunk")
groups[[3]] <- c(groups[[1]], groups[[2]])

# loop for each tree
output <- c()
host_distances <- c()
for(a in 1:length(all_trees)) {
	a_rep <- read.tree(all_trees[a])
	a_rep <- midpoint.root(a_rep)
	# loop for each species
	tree_distances <- c()
	for(g in 1:length(groups)) {
		group_of_interest <- groups[[g]]
		# calculate MRCA distance for each species (for the ant species tree)
		if(a == 1) {
			# calculate MRCA of entire group
			total_mrca <- getMRCA(species_tree, tip = group_of_interest)
			# calculate length of edges to reach common ancestor for each individual in group
			# and then average
			edge_lengths <- list()
			group_edge_lengths <- c()
			for(b in 1:length(group_of_interest)) {
				# set edge length as zero to add to
				edge_lengths[[b]] <- 0
				# what is the number of this individual?
				b_number <- match(group_of_interest[b], species_tree$tip.label)

				# loop throup edge table until reaching MRCA
				while_loop <- 0
				while(while_loop != total_mrca) {
					# add edge length
					edge_lengths[[b]] <- edge_lengths[[b]] + species_tree$edge.length[species_tree$edge[,2]==b_number]

					# change new number to that node and update the while_loop object to the node
					b_number <- species_tree$edge[species_tree$edge[,2]==b_number,1]
					while_loop <- b_number
				}
				# add the MRCA to the nodes_needed object
				group_edge_lengths <- c(group_edge_lengths, edge_lengths[[b]])
			}

			# calc the mean of the group edge lengths
			group_edge_lengths <- mean(group_edge_lengths)
			# add to vector
			host_distances <- c(host_distances, group_edge_lengths)
		}

		# calculate for the blochmannia tree
		# calculate MRCA of entire group
		total_mrca <- getMRCA(a_rep, tip = group_of_interest)
		# calculate length of edges to reach common ancestor for each individual in group
		# and then average
		edge_lengths <- list()
		group_edge_lengths <- c()
		for(b in 1:length(group_of_interest)) {
			# set edge length as zero to add to
			edge_lengths[[b]] <- 0
			# what is the number of this individual?
			b_number <- match(group_of_interest[b], a_rep$tip.label)

			# loop throup edge table until reaching MRCA
			while_loop <- 0
			while(while_loop != total_mrca) {
				# add edge length
				edge_lengths[[b]] <- edge_lengths[[b]] + a_rep$edge.length[a_rep$edge[,2]==b_number]

				# change new number to that node and update the while_loop object to the node
				b_number <- a_rep$edge[a_rep$edge[,2]==b_number,1]
				while_loop <- b_number
			}
			# add the MRCA to the nodes_needed object
			group_edge_lengths <- c(group_edge_lengths, edge_lengths[[b]])
		}

		# calc the mean of the group edge lengths
		group_edge_lengths <- mean(group_edge_lengths)
		# add to vector
		tree_distances <- c(tree_distances, group_edge_lengths)
	}
	output <- rbind(output, tree_distances)
}

# mean blochmannia edge lengths / host phylogeny edge lengths
output[,1] <- output[,1] / host_distances[1]
output[,2] <- output[,2] / host_distances[2]
output[,3] <- output[,3] / host_distances[3]

# add all outputs into a data frame
output2 <- data.frame(gene_number = as.numeric(sapply(strsplit(tree_names, "_"), "[[", 1)), gene_name = as.character(sapply(strsplit(tree_names, "-"), "[[", 2)), kf94 = kf94, bloch_tanaemyrmex_edge_length = output[,1], bloch_camponotus_edge_length = output[,2], edge_length = output[,3])

# write output
write.table(output2, file="blochmannia_tree_characteristics.txt", sep="\t", quote=F, row.names=F, col.names=T)

# read in output
x <- read.table("blochmannia_tree_characteristics.txt", stringsAsFactors=F, header=T)
# order by gene number
x <- x[order(x[,1]),]
# get absolute rate estimates from Camponotus molecular evolution rate
summary(x$edge_length) * 1.983877e-09

# read in outgroup annotations
flor <- read.table("BX248583_floridanus_annotations.tsv", stringsAsFactors=F, header=T, fill=T, row.names=NULL)
# keep only annotations that are for "CDS"
flor <- flor[flor[,2] == "CDS",]
# sort
flor <- flor[order(as.numeric(flor[,4])),]
# redo row names
rownames(flor) <- seq(from=1, to=nrow(flor), by=1)

# check that gene IDs in object x match those in the floridanus annotations
test <- c()
for(a in 1:nrow(x)) {
	if(x[a,2] == flor[x[a,1],1]) {
		test <- c(test, TRUE)
	} else {
		print(a)
		test <- c(test, FALSE)
	}
}

# if the above test matched everything, continue

# extract locations for each gene (outgroup genome coordinates)
coords <- c()
for(a in 1:nrow(x)) {
	coords <- c(coords, (as.numeric(flor[x[a,1],5]) + as.numeric(flor[x[a,1],4])) / 2)
}
# add to table
x <- cbind(x[,1:2], coords, x[,3:6])

### read in alignments to obtain sequence identity for each gene in the ingroup (excluding indels)
x_files <- list.files("bloch_fasta_trees", pattern="*_aligned_trimmed.fasta$", full.names=T)
x_files_numbers <- as.numeric(sapply(strsplit(x_files, "_"), "[[", 5))

# percent identity for ingroup for each gene
pid <- list()
for(a in 1:nrow(x)) {
	# read in fasta alignment
	a_name <- x_files[match(x[a,1], x_files_numbers)]
	a_rep <- read.fasta(a_name)
	# remove outgroups
	a_rep <- a_rep[2:17]
	# pid
	pid_rep <- list()
	for(b in 1:length(a_rep[[1]])) {
		if(length(unique(as.character(lapply(a_rep, "[[", b)))) == 1) {
			pid_rep[[b]] <- 1
		} else {
			pid_rep[[b]] <- 0
		}
	}
	pid[[a]] <- sum(unlist(pid_rep)) / length(a_rep[[1]])
}
PID <- unlist(pid)
# add to data frame
x <- cbind(x, PID)

# percent identity for tanaemyrmex for each gene
pid <- list()
for(a in 1:nrow(x)) {
	# read in fasta alignment
	a_name <- x_files[match(x[a,1], x_files_numbers)]
	a_rep <- read.fasta(a_name)
	# keep only tanaemyrmex
	a_rep <- a_rep[names(a_rep) %in% groups[[1]]]
	# pid
	pid_rep <- list()
	for(b in 1:length(a_rep[[1]])) {
		if(length(unique(as.character(lapply(a_rep, "[[", b)))) == 1) {
			pid_rep[[b]] <- 1
		} else {
			pid_rep[[b]] <- 0
		}
	}
	pid[[a]] <- sum(unlist(pid_rep)) / length(a_rep[[1]])
}
PID_tanaemyrmex <- unlist(pid)
# add to data frame
x <- cbind(x, PID_tanaemyrmex)

# percent identity for camponotus for each gene
pid <- list()
for(a in 1:nrow(x)) {
	# read in fasta alignment
	a_name <- x_files[match(x[a,1], x_files_numbers)]
	a_rep <- read.fasta(a_name)
	# keep only camponotus
	a_rep <- a_rep[names(a_rep) %in% groups[[2]]]
	# pid
	pid_rep <- list()
	for(b in 1:length(a_rep[[1]])) {
		if(length(unique(as.character(lapply(a_rep, "[[", b)))) == 1) {
			pid_rep[[b]] <- 1
		} else {
			pid_rep[[b]] <- 0
		}
	}
	pid[[a]] <- sum(unlist(pid_rep)) / length(a_rep[[1]])
}
PID_camponotus <- unlist(pid)
# add to data frame
x <- cbind(x, PID_camponotus)

# write table output
write.table(x, file="blochmannia_tree_characteristics2.txt", sep="\t", quote=F, row.names=F, col.names=T)

# 20kbp means for each stat
window_coords <- seq(from=0, to=700000, by=20000)
for(a in 1:length(window_coords)) {
	# skip the last row
	if(a != length(window_coords)) {
		a_start <- window_coords[a]
		a_end <- window_coords[a+1]
		a_middle <- mean(c(window_coords[a+1], window_coords[a]))
		a_rep <- x[x$coords >= a_start & x$coords <= a_end,]
		a_rep <- c(a_middle, apply(a_rep[,4:ncol(a_rep)], 2, mean))
		if(a == 1) {
			x_windows <- a_rep
		} else {
			x_windows <- rbind(x_windows, a_rep)
		}
	}
}
x_windows <- as.data.frame(x_windows)

#plot
par(mar=c(2,5,1,1))
par(mfrow=c(3,1))
plot(x$coords, x$kf94, pch=19, cex=0.3, xaxt="n", xlab="", col="gray70", ylab="KF94 Distance")
lines(x_windows$V1, x_windows$kf94, lty=2, lwd=2)
plot(x$coords, x$bloch_tanaemyrmex_edge_length, pch=19, cex=0.3, ylim=c(0,50), xaxt="n", xlab="", col="lightblue", ylab="Relative Evolution Rate")
points(x$coords, x$bloch_camponotus_edge_length, pch=19, cex=0.3, col="lightcoral")
points(x$coords, x$edge_length, pch=19, cex=0.3, col="gray70")
lines(x_windows$V1, x_windows$bloch_tanaemyrmex_edge_length, col="navyblue", lwd=2)
lines(x_windows$V1, x_windows$bloch_camponotus_edge_length, col="red3", lwd=2)
lines(x_windows$V1, x_windows$edge_length, lty=2, lwd=2)
plot(x$coords, x$PID_tanaemyrmex*100, pch=19, cex=0.3, col="lightblue", ylim=c(60, 100), ylab="Gene % Identity")
points(x$coords, x$PID_camponotus*100, pch=19, cex=0.3, col="lightcoral")
points(x$coords, x$PID*100, pch=19, cex=0.3, col="gray70")
lines(x_windows$V1, x_windows$PID_tanaemyrmex*100, col="navyblue", lwd=2)
lines(x_windows$V1, x_windows$PID_camponotus*100, col="red3", lwd=2)
lines(x_windows$V1, x_windows$PID*100, lty=2, lwd=2)