
# Full-cross-val workflow
nftime <- dim(data$hcst$data)['time']
## Leave-one-out:
k <- 1
cross <- CSTools:::.make.eval.train.dexes('leave-one-out', sdate_dim, NULL)
# Paralelized:
loops <- array(1:length(cross), c(loop = length(cross)))

source("modules/Indices/R/tmp/NAO.R")
source("modules/Indices/R/tmp/Utils.R")
source("modules/Indices/R/tmp/EOF.R")
source("modules/Indices/R/tmp/ProjectField.R")

res <- Apply(list(loops), target = NULL,
             fun = function(t) {
               # subset years:
               # training
               obs_tr <- Subset(data$obs$data, along = 'syear',
                                indices = cross[[t]]$train.dexes)
               hcst_tr <- Subset(data$hcst$data, along = 'syear',
                                 indices = cross[[t]]$train.dexes)
               # eval years
               hcst_ev <- Subset(data$hcst$data, along = 'syear',
                                 indices = cross[[t]]$eval.dexes)
               obs_ev <- Subset(data$obs$data, along = 'syear',
                                indices = cross[[t]]$eval.dexes)
               # compute climatology:
               clim_obs_tr <- MeanDims(obs_tr, 'syear')
               clim_hcst_tr <- MeanDims(hcst_tr, c('syear', 'ensemble'))
               # compute anomalies:
               ano_obs_tr <- s2dv::Ano(obs_tr, clim_obs_tr)
               ano_hcst_tr <- s2dv::Ano(hcst_tr, clim_hcst_tr) 
               ano_hcst_ev <- s2dv::Ano(hcst_ev, clim_hcst_tr)
               ano_obs_ev <- s2dv::Ano(obs_ev, clim_obs_tr)
               # compute NAO:
               nao <- NAO(exp = ano_hcst_tr, obs = ano_obs_tr, exp_cor = ano_hcst_ev,
                          ftime_avg = NULL, time_dim = 'syear',
                          memb_dim = 'ensemble',
                          space_dim = c('latitude', 'longitude'),
                          ftime_dim = 'time',
                          lat = data$obs$coords$latitude,
                          lon = data$obs$coords$longitude, 
                          ncores = recipe$Analysis$ncores)
               
               nao_obs_ev <- NAO(exp = ano_hcst_tr, obs = ano_obs_tr, exp_cor = ano_obs_ev,
                                 ftime_avg = NULL, time_dim = 'syear',
                                 memb_dim = 'ensemble',
                                 space_dim = c('latitude', 'longitude'),
                                 ftime_dim = 'time',
                                 lat = data$obs$coords$latitude,
                                 lon = data$obs$coords$longitude,
                                 ncores = recipe$Analysis$ncores)$exp_cor
               #Standarisation:
               # Need the nao_hcst (for the train.dexes) to standarize the eval.dexes?
               nao_hcst_ev <- Apply(list(nao$exp, nao$exp_cor),
                                    target_dims = c('syear', 'ensemble'),
                                    fun = function(x, y) {
                                      sd <- sqrt(var(as.vector(x), na.rm = TRUE))
                                      means <- mean(as.vector(x), na.rm = TRUE)
                                      res <- apply(y, c(1,2), function(z) {(z-means)/sd})},
                                    ncores = recipe$Analysis$ncores)$output1
               nao_obs_ev <- Apply(list(nao$obs, nao_obs_ev),
                                   target_dims = c('syear','ensemble'),
                                   fun = function(x, y) {
                                     sd <- sqrt(var(as.vector(x), na.rm = TRUE))
                                     means <- mean(as.vector(x), na.rm = TRUE)
                                     res <- apply(y, c(1,2),
                                                  function(z) {(z-means)/sd})},
                                   ncores = recipe$Analysis$ncores)$output1
               nao_obs_tr <- Apply(list(nao$obs), target_dims = 'syear',
                                   fun = function(x) {
                                     sd <- sqrt(var(as.vector(x), na.rm = TRUE))
                                     means <- mean(as.vector(x), na.rm = TRUE)
                                     res <- apply(x, 1,
                                                  function(z) {(z-means)/sd})},
                                   ncores = recipe$Analysis$ncores, output_dims = 'syear')$output1
               nao_hcst_tr <- Apply(list(nao$exp), target_dims = c('syear', 'ensemble'),
                                    fun = function(x) {
                                      sd  <- sqrt(var(as.vector(x), na.rm = TRUE))
                                      means <- mean(as.vector(x), na.rm = TRUE)
                                      res <- apply(x, c(1,2), function (z) {(z-means)/sd})},                                               ncores = recipe$Analysis$ncores)$output1
               #Category limits
               lims_nao_hcst_tr <-  Apply(nao_hcst_tr, target_dims = c('syear', 'ensemble'),
                                          fun = function(x) {quantile(as.vector(x), c(1/3, 2/3), na.rm = TRUE)},
                                          output_dims = 'probs', ncores = recipe$Analysis$ncores)$output1
               lims_nao_obs_tr <-  Apply(nao_obs_tr, target_dims = 'syear',
                                         fun = function(x) {quantile(as.vector(x), c(1/3, 2/3), na.rm = TRUE)},
                                         output_dims = 'probs', ncores = recipe$Analysis$ncores)$output1
               # store results:
               
               lims_nao_hcst_tr_res <- 
             }
             # name dimensions
             # compute probabilities
             # what to return?
               return(list(#ano_obs_tr = ano_obs_tr, ano_hcst_tr = ano_hcst_tr,
                 #ano_hcst_ev = ano_hcst_ev, ano_obs_ev = ano_obs_ev,
                 #clim_obs_tr = clim_obs_tr, clim_hcst_tr = clim_hcst_tr,
                 nao_hcst_tr = nao_hcst_tr, nao_hcst_ev = nao_hcst_ev,
                 nao_obs_ev = nao_obs_ev, nao_obs_tr = nao_obs_tr,
                 lims_nao_hcst_tr = lims_nao_hcst_tr, lims_nao_obs_tr = lims_nao_obs_tr))
             }, ncores = 1)


nmemb <- dim(data$hcst$data)['ensemble']

# RPS
source("modules/Crossval/R/tmp/GetProbs.R")
nao_hcst_probs_ev <- GetProbs(res$nao_hcst_ev, time_dim = 'loop', 
                              prob_thresholds = NULL,
                              bin_dim_abs = 'probs', indices_for_quantiles = NULL,
                              memb_dim = 'ensemble', abs_thresholds = res$lims_nao_hcst_tr,
                              ncores = recipe$Analysis$ncores)
res$lims_nao_obs_tr <- Subset(res$lims_nao_obs_tr, indices = 1, along = 'ensemble',
                              drop = 'selected')

# Error in Subset(res$lims_nao_obs_tr, indices = 1, along = "ensemble",  : 
#                   Could not match all dimension names in 'indices' with dimension names in input array 'x'.

nao_obs_probs_ev <- GetProbs(res$nao_obs_ev, time_dim = 'loop',
                             prob_thresholds = NULL,
                             bin_dim_abs = 'probs', indices_for_quantiles = NULL,
                             memb_dim = 'ensemble', abs_thresholds = res$lims_nao_obs_tr,
                             ncores = recipe$Analysis$ncores)
source("modules/Crossval/R/tmp/RPS.R")
rps <- RPS(exp = nao_hcst_probs_ev, obs = nao_obs_probs_ev, memb_dim = NULL,
           cat_dim = 'probs', cross.val = FALSE, time_dim = 'loop',
           Fair = fair, nmemb = nmemb,
           ncores = recipe$Analysis$ncores)
source("modules/Crossval/R/RPS_clim.R")
rps_clim <- Apply(list(nao_obs_probs_ev),
                  target_dims = c('probs', 'loop'),
                  RPS_clim, bin_dim_abs = 'probs', Fair = fair, cross.val = FALSE)$output1

# RPSS
source("modules/Crossval/R/tmp/RPSS.R")
source("modules/Crossval/R/tmp/RandomWalkTest.R")
rpss <- RPSS(exp = nao_hcst_probs_ev, obs = nao_obs_probs_ev,
             time_dim = 'loop', memb_dim = NULL,
             cat_dim = 'probs',
             Fair = fair, nmemb = nmemb,
             nmemb_ref = sdate_dim - k,
             # We should use a line like this 
             #abs_threshold = res$lims_ano_hcst_tr, 
             #prob_threshold = c(1/3, 2/3),
             cross.val = FALSE, 
             ncores = recipe$Analysis$ncores)
# CRPS
crps <- CRPS(exp  = res$nao_hcst_ev, obs = res$nao_obs_ev,
             time_dim = 'loop', memb_dim = 'ensemble',
             Fair = fair,
             ncores = recipe$Analysis$ncores)
res$nao_obs_ev <- Subset(res$nao_obs_ev, indices = 1, along = 'syear', drop = 'selected')
crps_clim <- CRPS(exp = res$nao_obs_tr, obs = res$nao_obs_ev,
                  time_dim = 'loop', memb_dim = 'syear',
                  Fair = fair,
                  ncores = recipe$Analysis$ncores)

# CRPSS
ref <- res$nao_obs_tr
dim(ref) <- c(ensemble = as.numeric(sdate_dim) -1, dat = 1, var = 1, sday = 1, sweek = 1,
              nftime, loop = as.numeric(sdate_dim))
res$nao_hcst_ev <- Subset(res$nao_hcst_ev, indices = 1, along = 'syear', drop = 'selected')

crpss <- CRPSS(exp = res$nao_hcst_ev, obs = res$nao_obs_ev, ref = ref,
               memb_dim = 'ensemble', Fair = fair,
               time_dim = 'loop', clim.cross.val = FALSE,
               ncores = recipe$Analysis$ncores)

# Error in CRPSS(exp = res$nao_hcst_ev, obs = res$nao_obs_ev, ref = ref,  : 
#                  Parameter 'time_dim' is not found in 'exp' or 'obs' dimension.
               

# Corr
enscorr <- s2dv::Corr(res$nao_hcst_ev, res$nao_obs_ev, 
                      dat_dim = 'dat', 
                      time_dim = 'loop', 
                      method = 'pearson', 
                      memb_dim = 'ensemble',
                      memb = F,
                      conf = F,
                      pval = F,
                      sign = T,
                      alpha = 0.05,
                      ncores = recipe$Analysis$ncores)

# Mean Bias not required for NAO
#mean_bias <- Bias(res$ano_hcst_ev, res$ano_obs_ev,
#mean_bias <- Bias(data$hcst$data, data$obs$data,
#                 time_dim = 'syear',
#                 memb_dim = 'ensemble',
#                 ncores = recipe$Analysis$ncores)
# Spread error ratio
source("modules/Crossval/R/tmp/SprErr.R")
enssprerr <- SprErr(exp = res$nao_hcst_ev, 
                    obs = res$nao_obs_ev,
                    memb_dim = 'ensemble',
                    time_dim = 'loop')
enssprerr$p.val <- enssprerr$p.val <= 0.05

rmss <- s2dv::RMSSS(exp = res$nao_hcst_ev,
                    obs = res$nao_obs_ev,
                    memb_dim = 'ensemble',
                    time_dim = 'loop',
                    sign = TRUE)

skill_metrics <- list(#mean_bias = mean_bias,
  enscorr = enscorr$corr,
  enscorr_significance = enscorr$sign, enssprerr = enssprerr$ratio,
  enssprerr_significance = enssprerr$p.val,
  #rps = rps,
  rpss = rpss$rpss, rpss_significance = rpss$sign, #crps = crps,
  crpss = crpss$crpss, crpss_significance = crpss$sign,
  rmsss = rmss$rmsss, rmsss_significance = rmss$sign)
skill_metrics <- lapply(skill_metrics, function(x) {
  x <- InsertDim(drop(x), len = 1, pos = 1, name = 'var')
  x <- InsertDim(x, len = 1, pos = 1, name = 'region')
  names(dim(x)) <- c('var', 'region', 'time')
  return(x)})
## Generate the NAO s2dv cube to correctly save NAO attributes:
lats <- data$hcst$coords$latitude
lons <- data$hcst$coords$longitude
dim(res$nao_hcst_ev) <- c(var = 1, ensemble = as.numeric(dim(data$hcst$data)['ensemble']), time = as.numeric(nftime), syear = as.numeric(sdate_dim))
dim(res$nao_obs_ev) <- c(var = 1, time = as.numeric(nftime), syear = as.numeric(sdate_dim))
hcst_dates <- data$hcst$attrs$Dates
hcst_dates <- drop(data$hcst$attrs$Dates)

nao <- list(hcst = s2dv_cube(
  data = res$nao_hcst_ev, 
  varName = "nao",
  metadata = list(
    region = list(name = "region",
                  long_name = "NAO region",
                  lats_range = paste(range(lats), collapse = ", "),
                  lons_range = paste(range(lons), collapse = ", ")),
    time = data$hcst$attrs$Variable$metadata$time,
    nao = list(units = 'adim', 
               long_name = 'North Atlantic Oscillation')),
  Dates = hcst_dates,
  Dataset = recipe$Analysis$Datasets$System$name),
  obs = s2dv_cube(
    data = res$nao_obs_ev, 
    varName = "nao",
    metadata = list(
      region = list(name = "region",
                    long_name = "NAO region",
                    lats_range = paste(range(lats), collapse = ", "),
                    lons_range = paste(range(lons), collapse = ", ")),
      time = data$obs$attrs$Variable$metadata$time,
      nao = list(units = 'adim', 
                 long_name = 'North Atlantic Oscillation')),
    Dates = data$obs$attrs$Dates,
    Dataset = recipe$Analysis$Datasets$Reference$name),
  fcst=NULL)


original <- recipe$Run$output_dir
recipe$Run$output_dir <- paste0(original, "/outputs/", folder, "/")
# Compute save metrics 
source("modules/Saving/Saving.R")
Saving <- save_metrics(recipe = recipe, data_cube = nao$hcst, metrics = skill_metrics, agg = 'region')
recipe$Run$output_dir <- original

source("plot_NAO.R")
#dim(res$nao_hcst_ev) <- c(ensemble = as.numeric(dim(data$hcst$data)['ensemble']), time = as.numeric(nftime), syear = as.numeric(sdate_dim))
#dim(res$nao_obs_ev) <- c(time = as.numeric(nftime), syear = as.numeric(sdate_dim))
#nao <- list(hcst = list(data = res$nao_hcst_ev),
#                                     obs = list(data = res$nao_obs_ev))

plot_NAO(recipe = recipe, nao = nao, data, metrics = skill_metrics)

# Convert plots to user-chosen format
if (!is.null(recipe$Analysis$Workflow$Visualization$file_format) &&
    tolower(recipe$Analysis$Workflow$Visualization$file_format) != "pdf") {
  extension <- tolower(recipe$Analysis$Workflow$Visualization$file_format)
  plot_files <- list.files(path = recipe$Run$output_dir, pattern = "\\.pdf$",
                           recursive = TRUE, full.names = TRUE)
  for (file in plot_files) {
    system_command <- paste("convert -density 300", file,
                            "-resize 40% -alpha remove",
                            paste0(tools::file_path_sans_ext(file), ".",
                                   extension))
    system(system_command)
  }
  unlink(plot_files)
  info(recipe$Run$logger,
       paste0("##### PLOT FILES CONVERTED TO ", toupper(extension), " #####"))
}


source("tools/add_logo.R")
add_logo(recipe, "tools/BSC_logo_95.jpg") #"rsz_rsz_bsc_logo.png")

