source("modules/Crossval/R/tmp/GetProbs.R")
  
Crossval_multimodel_NAO <- function(recipe, data) {
  cross.method <- recipe$Analysis$cross.method
  # TODO move check
  obsproj <- recipe$Analysis$Workflow$Indices$NAO$obsproj
  if (is.null(obsproj)) {
    obsproj <- TRUE
  }
  if (is.null(cross.method)) {
    cross.method <- 'leave-one-out'
  }
  categories <- recipe$Analysis$Workflow$Probabilities$percentiles
  categories <- lapply(categories, function (x) { 
                       sapply(x, function(y) {
                              eval(parse(text = y))})})
  ncores <- recipe$Analysis$ncores
  na.rm <- recipe$Analysis$remove_NAs 
  ## data dimensions 
  sdate_dim <- dim(data$hcst[[1]]$data)['syear']
  orig_dims <- names(dim(data$hcst[[1]]$data))
  cross <- CSTools:::.make.eval.train.dexes(eval.method = cross.method,
                                            amt.points = sdate_dim,
                                            amt.points_cor = NULL) # k = ?

  # Define output dims and names:
  nao_hcst_tr_aux <- list()
  nao_hcst_ev_aux <- list()

  nao_obs_tr_res <- list()
  nao_hcst_ev_res <- list()
  # Cross-val loop starts
  for (t in 1:length(cross)) {
    info(recipe$Run$logger, paste("crossval:", t))
    nao_hcst_tr_res <- list()
    nao_hcst_ev_res <- list()

    # Observations
    obs_tr <- Subset(data	$obs$data, along = 'syear',
   		     indices = cross[[t]]$train.dexes)
    obs_ev <- Subset(data$obs$data, along = 'syear',
                     indices = cross[[t]]$eval.dexes)
    clim_obs_tr <- MeanDims(obs_tr, 'syear')
    ano_obs_tr <- s2dv::Ano(obs_tr, clim_obs_tr,
                            ncores = ncores)
    ano_obs_ev <- s2dv::Ano(obs_ev, clim_obs_tr,
                            ncores = ncores)
    # NAO for individual models
    for (sys in 1:length(data$hcst)) {
      hcst_tr <- Subset(data$hcst[[sys]]$data, along = 'syear',
      	                indices = cross[[t]]$train.dexes)
      ## evaluation indices
      hcst_ev <- Subset(data$hcst[[sys]]$data, along = 'syear',
                        indices = cross[[t]]$eval.dexes)
      # compute climatology:
      clim_hcst_tr <- MeanDims(hcst_tr, c('syear', 'ensemble'))
      ano_hcst_tr <- s2dv::Ano(hcst_tr, clim_hcst_tr,
                               ncores = ncores)
      ano_hcst_ev <- s2dv::Ano(hcst_ev, clim_hcst_tr,
                               ncores = ncores)
      # compute NAO:
      nao <- NAO(exp = ano_hcst_tr, obs = ano_obs_tr, exp_cor = ano_hcst_ev,
                 ftime_avg = NULL, time_dim = 'syear',
                 memb_dim = 'ensemble',
                 space_dim = c('latitude', 'longitude'),
                 ftime_dim = 'time', obsproj = obsproj,
                 lat = data$obs$coords$latitude,
                 lon = data$obs$coords$longitude,
                 ncores = recipe$Analysis$ncores)

      nao_obs_ev <- NAO(exp = ano_hcst_tr, obs = ano_obs_tr,
			exp_cor = ano_obs_ev,
                        ftime_avg = NULL, time_dim = 'syear',
                        memb_dim = 'ensemble',
                        space_dim = c('latitude', 'longitude'),
                        ftime_dim = 'time', obsproj = obsproj,
                        lat = data$obs$coords$latitude,
                        lon = data$obs$coords$longitude,
                        ncores = recipe$Analysis$ncores)$exp_cor
      #Standarisation:
      # Need the nao_hcst (for the train.dexes) to standarize the eval.dexes?
      nao_hcst_ev <- Apply(list(nao$exp, nao$exp_cor),
                           target_dims = c('syear', 'ensemble'),
                           fun = function(x, y) {
                             sd <- sqrt(var(as.vector(x), na.rm = TRUE))
                             means <- mean(as.vector(x), na.rm = TRUE)
                           res <- apply(y, c(1,2), function(z) {(z-means)/sd})},
                           ncores = recipe$Analysis$ncores)$output1
      nao_obs_ev <- Apply(list(nao$obs, nao_obs_ev),
                         target_dims = c('syear','ensemble'),
                          fun = function(x, y) {
                            sd <- sqrt(var(as.vector(x), na.rm = TRUE))
                            means <- mean(as.vector(x), na.rm = TRUE)
                          res <- apply(y, c(1,2), function(z) {(z-means)/sd})},
                          ncores = recipe$Analysis$ncores)$output1
      nao_obs_tr <- Apply(list(nao$obs), target_dims = 'syear',
                          fun = function(x) {
                            sd <- sqrt(var(as.vector(x), na.rm = TRUE))
                            means <- mean(as.vector(x), na.rm = TRUE)
                            res <- apply(x, 1, function(z) {(z-means)/sd})},
                          ncores = recipe$Analysis$ncores,
                          output_dims = 'syear')$output1
      nao_hcst_tr <- Apply(list(nao$exp), target_dims = c('syear', 'ensemble'),
                           fun = function(x) {
                             sd  <- sqrt(var(as.vector(x), na.rm = TRUE))
                             means <- mean(as.vector(x), na.rm = TRUE)
                           res <- apply(x, c(1,2), function(z) {(z-means)/sd})},
                           ncores = recipe$Analysis$ncores)$output1
      # store results:
      nao_hcst_tr_res <- append(nao_hcst_tr_res, list(nao_hcst_tr))
      names(nao_hcst_tr_res)[length(nao_hcst_tr_res)] <- names(data$hcst)[[sys]]
      nao_hcst_ev_res <- append(nao_hcst_ev_res, list(nao_hcst_ev))
      names(nao_hcst_ev_res)[length(nao_hcst_ev_res)] <- names(data$hcst)[[sys]]
    }
    if (t == 1) {
      nao_hcst_ev_aux <- nao_hcst_ev_res
    } else {
      for (sys in 1:length(data$hcst)) {
        nao_hcst_ev_aux[[sys]] <- abind(nao_hcst_ev_aux[[sys]],
                                        nao_hcst_ev_res[[sys]],
                                 along = length(dim(nao_hcst_ev_res[[sys]])) + 1)
        nao_hcst_tr_aux[[sys]] <- abind(nao_hcst_tr_aux[[sys]],
                                        nao_hcst_tr_res[[sys]],
                                 along = length(dim(nao_hcst_tr_res[[sys]])) + 1)
      }
    }
 }
    nao_hcst_ev_aux <- lapply(1:length(nao_hcst_ev_aux), function(x) {
      names(dim(nao_hcst_ev_aux[[x]])) <- c(names(dim(nao_hcst_ev_res[[x]])), 'sample')
      return(nao_hcst_ev_aux[[x]])})
    nao_hcst_tr_aux <- lapply(1:length(nao_hcst_tr_aux), function(x) {
      names(dim(nao_hcst_tr_aux[[x]])) <- c(names(dim(nao_hcst_tr_res[[x]])), 'sample')
      return(nao_hcst_tr_aux[[x]])})

    # Observed NAO should be the same for all models.
    nao_obs_ev_res <- append(nao_obs_ev_res, list(nao_obs_ev))
    names(nao_obs_ev_res)[length(nao_obs_ev_res)] <- names(data$hcst)[[1]]


