# One function to compute all the possible metrics or statistics
# indv_members is the list returned by load_multimodel
  ## this is needed to compute crps/crpss but it not always will fit in mem
# ensmean is a list returned by load_multimodel_mean
  ## this reduce memory and it can be used to compute several metrics
# probs is a list returned by load_multimodel_probs
  ## this is needed to compute rps/rpss

multimodel_metrics <- function(recipe,
                               indv_membs = NULL,
                               ensmean = NULL,
                               probs = NULL) {
  ncores <- recipe$Analysis$ncores
  na.rm <- recipe$Analysis$remove_NAs
  cross.method <- recipe$Analysis$cross.method
  if (is.null(cross.method)) {
    cross.method <- 'leave-one-out'
  }
  requested_metrics <- strsplit(recipe$Analysis$Workflow$Skill$metric,
                                ", | |,")[[1]]

  if (!is.null(indv_membs)) {
    # conver $data elemenst to list to use multiApply:
    indv_membs <- append(list(obs = indv_membs$obs$data),
                         lapply(indv_membs$hcst, function(x) x$data))
    names(indv_membs)[-1] <- sapply(recipe$Analysis$Datasets$System, 
                                  '[[', 'name')
    if (any(c('crps', 'crpss') %in% requested_metrics)) {
      crps <- Apply(indv_membs, target_dims = c('syear', 'ensemble'),
                    fun = function(obs, ...) {
                      res <- abind(..., along = 2)
                      names(dim(res)) <- names(dim(obs))
                      obs <- Subset(obs, along = 'ensemble',
                                    indices = 1, drop = 'selected')
                      s2dv:::.CRPS(exp = res, obs = obs, dat_dim = NULL,
                                   time_dim = 'syear',
                                   memb_dim = 'ensemble')},
                    ncores = ncores)$output1
      skill_metrics$crps <- crps
      # Build the reference forecast:
      sdate_dim <- dim(indv_membs[[1]])['syear']
      cross <- CSTools:::.make.eval.train.dexes(eval.method = cross.method,
                                                amt.points = sdate_dim,
                                                amt.points_cor = NULL)
      tmp <- array(1:length(cross), c(syear = length(cross)))
      ref_clim <- Apply(list(indv_membs$obs, tmp),
                        target_dims = list(c('ensemble', 'syear'), NULL),
                        function(x, y) {
                          Subset(x, along = 'syear',
                                 indices = cross[[y]]$train.dexes, drop = T)},
                        ncores = ncores, output_dims = 'ensemble')$output1
      indv_membs <- append(list(ref = ref_clim), indv_membs)
      crps_clim <- Apply(indv_membs[1:2], target_dims = c('syear', 'ensemble'),
                         fun = function(ref, obs) {
                           obs <- Subset(obs, along = 'ensemble',
                                         indices = 1, drop = 'selected')
                           s2dv:::.CRPS(exp = ref, obs = obs, dat_dim = NULL,
                           time_dim = 'syear', memb_dim = 'ensemble')},
                         ncores = ncores)$output1
      skill_metrics$crps_clim <- crps_clim


      crpss <- Apply(indv_membs, target_dims = c('syear', 'ensemble'),
                     fun = function(ref, obs, ...) {
                       res <-  abind(..., along = 2)
                       names(dim(res)) <- names(dim(obs))
                       obs <- Subset(obs, along = 'ensemble',
                                     indices = 1, drop = 'selected')
                       CRPSS(exp = res, obs = obs, ref = ref,
                            memb_dim = 'ensemble', Fair = FALSE,
                            time_dim = 'syear', clim.cross.val = FALSE)},
                     ncores = ncores)
      skill_metrics$crpss <- crpss$crpss
      skill_metrics$crpss_significance <- crpss$sign


    }
  }
}
