
source("modules/Saving/Saving.R")
source("modules/Crossval/R/tmp/RPS.R")
source("modules/Crossval/R/RPS_clim.R")
source("modules/Crossval/R/CRPS_clim.R")
source("modules/Crossval/R/tmp/RPSS.R")
source("modules/Crossval/R/tmp/RandomWalkTest.R")
source("modules/Crossval/R/tmp/Corr.R")
source("modules/Crossval/R/tmp/Bias.R")
source("modules/Crossval/R/tmp/SprErr.R")
source("modules/Crossval/R/tmp/Eno.R")

## data_crossval is the result from function full_crossval_anomalies or similar.
## this is a list with the required elements:
  ## probs is a list with
    ## probs$hcst_ev and probs$obs_ev 
    ## probs$hcst_ev will have as many elements in the $Probabilities$percentiles
    ## each element will be an array with 'cat' dimension
    ## the same for probs$obs_ev
  ## hcst is a s2dv_cube for the post-processed hindcast for the evalutaion indices
    ## in this case cross validated anomalies
  ## obs is a s2dv_cube for the post-processed obs 
    ## in this case cross validated anomalies
  ## fcst is a s2dv_cube for the post-processed fcst
    ## in this case cross anomalies with the full hindcast period
    ## this object is not required for skill assessment
  ## hcst.full_val and obs.full_val are the original data to compute mean bias
  ## cat_lims used to compute the probabilities
    ## this object is not required for skill assessment
  ## ref_obs_tr is an array with the cross-validate observed anomalies
    ## to be used as reference forecast in the CRPSS and CRPS_clim
    ## it is computed from the training indices
## the recipe could be used to read the Percentiles 
## if fair is TRUE, the nmemb used to compute the probabilities is needed
  ## nmemb_ref is the number of year - 1 in case climatological forecast is the reference
Crossval_skill <- function(recipe, data_crossval,
                           fair = FALSE, nmemb = NULL, nmemb_ref = NULL) {
  ncores <- recipe$Analysis$ncores
  alpha <- recipe$Analysis$Skill$alpha
  na.rm <- recipe$Analysis$remove_NAs
  if (is.null(alpha)) {
    alpha <- 0.05
  }
  ## START SKILL ASSESSMENT:
  skill_metrics <- list()
  requested_metrics <- strsplit(recipe$Analysis$Workflow$Skill$metric,
                                ", | |,")[[1]]
  
  if ('rps' %in% requested_metrics) {
    rps <- RPS(exp = data_crossval$probs$hcst_ev[[1]],
               obs = data_crossval$probs$obs_ev[[1]], memb_dim = NULL,
               cat_dim = 'cat', cross.val = FALSE, time_dim = 'syear',
     	       Fair = fair, nmemb = nmemb,
               ncores = ncores)
    rps_clim <- Apply(list(data_crossval$probs$obs_ev[[1]]),
                      target_dims = c('cat', 'syear'),
                      RPS_clim, bin_dim_abs = 'cat', Fair = fair,
    		      cross.val = FALSE, ncores = ncores)$output1
    skill_metrics$rps <- rps
    skill_metrics$rps_clim <- rps_clim
  }
  if ('rpss' %in% requested_metrics) {
    rpss <- RPSS(exp = data_crossval$probs$hcst_ev[[1]],
                 obs = data_crossval$probs$obs_ev[[1]],
                 ref = NULL, # ref is 1/3 by default if terciles
                 time_dim = 'syear', memb_dim = NULL,
                 cat_dim = 'cat', nmemb = nmemb,
                 dat_dim = NULL,
                 prob_thresholds = 0.1, # un use param when providing probs
                 indices_for_clim = NULL,
                 Fair = fair, weights_exp = NULL, weights_ref = NULL, 
                 cross.val = FALSE, na.rm = na.rm, 
                 sig_method.type = 'two.sided.approx', alpha = alpha,
                 ncores = ncores)
    skill_metrics$rpss <- rpss$rpss
    skill_metrics$rpss_significance <- rpss$sign
  }

  if ('crps' %in% requested_metrics) {
    crps <- CRPS(exp  = data_crossval$hcst$data,
                 obs = data_crossval$obs$data,
                 time_dim = 'syear', memb_dim = 'ensemble',
                 Fair = fair,
                 ncores = ncores)
    skill_metrics$crps <- crps
    crps_clim <- CRPS(exp = data_crossval$ref_obs_tr,
                      obs = data_crossval$obs$data,
                      time_dim = 'syear', memb_dim = 'ensemble',
                      Fair = fair, ncores = ncores) 
    skill_metrics$crps_clim <- crps_clim
  }
  if ('crpss' %in% requested_metrics) {
    crpss <- CRPSS(exp = data_crossval$hcst$data,
                   obs = data_crossval$obs$data,
                   ref = data_crossval$ref_obs_tr,
                   memb_dim = 'ensemble', Fair = fair,
                   time_dim = 'syear', clim.cross.val = FALSE,
                   ncores = ncores)
    skill_metrics$crpss <- crpss$crpss
    skill_metrics$crpss_significance <- crpss$sign
  }

  if ('enscorr' %in% requested_metrics) {
    enscorr <- Corr(exp = data_crossval$hcst$data,
                    obs = data_crossval$obs$data, 
                    dat_dim = NULL,
                    time_dim = 'syear', 
                    method = 'pearson', 
                    memb_dim = 'ensemble',
                    memb = F,
                    conf = F,
                    pval = F,
                    sign = T,
                    alpha = alpha,
                    ncores = ncores)
    skill_metrics$enscorr <- enscorr
  }
  if ('mean_bias' %in% requested_metrics) {
    mean_bias <- Bias(exp = data_crossval$hcst.full_val$data,
                      obs = data_crossval$obs.full_val$data,
                      time_dim = 'syear',
                      memb_dim = 'ensemble',
                      alpha = alpha,
                      ncores = ncores)
    skill_metrics$mean_bias <- mean_bias$bias
    skill_metrics$mean_bias_significance <- mean_bias$sig
  }
  if ('enssprerr' %in% requested_metrics) {
    enssprerr <- SprErr(exp = data_crossval$hcst$data,
                        obs = data_crossval$obs$data,
                        memb_dim = 'ensemble', dat_dim = NULL,
                        time_dim = 'syear', pval = TRUE,
                        ncores = ncores)
    skill_metrics$SprErr <- enssprerr$ratio
    skill_metrics$SprErr_significance <- enssprerr$p.val <= alpha
  }
  if ('rms' %in% requested_metrics) {
    rms <- RMS(exp = data_crossval$hcst$data,
               obs = data_crossval$obs$data,
               memb_dim = 'ensemble', dat_dim = NULL,
               time_dim = 'syear', alpha = alpha,
               ncores = ncores)
    skill_metrics$rms <- rms$rms
  }
  if ('rmss' %in% requested_metrics) {
    rmss <- RMSSS(exp = data_crossval$hcst$data,
                  obs = data_crossval$obs$data,
                  ref = res$ref_obs_tr,
                  memb_dim = 'ensemble', dat_dim = NULL,
                  time_dim = 'syear', alpha = alpha, sign = TRUE,
                  ncores = ncores)
    skill_metrics$rmss <- rmss$rmss
    skill_metrics$rmss_significance <- rmss$sign
  }
  original <- recipe$Run$output_dir
  recipe$Run$output_dir <- paste0(original, "/outputs/Skill/")

  skill_metrics <- lapply(skill_metrics, function(x) {
  		   if (is.logical(x)) {
		     dims <- dim(x)
		     res <- as.numeric(x)
		     dim(res) <- dims
		   } else {
		     res <- x
		   }
		   return(res)
		      })
  # Save metrics
      save_metrics(recipe = recipe,
                   metrics = skill_metrics,
                   data_cube = data_crossval$hcst, agg = 'global',
                   outdir = recipe$Run$output_dir)

  recipe$Run$output_dir <- original
  # reduce dimension to work with Visualization module:
  skill_metrics <- lapply(skill_metrics, function(x) {drop(x)})
  skill_metrics <- lapply(skill_metrics, function(x){
                          InsertDim(x, pos = 1, len = 1, name = 'var')})
  return(skill_metrics)
}

