
## This script includes the code in the vignette "FOCUS_calibration_t2m.ipynb"
## to execute the workflow locally with the data already stored on your own
## machine (option 3.3) and with retrieve TRUE (option 5.1). Please read the 
## full documentation and instrunctions in the Gitlab and in the vignette.

#############################################################################
################## 1. Install the R packages needed #########################
#############################################################################

## Creating (in the binder environment or within your computer) the folder that will contain the R libraries that will be installed afterwards  
options(warn=-1) #To supress warnings from now on
dir.create(paste0(getwd(),"/R_libs/"))
.libPaths(paste0(getwd(),"/R_libs/"))

## Installation of the R packages (it may take a while, please be patient)
install.packages("devtools",destdir=(paste0(getwd(),"/R_libs/")), lib=(paste0(getwd(),"/R_libs/")), quiet=T)
library(devtools)
install.packages("startR", destdir=(paste0(getwd(),"/R_libs/")), lib=(paste0(getwd(),"/R_libs/")), quiet=T)
install.packages("multiApply", destdir=(paste0(getwd(),"/R_libs/")), lib=(paste0(getwd(),"/R_libs/")), quiet=T)
install.packages("ClimProjDiags", destdir=(paste0(getwd(),"/R_libs/")), lib=(paste0(getwd(),"/R_libs/")), quiet=T)
install.packages("CSTools", destdir=(paste0(getwd(),"/R_libs/")), lib=(paste0(getwd(),"/R_libs/")), quiet=T, dependencies=F)
install.packages("s2dv", destdir=(paste0(getwd(),"/R_libs/")), lib=(paste0(getwd(),"/R_libs/")), quiet=T)
install.packages("RColorBrewer", destdir=(paste0(getwd(),"/R_libs/")), lib=(paste0(getwd(),"/R_libs/")), quiet=T)
install.packages("tibble", destdir=(paste0(getwd(),"/R_libs/")), lib=(paste0(getwd(),"/R_libs/")), quiet=T)
install.packages("stringr", destdir=(paste0(getwd(),"/R_libs/")), lib=(paste0(getwd(),"/R_libs/")), quiet=T)
install.packages("R6", destdir=(paste0(getwd(),"/R_libs/")), lib=(paste0(getwd(),"/R_libs/")), quiet=T)
install.packages("lubridate", destdir=(paste0(getwd(),"/R_libs/")), lib=(paste0(getwd(),"/R_libs/")), quiet=T)
install.packages("ecmwfr", destdir=(paste0(getwd(),"/R_libs/")), lib=(paste0(getwd(),"/R_libs/")), quiet=T)

print("*** The libraries have been installed successfully ***")

#############################################################################
######################## 2. Load the libraries ##############################
#############################################################################

## Loading the libraries needed to conduct the analysis
## We use suppressPackageStartupMessages() to quiet the output while loading the libraries
options(warn=-1) #To supress warnings from now on
.libPaths(paste0(getwd(),"/R_libs/")) #Define the path where the libraries are stored
suppressPackageStartupMessages(library(startR,quietly=TRUE, warn.conflicts=FALSE))
suppressPackageStartupMessages(library(s2dv,quietly=TRUE, warn.conflicts=FALSE))
suppressPackageStartupMessages(library(multiApply,quietly=TRUE, warn.conflicts=FALSE))
suppressPackageStartupMessages(library(ClimProjDiags,quietly=TRUE, warn.conflicts=FALSE))
suppressPackageStartupMessages(library(RColorBrewer,quietly=TRUE, warn.conflicts=FALSE))
suppressPackageStartupMessages(library(CSTools,quietly=TRUE, warn.conflicts=FALSE))
suppressPackageStartupMessages(library(ecmwfr,quietly=TRUE, warn.conflicts=FALSE))
suppressPackageStartupMessages(library(lubridate,quietly=TRUE, warn.conflicts=FALSE))
suppressPackageStartupMessages(library(ncdf4,quietly=TRUE, warn.conflicts=FALSE))
suppressPackageStartupMessages(library(parallel,quietly=TRUE, warn.conflicts=FALSE))

print("*** The libraries have been loaded successfully ***")

#############################################################################
################ 3. Data and configuration parameters #######################
#############################################################################

# Variable
var_name <- 't2m'

# Period
years <- c(1995:2015) 

# Inisialisation month
ini <- 11

# Obtaining the vector of initialisation dates
sdate_forecast <- paste0(years,ini) 

# Forecast period
forecast_time <- indices(1:3)

# Region
lons.min <- 0        
lons.max <- 49       
lats.min <- -40      
lats.max <- -10 

#############################################################################
########### 3.3 Local execution of the notebook on your computer. ###########
###### Case 1: the seasonal data is already stored on your machine ##########
#############################################################################

exp_path <- paste0(getwd(),'/data/exp/ecmwf/system5c3s/monthly_mean/data_sample/$var$/$var$_$sdate$01.nc') 

obs_path <- paste0(getwd(),'/data/reanalysis/ecmwf/era5/monthly_mean/data_sample/$var$/$var$_$date$.nc') 

# Path of the reference grid for the interpolation
grid <- paste0(getwd(),'/reference_seas5.nc')

#############################################################################
####################### 4. Load seasonal data ###############################
#############################################################################

# Select the retrieve method
retrieve <- TRUE

# Load the hindcast 
exp <- startR::Start(# Path to files
  dat = exp_path,
  # Dimensions (they can be values, indices or "all")
  var = var_name,
  sdate = sdate_forecast,
  ensemble = 'all',
  time = forecast_time,
  latitude = values(list(lats.min, lats.max)),
  longitude = values(list(lons.min, lons.max)),
  # Reorder longitude and latitudes
  longitude_reorder = CircularSort(-180,180),
  latitude_reorder = Sort(decreasing = TRUE),
  # Interpolation
  transform = CDORemapper,
  transform_extra_cells = 8,
  transform_params = list(grid = grid,
                          method = 'conservative',
                          crop = c(lons.min, lons.max, lats.min, lats.max)),
  transform_vars = c('latitude', 'longitude'),
  # Synonims in case the variable names are different in the files
  synonims = list(latitude = c('lat', 'latitude'),
                  longitude = c('lon', 'longitude'),
                  ensemble = c('member','ensemble','number')),
  # Specify the metadata to return
  return_vars = list(latitude = 'dat', longitude = 'dat', time = 'sdate'),
  retrieve = retrieve, silent = FALSE, num_procs = 1)

# Here, we define the dates that will be used to retrieve the reanalysis data.
# They will be the same as the hindcast times.
dates <- attr(exp, 'Variables')$common$time

# Giving dates format
dates_file <- format(dates, '%Y%m') 

# Specifying the dimensions
dim(dates_file) <- c(sdate = length(sdate_forecast),
                     time = length(forecast_time))

obs <- startR::Start(# Path to files
  dat = obs_path,
  # Dimensions (they can be values, indices or "all")
  var = var_name,
  date = dates_file,
  latitude = values(list(lats.min, lats.max)),
  longitude = values(list(lons.min, lons.max)),
  # Reorder longitude and latitudes
  longitude_reorder = CircularSort(-180,180),
  latitude_reorder = Sort(decreasing = TRUE),
  # Interpolation
  transform = CDORemapper,
  transform_extra_cells = 8,
  transform_params = list(grid = grid,
                          method = 'conservative',
                          crop = c(lons.min, lons.max, lats.min, lats.max)),
  transform_vars = c('latitude', 'longitude'),
  split_multiselected_dims = TRUE,
  # Synonims in case the variable names are different in the files
  synonims = list(longitude = c('lon', 'longitude'),
                  latitude = c('lat', 'latitude')),
  # Specify the metadata to return
  return_vars = list(time = 'date',latitude = 'dat',longitude = 'dat'),
  retrieve = retrieve, silent = FALSE, num_procs = 1)

cat('Dimensions of hindcasts: \n')
if (isTRUE(retrieve)){
  print(dim(exp))
} else {
  print(attr(exp,'Dimensions'))
}
cat('Dimension of the observations: \n')
if (isTRUE(retrieve)){
  print(dim(obs))
} else {
  print(attr(obs,'Dimensions'))
}

lat <- as.numeric(attr(obs,'Variables')$dat1$latitude)
lon <- as.numeric(attr(obs,'Variables')$dat1$longitude)

cat('Latitudes: ')
cat(lat)
cat('\n')
cat('Longitudes: ')
cat(lon)

#############################################################################
####################### 5. Hindcast calibration #############################
#############################################################################

member_dim <- 'ensemble'
sdate_dim <- 'sdate'   
ftime_dim <-'time'                           
methods <- c('raw','bias','evmos','mse_min','crps_min','rpc-based')
cross_validation <- "leave-one-out"
alpha <- 0.05
na.rm <- FALSE
ncores <- 1

## Definition of the function used to calibrate the hindcast using different methods
calibrate_hcst <- function(exp, obs, methods, cross_validation = "leave-one-out",
                           sdate_dim = 'sdate', member_dim = 'ensemble', 
                           alpha = 0.05, na.rm = FALSE, ncores = NULL) {
  
  ## Let's first make homogeneous the units of both observations and the hindcast
  ## If you have chosen a variable which was not defined in section 3 you should add it here and make the proper unit conversion.
  switch(tolower(var_name),
         "tp"  = {exp <- exp* 86400 * 1000 ; obs<- obs* 1000},  # Final units: 'mm/day'
         "msl" = {exp <- exp/100 ; obs<- obs/100},              # Final units: 'hPa'
         "ssr" = {obs <- obs/86400})                            # Final units:'W/m2'
  
  # Loop over different calibration methods and store the calibrated predictions in a list
  calibrated <- list()
  for (method in methods){
    
    if (!identical(method,'raw')){
      calibrated[[method]] <- CSTools::Calibration(exp = exp, obs = obs, cal.method = method, eval.method = cross_validation,
                                                   multi.model = FALSE, na.fill = TRUE, na.rm = na.rm, memb_dim = member_dim, 
                                                   sdate_dim = sdate_dim, apply_to = 'sign', alpha = alpha, ncores = ncores)
    } else { # if "raw" is selected, return the raw forecast
      calibrated[[method]] <- exp
    }
    
  }
  
  ## Adding the observations to the list
  calibrated$obs <- obs
  
  ## Returning the list with the raw and calibrated hindcasts and the observations
  return(calibrated)
}

#############################################################################
############### 5.1. Calibrating when retrieve is set to TRUE ###############
#############################################################################

calibrated_hcst <- multiApply::Apply(data = list(exp = exp, 
                                                 obs = obs), 
                                     target_dims = list(exp = c(sdate_dim,member_dim),
                                                        obs = c(sdate_dim)), 
                                     fun = calibrate_hcst, 
                                     methods = methods, 
                                     cross_validation = cross_validation,
                                     alpha = alpha,
                                     sdate_dim = sdate_dim, 
                                     member_dim = member_dim, 
                                     na.rm = na.rm, 
                                     ncores = ncores)

#############################################################################
#################### 6. Forecast quality assessment #########################
#############################################################################

obs <- calibrated_hcst[['obs']]
calibrated_hcst$obs <- NULL

agg <- TRUE 

#if agg is TRUE, the aggregation along the forecast times is done (seasonal aggregation)
if (isTRUE(agg)){
  ## Raw and calibrated hindcasts
  for (method in methods){
    calibrated_hcst[[method]] <- multiApply::Apply(data = calibrated_hcst[[method]],
                                                   target_dims = ftime_dim, 
                                                   fun = mean, 
                                                   na.rm = na.rm,
                                                   ncores= ncores)[[1]]   
  }
  ## Observations
  obs <- multiApply::Apply(data = obs, target_dims = ftime_dim, fun = mean, na.rm = na.rm, ncores = ncores)[[1]]  
}     

# Dimensions of the observations
dim(obs)

for (method in methods){
  calibrated_hcst[[method]] <- drop(calibrated_hcst[[method]])
}
obs <- drop(obs)

# New dimensions of the observations
dim(obs)

#New dimensions of the calibrated forecasts
dim(calibrated_hcst$bias)

# Structure of the list with the calibrated forecast using different calibration methods
str(calibrated_hcst)

acc <- list() # ACC of the hindcasts
rpss <- list() # RPSS of the hindcasts vs climatology
crpss <- list() # CRPSS of the hindcasts vs climatology

acc_vs_raw <- list() # ACC of the hindcasts vs non-calibrated hindcasts
rpss_vs_raw <- list() # RPSS of the hindcasts vs non-calibrated hindcasts
crpss_vs_raw <- list() # CRPSS of the hindcasts vs non-calibrated hindcasts

for (method in methods){
  
  ## Skill scores vs climatology
  
  acc[[method]] <- s2dv::Corr(exp = calibrated_hcst[[method]], obs = obs, 
                              time_dim = "sdate", memb_dim = "ensemble", method = "pearson", memb = FALSE, 
                              pval = FALSE, conf = FALSE, sign = TRUE, alpha = alpha, ncores = ncores)
  
  rpss[[method]] <- s2dv::RPSS(exp = calibrated_hcst[[method]], obs = obs, ref = NULL, 
                               time_dim = "sdate", memb_dim = "ensemble", prob_thresholds = c(1/3, 2/3), # tercile categories
                               sig_method.type = "two.sided", alpha = alpha, ncores = ncores)
  
  crpss[[method]] <- s2dv::CRPSS(exp = calibrated_hcst[[method]], obs = obs, ref = NULL, 
                                 time_dim = "sdate", memb_dim = "ensemble",
                                 sig_method.type = "two.sided", alpha = alpha, ncores = ncores)
  
  ## Skill scores vs non-calibrated hindcasts
  
  acc_vs_raw[[method]] <- s2dv::DiffCorr(exp = calibrated_hcst[[method]], obs = obs, ref = calibrated_hcst[['raw']], 
                                         time_dim = "sdate", memb_dim = "ensemble", method = "pearson",
                                         pval = FALSE, sign = TRUE, test.type = "two-sided", alpha = alpha, ncores = ncores)
  
  rpss_vs_raw[[method]] <- s2dv::RPSS(exp = calibrated_hcst[[method]], obs = obs, ref = calibrated_hcst[['raw']], 
                                      time_dim = "sdate", memb_dim = "ensemble", prob_thresholds = c(1/3, 2/3), # tercile categories
                                      sig_method.type = "two.sided", alpha = alpha, ncores = ncores)
  
  crpss_vs_raw[[method]] <- s2dv::CRPSS(exp = calibrated_hcst[[method]], obs = obs, ref = calibrated_hcst[['raw']], 
                                        time_dim = "sdate", memb_dim = "ensemble",
                                        sig_method.type = "two.sided", alpha = alpha, ncores = ncores)
  
}

## Structure of the RPSS (similar for all metrics)
str(rpss)

#############################################################################
###################### 7. Visualise the results #############################
#############################################################################

if (isFALSE(agg)){
  target_time <- 2 #target_time could be between 1 and "length(forecast_time)"
  # Subset here the target_time in the arrays with the metrics. For example,
  # rpss$raw$rpss <- ClimProjDiags::Subset(x = rpss$raw$rpss, along = 'time', indices = target_time, drop = 'selected')
}

ref_period <- paste0(years[1],"-",years[length(years)]) # In our example ref_period <- "1995-2015"
if (agg == TRUE) {
  months_agg <- seq(ini + forecast_time[1]-1, ini + forecast_time[length(forecast_time)] - 1)
  ftime_info <- ""
  
  for (i in (1:length(months_agg))){
    aux_month <- as.numeric(months_agg[i])
    if(aux_month < 13){
      months_agg[i] <- month.abb[aux_month]  
    } else {
      aux_month <- aux_month - 12
      months_agg[i] <- month.abb[aux_month]
    }
    ftime_info <- paste0(ftime_info,substring(months_agg[i], 1, 1))
  }
  ft_string <- paste0("agg_",ftime_info)
  ftime_info <- paste0("Time agg. = ",ftime_info)
} else { # agg == FALSE
  aux_month <- as.numeric(ini+forecast_time[target_time]-1)
  
  if(aux_month < 13){
    aux_month <- month.abb[aux_month]
  } else {
    aux_month <- aux_month - 12
    aux_month <- month.abb[aux_month]
  }
  ftime_info <- paste0(aux_month)
  ftime_info <- paste0("Fcst time: ",ftime_info) 
  ft_string <- paste0("target_month_",aux_month) 
} 

#Let's give the initialisation string format
ini_str <- month.abb[ini]

# create the directory where the figures will be stored
outdir <- paste0(getwd(),"/Figures/",var_name,"/ini",ini,"/",ref_period,"/",ft_string,"/")  
dir.create(outdir, showWarnings = TRUE, recursive = TRUE, mode = "0777")

print(outdir)

#############################################################################
####### 7.1. Climatology of the calibrated and non-calibrated forecast ######
#############################################################################

clim_ensmean <- list()
for (method in methods){
  clim_ensmean[[method]] <- multiApply::Apply(calibrated_hcst[[method]],
                                              target_dims = c("sdate","ensemble"),
                                              fun = mean,
                                              na.rm = FALSE,
                                              ncores = ncores)[[1]] 
}

switch(tolower(var_name),
       "t2m" = {units <- c('K')}, 
       "tp"  = {units <- c('mm/day')},
       "msl" = {units <- c('hPa')},
       "ssr" = {units <- c('W/m2')})

num_colours <- 15  # The colour bar will have 15 colours
value_min_bar <- round(min(unlist(clim_ensmean))) 
value_max_bar <- round(max(unlist(clim_ensmean)))
diff_between_colours <- round((value_max_bar-value_min_bar)/num_colours)
value_max_bar <- value_min_bar + (diff_between_colours*num_colours)
triangle_end_inf <-isTRUE(value_min_bar > min(unlist(clim_ensmean)))   
triangle_end_sup <- isTRUE(value_max_bar < max(unlist(clim_ensmean)))
colours_palette <- "yellowred"

s2dv::PlotLayout(PlotEquiMap,plot_dims = c("latitude", "longitude"), nrow = 2,
                 var = clim_ensmean, lon = lon, lat = lat, 
                 title_scale = 0.45, colNA = "grey", brks = seq(value_min_bar,value_max_bar,diff_between_colours),
                 color_fun = clim.palette(colours_palette), title_margin_scale = 4,
                 toptitle = paste0("Climatology of the ensemble mean | ",var_name," | SEAS5/ERA5 \n Init. month = ",ini_str," | Reference period=",ref_period," | ", ftime_info),
                 titles = paste0(methods), units = paste0(var_name, " (",units,")"), filled.continents = FALSE, filled.oceans = "white",
                 country.borders = TRUE, coast_width = 1, lake_color = "black", width = 11, height = 7,
                 triangle_ends = c(triangle_end_inf,triangle_end_sup),res = 100, fileout = NULL) # fileout = paste0(outdir,"clim_ens_mean.png"))

#############################################################################
######### 7.2. Bias of the calibrated and non-calibrated forecasts ##########
#############################################################################

## Climatology of the observations
clim_obs <- s2dv::MeanDims(data = obs, dims = 'sdate', na.rm = FALSE)

## Bias of the climatology for the raw and calibrated hindcasts
clim_diff <- list()
for (method in methods){
  clim_diff[[method]] <- clim_ensmean[[method]] - clim_obs
}

s2dv::PlotLayout(PlotEquiMap,plot_dims = c("latitude", "longitude"), nrow = 2,
                 var = clim_diff, lon = lon,lat = lat,
                 colNA="grey", title_scale = 0.45, dot_symbol = 20, dot_size = 1, units = paste0(var_name, " (",units,")"),
                 toptitle = paste0("Bias of the climatology | ",var_name," | SEAS5/ERA5 \n Init. month=",ini_str," | Reference period=",ref_period, " | ", ftime_info),
                 brks = seq(-2,2,0.5),titles = methods,title_margin_scale = 4,
                 filled.continents = FALSE, filled.oceans = "white", country.borders = TRUE, coast_width = 1, width = 11, height = 7, res = 100,
                 fileout = NULL) # fileout = paste0(outdir,"climEnsMean_cal_vs_raw.png"))

#############################################################################
########## 7.3. Anomaly Correlation Coefficient (ACC) #######################
#############################################################################

str(acc)

s2dv::PlotLayout(PlotEquiMap,plot_dims = c("latitude", "longitude"), lon = lon,lat = lat, nrows = 2,
                 var = list(acc[['raw']]$corr, acc[['bias']]$corr, acc[['evmos']]$corr,
                            acc[['mse_min']]$corr, acc[['crps_min']]$corr, acc[['rpc-based']]$corr),
                 special_args = list(list(dots = drop(acc[['raw']]$sign)),
                                     list(dots = drop(acc[['bias']]$sign)),
                                     list(dots = drop(acc[['evmos']]$sign)),
                                     list(dots = drop(acc[['mse_min']]$sign)),
                                     list(dots = drop(acc[['crps_min']]$sign)),
                                     list(dots = drop(acc[['rpc-based']]$sign))),
                 title_scale = 0.45, brks = seq(-1,1,0.2), titles = paste0(methods),
                 toptitle = paste0("ACC for ",var_name, " | SEAS/ERA5 \n Init. month = ",ini_str," | Reference period = ",ref_period," | ", ftime_info),
                 colNA = "grey", dot_symbol = 20, dot_size = 1, filled.continents = FALSE, filled.oceans = "white", width = 11, height = 7,
                 triangle_ends = c(FALSE,FALSE), res = 100, title_margin_scale = 4, fileout = NULL) #paste0(outdir,"/ACC.png"))

#############################################################################
################## 7.4. Differences between the ACC #########################
#############################################################################

str(acc_vs_raw)
    
s2dv::PlotLayout(PlotEquiMap,plot_dims = c("latitude", "longitude"), lon = lon,lat = lat, nrows = 2,
                 var = list(acc_vs_raw[['raw']]$diff.corr, acc_vs_raw[['bias']]$diff.corr, acc_vs_raw[['evmos']]$diff.corr,
                            acc_vs_raw[['mse_min']]$diff.corr, acc_vs_raw[['crps_min']]$diff.corr, acc_vs_raw[['rpc-based']]$diff.corr),
                 special_args = list(list(dots = acc_vs_raw[['raw']]$sign),
                                     list(dots = acc_vs_raw[['bias']]$sign),
                                     list(dots = acc_vs_raw[['evmos']]$sign),
                                     list(dots = acc_vs_raw[['mse_min']]$sign),
                                     list(dots = acc_vs_raw[['crps_min']]$sign),
                                     list(dots = acc_vs_raw[['rpc-based']]$sign)),
                 title_scale = 0.45, brks = seq(-1,1,0.2), titles = paste0(methods),
                 toptitle = paste0("ACC differences for ",var_name, " | SEAS/ERA5 \n Init. month = ",ini_str," | Reference period = ",ref_period," | ", ftime_info),
                 colNA = "grey", dot_symbol = 20, dot_size = 1, filled.continents = FALSE, filled.oceans = "white", width = 11, height = 7,
                 triangle_ends = c(FALSE,FALSE), res = 100, title_margin_scale = 4, fileout = NULL) #paste0(outdir,"/ACC_differences.png"))

#############################################################################
############ 7.5. Ranked Probability Skill Score (RPSS) #####################
#############################################################################

str(rpss)

s2dv::PlotLayout(PlotEquiMap,plot_dims = c("latitude", "longitude"), lon = lon,lat = lat, nrows = 2,
                 var = list(rpss[['raw']]$rpss, rpss[['bias']]$rpss, rpss[['evmos']]$rpss,
                            rpss[['mse_min']]$rpss, rpss[['crps_min']]$rpss, rpss[['rpc-based']]$rpss),
                 special_args = list(list(dots = rpss[['raw']]$sign),
                                     list(dots = rpss[['bias']]$sign),
                                     list(dots = rpss[['evmos']]$sign),
                                     list(dots = rpss[['mse_min']]$sign),
                                     list(dots = rpss[['crps_min']]$sign),
                                     list(dots = rpss[['rpc-based']]$sign)),
                 title_scale = 0.45, brks = seq(0,1,0.1), color_fun = clim.palette('yellowred'), titles = paste0(methods),
                 toptitle = paste0("RPSS vs climatology for ",var_name, " | SEAS/ERA5 \n Init. month = ",ini_str," | Reference period = ",ref_period," | ", ftime_info),
                 colNA = "grey", dot_symbol = 20, dot_size = 1, filled.continents = FALSE, filled.oceans = "white", width = 11, height = 7,
                 triangle_ends = c(FALSE,FALSE), res = 100, title_margin_scale = 4, fileout = NULL) #paste0(outdir,"/RPSS.png"))

#############################################################################
#### 7.6. RPSS using as reference forecast the non-calibrated forecast ######
#############################################################################

str(rpss_vs_raw)

s2dv::PlotLayout(PlotEquiMap,plot_dims = c("latitude", "longitude"), lon = lon,lat = lat, nrows = 2,
                 var = list(rpss_vs_raw[['raw']]$rpss, rpss_vs_raw[['bias']]$rpss, rpss_vs_raw[['evmos']]$rpss,
                            rpss_vs_raw[['mse_min']]$rpss, rpss_vs_raw[['crps_min']]$rpss, rpss_vs_raw[['rpc-based']]$rpss),
                 special_args = list(list(dots = rpss_vs_raw[['raw']]$sign),
                                     list(dots = rpss_vs_raw[['bias']]$sign),
                                     list(dots = rpss_vs_raw[['evmos']]$sign),
                                     list(dots = rpss_vs_raw[['mse_min']]$sign),
                                     list(dots = rpss_vs_raw[['crps_min']]$sign),
                                     list(dots = rpss_vs_raw[['rpc-based']]$sign)),
                 title_scale = 0.45, brks = seq(0,1,0.1), color_fun = clim.palette('yellowred'), titles = paste0(methods),
                 toptitle = paste0("RPSS vs non-calibrated forecasts for ",var_name, " | SEAS/ERA5 \n Init. month = ",ini_str," | Reference period = ",ref_period," | ", ftime_info),
                 colNA = "grey", dot_symbol = 20, dot_size = 1, filled.continents = FALSE, filled.oceans = "white", width = 11, height = 7,
                 triangle_ends = c(FALSE,FALSE), res = 100, title_margin_scale = 4, fileout = NULL) #paste0(outdir,"/RPSS_ref_raw.png"))

#############################################################################
########## 7.7. Continuous Ranked Probability Skill Score (CRPSS) ###########
#############################################################################

str(crpss)

s2dv::PlotLayout(PlotEquiMap,plot_dims = c("latitude", "longitude"), lon = lon,lat = lat, nrows = 2,
                 var = list(crpss[['raw']]$crpss, crpss[['bias']]$crpss, crpss[['evmos']]$crpss,
                            crpss[['mse_min']]$crpss, crpss[['crps_min']]$crpss, crpss[['rpc-based']]$crpss),
                 special_args = list(list(dots = crpss[['raw']]$sign),
                                     list(dots = crpss[['bias']]$sign),
                                     list(dots = crpss[['evmos']]$sign),
                                     list(dots = crpss[['mse_min']]$sign),
                                     list(dots = crpss[['crps_min']]$sign),
                                     list(dots = crpss[['rpc-based']]$sign)),
                 title_scale = 0.45, brks = seq(0,1,0.1), color_fun = clim.palette('yellowred'), titles = paste0(methods),
                 toptitle = paste0("CRPSS vs climatology for ",var_name, " | SEAS/ERA5 \n Init. month = ",ini_str," | Reference period = ",ref_period," | ", ftime_info),
                 colNA = "grey", dot_symbol = 20, dot_size = 1, filled.continents = FALSE, filled.oceans = "white", width = 11, height = 7,
                 triangle_ends = c(FALSE,FALSE), res = 100, title_margin_scale = 4, fileout = NULL) #paste0(outdir,"/CRPSS.png"))

#############################################################################
#### 7.8. CRPSS using as reference forecast the non-calibrated forecast #####
#############################################################################

s2dv::PlotLayout(PlotEquiMap,plot_dims = c("latitude", "longitude"), lon = lon,lat = lat, nrows = 2,
                 var = list(crpss_vs_raw[['raw']]$crpss, crpss_vs_raw[['bias']]$crpss, crpss_vs_raw[['evmos']]$crpss,
                            crpss_vs_raw[['mse_min']]$crpss, crpss_vs_raw[['crps_min']]$crpss, crpss_vs_raw[['rpc-based']]$crpss),
                 special_args = list(list(dots = crpss_vs_raw[['raw']]$sign),
                                     list(dots = crpss_vs_raw[['bias']]$sign),
                                     list(dots = crpss_vs_raw[['evmos']]$sign),
                                     list(dots = crpss_vs_raw[['mse_min']]$sign),
                                     list(dots = crpss_vs_raw[['crps_min']]$sign),
                                     list(dots = crpss_vs_raw[['rpc-based']]$sign)),
                 title_scale = 0.45, brks = seq(0,1,0.1), color_fun = clim.palette('yellowred'), titles = paste0(methods),
                 toptitle = paste0("CRPSS vs non-calibrated forecasts for ",var_name, " | SEAS/ERA5 \n Init. month = ",ini_str," | Reference period = ",ref_period," | ", ftime_info),
                 colNA = "grey", dot_symbol = 20, dot_size = 1, filled.continents = FALSE, filled.oceans = "white", width = 11, height = 7,
                 triangle_ends = c(FALSE,FALSE), res = 100, title_margin_scale = 4, fileout = NULL) #paste0(outdir,"/CRPSS_ref_raw.png"))

print("Finished! :-)")
