#########################################################################################
#                               Sub-seasonal Skill Scores                               #
#########################################################################################

# Creation: 02/2016
# Author: Nicola Cortesi
# Aim: validate subseasonal forecast systems
# I/O: input monthly hindcasts formatted as in /esnas,
#       output: .png with the desired skill score
# Assumptions: it relies on Load() to import data
# Branch: skill_assessment

# you can run it from the terminal with the syntax:
#
# Rscript SkillScores.R <chunk to compute>
#
# i.e: to run only the chunk number 3, write:
#
# Rscript SkillScores.R 3
#
# if you don't specify any argument, or you run it from the R interface, it runs all the chunks in a sequential way.
# Use this last approach only if the computational speed is not a rm.

rm(list=ls())

##########################################################################################
#                            Load functions and libraries                                #
##########################################################################################

#load libraries and functions:
library(SpecsVerification) # for skill scores
library(s2dverification)   # for Load() function
library(easyVerification)  # for veriApply() function
library(abind)             # for merging arrays
library(future)            # for availableCores() function
library(multiApply)        # for Apply() function

source('/esarchive/scratch/ncortesi/scripts/Rfunctions.R')
source('/esarchive/scratch/ncortesi/scripts/Andrea/Calibration.R')

## source('~/GIT/s2dverification/R/Utils.R')
## file_path='/home/Earth/ngonzal2/R/ConfFiles/IC3.conf'       # Load() file path
## file_path <- '/home/Earth/ncortesi/Downloads/RESILIENCE/IC3.conf'

##########################################################################################
#                                   Power9 settings                                      #
##########################################################################################

args <- commandArgs(TRUE) # put into variable args the list with all the optional arguments with whom the script was run from the terminal

## number of cpus employed for validating with veryApply(). Notice that any number greater than ~10 in Power9 could exceed the available memory resources, since each core needs ~50 GB of RAM for this task!
if(length(args) == 0) { print("Running the script in a sequential way"); my.par <- TRUE; nmaxcores <- max(1, availableCores()-1) }
if(length(args) == 1) { print("Running the script in a parallel way"); my.par <- TRUE; nmaxcores <- max(1, availableCores()-1) }
if(length(args) > 1) stop("Only one argument is required")

chunk <- ifelse(length(args) == 1 ,TRUE, FALSE) # set variable 'chunk' to TRUE if we are running the script from the command line with 1 argument, FALSE otherwise
if(chunk) chunk.month <- as.integer(args[1])  # number of the month to run in this script (if chunk == TRUE)

##########################################################################################
#                                     User's settings                                    #
##########################################################################################

# working dir where to store the output maps and files in the /Data and /Maps subdirs:
##work.dir <- "/scratch/Earth/ncortesi/RESILIENCE/Subestacional"
##work.dir <- "/esarchive/scratch/ncortesi/Subestacional/S2S_11_members/Thursday/wind_module_test"
##work.dir <- "/esarchive/scratch/ncortesi/Subestacional/ECMWF_4_members_1996-2013"
##work.dir <- "/esarchive/scratch/ncortesi/Subestacional/ECMWF_11_members_1996-2013"
##work.dir <- "/esarchive/scratch/ncortesi/Subestacional/ECMWF_4_members_1996-2013_4_start_dates"
##work.dir <- "/esarchive/scratch/ncortesi/Subestacional/ECMWF_11_members_1996-2013_4_start_dates"
##work.dir <- "/esarchive/scratch/ncortesi/Subestacional/ECMWF_11_members_1996-2013_4_start_dates_test"
work.dir <- "/esarchive/scratch/ncortesi/Subestacional/ECMWF_11_members_1996-2015"

forecast.year <- 2016 ##2014 ##2016 #2014   # starting year of the weekly sequence of the forecasts
mes <- 1                                          # starting forecast month (1: january, 2: february, etc.)
day <- 7 #2 #2                                       # starting forecast day (usually the first Thursday of January of the forecast year,
                                                  # which in case of forecasts of 2014 is the 2 of January and in case of the forecasts of 2016 is the 7 of January)
                                                  # (for 2014 forecasts, also Tuesday forecasts were downloaded. They start on the 7th of January)
# path of the monthly forecast system files:
##forecast.dir <- paste0('/esarchive/exp/ecmwf/s2s-monthly_ensforhc/weekly_mean/$VAR_NAME$_f24h/',forecast.year,'$MONTH$$DAY$/$VAR_NAME$_$START_DATE$.nc')
##forecast.dir <- paste0('/esarchive/exp/ECMWF/monthly/ensforhc/weekly_mean/$VAR_NAME$_f6h/', forecast.year,'$MONTH$$DAY$00/$VAR_NAME$_$START_DATE$00.nc')
##forecast.dir <- paste0('/esarchive/exp/ecmwf/monthly_ensforhc/weekly_mean/$VAR_NAME$_f6h/', forecast.year,'$MONTH$$DAY$00/$VAR_NAME$_$START_DATE$00.nc') ## forecast year: 2014
forecast.dir <- paste0('/esarchive/exp/ecmwf/monthly_ensforhc/weekly_mean/$VAR_NAME$_f6h/', forecast.year,'$MONTH$$DAY$/$VAR_NAME$_$START_DATE$.nc') # as previous but without 00 (for forecast year > 2014 belonging to the 11-members forecasts)
##forecast.dir <- paste0('/esarchive/exp/ecmwf/s2s-monthly_ensforhc/original_files/with_cdo/weekly_mean/$VAR_NAME$_f24h/', forecast.year,'$MONTH$$DAY$/$VAR_NAME$_$START_DATE$.nc')

##forecast.dir <- paste0('/esarchive/scratch/jginer/wind-comparison/ecmwf/weekly_mean/sfcWind_f6h/', forecast.year,'$MONTH$$DAY$/$VAR_NAME$_$START_DATE$.nc') # test with daily u and v

##forecast.dir <- paste0('/esarchive/scratch/jginer/wind-comparison/s2s/weekly_mean/sfcWind_f24h/', forecast.year,'$MONTH$$DAY$/$VAR_NAME$_$START_DATE$.nc') # test with daily u and v

# path of the weekly reanalysis data of the variable 'var' to study:
##rean.dir <- '/esarchive/recon/ecmwf/erainterim/weekly_mean/$VAR_NAME$-ecmwf-od_f6h/$VAR_NAME$_$START_DATE$.nc' ## 1994-2013 means for Thursday 2014 start dates (2 Jan., 9th, etc)
rean.dir <- '/esarchive/recon/ecmwf/erainterim/weekly_mean/$VAR_NAME$-ecmwf_f6h/$VAR_NAME$_$START_DATE$.nc' ## 1996-2015 means for Thrusday 2016 start dates (7 Jan, 14, ..)
                                                                                                          ## and for Tuesday start dates too (4 Jan, 11, ..)
#rean.dir <- '/esarchive/scratch/jginer/wind-comparison/erainterim/weekly_mean/$VAR_NAME$_f6h/$VAR_NAME$_$START_DATE$.nc' ## test for S2S
rean.name <- "ERA-Interim"

cfs.name <- 'ECMWF'               # name of the climate forecast system (CFS) used for predictions (to be used in map titles)

var.name <- 'sfcWind' #'tas'      # forecast variable to verify. It is the name used inside the netCDF files and as suffix in map filenames, i.e: 'sfcWind' or 'psl'
var.name.map <- "10-m wind speed" #'2m Temperature'  #'10m Wind Speed'   # forecast variable to verify (name used in the map titles), i.e: '10m Wind Speed' or 'SLP'

# if TRUE, interpolate the reanalysis to the same resolution of the model; if FALSE, interpolate the model to the same res.of the reanalysis:
# notice that if the reanalysis grid is the same as the model grid, no interpolation is performed.
rean.int <- FALSE

# choose one or more skills scores to compute between EnsCorr, FairRPSS, FairCRPSS and/or RelDiagr:
my.score.name <- c('FairRpss','FairCrpss','EnsCorr','RelDiagr')

# number of resamples considered in the bootstrapping of the FairRPSS and the FairCRPSS (should be AT LEAST 1000, better with 5000-10000) :
n.boot <- 100 #1000               

yr1.hind <- 1996 #1994 #1994               # first hindcast year
yr2.hind <- 2015 #2015 #2013 #2013         # last hindcast year (usually the forecast year -1)

veri.month <- 7 ## 8 #6:12 #1:12 #1:12      # select the month(s) you want to compute the chosen skill scores

## set it to TRUE if you want to bias correct the forecast with the calibration method before validation them:
calibration <- FALSE

## if you know some input forecast files can have NA values, set this option to TRUE so you can still validate them, by firstly removing forecasts with NA.
## Computation takes longer, so set this option to TRUE only if you are sure there are missing data in the forecasts:
na.remove <- FALSE  

four.start.dates <- TRUE ## if TRUE, discard in the validation the fifth start dates of the month if present.

leadtime.week <- c('5-11','12-18','19-25','26-32') # which leadtimes are available in the weekly dataset

my.prob <- c(1/3,2/3)      # quantiles used for FairRPSS and the RelDiagr (usually they are the terciles)
#conf.level <- c(0.025, 0.975) # percentiles employed for the calculation of the confidence level for the Rpss and Crpss (can be more than 2 if needed)

#int.lat <- NULL #1:160     # latitude position of grid points selected for the Reliability Diagram (if you want to subset a region; if not, put NULL)
#int.lon <- NULL #1:320     # longitude position of grid points selected for the Reliability Diagram (if you want to subset a region; if not, put NULL)

# coordinates of a chosen point in the world to plot the time series over the 52 maps (they must be the same values in objects la y lo)
#my.lat <- 55.3197120 
#my.lon <- 0.5625


###################################### Derived variables #############################################################################

#source(paste0(workdir,'/ColorBarV.R'))  # Color scale used for maps
n.categ <- 1 + length(my.prob)  # number of forecasting categories (usually 3 if terciles are used)

# blue-yellow-red colors:
#col <- ifelse(!mare, as.character(read.csv("/scratch/Earth/ncortesi/RESILIENCE/rgbhex.csv",header=F)[,1]), as.character(read.csv("/gpfs/projects/bsc32//bsc32842/scripts/rgbhex.csv",header=F)[,1]))  
#col<-as.character(read.csv("/home/Earth/vtorralb/rgbhex.csv",header=F)[,1])  # blue-yellow-red colors
col <- c("#0C046E", "#1914BE", "#2341F7", "#2F55FB", "#3E64FF", "#528CFF", "#64AAFF", "#82C8FF", "#A0DCFF", "#B4F0FF", "#FFFBAF", "#FFDD9A", "#FFBF87", "#FFA173", "#FF7055", "#FE6346", "#F7403B", "#E92D36", "#C80F1E", "#A50519")

sdates.seq <- weekly.seq(forecast.year, mes, day)  # load the sequence of dates corresponding to all the thursday of the year
pos.bis <- which(sdates.seq == paste0(forecast.year,"0229")) # find if there is a startdate at the 29th of february and return its position in the vector sdates.seq
if(l(pos.bis) != 0) sdates.seq <- sdates.seq[-pos.bis] # if there is a startdate at the 29th of february, remove it.

n.leadtimes <- length(leadtime.week)
n.yrs.hind <- yr2.hind - yr1.hind + 1
#my.month.short <- substr(my.month,1,3)

## extract geographic coordinates; do only ONCE for each preducton system and then save the lat/lon in a coordinates.RData and comment these rows to use function load() below: 
#data.hindcast <- Load(var='sfcWind', exp = 'EnsEcmwfWeekHind', 
#	      	 obs = NULL, sdates=paste0(yr1.hind:yr2.hind,substr(sdates.seq[startdate],5,6),substr(sdates.seq[startdate],7,8)),
#		 nleadtime = 1, leadtimemin = 1, leadtimemax = 1, output = 'lonlat', configfile = file_path, method='distance-weighted' )

#lats<-data.hindcast$lat
#lons<-data.hindcast$lon
#save(lats,lons,file=paste0(workdir,'/coordinates.RData'))
#rm(data.hindcast);gc()

# format geographic coordinates to use with PlotEquiMap:
#load(paste0(workdir,'/coordinates.RData'))
#n.lon <- length(lons)
#n.lat <- length(lats)
#n.lonr <- n.lon - ceiling(length(lons[lons<180 & lons > 0]))
#la<-rev(lats)
#lo<-c(lons[c((n.lonr+1):n.lon)]-360,lons[1:n.lonr])

## check if the two grids are the same:
rean.grid <- Load(var = var.name, exp = list(list(path=rean.dir)),
                  obs = NULL, sdates=paste0(yr1.hind,substr(sdates.seq[1],5,6),substr(sdates.seq[1],7,8)),
                  nleadtime = 1, leadtimemin = 1, leadtimemax = 1, output = 'lonlat', nprocs=1) 

model.grid <- Load(var = var.name, exp = list(list(path = forecast.dir)),
                  obs = NULL, sdates=paste0(yr1.hind, substr(sdates.seq[1],5,6), substr(sdates.seq[1],7,8)),
                  nleadtime = 1, leadtimemin = 1, leadtimemax = 1, output = 'lonlat', nprocs=1)

## number of hindcast members (determined loading the data of 1 year only)
n.members <- dim(model.grid$mod)[2]

if(identical(rean.grid$lon, model.grid$lon)) disable.interp <- TRUE else { disable.interp <- FALSE; print("The observed and forecasted grids are different; interpolation is necessary") }
rm(rean.grid, model.grid)
gc()

# Load only 1 year of the model lat and lon data, if we want to interpolate the model to the reanalysis:
if(!rean.int) data.grid <- Load(var = var.name, exp = list(list(path=rean.dir)),
                  obs = NULL, sdates=paste0(yr1.hind,substr(sdates.seq[1],5,6),substr(sdates.seq[1],7,8)),
                  nleadtime = 1, leadtimemin = 1, leadtimemax = 1, output = 'lonlat', nprocs=1) 

## Load only 1 year of the model lat and lon data, if we want to interpolate the reanalysis to the model:
if(rean.int) data.grid <- Load(var = var.name, exp = list(list(path=forecast.dir)),
                  obs = NULL, sdates=paste0(yr1.hind, substr(sdates.seq[1],5,6), substr(sdates.seq[1],7,8)),
                  nleadtime = 1, leadtimemin = 1, leadtimemax = 1, output = 'lonlat', nprocs=1)

lons <- data.grid$lon
lats <- data.grid$lat
n.lon <- length(lons)  ## ERA-Interim: 256x512 points (res. ~0.70); ECMWF from S2S project: 121x240 points; ECMWF original: 256x512 points
n.lat <- length(lats)
my.grid <- paste0('r',n.lon,'x',n.lat)

rm(data.grid)
gc()
    
#########################################################################################
#     Calculate and save up to all the chosen Skill Scores for a selected period        #
#########################################################################################
# if we run the script from the terminal with an argument, it only computes the month we specify in the second argument and save the results for that month
if(chunk) veri.month <- chunk.month
    
for(month in veri.month){
    #month <- 1 # for the debug

    # select the startdates (weeks) you want to compute:
    my.startdates <- which(as.integer(substr(sdates.seq,5,6)) == month) #startdates.monthly[[month]] #c(1:5)

    ## in case we are comparing with another system, there must be exactly the same number of start dates for each month, so the last start date is discarded:
    if(four.start.dates) my.startdates <- my.startdates[1:4]
    
    startdate.name <- my.month[month]                       # name given to the period of the selected startdates  # i.e:"January" 
    n.startdates <- length(my.startdates)                   # number of startdates in the selected month
    n.yrs <- n.startdates * n.yrs.hind                      # number of total years for the selected month
    
    print(paste0("Computing Skill Scores for ",startdate.name))

    # Merge all selected startdates:
    hind.dim <- c(n.members, n.yrs.hind * n.startdates, n.leadtimes, n.lat, n.lon)  # dimensions of the hindcast array
        
    ##my.FairRpss <- my.FairCrpss <- my.EnsCorr <- array(NA,c(n.leadtimes, n.lat,n.lon))
    #my.FairRpssConf <- my.FairCrpssConf <- array(NA, c(length(conf.level), n.leadtimes, n.lat, n.lon))
    my.FairRpss.pvalue <- my.FairCrpss.pvalue <- my.EnsCorr.pvalue <- array(NA, c(n.leadtimes, n.lat, n.lon))
    
    anom.rean.chunk <- anom.hindcast.mean.chunk <- array(NA, c(n.startdates * n.yrs.hind, n.leadtimes, n.lat, n.lon))
    anom.hindcast.chunk <- array(NA, c(n.members, n.startdates * n.yrs.hind, n.leadtimes, n.lat, n.lon))
    #my.interv<-(1 + sub.size*(s-1)):((sub.size*s) + add.last)          # longitude interval where to load data

    for(startdate in my.startdates){
        print(paste0("Start date num.", startdate))
        
	  pos.startdate <- which(startdate == my.startdates) # count of the number of stardates already loaded+1
          my.time.interv <- ( 1 + (pos.startdate-1) * n.yrs.hind) : (pos.startdate * n.yrs.hind)  # time interval where to load data
          
          print(paste0('Computing reanalysis anomalies for startdate ', which(startdate == my.startdates),'/',length(my.startdates)))

          # load weekly var rean data in: /esarchive/reconstructions/ecmwf/eraint/weekly_mean:
          if(!rean.int || disable.interp) data.rean <- Load(var = var.name, exp = list(list(path=rean.dir)),
                          obs = NULL, sdates=paste0(yr1.hind:yr2.hind,substr(sdates.seq[startdate],5,6),substr(sdates.seq[startdate],7,8)),
                          nleadtime = n.leadtimes, leadtimemin = 1, leadtimemax = n.leadtimes, output = 'lonlat', nprocs=1) ##availableCores())
          
          ## notice that if model grid is the same as rean grid, rean grid is not interpolated!
          if(rean.int && !disable.interp) data.rean <- Load(var = var.name, exp = list(list(path=rean.dir)),
                          obs = NULL, sdates=paste0(yr1.hind:yr2.hind,substr(sdates.seq[startdate],5,6),substr(sdates.seq[startdate],7,8)),
                          nleadtime = n.leadtimes, leadtimemin = 1, leadtimemax = n.leadtimes, output = 'lonlat', grid=my.grid, method='bilinear', nprocs=1)
          ## note that extra cores are only useful for interpolating, not for loading data
          
          # dim(data.rean$mod) # [1,1,20,4,320,640]

          # Mean of the 20 years (for each point and leadtime):
          clim.rean <- apply(data.rean$mod, c(1,2,4,5,6), mean)[1,1,,,] # average of all years for each leadtime and pixel
          clim.rean <- InsertDim(clim.rean, 1, n.yrs.hind) # repeat the same values <n.years> times to be able to calculate the anomalies #[1,1,20,4,320,640]
          
          anom.rean <- data.rean$mod[1,1,,,,] - clim.rean 
          #anom.rean<-InsertDim(anom.rean[1,1,,,,],1,1) # [1,20,4,320,640]

          rm(data.rean, clim.rean)
          gc()
          
	  #load(paste0(workdir,'/Data_',var.name,'/anom_rean_startdate',startdate,'.RData'))
	  #anom.rean <- drop(anom.rean)
	  anom.rean.chunk[my.time.interv,,,] <- anom.rean
          
          # Load hindcast data:
          print(paste0('Computing forecast anomalies for startdate ', which(startdate == my.startdates),'/',length(my.startdates)))
        
          # Load weekly var subseasonal data in: /esarchive/exp/ECMWF/monthly/ensforhc/weekly_mean and interpolate them to the same ERA-Interim resolution (512x256)
          if(rean.int || disable.interp) data.hindcast <- Load(var = var.name, exp = list(list(path=forecast.dir)),
                              obs = NULL, sdates = paste0(yr1.hind:yr2.hind, substr(sdates.seq[startdate],5,6), substr(sdates.seq[startdate],7,8)),
                              nleadtime = n.leadtimes, leadtimemin = 1, leadtimemax = n.leadtimes, output = 'lonlat', nprocs=1)

          ## notice that if model grid is the same as rean grid, model grid is not interpolated!
          if(!rean.int && !disable.interp) data.hindcast <- Load(var = var.name, exp = list(list(path=forecast.dir)),
                              obs = NULL, sdates = paste0(yr1.hind:yr2.hind, substr(sdates.seq[startdate],5,6), substr(sdates.seq[startdate],7,8)),
                              nleadtime = n.leadtimes, leadtimemin = 1, leadtimemax = n.leadtimes, output = 'lonlat', grid=my.grid, method='bilinear', nprocs=nmaxcores)  
          ## note that nmaxcores= ~20 cores is the optimal value for this interpolation to be fast and the RAM needed is about 10 GB for start date;
     
          
          ## ## for the debug (draw each submap to detect where is the bad data):
          ## for (i in 1:11) for(j in 1:18) for(k in 2:2) PlotEquiMap(data.hindcast$mod[1,i,j,k,,], lon=lons, lat=lats, filled.continents=FALSE )
          ## PlotEquiMap(data.hindcast$mod[1,2,15,2,,], lon=lons, lat=lats, filled.continents=FALSE ) ## bad model
          
          # Mean of the 4 members of the Hindcasts and of the 20 years (for each point and leadtime);
          clim.hindcast <- apply(data.hindcast$mod, c(1,4,5,6), mean, na.rm=na.remove)[1,,,] # average of all years and members for each leadtime and pixel

          clim.hindcast <- InsertDim(InsertDim(clim.hindcast, 1, n.yrs.hind), 1, n.members)  # repeat the same values <n.members> and <n.yrs.hind> times to calc. anomalies
          anom.hindcast <- data.hindcast$mod[1,,,,,] - clim.hindcast

          rm(data.hindcast, clim.hindcast)
          gc()

          ## average the anomalies over all the hindcast members to compute the ensemble mean [1,20,4,320,640]
          print("Computing Ensemble Mean. Please wait...")
          anom.hindcast.mean <- apply(anom.hindcast, c(2,3,4,5), mean, na.rm = FALSE) ## serial version of the below command, actually much faster!!! (95s vs 558s)
          ## system.time(anom.hindcast.mean <- Apply(data = anom.hindcast, margins = c(2,3,4,5), AtomicFun = "mean", na.rm = FALSE, ncores = nmaxcores))
          gc()
              
	  #load(paste0(workdir,'/Data_',var.name,'/anom_hindcast_startdate',startdate,'.RData')) 
	  ##anom.hindcast <- drop(anom.hindcast)
	  anom.hindcast.chunk[,my.time.interv,,,] <- anom.hindcast
          
	  rm(anom.hindcast, anom.rean)
          gc()
                   
	  ##anom.hindcast.mean <- drop(anom.hindcast.mean)
	  anom.hindcast.mean.chunk[my.time.interv,,,] <- anom.hindcast.mean
              
	  rm(anom.hindcast.mean)
          gc()	  

          ## test hindcast quality:
          ## for (i in 1:11) for(j in 1:18) for(k in 1:1) PlotEquiMap(anom.hindcast.chunk[i,j,k,,], lon=lons, lat=lats, filled.continents=FALSE )
          
    } # end load data and conversion to anomalies
    

    ## calibrate forecasts:
    if(calibration){
        print("Calibrating forecast anomalies. Please wait...")
        
        anom.hindcast.cal <- NA * anom.hindcast.chunk        
        names(dim(anom.rean.chunk)) <- c("year","lead","lat","lon") ## necessary to run Apply()
        names(dim(anom.hindcast.chunk)) <- c("member","year","lead","lat","lon") ## necessary to run Apply()

        ## reference: [sdate, year]; forecast: [sdate, year, ensemble]
        ## output: array [ensemble, sdate, syear]
        anom.hindcast.cal <- Apply(data=list(anom.rean.chunk, anom.hindcast.chunk), margins=list(c(2,3,4), c(3,4,5)), AtomicFun='Calibration', anom=TRUE, ncores=availableCores()-1)[[1]] ## [[1]] is to convert the list in array

        anom.hindcast.chunk <- anom.hindcast.cal
        rm(anom.hindcast.cal)
        gc()

        save(anom.hindcast.chunk, file=paste0(work.dir,'/',var.name,'_',my.month[month],'_calibrated_anomalies.RData')) ## save anom.hindcast.chunk
        ##load(file=paste0(work.dir,'/',var.name,'_',my.month[month],'_calibrated_anomalies.RData')) ## load anom.hindcast.chunk

        ## Note that with 9 cpus, the equivalent serial version below is 3 times slower!
        ## for(lon in 1:10){
        ##     print(lon)
        ##     for(lat in 1:10){
        ##         for (lead in 1:n.leadtimes){
        ##             anom.hindcast.cal[,,lead,lat,lon] <- Calibration(anom.rean.chunk[,lead,lat,lon], anom.hindcast.chunk[,,lead,lat,lon], anom=TRUE)
        ##         }
        ##     }
        ## }

    }
    
    if(any(my.score.name == "FairRpss")){
        print("Validating FairRpss. Please wait...")
        
        ##my.FairRpss.sub <- veriApply("FairRpss", fcst=anom.hindcast.sub, obs=anom.rean.sub, prob=my.prob, tdim=2, ensdim=1, parallel=FALSE, ncpus=nmaxcores)

        ## count number of eventual NA values (IT IS VERY, VERY SLOW!!!!):
        ##n.miss <- length(which(is.na(anom.hindcast.chunk)))
        ##if(n.miss > 0) print(paste0("Warning: ", n.miss," missing values were found in the hindcasts"))

        ## do NOT set palallel=TRUE or it will eat all the memory of the node, since each job would use dozens of GB or RAM!!!
        my.FairRps.chunk <- veriApply("FairRps", fcst=anom.hindcast.chunk, obs=anom.rean.chunk, prob=my.prob, tdim=2, ensdim=1, parallel=my.par, ncpus=nmaxcores, na.rm=na.remove)
        gc()
        ##my.FairRps.chunk.double <- veriApply("FairRps", fcst=anom.hindcast.chunk.double, obs=anom.rean.chunk.double, prob=my.prob, tdim=2, ensdim=1, parallel=FALSE, ncpus=nmaxcores)

        print("Validating FairRpss. Please wait......")

        ## for the debug:
        ## tm1 <- array(rnorm(11*72*4*256*512),c(11,72,4,256,512))
        ##  tm2 <- array(rnorm(72*4*256*512), c(72,4,256,512))
        ## test <- veriApply("FairRps", fcst=tm1, obs=tm2, prob=c(1/3,2/3), tdim=2, ensdim=1, parallel=TRUE, ncpus=5, na.rm=na.remove)
        my.FairRps.chunk.sum <- apply(my.FairRps.chunk, c(2,3,4), sum, na.rm=TRUE)
        
        ## the prob.for the climatological ensemble can be set without using convert2prob and they are the same for all points and leadtimes (33.33% for each tercile):
        ens.ref <- array(33.33, c(n.startdates * n.yrs.hind, 3)) 

        ## the observed probabilities vary from point to point and for each leadtime:   [obs]= [3*n.years*n.startdates, n.leadtimes, lat, lon]
        ## this function does not clean memory properly in Power9: it increases from 26 GB to 150+ GB without creating any new var!!!
        ## If this script is run from the interpreter, memory seems to be released but htop doesn't decrease the value, while if it is run from the terminal
        ## it won't free memory soo when computing the FairCrpss in parallel it will give a segmentation fault error due to lack of memory
        print("Converting observed anomalies to probabilities. Please wait...")
        obs <- apply(anom.rean.chunk, c(2,3,4), function(x){convert2prob(x, prob <- my.prob)})
        ##obs <- Apply(data = anom.rean.chunk, margins = c(2,3,4), AtomicFun = "convert2prob", prob = my.prob, ncores = nmaxcores) ## not working
        
        for(i in 1:10) gc(reset = T, full = T) ## even this command cannot clean memory after apply!!! (it increases from 26 GB to 150+ GB without creating any new var!!!)
        
        ## reshape obs to be able to apply EnsRps(ens, obs) below:  
        obs2 <- InsertDim(obs, 1, 3)
        obs2[2,1:(n.startdates * n.yrs.hind),,,] <- obs2[1,(n.startdates * n.yrs.hind + 1:(n.startdates * n.yrs.hind)),,,]
        obs2[3,1:(n.startdates * n.yrs.hind),,,] <- obs2[1,(2 * n.startdates * n.yrs.hind + 1:(n.startdates * n.yrs.hind)),,,]
        obs3 <- obs2[,1:(n.startdates * n.yrs.hind),,,,drop=F] # drop=F is used not to loose the dimension(s) with only 1 element
        
        my.Rps.clim.chunk <- apply(obs3, c(3,4,5), function(x){ EnsRps(ens.ref,t(x), format="members") })
        my.Rps.clim.chunk.sum <- apply(my.Rps.clim.chunk, c(2,3,4), sum, na.rm=TRUE)
                         
        my.FairRpss.chunk <- 1 - (my.FairRps.chunk.sum / my.Rps.clim.chunk.sum)
        
        rm(ens.ref, obs, obs2, obs3, my.FairRps.chunk.sum, my.Rps.clim.chunk.sum)
        gc()

        ## ## Not-Fair RPSS (without bootstrapping):
        ## my.Rps.chunk <- veriApply("EnsRps", fcst=anom.hindcast.chunk, obs=anom.rean.chunk, prob=my.prob, tdim=2, ensdim=1, parallel=my.par, ncpus=nmaxcores, na.rm=FALSE)
        ## my.Rps.chunk.sum <- apply(my.Rps.chunk, c(2,3,4), sum, na.rm=TRUE)
        ## my.Rpss.chunk <- 1 - (my.Rps.chunk.sum / my.Rps.clim.chunk.sum)
        ## rm(my.Rps.chunk.sum)

        ## ######################### bootstrapping ##############################
        print("Bootstrapping FairRpss. Please wait...")

        my.FairRpssBoot <- array(NA, c(n.boot, n.leadtimes, n.lat, n.lon)) # define it only now because for n.boot=1000, it is a 2 GB object!!!
        
        for(b in 1:n.boot){
            cat(paste0("Bootstrapping FairRpss: resampling n. ",b,"/",n.boot,"\n"))
            yrs.sampled <- sample(1:n.yrs, replace=TRUE)

            ##for(y in 1:n.yrs) anom.rean.chunk.sampled[y,,,] <- anom.rean.chunk[yrs.sampled[y],,,]
            ##for(y in 1:n.yrs) anom.hindcast.chunk.sampled[,y,,,] <- anom.hindcast.chunk[,yrs.sampled[y],,,]

            my.FairRps.sampled <- my.FairRps.chunk              
            my.Rps.clim.sampled <- my.Rps.clim.chunk
            
            for(y in 1:n.yrs) my.FairRps.sampled[y,,,] <- my.FairRps.chunk[yrs.sampled[y],,,]
            for(y in 1:n.yrs) my.Rps.clim.sampled[y,,,] <- my.Rps.clim.chunk[yrs.sampled[y],,,]
            
            ##my.FairRps.chunk <- veriApply("FairRps", fcst=anom.hindcast.chunk.sampled, obs=anom.rean.chunk.sampled, prob=my.prob, tdim=2, ensdim=1, parallel=FALSE, ncpus=nmaxcores)
            ##ens <- array(33,c(n.startdates*n.yrs.hind,3)) 
            ##obs <- apply(anom.rean.chunk.sampled, c(2,3,4), function(x){convert2prob(x, prob <- my.prob)})
            ##obs2 <- InsertDim(obs,1,3)
            ##obs2[2,1:(n.startdates*n.yrs.hind),,,] <- obs2[1,(n.startdates*n.yrs.hind + 1:(n.startdates*n.yrs.hind)),,,]
            ##obs2[3,1:(n.startdates*n.yrs.hind),,,] <- obs2[1,(2*n.startdates*n.yrs.hind + 1:(n.startdates*n.yrs.hind)),,,]
            ##obs3 <- obs2[,1:(n.startdates*n.yrs.hind),,,,drop=F]
            ##my.Rps.clim.chunk <- apply(obs3,c(3,4,5), function(x){ EnsRps(ens,t(x),format="members") })
                
            my.FairRps.sampled.sum <- apply(my.FairRps.sampled, c(2,3,4), sum, na.rm=TRUE)
            my.Rps.clim.sampled.sum <- apply(my.Rps.clim.sampled, c(2,3,4), sum, na.rm=TRUE)                
            my.FairRpss.sampled <- 1 - (my.FairRps.sampled.sum / my.Rps.clim.sampled.sum)
            
            my.FairRpssBoot[b,,,] <- my.FairRpss.sampled
            gc()
        }

        rm(my.FairRps.sampled, my.Rps.clim.sampled, my.FairRps.sampled.sum,  my.Rps.clim.sampled.sum,  my.FairRpss.sampled, yrs.sampled)
        gc()

        ## save the partial results of the validation for measuring the significance of the difference between two model versions:
        save(my.FairRps.chunk, my.Rps.clim.chunk, file=paste0(work.dir,'/',var.name,'_',my.month[month],'_Rps.RData'))
        rm(my.FairRps.chunk, my.Rps.clim.chunk)
        gc()

        ## test if the bootstrapped mean value is significantly different from this value or not:        
        my.FairRpssBootMean <- apply(my.FairRpssBoot, c(2,3,4), mean, na.rm=TRUE)

        my.test.value <- 0 
        my.FairRpss.diff1 <- my.FairRpssBootMean + abs(my.test.value - my.FairRpssBootMean)
        my.FairRpss.diff2 <- my.FairRpssBootMean - abs(my.test.value - my.FairRpssBootMean)
        
        for(i in 1:dim(my.FairRpssBoot)[2]){
            for(j in 1:dim(my.FairRpssBoot)[3]){
                for(k in 1:dim(my.FairRpssBoot)[4]){
                    extr1 <- which(my.FairRpssBoot[,i,j,k] > my.FairRpss.diff1[i,j,k])
                    extr2 <- which(my.FairRpssBoot[,i,j,k] < my.FairRpss.diff2[i,j,k])
                    if(my.test.value != 0) my.FairRpss.pvalue[i,j,k] <- (length(extr1) + length(extr2)) / n.boot  # two-tailed test
                    if(my.test.value == 0) my.FairRpss.pvalue[i,j,k] <- ifelse(my.FairRpssBootMean[i,j,k] > 0, length(extr2)/n.boot, length(extr1)/n.boot)
                }
            }
        }

        rm(my.FairRpssBoot, my.FairRpssBootMean, my.FairRpss.diff1, my.FairRpss.diff2, extr1, extr2)
        gc()
        
        ##which(my.FairRpss.pvalue > 0.05) # points where the FairRPSS is significant (i.e: whose FairRPSS is not significantly different from the population mean)
        ## calculate the percentiles of the skill score in case you want to compute the confidence interval of the 95%:
        ##my.FairRpssConf[,,,chunk$int[[c]]] <- apply(my.FairRpssBoot[,,,chunk$int[[c]]], c(2,3,4), function(x) quantile(x, probs=conf.level, type=8))

        ## save the results for the chunk until now (in case the job crashes before the end):
        save(my.FairRpss.chunk, my.FairRpss.pvalue, file=paste0(work.dir,'/',var.name,'_',my.month[month],'.RData'))
        ##load(paste0(work.dir,'/',var.name,'_',my.month[month],'.RData'))
        
    } # close if on FairRpss

    if(any(my.score.name=="FairCrpss")){
        print("Validating FairCrpss. Please wait...")

        ## my.FairCrpss.chunk <- veriApply("FairCrpss", fcst=anom.hindcast.chunk, obs=anom.rean.chunk, tdim=2, ensdim=1, parallel=FALSE, ncpus=1)

        ##Numerador:
        ## Power9 is not cleaning memory properly!!!
        my.FairCrps.chunk <- veriApply("FairCrps", fcst=anom.hindcast.chunk, obs=anom.rean.chunk, tdim=2, ensdim=1, parallel=FALSE, ncpus=1, na.rm=na.remove)

        print("Validating FairCrpss. Please wait......")

        my.FairCrps.chunk.sum <- apply(my.FairCrps.chunk,c(2,3,4), sum, na.rm=TRUE)

        ## Denominador:
        print("Validating FairCrpss. Please wait.........")

        my.Crps.clim.chunk <- array(NA, c(n.startdates * n.yrs.hind, n.leadtimes, n.lat, n.lon))
        anom.hind.clim <- array(NA, c(n.members, n.startdates * n.yrs.hind, n.leadtimes, n.lat, n.lon))
        anom.rean.clim <- array(NA, c(n.startdates * n.yrs.hind, n.startdates * n.yrs.hind, n.leadtimes, n.lat, n.lon))

        for(i in 1:n.leadtimes){
            print(paste0("leadtime= ",i))
            for(j in 1:n.lat){
                for(k in 1:n.lon){
                    ## extract a random sample with no replacement of the hindcast:    
                    ## anom.hind.clim[,,i,j,k] <- array(sample(anom.hindcast.chunk[,,i,j,k],n.startdates*n.yrs.hind*n.members),c(n.startdates*n.yrs.hind,n.members))
                    anom.rean.clim[,,i,j,k] <- t(array(anom.rean.chunk[,i,j,k],c(n.startdates*n.yrs.hind, n.startdates*n.yrs.hind)))
                    my.Crps.clim.chunk[,i,j,k] <- EnsCrps((anom.rean.clim[,,i,j,k]), anom.rean.chunk[,i,j,k])
                }
            }
        }

        print("Validating FairCrpss. Please wait............")

        ## anom.hind.clim <- apply(anom.hindcast.chunk,c(3,4,5), sample)
        ## anom.all.chunk <- abind(anom.hind.clim,anom.rean.chunk, along=1) # merge exp and obs together to use apply:
        ## my.Crps.clim.chunk <- apply(anom.all.chunk, c(3,4,5), function(x){ EnsCrps(t(x[1:n.members,]), x[n.members+1,])})
        my.Crps.clim.chunk.sum <- apply(my.Crps.clim.chunk, c(2,3,4), sum, na.rm=TRUE)

        ## Skill Score:
        my.FairCrpss.chunk <- 1 - (my.FairCrps.chunk.sum / my.Crps.clim.chunk.sum)

        rm(my.FairCrps.chunk.sum, my.Crps.clim.chunk.sum)
        gc()

        ## Not-Fair CRPSS (without bootstrapping):
        ##
        ## ## Function like EnsCrps, but without the initial control which is bugged
        ## EnsCrps2 <- function (ens, obs, R.new = NA) { 
        ##    crps = enscrps_cpp(ens, obs, R.new)
        ##    return(crps)
        ## }
        ##
        ## my.Crps.chunk <- veriApply("EnsCrps2", fcst=anom.hindcast.chunk, obs=anom.rean.chunk, prob=my.prob, tdim=2, ensdim=1, parallel=FALSE, ncpus=1, na.rm=na.remove)
        ## my.Crps.chunk.sum <- apply(my.Crps.chunk, c(2,3,4), sum, na.rm=T)
        ## my.Crpss.chunk <- 1 - (my.crps.chunk.sum / my.Crps.clim.chunk.sum)
        ## rm(my.Crps.chunk.sum)
        
            
        ## ############################### bootstrapping ###########################################
        print("Bootstrapping FairCrpss. Please wait...")

        my.FairCrpssBoot <- array(NA, c(n.boot, n.leadtimes, n.lat, n.lon))  # define it only now, because for n.boot = 1000, it is a 2 GB object!!!
           
        for(b in 1:n.boot){
            cat(paste0("Bootstrapping FairCrpss: resampling n. ",b,"/",n.boot,"\n"))
            
            yrs.sampled <- sample(1:n.yrs, replace=TRUE)
            
            my.FairCrps.sampled <- my.FairCrps.chunk                
            my.Crps.clim.sampled <- my.Crps.clim.chunk
            
            for(y in 1:n.yrs) my.FairCrps.sampled[y,,,] <- my.FairCrps.chunk[yrs.sampled[y],,,]
            for(y in 1:n.yrs) my.Crps.clim.sampled[y,,,] <- my.Crps.clim.chunk[yrs.sampled[y],,,]

            my.FairCrps.sampled.sum <- apply(my.FairCrps.sampled,c(2,3,4), sum)
            my.Crps.clim.sampled.sum <- apply(my.Crps.clim.sampled,c(2,3,4), sum)
            
            my.FairCrpss.sampled <- 1-(my.FairCrps.sampled.sum / my.Crps.clim.sampled.sum)

            my.FairCrpssBoot[b,,,] <- my.FairCrpss.sampled
            
            gc()
        }

        rm(my.FairCrps.sampled,  my.Crps.clim.sampled, my.FairCrpss.sampled , yrs.sampled)
        gc()
        
        my.FairCrpssBootMean <- apply(my.FairCrpssBoot,c(2,3,4),mean,na.rm=TRUE)

        print("Bottstrapping FairCrpss. Please wait......")

        my.test.value <- 0 # my.FairCrpss.chunk   # test if the bootstrapped mean value is significantly different from this value or not
        
        my.FairCrpss.diff1 <- my.FairCrpssBootMean + abs(my.test.value - my.FairCrpssBootMean)
        my.FairCrpss.diff2 <- my.FairCrpssBootMean - abs(my.test.value - my.FairCrpssBootMean)
        
        for(i in 1:dim(my.FairCrpssBoot)[2]){
            for(j in 1:dim(my.FairCrpssBoot)[3]){
                for(k in 1:dim(my.FairCrpssBoot)[4]){
                    extr1 <- which(my.FairCrpssBoot[,i,j,k] > my.FairCrpss.diff1[i,j,k])
                    extr2 <- which(my.FairCrpssBoot[,i,j,k] < my.FairCrpss.diff2[i,j,k])
                    if(my.test.value != 0) my.FairCrpss.pvalue[i,j,k] <- (length(extr1) + length(extr2)) / n.boot
                    if(my.test.value == 0) my.FairCrpss.pvalue[i,j,k] <- ifelse(my.FairCrpssBootMean[i,j,k] > 0, length(extr2)/n.boot, length(extr1)/n.boot)
                }
            }
        }

        rm(my.FairCrpssBoot, my.FairCrpssBootMean,  my.FairCrpss.diff1, my.FairCrpss.diff2,  extr1,  extr2)
        ## which(my.FairCrpss.pvalue > 0.05) # points where thw FairCRPSS is significant (i.e: whose FairRPSS  not significantly different from the population mean)
        
        ## calculate the percentiles of the skill score if you want to measure hte confidence interval:
        ## my.FairCrpssConf[,,,chunk$int[[c]]] <- apply(my.FairCrpssBoot[,,,chunk$int[[c]]], c(2,3,4), function(x) quantile(x, probs=conf.level, type=8))
        
        ## save the results for the chunk until now:
        save(my.FairRpss.chunk, my.FairRpss.pvalue, my.FairCrpss.chunk, my.FairCrpss.pvalue, file=paste0(work.dir,'/',var.name,'_',my.month[month],'.RData'))

        ## save also the results for measuring the significance of the difference between two model versions:
        ## save(my.FairCrps.chunk, my.Crps.clim.chunk, file=paste0(work.dir,'/',var.name,'_',my.month[month],'_CRPS.RData'))
        
    } # close if on FairCrpss

    if(any(my.score.name=="EnsCorr")){
        print("Validating EnsCorr. Please wait...")

        ## ensemble mean correlation for the selected startdates (first dim, 'week') and all leadtimes (second dim):
        my.EnsCorr.chunk <- my.PValue.chunk <- array(NA, c(n.leadtimes, n.lat, n.lon))

        for(i in 1:n.leadtimes){
	    for(j in 1:n.lat){
                for(k in 1:n.lon){                      
                    my.EnsCorr.chunk[i,j,k] <- cor(anom.hindcast.mean.chunk[,i,j,k], anom.rean.chunk[,i,j,k], use="complete.obs")
                    ## option 'na.method' is chosen from the following list: c("all.obs", "complete.obs", "pairwise.complete.obs", "everything", "na.or.complete")
                    
                    my.EnsCorr.pvalue[i,j,k] <- cor.test(anom.hindcast.mean.chunk[,i,j,k], anom.rean.chunk[,i,j,k], use="complete.obs")$p.value
                    
                }
	    }
        }

        ## save also the results for measuring the significance of the difference between two model versions:
        save(anom.hindcast.mean.chunk, anom.rean.chunk, file=paste0(work.dir,'/',var.name,'_',my.month[month],'_mean_anom.RData'))
    
        ##my.EnsCorr[,,chunk$int[[c]]] <- my.EnsCorr.chunk
        ##my.PValue[,,chunk$int[[c]]] <- my.PValue.chunk

        ##rm(anom.hindcast.chunk,anom.hindcast.mean.chunk,my.EnsCorr.chunk,my.PValue.chunk)

    }

    rm(anom.hindcast.mean.chunk)
    gc()
                                        
    ## the Reliability diagram cannot be computed splitting the data in chunk:
    if(any(my.score.name=="RelDiagr")){ # in this case the computation cannot be split in groups of grid points, but can be split for leadtime instead:
        print("Validating Reliability Diagrams. Please wait...")

            obs.chunk.prob <- ens.chunk.prob <- list()
            
            int1 <- 1:(n.startdates*n.yrs.hind)  # time interval of the data of the first tercile
            int2 <- (1+(n.startdates*n.yrs.hind)):(2*n.startdates*n.yrs.hind)  # time interval of the data of the second tercile
            int3 <- (1+(2*n.startdates*n.yrs.hind)):(3*n.startdates*n.yrs.hind)  # time interval of the data of the third tercile
                
            for(lead in 1:n.leadtimes){
                cat(paste0('leadtime n.: ',lead,'/',n.leadtimes,'\n'))

                anom.hindcast.lead <- anom.hindcast.chunk[,,lead,,] # extract only the lead time we are interested in
                anom.hindcast.chunk.perm <- aperm(anom.hindcast.lead, c(2,1,3,4))  # swap the first two dimensions to put the years as first dimension (needed for convert2prob)
                gc()

                cat('Computing the Reliability Diagram. Please wait... \n')
                ens.chunk <- apply(anom.hindcast.chunk.perm, c(3,4), convert2prob, prob = my.prob) # its dimens. are: [n.startdates*n.yrs.hind*n.forecast.categories, n.lat, n.lon]
                anom.rean.lead <- anom.rean.chunk[,lead,,]
                
                cat('Computing the Reliability Diagram. Please wait...... \n')

                obs.chunk <- apply(anom.rean.lead, c(2,3), convert2prob, prob = my.prob) # the first n.members*n.yrs.hind elements belong to the first tercile, and so on.
                ens.chunk.prob[[lead]] <- apply(ens.chunk, c(2,3), function(x) count2prob(matrix(x, n.startdates*n.yrs.hind, n.categ), type=4)) # type=4 is the only method with sum(prob)=1 !!
                cat('Computing the Reliability Diagram. Please wait......... \n')

                obs.chunk.prob[[lead]] <- apply(obs.chunk, c(2,3), function(x) count2prob(matrix(x, n.startdates*n.yrs.hind, n.categ), type=4)) 

                #print(my.RelDiagr) # for debugging          
                rm(ens.chunk, obs.chunk)
                gc()

            } # next lead
            
        } # close if on RelDiagr

        print("Saving results...")

        # save the results for the chunk:
        save(my.FairRpss.chunk, my.FairRpss.pvalue, my.FairCrpss.chunk, my.FairCrpss.pvalue, my.EnsCorr.chunk, my.EnsCorr.pvalue, obs.chunk.prob, ens.chunk.prob, int1, int2, int3, work.dir, rean.dir, rean.name, forecast.year, forecast.dir, cfs.name, var.name, var.name.map, mes, day, yr1.hind, yr2.hind, leadtime.week, n.members, my.prob, n.boot, n.categ, sdates.seq, n.leadtimes, n.yrs.hind, lons, lats, n.lon, n.lat, my.grid, month, my.startdates, startdate.name, n.startdates, n.yrs, file=paste0(work.dir,'/',var.name,'_',my.month[month],'.RData'))

        # if you only compute the RelDiagr, save this:
        #save(obs.chunk.prob, ens.chunk.prob, int1, int2, int3, work.dir, rean.dir, rean.name, forecast.year, forecast.dir, cfs.name, var.name, var.name.map, mes, day, yr1.hind, yr2.hind, leadtime.week, n.members, my.prob, n.boot, n.categ, sdates.seq, n.leadtimes, n.yrs.hind, lons, lats, n.lon, n.lat, my.grid, veri.month, my.startdates, startdate.name, n.startdates, n.yrs, file=paste0(work.dir,'/',var.name,'_',my.month[month],'.RData'))
    
        print("Saving results...done")

} # next m (month)


print("Finished!")


