#########################################################################################
#                               Sub-seasonal Skill Scores                               #
#########################################################################################
# you can run it from the terminal with the syntax:
#
# Rscript SkillScores_v7.R <chunk to compute>
#
# i.e: to run only the chunk number 3, write:
#
# Rscript SkillScores_v4.R 3
#
# if you don't specify any argument, or you run it from the R interface, it runs all the chunks in a sequential way.
# Use this last approach only if the computational speed is not a problem.

rm(list=ls())

##########################################################################################
#                            Load functions and libraries                                #
##########################################################################################

#load libraries and functions:
library(SpecsVerification) # for skill scores
library(s2dverification)   # for Load() function
library(easyVerification)  # for veriApply() function
library(abind)             # for merging arrays
source('/home/Earth/ncortesi/scripts/Rfunctions.R')

##########################################################################################
#                                MareNostrum settings                                    #
##########################################################################################

args <- commandArgs(TRUE) # put into variable args the list with all the optional arguments with whom the script was run from the terminal

if(length(args) == 0) print("Running the script in a sequential way")
if(length(args) > 1) stop("Only one argument is required")

chunk <- ifelse(length(args) == 1 ,TRUE, FALSE) # set variable 'chunk' to TRUE if we are running the script from the command line with 1 argument, FALSE otherwise

if(chunk) chunk.month <- as.integer(args[1])  # number of the month to run in this script (if chunk == TRUE)

##########################################################################################
#                                     User's settings                                    #
##########################################################################################

# working dir where to put the output maps and files in the /Data and /Maps subdirs:
work.dir <- "/scratch/Earth/ncortesi/RESILIENCE/Subestacional"

# path of the weekly var rean data:
#rean.dir <- '/esarchive/old-files/recon_ecmwf_erainterim/weekly_mean/$VAR_NAME$_f6h/$VAR_NAME$_$START_DATE$.nc'

rean.dir <- '/esnas/recon/ecmwf/erainterim/weekly_mean/$VAR_NAME$-ecmwf_f6h/$VAR_NAME$_$START_DATE$.nc'
rean.name <- "ERA-Interim"

# path of the monthly forecast system files:
forecast.year <- 2014                    # starting year of the weekly sequence of the forecasts
forecast.dir <- paste0('/esnas/exp/ECMWF/monthly/ensforhc/weekly_mean/$VAR_NAME$_f6h/',forecast.year,'$MONTH$$DAY$00/$VAR_NAME$_$START_DATE$00.nc')

cfs.name <- 'ECMWF'                      # name of the climate forecast system (CFS) used for monthly predictions (to be used in map titles)
var.name <- 'sfcWind'                    # forecast variable. It is the name used inside the netCDF files and as suffix in map filenames, i.e: 'sfcWind' or 'psl'
var.name.map <- '10m Wind Speed'         # forecast variable to verify (name used in the map titles), i.e: '10m Wind Speed' or 'SLP'

mes <- 1                                 # starting forecast month (1: january, 2: february, etc.)
day <- 2                                 # starting forecast day

yr1.hind <- 1994 #1994         # first hindcast year
yr2.hind <- 2013 #2013         # last hindcast year (usually the forecast year -1)

leadtime.week <- c('5-11','12-18','19-25','26-32') # which leadtimes are available in the weekly dataset
n.members <- 4  # number of hindcast members

my.score.name <- c('FairRpss','FairCrpss','EnsCorr','RelDiagr')  # choose one or more skills scores to compute between EnsCorr, FairRPSS, FairCRPSS and/or RelDiagr
            
veri.month <- 1:12      # select the month(s) you want to compute the chosen skill scores

my.prob <- c(1/3,2/3)      # quantiles used for FairRPSS and the RelDiagr (usually they are the terciles)
#conf.level <- c(0.025, 0.975) # percentiles employed for the calculation of the confidence level for the Rpss and Crpss (can be more than 2 if needed)

int.lat <- NULL #1:160     # latitude position of grid points selected for the Reliability Diagram (if you want to subset a region; if not, put NULL)
int.lon <- NULL #1:320     # longitude position of grid points selected for the Reliability Diagram (if you want to subset a region; if not, put NULL)

n.boot <- 1000               # number of resamples considered in the bootstrapping of the FairRPSS and the FairCRPSS

# coordinates of a chosen point in the world to plot the time series over the 52 maps (they must be the same values in objects la y lo)
#my.lat <- 55.3197120 
#my.lon <- 0.5625

###################################### Derived variables #############################################################################

#source(paste0(workdir,'/ColorBarV.R'))  # Color scale used for maps
n.categ <- 1+length(my.prob)  # number of forecasting categories (usually 3 if terciles are used)

# blue-yellow-red colors:
#col <- ifelse(!mare, as.character(read.csv("/scratch/Earth/ncortesi/RESILIENCE/rgbhex.csv",header=F)[,1]), as.character(read.csv("/gpfs/projects/bsc32//bsc32842/scripts/rgbhex.csv",header=F)[,1]))  
#col<-as.character(read.csv("/home/Earth/vtorralb/rgbhex.csv",header=F)[,1])  # blue-yellow-red colors
col <- c("#0C046E", "#1914BE", "#2341F7", "#2F55FB", "#3E64FF", "#528CFF", "#64AAFF", "#82C8FF", "#A0DCFF", "#B4F0FF", "#FFFBAF", "#FFDD9A", "#FFBF87", "#FFA173", "#FF7055", "#FE6346", "#F7403B", "#E92D36", "#C80F1E", "#A50519")

sdates.seq <- weekly.seq(forecast.year,mes,day)  # load the sequence of dates corresponding to all the thursday of the year
n.leadtimes <- length(leadtime.week)
n.yrs.hind <- yr2.hind-yr1.hind+1
#my.month.short <- substr(my.month,1,3)

## extract geographic coordinates; do only ONCE for each preducton system and then save the lat/lon in a coordinates.RData and comment these rows to use function load() below: 
#data.hindcast <- Load(var='sfcWind', exp = 'EnsEcmwfWeekHind', 
#	      	 obs = NULL, sdates=paste0(yr1.hind:yr2.hind,substr(sdates.seq[startdate],5,6),substr(sdates.seq[startdate],7,8)),
#		 nleadtime = 1, leadtimemin = 1, leadtimemax = 1, output = 'lonlat', configfile = file_path, method='distance-weighted' )

#lats<-data.hindcast$lat
#lons<-data.hindcast$lon
#save(lats,lons,file=paste0(workdir,'/coordinates.RData'))
#rm(data.hindcast);gc()

# format geographic coordinates to use with PlotEquiMap:
#load(paste0(workdir,'/coordinates.RData'))
#n.lon <- length(lons)
#n.lat <- length(lats)
#n.lonr <- n.lon - ceiling(length(lons[lons<180 & lons > 0]))
#la<-rev(lats)
#lo<-c(lons[c((n.lonr+1):n.lon)]-360,lons[1:n.lonr])

# load only 1 year of weekly var rean data from reanalysis to get lat and lon:        
data.rean <- Load(var = var.name, exp = list(list(path=rean.dir)),
                  obs = NULL, sdates=paste0(yr1.hind,substr(sdates.seq[1],5,6),substr(sdates.seq[1],7,8)),
                  nleadtime = 1, leadtimemin = 1, leadtimemax = 1, output = 'lonlat', nprocs=1) 

lons <- data.rean$lon
lats <- data.rean$lat
n.lon <- length(lons)
n.lat <- length(lats)
my.grid<-paste0('r',n.lon,'x',n.lat)

#file_path='/home/Earth/ngonzal2/R/ConfFiles/IC3.conf'              # Load() file path
#file_path <- '/home/Earth/ncortesi/Downloads/RESILIENCE/IC3.conf'

#########################################################################################
#     Calculate and save up to all the chosen Skill Scores for a selected period        #
#########################################################################################
# if we run the script from the terminal with an argument, it only computes the month we specify in the second argument and save the results for that month
if(chunk) veri.month <- chunk.month
    
for(month in veri.month){
    #month=1 # for the debug

    # select the startdates (weeks) you want to compute:
    my.startdates <- which(as.integer(substr(sdates.seq,5,6)) == month) #startdates.monthly[[month]] #c(1:5)
    
    startdate.name <- my.month[month]                       # name given to the period of the selected startdates  # i.e:"January" 
    n.startdates <- length(my.startdates)                   # number of startdates in the selected month
    n.yrs <- length(my.startdates)*n.yrs.hind               # number of total years for the selected month
    
    print(paste0("Computing Skill Scores for ",startdate.name))

    # Merge all selected startdates:
    hind.dim <- c(n.members, n.yrs.hind*n.startdates, n.leadtimes, n.lat, n.lon)  # dimensions of the hindcast array
        
    my.FairRpss <- my.FairCrpss <- my.EnsCorr <- array(NA,c(n.leadtimes, n.lat,n.lon))
    my.FairRpssBoot <- my.FairCrpssBoot <- array(NA, c(n.boot, n.leadtimes, n.lat, n.lon))
    #my.FairRpssConf <- my.FairCrpssConf <- array(NA, c(length(conf.level), n.leadtimes, n.lat, n.lon))
    my.FairRpss.pvalue <- my.FairCrpss.pvalue <- my.EnsCorr.pvalue <- array(NA, c(n.leadtimes, n.lat, n.lon))
    
    anom.rean.chunk <- anom.hindcast.mean.chunk <- array(NA, c(n.startdates*n.yrs.hind, n.leadtimes, n.lat, n.lon))
    anom.rean.clim <- array(NA, c(n.startdates*n.yrs.hind, n.startdates*n.yrs.hind, n.leadtimes, n.lat, n.lon))
    anom.hindcast.chunk <- array(NA, c(n.members, n.startdates*n.yrs.hind, n.leadtimes, n.lat, n.lon))
    #my.interv<-(1 + sub.size*(s-1)):((sub.size*s) + add.last)          # longitude interval where to load data

    for(startdate in my.startdates){
	  pos.startdate <- which(startdate == my.startdates) # count of the number of stardates already loaded+1
          my.time.interv <- (1+(pos.startdate-1)*n.yrs.hind):(pos.startdate*n.yrs.hind)  # time interval where to load data
          
          print(paste0('Computing reanalysis anomalies for startdate ', which(startdate==my.startdates),'/',length(my.startdates)))

          # load weekly var rean data in: /esnas/reconstructions/ecmwf/eraint/weekly_mean
          data.rean <- Load(var = var.name, exp = list(list(path=rean.dir)),
                          obs = NULL, sdates=paste0(yr1.hind:yr2.hind,substr(sdates.seq[startdate],5,6),substr(sdates.seq[startdate],7,8)),
                          nleadtime = n.leadtimes, leadtimemin = 1, leadtimemax = n.leadtimes, output = 'lonlat', nprocs=1) 

          # dim(data.rean$mod) # [1,1,20,4,320,640]

          # Mean of the 20 years (for each point and leadtime)
          clim.rean <- apply(data.rean$mod,c(1,2,4,5,6),mean)[1,1,,,] # average of all years for each leadtime and pixel
          clim.rean <- InsertDim(InsertDim(InsertDim(clim.rean,1,n.yrs.hind),1,1),1,1) # repeat the same values <n.years> times to be able to calculate the anomalies #[1,1,20,4,320,640]
          anom.rean <- data.rean$mod - clim.rean 
          #anom.rean<-InsertDim(anom.rean[1,1,,,,],1,1) # [1,20,4,320,640]

          rm(data.rean, clim.rean)
          gc()
          
	  #load(paste0(workdir,'/Data_',var.name,'/anom_rean_startdate',startdate,'.RData'))
	  #anom.rean <- drop(anom.rean)
	  anom.rean.chunk[my.time.interv,,,] <- anom.rean
          
          # Load hindcast data:
          print(paste0('Computing forecast anomalies for startdate ', which(startdate==my.startdates),'/',length(my.startdates)))
        
          # Load weekly var subseasonal data in: /esnas/exp/ECMWF/monthly/ensforhc/weekly_mean and interpolate them to the same ERA-Interim resolution (512x256)
          data.hindcast <- Load(var = var.name, exp = list(list(path=forecast.dir)),
                              obs = NULL, sdates = paste0(yr1.hind:yr2.hind, substr(sdates.seq[startdate],5,6), substr(sdates.seq[startdate],7,8)),
                              nleadtime = n.leadtimes, leadtimemin = 1, leadtimemax = n.leadtimes, output = 'lonlat',  grid=my.grid, method='bilinear', nprocs=1) 
        
          # dim(data.hindcast$mod) # [1,4,20,4,320,640]

          # Mean of the 4 members of the Hindcasts and of the 20 years (for each point and leadtime);
          clim.hindcast <- apply(data.hindcast$mod,c(1,4,5,6),mean)[1,,,] # average of all years and members for each leadtime and pixel
          clim.hindcast <- InsertDim(InsertDim(InsertDim(clim.hindcast,1,n.yrs.hind),1,n.members),1,1)  # repeat the same values <n.members> and <n.yrs.hind> times to calc. anomalies
          anom.hindcast <- data.hindcast$mod - clim.hindcast
        
          rm(data.hindcast,clim.hindcast)
          gc()

          # average the anomalies over the 4 hindcast members to compute the ensemble mean [1,20,4,320,640]
          anom.hindcast.mean<-apply(anom.hindcast,c(1,3,4,5,6),mean)

          gc()
              
	  #load(paste0(workdir,'/Data_',var.name,'/anom_hindcast_startdate',startdate,'.RData')) 
	  anom.hindcast <- drop(anom.hindcast)
	  anom.hindcast.chunk[,my.time.interv,,,] <- anom.hindcast
          
	  rm(anom.hindcast, anom.rean)
          gc()
                   
	  anom.hindcast.mean <- drop(anom.hindcast.mean)
	  anom.hindcast.mean.chunk[my.time.interv,,,] <- anom.hindcast.mean
              
	  rm(anom.hindcast.mean)
          gc()	  

    } # end load data and conversion to anomalies
    
        
    if(any(my.score.name=="FairRpss")){            
	    #my.FairRpss.sub <- veriApply("FairRpss", fcst=anom.hindcast.sub, obs=anom.rean.sub, prob=my.prob, tdim=2, ensdim=1, parallel=FALSE, ncpus=n.cpus)
            my.FairRps.chunk <- veriApply("FairRps", fcst=anom.hindcast.chunk, obs=anom.rean.chunk, prob=my.prob, tdim=2, ensdim=1, parallel=FALSE, ncpus=1)
            #my.FairRps.chunk.double <- veriApply("FairRps", fcst=anom.hindcast.chunk.double, obs=anom.rean.chunk.double, prob=my.prob, tdim=2, ensdim=1, parallel=FALSE, ncpus=n.cpus)
            my.FairRps.chunk.sum <- apply(my.FairRps.chunk, c(2,3,4), sum)

        
            # the prob.for the climatological ensemble can be set without using convert2prob and they are the same for all points and leadtimes (33% for each tercile):
            ens.ref <- array(33,c(n.startdates*n.yrs.hind,3)) 

            # the observed probabilities vary from point to point and for each leadtime:  
            obs <- apply(anom.rean.chunk, c(2,3,4), function(x){convert2prob(x, prob <- my.prob)})

            # reshape obs to be able to apply EnsRps(ens, obs) below:  
            obs2 <- InsertDim(obs,1,3)
            obs2[2,1:(n.startdates*n.yrs.hind),,,] <- obs2[1,(n.startdates*n.yrs.hind + 1:(n.startdates*n.yrs.hind)),,,]
            obs2[3,1:(n.startdates*n.yrs.hind),,,] <- obs2[1,(2*n.startdates*n.yrs.hind + 1:(n.startdates*n.yrs.hind)),,,]
            obs3 <- obs2[,1:(n.startdates*n.yrs.hind),,,,drop=F] # drop=F is used not to loose the dimension(s) with only 1 element
           
            my.Rps.clim.chunk <- apply(obs3,c(3,4,5), function(x){ EnsRps(ens.ref,t(x)) })
            my.Rps.clim.chunk.sum <- apply(my.Rps.clim.chunk, c(2,3,4), sum)
                         
	    my.FairRpss.chunk <- 1-(my.FairRps.chunk.sum / my.Rps.clim.chunk.sum)
              
	    rm(ens.ref, obs, obs2, obs3, my.FairRps.chunk.sum, my.Rps.clim.chunk.sum)
            gc()
              
            # bootstrapping:                
            for(b in 1:n.boot){
                cat(paste0("Bootstrapping FairRpss: resampling n. ",b,"/",n.boot,"\n"))
                yrs.sampled <- sample(1:n.yrs, replace=TRUE)

                #for(y in 1:n.yrs) anom.rean.chunk.sampled[y,,,] <- anom.rean.chunk[yrs.sampled[y],,,]
                #for(y in 1:n.yrs) anom.hindcast.chunk.sampled[,y,,,] <- anom.hindcast.chunk[,yrs.sampled[y],,,]

                my.FairRps.sampled <- my.FairRps.chunk              
                my.Rps.clim.sampled <- my.Rps.clim.chunk
                
                for(y in 1:n.yrs) my.FairRps.sampled[y,,,] <- my.FairRps.chunk[yrs.sampled[y],,,]
                for(y in 1:n.yrs) my.Rps.clim.sampled[y,,,] <- my.Rps.clim.chunk[yrs.sampled[y],,,]
           
                #my.FairRps.chunk <- veriApply("FairRps", fcst=anom.hindcast.chunk.sampled, obs=anom.rean.chunk.sampled, prob=my.prob, tdim=2, ensdim=1, parallel=FALSE, ncpus=n.cpus)
                #ens <- array(33,c(n.startdates*n.yrs.hind,3)) 
                #obs <- apply(anom.rean.chunk.sampled, c(2,3,4), function(x){convert2prob(x, prob <- my.prob)})
                #obs2 <- InsertDim(obs,1,3)
                #obs2[2,1:(n.startdates*n.yrs.hind),,,] <- obs2[1,(n.startdates*n.yrs.hind + 1:(n.startdates*n.yrs.hind)),,,]
                #obs2[3,1:(n.startdates*n.yrs.hind),,,] <- obs2[1,(2*n.startdates*n.yrs.hind + 1:(n.startdates*n.yrs.hind)),,,]
                #obs3 <- obs2[,1:(n.startdates*n.yrs.hind),,,,drop=F]
                #my.Rps.clim.chunk <- apply(obs3,c(3,4,5), function(x){ EnsRps(ens,t(x)) })
                
                my.FairRps.sampled.sum <- apply(my.FairRps.sampled,c(2,3,4), sum)
                my.Rps.clim.sampled.sum <- apply(my.Rps.clim.sampled,c(2,3,4), sum)                
                my.FairRpss.sampled <- 1-(my.FairRps.sampled.sum/my.Rps.clim.sampled.sum)

                my.FairRpssBoot[b,,,] <- my.FairRpss.sampled

                gc()
            }

            rm(my.FairRps.sampled, my.Rps.clim.sampled, my.FairRps.sampled.sum,  my.Rps.clim.sampled.sum,  my.FairRpss.sampled, yrs.sampled)
            gc()

            my.FairRpssBootMean <- apply(my.FairRpssBoot,c(2,3,4),mean,na.rm=T)

            my.test.value <- 0 # my.FairRpss.chunk   # test if the bootstrapped mean value is significantly different from this value or not

            my.FairRpss.diff1 <- my.FairRpssBootMean + abs(my.test.value - my.FairRpssBootMean)
            my.FairRpss.diff2 <- my.FairRpssBootMean - abs(my.test.value - my.FairRpssBootMean)
           
            for(i in 1:dim(my.FairRpssBoot)[2]){
                for(j in 1:dim(my.FairRpssBoot)[3]){
                    for(k in 1:dim(my.FairRpssBoot)[4]){
                        extr1 <- which(my.FairRpssBoot[,i,j,k] > my.FairRpss.diff1[i,j,k])
                        extr2 <- which(my.FairRpssBoot[,i,j,k] < my.FairRpss.diff2[i,j,k])
                        if(my.test.value != 0) my.FairRpss.pvalue[i,j,k] <- (length(extr1) + length(extr2)) / n.boot  # two-tailed test
                        if(my.test.value == 0) my.FairRpss.pvalue[i,j,k] <- ifelse(my.FairRpssBootMean[i,j,k] > 0, length(extr2)/n.boot, length(extr1)/n.boot)

                    }
                }
            }

            rm(my.FairRpssBootMean, my.FairRpss.diff1, my.FairRpss.diff2, extr1, extr2)
            gc()
        
            #which(my.FairRpss.pvalue > 0.05) # points where the FairRPSS is significant (i.e: whose FairRPSS is not significantly different from the population mean)

            # calculate the percentiles of the skill score in case you want to compute the confidence interval of the 95%:
            #my.FairRpssConf[,,,chunk$int[[c]]] <- apply(my.FairRpssBoot[,,,chunk$int[[c]]], c(2,3,4), function(x) quantile(x, probs=conf.level, type=8))
                    
       } # close if on FairRpss

       if(any(my.score.name=="FairCrpss")){

            #my.FairCrpss.chunk <- veriApply("FairCrpss", fcst=anom.hindcast.chunk, obs=anom.rean.chunk, tdim=2, ensdim=1, parallel=FALSE, ncpus=1)

            #Numerador:
	    my.FairCrps.chunk <- veriApply("FairCrps", fcst=anom.hindcast.chunk, obs=anom.rean.chunk, tdim=2, ensdim=1, parallel=FALSE, ncpus=1)
            my.FairCrps.chunk.sum <- apply(my.FairCrps.chunk,c(2,3,4), sum)

            # Denominador:
            my.Crps.clim.chunk <- array(NA, c(n.startdates*n.yrs.hind, n.leadtimes, n.lat, n.lon))
            anom.hind.clim <- array(NA, c(n.members, n.startdates*n.yrs.hind, n.leadtimes, n.lat, n.lon))
           
            for(i in 1:n.leadtimes){
                print(paste0("leadtime= ",i))
                for(j in 1:n.lat){
                    for(k in 1:n.lon){
                        # extract a random sample with no replacement of the hindcast:    
                        #anom.hind.clim[,,i,j,k] <- array(sample(anom.hindcast.chunk[,,i,j,k],n.startdates*n.yrs.hind*n.members),c(n.startdates*n.yrs.hind,n.members))
                        anom.rean.clim[,,i,j,k] <- t(array(anom.rean.chunk[,i,j,k],c(n.startdates*n.yrs.hind, n.startdates*n.yrs.hind)))
                        my.Crps.clim.chunk[,i,j,k] <- EnsCrps((anom.rean.clim[,,i,j,k]), anom.rean.chunk[,i,j,k])
                    }
                }
            }

            ### anom.hind.clim <- apply(anom.hindcast.chunk,c(3,4,5), sample)
            ### anom.all.chunk <- abind(anom.hind.clim,anom.rean.chunk, along=1) # merge exp and obs together to use apply:
            ### my.Crps.clim.chunk <- apply(anom.all.chunk, c(3,4,5), function(x){ EnsCrps(t(x[1:n.members,]), x[n.members+1,])})

            my.Crps.clim.chunk.sum <- apply(my.Crps.clim.chunk,c(2,3,4), sum)

            ## Skill Score:
            my.FairCrpss.chunk <- 1 - (my.FairCrps.chunk.sum / my.Crps.clim.chunk.sum)

            #rm(anom.all.chunk, my.FairCrps.chunk.sum, my.Crps.clim.chunk.sum)

            rm(my.FairCrps.chunk.sum,  my.Crps.clim.chunk.sum)
            gc()
            
            ### bootstrapping:              
            for(b in 1:n.boot){
                cat(paste0("Bootstrapping FairCrpss: resampling n. ",b,"/",n.boot,"\n"))
 
                yrs.sampled <- sample(1:n.yrs, replace=TRUE)
                
                my.FairCrps.sampled <- my.FairCrps.chunk                
                my.Crps.clim.sampled <- my.Crps.clim.chunk
                
                for(y in 1:n.yrs) my.FairCrps.sampled[y,,,] <- my.FairCrps.chunk[yrs.sampled[y],,,]
                for(y in 1:n.yrs) my.Crps.clim.sampled[y,,,] <- my.Crps.clim.chunk[yrs.sampled[y],,,]

                my.FairCrps.sampled.sum <- apply(my.FairCrps.sampled,c(2,3,4), sum)
                my.Crps.clim.sampled.sum <- apply(my.Crps.clim.sampled,c(2,3,4), sum)
                
                my.FairCrpss.sampled <- 1-(my.FairCrps.sampled.sum / my.Crps.clim.sampled.sum)

                my.FairCrpssBoot[b,,,] <- my.FairCrpss.sampled
                
                gc()
             }

             rm(my.FairCrps.sampled,  my.Crps.clim.sampled, my.FairCrpss.sampled , yrs.sampled)
             gc()
           
             my.FairCrpssBootMean <- apply(my.FairCrpssBoot,c(2,3,4),mean,na.rm=T)

             my.test.value <- 0 # my.FairCrpss.chunk   # test if the bootstrapped mean value is significantly different from this value or not
            
             my.FairCrpss.diff1 <- my.FairCrpssBootMean + abs(my.test.value - my.FairCrpssBootMean)
             my.FairCrpss.diff2 <- my.FairCrpssBootMean - abs(my.test.value - my.FairCrpssBootMean)
           
             for(i in 1:dim(my.FairCrpssBoot)[2]){
                 for(j in 1:dim(my.FairCrpssBoot)[3]){
                     for(k in 1:dim(my.FairCrpssBoot)[4]){
                         extr1 <- which(my.FairCrpssBoot[,i,j,k] > my.FairCrpss.diff1[i,j,k])
                         extr2 <- which(my.FairCrpssBoot[,i,j,k] < my.FairCrpss.diff2[i,j,k])
                         if(my.test.value != 0) my.FairCrpss.pvalue[i,j,k] <- (length(extr1) + length(extr2)) / n.boot
                         if(my.test.value == 0) my.FairCrpss.pvalue[i,j,k] <- ifelse(my.FairCrpssBootMean[i,j,k] > 0, length(extr2)/n.boot, length(extr1)/n.boot)

                     }
                 }
             }

             rm(my.FairCrpssBootMean,  my.FairCrpss.diff1, my.FairCrpss.diff2,  extr1,  extr2)
             ###which(my.FairCrpss.pvalue > 0.05) # points where thw FairCRPSS is significant (i.e: whose FairRPSS  not significantly different from the population mean)
           
             # calculate the percentiles of the skill score if you want to measure hte confidence interval:
             #my.FairCrpssConf[,,,chunk$int[[c]]] <- apply(my.FairCrpssBoot[,,,chunk$int[[c]]], c(2,3,4), function(x) quantile(x, probs=conf.level, type=8)) 
        
        } # close if on FairCrpss

        if(any(my.score.name=="EnsCorr")){
	  # ensemble mean correlation for the selected startdates (first dim, 'week') and all leadtimes (second dim):
	  my.EnsCorr.chunk <- my.PValue.chunk <- array(NA, c(n.leadtimes, n.lat, n.lon))

	  for(i in 1:n.leadtimes){
	    for(j in 1:n.lat){
	      for(k in 1:n.lon){                      
		my.EnsCorr.chunk[i,j,k] <- cor(anom.hindcast.mean.chunk[,i,j,k],anom.rean.chunk[,i,j,k], use="complete.obs")
	        # option 'na.method' is chosen from the following list: c("all.obs", "complete.obs", "pairwise.complete.obs", "everything", "na.or.complete")
               
                my.EnsCorr.pvalue[i,j,k] <- cor.test(anom.hindcast.mean.chunk[,i,j,k],anom.rean.chunk[,i,j,k], use="complete.obs")$p.value
                
	      }
	    }
	  }

	  #my.EnsCorr[,,chunk$int[[c]]] <- my.EnsCorr.chunk
	  #my.PValue[,,chunk$int[[c]]] <- my.PValue.chunk

	  #rm(anom.hindcast.chunk,anom.hindcast.mean.chunk,my.EnsCorr.chunk,my.PValue.chunk)
        }

        #rm(anom.rean.chunk)
        gc()
        #mem()

        # the Reliability diagram cannot be computed splitting the data in chunk, so it is computed only if the script is run from the R interface or with no arguments:

        if(any(my.score.name=="RelDiagr")){ # in this case the computation cannot be split in groups of grid points, but can be split for leadtime instead:   
            my.RelDiagr<-list()
            
            for(lead in 1:n.leadtimes){
                cat(paste0('leadtime n.: ',lead,'/',n.leadtimes,'\n'))

                anom.hindcast.lead <- anom.hindcast.chunk[,,lead,,] # extract only the lead time we are interested in
                anom.hindcast.chunk.perm <- aperm(anom.hindcast.lead,c(2,1,3,4))  # swap the first two dimensions to put the years as first dimension (needed for convert2prob)
                gc()

                cat('Computing the Reliability Diagram. Please wait... \n')
                ens.chunk<-apply(anom.hindcast.chunk.perm, c(3,4), convert2prob, prob = my.prob) # its dimens. are: [n.startdates*n.yrs.hind*n.forecast.categories, n.lat, n.lon]
                anom.rean.lead <- anom.rean.chunk[,lead,,]
                
                cat('Computing the Reliability Diagram. Please wait...... \n')

                obs.chunk<-apply(anom.rean.lead,c(2,3), convert2prob, prob = my.prob) # the first n.members*n.yrs.hind elements belong to the first tercile, and so on.
                ens.chunk.prob<-apply(ens.chunk, c(2,3), function(x) count2prob(matrix(x, n.startdates*n.yrs.hind, n.categ), type=4)) # type=4 is the only method with sum(prob)=1 !!
                cat('Computing the Reliability Diagram. Please wait......... \n')

                obs.chunk.prob<-apply(obs.chunk, c(2,3), function(x) count2prob(matrix(x, n.startdates*n.yrs.hind, n.categ), type=4)) 

                if(is.null(int.lat)) int.lat <- 1:n.lat  # if there is no region selected, select all the world
                if(is.null(int.lon)) int.lon <- 1:n.lon

                cat('Computing the Reliability Diagram. Please wait............ \n')

                int1 <- 1:(n.startdates*n.yrs.hind)  # time interval of the data of the first tercile
                my.RelDiagr[[lead]] <- ReliabilityDiagram(ens.chunk.prob[int1,int.lat,int.lon], obs.chunk.prob[int1,int.lat,int.lon], bins=5, nboot=0, plot=FALSE, plot.refin=F) #tercile 1

                int2 <- (1+(n.startdates*n.yrs.hind)):(2*n.startdates*n.yrs.hind)  # time interval of the data of the second tercile
                my.RelDiagr[[n.leadtimes+lead]] <- ReliabilityDiagram(ens.chunk.prob[int2,int.lat,int.lon], obs.chunk.prob[int2,int.lat,int.lon], bins=5, nboot=0, plot=FALSE, plot.refin=F)

                int3 <- (1+(2*n.startdates*n.yrs.hind)):(3*n.startdates*n.yrs.hind)  # time interval of the data of the third tercile
                my.RelDiagr[[2*n.leadtimes+lead]] <- ReliabilityDiagram(ens.chunk.prob[int3,int.lat,int.lon], obs.chunk.prob[int3,int.lat,int.lon], bins=5, nboot=0, plot=FALSE, plot.refin=F)

                #print(my.RelDiagr) # for debugging          
                rm(ens.chunk, obs.chunk, ens.chunk.prob, obs.chunk.prob, int1, int2, int3)
                gc()

            } # next lead
            
        } # close if on RelDiagr

        print("Saving results...")

        # save the results for the chunk:
        save(my.FairRpss.chunk, my.FairRpss.pvalue, my.FairCrpss.chunk, my.FairCrpss.pvalue, my.EnsCorr.chunk, my.EnsCorr.pvalue, my.RelDiagr, my.FairRpssBoot, my.FairCrpssBoot, work.dir, rean.dir, rean.name, forecast.year, forecast.dir, cfs.name, var.name, var.name.map, mes, day, yr1.hind, yr2.hind, leadtime.week, n.members, my.prob, int.lat, int.lon, n.boot, n.categ, sdates.seq, n.leadtimes, n.yrs.hind, lons, lats, n.lon, n.lat, my.grid, veri.month, my.startdates, startdate.name, n.startdates, n.yrs, file=paste0(work.dir,'/',var.name,'_',my.month[month],'.RData'))
    
        #if(any(my.score.name=="FairRpss")) save(my.FairRpss.chunk, file=paste0(work.dir,'/',var.name,'/FairRpss_',startdate.name,'.RData'))
    
        print("Saving results...done")

} # next m (month)


print("Finished!")
