rm(list=ls())

library(s2dverification)
library(multiApply)
library(abind)
library(future) # for availableCores() function
source('/esarchive/scratch/ncortesi/scripts/Rfunctions.R')
source('/esarchive/scratch/ncortesi/scripts/Vero/VeroRegimes.R')

#######################################################################################################
##                                                                                                   ##
##   Validation of WR forecasts at subseasonal scale with the assign method (final methodology)      ##
##                                                                                                   ##
#######################################################################################################

## important: even if Load() function is able to interpolate grid, when loading 1 data chunk, Load() loads and interpolates the whole grid instead than
## only 1 chunk, so it is up to 100 times slower than what should be. To solve this issue, an interpolation function by chunks is being developed.
## In meantime, a temporary solution is to load hindcasts from a new dataset
## with the interpolated values of the hindcast dataset to the target grid (i.e: ERA-Interim reanalysis) with the below commands exectuted from terminal:
##
## gridTemplate=/esarchive/recon/ecmwf/erainterim/6hourly/psl/psl_197901.nc
## input=/esarchive/exp/ecmwf/monthly_ensforhc/6hourly/psl/2014010200/psl_1994010200.nc
## output=/esarchive/scratch/ncortesi/Subestacional/hindcasts_data/6hourly/psl/psl_1994010200.nc
## cdo -f nc -remapbil,$gridTemplate $input $output
##
## (the generic function is: cdo -f nc -regriddingMethod,gridTemplate.nc input.nc output.nc)
##
## you can interpolate the whole hindcast dataset faster with a loop replacing last line with:
##
## for week in 0102 0109 0116 0123 0130 0206 0213 0220 0227 0306 0313 0320 0327 0403 0410 0417 0424 0501 0508 0515 0522 0529 0605 0612 0619 0626 0703 0710 0717 0724 0731 0807 0814 0821 0828 0904 0911 0918 0925 1002 1009 1016 1023 1030 1106 1113 1120 1127 1204 1211 1218 1225; do for year in {1994..2013}; do cdo -f nc -remapbil,$gridTemplate /esarchive/exp/ecmwf/monthly_ensforhc/6hourly/psl/2014${week}00/psl_${year}${week}00.nc /esarchive/scratch/ncortesi/Subestacional/hindcasts_data/6hourly/psl/psl_${year}${week}00.nc; done; done
##
## or with the line below for the forecasts of 2018:
## (in this case, before interpolating you have to remove variable "realization" or remapbil won't function as this variable is seen by CDO as a grid!!!)
## (moreover, file size is so big that it's better to interpolate in parallel many files at the same time with the script interp9.sh as shown below)
##
## for week in 0104 0111 0118 0125 0201 0208 0215 0222 0301 0308 0315 0322 0329 0405 0412 0419 0426 0503 0510 0517 0524 0531 0607 0614 0621 0628 0705 0712 0719 0726 0802 0809 0816 0823 0830 0906 0913 0920 0927 1004 1011 1018 1025 1101 1108 1115 1122 1129 1206 1213 1220 1227; do for year in {1998..2017}; do rm -f chunk$year$week.*; sbatch -J int$year$week --output=chunk$year$week.out --error=chunk$year$week.err --export=week=$week,year=$year /esarchive/scratch/ncortesi/scripts/interp9.sh; done; done
##
## once the new dataset is ready, you'll always load this one instead of the original one.

## folder with the input weekly clustering (one .RData for each week of the year):
work.dir <- "/esarchive/scratch/ncortesi/Regimes/87_WRs_subseasonal/weekly_1979-2017_starting_from_first_of_January_of_2014_ERA-Interim"
##work.dir <- "/esarchive/scratch/ncortesi/Regimes/87_WRs_subseasonal/weekly_1979-2017_starting_from_first_of_January_of_2018_ERA-Interim"

## 6-hourly data (not used unless you have no daily means data available on /esarchive and you have to average them by yourself):
##forecast.dir <- paste0('/esarchive/exp/ecmwf/monthly_ensforhc/6hourly/$VAR_NAME$/', forecast.year,'$MONTH$$DAY$/$VAR_NAME$_$START_DATE$.nc')  ## forecast year: 2016
##forecast.dir <- paste0('/esarchive/exp/ecmwf/monthly_ensforhc/6hourly/$VAR_NAME$/', forecast.year,'$MONTH$$DAY$00/$VAR_NAME$_$START_DATE$00.nc') ## forecast year: 2014
##forecast.dir.interp <- paste0('/esarchive/scratch/ncortesi/Subestacional/hindcasts_data/6hourly/$VAR_NAME$/$VAR_NAME$_$START_DATE$00.nc') ## forecast year: 2014
##forecast.dir.interp <- paste0('/gpfs/projects/bsc32/bsc32842/6hourly/$VAR_NAME$/$VAR_NAME$_$START_DATE$00.nc') ## forecast year: 2014
##forecast.dir.interp <- paste0('/nvme1/ncortesi/$VAR_NAME$/$VAR_NAME$_$START_DATE$00.nc') ## forecast year: 2014
##forecast.dir.interp <- paste0('/scratch/ncortesi/$VAR_NAME$/$VAR_NAME$_$START_DATE$00.nc') ## forecast year: 2014

## daily data:
##forecast.dir <- paste0('/esarchive/exp/ecmwf/monthly_ensforhc/daily_mean/$VAR_NAME$_f6h/', forecast.year,'$MONTH$$DAY$00/$VAR_NAME$_$START_DATE$.nc') ## forecast year: 2014
##forecast.dir.interp <- paste0('/esarchive/scratch/ncortesi/Subestacional/hindcasts_data/daily_mean/psl_f6h/$VAR_NAME$_$START_DATE$.nc') ## forecast year: 2014
Rdata.dir <- "/gpfs/projects/bsc32/bsc32842/daily/forecasts_2014"
##Rdata.dir <- "/gpfs/projects/bsc32/bsc32842/daily/forecasts_2018"

yr1.obs <- 1979 # first year of psl data from the reanalysis
yr2.obs <- 2017 # last year of psl data from the reanalysis 
    
forecast.year <- 2014 #2018 #2014              # starting year of the weekly sequence of the forecasts
mes <- 1                                 # starting forecast month (1: january, 2: february, etc.)
day <- 2 #4 #2                              # starting forecast day (usually the first Thursday of January of the forecast year,
                                         # which in case of forecasts of 2014 is the 2 of January and in case of the forecasts of 2016 is the 7 of January)
                                         # (for 2014 forecasts, also Tuesday forecasts were downloaded. They start on the 7th of January)
yr1.hind <- 1994 #1998 ## 1994                 # first hindcast year
yr2.hind <- 2013 #2016 ## 2017 ## 2013         # last hindcast year (usually the forecast year -1)

n.members <- 4 #11 # 4

n.clusters <- 4

## if TRUE, reorder the WR numbers to use the same order of the observed weekly classification created by us:
reordering <- TRUE

##################################################################################################################################
## Load Subseasonal SLP hindcast data for one start date and lead time (1 week), interpolate it to the same grid of the reanalysis 
## and finally load SLP monthly reanalysis data to assign to each day an observed WR

n.yrs.hind <- yr2.hind - yr1.hind + 1
n.yrs.obs <- yr2.obs - yr1.obs + 1

sdates.seq <- weekly.seq(forecast.year, mes, day)  # load the sequence of dates corresponding to all the thursday of the year (for forecast data)
pos.bis <- which(sdates.seq == paste0(forecast.year, "0229")) # find if there is a startdate at the 29th of february and return its position in the vector sdates.seq
if(length(pos.bis) != 0) sdates.seq <- sdates.seq[-pos.bis] # if there is a start date that starts exactly on the 29th of February, remove it
n.sdates <- length(sdates.seq) ## number of start dates

## chunk is a number from 1 to the number of start dates sdates (52)
if(length(commandArgs(TRUE)) != 0) chunk <- as.integer(commandArgs(TRUE)) else stop("You need to provide an external parameter with the chunk number")

## detect lon and lat of reanalysis to be able to interpolate the forecasts to the reanalysis grid (forecasts of ECMWF are 320x640 or higher, reanalysis (ERA-Int) is 256x512):
test <- Load('psl', exp = NULL, 
                 obs = 'erainterim',
                 sdates = '20010101',
                 storefreq='daily',
                 leadtimemin = 1, leadtimemax = 1,
                 latmin = 27, latmax = 81, lonmin = 274.5, lonmax = 45,
                 output = 'lonlat', nproc=1) ## load 1 day only

## lon.rean <- data.grid$lon
## lat.rean <- data.grid$lat
## n.lon.rean <- length(lon.rean) 
## n.lat.rean <- length(lat.rean)
## grid.rean <- paste0('r', n.lon.rean,'x',n.lat.rean) ## [256 x 512 in case of era-interim]

## note that sdate is the hindcast start date, not the observed start date of the reference WR classification that starts on the 1st of January, so they differ!!!
print(paste0("Loading Start Date num. ", chunk, "/", n.sdates))

sdates <- paste0(yr1.hind:yr2.hind, substr(sdates.seq[chunk],5,6), substr(sdates.seq[chunk],7,8)) ## same start date but for all hindcast years

## load the anomalies of the chunk if they were already saved as .RData; if not, compute them and save them:
if(file.exists(paste0(Rdata.dir,"/anom_for_chunk_", chunk, ".RData"))){
    load(paste0(Rdata.dir,"/anom_for_chunk_", chunk, ".RData"))
} else {
    
    ## load forecasts for all the years of start date 'chunk' and ALL FORECAST DAYS associated to it (32) already interpolated to the same resolution of the reanalysis:
    ## note that the first forecast day is not equal to startdate+1 but to startdate!!!    
    system.time(data <- Load('psl', exp = list(list(path=forecast.dir.interp)),
                             obs = NULL,
                             sdates = sdates,
                             storefreq='daily',
                             leadtimemin = 1, leadtimemax = 32, ##129, ## replace 129 with 32 when you'll load daily data directly
                             latmin = 27, latmax = 81,
                             lonmin = 274.5, lonmax = 45, output = 'lonlat', nprocs=1)) ##max(1, availableCores()))) ## it's faster with more cores, even without interpolation
    ##grid=grid.rean, method='bilinear', nprocs=max(1, availableCores())))

    ## to load 1 chunk only (1 latitude value only from 81 to 27 degrees north, equivalent to chunks number 14 to 90 (on ERA-Interim grid)):
    ## data <- Load('psl', exp = list(list(path=forecast.dir.interp)),
    ##                           obs = NULL,
    ##                           sdates = sdates,
    ##                           storefreq='daily',
    ##                           leadtimemin = 1, leadtimemax = 129,
    ##                           latmin = lat.rean[chunk], latmax = lat.rean[chunk],
    ##                           output = 'lonlat', nprocs=1)

    ##n.members <- dim(data$mod)[2]
    n.lat <-  dim(data$mod)[5]
    n.lon <-  dim(data$mod)[6]

    sixhourly <- FALSE
    if(sixhourly){
        ## In case you are loading 6-hourly data, you have to convert them to daily means before:
        data.daily <- array(NA, c(1, n.members, n.yrs.hind, 32, n.lat, n.lon))       ## [dataset x members x sdate x time x lat x lon]
        my.names <- names(dim(data$mod))

        ##for(dd in 1:32) { print(paste0("Day ", dd, " of 32")); data.daily[1,1:n.members,1:n.yrs.hind, dd, 1:n.lat,1:n.lon] <-  apply(data$mod, c(1,2,3,5,6), function(x){mean(x[(dd-1)*4 + 1:4])})}

        ## above loop is twice faster with .Internal(mean) used below, but there must be no NA values:
        for(dd in 1:32) { print(paste0("Day ", dd, " of 32")); data.daily[1, 1:n.members, 1:n.yrs.hind, dd, 1:n.lat, 1:n.lon] <-  apply(data$mod, c(1,2,3,5,6), function(x){.Internal(mean(x[(dd-1)*4 + 1:4]))})}

        data$mod <- data.daily
        names(dim(data$mod)) <- my.names ## re-assign the dimension names because they were deleted by the previous command

        rm(data.daily)
    }

    ## remove the eventual forecasts of the 29th of February: (they are present for the forecasts of 2016, but not of 2014 or 2018):
    ## if(leap.year(forecast.year)) data.daily <- data.daily[1,,,-,,] ##sistema!!!

    ## measure forecast climatology for the 32 days loaded:
    clim <- Clim(var_exp = data$mod, var_obs = data$mod, memb=F)

    ## apply the LOESS filter to the 32-days climatology:
    clim_smoothed_for <- aperm(apply(clim$clim_exp, c(1,3,4), Loess, loess_span=1), c(2,1,3,4))

    ## measure forecast anomalies:
    anom_for <- Ano(data$mod, clim_smoothed_for)

    ## convert forecast anomalies from Pascal to hPa;
    anom_for <- anom_for / 100

    ## save SLP anomalies. Subsequent calls will read this file instead of loading data fron esarchive or gpfs:
    ##dir.create("/nvme1/ncortesi")
    save(anom_for, file=paste0(Rdata.dir,"/anom_for_chunk_", chunk, ".RData"))

} ## close if

## in case of the forecasts of 2018 we have to remove tha SLP hindcast data for year 2017 which was nor removed in the .RData:
if(forecast.year == 2018) anom_for <- anom_for[,,1:19,,,, drop=FALSE] 
        
## For each forecast day of the start date 'chunk', identify to which week of the year (starting from the 1st of January) it belongs:
print("Loading finished")

## position of the 32 forecast days on the SAME MONTH of the hindcast start date
forecast.daily.seq <- as.numeric(substr(sdates.seq[chunk],7,8)) + 1:32 - 1 

## month of the currently selected start date:
month.sdate <- as.numeric(substr(sdates.seq[chunk],5,6))

## get the total number of days BEFORE the month of the start date (for a no leap year):
if(month.sdate > 1) n.days.previous <- n.days.in.a.monthly.period(1, month.sdate - 1, 2001) else n.days.previous <- 0

## get the sequence of forecast days counting from the START of the YEAR instead than from the start of the month of the start date:
days.seq <- n.days.previous + forecast.daily.seq

## get the sequence of monthly numbers corresponding to the sequence of forecast days:
monthly.seq <- seq.months.in.a.year(2001)[days.seq]

## if the forecast days extend to the following year, assign them to January:
if(length(which(is.na(monthly.seq))) > 0) monthly.seq[which(is.na(monthly.seq))] <- 1 

## sequence of weekly numbers STARTING FROM THE FIRST Of JANUARY, each one corresponding to one of the 32 forecast days:
weekly.seq <- seq.weeks.in.a.year(2001)[days.seq]

##The last week of the year is the number 53 and it is made up by 1 day ony, so it isn't used in the WR classification and it is included in week 52:
weekly.seq[which(weekly.seq == 53)] <- 52

## if the forecast days extend to the following year, assign them to January's weeks:
if(length(which(is.na(weekly.seq))) > 0) weekly.seq[which(is.na(weekly.seq))] <- rep(1:5, each=7)[1:length(which(is.na(weekly.seq)))]

## check all the sequences (for the debug):
##cbind(forecast.days=1:length(days.seq), days.seq, monthly.seq, weekly.seq) 

## identify which month(s) to load:
months2load <- unique(monthly.seq)

## identify which week(s) to load:
weeks2load <- unique(weekly.seq)

## array which will store the cluster sequence of the predicted WRs for all the 32 forecast days of the selected sdate:
WR.seq.for <- array(NA, c(1, n.members, n.yrs.hind, 32))

## array which will store the cluster sequence of the observed WRs for all the 32 forecast days of the selected sdate:
WR.seq.rean <- array(NA, c(n.yrs.hind, 32)) 

## vector which will store the % of coincident WRs for all the 32 forecast days of the selected sdate:
WR.seq.coinc <- WR.seq.coinc.perc <- array(0, 32)

## other arrays for the validation:
WR.sp.cor <- array(NA, c(n.clusters, 32))

## remember that weeks start from the 1st of January of a NO LEAP year!
print("Loading weekly WR classification. Please wait...")
for(week in weeks2load){
    ##print(paste0("Week number ", week))

    ## extract only forecast anomalies of days belonging to that week:
    pos.week <- which(weekly.seq == week) ## pos.week is the position of the numbers of the forecast days which belongs to the selected week
    anom_for_same_week <- anom_for[,,, pos.week,,,drop=FALSE] ## 'anom_for' are the forecasted anomalies for the 32 forecast days 
    
    ## load 'WR_obs' variable with the cluster data of the same month of the forecast day considered:
    ## WR_obs$composite is a 3D array with the spatial patterns of the 4 WRs and WR_obs$cluster is a vector with obs daily sequence for all years of the loaded period
    ## WR_obs data is only valid for the central week of the 5-week mobile window used.
    load(paste0(work.dir, "/observed_regimes_week_", week,".RData"))
    
    ## measure distance between the SLP anomalies of each forecast day and those of the obs. WRs of the reference month(s), and assign to each forecast day the closer WR:
    names(dim(WR_obs$composite)) <- c("lat", "lon", "nclust") ## add names to the dimensions as they are needed by RegimeAssign() function below

    ## Unfortunately function RegimesAssign() doesn't work in chunks because it depends on Apply() which automatically tries to employ all cores of the Power9 node.
    ## For this reason, I had to replace Apply() with apply() below:
    ## WR.seq.for[,,, pos.week] <- RegimesAssign(var_ano = anom_for_same_week, ref_maps = WR_obs$composite, lats = test$lat, method = 'distance')$cluster
    posdim <- which(names(dim(WR_obs$composite)) == 'nclust')
    poslat <- which(names(dim(WR_obs$composite)) == 'lat')
    poslon <- which(names(dim(WR_obs$composite)) == 'lon')
    poslat_ano <- which(names(dim(anom_for_same_week)) == 'lat')
    poslon_ano <- which(names(dim(anom_for_same_week)) == 'lon')
    nclust <- dim(WR_obs$composite)[posdim]
    nlat <- dim(WR_obs$composite)[poslat]
    nlon <- dim(WR_obs$composite)[poslon]
    
    ## WR.seq.for[,,, pos.week] <- apply(X = anom_for_same_week, MARGIN = c((1:length(dim(anom_for_same_week)))[-c(poslat_ano, poslon_ano)]),
    ##                                  FUN = 'anom2regime', ref = WR_obs$composite, lat = test$lat, method = 'distance')        
    latWeights <- InsertDim(sqrt(cos(test$lat * pi / 180)), 2, dim(WR_obs$composite)[poslon]) ## rm sqrt?

    assign.method <- "RMSE" #"cor"
    if(assign.method == "RMSE"){
        ## measure RMSE between observed and forecasted WR anomalies an take the lowest values:
        temp1 <- apply(X = anom_for_same_week, MARGIN = 1:4, FUN = function(x){mean((as.vector(x * latWeights) - as.vector(WR_obs$composite[,,1] * latWeights))^2)})
        temp2 <- apply(X = anom_for_same_week, MARGIN = 1:4, FUN = function(x){mean((as.vector(x * latWeights) - as.vector(WR_obs$composite[,,2] * latWeights))^2)})
        temp3 <- apply(X = anom_for_same_week, MARGIN = 1:4, FUN = function(x){mean((as.vector(x * latWeights) - as.vector(WR_obs$composite[,,3] * latWeights))^2)})
        temp4 <- apply(X = anom_for_same_week, MARGIN = 1:4, FUN = function(x){mean((as.vector(x * latWeights) - as.vector(WR_obs$composite[,,4] * latWeights))^2)})
        temp <- abind(temp1, temp2, temp3, temp4, along = 5)
        WR.seq.for[1,,, pos.week] <- apply(temp, 1:4, function(x){ which(x == min(x)) })         
        rm(temp, temp1, temp2, temp3, temp4)
    } else {
        ## Alterantively, measure the spatial correlation instead of the RMSE and take the highest values:
        temp1 <- apply(X = anom_for_same_week, MARGIN = 1:4, FUN = function(x){cor(as.vector(x * latWeights), as.vector(WR_obs$composite[,,1] * latWeights))})
        temp2 <- apply(X = anom_for_same_week, MARGIN = 1:4, FUN = function(x){cor(as.vector(x * latWeights), as.vector(WR_obs$composite[,,2] * latWeights))})
        temp3 <- apply(X = anom_for_same_week, MARGIN = 1:4, FUN = function(x){cor(as.vector(x * latWeights), as.vector(WR_obs$composite[,,3] * latWeights))})
        temp4 <- apply(X = anom_for_same_week, MARGIN = 1:4, FUN = function(x){cor(as.vector(x * latWeights), as.vector(WR_obs$composite[,,4] * latWeights))})

        temp <- abind(temp1, temp2, temp3, temp4, along = 5)
        WR.seq.for[1,,, pos.week] <- apply(temp, 1:4, function(x){ which(x == max(x)) })         
        rm(temp, temp1, temp2, temp3, temp4)
    }
    
    ## reorder the observed and forecasted WR numbers to use the same order of the observed weekly classification created by us:
    if(reordering){
        cl <- array(NA, c(52,4))
        cl[1,] <- c(3,2,1,4); cl[2,] <- c(1,3,4,2); cl[3,] <- c(4,1,2,3); cl[4,] <- c(2,3,4,1); cl[5,] <- c(3,2,4,1)
        cl[6,] <- c(4,3,1,2); cl[7,] <- c(4,1,3,2); cl[8,] <- c(1,3,2,4); cl[9,] <- c(3,4,2,1); cl[10,] <- c(1,4,3,2)
        cl[11,] <- c(2,1,3,4); cl[12,] <- c(3,2,4,1); cl[13,] <- c(3,4,2,1); cl[14,] <- c(3,4,1,2); cl[15,] <- c(1,3,4,2)
        cl[16,] <- c(2,4,1,3); cl[17,] <- c(2,1,4,3); cl[18,] <- c(1,2,4,3); cl[19,] <- c(2,3,1,4); cl[20,] <- c(3,1,2,4)
        cl[21,] <- c(2,3,1,4); cl[22,] <- c(4,3,2,1); cl[23,] <- c(4,2,1,3); cl[24,] <- c(4,1,3,2); cl[25,] <- c(4,3,1,2)
        cl[26,] <- c(3,2,1,4); cl[27,] <- c(3,2,4,1); cl[28,] <- c(4,2,1,3); cl[29,] <- c(3,1,4,2); cl[30,] <- c(4,1,3,2)
        cl[31,] <- c(4,3,2,1); cl[32,] <- c(3,4,1,2); cl[33,] <- c(1,3,2,4); cl[34,] <- c(1,4,3,2); cl[35,] <- c(2,1,4,3)
        cl[36,] <- c(3,4,2,1); cl[37,] <- c(2,3,4,1); cl[38,] <- c(1,4,2,3); cl[39,] <- c(3,1,4,2); cl[40,] <- c(3,1,2,4)
        cl[41,] <- c(1,3,4,2); cl[42,] <- c(2,4,3,1); cl[43,] <- c(3,4,1,2); cl[44,] <- c(3,2,4,1); cl[45,] <- c(1,2,3,4)
        cl[46,] <- c(4,3,1,2); cl[47,] <- c(4,3,2,1); cl[48,] <- c(2,3,1,4); cl[49,] <- c(4,1,3,2); cl[50,] <- c(3,1,4,2)
        cl[51,] <- c(2,3,4,1); cl[52,] <- c(3,4,1,2)

        ## reorder WR spatial patterns of the loaded 5 weeks: (so that now the first WR corresponds to the first one in the rearranged composition)
        WR_obs$composite <- abind(WR_obs$composite[,, cl[week, 1]], WR_obs$composite[,, cl[week, 2]], WR_obs$composite[,, cl[week, 3]], WR_obs$composite[,, cl[week, 4]], along = 3)

        new.wr <- match(1:4, cl[week,]) ## or c(which(cl[week,] == 1), which(cl[week,] == 2), which(cl[week,] == 3), which(cl[week,] == 4))
              
        ## reorder observed WR sequence of the loaded 5 weeks:
        temp <- WR_obs$cluster
        for(wr in 1:n.clusters) temp[which(WR_obs$cluster == wr)] <- new.wr[wr]
        WR_obs$cluster <- temp; rm(temp)

        ## reorder forecasted WR sequence of the loaded week:
        temp <- WR.seq.for[1,,, pos.week]
        for(wr in 1:n.clusters) temp[which(WR.seq.for[1,,, pos.week] == wr)] <- new.wr[wr]
        WR.seq.for[1,,, pos.week] <- temp; rm(temp)
    }

    ## ## check observed weekly regime patterns (for the debug):
    ## PlotEquiMap(WR_obs$composite[,,1], test$lon, test$lat, filled.continents=FALSE, brks=seq(-16,16,2))
    ## PlotEquiMap(WR_obs$composite[,,2], test$lon, test$lat, filled.continents=FALSE, brks=seq(-16,16,2))
    ## PlotEquiMap(WR_obs$composite[,,3], test$lon, test$lat, filled.continents=FALSE, brks=seq(-16,16,2))
    ## PlotEquiMap(WR_obs$composite[,,4], test$lon, test$lat, filled.continents=FALSE, brks=seq(-16,16,2))

    ## reshape the daily sequence of observed weekly WRs in an array [years, days]:
    WR.seq.obs.array <- array(WR_obs$cluster, c(n.yrs.obs, 35))  ## 35 is the number of days of the mobile windows of 5 weeks used to classify observed WRs
    
    ## select only the reanalysis years within the hindcasts years to be able to compare the observed WRs with the forecasted ones:
    yr1 <- yr1.hind - yr1.obs + 1
    yr2 <- yr2.hind - yr1.obs + 1
    ##WR.seq.obs.array <- WR.seq.obs.array[yr1:yr2,]
    WR.seq.obs.array <- WR.seq.obs.array[yr1:(yr2+1),] ## load one year more to be able to assign obs WRs for forecast days beyond the 31th of December (see below)

    ## select only days of the central week (of the 5 weeks):
    if(week == 52) week.length <- 8 else week.length <- 7 ## include the 8th day in case of last week
    WR.seq.obs.array <- WR.seq.obs.array[, 15:(14 + week.length)]
    
    ## Select only the part of WR.seq.obs.array inside the forecast days of the selected week.
    ## Note that WR.seq.rean[,X] indicates the observed WRs corresponding to the forecast days, while WR.seq.obs.array indicates the observed WRs
    ## corresponding to the calendar days, so you have to convert the second to the first with the command below:
    n.days.week <- length(which(weekly.seq == week)) ## number of forecast days inside the week loaded
    if (week != 52){
        if (week == weeks2load[1]) pos.fd <- (7 - n.days.week + 1):7 else pos.fd <- 1:n.days.week
    } else {
        if (week == weeks2load[1]) pos.fd <- (8 - n.days.week + 1):8 else pos.fd <- 1:n.days.week ## last week (52th) can be of 8 days instead of 7
    }
    
    WR.seq.rean[, pos.week] <- WR.seq.obs.array[1:n.yrs.hind, pos.fd] 

    ## correct in case the forecast days enter in the following year:
    if(week != weeks2load[1] && weeks2load[2] < weeks2load[1]) WR.seq.rean[, pos.week] <- WR.seq.obs.array[-1, pos.fd]
    ## if(week != weeks2load[1] && weeks2load[2] < weeks2load[1]){
    ##     WR.seq.rean[1:(n.yrs.hind - 1), pos.week] <- WR.seq.obs.array[-1, pos.fd]
    ##     WR.seq.rean[n.yrs.hind, pos.week] <- NA ##WR.seq.obs.array[n.yrs.hind+1, pos.fd] ## here we should place the WR sequence of year 2018 but it is not available, so we place NA instead
    ## }

    ## compare the forecasted WR sequence with the observed one to get the number of coincident regimes for each forecast day (summed over all members and all hindcast years):
    for(d in pos.week){
        for(y in 1:n.yrs.hind){
            WR.seq.coinc[d] <- WR.seq.coinc[d] + length(which(WR.seq.for[1,,y,d] == WR.seq.rean[y,d]))
        }
    }
    
    ## for each WR and forecasted day, measure the spatial correlations between predicted and observed WR patterns:
    for(wr in 1:n.clusters) {
        for(d in pos.week){
            ## hindcast years and members with the same assigned WR:
            n.dd <- length(which(WR.seq.for[1,,,d] == wr))

            if(n.dd > 0){
                ## print(WR.seq.for[1,,,d])
                same.wr <- which(WR.seq.for[1,,,d] == wr, arr.ind = TRUE)
                
                anom_for_same_day_and_wr <- 0
                for(dd in 1:n.dd) {
                    anom_for_same_day_and_wr <- anom_for_same_day_and_wr + anom_for_same_week[1,same.wr[dd, 1], same.wr[dd, 2], which(pos.week == d),,]
                    ## PlotEquiMap(anom_for_same_week[same.wr[dd, 1], same.wr[dd, 2], wr,,], lon=test$lon, lat=test$lat, filled.continents=FALSE, brks=my.brks, toptitle="Hindcasted") ## for debug
                }

                ## ensemble mean of SLP hindcast anomalies during days and members of predicted regime wr
                anom_for_same_day_and_wr <- anom_for_same_day_and_wr / n.dd 
                
                WR.sp.cor[wr, d] <- cor(as.vector(WR_obs$composite[,, wr]), as.vector(anom_for_same_day_and_wr))

                ## ## plot the WR hindcasted ensemble pattern for debug:
                ## my.brks <- c(-100,seq(-21,-1,2),0,seq(1,21,2),100) ## 23 intervals
                ## PlotEquiMap(WR_obs$composite[,, wr], lon=test$lon, lat=test$lat, filled.continents=FALSE, brks=my.brks, toptitle="Observed")
                ## x11()
                ## PlotEquiMap(anom_for_same_day_and_wr, lon=test$lon, lat=test$lat, filled.continents=FALSE, brks=my.brks, toptitle="Hindcasted")

                ## ## plot the same as above but for the forecasted WR pattern of a particular day, year and member:
                ## year <- 20
                ## member <- 4
                ## PlotEquiMap(anom_for_same_week[member, year, d,,], lon=test$lon, lat=test$lat, filled.continents=FALSE, brks=my.brks, toptitle="Hindcasted")
                ## PlotEquiMap(anom_for[1, member, year, d,,], lon=test$lon, lat=test$lat, filled.continents=FALSE, brks=my.brks, toptitle="Hindcasted")
               
            } else {
                WR.sp.cor[wr, d] <- NA
            }
        }
        
    } ## close for on wr

} ## close for on week

## convert to a %:
WR.seq.coinc.perc <- 100 * WR.seq.coinc / (n.yrs.hind * n.members)

## for the debug: 
cbind(forecast.days=1:length(days.seq), days.seq, monthly.seq, weekly.seq, n.coinc.days=WR.seq.coinc, perc.coinc=WR.seq.coinc.perc)

## for the debug: for each hindcast year, it shows the observed (first column) and forecasted (other columns) WRs of each ensemble member:
## cbind(WR.yr1=WR.seq.rean[1,], t(WR.seq.for[1,,1,]), WR.yr2=WR.seq.rean[2,], t(WR.seq.for[1,,2,]), WR.yr3=WR.seq.rean[3,], t(WR.seq.for[1,,3,]), WR.yr4=WR.seq.rean[4,], t(WR.seq.for[1,,4,]), WR.yr5=WR.seq.rean[5,], t(WR.seq.for[1,,5,]), WR.yr6=WR.seq.rean[6,], t(WR.seq.for[1,,6,]), WR.yr7=WR.seq.rean[7,], t(WR.seq.for[1,,7,]), WR.yr8=WR.seq.rean[8,], t(WR.seq.for[1,,8,]), WR.yr9=WR.seq.rean[9,], t(WR.seq.for[1,,9,]), WR.yr10=WR.seq.rean[10,], t(WR.seq.for[1,,10,]), WR.yr11=WR.seq.rean[11,], t(WR.seq.for[1,,11,]), WR.yr12=WR.seq.rean[12,], t(WR.seq.for[1,,12,]), WR.yr13=WR.seq.rean[13,], t(WR.seq.for[1,,13,]), WR.yr14=WR.seq.rean[14,], t(WR.seq.for[1,,14,]), WR.yr15=WR.seq.rean[15,], t(WR.seq.for[1,,15,]), WR.yr16=WR.seq.rean[16,], t(WR.seq.for[1,,16,]), WR.yr17=WR.seq.rean[17,], t(WR.seq.for[1,,17,]), WR.yr18=WR.seq.rean[18,], t(WR.seq.for[1,,18,]), WR.yr19=WR.seq.rean[19,], t(WR.seq.for[1,,19,])) ##, WR.yr20=WR.seq.rean[20,], t(WR.seq.for[1,,20,])) 

print("Saving results")
save(chunk, WR.seq.coinc, WR.seq.coinc.perc, WR.seq.for, WR.seq.rean, WR.sp.cor, yr1, yr2, n.yrs.hind, n.members, work.dir, file=paste0(work.dir, "/Assign_table_startdate_",chunk,".Rdata"))

