
# Creation:  6/2016
# Author: Nicola Cortesi
# Aim: To compute the four Euro-Atlantic weather regimes from a reanalysis or from a forecast system
#
# I/O: input must be in a format compatible with Load(); 1 output .RData file is created in the 'workdir' folder.
#      You can also run this script from terminal with: Rscript ~/scripts/weather_regimes_maps.R
#
# Assumption:  its output are needed by the scripts weather_regimes_impact.R and  weather_regimes.R
#
# Branch: weather_regimes

rm(list=ls())
library(s2dverification)  # for the function Load()
library(abind)
library(reshape2) # after each undate of s2dverification, it's better to remove and install this package again because sometimes it gets broken! 

SMP <- FALSE   # if TRUE, the script is assumed to run on the SMP Machine


if(SMP){
   source('/gpfs/projects/bsc32/bsc32842/Rfunctions.R')
   workdir <- "/gpfs/projects/bsc32/bsc32842/RESILIENCE"  # working dir where output files will be generated
} else {
   source('/home/Earth/ncortesi/scripts/Rfunctions.R')
   workdir <- "/scratch/Earth/ncortesi/RESILIENCE/Regimes"  # working dir where output files will be generated
}

# module load NCO # : load it from the terminal before running this script interactively in the SMP machine, if you didn't load it in the .bashrc

# available reanalysis for the geopotential (g500) or the geopotential height (z500 = g500/9.81) and for var data:
ERAint <- '/esnas/reconstructions/ecmwf/eraint/$STORE_FREQ$_mean/$VAR_NAME$_f6h/$VAR_NAME$_$YEAR$$MONTH$.nc'
#ERAint <- '/esarchive/old-files/recon_ecmwf_erainterim/$STORE_FREQ$_mean/$VAR_NAME$_f6h/$VAR_NAME$_$YEAR$$MONTH$.nc'
ERAint.name <- "ERA-Interim"

JRA55 <- '/esnas/recon/jma/jra55/$STORE_FREQ$_mean/$VAR_NAME$_f6h/$VAR_NAME$_$YEAR$$MONTH$.nc'
JRA55.name <- "JRA-55"

rean <- ERAint #JRA55 #ERAint  # choose one of the two above reanalysis from where to load the input psl data

# available forecast systems for the psl and var data:
#ECMWF_S4 <- list(path = paste0('/esnas/exp/ECMWF/seasonal/0001/s004/m001/$STORE_FREQ$_mean/$VAR_NAME$_f6h/$VAR_NAME$_$START_DATE$.nc'))
#ECMWF_S4 <- '/esnas/exp/ECMWF/seasonal/0001/s004/m001/$STORE_FREQ$_mean/psl_f6h/psl_$START_DATE$.nc'
ECMWF_S4 <- '/esnas/exp/ecmwf/system4_m1/$STORE_FREQ$_mean/$VAR_NAME$_f6h/$VAR_NAME$_$START_DATE$.nc'
ECMWF_S4.name <- "ECMWF-S4"
member.name <- "ensemble"   # name of the dimension with the ensemble members inside the netCDF files of S4

ECMWF_monthly <- '/esnas/exp/ECMWF/monthly/ensforhc/6hourly/$VAR_NAME$/2014$MONTH$$DAY$00/$VAR_NAME$_$START_DATE$00.nc'
ECMWF_monthly.name <- "ECMWF-Monthly"

# choose a forecast system:
forecast <- ECMWF_S4  

# put 'rean' to load pressure fields and var from reanalisis, or put 'forecast' to load them from forecast systems:
fields <- forecast #rean #forecast  

psl <- "psl" #"g500"                           # pressure variable to use from the chosen reanalysis
psl.name <- "SLP" # "Z500"                     # the name to show in the maps

year.start <- 1981 #1979 #1981 #1982 #1981 #1994 #1979 #1981
year.end <- 2015 #2016 #2013 #2015

missing.forecasts <- FALSE # set it to TRUE if there can be missing hindcasts files in the forecast psl data, so it will load only the available years and deals with NA
res <- 0.75                # set the resolution you want to interpolate the psl and var data when loading it (i.e: set to 0.75 to use the same res of ERA-Interim)
                           # interpolation method is BILINEAR.

LOESS <- TRUE             # To apply the LOESS filter or not to the climatology
running.cluster <- TRUE    # add to the clustering also the daily SLP data of the two closer months to the month to use (only for monthly analysis)
lat.weighting <- FALSE      # set it to true to weight psl data on latitude before applying the cluster analysis
sequences <-FALSE          # set it to TRUE if you want to select only days beloning to sequences of at least 5 consecutive days belonging to the same regime.

PCA <- FALSE                # set it to TRUE if you want to apply the PCA before the k-means cluster analysis, FALSE otherwise
variance.explained <- 0.8  # the user can set here the minimum % of variance explained by the PCs (typically 0.8 which is equal to 80%)

# Coordinates of the box enclosing the study region (the Northern Atlantic in this case):
lat.max <- 81 #81 #80 #70
lat.min <- 27 #30 #20 #30
lon.max <- 45 #36 #30 #40 
lon.min <- 274.5 #274.5 #270 #280 # put a positive number here because the geopotential has only positive vaues of longitud!

# for both reanalysis and forecasts:
WR.period <- 9 # 13:16 #1:12 #13:16  # 1-12: Jan-Dec; 13-16: Winter-Spring-Summer-Autumn  [bypassed by the optional argument of the script]

# Only for Seasonal forecasts:
startM <- 9                                      # start month (1=January, 12=December)  [bypassed by the optional arguments of the script]
leadM <- 0                                       # select only the data of one lead month:  [bypassed by the optional arguments of the script]
n.members <- 15                                  # number of members of the forecast system to load
n.leadtimes <- 216                               # number of leadtimes of the forecast system to load

# Only for Monthly forecasts:
#leadtime <- 1:7 #5:11 #1                        # if fields=forecast, set the forecast time (in days) you want to compute the weather regimes.
#startdate.shift <- 1                            # if leadtime=5:11, it's better to shift all startdates of the season 1 week in the past, to fit the exact season of obs.data
                                                 # if leadtime=12:18, it's better to shift all startdates of the season 2 weeks in the past, and so on.
#forecast.year <- year.end+1                     # if fields=forecast, set the forecast year (it is usually the year after year.end)
#mes=1                                           # if fields=forecast, set the month of the first startdate of the forecast year
#day=2                                           # if fields=forecast, set the day of the first startdate of the forecast year

#############################################################################################################################
rean.name <- unname(ifelse(rean == ERAint, ERAint.name, JRA55.name))
forecast.name <- unname(ifelse(forecast == ECMWF_S4, ECMWF_S4.name, ECMWF_monthly.name))
fields.name <- unname(ifelse(fields == rean, rean.name, forecast.name))
n.years <- year.end - year.start + 1

# in case the script is run with one argument, it is assumed a reanalysis is being used and the argument becomes the month we want to perform the clustering;
# in case the script is run with two arguments, it is assumed a forecast is used and the first argument represents the startmonth and the second one the leadmonth.
# in case the script is run with no arguments, the values of the variables inside the script are used:
script.arg <- as.integer(commandArgs(TRUE))

if(length(script.arg) == 0){
   start.month <- startM
   #WR.period <- start.month
   if(fields.name == forecast.name) lead.month <- leadM
}

# in case the script is run with 1 argument, it is assumed you are using a reanalysis:
if(length(script.arg) == 1){
   fields <- rean
   fields.name <- rean.name
   WR.period <-  script.arg[1]      
}

if(length(script.arg) >= 2){
   fields <- forecast
   fields.name <- forecast.name
   start.month <- script.arg[1]                 
   lead.month <- script.arg[2]
   WR.period <- start.month
}

if(length(script.arg) == 3){ # in this case, we are running the script in the SMP machine and we need to override the variables below:
   source('/gpfs/projects/bsc32/bsc32842/Rfunctions.R') # for the calendar function
   workdir <- "/gpfs/projects/bsc32/bsc32842/RESILIENCE"  # working dir
}

if(length(script.arg) >= 2) {lead.comment <- paste0("Lead month: ", lead.month)} else {lead.comment <- ""}
cat(paste0("Field: ", fields.name, " Period: ", WR.period, " ", lead.comment, "\n"))

if(lat.min >= lat.max) stop("lat.min cannot be greater than lat.max")

#days.period <- n.days.period <- period.length <- list()
#for (pp in 1:17){
#    days.period[[pp]] <- NA
#    for(y in year.start:year.end) days.period[[pp]] <- c(days.period[[pp]], n.days.in.a.future.year(year.start, y) + pos.period(y,pp))
#    days.period[[pp]] <- days.period[[pp]][-1] # remove the NA at the begin that was introduced only to be able to execute the above command
#    # number of days belonging to that period from year.start to year.end:
#    n.days.period[[pp]] <- length(days.period[[pp]])
#    # Number of days belonging to that period in a single year: 
#    period.length[[pp]] <- n.days.in.a.period(pp,1999) #+ ifelse(period==13 | period==2, 1, 0)  # using year 1999 introduce a small error in winter season length because of #bisestile years
#}

# Create a regular lat/lon grid to interpolate the data to the chosen res: (Notice that the regular grid is always centered at lat=0 and lon=0!)
n.lon.grid <- 360/res
n.lat.grid <- (180/res)+1
my.grid <- paste0('r',n.lon.grid,'x',n.lat.grid)
#domain<-list()
#domain$lon <- seq(0,359.999999999,res) 
#domain$lat <- c(rev(seq(0,90,res)),seq(res[1],-90,-res))

cat("Loading lat/lon. Please wait...\n")
# load only 1 day of pressure data to detect the minimum and maximum lat and lon values which identify only the North Atlantic area:
if(fields.name == rean.name) domain <- Load(var = psl, exp = NULL, obs = list(list(path=fields)), '19990101', storefreq = 'daily', leadtimemax = 1, output = 'lonlat')

### Real test:
### lat <- 
### domain <- Load(var = "psl", exp = list(list(path="/esnas/exp/ecmwf/system4_m1/$STORE_FREQ$_mean/$VAR_NAME$_f6h/$VAR_NAME$_$START_DATE$.nc")), obs=NULL, sdates=paste0(1982:2011,'0101'), storefreq = 'daily', leadtimemax=n.leadtimes, nmember=n.members, output = 'lonlat', latmin = lat[chunk], latmax = lat[chunk+2], nprocs=1)

### if (fields.name="pippo"){
    
# in the case of S4, we also interpolate it to the same resolution of the reanalysis:
# (notice that if the S4 grid has the same grid of the chosen grid (typically ERA-Interim), Load() leaves the S4 grid unchanged)
if(fields.name == ECMWF_S4.name) domain <- Load(var = psl, exp = list(list(path=fields)), obs=NULL, sdates=paste0(year.start,'0101'), storefreq = 'daily', dimnames=list(member=member.name), leadtimemax=1, nmember=1, output = 'lonlat', grid=my.grid, method='bilinear', nprocs=1)

#domain <- Load(var = "psl", exp = list(list(path="/esnas/exp/ecmwf/system4_m1/$STORE_FREQ$_mean/$VAR_NAME$_f6h/$VAR_NAME$_$START_DATE$.nc")), obs=NULL, sdates='19820101', storefreq = 'daily', leadtimemax=1, nmember=1, output = 'lonlat', grid='r480x241', lonmax=100)

if(fields.name == ECMWF_monthly.name) domain <- Load(var = psl, exp = NULL, obs = list(list(path=fields)), paste0(year.start,'0102'), storefreq = 'daily', leadtimemax = 1, output = 'lonlat')    

n.lon <- length(domain$lon)
n.lat <- length(domain$lat)

pos.lat.max <- which(lat.max - round(domain$lat,4) >= 0)[1]           # select the first latitude of the domain below the maximum latitude
pos.lat.min <- tail(which(round(domain$lat,4) - lat.min >= 0),1)      # select the last latitude of the domain over the minumum latitude
pos.lon.max <- tail(which(lon.max - round(domain$lon,4) >= 0),1)
pos.lon.min <- which(round(domain$lon,4) - lon.min >= 0)[1]

pos.lat <- if(pos.lat.min < pos.lat.max) {pos.lat.min:pos.lat.max} else {pos.lat.max:pos.lat.min}
pos.lon <- c(1:pos.lon.max,pos.lon.min:length(domain$lon)) # beware that it only consider the case where the study area cross the meridian of Greenwhich!
n.pos.lat <- length(pos.lat)
n.pos.lon <- length(pos.lon)

lat <- domain$lat[pos.lat] # lat values of chosen area only (it is smaller than the whole spatial domain loaded by the data)
#if (lat[2] < lat[1]) lat <- rev(lat) # reverse lat values if they are in decreasing order
lon <- domain$lon[pos.lon] # lon values of chosen area only

#lat.min.area <- domain$lat[pos.lat.min] # min and max lat/lon of the chosen area only (same as lat.max, but its values correspond to actual values in the lat vector)
#lat.max.area <- domain$lat[pos.lat.max]
#lon.min.area <- domain$lon[pos.lon.min]
#lon.max.area <- domain$lon[pos.lon.max]

#rm(domain)

# Load psl data (interpolating it to the same reanlaysis grid, if necessary):
cat("Loading data. Please wait...\n")

if(fields.name == rean.name){ # Load daily psl data of all years in the reanalysis case:
    psleuFull <-array(NA,c(1,1,n.years,365, n.pos.lat, n.pos.lon))

    #for (y in year.start:year.end){
    #    var <- Load(var = psl, exp = NULL, obs = list(list(path=fields)), paste0(y,'0101'), storefreq = 'daily', leadtimemax = n.days.in.a.year(y), output = 'lonlat',
    #                latmin = lat.min, latmax = lat.max, lonmin = lon.min, lonmax = lon.max, nprocs=1)
    #    psleuFull[seq.days.in.a.future.year(year.start, y),,] <- var$obs
    #    rm(var)
    #    gc()
    #}

    psleuFull366 <-  Load(var = psl, exp = NULL, obs = list(list(path=fields)), paste0(year.start:year.end,'0101'), storefreq = 'daily', leadtimemax = 366, output = 'lonlat', latmin = lat.min, latmax = lat.max, lonmin = lon.min, lonmax = lon.max)$obs

    # remove bisestile days to have all arrays with the same dimensions:
    cat("Removing bisestile days. Please wait...\n")
    psleuFull[,,,,,] <- psleuFull366[,,,1:365,,]

    for(y in year.start:year.end){
        y2 <- y - year.start + 1
        if(n.days.in.a.year(y) == 366) psleuFull[,,y2,60:365,,] <- psleuFull366[,,y2,61:366,,] # take the march to december period removing the 29th of February
    }

    rm(psleuFull366)
    gc()

    # convert psl in daily anomalies with the LOESS filter:
    cat("Calculating anomalies. Please wait...\n")
    pslPeriodClim <- apply(psleuFull, c(1,2,4,5,6), mean, na.rm=T)

    if(LOESS == TRUE){
        pslPeriodClimLoess <- pslPeriodClim
        for(i in n.pos.lat){
            for(j in n.pos.lon){
                my.data <- data.frame(ens.mean=pslPeriodClim[1,1,,i,j], day=1:365)
                my.loess <- loess(ens.mean ~ day, my.data, span=0.35)
                pslPeriodClimLoess[1,1,,i,j] <- predict(my.loess)
            }
        }

        rm(pslPeriodClim)
        gc()

        pslPeriodClim2 <- InsertDim(pslPeriodClimLoess, 3, n.years)
        
    } else {
        pslPeriodClim2  <- InsertDim(pslPeriodClim, 3, n.years)
        
        rm(pslPeriodClim)
        gc()
    }
    
    ## s4.data <- data.frame(s4=pslPeriodClim[1,], day=1:216)
    ## s4.loess <- loess(s4 ~ day, s4.data, span=0.35)
    ## s4.pred <- predict(s4.loess)
        
    psleuFull <- psleuFull - pslPeriodClim2    
    rm(pslPeriodClim2)
    gc()
    
} # close if on fields.name == rean

if(fields.name == ECMWF_S4.name){ # in the forecast case, we load only the data for 1 start month at time and all the lead times (since each month needs ~3 GB of memory)    
    if(start.month <= 9) {start.month.char <- paste0("0",as.character(start.month))} else {start.month.char <- start.month}

    fields2 <- gsub("\\$STORE_FREQ\\$","daily",fields)
    fields3 <- gsub("\\$VAR_NAME\\$",psl,fields2)

    if(missing.forecasts == TRUE){
        years <- c()
        for(y in year.start:year.end){
            fields4 <- gsub("\\$START_DATE\\$",paste0(y, start.month.char,'01'),fields3)

            char.lead <- nchar(system(paste0("ncks -m ",fields4,"| grep -E -i 'psl size' | cut -d '*' -f1"), intern=T))
            # beware, in this way it can only take into account leadtimes from 100 to 999, but it is the only way in the SMP machine:
            #num.lead <- as.integer(substr(system(paste0("ncks -m ",fields4,"| grep -E -i 'psl size' | cut -d '*' -f1"), intern=T),char.lead-3,char.lead)) 
            num.lead <- as.integer(system(paste0("ncks -m ",fields4,"| grep -E -i 'psl size' | cut -d '=' -f2 | cut -d '*' -f1"), intern=T)) # don't work well on the SMP
            num.memb <- as.integer(system(paste0("ncks -m ",fields4,"| grep -E -i 'psl size' | cut -d '=' -f2 | cut -d '*' -f2"), intern=T))
            num.lat  <- as.integer(system(paste0("ncks -m ",fields4,"| grep -E -i 'psl size' | cut -d '=' -f2 | cut -d '*' -f3"), intern=T))
            num.lon  <- as.integer(system(paste0("ncks -m ",fields4,"| grep -E -i 'psl size' | cut -d '=' -f2 | cut -d '*' -f4"), intern=T))

            #if(num.lead == n.leadtimes && num.memb >= n.members && num.lat == 181 && num.lon == 360) years <- c(years,y)
            if(num.lead == n.leadtimes && num.memb >= n.members) years <- c(years,y)      
        }

    } else {
        years <- year.start:year.end
    }

    n.years.full <- length(years)

    if(running.cluster == TRUE) {

        start.month1 <- ifelse(start.month == 1, 12, start.month-1)
        if(start.month1 <= 9) {start.month1.char <- paste0("0",as.character(start.month1))} else {start.month1.char <- start.month1}
        
        start.month2.char <- start.month.char
        
        start.month3 <- ifelse(start.month == 12, 1, start.month+1)
        if(start.month3 <= 9) {start.month3.char <- paste0("0",as.character(start.month3))} else {start.month3.char <- start.month3}

        # load three months of data, inserting all them in the third dimension of the array, one month at time:
        psleuFull1 <- Load(var = psl, exp = list(list(path=fields)), obs = NULL, sdates=paste0(years, start.month1.char,'01'), dimnames=list(member=member.name), nmember=n.members, leadtimemax=n.leadtimes, storefreq='daily', output = 'lonlat', latmin = lat.min, latmax = lat.max, lonmin = lon.min, lonmax = lon.max, grid=my.grid, method='bilinear')$mod
        gc()

        psleuFull2 <- Load(var = psl, exp = list(list(path=fields)), obs = NULL, sdates=paste0(years, start.month2.char,'01'), dimnames=list(member=member.name), nmember=n.members, leadtimemax=n.leadtimes, storefreq='daily', output = 'lonlat', latmin = lat.min, latmax = lat.max, lonmin = lon.min, lonmax = lon.max, grid=my.grid, method='bilinear')$mod
        gc()

        psleuFull3 <- Load(var = psl, exp = list(list(path=fields)), obs = NULL, sdates=paste0(years, start.month3.char,'01'), dimnames=list(member=member.name), nmember=n.members, leadtimemax=n.leadtimes, storefreq='daily', output = 'lonlat', latmin = lat.min, latmax = lat.max, lonmin = lon.min, lonmax = lon.max, grid=my.grid, method='bilinear')$mod
        gc()

    } else {
        
        psleuFull <- Load(var = psl, exp = list(list(path=fields)), obs = NULL, sdates=paste0(years, start.month.char,'01'), dimnames=list(member=member.name), nmember=n.members, leadtimemax=n.leadtimes, storefreq='daily', output = 'lonlat', latmin = lat.min, latmax = lat.max, lonmin = lon.min, lonmax = lon.max, grid=my.grid, method='bilinear', nprocs=1)$mod
    
    }

     if(LOESS == TRUE){
         # convert psl in daily anomalies with the LOESS filter:
         cat("Calculating anomalies. Please wait...\n")
         
         if(running.cluster == TRUE) {     
            pslPeriodClim1 <- apply(psleuFull1, c(1,4,5,6), mean, na.rm=T)
            pslPeriodClim2 <- apply(psleuFull2, c(1,4,5,6), mean, na.rm=T)
            pslPeriodClim3 <- apply(psleuFull3, c(1,4,5,6), mean, na.rm=T)
    
            pslPeriodClimLoess1 <- pslPeriodClim1
            pslPeriodClimLoess2 <- pslPeriodClim2
            pslPeriodClimLoess3 <- pslPeriodClim3

            for(i in n.pos.lat){
                for(j in n.pos.lon){
                    my.data1 <- data.frame(ens.mean=pslPeriodClim1[1,,i,j], day=1:n.leadtimes)
                    my.loess1 <- loess(ens.mean ~ day, my.data1, span=0.35)
                    pslPeriodClimLoess1[1,,i,j] <- predict(my.loess1)
                    rm(my.data1, my.loess1)
                    gc()

                    my.data2 <- data.frame(ens.mean=pslPeriodClim2[1,,i,j], day=1:n.leadtimes)
                    my.loess2 <- loess(ens.mean ~ day, my.data2, span=0.35)
                    pslPeriodClimLoess2[1,,i,j] <- predict(my.loess2)
                    rm(my.data2, my.loess2)
                    gc()

                    my.data3 <- data.frame(ens.mean=pslPeriodClim3[1,,i,j], day=1:n.leadtimes)
                    my.loess3 <- loess(ens.mean ~ day, my.data3, span=0.35)
                    pslPeriodClimLoess3[1,,i,j] <- predict(my.loess3)
                    rm(my.data3, my.loess3)
                    gc()
                }
            }

            rm(pslPeriodClim1, pslPeriodClim2, pslPeriodClim3)
            gc()

            pslPeriodClimDos1 <- InsertDim(InsertDim(pslPeriodClimLoess1, 2, n.years.full), 2, n.members)
            pslPeriodClimDos2 <- InsertDim(InsertDim(pslPeriodClimLoess2, 2, n.years.full), 2, n.members)
            pslPeriodClimDos3 <- InsertDim(InsertDim(pslPeriodClimLoess3, 2, n.years.full), 2, n.members)

            pslPeriodClimDos <- unname(abind(pslPeriodClimDos1, pslPeriodClimDos2, pslPeriodClimDos3, along=3))
            rm(pslPeriodClimDos1, pslPeriodClimDos2, pslPeriodClimDos3)
            gc()
            
            psleuFull <- unname(abind(psleuFull1, psleuFull2, psleuFull3, along=3)) # add the three months in the year dimension
            rm(psleuFull1, psleuFull2, psleuFull3)
            gc()
                
         } else { # in case of no running cluster:
    
            pslPeriodClim <- apply(psleuFull, c(1,4,5,6), mean, na.rm=T)

            pslPeriodClimLoess <- pslPeriodClim
            for(i in n.pos.lat){
                for(j in n.pos.lon){
                    my.data <- data.frame(ens.mean=pslPeriodClim[1,,i,j], day=1:n.leadtimes)
                    my.loess <- loess(ens.mean ~ day, my.data, span=0.35)
                    pslPeriodClimLoess[1,,i,j] <- predict(my.loess)
                }
            }

            rm(pslPeriodClim)
            gc()

            pslPeriodClimDos <- InsertDim(InsertDim(pslPeriodClimLoess, 2, n.years.full), 2, n.members)
             
         } # close if on running cluster
         
     } else { #in case of no LOESS:
         
       if(running.cluster == TRUE) {
          # in this case, the climatology is measured FOR EACH MONTH INDIPENDENTLY, instead of using a seasonal value: 
          pslPeriodClim1 <- apply(psleuFull1, c(1,4,5,6), mean, na.rm=T)
          pslPeriodClim2 <- apply(psleuFull2, c(1,4,5,6), mean, na.rm=T)
          pslPeriodClim3 <- apply(psleuFull3, c(1,4,5,6), mean, na.rm=T)

          pslPeriodClim <- unname(abind(pslPeriodClim1, pslPeriodClim2, pslPeriodClim3))
           
          pslPeriodClimDos <- InsertDim(InsertDim(pslPeriodClim, 2, n.years.full), 2, n.members)
          rm(pslPeriodClim)
          gc()

          psleuFull <- unname(abind(psleuFull1, psleuFull2, psleuFull3, along=3))
          rm(psleuFull1, psleuFull2, psleuFull3)
          gc()
           
       } else {# in case of no running cluster:
           
          pslPeriodClimDos <- InsertDim(InsertDim(pslPeriodClim, 2, n.years.full), 2, n.members)
          rm(pslPeriodClim)
          gc()
       }
     }
    
     ## s4.data <- data.frame(s4=pslPeriodClim[1,], day=1:216)
     ## s4.loess <- loess(s4 ~ day, s4.data, span=0.35)
     ## s4.pred <- predict(s4.loess)
        
     psleuFull <- psleuFull - pslPeriodClimDos    
     rm(pslPeriodClimDos)
     gc()
    
} # close if on fields.name=forecasts.name

if(fields.name == ECMWF_monthly.name){
    # Load 6-hourly psl data in the forecast case:    
    psleuFull <-array(NA, c(n.sdates*n.years, n.leadtimes, n.pos.lat, n.pos.lon))  # daily order: 19940102 19950102 ... 20130102 19940109 19950109 ... 20130109 ...
    n.leadtimes <- length(leadtime)
    sdates <- weekly.seq(forecast.year,mes,day)
    n.sdates <- length(sdates)
    
    for (startdate in 1:n.sdates){
       for(lday in leadtime){
          var <- Load(var = psl, exp = NULL, obs = list(list(path=fields)), paste0(year.start:year.end, substr(sdates[startdate],5,6), substr(sdates[startdate],7,8) ), storefreq = 'daily', leadtimemin = lday*4-3, leadtimemax = lday*4, output = 'lonlat', latmin = lat.min, latmax = lat.max, lonmin = lon.min, lonmax = lon.max)
          
          mean.lday <- apply(var$obs, c(3,5,6), mean, na.rm=T)
          rm(var)
          gc()
                           
          psleuFull[(1+(startdate-1)*n.years):(startdate*n.years), lday-leadtime[1]+1,,] <- mean.lday
          rm(mean.lday)
          gc()
       }
    }
}    

if(psl=="g500") psleuFull <- psleuFull/9.81   # convert geopotential to geopotential height (in m)
if(psl=="psl") psleuFull <- psleuFull/100   # convert MSLP in Pascal to MSLP in hPa
gc()

#save.image(file=paste0(workdir,"/weather_regimes_",fields.name,"_",year.start,"-",year.end,".RData"))
#load(file=paste0(workdir,"/weather_regimes_",fields.name,"_",year.start,"-",year.end,".RData"))

#save.image("/scratch/Earth/ncortesi/RESILIENCE/Regimes/test.RData")

# compute the cluster analysis:
my.PCA <- list()
my.cluster <- list()
my.cluster.array <- list()
tot.variance <- list()
n.pcs <- my.cluster <- c()
    
#WR.period=1 # for the debug   
for(p in WR.period){
    # Select only the data of the month/season we want:
    if(fields.name == rean.name){

        #pslPeriod <- psleuFull[days.period[[p]],,] # select only days in the chosen period (i.e: winter)
        if(running.cluster == TRUE) {
            pslPeriod <- psleuFull[1,1,,pos.month.extended(2001,p),,] # select all days in the period of 3 months centered on the target month p
        } else {
            pslPeriod <- psleuFull[1,1,,pos.period(2001,p),,] # select only days in the chosen period (i.e: winter)
        }
        
        # weight the pressure fields based on latitude (apply it to anomalies, not to absolute values!):
        if(lat.weighting){
            lat.weighted <- cos(lat*pi/180)^0.5
            lat.weighted.array <- InsertDim(InsertDim(InsertDim(lat.weighted,2,n.pos.lon), 1, length(pos.month.extended(2001,p))), 1, n.years)
            pslPeriod <- pslPeriod * lat.weighted.array
            rm(lat.weighted.array)
            gc()
        }

        gc()
        cat("Preformatting data for clustering. Please wait...\n")
        psl.melted <- melt(pslPeriod[,,,, drop=FALSE], varnames=c("Year","Day","Lat","Lon"))
        gc()
        
        cat("Preformatting data for clustering. Please wait......\n")
        
        # this function eats much memory!!!
        psl.kmeans <- unname(acast(psl.melted, Year + Day ~ Lat + Lon))
        #gc()
        
        # select period data from psleuFull to use it as input of the cluster analysis:    
        #psl.kmeans <- pslPeriod
        #dim(psl.kmeans) <- c(n.days.period[[p]], n.pos.lat*n.pos.lon) # convert array in a matrix!
        #rm(pslPeriod, pslPeriod.weighted)
    }

    # in case of S4 we don't need to select anything because we already loaded only 1 month of data!
    if(fields.name == ECMWF_S4.name){
        if(running.cluster == TRUE && p < 13) {
            # if you want to fully implement the running cluster of the monthly S4 data, you have to finish selecting automatically the 3-months running period
            # generalizing the command below (which at present only work for the month of January and lead time 0):

            # Select ASO and lead time 0, for our case study:
            pslPeriod <- psleuFull[,,,1:30,,, drop=FALSE]
                
        } else {
            
            # select data only for the startmonth and leadmonth to study:
            cat("Extracting data. Please wait...\n")
            chosen.month <- start.month + lead.month   # find which month we want to select data
            if(chosen.month > 12) chosen.month <- chosen.month - 12
            
            # remove the 29th of February to have the same n. of elements for all years
            i=0
            for(y in years){
                i=i+1
                leadtime.min <- 1 + n.days.in.a.monthly.period(start.month, chosen.month, y) - n.days.in.a.month(chosen.month, y)
                leadtime.max <- leadtime.min + n.days.in.a.month(chosen.month, y) - 1
                                        #leadtime.max <- 61 
                
                if (n.days.in.a.month(chosen.month, y) == 29) leadtime.max <- leadtime.max - 1 
                int.leadtimes <- leadtime.min:leadtime.max
                num.leadtimes <- leadtime.max - leadtime.min + 1 # number of days in the chosen month (different from 'n.leadtimes',which is the total number of days in the file!)
                 # by construction it is equal to: n.days.in.a.month(chosen.month, y)
                if(y == years[1]) {
                    pslPeriod <- psleuFull[,,1,int.leadtimes,,,drop=FALSE]
                } else {
                    pslPeriod <- unname(abind(pslPeriod, psleuFull[,,i,int.leadtimes,,,drop=FALSE], along=3))
                }
            }

        }
        
        # weight the pressure fields based on latitude (apply it to anomalies, not to absolute values!):
        if(lat.weighting){
            lat.weighted <- cos(lat*pi/180)^0.5
            lat.weighted.array <- InsertDim(InsertDim(InsertDim(lat.weighted,2,n.pos.lon), 1, length(pos.month.extended(2001,p))), 1, n.years)
            pslPeriod <- pslPeriod * lat.weighted.array  ######### adapt!!!!!!!!!!!
            rm(lat.weighted.array)
            gc()
        }

        # note that the data order in psl.kmeans[,X] is: year1day1, year1day2, ... , year1day31, year2day1, year2day2, ... , year2day28
        gc()
        cat("Preformatting data. Please wait...\n")
        psl.melted <- melt(pslPeriod[1,,,,,, drop=FALSE], varnames=c("Exp","Member","StartYear","LeadDay","Lat","Lon"))
        gc()

        save(psl.melted,file=paste0(workdir,"/psl_melted.RData"))
        
        cat("Preformatting data. Please wait......\n")
        # This function eats much memory!!!
        psl.kmeans <- unname(acast(psl.melted, Member + StartYear + LeadDay ~ Lat + Lon))
        #gc()

        save(psl.kmeans,file=paste0(workdir,"/psl_kmeans.RData"))
        
        my.cluster <- kmeans(psl.kmeans, centers=4, iter.max=100, nstart=30, trace=FALSE)
        save(my.cluster,file=paste0(workdir,"/my_cluster.RData"))

        
        #psl.kmeans <- array(NA,c(dim(pslPeriod)))
        #n.rows <- nrow(psl.melted)
        #a <- psl.melted$Member
        #b <- psl.melted$StartYear
        #c <- psl.melted$LeadDay
        #d <- psl.melted$Lat
        #e <- psl.melted$Lon
        #f <- psl.melted$value

        # this function to convert a data.frame in an array is much less memory-eater than acast but a bit slower:
        #for(i in 1:n.rows) psl.kmeans[1,a[i],b[i],c[i],d[i],e[i]] <- f[i]
        gc()
        #rm(pslPeriod); gc()
    }

    if(fields.name == ECMWF_monthly.name){
        my.startdates.period <- months.period(forecast.year,mes,day,p) # get which startdates belong to the chosen period (remember that there are no bisestile day left!)

        days.period <- list() # get which days inside psleuFull belong to the chosen period
        for(startdate in my.startdates.period){
           days.period[[p]] <- c(days.period[[p]], (1+(startdate-1)*n.years):(startdate*n.years))
        }
        
        # in winter you must remove the 2nd startdate (20140109) because its data is bad, as you can see with: plot(psl.kmeans[,1:2])
        # if(fields.name == forecast.name && period == 13) psl.kmeans[21:40,] <- NA
    }

        
    # compute the clustering:
    cat("Clustering data. Please wait...\n")
    if(PCA){ 
       my.seq <- seq(1, n.pos.lat*n.pos.lon, 16) # select only 1 point of 9
       pslcut <- psl.kmeans[,my.seq]
  
       my.PCA <- princomp(pslcut,cor=FALSE)
       tot.variance <- head(cumsum(my.PCA$sdev^2/sum(my.PCA$sdev^2)),50)  # check how many PCAs to keep basing on the sum of their expl. variance
       n.pcs <- head(as.numeric(which(tot.variance > variance.explained)),1) # select only the pcs that explains at least 80% of variance
       my.cluster <- kmeans(my.PCA$scores[,1:n.pcs], centers=4, iter.max=100, nstart=30)  # 4 is the number of clusters
       rm(pslcut)
    } else {  # 4 is the number of clusters:

       my.cluster <- kmeans(psl.kmeans, centers=4, iter.max=100, nstart=30, trace=FALSE)
        
       gc()
        
       #save(my.cluster, file=paste0(workdir,"/",fields.name,"_",my.period[p],"_leadmonth_",lead.month,"_series.RData"))
        
       #if(fields.name == rean.name){
           #my.cluster[[p]] <- kmeans(psl.kmeans[which(!is.na(psl.kmeans[,1])),], centers=4, iter.max=100, nstart=30, trace=FALSE)
           # save the time series output of the cluster analysis:
           #save(my.cluster, my.cluster.array, my.PCA, tot.variance, n.pcs, file=paste0(workdir,"/",fields.name,"_",my.period[p],"_series.RData"))
       #}
        
       if(fields.name == ECMWF_S4.name) {
           my.cluster.array <- array(my.cluster$cluster, c(num.leadtimes, n.years.full, n.members)) # in reverse order compared to the definition of psl.kmeans

           # save the time series output of the cluster analysis:
           #save(my.cluster, my.cluster.array, my.PCA, tot.variance, n.pcs, file=paste0(workdir,"/",fields.name,"_",my.period[p],"_leadmonth_",lead.month,"_series.RData"))
           
           #plot(SMA(my.cluster$centers[1,],201),type="l",col="gold",ylim=c(-20,20));lines(SMA(my.cluster$centers[2,],201),type="l",col="red"); lines(SMA(my.cluster$centers[3,],201),type="l",col="green");lines(SMA(my.cluster$centers[4,],201),type="l",col="blue");lines(SMA(psl.kmeans[29,],201),type="l",col="gray");lines(rep(0,14410),type="l",col="black",lty=2)
       }
    }
    
    rm(psl.kmeans)
    gc()
    
#} # close for on 'p'

#save.image(file=paste0(workdir,"/weather_regimes_",fields.name,"_",year.start,"-",year.end,".RData"))
#load(file=paste0(workdir,"/weather_regimes_",fields.name,"_",year.start,"-",year.end,".RData"))

#my.grid<-paste0('r',n.lon,'x',n.lat)
#coord <- Load(var = 'sfcWind', exp = list(ECMWF_monthly), obs = NULL, sdates=paste0(2013,'0102'), leadtimemin = 1, leadtimemax=1,  output = 'lonlat', nprocs=1, grid=my.grid, method='bilinear')

# measure regime anomalies:
    
#for(p in WR.period){ # 1-12: Jan-Dec; 13-16: Winter-Spring-Summer-Autumn; 17: Year  
    
    if(fields.name == rean.name){

      if(running.cluster == TRUE && p < 13){ # select only the days of the 3-months cluster series inside the target month p
          n.days.period <- length(pos.month.extended(2001,p))  # total number of days in the three months
          
          days.month <- days.month.new <- days.month.full <- c()
          days.month <- which(pos.month.extended(2001,p) == pos.month(2001,p)[1])-1 + 1:length(pos.month(2001,p))
          
          for(y in 1:n.years){
              days.month.new <- days.month + n.days.period*(y-1)
              days.month.full <- c(days.month.full, days.month.new)
              #print(days.month.new)
              #print(days.month)
          }
          
          cluster.sequence <- my.cluster$cluster[days.month.full] # select only the days of the cluster series inside the target month p

      } else {
          cluster.sequence <- my.cluster$cluster
      }
      
      if(sequences){ # it works only for rean data!!!
          # for each of the 4 clusters, remove the days not belonging to sequences of at least 5 days:
          # warning: first 4 days and last 4 days are not take into account y the algorithm and are treated as not belonging to any sequence (sequ=FALSE)    
          wrdiff <- diff(my.cluster$cluster)

          sequ <- rep(FALSE, n.days.period[[p]])
          for(i in 5:(n.days.period[[p]]-5)){
              sequ[i] <- TRUE
              if(wrdiff[i] != 0 & wrdiff[i+1] != 0) sequ[i] = FALSE
              if(wrdiff[i] != 0 & wrdiff[i+2] != 0) sequ[i] = FALSE
              if(wrdiff[i] != 0 & wrdiff[i+3] != 0) sequ[i] = FALSE
              if(wrdiff[i] != 0 & wrdiff[i+4] != 0) sequ[i] = FALSE
              if(wrdiff[i-1] != 0 & wrdiff[i] != 0) sequ[i] = FALSE
              if(wrdiff[i-1] != 0 & wrdiff[i+1] != 0) sequ[i] = FALSE
              if(wrdiff[i-1] != 0 & wrdiff[i+2] != 0) sequ[i] = FALSE
              if(wrdiff[i-1] != 0 & wrdiff[i+3] != 0) sequ[i] = FALSE
              if(wrdiff[i-2] != 0 & wrdiff[i] != 0) sequ[i] = FALSE
              if(wrdiff[i-2] != 0 & wrdiff[i+1] != 0) sequ[i] = FALSE
              if(wrdiff[i-2] != 0 & wrdiff[i+2] != 0) sequ[i] = FALSE
              if(wrdiff[i-3] != 0 & wrdiff[i] != 0) sequ[i] = FALSE
              if(wrdiff[i-3] != 0 & wrdiff[i+1] != 0) sequ[i] = FALSE
              if(wrdiff[i-4] != 0 & wrdiff[i] != 0) sequ[i] = FALSE
          } # close for loop on i
          
          # sequence of daily cluster numbers without the days that doesn't belong to sequences of 5 or more days:
          cluster.sequence <- my.cluster$cluster * sequ
          rm(wrdiff, sequ)
      }
      
      # Replace the days belonging to a regime with the days belonging to a sequence of at least 5 days of that regime:
      wr1 <- which(cluster.sequence == 1)
      wr2 <- which(cluster.sequence == 2)
      wr3 <- which(cluster.sequence == 3)
      wr4 <- which(cluster.sequence == 4)
      
      # yearly frequency of each regime:    
      wr1y <- wr2y <- wr3y <-wr4y <- c()
      np <- n.days.in.a.period(p,2001)
      
      for(y in year.start:year.end){
        # for both reanalysis and S4, the time order is always: year1day1, year1day2, ..., year1day31, year2day1, year2day2, ..., year2day28, etc,
        wr1y[y-year.start+1] <- length(which(cluster.sequence[(y-year.start)*np + (1:np)] == 1))
        wr2y[y-year.start+1] <- length(which(cluster.sequence[(y-year.start)*np + (1:np)] == 2))
        wr3y[y-year.start+1] <- length(which(cluster.sequence[(y-year.start)*np + (1:np)] == 3))
        wr4y[y-year.start+1] <- length(which(cluster.sequence[(y-year.start)*np + (1:np)] == 4))
      }

      # convert to frequencies in %:
      wr1y <- wr1y/np
      wr2y <- wr2y/np
      wr3y <- wr3y/np
      wr4y <- wr4y/np

      gc()
      
      pslmat <- unname(acast(psl.melted, Year + Day ~ Lat ~ Lon))

      if(running.cluster == TRUE){
         pslmat.new <- pslmat
         pslmat <- pslmat.new[days.month.full,,]
         gc()
      }
        
      pslwr1 <- pslmat[wr1,,, drop=F]
      pslwr2 <- pslmat[wr2,,, drop=F]
      pslwr3 <- pslmat[wr3,,, drop=F]
      pslwr4 <- pslmat[wr4,,, drop=F]

      pslwr1mean <- apply(pslwr1, c(2,3), mean, na.rm=T)
      pslwr2mean <- apply(pslwr2, c(2,3), mean, na.rm=T)
      pslwr3mean <- apply(pslwr3, c(2,3), mean, na.rm=T)
      pslwr4mean <- apply(pslwr4, c(2,3), mean, na.rm=T)

      # spatial mean for each day to save for the meaure of the FairRPSS:
      pslwr1spmean <- apply(pslwr1, 1, mean, na.rm=T)
      pslwr2spmean <- apply(pslwr2, 1, mean, na.rm=T)
      pslwr3spmean <- apply(pslwr3, 1, mean, na.rm=T)
      pslwr4spmean <- apply(pslwr4, 1, mean, na.rm=T)
      
      # save all the data necessary to redraw the graphs when we know the right regime:
      save(my.cluster, lon, lon.max, lon.min, lat, lat.max, lat.min, pos.lat, pos.lon, n.pos.lat, n.pos.lon, psl, psl.name, my.period, p, workdir, rean.name, fields.name, year.start, year.end, n.years, WR.period, pslwr1mean, pslwr2mean, pslwr3mean, pslwr4mean, pslwr1spmean, pslwr2spmean, pslwr3spmean, pslwr4spmean, wr1y,wr2y,wr3y,wr4y,file=paste0(workdir,"/",fields.name,"_",my.period[p],"_psl.RData"))

            # just to save the plots of the ERA-Interim monthly regime anomalies with the running cluster now instead than loading the next script:
            EU <- c(1:which(lon >= lon.max)[1], (which(lon >= 337)[1]:length(lon)))  # restrict area to continental europe only
             if(psl == "g500"){
                my.brks <- c(-300,seq(-200,200,10),300) # % Mean anomaly of a WR
                my.brks2 <- c(-300,seq(-200,200,10),300) # % Mean anomaly of a WR for the contour lines
                values.to.plot <- c(-200, -100, 0, 100, 200)  # values we want to show in the labels
            }

            if(psl == "psl"){
                my.brks <- c(seq(-19,-1,2),0,seq(1,19,2)) # % Mean anomaly of a WR
                my.brks2 <- my.brks #c(seq(-20,20,2)) # % Mean anomaly of a WR for the contour lines
                values.to.plot <- my.brks #c(-17,-15,-13,-11,-9,-7,-5,-3,-1,0,1,3,5,7,9,11,13,15,17)
            }
        
            my.cols <- colorRampPalette(rev(brewer.pal(11,"RdBu")))(length(my.brks)-1)    # blue--white--red colors
            my.cols[floor(length(my.cols)/2)] <- my.cols[floor(length(my.cols)/2)+1] <- "white"

            map1 <- pslwr1mean
            map2 <- pslwr2mean
            map3 <- pslwr3mean
            map4 <- pslwr4mean
        
            png(filename=paste0(workdir,"/",fields.name,"_",my.period[p],"_cluster1.png"),width=1000,height=1200)
            PlotEquiMap2(rescale(map1,my.brks[1],tail(my.brks,1)), lon, lat, filled.continents=FALSE, brks=my.brks, cols=my.cols, sizetit=1.2, contours=t(map1), brks2=my.brks2, intxlon=10, intylat=10, drawleg=F, cex.lab=1.5, xlabel.dist=1.5, contours.lty="F1FF1F")
            dev.off()
            png(filename=paste0(workdir,"/",fields.name,"_",my.period[p],"_cluster2.png"),width=1000,height=1200)
            PlotEquiMap2(rescale(map2,my.brks[1],tail(my.brks,1)), lon, lat, filled.continents=FALSE, brks=my.brks, cols=my.cols, sizetit=1.2, contours=t(map2), brks2=my.brks2, intxlon=10, intylat=10, drawleg=F, cex.lab=1.5, contours.lty="F1FF1F")
            dev.off()
            png(filename=paste0(workdir,"/",fields.name,"_",my.period[p],"_cluster3.png"),width=1000,height=1200)
            PlotEquiMap2(rescale(map3,my.brks[1],tail(my.brks,1)), lon, lat, filled.continents=FALSE, brks=my.brks, cols=my.cols, sizetit=1.2, contours=t(map3), brks2=my.brks2, intxlon=10, intylat=10, drawleg=F, cex.lab=1.5, contours.lty="F1FF1F")
            dev.off()
            png(filename=paste0(workdir,"/",fields.name,"_",my.period[p],"_cluster4.png"),width=1000,height=1200)
            PlotEquiMap2(rescale(map4,my.brks[1],tail(my.brks,1)), lon, lat, filled.continents=FALSE, brks=my.brks, cols=my.cols, sizetit=1.2, contours=t(map4), brks2=my.brks2, intxlon=10, intylat=10, drawleg=F, cex.lab=1.5, contours.lty="F1FF1F")
            dev.off()


        
      rm(pslwr1mean,pslwr2mean,pslwr3mean,pslwr4mean)
      rm(pslwr1,pslwr2,pslwr3,pslwr4)     
      gc()    
      
    }
        
    if(fields.name == ECMWF_S4.name){
      cat("Computing regime anomalies and frequencies. Please wait...\n")

      #load(file=paste0(workdir,"/",fields.name,"_",my.period[p],"_leadmonth_",lead.month,"_ClusterData.RData"))
      
      cluster.sequence <- my.cluster$cluster #as.vector(my.cluster.array) 

      wr1 <- which(cluster.sequence == 1)
      wr2 <- which(cluster.sequence == 2)
      wr3 <- which(cluster.sequence == 3)
      wr4 <- which(cluster.sequence == 4)

      wr1y <- apply(my.cluster.array == 1, 2, sum)
      wr2y <- apply(my.cluster.array == 2, 2, sum)
      wr3y <- apply(my.cluster.array == 3, 2, sum)
      wr4y <- apply(my.cluster.array == 4, 2, sum)

      # convert to frequencies in %:
      wr1y <- wr1y/(num.leadtimes*n.members) # in this case, n.leadtimes is the number of days of one month of the cluser time series
      wr2y <- wr2y/(num.leadtimes*n.members)
      wr3y <- wr3y/(num.leadtimes*n.members)
      wr4y <- wr4y/(num.leadtimes*n.members)

      # same as above, but to measure the maximum frequence between all the members:
      wr1yMax <- apply(apply(my.cluster.array == 1, c(2,3), sum),1,max)
      wr2yMax <- apply(apply(my.cluster.array == 2, c(2,3), sum),1,max)
      wr3yMax <- apply(apply(my.cluster.array == 3, c(2,3), sum),1,max)
      wr4yMax <- apply(apply(my.cluster.array == 4, c(2,3), sum),1,max)

      wr1yMax <- wr1yMax/num.leadtimes
      wr2yMax <- wr2yMax/num.leadtimes
      wr3yMax <- wr3yMax/num.leadtimes
      wr4yMax <- wr4yMax/num.leadtimes

      wr1yMin <- apply(apply(my.cluster.array == 1, c(2,3), sum),1,min)
      wr2yMin <- apply(apply(my.cluster.array == 2, c(2,3), sum),1,min)
      wr3yMin <- apply(apply(my.cluster.array == 3, c(2,3), sum),1,min)
      wr4yMin <- apply(apply(my.cluster.array == 4, c(2,3), sum),1,min)

      wr1yMin <- wr1yMin/num.leadtimes
      wr2yMin <- wr2yMin/num.leadtimes
      wr3yMin <- wr3yMin/num.leadtimes
      wr4yMin <- wr4yMin/num.leadtimes

      cat("Computing regime anomalies and frequencies. Please wait......\n")

      # in this case, must use psl.melted instead of psleuFull. Both are already in anomalies:
      # (psl.melted depends on the startdate and leadtime, psleuFull no)
      gc()
      pslmat <- unname(acast(psl.melted, Member + StartYear + LeadDay ~ Lat ~ Lon))
      #pslmat <- unname(acast(melt(pslPeriod[1,1:2,,,,, drop=FALSE],varnames=c("Exp","Member","StartYear","LeadDay","Lat","Lon")), Member ~ StartYear + LeadDay ~ Lat ~ Lon))
      gc()

      #for(a in 1:2){
      #    for(b in 1:3){
      #        for(c in 1:4){
      #             pslmat[1, a + (b-1)*a + (c-1)*a*b,,] <- pslPeriod[1,a,b,c,,]
      #        }
      #    }
      #}                                 
      
      pslwr1 <- pslmat[wr1,,, drop=F]
      pslwr2 <- pslmat[wr2,,, drop=F]
      pslwr3 <- pslmat[wr3,,, drop=F]
      pslwr4 <- pslmat[wr4,,, drop=F]

      pslwr1mean <- apply(pslwr1, c(2,3), mean, na.rm=T)
      pslwr2mean <- apply(pslwr2, c(2,3), mean, na.rm=T)
      pslwr3mean <- apply(pslwr3, c(2,3), mean, na.rm=T)
      pslwr4mean <- apply(pslwr4, c(2,3), mean, na.rm=T)

      # spatial mean for each day to save for the meaure of the FairRPSS:
      pslwr1spmean <- apply(pslwr1, 1, mean, na.rm=T)
      pslwr2spmean <- apply(pslwr2, 1, mean, na.rm=T)
      pslwr3spmean <- apply(pslwr3, 1, mean, na.rm=T)
      pslwr4spmean <- apply(pslwr4, 1, mean, na.rm=T)
      
      rm(pslmat)
      gc()

      # save all the data necessary to draw the graphs (but not the impact maps)
      save(my.cluster, my.cluster.array, lon, lon.max, lat, pos.lat, pos.lon, n.pos.lat, n.pos.lon, psl, psl.name, my.period, p, workdir,rean.name,fields.name, year.start,year.end, years, n.years, n.years.full, WR.period, pslwr1mean, pslwr2mean, pslwr3mean, pslwr4mean, pslwr1spmean, pslwr2spmean, pslwr3spmean, pslwr4spmean, wr1y, wr2y, wr3y, wr4y, wr1yMax, wr2yMax, wr3yMax, wr4yMax, wr1yMin, wr2yMin, wr3yMin, wr4yMin, n.leadtimes, num.leadtimes, file=paste0(workdir,"/",fields.name,"_",my.period[p],"_leadmonth_",lead.month,"_psl.RData"))
      
      rm(pslwr1mean,pslwr2mean,pslwr3mean,pslwr4mean)
      rm(pslwr1,pslwr2,pslwr3,pslwr4)
      gc()   
      
    }

    # for the monthly system, the time order is always:  year1day1, year2day1, ... , yearNday1, year1day2, year2day2, ..., yearNday2, etc.:
    if(fields.name == ECMWF_monthly.name) {
       if(fields.name == ECMWF_monthly.name){
       my.startdates.period <- months.period(forecast.year,mes,day,p)

       #if(p == 13) my.startdates.period <- my.startdates.period[-2]  # remove the second startdate because it is broken
       
       days.period <- period.length <- list()
       for(startdate in my.startdates.period){
          days.period[[p]] <- c(days.period[[p]], (1+(startdate-1)*(year.end-year.start+1)):(startdate*(year.end-year.start+1)))
       }
       period.length[[p]] <- length(my.startdates.period) # number of Thusdays in the period
      }

      wr1y <- wr2y <- wr3y <-wr4y <- c()
      wr1y[y-year.start+1] <- length(which(cluster.sequence[seq((y-year.start+1),length(cluster.sequence), n.years)] == 1))
      wr2y[y-year.start+1] <- length(which(cluster.sequence[seq((y-year.start+1),length(cluster.sequence), n.years)] == 2))
      wr3y[y-year.start+1] <- length(which(cluster.sequence[seq((y-year.start+1),length(cluster.sequence), n.years)] == 3))
      wr4y[y-year.start+1] <- length(which(cluster.sequence[seq((y-year.start+1),length(cluster.sequence), n.years)] == 4))
    }

     cat("Finished!\n")
} # close the for loop on 'p' 





















