## Note that you have to divide by 4 the GB of data loaded, because in the NetCDF file they are 4 times smaller
## to open ecFlow GUI from terminal:
## ecFlow_vi &

### load data and interpolate it to the same grid of an existing NetCDF:

library(startR, lib.loc = '/gpfs/projects/bsc32/share/R_libs/3.5/')

obs.path <- "/esarchive/recon/ecmwf/erainterim/monthly_mean/$var$_f6h/$var$_$sdate$.nc"

obs <- Start(dat = obs.path,
             var = 'tas',
             sdate = paste0(2005:2007, '11'),
             time = 'all',
             latitude = values(list(-90, 90)),
             longitude = values(list(0, 359.99)),
             latitude_reorder = Sort(),
             longitude_reorder = CircularSort(0, 360),
             transform = CDORemapper,
             transform_params = list(
               grid = paste0('/esarchive/exp/ecmwf/system5c3s/',
                             'monthly_mean/tas_f6h/tas_20181101.nc'),
               method = 'con'
             ),
             transform_vars = c('latitude', 'longitude'),
             return_vars = list(latitude = 'dat',
                                longitude = 'dat'),
             retrieve = TRUE)

lons <- attr(obs, 'Variables')$dat1$longitude
lats <- attr(obs, 'Variables')$dat1$latitude

library(s2dverification, lib.loc = '/gpfs/projects/bsc32/share/R_libs/3.5/')

PlotLayout(PlotEquiMap, c('longitude', 'latitude'), obs, lons, lats)





### IN ORDER TO LOAD DATA DIRECTLY FROM ESARCHIVE:
#  - comment the parameters 'data_dir' and 'special_setup'
# IN ORDER TO TRANSFER DATA ONTO GPFS WITH RSYNC AND LOAD FROM THERE:
#  - comment only the parameter 'special_setup'
# IN ORDER TO TRANSFER DATA ONTO GPFS WITH DTRSYNC AND LOAD FROM THERE:
#  - leave all uncommented
# BE AWARE YOUR TARGET DATA MAY BE ALREADY CACHED ON 'data_dir'
#  - if you want to trigger transfer, remove it or specify a new 'data_dir'

## module load R CDO ecFlow
## library(startR)                         

# Declare the data source
##repos <- '/esarchive/exp/ecmwf/system5_m1/6hourly/$var$/$var$_$sdate$.nc'
##data <- Start(dat = repos,
##              var = 'tas',
##              sdate = '19810101',   # PUT ERROR MESSAGE IF USING DATA_DIR AND indices()
##              ensemble = 'all',
##              time = 'all',
##              latitude = values(list(10, 15)),
##              latitude_reorder = Sort(),
##              longitude = values(list(30, 35)),
##              return_vars = list(longitude = NULL,
##                                 latitude = NULL))

# Build a workflow
##fun <- function(x) {
##  r <- apply(x + 1, 2, mean)
##  first <- 1
##  last <- length(r)
##  mid <- ceiling(length(r)/2)
##  q2 <- ceiling(mid/2)
##  q3 <- mid + q2
##  array(
##    c(mean(r[first:(q2 - 1)]), mean(r[q2:(mid - 1)]),
##      mean(r[mid:(q3 - 1)]), mean(r[q3:last])),
##    dim = c(time = 4)
##  )
##}
##step <- Step(fun, 
##             target_dims = c('ensemble', 'time'), 
##             output_dims = c('time'))
##wf <- AddStep(data, step)

# Define HPC and computation parameters, and run
##res <- Compute(wf,
##               chunks = list(sdate = 1, latitude = 1), 
##               threads_load = 1,
##               threads_compute = 1,
##               cluster = list(queue_host = 'nic9',
##                              queue_type = 'slurm',
##                              data_dir = '/gpfs/projects/bsc32/share/startR_data_repos/',
##                              temp_dir = '/gpfs/scratch/bsc32/bsc32842/startR_tests/',
##                              lib_dir = '/gpfs/projects/bsc32/share/R_libs/3.5/',
##                              r_module = 'R/3.5.0-foss-2018b',
##                              cores_per_job = 1,
##                              job_wallclock = '4:00:00',
##                              max_jobs = 1,
##                              extra_queue_params = list('#SBATCH --mem-per-cpu=20000'),
##                              bidirectional = FALSE,
##                              polling_period = 10,
##                              special_setup = 'marenostrum4'
##                             ),
##               ecflow_suite_dir = '/home/Earth/ncortesi/tests_startr/',
##               wait = FALSE)

## to load from esarchive:
##res <- Compute(wf,
##               chunks = list(sdate = 1, latitude = 1), 
##               threads_load = 1,
##               threads_compute = 1,
##               cluster = list(queue_host = 'nic9',
##                              queue_type = 'slurm',
##                              temp_dir = '/gpfs/scratch/bsc32/bsc32842/startR_tests/',
##                              lib_dir = '/gpfs/projects/bsc32/share/R_libs/3.5/',
##                              r_module = 'R/3.5.0-foss-2018b',
##                              cores_per_job = 1,
##                              job_wallclock = '4:00:00',
##                              max_jobs = 1,
##                              extra_queue_params = list('#SBATCH --mem-per-cpu=20000'),
##                              bidirectional = FALSE,
##                              polling_period = 10,
##                             ),
##               ecflow_suite_dir = '/home/Earth/ncortesi/tests_startr/',
##               wait = FALSE)

## result <- Collect(res, wait = TRUE)


#module load R CDO ecFlow
library(startR)

# Declare the data source
repos <- '/esarchive/exp/ecmwf/monthly_ensforhc/6hourly/$var$/$year$/$var$_$sdate$.nc'
data <- Start(dat = repos,
              var = 'psl',
              year = '20180101',
              sdate = indices(1:8),   # PUT ERROR MESSAGE IF USING DATA_DIR AND indices()
              ensemble = 'all',
              time = indices(1:10),
              latitude = 'all', #values(list(10, 15)),
              latitude_reorder = Sort(),
              longitude = 'all', #values(list(30, 35)),
              return_vars = list(longitude = NULL,
                                 latitude = NULL))

# Build a workflow
fun <- function(x) {
  array(0, dim = c(time = 1))
  ##array(0, dim = c(time = 1))
}

step <- Step(fun, 
             target_dims = c('ensemble', 'time'), 
             output_dims = c('time'))

wf <- AddStep(data, step)

# Define HPC and computation parameters, and run
## res <- Compute(wf,
##                chunks = list(sdate = 1, latitude = 1), 
##                threads_load = 1,
##                threads_compute = 1,
##                cluster = list(queue_host = 'nic9',
##                               queue_type = 'slurm',
##                               data_dir = '/gpfs/projects/bsc32/share/startR_data_repos/',
##                               temp_dir = '/gpfs/scratch/bsc32/bsc32842/startR_tests/',
##                               lib_dir = '/gpfs/projects/bsc32/share/R_libs/3.5/',
##                               r_module = 'R/3.5.0-foss-2018b',
##                               cores_per_job = 1,
##                               job_wallclock = '4:00:00',
##                               max_jobs = 1,
##                               extra_queue_params = list('#SBATCH --mem-per-cpu=20000'),
##                               bidirectional = FALSE,
##                               polling_period = 10,
##                               special_setup = 'marenostrum4'
##                              ),
##                ecflow_suite_dir = '/esarchive/scratch/ncortesi/tests_startr/',
##                wait = FALSE)

## to load from /esarchive:
res <- Compute(wf,
               chunks = list(sdate = 1, latitude = 1), 
               threads_load = 1,
               threads_compute = 1,
               cluster = list(queue_host = 'nic9',
                              queue_type = 'slurm',
                              temp_dir = '/gpfs/scratch/bsc32/bsc32842/startR_tests/',
                              lib_dir = '/gpfs/projects/bsc32/share/R_libs/3.5/',
                              r_module = 'R/3.5.0-foss-2018b',
                              cores_per_job = 1,
                              job_wallclock = '4:00:00',
                              max_jobs = 1,
                              extra_queue_params = list('#SBATCH --mem-per-cpu=20000'),
                              bidirectional = FALSE,
                              polling_period = 10
                             ),
               ecflow_suite_dir = '/esarchive/scratch/ncortesi/tests_startr/',
               wait = FALSE)

result <- Collect(res, wait = TRUE)

## times from /gpfs: 240 241 239 223
