  library(startR)
  
# exp data
  repos <- paste0('/esarchive/exp/ecmwf/system4_m1/monthly_mean/',
                  '$var$_f6h/$var$_$sdate$.nc')
  sdates <- sapply(2012:2016, function(x) paste0(x, sprintf('%02d', 1:12), '01'))

  exp <- Start(dat = repos,
                     var = 'tas',
                     sdate = sdates,
                     time = indices(1),
                     ensemble = 'all',
                     latitude = 'all',
                     longitude = 'all',
                     synonims = list(longitude = c('lon', 'longitude'),
                                     latitude = c('lat', 'latitude')),
                     retrieve = F)

# obs data
  repos_obs <- paste0('/esarchive/recon/ecmwf/erainterim/monthly_mean/',
                      '$var$_f6h/$var$_$sdate$.nc')
  sdates_obs <- (sapply(2012:2016, function(x) paste0(x, sprintf('%02d', 1:12))))

  obs <- Start(dat = repos_obs,
                     var = 'tas',
                     sdate = sdates_obs,
                     time = indices(1),
                     latitude = 'all',
                     longitude = 'all',
                     synonims = list(longitude = c('lon', 'longitude'),
                                     latitude = c('lat', 'latitude')),
                     retrieve = F)


  func <- function(x, y) {
    crps <- mean(SpecsVerification::EnsCrps(x, y, R.new = Inf), na.rm = T)
    return(crps)
  }
  step <- Step(func, target_dims = list(c('sdate', 'ensemble'), c('sdate')),
                     output_dims = NULL)
  wf <- AddStep(list(exp, obs), step)
 

# Compute() on fatnodes
  res <- Compute(wf,
                 chunks = list(latitude = 2,
                               longitude = 2),
                 threads_load = 2,
                 threads_compute = 4,
                 cluster = list(queue_host = 'fat',
                                queue_type = 'slurm',
                                cores_per_job = 2,
                                job_wallclock = '01:10:00',
                                max_jobs = 4,
                                bidirectional = TRUE
                 ),
                 ecflow_suite_dir = '/home/Earth/aho/startR_local/'
         )


# Compute() on Power 9
 res <- Compute(wf,
                 chunks = list(latitude = 2,
                               longitude = 2),
                 threads_load = 2,
                 threads_compute = 4,
                 cluster = list(queue_host = 'p1',
                                queue_type = 'slurm',
                                temp_dir = '/gpfs/scratch/bsc32/bsc32734/startR_hpc/',
                                r_module = 'R/3.5.0-foss-2018b',
                                job_wallclock = '00:30:00',
                                cores_per_job = 4,
                                max_jobs = 4,
                                bidirectional = FALSE,
                                polling_period = 50),
                 ecflow_suite_dir = '/home/Earth/aho/startR_local/',
                 wait = TRUE
                 )

# Compute() on Nord 3
 res <- Compute(wf,
                 chunks = list(latitude = 2,
                               longitude = 2),
                 threads_load = 2,
                 threads_compute = 4,
                 cluster = list(queue_host = 'nord3',
                                queue_type = 'lsf',
                                temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/',
                                job_wallclock = '00:30',
                                extra_queue_params = list('#BSUB -q bsc_es'),
                                cores_per_job = 4,
                                max_jobs = 4,
                                bidirectional = FALSE,
                                polling_period = 50),
                 ecflow_suite_dir = '/home/Earth/nperez/startR_local/',
                 wait = TRUE)
# Results
 dim(res$output1)
#      dat       var      time  latitude longitude 
#        1         1         1       256       512 
 summary(res$output1)
#   Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
#0.09882 0.37815 0.71648 0.83638 1.20353 6.23452 


# Plotting
library(s2dverification)

  lon <- seq(from = 0, to = 359, length.out = 512)
  lat <- seq(from = 90, to = -90, length.out = 256)

  brks <- seq(-0.1, 4, length.out = 11)
  PlotEquiMap(res$output1[1, 1, 1, , ], lon, lat,
              color_fun = clim.palette('yellowred'), brks = brks,
              filled.continents = FALSE, triangle_ends = c(TRUE, TRUE),
              toptitle = 'ECMWF monthly mean tas CRPS 2012-2016', title_scale = 0.6)


