library(startR)

path <- '/esarchive/exp/monarch/t0kc/regional/3hourly/'
vars <- list.files(path)

path <- list(list(name = 'raw', 
                  path = paste0(path, '$var$/$var$-$member$_$date$03.nc')),
             list(name = 'ana',
                  path = paste0(path, '$var$/$var$-$member$_$date$03_an.nc')))
date <- '20131221'
startR:::.warning("Data reading...")
#for (variable in vars) {
#variable = vars[1]
data <- Start(dat = path[[1]],
              var = vars,
              date = date,
              member = 'all',
              time = 'all',
              lev = 'all',
              rlat = 'all',
              rlon = 'all',
              retrieve = FALSE, num_procs = 1)
time1 <- Sys.time()
startR:::.warning("Data point")

stats <- function(x, probs = 0.75) {
  # x has member dimension
  # more than one prob is possible
  perc <- quantile(as.vector(x), probs)
  av <- mean(x, na.rm = TRUE) # to be confirmed
  sd <- sd(x, na.rm = TRUE)
  max <- max(x, na.rm = TRUE)
  median <- median(x, na.rm = TRUE)
  res <- array(c(av, sd, median, max, perc), c(stats = (4 + length(probs))))
  return(res)
}

step <- Step(fun = stats,
               target_dims = 'member',
               output_dims = 'stats')
  
workflow <- AddStep(data, step)

 queue_host = 'nord3'   #your own host name for power9
  temp_dir =  '/gpfs/scratch/bsc32/bsc32339/startR_hpc/'
  ecflow_suite_dir = '/home/Earth/nperez/startR_local/'  #your own local directory
#------------------------------------------------------------
  res <- Compute(workflow,
                 chunks = list(var = 28, time = 8),
                 threads_load = 1,
                 threads_compute = 16,
                 cluster = list(queue_host = queue_host,
                                queue_type = 'lsf',
                                extra_queue_params = list('#BSUB -q bsc_debug'),
                                cores_per_job = 16,
                                temp_dir = temp_dir,
                                polling_period = 10,
                                job_wallclock = '03:00',
                                max_jobs = 224,
                                bidirectional = FALSE),
                 ecflow_suite_dir = ecflow_suite_dir,
                 wait = TRUE, debug = TRUE)


