library(startR)

path <- '/esarchive/oper/thredds-dust/monarch-dustclim/3hourly/$var$-av_an/$var$_$year$$month$$day$03_av_an.nc'

data <- Start(dataset = path,
                     var = 'od550du',
                     year = as.character(2007:2008),
  month = sprintf('%02d', 1:2),
  day = sprintf('%02d', 1:2),
  time = 'all',
  rlat = 'all', #indices(100:200),
  rlon = 'all', #indices(200:300),
  return_vars = list(time = NULL,
  rlat = NULL, rlon = NULL,
  lat = NULL, lon = NULL),
  retrieve = FALSE, num_procs = 1)   

Statistics <- function(data, probs = c(0.25, 0.95)) {
  prom <- mean(data, na.rm = TRUE)
  perc <- quantile(data, probs = probs)
  sd <- sd(data, na.rm = TRUE)
  res <- array(c(prom, perc, sd), c(stats = 4))
  return(res)
}
## Test the function with Apply:
#res <- Apply(list(data), target_dims = c('month', 'day', 'time'), fun = Statistics, ncores = 4)


step <- Step(fun = Statistics,
               target_dims = c('month', 'day', 'time'),
               output_dims = 'stats')
  
workflow <- AddStep(data, step)
  
  res <- Compute(workflow,
                 chunks = list(year = 2),
                 threads_load = 1,
                 threads_compute = 4)

#-----------modify according to your personal info---------
  queue_host = 'nord3'   #your own host name for power9
  temp_dir =  '/gpfs/scratch/bsc32/bsc32339/startR_hpc/'
  ecflow_suite_dir = '/home/Earth/nperez/startR_local/'  #your own local directory
#------------------------------------------------------------
  res <- Compute(workflow,
                 chunks = list(year = 2),
                 threads_load = 1,
                 threads_compute = 4,
                 cluster = list(queue_host = queue_host,
                                queue_type = 'lsf',
                                extra_queue_params = list('#BSUB -q debug'),
                                cores_per_job = 4,
                                temp_dir = temp_dir,
                                polling_period = 10,
                                job_wallclock = '01:00',
                                max_jobs = 2,
                                bidirectional = FALSE),
                 ecflow_suite_dir = ecflow_suite_dir,
                 wait = TRUE, debug = TRUE)

