# I don't know how to create a bigdata object.
final_dims = c(a = 5, b = 2)
data_array <- bigmemory::big.matrix(nrow = prod(final_dims), ncol = 1)
shared_matrix_pointer <- bigmemory::describe(data_array)
class(data_array)
rm(list=ls())
getwd()
setwd("/esarchive/scratch/nperez/git/Flor/Nord3/")
library(startR)
source("../source_dir.R")
source("../source_lines.R")
source_dir("/esarchive/scratch/nperez/git/Flor/Nord3/R/R/")

library(startR)
data <- Start(dataset = '/esarchive/exp/ecearth/a1ua/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/$member$/day/$var$/gr/v20190713/$var$_day_EC-Earth3_dcppA-hindcast_s$sdate$-$member$_gr_$fyear$.nc',
              var = 'tasmin',
              sdate = paste0(1961),
              time = 'first',
              lat = values(list(-90,90)),
              lon = values(list(0,359.9)),
              fyear = indices(1:5),
              member = 'all',
              fyear_depends = 'sdate',
              return_vars = list(lat = 'dataset', lon = 'dataset'),
              num_procs = 1, retrieve = FALSE)

#names(dim(data)) <- "A"
print("IS working")

fun <- function(data){
  data <- multiApply::Apply(data = data, target_dims = c('time','fyear'), fun = mean, ncores = 4)$output1
  return(data)
}

step <- Step(fun = fun,
             use_libraries = c('startR','multiApply'),
             target_dims = c('dataset','var','time','fyear'), 
             output_dims = c('dataset','var'))

wf = AddStep(inputs = data, step_fun = step)

 queue_host = 'nord3'   #your own host name for power9
  temp_dir =  '/gpfs/scratch/bsc32/bsc32339/startR_hpc/'
  ecflow_suite_dir = '/home/Earth/nperez/startR_local/'  #your own local directory

res = Compute(workflow = wf,
              chunks = list(member = 10),
              threads_load = 2,
              threads_compute = 8,
              cluster = list(queue_host = 'nord3',
                             queue_type = 'lsf',
                             temp_dir = temp_dir,
                             cores_per_job = 8,
                             job_wallclock = '02:00',
                             max_jobs = 10,
                             extra_queue_params = list('#BSUB -q bsc_es -M 3000'), # node exclusive: #BSUB -x
                             bidirectional = FALSE,
                             polling_period = 10),
              ecflow_suite_dir = ecflow_suite_dir,
              wait = TRUE)





