library(multiApply)          # Apply() function
library(startR)              # Start() to load data

source('/home/Earth/aho/aho-testtest/startR/issue/issue23/ByChunks.R')
source('/home/Earth/aho/aho-testtest/startR/issue/issue23/Compute.R')

exp_path <- paste0('/esarchive/exp/ncep/cfs-v2/weekly_mean/s2s/$var$_f24h/',
                   '$var$_$sdate$.nc')
obs_path <- paste0("/esarchive/recon/ecmwf/era5/weekly_mean/",
                   "$var$_f1h/$var$_$file_date$.nc")

hcst.sdates <- c("19990104", "19990107", "19990111", "19990114", "19990118", "19990121", "19990125", "19990128", "20000104", "20000107", "20000111", "20000114",
"20000118", "20000121", "20000125", "20000128")


exp <- Start(dat = exp_path, 
             var = 'tas',
             sdate = hcst.sdates,   #YYYYMMDD
             time = indices(1:4),
             latitude =  indices(1),#values(list(lats.min, lats.max)),
             longitude = indices(1),#values(list(lons.min, lons.max)),
             ensemble = 'all',
             return_vars = list(lat = NULL,
                                lon = NULL,
                                time = c('sdate')),
             synonims = list(latitude = c('lat', 'latitude'),
                             longitude = c('lon', 'longitude')), 
             retrieve = F)

# Obtain dates to load from obs
dates <- attr(exp, 'Variables')$common$time
file_date <- sapply(dates, format, '%Y%m%d')
dim(file_date) <- dim(dates)

obs <- Start(dat = obs_path, 
             var = 'tas',
             file_date = file_date,  # dim('sdate','time') 
             latitude = indices(1),  #values(list(lats.min, lats.max)),
             longitude = indices(1),  #values(list(lons.min, lons.max)),
             return_vars = list(latitude = 'dat',
                                     longitude = 'dat',
                                     time = 'file_date'),
             split_multiselected_dims = TRUE,
             retrieve = F)


score_calc <- function(forecast, reference){

#source('/home/Earth/aho/aho-testtest/startR/issue/issue23/ByChunks.R')  #fat #not work
#source('/home/Earth/aho/aho-testtest/startR/issue/issue23/Compute.R')
#source('/home/bsc32/bsc32734/ByChunks_issue23.R')   #p9
#source('/home/bsc32/bsc32734/Compute_issue23.R')

  em <- Apply(forecast, 'ensemble', mean, na.rm = TRUE)[[1]]
  bias <- Apply(reference, c('sdate'), function(x) x - em)[[1]]
  dim(bias) <- c(length(bias))
  return(bias)

}

step <- Step(score_calc,
             list(forecast = c('sdate', 'ensemble'),
                  reference = c('sdate')),
             list(bias = c('sdate')),
                 use_libraries = c('multiApply'))

wf <- AddStep(list(exp, obs), step)

res <- Compute(wf,    
               chunks = list(time = 4)) #,
                        #     latitude = 1,
                        #     longitude = 1))


#Compute() on fatnodes
#  res <- Compute(wf,
#                 #chunks = list(latitude = 2,
#                 #              longitude = 2),#$output1
#                 chunks = list(time = 4),
#                 threads_load = 2,
#                 threads_compute = 4,
#                 cluster = list(queue_host = 'fat',
#                                queue_type = 'slurm',
#                                cores_per_job = 2,
#                                r_module = '/home/Earth/aho/aho-testtest/startR/issue/issue23/',
#                                job_wallclock = '00:10:00',
#                                max_jobs = 4,
#                                bidirectional = TRUE
#                 ),
#                 ecflow_suite_dir = '/home/Earth/aho/startR_local/',
#                 wait = TRUE)


#-------------------user-defined---------------------
  queue_host <- 'p1'
  temp_dir <- '/gpfs/scratch/bsc32/bsc32734/startR_hpc/'
  ecflow_suite_dir <- '/home/Earth/aho/startR_local/'
#----------------------------------------------------

#   res <- Compute(wf,
#                  chunks = list(time = 4),
#                   #chunks = list(latitude = 2,
#                   #              longitude = 2), $output1
#                  threads_load = 2,
#                  threads_compute = 4,
#                  cluster = list(queue_host = queue_host,
#                                 queue_type = 'slurm',
#                                 temp_dir = temp_dir,
#                                 #lib_dir = '/gpfs/projects/bsc32/share/R_libs/3.5/',
#                                 r_module = 'R/3.5.0-foss-2018b',
#                                 #r_module = 'startR/0.1.2-foss-2018b-R-3.5.0',
#                                 job_wallclock = '00:10:00',
#                                 cores_per_job = 4,
#                                 max_jobs = 4,
#                                 bidirectional = FALSE,
#                                 polling_period = 50
#                                ),
#                  ecflow_suite_dir = ecflow_suite_dir,
#                  #ecflow_suite_dir = '/esarchive/scratch/aho/ecFlow/',
#                  wait = TRUE
#                  )   $output1

