# Load startR
library(startR)

# Define a region
lons.min <- 0
lons.max <- 10
lats.min <- 0
lats.max <- 10

# Declaration of data sources
exp <- Start(dat = '/esarchive/exp/ecmwf/system4_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc',
             var = 'tas',
             sdate = paste0(2000:2010, '0101'),
             ensemble = 'all',
             time = indices(1),
             latitude = values(list(lats.min, lats.max)),
             latitude_reorder = Sort(),
             longitude = values(list(lons.min, lons.max)),
             synonims = list(latitude = c('lat', 'latitude'),
                             longitude = c('lon', 'longitude')),
             return_vars = list(latitude = 'dat',
                                longitude = 'dat',
                                time = c('sdate')),
             retrieve = FALSE)

obs <- Start(dat = '/esarchive/recon/ecmwf/erainterim/monthly_mean/$var$_f6h/$var$_$sdate$.nc',
             var = 'tas',
             sdate = paste0(2000:2010, '02'),
             time = indices(1),
             latitude = values(list(lats.min, lats.max)),
             latitude_reorder = Sort(),
             longitude = values(list(lons.min, lons.max)),
             synonims = list(latitude = c('lat', 'latitude'),
                             longitude = c('lon', 'longitude')),
             return_vars = list(latitude = 'dat',
                                longitude = 'dat',
                                time = c('sdate')),
             retrieve = FALSE)

# Define of the workflow

# Function
wrap_cal <- function(obs, exp) {
    obs <- s2dv::InsertDim(obs, 1, 1, name = 'ensemble')
    #calibrated <- CSTools:::.cal(obs = obs, var_exp = exp) # CSTools version 1.0.1 or earlier
     calibrated <- CSTools:::.cal(exp = exp, obs = obs,
                                 cal.method = "mse_min",
                                 eval.method = "leave-one-out",
                                 multi.model = FALSE,
                                 na.fill = TRUE, na.rm = TRUE,
                                 apply_to = 'sign', alpha = 0.1)
    return(calibrated)
}

step <- Step(wrap_cal,
             target_dims = list(obs = c('sdate'), exp = c('ensemble', 'sdate')),
             output_dims = c('ensemble', 'sdate'))

# workflow of operations
wf <- AddStep(list(obs = obs, exp = exp), step)


# Execution
res <- Compute(wf, chunks = list(latitude = 2, longitude = 2),
               threads_load = 2, threads_compute = 4)

# Declaration of HPC and execution
## ECFlow is required
# On Fatnodes
res_fat2 <- Compute(wf,
                    chunks = list(latitude = 2, longitude = 2),
                    threads_load = 2,
                    threads_compute = 4,
                    cluster = list(queue_host = "bsceslogin01.bsc.es",
                                   cores_per_job = 2,
                                   max_jobs = 4, job_wallclock = '00:10:00'),
                    ecflow_suite_dir = "/esarchive/scratch/nperez/ecflow") # your path!

# On Nord3
res_nord3 <- Compute(wf,
                    chunks = list(latitude = 2, longitude = 2),
                    threads_load = 2,
                    threads_compute = 4,
                    cluster = list(queue_host = "nord3",
                                   queue_type = 'lsf',
                                   extra_queue_params = list('#BSUB -q bsc_es'),
                                   cores_per_job = 2,
                                   temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/',
                                   polling_period = 10,
                                   job_wallclock = '01:00',
                                   max_jobs = 4,
                                   bidirectional = FALSE),
                    ecflow_suite_dir = "/esarchive/scratch/nperez/ecflow") # your path!

# Results
dim(res$output1)
# ensemble     sdate       dat       var      time  latitude longitude 
#       15        11         1         1         1        14        15 
summary(res$output1)
#   Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
#  293.1   300.6   301.1   301.1   301.6   306.8 

