diff --git a/inst/doc/usecase/ex2_1_timedim.R b/inst/doc/usecase/ex2_1_timedim.R index 2365e00bf8b5e349d23964ebf75f416759ef3acc..f06b766dd1fff0db1ef3a02b0c1005ad376e2752 100644 --- a/inst/doc/usecase/ex2_1_timedim.R +++ b/inst/doc/usecase/ex2_1_timedim.R @@ -1,7 +1,7 @@ # ----------------------------------------------------------------- # Function working on time dimension e.g.: Season # ------------------------------------------------------------------ - +library(startR) repos <- '/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc' data <- Start(dat = repos, var = 'tas', @@ -47,9 +47,9 @@ ## on Power9 #-----------modify according to your personal info--------- - queue_host = 'p1' #your own host name for power9 - temp_dir = '/gpfs/scratch/bsc32/bsc32734/startR_hpc/' - ecflow_suite_dir = '/home/Earth/aho/startR_local/' #your own local directory + queue_host = 'cte-power' #your own host name for power9 + temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' + ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory #------------------------------------------------------------ res <- Compute(wf1, chunks = list(ensemble = 20, @@ -60,10 +60,34 @@ queue_type = 'slurm', cores_per_job = 2, temp_dir = temp_dir, - r_module = 'R/3.5.0-foss-2018b', + r_module = 'R/3.6.1-foss-2018b', polling_period = 10, job_wallclock = '01:00:00', max_jobs = 40, bidirectional = FALSE), ecflow_suite_dir = ecflow_suite_dir, wait = TRUE) + +## on Nord3 +#-----------modify according to your personal info--------- + queue_host = 'nord3' #your own host name for power9 + temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' + ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory +#------------------------------------------------------------ + res <- Compute(wf1, + chunks = list(ensemble = 20, + sdate = 2), + threads_load = 2, + threads_compute = 4, + cluster = list(queue_host = queue_host, + queue_type = 'lsf', + extra_queue_params = list('#BSUB -q bsc_es'), + cores_per_job = 2, + temp_dir = temp_dir, + polling_period = 10, + job_wallclock = '01:00', + max_jobs = 40, + bidirectional = FALSE), + ecflow_suite_dir = ecflow_suite_dir, + wait = TRUE) + diff --git a/inst/doc/usecase/ex2_2_attr.R b/inst/doc/usecase/ex2_2_attr.R index 4a73ae2f48c7f6e6cb798e76ce6c54a0e3c00355..f06ec46c76128aa5303468eccb220dfcf228fcea 100644 --- a/inst/doc/usecase/ex2_2_attr.R +++ b/inst/doc/usecase/ex2_2_attr.R @@ -1,7 +1,7 @@ #----------------------------------------------------------------- # Function using attributes of the data e.g.: latitudinal correction # ------------------------------------------------------------------ - +library(startR) repos <- '/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc' data <- Start(dat = repos, var = 'tas', @@ -27,6 +27,7 @@ wf2 <- AddStep(data, step2) ## locally +library(multiApply) res2 <- Compute(workflow = wf2, chunks = list(ensemble = 2, sdate = 2)) @@ -48,9 +49,9 @@ ## on Power9 #-----------modify according to your personal info--------- - queue_host = 'p1' #your own host name for power9 - temp_dir = '/gpfs/scratch/bsc32/bsc32734/startR_hpc/' - ecflow_suite_dir = '/home/Earth/aho/startR_local/' #your own local directory + queue_host = 'cte-power' #your own host name for power9 + temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' + ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory #------------------------------------------------------------ res2_P <- Compute(wf2, chunks = list(ensemble = 20, @@ -61,7 +62,7 @@ queue_type = 'slurm', cores_per_job = 2, temp_dir = temp_dir, - r_module = 'R/3.5.0-foss-2018b', + r_module = 'R/3.6.1-foss-2018b', #extra_queue_params = list('#SBATCH --mem-per-cpu=3000'), polling_period = 10, job_wallclock = '01:00:00', @@ -69,3 +70,26 @@ bidirectional = FALSE), ecflow_suite_dir = ecflow_suite_dir, #your own local directory wait = TRUE) + +## on Nord3 +#-----------modify according to your personal info--------- + queue_host = 'nord3' #your own host name for power9 + temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' + ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory +#------------------------------------------------------------ + res2_Nord <- Compute(wf2, + chunks = list(ensemble = 20, + sdate = 2), + threads_load = 2, + threads_compute = 4, + cluster = list(queue_host = queue_host, #your own host name for power9 + queue_type = 'lsf', + cores_per_job = 2, + temp_dir = temp_dir, + extra_queue_params = list('#BSUB -q bsc_es'), + polling_period = 10, + job_wallclock = '01:00', + max_jobs = 40, + bidirectional = FALSE), + ecflow_suite_dir = ecflow_suite_dir, #your own local directory + wait = TRUE) diff --git a/inst/doc/usecase/ex2_3_cdo.R b/inst/doc/usecase/ex2_3_cdo.R index cd7d4a368dc5c9964adbab98ef884ccb25903ba9..0262de38a50ee259196d275abfe4ddef56d940bb 100644 --- a/inst/doc/usecase/ex2_3_cdo.R +++ b/inst/doc/usecase/ex2_3_cdo.R @@ -1,7 +1,7 @@ # -------------------------------------------------------------- # Function doing regridding CDO (e.g.: s2dverification::CDORemap) #--------------------------------------------------------------- - +library(startR) repos <- '/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc' data <- Start(dat = repos, var = 'tas', @@ -51,9 +51,9 @@ ## on Power9 #-----------modify according to your personal info--------- - queue_host = 'p1' #your own host name for power9 - temp_dir = '/gpfs/scratch/bsc32/bsc32734/startR_hpc/' - ecflow_suite_dir = '/home/Earth/aho/startR_local/' #your own local directory + queue_host = 'cte-power' #your own host name for power9 + temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' + ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory #------------------------------------------------------------ res3_P <- Compute(wf3, chunks = list(ensemble = 2, @@ -74,3 +74,27 @@ ecflow_suite_dir = ecflow_suite_dir, #your own local directory wait = TRUE) + +## on Nord 3 +#-----------modify according to your personal info--------- + queue_host = 'nord3' #your own host name for power9 + temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' + ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory +#------------------------------------------------------------ + res3_N <- Compute(wf3, + chunks = list(ensemble = 2, + sdate = 2), + threads_load = 2, + threads_compute = 4, + cluster = list(queue_host = queue_host, #your own host name for power9 + queue_type = 'lsf', + cores_per_job = 1, + temp_dir = temp_dir, + CDO_module = 'CDO', + extra_queue_params = list('#BSUB -q bsc_es'), + polling_period = 100, + job_wallclock = '01:00', + max_jobs = 4, + bidirectional = FALSE), + ecflow_suite_dir = ecflow_suite_dir, #your own local directory + wait = TRUE) diff --git a/inst/doc/usecase/ex2_4_two_func.R b/inst/doc/usecase/ex2_4_two_func.R index 799bd3f677f737bd9b0e37655b7e398beba11cb3..4f3da94e554a396c29aa6f7a84646787ac103934 100644 --- a/inst/doc/usecase/ex2_4_two_func.R +++ b/inst/doc/usecase/ex2_4_two_func.R @@ -1,7 +1,7 @@ # -------------------------------------------------------------- # Two functions (e.g.: s2dverification::CDORemap and Season) #--------------------------------------------------------------- - +library(startR) repos <- '/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc' data <- Start(dat = repos, var = 'tas', @@ -52,9 +52,9 @@ ## on Power9 #-----------modify according to your personal info----------- - queue_host = 'p1' #your own host name for power9 - temp_dir = '/gpfs/scratch/bsc32/bsc32734/startR_hpc/' - ecflow_suite_dir = '/home/Earth/aho/startR_local/' #your own local directory + queue_host = 'cte-power' #your own host name for power9 + temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' + ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory #------------------------------------------------------------ res4_P <- Compute(wf4, chunks = list(ensemble = 2, @@ -65,7 +65,7 @@ queue_type = 'slurm', cores_per_job = 1, temp_dir = temp_dir, - r_module = 'R/3.5.0-foss-2018b', + r_module = 'R/3.6.1-foss-2018b', CDO_module = 'CDO/1.9.5-foss-2018b', extra_queue_params = list('#SBATCH --mem-per-cpu=3000'), polling_period = 10, @@ -75,3 +75,26 @@ ecflow_suite_dir = ecflow_suite_dir, #your own local directory wait = TRUE) +## on Nord 3 +#-----------modify according to your personal info----------- + queue_host = 'nord3' #your own host name for power9 + temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' + ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory +#------------------------------------------------------------ + res4_N <- Compute(wf4, + chunks = list(ensemble = 2, + sdate = 2), + threads_load = 2, + threads_compute = 4, + cluster = list(queue_host = queue_host, #your own host name for power9 + queue_type = 'lsf', + cores_per_job = 1, + temp_dir = temp_dir, + CDO_module = 'CDO', + extra_queue_params = list('#SBATCH -q bsc_es'), + polling_period = 10, + job_wallclock = '01:00', + max_jobs = 6, + bidirectional = FALSE), + ecflow_suite_dir = ecflow_suite_dir, #your own local directory + wait = TRUE) diff --git a/inst/doc/usecase/ex2_6_ext_param_func.R b/inst/doc/usecase/ex2_6_ext_param_func.R index 1bd74a852d42c656a9c7503dbb81edb3a554a145..04baeb23b968b36dd6b819d09da1d9e1133a97df 100644 --- a/inst/doc/usecase/ex2_6_ext_param_func.R +++ b/inst/doc/usecase/ex2_6_ext_param_func.R @@ -113,12 +113,29 @@ cluster = list(queue_host = 'cte-power', #user-specific queue_type = 'slurm', temp_dir = '/gpfs/scratch/bsc32/bsc32339/', #user-specific - r_module = 'R/3.5.0-foss-2018b', + r_module = 'R/3.6.1-foss-2018b', cores_per_job = 2, job_wallclock = '10:00:00', max_jobs = 4, bidirectional = FALSE, - polling_period = 10) + polling_period = 10), ecflow_suite_dir = '/esarchive/scratch/nperez/ecFlow', #user-specific - wait = FALSE) + wait = TRUE) + +# ---- On Nord 3: + res <- Compute(workflow$strat, + chunks = list(latitude = 2, longitude = 2), + threads_load = 2, + threads_compute = 2, + cluster = list(queue_host = 'nord3', #user-specific + queue_type = 'lsf', + temp_dir = '/gpfs/scratch/bsc32/bsc32339/', #user-specific + cores_per_job = 2, + extra_queue_params = list('#BSUB -q bsc_es'), + job_wallclock = '10:00', + max_jobs = 4, + bidirectional = FALSE, + polling_period = 10), + ecflow_suite_dir = '/esarchive/scratch/nperez/ecFlow', #user-specific + wait = TRUE) diff --git a/inst/doc/usecase/ex2_7_seasonal_forecast_verification.R b/inst/doc/usecase/ex2_7_seasonal_forecast_verification.R index 7c7d5d84f37ae0ff519f3eecf32ae6d633041fc0..f24fa42a45b4f2a4fad8505bdd74b6019608557c 100644 --- a/inst/doc/usecase/ex2_7_seasonal_forecast_verification.R +++ b/inst/doc/usecase/ex2_7_seasonal_forecast_verification.R @@ -77,6 +77,23 @@ wait = TRUE ) +# Compute() on Nord 3 + res <- Compute(wf, + chunks = list(latitude = 2, + longitude = 2), + threads_load = 2, + threads_compute = 4, + cluster = list(queue_host = 'nord3', + queue_type = 'lsf', + temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/', + job_wallclock = '00:30', + extra_queue_params = list('#BSUB -q bsc_es'), + cores_per_job = 4, + max_jobs = 4, + bidirectional = FALSE, + polling_period = 50), + ecflow_suite_dir = '/home/Earth/nperez/startR_local/', + wait = TRUE) # Results dim(res$output1) # dat var time latitude longitude diff --git a/inst/doc/usecase/ex2_8_calibration.R b/inst/doc/usecase/ex2_8_calibration.R index a8bd575799b644d4c0557c987f1e8f75921f3b0b..d02afc1d63452bf2aa04105c38dc0c19c147fa4a 100644 --- a/inst/doc/usecase/ex2_8_calibration.R +++ b/inst/doc/usecase/ex2_8_calibration.R @@ -43,11 +43,11 @@ wrap_cal <- function(obs, exp) { names(dim(obs))<- c('member', 'sdate') exp <- t(exp) names(dim(exp))<- c('member', 'sdate') - calibrated <- CSTools:::.cal(var_obs = obs, var_exp = exp) # CSTools version 1.0.1 or earlier - # calibrated <- CSTools:::.cal(mod = exp, obs = obs, - # cal.method = "mse_min", - # eval.method = "leave-one-out", - # multi.model = FALSE) + #calibrated <- CSTools:::.cal(obs = obs, var_exp = exp) # CSTools version 1.0.1 or earlier + calibrated <- CSTools:::.cal(exp = exp, obs = obs, + cal.method = "mse_min", + eval.method = "leave-one-out", + multi.model = FALSE) return(calibrated) } @@ -65,6 +65,7 @@ res <- Compute(wf, chunks = list(latitude = 2, longitude = 2), # Declaration of HPC and execution ## ECFlow is required +# On Fatnodes res_fat2 <- Compute(wf, chunks = list(latitude = 2, longitude = 2), threads_load = 2, @@ -74,6 +75,22 @@ res_fat2 <- Compute(wf, max_jobs = 4, job_wallclock = '00:10:00'), ecflow_suite_dir = "/esarchive/scratch/nperez/ecflow") # your path! +# On Nord3 +res_nord3 <- Compute(wf, + chunks = list(latitude = 2, longitude = 2), + threads_load = 2, + threads_compute = 4, + cluster = list(queue_host = "nord3", + queue_type = 'lsf', + extra_queue_params = list('#BSUB -q bsc_es'), + cores_per_job = 2, + temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/', + polling_period = 10, + job_wallclock = '01:00', + max_jobs = 4, + bidirectional = FALSE), + ecflow_suite_dir = "/esarchive/scratch/nperez/ecflow") # your path! + # Results dim(res$output1) # ensemble sdate dat var time latitude longitude