From 3ceab94a4ace5a75e148729d8eda146b851f327f Mon Sep 17 00:00:00 2001 From: aho Date: Wed, 5 Oct 2022 19:24:24 +0200 Subject: [PATCH] Revise Compute() to use nord3v2 --- inst/doc/usecase/ex2_10_existing_mask.R | 14 +++++----- inst/doc/usecase/ex2_1_timedim.R | 5 ++-- inst/doc/usecase/ex2_2_attr.R | 7 +++-- inst/doc/usecase/ex2_3_cdo.R | 7 +++-- inst/doc/usecase/ex2_4_two_func.R | 7 +++-- inst/doc/usecase/ex2_6_ext_param_func.R | 18 ++++++++----- .../ex2_7_seasonal_forecast_verification.R | 11 ++++---- inst/doc/usecase/ex2_8_calibration.R | 27 +++++++++---------- 8 files changed, 45 insertions(+), 51 deletions(-) diff --git a/inst/doc/usecase/ex2_10_existing_mask.R b/inst/doc/usecase/ex2_10_existing_mask.R index 20d2db7..93c1aba 100644 --- a/inst/doc/usecase/ex2_10_existing_mask.R +++ b/inst/doc/usecase/ex2_10_existing_mask.R @@ -67,9 +67,9 @@ res <- Compute(workflow = wf_mask, chunks = list(lat = 2, lon = 2)) -# Submit to Nord3 +# Submit to Nord3v2 #-------------------user-defined--------------------- - queue_host <- 'nord1' + queue_host <- 'nord4' temp_dir <- '/gpfs/scratch/bsc32/bsc32734/startR_hpc/' ecflow_suite_dir <- '/home/Earth/aho/startR_local/' #---------------------------------------------------- @@ -79,12 +79,11 @@ res <- Compute(workflow = wf_mask, threads_load = 2, threads_compute = 4, cluster = list(queue_host = queue_host, - queue_type = 'lsf', + queue_type = 'slurm', temp_dir = temp_dir, cores_per_job = 2, job_wallclock = '05:00', max_jobs = 4, - extra_queue_params = list('#BSUB -q bsc_es'), bidirectional = FALSE, polling_period = 10 ), @@ -201,9 +200,9 @@ res <- Compute(workflow = wf_mask, chunks = list(lat = 2, lon = 2)) -# Submit to Nord3 +# Submit to Nord3v2 #-------------------user-defined--------------------- - queue_host <- 'nord1' + queue_host <- 'nord4' temp_dir <- '/gpfs/scratch/bsc32/bsc32734/startR_hpc/' ecflow_suite_dir <- '/home/Earth/aho/startR_local/' #---------------------------------------------------- @@ -213,12 +212,11 @@ res <- Compute(workflow = wf_mask, threads_load = 2, threads_compute = 4, cluster = list(queue_host = queue_host, - queue_type = 'lsf', + queue_type = 'slurm', temp_dir = temp_dir, cores_per_job = 2, job_wallclock = '05:00', max_jobs = 4, - extra_queue_params = list('#BSUB -q bsc_es'), bidirectional = FALSE, polling_period = 10 ), diff --git a/inst/doc/usecase/ex2_1_timedim.R b/inst/doc/usecase/ex2_1_timedim.R index f8253c8..15ef37d 100644 --- a/inst/doc/usecase/ex2_1_timedim.R +++ b/inst/doc/usecase/ex2_1_timedim.R @@ -68,7 +68,7 @@ library(startR) ## on Nord3 #-----------modify according to your personal info--------- - queue_host = 'nord3' #your own host name for power9 + queue_host = 'nord4' temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory #------------------------------------------------------------ @@ -78,8 +78,7 @@ library(startR) threads_load = 2, threads_compute = 4, cluster = list(queue_host = queue_host, - queue_type = 'lsf', - extra_queue_params = list('#BSUB -q bsc_es'), + queue_type = 'slurm', cores_per_job = 2, temp_dir = temp_dir, polling_period = 10, diff --git a/inst/doc/usecase/ex2_2_attr.R b/inst/doc/usecase/ex2_2_attr.R index f06ec46..fbd2eac 100644 --- a/inst/doc/usecase/ex2_2_attr.R +++ b/inst/doc/usecase/ex2_2_attr.R @@ -71,9 +71,9 @@ library(multiApply) ecflow_suite_dir = ecflow_suite_dir, #your own local directory wait = TRUE) -## on Nord3 +## on Nord3v2 #-----------modify according to your personal info--------- - queue_host = 'nord3' #your own host name for power9 + queue_host = 'nord4' #your own host name for power9 temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory #------------------------------------------------------------ @@ -83,10 +83,9 @@ library(multiApply) threads_load = 2, threads_compute = 4, cluster = list(queue_host = queue_host, #your own host name for power9 - queue_type = 'lsf', + queue_type = 'slurm', cores_per_job = 2, temp_dir = temp_dir, - extra_queue_params = list('#BSUB -q bsc_es'), polling_period = 10, job_wallclock = '01:00', max_jobs = 40, diff --git a/inst/doc/usecase/ex2_3_cdo.R b/inst/doc/usecase/ex2_3_cdo.R index 8c398f7..9dca045 100644 --- a/inst/doc/usecase/ex2_3_cdo.R +++ b/inst/doc/usecase/ex2_3_cdo.R @@ -75,9 +75,9 @@ library(startR) wait = TRUE) -## on Nord 3 +## on Nord3v2 #-----------modify according to your personal info--------- - queue_host = 'nord3' #your own host name for power9 + queue_host = 'nord4.bsc.es' temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory #------------------------------------------------------------ @@ -87,11 +87,10 @@ library(startR) threads_load = 2, threads_compute = 4, cluster = list(queue_host = queue_host, #your own host name for power9 - queue_type = 'lsf', + queue_type = 'slurm', cores_per_job = 1, temp_dir = temp_dir, CDO_module = 'CDO', - extra_queue_params = list('#BSUB -q bsc_es'), polling_period = 100, job_wallclock = '01:00', max_jobs = 4, diff --git a/inst/doc/usecase/ex2_4_two_func.R b/inst/doc/usecase/ex2_4_two_func.R index 2e638bd..c76eda9 100644 --- a/inst/doc/usecase/ex2_4_two_func.R +++ b/inst/doc/usecase/ex2_4_two_func.R @@ -72,9 +72,9 @@ summary(res4$output1) ecflow_suite_dir = ecflow_suite_dir, #your own local directory wait = TRUE) -## on Nord 3 +## on Nord3v2 #-----------modify according to your personal info----------- - queue_host = 'nord3' #your own host name for power9 + queue_host = 'nord4.bsc.es' temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory #------------------------------------------------------------ @@ -84,11 +84,10 @@ summary(res4$output1) threads_load = 2, threads_compute = 4, cluster = list(queue_host = queue_host, #your own host name for power9 - queue_type = 'lsf', + queue_type = 'slurm', cores_per_job = 1, temp_dir = temp_dir, CDO_module = 'CDO', - extra_queue_params = list('#SBATCH -q bsc_es'), polling_period = 10, job_wallclock = '01:00', max_jobs = 6, diff --git a/inst/doc/usecase/ex2_6_ext_param_func.R b/inst/doc/usecase/ex2_6_ext_param_func.R index 8bd128f..760e30d 100644 --- a/inst/doc/usecase/ex2_6_ext_param_func.R +++ b/inst/doc/usecase/ex2_6_ext_param_func.R @@ -106,7 +106,7 @@ res$strat[1:5, 1:2, 1] # ------------------------------------------------------------- -# ---- To be tried for comparison of results: +#Power9 res <- Compute(workflow$strat, chunks = list(latitude = 2, longitude = 2), threads_load = 2, @@ -123,20 +123,24 @@ res$strat[1:5, 1:2, 1] ecflow_suite_dir = '/esarchive/scratch/nperez/ecFlow', #user-specific wait = TRUE) -# ---- On Nord 3: +# Nord3-v2 +#-------------------user-defined--------------------- + queue_host <- 'nord4' + temp_dir <- '/gpfs/scratch/bsc32/bsc32734/startR_hpc/' + ecflow_suite_dir <- '/home/Earth/aho/startR_local/' +#---------------------------------------------------- res <- Compute(workflow$strat, chunks = list(latitude = 2, longitude = 2), threads_load = 2, threads_compute = 2, - cluster = list(queue_host = 'nord3', #user-specific - queue_type = 'lsf', - temp_dir = '/gpfs/scratch/bsc32/bsc32339/', #user-specific + cluster = list(queue_host = queue_host, #user-specific + queue_type = 'slurm', + temp_dir = temp_dir, #user-specific cores_per_job = 2, - extra_queue_params = list('#BSUB -q bsc_es'), job_wallclock = '10:00', max_jobs = 4, bidirectional = FALSE, polling_period = 10), - ecflow_suite_dir = '/esarchive/scratch/nperez/ecFlow', #user-specific + ecflow_suite_dir = ecflow_suite_dir, #user-specific wait = TRUE) diff --git a/inst/doc/usecase/ex2_7_seasonal_forecast_verification.R b/inst/doc/usecase/ex2_7_seasonal_forecast_verification.R index b28e6a6..b752383 100644 --- a/inst/doc/usecase/ex2_7_seasonal_forecast_verification.R +++ b/inst/doc/usecase/ex2_7_seasonal_forecast_verification.R @@ -77,22 +77,21 @@ wait = TRUE ) -# Compute() on Nord 3 +# Compute() on Nord3v2 res <- Compute(wf, chunks = list(latitude = 2, longitude = 2), threads_load = 2, threads_compute = 4, - cluster = list(queue_host = 'nord3', - queue_type = 'lsf', - temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/', + cluster = list(queue_host = 'nord4', # your alias + queue_type = 'slurm', + temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/', # your own path job_wallclock = '00:30', - extra_queue_params = list('#BSUB -q bsc_es'), cores_per_job = 4, max_jobs = 4, bidirectional = FALSE, polling_period = 50), - ecflow_suite_dir = '/home/Earth/nperez/startR_local/', + ecflow_suite_dir = '/home/Earth/nperez/startR_local/', # your own path wait = TRUE) # Results dim(res$output1) diff --git a/inst/doc/usecase/ex2_8_calibration.R b/inst/doc/usecase/ex2_8_calibration.R index 23ab3b0..a36f349 100644 --- a/inst/doc/usecase/ex2_8_calibration.R +++ b/inst/doc/usecase/ex2_8_calibration.R @@ -66,31 +66,28 @@ res <- Compute(wf, chunks = list(latitude = 2, longitude = 2), # Declaration of HPC and execution ## ECFlow is required -# On Fatnodes -res_fat2 <- Compute(wf, - chunks = list(latitude = 2, longitude = 2), - threads_load = 2, - threads_compute = 4, - cluster = list(queue_host = "bsceslogin01.bsc.es", - cores_per_job = 2, - max_jobs = 4, job_wallclock = '00:10:00'), - ecflow_suite_dir = "/esarchive/scratch/nperez/ecflow") # your path! -# On Nord3 +# On Nord3v2 +#-------------------user-defined--------------------- + queue_host <- 'nord4.bsc.es' + temp_dir <- '/gpfs/scratch/bsc32/bsc32734/startR_hpc/' + ecflow_suite_dir <- '/home/Earth/aho/startR_local/' +#---------------------------------------------------- res_nord3 <- Compute(wf, chunks = list(latitude = 2, longitude = 2), threads_load = 2, threads_compute = 4, - cluster = list(queue_host = "nord3", - queue_type = 'lsf', - extra_queue_params = list('#BSUB -q bsc_es'), + cluster = list(queue_host = queue_host, + queue_type = 'slurm', + extra_queue_params = list("#SBATCH --constraint=medmem"), cores_per_job = 2, - temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/', + temp_dir = temp_dir, polling_period = 10, job_wallclock = '01:00', max_jobs = 4, bidirectional = FALSE), - ecflow_suite_dir = "/esarchive/scratch/nperez/ecflow") # your path! + ecflow_suite_dir = ecflow_suite_dir, + wait = T) # Results dim(res$output1) -- GitLab