diff --git a/R/ByChunks_autosubmit.R b/R/ByChunks_autosubmit.R index 16d273d04c4955478f179e62a2f9283c961313e4..bf63878d861ab1c77f9f1f16073be9052d83cb34 100644 --- a/R/ByChunks_autosubmit.R +++ b/R/ByChunks_autosubmit.R @@ -198,6 +198,7 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', r_module = 'R', CDO_module = NULL, autosubmit_module = 'autosubmit', + conda_env = NULL, node_memory = NULL, # not used cores_per_job = NULL, job_wallclock = '01:00:00', @@ -215,7 +216,7 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', if (any(!(names(cluster) %in% c('queue_host', 'queue_type', 'data_dir', 'temp_dir', 'lib_dir', 'init_commands', 'r_module', 'CDO_module', 'autosubmit_module', - 'ecflow_module', 'node_memory', + 'ecflow_module', 'conda_env', 'node_memory', 'cores_per_job', 'job_wallclock', 'max_jobs', 'extra_queue_params', 'bidirectional', 'polling_period', 'special_setup', 'expid', 'hpc_user', @@ -235,7 +236,7 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', cluster <- default_cluster ### queue_host - support_hpcs <- c('local', 'nord3') # names in platforms.yml + support_hpcs <- c('local', 'nord3', 'nord4') # names in platforms.yml if (is.null(cluster$queue_host) || !cluster$queue_host %in% support_hpcs) { stop("Cluster component 'queue_host' must be one of the following: ", paste(support_hpcs, collapse = ','), '.') @@ -265,12 +266,14 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', "character strings.") } ### r_module - if (!is.character(cluster[['r_module']])) { - stop("The component 'r_module' of the parameter 'cluster' must be a character string.") - } - if ((nchar(cluster[['r_module']]) < 1) || (grepl(' ', cluster[['r_module']]))) { - stop("The component 'r_module' of the parameter 'cluster' must have at least one character ", - "and contain no blank spaces.") + if (!is.null(cluster[['r_module']])) { + if (!is.character(cluster[['r_module']])) { + stop("The component 'r_module' of the parameter 'cluster' must be a character string.") + } + if ((nchar(cluster[['r_module']]) < 1) || (grepl(' ', cluster[['r_module']]))) { + stop("The component 'r_module' of the parameter 'cluster' must have at least one character ", + "and contain no blank spaces.") + } } ### CDO_module if (!is.null(cluster[['CDO_module']])) { @@ -278,11 +281,27 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', stop("The component 'CDO_module' of the parameter 'cluster' must be a character string.") } if (nchar(cluster[['CDO_module']]) < 1 || grepl(' ', cluster[['CDO_module']])) { - warning("The component 'CDO_module' of parameter 'cluster' must have ", + warning("The component 'CDO_module' of parameter 'cluster' is longer ", " than 1 and only the first element will be used.") } cluster[['r_module']] <- paste(cluster[['r_module']], cluster[['CDO_module']]) } + ### conda environment + if (!is.null(cluster[['conda_env']])) { + if (!is.character(cluster[['conda_env']])) { + stop("The component 'conda_env' of the parameter 'cluster' must be a character string.") + } + if (nchar(cluster[['conda_env']]) < 1 || grepl(' ', cluster[['conda_env']])) { + warning("The component 'conda_env' of parameter 'cluster' must have at ", + "least one character and contain no blank spaces.") + } + if (!is.null(cluster[['r_module']])) { + warning("Both 'conda_env' and 'r_module' cluster components have been ", + "specified. 'conda_env' will be used.") + } + } else { + cluster[['conda_env']] <- NULL + } ### autosubmit_module if (!is.character(cluster[['autosubmit_module']])) { stop("The component 'autosubmit_module' of the parameter 'cluster' must be a character string.") @@ -331,8 +350,10 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', stop("The component 'polling_period' of the parameter 'cluster' must be numeric.") } cluster[['polling_period']] <- round(cluster[['polling_period']]) - ### special_setup - if (!(cluster[['special_setup']] %in% c('none', 'marenostrum4'))) { + if (is.null(cluster[['special_setup']])) { + cluster[['special_setup']] <- 'none' + } + if (!(cluster[['special_setup']] %in% c('none', 'marenostrum4', 'nord4'))) { stop("The value provided for the component 'special_setup' of the parameter ", "'cluster' is not recognized.") } @@ -586,6 +607,13 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', if (cluster[['special_setup']] == 'marenostrum4') { file_spec <- paste(paste0("/gpfs/archive/bsc32/", files_to_send), collapse = ' ') + ## TODO: Use transfer machine? + # system(paste0("ssh bsc032762@transfer1.bsc.es 'mkdir -p remote_data_dir,", + # ' ; rsync -Rrav ', '\'', file_spec, '\' "', remote_data_dir, '/"', + # " ; sleep 10 '")) + + ## TODO: + ## This bit may need to be changed system(paste0("ssh ", cluster[['queue_host']], " 'mkdir -p ", remote_data_dir, ' ; module load transfer ; cd ', remote_autosubmit_suite_dir_suite, ' ; dtrsync -Rrav ', '\'', file_spec, '\' "', remote_data_dir, '/"', diff --git a/R/ByChunks_ecflow.R b/R/ByChunks_ecflow.R index 6292448cea2503641ec999e39ffbad2c5fb31653..eab03fec9526694c9ea48d13e9c8848e834a1d19 100644 --- a/R/ByChunks_ecflow.R +++ b/R/ByChunks_ecflow.R @@ -166,6 +166,7 @@ ByChunks_ecflow <- function(step_fun, cube_headers, ..., chunks = 'auto', init_commands = list(''), r_module = 'R', CDO_module = NULL, + conda_env = NULL, ecflow_module = 'ecFlow', node_memory = NULL, cores_per_job = NULL, @@ -184,8 +185,8 @@ ByChunks_ecflow <- function(step_fun, cube_headers, ..., chunks = 'auto', } if (any(!(names(cluster) %in% c('queue_host', 'queue_type', 'data_dir', 'temp_dir', 'lib_dir', 'init_commands', - 'r_module', 'CDO_module', 'autosubmit_module', - 'ecflow_module', 'node_memory', + 'r_module', 'CDO_module', 'conda_env', + 'autosubmit_module', 'ecflow_module', 'node_memory', 'cores_per_job', 'job_wallclock', 'max_jobs', 'extra_queue_params', 'bidirectional', 'polling_period', 'special_setup', 'expid', 'hpc_user')))) { @@ -272,6 +273,23 @@ ByChunks_ecflow <- function(step_fun, cube_headers, ..., chunks = 'auto', } cluster[['r_module']] <- paste(cluster[['r_module']], cluster[['CDO_module']]) } + ### conda environment + if (!is.null(cluster[['conda_env']])) { + if (!is.character(cluster[['conda_env']])) { + stop("The component 'conda_env' of the parameter 'cluster' must be a character string.") + } + if (nchar(cluster[['conda_env']]) < 1 || grepl(' ', cluster[['conda_env']])) { + warning("The component 'conda_env' of parameter 'cluster' must have at ", + "least one character and contain no blank spaces.") + } + if (!is.null(cluster[['r_module']])) { + warning("Both 'conda_env' and 'r_module' cluster components have been ", + "specified. 'conda_env' will be used.") + } + } else { + cluster$conda_env <- NULL + } + if (!is.character(cluster[['ecflow_module']])) { stop("The component 'ecflow_module' of the parameter 'cluster' must be a character string.") } @@ -295,7 +313,7 @@ ByChunks_ecflow <- function(step_fun, cube_headers, ..., chunks = 'auto', stop("The component 'extra_queue_params' of the parameter 'cluster' must be a list of ", "character strings.") } - if (!(cluster[['special_setup']] %in% c('none', 'marenostrum4'))) { + if (!(cluster[['special_setup']] %in% c('none', 'marenostrum4', 'nord4'))) { stop("The value provided for the component 'special_setup' of the parameter ", "'cluster' is not recognized.") } @@ -505,12 +523,47 @@ ByChunks_ecflow <- function(step_fun, cube_headers, ..., chunks = 'auto', paste0('%include "./', cluster[['queue_type']], '.h"'), chunk_ecf_script_lines) } + # Special setup: Nord4 singularity wrapper + special_setup_commands <- list("") + if (cluster[['special_setup']] == "nord4") { + if ('conda_env' %in% names(cluster)) { + special_setup_commands <- list(paste0("module load nord3-singu \n", + "module load bsc/current \n", + "module use /gpfs/projects/bsc32/software/suselinux/11/modules/all \n", + "unset PYTHONSTARTUP \n", + "alias ecflow_client='nord3_singu_es ecflow_client' \n", + "shopt -s expand_aliases \n")) + } else { + special_setup_commands <- list(paste0("module load nord3-singu \n", + "module load bsc/current \n", + "module use /gpfs/projects/bsc32/software/suselinux/11/modules/all \n", + "unset PYTHONSTARTUP \n", + "alias cdo='nord3_singu_es cdo' \n", + "alias Rscript='nord3_singu_es Rscript' \n", + "alias ecflow_client='nord3_singu_es ecflow_client' \n", + "shopt -s expand_aliases \n")) + } + } + chunk_ecf_script_lines <- gsub('^include_special_setup', + paste0(paste0(special_setup_commands, collapse = '\n'), '\n'), + chunk_ecf_script_lines) + # Init commands chunk_ecf_script_lines <- gsub('^include_init_commands', paste0(paste0(cluster[['init_commands']], collapse = '\n'), '\n'), chunk_ecf_script_lines) - chunk_ecf_script_lines <- gsub('^include_module_load', - paste0('module load ', cluster[['r_module']]), - chunk_ecf_script_lines) + + if ('conda_env' %in% names(cluster)) { + chunk_ecf_script_lines <- gsub('^include_module_load', + paste0('set +eu \n', + 'source /gpfs/projects/bsc32/software/suselinux/11/software/Miniconda3/4.7.10/etc/profile.d/conda.sh \n', + 'conda activate ', cluster[['conda_env']], '\n', + 'set -eu'), + chunk_ecf_script_lines) + } else { + chunk_ecf_script_lines <- gsub('^include_module_load', + paste0('module load ', cluster[['r_module']]), + chunk_ecf_script_lines) + } ecf_vars <- paste0('%', as.vector(sapply(chunked_dims, function(x) { c(toupper(x), paste0(toupper(x), '_N')) @@ -639,7 +692,7 @@ ByChunks_ecflow <- function(step_fun, cube_headers, ..., chunks = 'auto', suite <- add_line(suite, paste0('edit ', toupper(chunked_dims[i]), ' ', 1), tabs) suite <- add_line(suite, paste0('edit ', toupper(chunked_dims[i]), '_N ', chunks[[chunked_dims[i]]]), tabs) } - + # Iterate through chunks chunk_array <- array(1:prod(unlist(chunks)), dim = (unlist(chunks))) arrays_of_results <- vector('list', length(attr(step_fun, 'OutputDims'))) diff --git a/R/Utils.R b/R/Utils.R index 242ec70e1cc27b069ba3e68251dbce5dc3535293..28840e1968002b8218c8767fcb9e2f6630b1b346 100644 --- a/R/Utils.R +++ b/R/Utils.R @@ -893,15 +893,47 @@ write_autosubmit_bash <- function(chunks, cluster, autosubmit_suite_dir) { # Rewrite chunk_args= bash_script_lines <- gsub('^chunk_args=*', paste0('chunk_args=', chunk_args), bash_script_lines) - # Include init commands + # Include Nord4 Singularity wrapper special setup + special_setup_commands <- list("") + if (cluster[['special_setup']] == "nord4") { + if ('conda_env' %in% names(cluster)) { + special_setup_commands <- list(paste0("module load nord3-singu \n", + "module load bsc/current \n", + "module use /gpfs/projects/bsc32/software/suselinux/11/modules/all \n", + "unset PYTHONSTARTUP \n", + "alias ecflow_client='nord3_singu_es ecflow_client' \n", + "shopt -s expand_aliases \n")) + } else { + special_setup_commands <- list(paste0("module load nord3-singu \n", + "module load bsc/current \n", + "module use /gpfs/projects/bsc32/software/suselinux/11/modules/all \n", + "unset PYTHONSTARTUP \n", + "alias cdo='nord3_singu_es cdo' \n", + "alias Rscript='nord3_singu_es Rscript' \n", + "alias ecflow_client='nord3_singu_es ecflow_client' \n", + "shopt -s expand_aliases \n")) + } + } + bash_script_lines <- gsub('^include_special_setup', + paste0(paste0(special_setup_commands, collapse = '\n'), '\n'), + bash_script_lines) + # Include init commands bash_script_lines <- gsub('^include_init_commands', paste0(paste0(cluster[['init_commands']], collapse = '\n'), '\n'), - bash_script_lines) # Rewrite include_module_load - bash_script_lines <- gsub('^include_module_load', - paste0('module load ', cluster[['r_module']]), - bash_script_lines) + if ('conda_env' %in% names(cluster)) { + bash_script_lines <- gsub('^include_module_load', + paste0('set +eu \n', + 'source /gpfs/projects/bsc32/software/suselinux/11/software/Miniconda3/4.7.10/etc/profile.d/conda.sh \n', + 'conda activate ', cluster[['conda_env']], '\n', + 'set -eu'), + bash_script_lines) + } else { + bash_script_lines <- gsub('^include_module_load', + paste0('module load ', cluster[['r_module']]), + bash_script_lines) + } # Rewrite cd run_dir # If run_dir is not specified, the script will run under ${proj_dir} if (!is.null(cluster[['run_dir']])) { @@ -945,6 +977,9 @@ write_autosubmit_confs <- function(chunks, cluster, autosubmit_suite_dir) { conf$config$AUTOSUBMIT_VERSION <- cluster['autosubmit_version'] #Q: Should it be the total amount of chunk? + ## TODO: Fix this; it is not generalizable + # conf$config$AUTOSUBMIT_VERSION <- stringr::str_extract(cluster['autosubmit_module'], + # "(?<=/)(.+)(?=\\-foss)") conf$config$MAXWAITINGJOBS <- as.integer(prod(unlist(chunks))) # total amount of chunk #NOTE: Nord3 max. amount of queued jobs is 366 if (conf$config$MAXWAITINGJOBS > 366) conf$config$MAXWAITINGJOBS <- 366 @@ -952,10 +987,12 @@ write_autosubmit_confs <- function(chunks, cluster, autosubmit_suite_dir) { ############################################################ } else if (conf_type == "expdef") { + proj_dir <- paste0("STARTR_CHUNKING_", cluster$expid) conf$default$EXPID <- cluster$expid conf$default$HPCARCH <- cluster$queue_host # PROJECT_PATH should be where submit.sh and load....R stored --> local startR_autosubmit folder, under expID/ - conf$local$PROJECT_PATH <- file.path(autosubmit_suite_dir, paste0("STARTR_CHUNKING_", cluster$expid)) + conf$local$PROJECT_PATH <- file.path(autosubmit_suite_dir, proj_dir) + conf$project$PROJECT_DESTINATION <- proj_dir ############################################################ } else if (conf_type == "jobs") { diff --git a/README.md b/README.md index 010fafa415b6e101d5d9cd8b123d28be72a49a75..f85ec76b6b6c4cfa814a399424634cfc76c37a89 100644 --- a/README.md +++ b/README.md @@ -100,22 +100,24 @@ wf <- AddStep(data, step) res <- Compute(wf, chunks = list(latitude = 2, longitude = 2), - threads_load = 1, - threads_compute = 2, - cluster = list(queue_host = 'p9login1.bsc.es', - queue_type = 'slurm', - temp_dir = '/gpfs/scratch/bsc32/bsc32473/startR_tests/', - lib_dir = '/gpfs/projects/bsc32/share/R_libs/3.5/', - r_module = 'R/3.5.0-foss-2018b', - cores_per_job = 2, - job_wallclock = '00:10:00', - max_jobs = 4, - extra_queue_params = list('#SBATCH --qos=bsc_es'), - bidirectional = FALSE, - polling_period = 10 - ), - ecflow_suite_dir = '/home/Earth/nmanuben/test_startR/', - wait = TRUE) + threads_load = 2, + threads_compute = 4, + cluster = list(queue_host = 'n4login1.bsc.es', + queue_type = 'slurm', + temp_dir = '/gpfs/scratch/bsc32/bsc032473/startR_tests/', + r_module = 'R/4.1.2-foss-2019b', + cores_per_job = 4, + job_wallclock = '00:10:00', + max_jobs = 4, + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es'), + bidirectional = FALSE, + special_setup = 'nord4', + polling_period = 10 + ), + ecflow_suite_dir = ecflow_suite_dir, + wait = TRUE + ) + ``` #### 4. Monitoring the execution diff --git a/inst/chunking/Autosubmit/platforms.yml b/inst/chunking/Autosubmit/platforms.yml index f8d8f70c11902875dd39a90b52c11a0080c2ef59..44d5f6abaa3b7b2c2aba05fcd8e93ec63a47fee5 100644 --- a/inst/chunking/Autosubmit/platforms.yml +++ b/inst/chunking/Autosubmit/platforms.yml @@ -12,3 +12,32 @@ Platforms: CUSTOM_DIRECTIVES: # "['#SBATCH --exclusive']" "['#SBATCH --constraint=medmem']" # MAX_WALLCLOCK: '48:00' #Q: ARE THESE SETTING CORRECT? + mn5: + TYPE: slurm + HOST: glogin1.bsc.es + PROJECT: bsc32 + USER: + SERIAL_QUEUE: gp_debug + QUEUE: gp_bsces + SCRATCH_DIR: /gpfs/scratch/ + TEMP_DIR: '' + ADD_PROJECT_TO_HOST: "false" + CUSTOM_DIRECTIVES: + amd: + TYPE: slurm + HOST: amdlogin1.bsc.es + PROJECT: bsc32 + USER: + SCRATCH_DIR: /gpfs/scratch/ + TEMP_DIR: '' + ADD_PROJECT_TO_HOST: "false" + CUSTOM_DIRECTIVES: + nord4: + TYPE: SLURM + HOST: n4login1.bsc.es + PROJECT: bsc32 + ADD_PROJECT_TO_HOST: "false" + USER: + SCRATCH_DIR: /gpfs/scratch/ + QUEUE: bsc_es + CUSTOM_DIRECTIVES: diff --git a/inst/chunking/Autosubmit/startR_autosubmit.sh b/inst/chunking/Autosubmit/startR_autosubmit.sh index 63d9e6124b510a4b5a93ef529c71be9a3fec0264..be8ce1d5ef754b737b2b8d8bde4950111f18b8a7 100644 --- a/inst/chunking/Autosubmit/startR_autosubmit.sh +++ b/inst/chunking/Autosubmit/startR_autosubmit.sh @@ -11,6 +11,7 @@ chunknum="${chunknum:5}" #e.g., CHUNK_0 # e.g., chunk_args=(%JOBS."${chunknum}".dat% %JOBS."${chunknum}".dat_N% %JOBS."${chunknum}".var% %JOBS."${chunknum}".var_N% %JOBS."${chunknum}".sdate% %JOBS."${chunknum}".sdate_N%) chunk_args= +include_special_setup include_init_commands include_module_load diff --git a/inst/chunking/ecFlow/Chunk.ecf b/inst/chunking/ecFlow/Chunk.ecf index 5a265fb0db82b49cce435e3d04306ada59b51ac2..3471781272c8823569cb576328ebaa8214af08bc 100644 --- a/inst/chunking/ecFlow/Chunk.ecf +++ b/inst/chunking/ecFlow/Chunk.ecf @@ -3,6 +3,7 @@ include_queue_header date --rfc-3339=seconds > %REMOTE_ECF_HOME%/%ECF_NAME%.setup_time +include_special_setup include_init_commands %include "./head.h" diff --git a/inst/doc/practical_guide.md b/inst/doc/practical_guide.md index 637474ef1cbfd78e3a0469fde8fa06587df8daac..691d657ad6e3b60de03f43a07e534bfedb45daca 100644 --- a/inst/doc/practical_guide.md +++ b/inst/doc/practical_guide.md @@ -58,7 +58,7 @@ Afterwards, you will need to understand and use five functions, all of them incl - **Compute()**, for specifying the HPC to be employed, the execution parameters (e.g. number of chunks and cores), and to trigger the computation - **Collect()** and the **EC-Flow graphical user interface**, for monitoring of the progress and collection of results -Next, you can see an example startR script performing the ensemble mean of a small data set on an HPC cluster such as Nord3-v2 or CTE-Power9, for you to get a broad picture of how the startR functions interact and the information that is represented in a startR script. Note that the `queue_host`, `temp_dir` and `ecflow_suite_dir` parameters in the `Compute()` call are user-specific. +Next, you can see an example startR script performing the ensemble mean of a small data set on an HPC cluster such as Nord4, for you to get a broad picture of how the startR functions interact and the information that is represented in a startR script. Note that the `queue_host`, `temp_dir` and `ecflow_suite_dir` parameters in the `Compute()` call are user-specific. ```r library(startR) @@ -89,15 +89,16 @@ wf <- AddStep(data, step) longitude = 2), threads_load = 2, threads_compute = 4, - cluster = list(queue_host = 'nord4.bsc.es', + cluster = list(queue_host = 'n4login1.bsc.es', queue_type = 'slurm', temp_dir = '/gpfs/scratch/bsc32/bsc32734/startR_hpc/', cores_per_job = 4, job_wallclock = '00:10:00', max_jobs = 4, - extra_queue_params = list('#SBATCH --constraint=medmem'), + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es'), bidirectional = FALSE, - polling_period = 10 + polling_period = 10, + special_setup = 'nord4' ), ecflow_suite_dir = '/home/Earth/aho/startR_local/', wait = TRUE @@ -130,7 +131,16 @@ After following these steps for the connections in both directions (although fro Do not forget adding the following lines in your .bashrc on the HPC machine. -If you are planning to run it on Nord3-v2, you have to add: +If you are planning to run it on Nord4, you have to add: +elif [ $BSC_MACHINE == "nord4" ]; then + module load nord3-singu + module load bsc/current + module use /gpfs/projects/bsc32/software/suselinux/11/modules/all + unset PYTHONSTARTUP +fi + + +If you are using Nord3v2: ``` if [ $BSC_MACHINE == "nord3v2" ]; then module purge @@ -589,10 +599,10 @@ You will need to add two parameters to your `Compute()` call: `cluster` and `ecf The parameter `ecflow_suite_dir` expects a path to a folder in the workstation where to store temporary files generated for the automatic management of the workflow. As you will see later, the EC-Flow workflow manager is used transparently for this purpose. -The parameter `cluster` expects a list with a number of components that will have to be provided a bit differently depending on the HPC you want to run on. You can see next an example cluster configuration that will execute the previously defined workflow on Nord3-v2. +The parameter `cluster` expects a list with a number of components that will have to be provided a bit differently depending on the HPC you want to run on. You can see next an example cluster configuration that will execute the previously defined workflow on Nord4. ```r # user-defined - temp_dir <- '/gpfs/scratch/bsc32/bsc32734/startR_hpc/' + temp_dir <- '/gpfs/scratch/bsc32/bsc032734/startR_hpc/' ecflow_suite_dir <- '/home/Earth/aho/startR_local/' res <- Compute(wf, @@ -600,15 +610,16 @@ The parameter `cluster` expects a list with a number of components that will hav longitude = 2), threads_load = 2, threads_compute = 4, - cluster = list(queue_host = 'nord4.bsc.es', + cluster = list(queue_host = 'n4login1.bsc.es', queue_type = 'slurm', temp_dir = temp_dir, r_module = 'R/4.1.2-foss-2019b' cores_per_job = 4, job_wallclock = '00:10:00', max_jobs = 4, - extra_queue_params = list('#SBATCH --constraint=medmem'), + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es'), bidirectional = FALSE, + special_setup = 'nord4', polling_period = 10 ), ecflow_suite_dir = ecflow_suite_dir, @@ -622,13 +633,15 @@ The cluster components and options are explained next: - `temp_dir`: directory on the HPC where to store temporary files. Must be accessible from the HPC login node and all HPC nodes. - `lib_dir`: directory on the HPC where the startR R package and other required R packages are installed, accessible from all HPC nodes. These installed packages must be compatible with the R module specified in `r_module`. This parameter is optional; only required when the libraries are not installed in the R module. - `r_module`: name of the UNIX environment module to be used for R. If not specified, 'module load R' will be used. +- `cdo_module`: name of the UNIX environment module to be used for CDO. If not specified, no CDO module will be loaded. +- `conda_env`: path to the conda environment containing R (and CDO if needed). If both `r_module` and `conda_env` are specified, the conda environment will take preference. - `cores_per_job`: number of computing cores to be requested when submitting the job for each chunk to the HPC queue. Each node (should be core?) may be capable of supporting more than one computing thread. - `job_wallclock`: amount of time to reserve the resources when submitting the job for each chunk. Must follow the specific format required by the specified `queue_type`. - `max_jobs`: maximum number of jobs (chunks) to be queued simultaneously onto the HPC queue. Submitting too many jobs could overload the bandwidth between the HPC nodes and the storage system, or could overload the queue system. - `extra_queue_params`: list of character strings with additional queue headers for the jobs to be submitted to the HPC. Mainly used to specify the amount of memory to book for each job (e.g. '#SBATCH --mem-per-cpu=30000'; __NOTE: this line does not work on Nord3v2__), to request special queuing (e.g. '#SBATCH --qos=bsc_es'), or to request use of specific software (e.g. '#SBATCH --reservation=test-rhel-7.5'). - `bidirectional`: whether the connection between the R workstation and the HPC login node is bidirectional (TRUE) or unidirectional from the workstation to the login node (FALSE). - `polling_period`: when the connection is unidirectional, the workstation will ask the HPC login node for results each `polling_period` seconds. An excessively small value can overload the login node or result in temporary banning. -- `special_setup`: name of the machine if the computation requires an special setup. Only Marenostrum 4 needs this parameter (e.g. special_setup = 'marenostrum4'). +- `special_setup`: name of the machine if the computation requires an special setup. Nord4 requires this parameter (special_setup = 'nord4'). The parameter for the old Marenostrum 4 machine (i.e. special_setup = 'marenostrum4') will be deprecated in a future release. After the `Compute()` call is executed, an EC-Flow server is automatically started on your workstation, which will orchestrate the work and dispatch jobs onto the HPC. Thanks to the use of EC-Flow, you will also be able to monitor visually the progress of the execution. See the "Collect and the EC-Flow GUI" section. @@ -722,16 +735,17 @@ You can see one example of cluster configuration below. res <- Compute(wf, chunks = list(sdate = 2), threads_compute = 4, threads_load = 2, cluster = list( - queue_host = 'nord3', + queue_host = 'nord4', expid = , - hpc_user = "bsc32xxx", + hpc_user = "bsc032xxx", r_module = "R/4.1.2-foss-2019b", CDO_module = "CDO/1.9.8-foss-2019b", autosubmit_module = 'autosubmit/4.0.0b-foss-2015a-Python-3.7.3', autosubmit_version = '4.0.0b', cores_per_job = 4, job_wallclock = '01:00:00', - max_jobs = 4 + max_jobs = 4, + special_setup = 'nord4' ), workflow_manager = 'autosubmit', autosubmit_suite_dir = "/home/Earth//startR_local_autosubmit/", @@ -743,16 +757,17 @@ You can see one example of cluster configuration below. res <- Compute(wf, chunks = list(sdate = 2), threads_compute = 4, threads_load = 2, cluster = list( - queue_host = 'nord3', + queue_host = 'nord4', expid = , - hpc_user = "bsc32xxx", + hpc_user = "bsc032xxx", r_module = "R/4.1.2-foss-2019b", CDO_module = "CDO/1.9.8-foss-2019b", autosubmit_module = 'autosubmit/4.0.98-foss-2021b-Python-3.9.6', autosubmit_version = '4.0.98', cores_per_job = 4, job_wallclock = '01:00:00', - max_jobs = 4 + max_jobs = 4, + special_setup = 'nord4' ), workflow_manager = 'autosubmit', autosubmit_suite_dir = "/home/Earth//startR_local_autosubmit/", @@ -773,7 +788,8 @@ To have the good practice, note down the expid if it is automatically created by - `run_dir`: The directory to run the startR jobs. It is useful when the self-defined function has relative path. - `init_commands`: The initial commands in bash script before R script runs. For example, the modules required by computation can be loaded here. - `r_module`: Name of the UNIX environment module to be used for R. If not specified, `module load R` will be used. -- `CDO_module`: Name of the UNIX environment module to be used for CDO. If not specified, it is NULL and no CDO module will be loaded. Make sure to assign it if `tranform` is required in Start(). +- `CDO_module`: Name of the UNIX environment module to be used for CDO. If not specified, it is NULL and no CDO module will be loaded. Make sure to assign it if `transform` is required in Start(). +- `conda_env`: path to the conda environment containing R (and CDO if needed). If both `r_module` and `conda_env` are specified, the conda environment will take preference. - `autosubmit_module`: The name of the Autosubmit module. If not specified, `module load autosubmit` will be used. - `autosubmit_version`: The Autosubmit version (e.g. '4.0.0b'). If not specified, it will be parsed from the `autosubmit_module`. If it cannot be parsed from this parameter, it will be set to '4.0.0b' by default. - `cores_per_job`: Number of computing cores to be requested when submitting the job for each chunk to the HPC queue. @@ -783,7 +799,7 @@ It is corresponded to the parameter "THREADS" in _jobs.yml_ and "PROCESSORS_PER_ - `extra_queue_params`: list of character strings for additional queue headers for the jobs to be submitted to the HPC. For example, to constraint using medmem node ('#SBATCH --constraint=medmem'); to request an exclusive mode ('#SBATCH --exclusive'). - `polling_period`: when the connection is unidirectional, the workstation will ask the HPC login node for results each `polling_period` seconds. An excessively small value can overload the login node or result in temporary banning. -- `special_setup`: name of the machine if the computation requires an special setup. Only Marenostrum 4 needs this parameter (e.g. special_setup = 'marenostrum4'). +- `special_setup`: name of the machine if the computation requires an special setup. Nord4 requires this parameter (special_setup = 'nord4'). The parameter for the old Marenostrum 4 machine (i.e. special_setup = 'marenostrum4') will be deprecated in a future release. After the Compute() call is executed, you can monitor the status on [Autosubmit GUI](https://earth.bsc.es/autosubmitapp/). @@ -798,15 +814,16 @@ Usually, in use cases where large data inputs are involved, it is convenient to longitude = 2), threads_load = 2, threads_compute = 4, - cluster = list(queue_host = 'nord4.bsc.es', + cluster = list(queue_host = 'n4login1.bsc.es', queue_type = 'slurm', - temp_dir = '/gpfs/scratch/bsc32/bsc32734/startR_hpc/', + temp_dir = '/gpfs/scratch/bsc32/bsc032734/startR_hpc/', cores_per_job = 4, job_wallclock = '00:10:00', max_jobs = 4, - extra_queue_params = list('#SBATCH --constraint=medmem'), + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es'), bidirectional = FALSE, - polling_period = 10 + polling_period = 10, + special_setup = 'nord4' ), ecflow_suite_dir = '/home/Earth/aho/startR_local/', wait = FALSE @@ -1127,7 +1144,38 @@ r <- Compute(wf, ## 7. Compute() cluster templates -### Nord3-v2 + +### Nord4 + +```r +# Using Autosubmit +cluster = list(queue_host = 'n4login1.bsc.es', + expid = NULL, # NOTE: fill with your own expid if you already have one + hpc_user = 'bsc032762', + r_module = 'R/4.1.2-foss-2019b', + CDO_module = "CDO/1.9.8-foss-2019b", + autosubmit_module = 'autosubmit/4.0.98-foss-2015a-Python-3.7.3', + cores_per_job = 4, + job_wallclock = '01:00:00', + max_jobs = 4, + special_setup = 'nord4' + ) + +# Using ecFlow +cluster = list(queue_host = 'n4login1.bsc.es', + r_module = "R/4.1.2-foss-2019b", + temp_dir = '/gpfs/scratch/bsc32/bsc032762/startR_hpc/', + CDO_module = "CDO/1.9.8-foss-2019b", + cores_per_job = 4, + job_wallclock = '01:00:00', + max_jobs = 4, + special_setup = "nord4", + bidirectional = FALSE, + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es') + ) +``` + +### Nord3-v2 (deprecated) ```r # Using ecFlow @@ -1160,7 +1208,7 @@ cluster = list(queue_host = 'nord1.bsc.es', ) ``` -### CTE-Power9 +### CTE-Power9 (deprecated) ```r cluster = list(queue_host = 'p9login1.bsc.es', @@ -1186,7 +1234,7 @@ cluster = list(queue_host = 'bsceslogin01.bsc.es', ) ``` -### Marenostrum 4 +### Marenostrum 4 (deprecated) ```r cluster = list(queue_host = 'mn2.bsc.es', diff --git a/inst/doc/usecase/ex2_10_existing_mask.R b/inst/doc/usecase/ex2_10_existing_mask.R index 93c1abab2dec03e99a552dc2c7aba741c015e49c..8e707124ce11b065541de62333e2445e0798f1df 100644 --- a/inst/doc/usecase/ex2_10_existing_mask.R +++ b/inst/doc/usecase/ex2_10_existing_mask.R @@ -67,6 +67,34 @@ res <- Compute(workflow = wf_mask, chunks = list(lat = 2, lon = 2)) + + +# Submit to Nord4 +#-------------------user-defined--------------------- + queue_host <- 'n4login1.bsc.es' + temp_dir <- '/gpfs/scratch/bsc32/bsc032734/startR_hpc/' + ecflow_suite_dir <- '/home/Earth/aho/startR_local/' +#---------------------------------------------------- + res <- Compute(wf_mask, + chunks = list(lat = 2, + lon = 2),#$output1 + threads_load = 2, + threads_compute = 4, + cluster = list(queue_host = queue_host, + queue_type = 'slurm', + temp_dir = temp_dir, + cores_per_job = 2, + job_wallclock = '05:00', + max_jobs = 4, + bidirectional = FALSE, + polling_period = 10, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es') + ), + ecflow_suite_dir = ecflow_suite_dir, + wait = TRUE + ) + # Submit to Nord3v2 #-------------------user-defined--------------------- queue_host <- 'nord4' @@ -200,10 +228,10 @@ res <- Compute(workflow = wf_mask, chunks = list(lat = 2, lon = 2)) -# Submit to Nord3v2 +# Submit to Nord4 #-------------------user-defined--------------------- queue_host <- 'nord4' - temp_dir <- '/gpfs/scratch/bsc32/bsc32734/startR_hpc/' + temp_dir <- '/gpfs/scratch/bsc32/bsc032734/startR_hpc/' ecflow_suite_dir <- '/home/Earth/aho/startR_local/' #---------------------------------------------------- res <- Compute(wf_mask, @@ -219,6 +247,8 @@ res <- Compute(workflow = wf_mask, max_jobs = 4, bidirectional = FALSE, polling_period = 10 + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es') ), ecflow_suite_dir = ecflow_suite_dir, wait = TRUE diff --git a/inst/doc/usecase/ex2_1_timedim.R b/inst/doc/usecase/ex2_1_timedim.R index 0c0f36c0a981e0abfd45e23fcd86d8f22406919f..f9872affeb15bc2892cbb27c284c495af96efd13 100644 --- a/inst/doc/usecase/ex2_1_timedim.R +++ b/inst/doc/usecase/ex2_1_timedim.R @@ -43,31 +43,8 @@ library(startR) # 220.5 273.3 282.7 279.9 295.6 306.8 # ---------------------------------------------------------------- -## on Power9 -#-----------modify according to your personal info--------- - queue_host <- 'cte-power' #your own host name for power9 - temp_dir <- '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' - ecflow_suite_dir <- '/home/Earth/nperez/startR_local/' #your own local directory -#------------------------------------------------------------ - res <- Compute(wf1, - chunks = list(ensemble = 20, - sdate = 2), - threads_load = 2, - threads_compute = 4, - cluster = list(queue_host = queue_host, - queue_type = 'slurm', - cores_per_job = 2, - temp_dir = temp_dir, - r_module = 'R/3.6.1-foss-2018b', - polling_period = 10, - job_wallclock = '01:00:00', - max_jobs = 40, - bidirectional = FALSE), - ecflow_suite_dir = ecflow_suite_dir, - wait = TRUE) - -## on Nord3 with ecFlow +## on Nord4 with ecFlow #-----------modify according to your personal info--------- queue_host <- 'nord4' temp_dir <- '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' @@ -85,15 +62,17 @@ library(startR) polling_period = 10, job_wallclock = '01:00', max_jobs = 40, - bidirectional = FALSE), + bidirectional = FALSE, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es') + ), ecflow_suite_dir = ecflow_suite_dir, wait = TRUE) - -## on Nord3 with Autosubmit +## on Nord4 with Autosubmit #-----------modify according to your personal info--------- - hpc_user <- "bsc32xxx" + hpc_user <- "bsc032xxx" expid <- "xxxx" # autosubmit exp id; can be NULL autosubmit_suite_dir <- "/home/Earth//startR_local_autosubmit/" #------------------------------------------------------------ @@ -102,7 +81,7 @@ library(startR) threads_load = 2, threads_compute = 4, cluster = list( - queue_host = 'nord3', + queue_host = 'nord4', r_module = "R/4.1.2-foss-2019b", autosubmit_module = 'autosubmit/4.0.0b-foss-2015a-Python-3.7.3', cores_per_job = 2, @@ -110,6 +89,7 @@ library(startR) max_jobs = 40, polling_period = 10, extra_queue_params = list('#SBATCH --constraint=medmem', '#SBATCH --exclusive'), + special_setup = 'nord4', expid = NULL, hpc_user = hpc_user ), diff --git a/inst/doc/usecase/ex2_2_attr.R b/inst/doc/usecase/ex2_2_attr.R index fbd2eac9e24207f5d51498ca9c8974daf23e72bb..2968e932ee0081482220186b5096ff38996c44de 100644 --- a/inst/doc/usecase/ex2_2_attr.R +++ b/inst/doc/usecase/ex2_2_attr.R @@ -47,13 +47,13 @@ library(multiApply) # 13.19 169.60 237.60 217.90 284.10 305.10 # ----------------------------------------------------------------- -## on Power9 +## on Nord4 #-----------modify according to your personal info--------- - queue_host = 'cte-power' #your own host name for power9 - temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' + queue_host = 'nord4' #your own host name for nord4 + temp_dir = '/gpfs/scratch/bsc32/bsc032339/startR_hpc/' ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory #------------------------------------------------------------ - res2_P <- Compute(wf2, + res2_Nord <- Compute(wf2, chunks = list(ensemble = 20, sdate = 2), threads_load = 2, @@ -62,22 +62,23 @@ library(multiApply) queue_type = 'slurm', cores_per_job = 2, temp_dir = temp_dir, - r_module = 'R/3.6.1-foss-2018b', - #extra_queue_params = list('#SBATCH --mem-per-cpu=3000'), polling_period = 10, - job_wallclock = '01:00:00', + job_wallclock = '01:00', max_jobs = 40, - bidirectional = FALSE), + bidirectional = FALSE, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es') + ), ecflow_suite_dir = ecflow_suite_dir, #your own local directory wait = TRUE) - -## on Nord3v2 + +## on Power9 #-----------modify according to your personal info--------- - queue_host = 'nord4' #your own host name for power9 + queue_host = 'cte-power' #your own host name for power9 temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory #------------------------------------------------------------ - res2_Nord <- Compute(wf2, + res2_P <- Compute(wf2, chunks = list(ensemble = 20, sdate = 2), threads_load = 2, @@ -86,9 +87,12 @@ library(multiApply) queue_type = 'slurm', cores_per_job = 2, temp_dir = temp_dir, + r_module = 'R/3.6.1-foss-2018b', + #extra_queue_params = list('#SBATCH --mem-per-cpu=3000'), polling_period = 10, - job_wallclock = '01:00', + job_wallclock = '01:00:00', max_jobs = 40, bidirectional = FALSE), ecflow_suite_dir = ecflow_suite_dir, #your own local directory wait = TRUE) + diff --git a/inst/doc/usecase/ex2_3_cdo.R b/inst/doc/usecase/ex2_3_cdo.R index 9dca0450a20bced2b71136aa5a2427766fa60228..bc9d1d6c3bbf7a5e5feeb126236d33f2711433b2 100644 --- a/inst/doc/usecase/ex2_3_cdo.R +++ b/inst/doc/usecase/ex2_3_cdo.R @@ -49,36 +49,11 @@ library(startR) # 206.3 269.2 283.3 278.8 296.4 314.2 # -------------------------------------------------------------------- -## on Power9 -#-----------modify according to your personal info--------- - queue_host = 'cte-power' #your own host name for power9 - temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' - ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory -#------------------------------------------------------------ - res3_P <- Compute(wf3, - chunks = list(ensemble = 2, - sdate = 2), - threads_load = 2, - threads_compute = 4, - cluster = list(queue_host = queue_host, #your own host name for power9 - queue_type = 'slurm', - cores_per_job = 1, - temp_dir = temp_dir, - r_module = 'R/3.5.0-foss-2018b', - CDO_module = 'CDO/1.9.5-foss-2018b', - extra_queue_params = list('#SBATCH --mem-per-cpu=3000'), - polling_period = 100, - job_wallclock = '01:00:00', - max_jobs = 4, - bidirectional = FALSE), - ecflow_suite_dir = ecflow_suite_dir, #your own local directory - wait = TRUE) - -## on Nord3v2 +## on Nord4 #-----------modify according to your personal info--------- - queue_host = 'nord4.bsc.es' - temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' + queue_host = 'n4login1.bsc.es' + temp_dir = '/gpfs/scratch/bsc32/bsc032339/startR_hpc/' ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory #------------------------------------------------------------ res3_N <- Compute(wf3, @@ -94,6 +69,8 @@ library(startR) polling_period = 100, job_wallclock = '01:00', max_jobs = 4, - bidirectional = FALSE), + bidirectional = FALSE, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es')), ecflow_suite_dir = ecflow_suite_dir, #your own local directory wait = TRUE) diff --git a/inst/doc/usecase/ex2_4_two_func.R b/inst/doc/usecase/ex2_4_two_func.R index c76eda994a17a7efbca2fbf1a07e8e96bf152702..e8411bc076b6f5efc1c73d66c30c2b42e1a691e9 100644 --- a/inst/doc/usecase/ex2_4_two_func.R +++ b/inst/doc/usecase/ex2_4_two_func.R @@ -47,35 +47,10 @@ summary(res4$output1) # 227.3 259.6 280.8 277.1 296.2 306.7 # ------------------------------------------------------------------ -## on Power9 +## on Nord4 #-----------modify according to your personal info----------- - queue_host = 'cte-power' #your own host name for power9 - temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' - ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory -#------------------------------------------------------------ - res4_P <- Compute(wf4, - chunks = list(ensemble = 2, - sdate = 2), - threads_load = 2, - threads_compute = 4, - cluster = list(queue_host = queue_host, #your own host name for power9 - queue_type = 'slurm', - cores_per_job = 1, - temp_dir = temp_dir, - r_module = 'R/3.6.1-foss-2018b', - CDO_module = 'CDO/1.9.5-foss-2018b', - extra_queue_params = list('#SBATCH --mem-per-cpu=3000'), - polling_period = 10, - job_wallclock = '01:00:00', - max_jobs = 6, - bidirectional = FALSE), - ecflow_suite_dir = ecflow_suite_dir, #your own local directory - wait = TRUE) - -## on Nord3v2 -#-----------modify according to your personal info----------- - queue_host = 'nord4.bsc.es' - temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' + queue_host = 'n4login1.bsc.es' + temp_dir = '/gpfs/scratch/bsc32/bsc032339/startR_hpc/' ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory #------------------------------------------------------------ res4_N <- Compute(wf4, @@ -83,7 +58,7 @@ summary(res4$output1) sdate = 2), threads_load = 2, threads_compute = 4, - cluster = list(queue_host = queue_host, #your own host name for power9 + cluster = list(queue_host = queue_host, #your own host name for nord4 queue_type = 'slurm', cores_per_job = 1, temp_dir = temp_dir, @@ -91,6 +66,8 @@ summary(res4$output1) polling_period = 10, job_wallclock = '01:00', max_jobs = 6, - bidirectional = FALSE), + bidirectional = FALSE, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es')), ecflow_suite_dir = ecflow_suite_dir, #your own local directory wait = TRUE) diff --git a/inst/doc/usecase/ex2_5_rainFARM.R b/inst/doc/usecase/ex2_5_rainFARM.R index 8b58901bcdf9c5a791fc7be8d4f9510604364033..379e81c3a75f92028e8c92c9fcb6a988dd681fde 100644 --- a/inst/doc/usecase/ex2_5_rainFARM.R +++ b/inst/doc/usecase/ex2_5_rainFARM.R @@ -99,11 +99,11 @@ res <- Compute(workflow, threads_compute = 4) -#========= OPTION 2: Compute ON NORD3 ============ +#========= OPTION 2: Compute ON NORD4 ============ #-----------modify according to your personal info--------- - queue_host = 'nord3' # your own host name for nord3v2 - temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' + queue_host = 'nord4' # your own host name for nord4 + temp_dir = '/gpfs/scratch/bsc32/bsc032339/startR_hpc/' ecflow_suite_dir = '/home/Earth/nperez/startR_local/' # your own local directory #------------------------------------------------------------ @@ -119,7 +119,9 @@ res <- Compute(workflow, polling_period = 10, job_wallclock = '01:00:00', max_jobs = 4, - bidirectional = FALSE), + bidirectional = FALSE, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es')), ecflow_suite_dir = ecflow_suite_dir, wait = TRUE) diff --git a/inst/doc/usecase/ex2_6_ext_param_func.R b/inst/doc/usecase/ex2_6_ext_param_func.R index 760e30d2c1e8422d560c9ff6d10da5206138bf71..970c3380b23af9cb0baa420a8a99aa022717a20e 100644 --- a/inst/doc/usecase/ex2_6_ext_param_func.R +++ b/inst/doc/usecase/ex2_6_ext_param_func.R @@ -106,27 +106,10 @@ res$strat[1:5, 1:2, 1] # ------------------------------------------------------------- -#Power9 - res <- Compute(workflow$strat, - chunks = list(latitude = 2, longitude = 2), - threads_load = 2, - threads_compute = 2, - cluster = list(queue_host = 'cte-power', #user-specific - queue_type = 'slurm', - temp_dir = '/gpfs/scratch/bsc32/bsc32339/', #user-specific - r_module = 'R/3.6.1-foss-2018b', - cores_per_job = 2, - job_wallclock = '10:00:00', - max_jobs = 4, - bidirectional = FALSE, - polling_period = 10), - ecflow_suite_dir = '/esarchive/scratch/nperez/ecFlow', #user-specific - wait = TRUE) - -# Nord3-v2 +# Nord4 #-------------------user-defined--------------------- queue_host <- 'nord4' - temp_dir <- '/gpfs/scratch/bsc32/bsc32734/startR_hpc/' + temp_dir <- '/gpfs/scratch/bsc32/bsc032734/startR_hpc/' ecflow_suite_dir <- '/home/Earth/aho/startR_local/' #---------------------------------------------------- res <- Compute(workflow$strat, @@ -140,7 +123,9 @@ res$strat[1:5, 1:2, 1] job_wallclock = '10:00', max_jobs = 4, bidirectional = FALSE, - polling_period = 10), + polling_period = 10, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es')), ecflow_suite_dir = ecflow_suite_dir, #user-specific wait = TRUE) diff --git a/inst/doc/usecase/ex2_7_seasonal_forecast_verification.R b/inst/doc/usecase/ex2_7_seasonal_forecast_verification.R index b752383453fb1db660cf692c599a0a08fb179f6d..3d48bf146329aaaeb6a3f69a88d881027e0a29ca 100644 --- a/inst/doc/usecase/ex2_7_seasonal_forecast_verification.R +++ b/inst/doc/usecase/ex2_7_seasonal_forecast_verification.R @@ -77,7 +77,7 @@ wait = TRUE ) -# Compute() on Nord3v2 +# Compute() on Nord4 res <- Compute(wf, chunks = list(latitude = 2, longitude = 2), @@ -85,11 +85,13 @@ threads_compute = 4, cluster = list(queue_host = 'nord4', # your alias queue_type = 'slurm', - temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/', # your own path + temp_dir = '/gpfs/scratch/bsc32/bsc032339/startR_hpc/', # your own path job_wallclock = '00:30', cores_per_job = 4, max_jobs = 4, bidirectional = FALSE, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es'), polling_period = 50), ecflow_suite_dir = '/home/Earth/nperez/startR_local/', # your own path wait = TRUE) diff --git a/inst/doc/usecase/ex2_8_calibration.R b/inst/doc/usecase/ex2_8_calibration.R index a36f349f59f6c3840363ba2f3f25298bb5dfd7d0..94cc429e6e40c12b3014083eee6231b5c88742d2 100644 --- a/inst/doc/usecase/ex2_8_calibration.R +++ b/inst/doc/usecase/ex2_8_calibration.R @@ -67,27 +67,30 @@ res <- Compute(wf, chunks = list(latitude = 2, longitude = 2), # Declaration of HPC and execution ## ECFlow is required -# On Nord3v2 +# On Nord4 #-------------------user-defined--------------------- - queue_host <- 'nord4.bsc.es' - temp_dir <- '/gpfs/scratch/bsc32/bsc32734/startR_hpc/' + queue_host <- 'n4login1.bsc.es' + temp_dir <- '/gpfs/scratch/bsc32/bsc032734/startR_hpc/' ecflow_suite_dir <- '/home/Earth/aho/startR_local/' #---------------------------------------------------- -res_nord3 <- Compute(wf, - chunks = list(latitude = 2, longitude = 2), - threads_load = 2, - threads_compute = 4, - cluster = list(queue_host = queue_host, - queue_type = 'slurm', - extra_queue_params = list("#SBATCH --constraint=medmem"), - cores_per_job = 2, - temp_dir = temp_dir, - polling_period = 10, - job_wallclock = '01:00', - max_jobs = 4, - bidirectional = FALSE), - ecflow_suite_dir = ecflow_suite_dir, - wait = T) +res <- Compute(wf, + chunks = list(latitude = 2, longitude = 2), + threads_load = 2, + threads_compute = 4, + cluster = list(queue_host = queue_host, + queue_type = 'slurm', + extra_queue_params = list("#SBATCH -A bsc32", + "#SBATCH -q bsc_es", + "#SBATCH --constraint=highmem"), + cores_per_job = 2, + temp_dir = temp_dir, + polling_period = 10, + job_wallclock = '01:00', + max_jobs = 4, + bidirectional = FALSE, + special_setup = 'nord4'), + ecflow_suite_dir = ecflow_suite_dir, + wait = T) # Results dim(res$output1) diff --git a/inst/doc/usecase/ex3_1_SubseasonalECVHindcast.md b/inst/doc/usecase/ex3_1_SubseasonalECVHindcast.md index e554c66ddbb3d8f86334b111a7bb34b69cbb8874..21262ce56f3bcf2503fc727e62a0b638f0b580d5 100644 --- a/inst/doc/usecase/ex3_1_SubseasonalECVHindcast.md +++ b/inst/doc/usecase/ex3_1_SubseasonalECVHindcast.md @@ -196,24 +196,26 @@ Finally, execute the analysis, for instance, in Nord3v2: ```r #-----------modify according to your personal info--------- - queue_host = 'nord4' #your own host name for Nord3v2 - temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' + queue_host = 'nord4' #your own host name for Nord4 + temp_dir = '/gpfs/scratch/bsc32/bsc032339/startR_hpc/' ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory #------------------------------------------------------------ result <- Compute(wf, - chunks = list(time = 4, longitude = 2, latitude = 2), - threads_load = 2, - threads_compute = 12, - cluster = list(queue_host = queue_host, - queue_type = 'slurm', - cores_per_job = 12, - temp_dir = temp_dir, - polling_period = 10, - job_wallclock = '03:00:00', - max_jobs = 16, - bidirectional = FALSE), - ecflow_suite_dir = ecflow_suite_dir, - wait = TRUE) + chunks = list(time = 4, longitude = 2, latitude = 2), + threads_load = 2, + threads_compute = 12, + cluster = list(queue_host = queue_host, + queue_type = 'slurm', + cores_per_job = 12, + temp_dir = temp_dir, + polling_period = 10, + job_wallclock = '03:00:00', + max_jobs = 16, + bidirectional = FALSE, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es')), + ecflow_suite_dir = ecflow_suite_dir, + wait = TRUE) ``` *Notice that the execution of `Compute` may last for ~2 hours each chunk. Consider set `wait` as false (see [practical guide](https://earth.bsc.es/gitlab/es/startR/-/blob/master/inst/doc/practical_guide.md#collect-and-the-ec-flow-gui)).*