diff --git a/MODULES b/MODULES index 7ed13edc7a7ef0e1532688570915ade26ea28ad6..e88620be04ae598f6c9310e66751ae7967880d21 100644 --- a/MODULES +++ b/MODULES @@ -22,6 +22,16 @@ elif [[ $BSC_MACHINE == "mn5" ]]; then source /gpfs/projects/bsc32/software/suselinux/11/software/Miniconda3/4.7.10/etc/profile.d/conda.sh conda activate /gpfs/projects/bsc32/repository/apps/conda_envs/SUNSET-env_2.0.0 +elif [[ $BSC_MACHINE == "amd" ]]; then + + module purge + module load CDO/1.9.10-foss-2019b + module load R/4.1.2-foss-2019b + module load GEOS/3.7.2-foss-2019b-Python-3.7.4 + module load GDAL/3.5.0-foss-2019b-Python-3.7.4 + module load PROJ/9.0.0-GCCcore-8.3.0 + module load Phantomjs/2.1.1 + elif [[ $HOSTNAME == "bsceshub02.bsc.es" ]]; then module purge diff --git a/autosubmit/auto-multimodel.sh b/autosubmit/auto-multimodel.sh index a9912666046bf3d8fc33643f764d864fa390963d..0ca01aebab0cac73d504afd010a92a983f9d92e2 100644 --- a/autosubmit/auto-multimodel.sh +++ b/autosubmit/auto-multimodel.sh @@ -1,18 +1,25 @@ #!/bin/bash ############ AUTOSUBMIT INPUTS ############ -proj_dir=%PROJDIR% +projdir=%PROJDIR% outdir=%common.OUTDIR% script=%common.SCRIPT% +tmpdir=%common.TMPDIR% SPLIT=%SPLIT% ############################### -cd $proj_dir +if [ -d $projdir ]; then + cd $projdir + srcdir=${outdir} +else + cd ${tmpdir}/sunset/$(basename ${outdir})/ + srcdir=${tmpdir}/$(basename ${outdir}) +fi -source split_to_recipe -# atomic_recipe_number=$(printf "%02d" $CHUNK) -atomic_recipe=${outdir}/logs/recipes/multimodel/atomic_recipe_sys-Multimodel${recipe}.yml +atomic_recipe=${srcdir}/logs/recipes/multimodel/atomic_recipe_sys-Multimodel${recipe}.yml +set +eu source MODULES +set -eu Rscript ${script} ${atomic_recipe} diff --git a/autosubmit/auto-scorecards.sh b/autosubmit/auto-scorecards.sh index c30f643f3be53f216ead66675a9545a0e159198a..7722e3b5d2bce0a3e43fae7726ffe671d68b2a30 100644 --- a/autosubmit/auto-scorecards.sh +++ b/autosubmit/auto-scorecards.sh @@ -1,15 +1,24 @@ #!/bin/bash ############ AUTOSUBMIT INPUTS ############ -proj_dir=%PROJDIR% +projdir=%PROJDIR% outdir=%common.OUTDIR% recipe=%common.RECIPE% +tmpdir=%common.TMPDIR% ############################### -cd $proj_dir +if [ -d $projdir ]; then + cd $projdir + srcdir=${outdir} +else + cd ${tmpdir}/sunset/$(basename ${outdir})/ + srcdir=${tmpdir}/$(basename ${outdir}) +fi -recipe=${outdir}/logs/recipes/${recipe} +recipe=${srcdir}/logs/recipes/${recipe} +set +eu source MODULES +set -eu -Rscript modules/Scorecards/execute_scorecards.R ${recipe} ${outdir} +Rscript modules/Scorecards/execute_scorecards.R ${recipe} ${srcdir} diff --git a/autosubmit/auto-transfer_recipes.sh b/autosubmit/auto-transfer_recipes.sh new file mode 100644 index 0000000000000000000000000000000000000000..4b332e8bd3ad45cab9fa9970bdbd14fd42404ad4 --- /dev/null +++ b/autosubmit/auto-transfer_recipes.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +############ AUTOSUBMIT INPUTS ############ +proj_dir=%PROJDIR% +outdir=%common.OUTDIR% +tmpdir=%common.TMPDIR% +############################### + +# cd $proj_dir + +# username=$(whoami) + +destdir=${tmpdir}/$(basename ${outdir}) + +# Copy recipes +mkdir -p $destdir +cp -r /gpfs/archive/bsc32/${outdir}/* ${destdir} + +# Copy repository with code +mkdir -p ${tmpdir}/sunset/$(basename ${outdir})/ +cp -r /gpfs/archive/bsc32/${proj_dir}/* ${tmpdir}/sunset/$(basename ${outdir})/ diff --git a/autosubmit/auto-transfer_results.sh b/autosubmit/auto-transfer_results.sh new file mode 100644 index 0000000000000000000000000000000000000000..fda0d7681702fe667fa74d1e53c03de9286a98d8 --- /dev/null +++ b/autosubmit/auto-transfer_results.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +############ AUTOSUBMIT INPUTS ############ +proj_dir=%PROJDIR% +outdir=%common.OUTDIR% +tmpdir=%common.TMPDIR% +############################### + +srcdir=${tmpdir}/$(basename ${outdir}) + +cp -r ${srcdir}/* /gpfs/archive/bsc32/${outdir} + +rm -r $srcdir + +rm -r ${tmpdir}/sunset/$(basename ${outdir})/ diff --git a/autosubmit/auto-verification.sh b/autosubmit/auto-verification.sh index e909dbfb433ff6d59816734973547e918513bf76..d323c45103c593933eccfa18758a58d9c28e5e8c 100644 --- a/autosubmit/auto-verification.sh +++ b/autosubmit/auto-verification.sh @@ -1,19 +1,27 @@ #!/bin/bash ############ AUTOSUBMIT INPUTS ############ -proj_dir=%PROJDIR% +projdir=%PROJDIR% outdir=%common.OUTDIR% +tmpdir=%common.TMPDIR% script=%common.SCRIPT% CHUNK=%CHUNK% ############################### -cd $proj_dir +if [ -d $projdir ]; then + cd $projdir + srcdir=${outdir} +else + cd ${tmpdir}/sunset/$(basename ${outdir})/ + srcdir=${tmpdir}/$(basename ${outdir}) +fi source chunk_to_recipe -# atomic_recipe_number=$(printf "%02d" $CHUNK) -atomic_recipe=${outdir}/logs/recipes/atomic_recipe_${recipe}.yml +atomic_recipe=${srcdir}/logs/recipes/atomic_recipe_${recipe}.yml +set +eu source MODULES +set -eu Rscript ${script} ${atomic_recipe} diff --git a/autosubmit/conf_gpfs/autosubmit.yml b/autosubmit/conf_gpfs/autosubmit.yml new file mode 100644 index 0000000000000000000000000000000000000000..0fd5d5c6aaf61945d131da77cda08d8d1fdd86cd --- /dev/null +++ b/autosubmit/conf_gpfs/autosubmit.yml @@ -0,0 +1,22 @@ +config: + EXPID: + AUTOSUBMIT_VERSION: 4.0.0b0 + MAXWAITINGJOBS: 16 + # Default maximum number of jobs to be running at the same time at any platform + # Default: 6 + TOTALJOBS: 16 + SAFETYSLEEPTIME: 10 + RETRIALS: 0 +mail: + NOTIFICATIONS: + TO: +communications: + # Communications library used to connect with platforms: paramiko or saga. + # Default: paramiko + API: paramiko +storage: + # Defines the way of storing the progress of the experiment. The available options are: + # A PICKLE file (pkl) or an SQLite database (db). Default: pkl + TYPE: pkl + # Defines if the remote logs will be copied to the local platform. Default: True. + COPY_REMOTE_LOGS: True diff --git a/autosubmit/conf_gpfs/expdef.yml b/autosubmit/conf_gpfs/expdef.yml new file mode 100644 index 0000000000000000000000000000000000000000..8dc29b27843729afa89be242a0d0de96bad1b3ec --- /dev/null +++ b/autosubmit/conf_gpfs/expdef.yml @@ -0,0 +1,44 @@ +DEFAULT: + EXPID: + HPCARCH: nord3v2 +experiment: + DATELIST: + MEMBERS: fc0 + CHUNKSIZEUNIT: month + CHUNKSIZE: 1 + NUMCHUNKS: + CHUNKINI: 1 + CALENDAR: standard +project: + PROJECT_TYPE: local + # Destination folder name for project. type: STRING, default: leave empty, + PROJECT_DESTINATION: auto-s2s +# If PROJECT_TYPE is not git, no need to change +git: + # Repository URL STRING: 'https://github.com/torvalds/linux.git' + PROJECT_ORIGIN: https://earth.bsc.es/gitlab/es/auto-s2s.git + # Select branch or tag, STRING, default: 'master', help: {'master' (default), 'develop', 'v3.1b', ...} + PROJECT_BRANCH: master + # type: STRING, default: leave empty, help: if model branch is a TAG leave empty + PROJECT_COMMIT: '' +svn: + PROJECT_URL: '' + PROJECT_REVISION: '' +# If PROJECT_TYPE is not local, no need to change +local: + # type: STRING, help: /foo/bar/ecearth + PROJECT_PATH: /esarchive/scratch/vagudets/repos/auto-s2s/ +# If PROJECT_TYPE is none, no need to change +project_files: + # Where is PROJECT CONFIGURATION file location relative to project root path + FILE_PROJECT_CONF: '' + # Where is JOBS CONFIGURATION file location relative to project root path + FILE_JOBS_CONF: '' + # Default job scripts type in the project. type: STRING, default: bash, supported: 'bash', 'python' or 'r' + JOB_SCRIPTS_TYPE: '' +rerun: + # Is a rerun or not? [Default: Do set FALSE]. BOOLEAN: TRUE, FALSE + RERUN: FALSE + # If RERUN: TRUE then supply the list of chunks to rerun + # LIST: [ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ] + CHUNKLIST: '' diff --git a/autosubmit/conf_gpfs/jobs.yml b/autosubmit/conf_gpfs/jobs.yml new file mode 100644 index 0000000000000000000000000000000000000000..9f9ce28148dd78dc3fcc97c268236475c47b3004 --- /dev/null +++ b/autosubmit/conf_gpfs/jobs.yml @@ -0,0 +1,41 @@ +JOBS: + transfer_recipes: + FILE: autosubmit/auto-transfer_recipes.sh + RUNNING: once + WALLCLOCK: 00:10 + PLATFORM: transfer + NOTIFY_ON: + PROCESSORS: 1 + verification: + FILE: autosubmit/auto-verification.sh + RUNNING: chunk + WALLCLOCK: + NOTIFY_ON: + PLATFORM: + PROCESSORS: + # SPLITS: # n_atomic_recipes, number of atomic recipes + multimodel: + FILE: autosubmit/auto-multimodel.sh + RUNNING: once + WALLCLOCK: + NOTIFY_ON: + PLATFORM: + PROCESSORS: + DEPENDENCIES: + verification: + SPLITS_FROM: + SPLITS: # n_atomic_recipes/n_models = n_multimodels + scorecards: + FILE: autosubmit/auto-scorecards.sh + WALLCLOCK: 00:10 + PLATFORM: + NOTIFY_ON: + PROCESSORS: 1 + DEPENDENCIES: + transfer_results: + FILE: autosubmit/auto-transfer_results.sh + RUNNING: once + WALLCLOCK: 00:10 + PLATFORM: transfer + NOTIFY_ON: + PROCESSORS: 1 diff --git a/autosubmit/conf_gpfs/platforms.yml b/autosubmit/conf_gpfs/platforms.yml new file mode 100644 index 0000000000000000000000000000000000000000..03f4b9401d687792ca8b0554c7dd94cd95865294 --- /dev/null +++ b/autosubmit/conf_gpfs/platforms.yml @@ -0,0 +1,33 @@ +## TODO: Change platform +Platforms: + nord3v2: + TYPE: slurm + HOST: nord4.bsc.es + USER: + PROJECT: bsc32 ## TO BE CHANGED + SCRATCH_DIR: /gpfs/scratch/ ## TO BE CHANGED + PROCESSORS_PER_NODE: 16 + SERIAL_QUEUE: debug + QUEUE: bsc_es + mn5: + TYPE: slurm + HOST: glogin1.bsc.es + PROJECT: bsc32 + USER: + QUEUE: gp_bsces + SCRATCH_DIR: /gpfs/scratch/ + TEMP_DIR: '' + amd: + TYPE: slurm + HOST: amdlogin1.bsc.es + PROJECT: bsc32 + USER: + SCRATCH_DIR: /gpfs/scratch/ + TEMP_DIR: '' + transfer: + TYPE: slurm + HOST: transfer2.bsc.es + PROJECT: bsc32 + SCRATCH_DIR: /gpfs/scratch/ + USER: + diff --git a/autosubmit/conf_gpfs/proj.yml b/autosubmit/conf_gpfs/proj.yml new file mode 100644 index 0000000000000000000000000000000000000000..679cf63b1ced38fd833d28ea9acfa145a1e9bc4f --- /dev/null +++ b/autosubmit/conf_gpfs/proj.yml @@ -0,0 +1,4 @@ +common: + MODULES: "MODULES" + OUTDIR: + SCRIPT: diff --git a/conf/archive_seasonal.yml b/conf/archive_seasonal.yml index 4ab68ab118ffaa186811c74e17820681c118f6ed..a9caa6fd9983e2a385756a04a1562d861a9ec0f3 100644 --- a/conf/archive_seasonal.yml +++ b/conf/archive_seasonal.yml @@ -33,7 +33,7 @@ gpfs: hcst: 25 calendar: "proleptic_gregorian" time_stamp_lag: "0" - reference_grid: "/gpfs/projects/bsc32/esarchive_cache/exp/ecmwf/system51c3s/monthly_mean/tas_f6h/tas_20180501.nc" + reference_grid: "conf/grid_description/griddes_system51c3s.txt" land_sea_mask: "/gpfs/projects/bsc32/esarchive_cache/exp/ecmwf/system51c3s/constant/lsm/lsm.nc" CMCC-SPS3.5: name: "CMCC-SPS3.5" diff --git a/conf/autosubmit.yml b/conf/autosubmit.yml index 25872a0ef510b95ebf0f8dbe911bf5a544d3d5ff..99b29f3c8022de791317ce78d2312143ba916c5b 100644 --- a/conf/autosubmit.yml +++ b/conf/autosubmit.yml @@ -5,6 +5,7 @@ esarchive: conf_format: yaml experiment_dir: /esarchive/autosubmit/ userID: bsc032 + tmp_dir: mars: platform: NORD3 ## TO BE CHANGED module_version: autosubmit/4.0.0b-foss-2015a-Python-3.7.3 ## TO BE CHANGED @@ -12,3 +13,12 @@ mars: conf_format: yaml experiment_dir: /esarchive/autosubmit/ ## TO BE CHANGED userID: bsc32 ## TO BE CHANGED + tmp_dir: +gpfs: + platform: mn5 + module_version: autosubmit/4.0.98-foss-2015a-Python-3.7.3 + auto_version: 4.0.98 + conf_format: yaml + experiment_dir: /esarchive/autosubmit/ ## TO BE CHANGED + userID: bsc032 + tmp_dir: /gpfs/scratch/bsc32/$user$/tmp/ diff --git a/launch_SUNSET.sh b/launch_SUNSET.sh index 6149a9639604942f58897363dd0c854f010b79a2..deb4f1124688c9b030ac7bf4e138d4cfc6b6c1b1 100644 --- a/launch_SUNSET.sh +++ b/launch_SUNSET.sh @@ -127,6 +127,13 @@ if [[ $run_method == "sbatch" ]]; then logdir=${outdir}/logs/slurm/ mkdir -p $logdir echo "Slurm job logs will be stored in $logdir" + + # Is machine MN5? + if [[ $BSC_MACHINE == "mn5" ]]; then + platform_params="-A bsc32 -q gp_bsces" + else + platform_params="" + fi # Launch one job per atomic recipe cd $codedir @@ -141,7 +148,7 @@ if [[ $run_method == "sbatch" ]]; then outfile=${logdir}/run-${job_name}.out errfile=${logdir}/run-${job_name}.err # Send batch job and capture job ID - job_ID=$(sbatch --parsable --job-name="SUNSET_verification" --output=$outfile --error=$errfile --time=$wallclock --cpus-per-task=$cpus $custom_directives conf/slurm_templates/run_parallel_workflow.sh ${script} ${atomic_recipe}) + job_ID=$(sbatch --parsable --job-name="SUNSET_verification" $platform_params --output=$outfile --error=$errfile --time=$wallclock --cpus-per-task=$cpus $custom_directives conf/slurm_templates/run_parallel_workflow.sh ${script} ${atomic_recipe}) # Add job ID to array verification_job_list+=($job_ID) echo "Submitted batch job $job_ID" @@ -156,7 +163,7 @@ if [[ $run_method == "sbatch" ]]; then outfile=${logdir}/run-multimodel-${job_name}.out errfile=${logdir}/run-multimodel-${job_name}.err # Send batch job and capture job ID - job_ID=$(sbatch --parsable --dependency=afterok:$(IFS=,; echo "${verification_job_list[*]}") --kill-on-invalid-dep=yes --job-name="SUNSET_multimodel" --output=$outfile --error=$errfile --time=$wallclock --cpus-per-task=$cpus $custom_directives conf/slurm_templates/run_parallel_workflow.sh ${script} ${atomic_recipe}) + job_ID=$(sbatch --parsable --dependency=afterok:$(IFS=,; echo "${verification_job_list[*]}") --kill-on-invalid-dep=yes --job-name="SUNSET_multimodel" $platform_params --output=$outfile --error=$errfile --time=$wallclock --cpus-per-task=$cpus $custom_directives conf/slurm_templates/run_parallel_workflow.sh ${script} ${atomic_recipe}) # Add job ID to array multimodel_job_list+=($job_ID) echo "Submitted batch job $job_ID" @@ -171,7 +178,7 @@ if [[ $run_method == "sbatch" ]]; then echo "Submitting scorecards jobs..." outfile=${logdir}/run-scorecards.out errfile=${logdir}/run-scorecards.err - sbatch --dependency=afterok:$(IFS=,; echo "${verification_job_list[*]} ${multimodel_job_list[*]}") --output=$outfile --error=$errfile --time=01:00:00 conf/slurm_templates/run_scorecards.sh ${recipe} ${outdir} + sbatch --dependency=afterok:$(IFS=,; echo "${verification_job_list[*]} ${multimodel_job_list[*]}") $platform_params --output=$outfile --error=$errfile --time=01:00:00 conf/slurm_templates/run_scorecards.sh ${recipe} ${outdir} fi fi diff --git a/modules/Scorecards/Scorecards_calculations.R b/modules/Scorecards/Scorecards_calculations.R index 093aa4260d7e6f36209b71d6e3579f469f43383e..f9f0f478a2a8b2fda78518b6bc93fdd38ebb7fbd 100644 --- a/modules/Scorecards/Scorecards_calculations.R +++ b/modules/Scorecards/Scorecards_calculations.R @@ -360,7 +360,6 @@ Scorecards_calculations <- function(recipe, data, skill_metrics, } ## Save metric result in array - browser() aggr_metrics[,,,which(metrics == met)] <- Reorder(data = mean_bias, order = c('var', 'time', 'region')) aggr_significance[,,,which(metrics == met)] <- Reorder(data = sign_mean_bias, diff --git a/modules/Scorecards/execute_scorecards.R b/modules/Scorecards/execute_scorecards.R index 2fa27549a20f414ae20d3db188b7918bd5ec26b0..763877b62a2bbf94b98b188bd3cb6207e0e831a2 100644 --- a/modules/Scorecards/execute_scorecards.R +++ b/modules/Scorecards/execute_scorecards.R @@ -1,5 +1,6 @@ source('tools/libs.R') source('modules/Scorecards/Scorecards_plotting.R') +Sys.setenv(OPENSSL_CONF="/dev/null") args = commandArgs(trailingOnly = TRUE) recipe_file <- args[1] diff --git a/recipes/examples/scorecards_on_MN5.yml b/recipes/examples/scorecards_on_MN5.yml new file mode 100644 index 0000000000000000000000000000000000000000..32decb69b7a9ccd4c3d836a5763949aafba5ae83 --- /dev/null +++ b/recipes/examples/scorecards_on_MN5.yml @@ -0,0 +1,100 @@ +Description: + Author: An-Chi Ho + Info: Compute Skills and Plot Scorecards with Autosubmit + +Analysis: + Horizon: seasonal + Variables: + - {name: tas, freq: monthly_mean} + Datasets: + System: # multiple systems for single model, split if Multimodel = F + - {name: ECMWF-SEAS5.1} + Multimodel: + execute: False # single option + Reference: + - {name: ERA5} + Time: + sdate: # list, split + - '0101' + - '0201' + - '0301' + - '0401' + - '0501' + - '0601' + - '0701' + - '0801' + - '0901' + - '1001' + - '1101' + - '1201' + fcst_year: + hcst_start: '1993' # single option + hcst_end: '2003' # single option + ftime_min: 1 # single option + ftime_max: 6 # single option + Region: # multiple lists, split? Add region name if length(Region) > 1 + - {name: "global", latmin: -90, latmax: 90, lonmin: 0, lonmax: 359.9} + Regrid: + method: bilinear + type: to_system + Workflow: + Anomalies: + compute: yes + cross_validation: no + save: 'none' + Calibration: + method: raw + save: 'none' + Skill: + metric: mean_bias EnsCorr rps rpss crps crpss EnsSprErr # list, don't split + cross_validation: yes + save: 'all' + Statistics: + metric: cov std n_eff spread + save: 'all' + Probabilities: + percentiles: [[1/3, 2/3]] + save: 'none' + Scorecards: + execute: yes # yes/no + regions: + Extra-tropical NH: {lon.min: 0, lon.max: 360, lat.min: 30, lat.max: 90} + Tropics: {lon.min: 0, lon.max: 360, lat.min: -30, lat.max: 30} + Extra-tropical SH : {lon.min: 0, lon.max: 360, lat.min: -30, lat.max: -90} + start_months: 'all' + metric: mean_bias enscorr rpss crpss enssprerr + metric_aggregation: 'score' + inf_to_na: TRUE # Optional, bool: set inf values in data to NA, default is FALSE table_label: NULL + fileout_label: NULL + col1_width: NULL + col2_width: NULL + calculate_diff: FALSE + ncores: 8 + remove_NAs: no # bool, don't split + Output_format: Scorecards # string, don't split + +################################################################################ +## Run CONFIGURATION +################################################################################ +Run: + Loglevel: INFO + Terminal: yes + filesystem: gpfs + output_dir: /esarchive/scratch/vagudets/auto-s2s-outputs/ + code_dir: /home/Earth/vagudets/git/auto-s2s/ + autosubmit: yes + # fill only if using autosubmit + auto_conf: + script: use_cases/ex1_2_autosubmit_scorecards/ex1_2-script.R # replace with the path to your script + expid: a7rn # replace with your EXPID + hpc_user: bsc032762 # replace with your hpc username + wallclock: 03:00 # hh:mm + processors_per_job: 16 + platform: mn5 + custom_directives: # ['#SBATCH --exclusive'] + email_notifications: yes # enable/disable email notifications. Change it if you want to. + email_address: victoria.agudetse@bsc.es # replace with your email address + notify_completed: yes # notify me by email when a job finishes + notify_failed: no # notify me by email when a job fails + + diff --git a/recipes/examples/scorecards_on_nord3v2.yml b/recipes/examples/scorecards_on_nord3v2.yml new file mode 100644 index 0000000000000000000000000000000000000000..06333966f7934c7b178e860922ddc193e9f25356 --- /dev/null +++ b/recipes/examples/scorecards_on_nord3v2.yml @@ -0,0 +1,100 @@ +Description: + Author: An-Chi Ho + Info: Compute Skills and Plot Scorecards with Autosubmit + +Analysis: + Horizon: seasonal + Variables: + - {name: tas, freq: monthly_mean} + Datasets: + System: # multiple systems for single model, split if Multimodel = F + - {name: ECMWF-SEAS5.1} + Multimodel: + execute: False # single option + Reference: + - {name: ERA5} + Time: + sdate: # list, split + - '0101' + - '0201' + - '0301' + - '0401' + - '0501' + - '0601' + - '0701' + - '0801' + - '0901' + - '1001' + - '1101' + - '1201' + fcst_year: + hcst_start: '1993' # single option + hcst_end: '2003' # single option + ftime_min: 1 # single option + ftime_max: 6 # single option + Region: # multiple lists, split? Add region name if length(Region) > 1 + - {name: "global", latmin: -90, latmax: 90, lonmin: 0, lonmax: 359.9} + Regrid: + method: bilinear + type: to_system + Workflow: + Anomalies: + compute: yes + cross_validation: no + save: 'none' + Calibration: + method: raw + save: 'none' + Skill: + metric: mean_bias EnsCorr rps rpss crps crpss EnsSprErr # list, don't split + cross_validation: yes + save: 'all' + Statistics: + metric: cov std n_eff spread + save: 'all' + Probabilities: + percentiles: [[1/3, 2/3]] + save: 'none' + Scorecards: + execute: yes # yes/no + regions: + Extra-tropical NH: {lon.min: 0, lon.max: 360, lat.min: 30, lat.max: 90} + Tropics: {lon.min: 0, lon.max: 360, lat.min: -30, lat.max: 30} + Extra-tropical SH : {lon.min: 0, lon.max: 360, lat.min: -30, lat.max: -90} + start_months: 'all' + metric: mean_bias enscorr rpss crpss enssprerr + metric_aggregation: 'score' + inf_to_na: TRUE # Optional, bool: set inf values in data to NA, default is FALSE table_label: NULL + fileout_label: NULL + col1_width: NULL + col2_width: NULL + calculate_diff: FALSE + ncores: 8 + remove_NAs: no # bool, don't split + Output_format: Scorecards # string, don't split + +################################################################################ +## Run CONFIGURATION +################################################################################ +Run: + Loglevel: INFO + Terminal: yes + filesystem: esarchive + output_dir: /esarchive/scratch/vagudets/auto-s2s-outputs/ + code_dir: /home/Earth/vagudets/git/auto-s2s/ + autosubmit: yes + # fill only if using autosubmit + auto_conf: + script: use_cases/ex1_2_autosubmit_scorecards/ex1_2-script.R # replace with the path to your script + expid: a6wq # replace with your EXPID + hpc_user: bsc032762 # replace with your hpc username + wallclock: 03:00 # hh:mm + processors_per_job: 16 + platform: nord3v2 + custom_directives: ['#SBATCH --exclusive'] + email_notifications: yes # enable/disable email notifications. Change it if you want to. + email_address: victoria.agudetse@bsc.es # replace with your email address + notify_completed: yes # notify me by email when a job finishes + notify_failed: no # notify me by email when a job fails + + diff --git a/tools/divide_recipe.R b/tools/divide_recipe.R index ceb696f44a1a7979c9bc5a94a522413157a13976..c478b20dedd81a1f0a0a1645b34226ade21950dc 100644 --- a/tools/divide_recipe.R +++ b/tools/divide_recipe.R @@ -21,6 +21,10 @@ divide_recipe <- function(recipe) { Run = recipe$Run[c("Loglevel", "output_dir", "Terminal", "code_dir", "logfile", "filesystem")]) + if (!is.null(recipe$Run$tmp_dir)) { + beta_recipe$Run$tmp_dir <- recipe$Run$tmp_dir + } + # duplicate recipe by independent variables: # If a single variable is not given inside a list, rebuild structure if (any(c("name", "freq", "units") %in% names(recipe$Analysis$Variables))) { diff --git a/tools/prepare_outputs.R b/tools/prepare_outputs.R index eab9c4fe63ff7a3070f008451cc32f1617e86dd9..da192448ab2885165672b8bf3296cb103d5c7d6b 100644 --- a/tools/prepare_outputs.R +++ b/tools/prepare_outputs.R @@ -32,7 +32,7 @@ prepare_outputs <- function(recipe_file, recipe$recipe_path <- recipe_file recipe$name <- tools::file_path_sans_ext(basename(recipe_file)) - output_dir = recipe$Run$output_dir + output_dir <- recipe$Run$output_dir # Create output folders if (!uniqueID) { folder_name <- paste0(gsub(".yml", "", gsub("/", "_", recipe$name))) @@ -127,9 +127,18 @@ prepare_outputs <- function(recipe_file, } else { recipe <- check_recipe(recipe) } + + # If autosubmit is used, a temporary directory needs to be created if the + # filesystem is not accessible from the autosubmit machine. + tmp_dir <- read_yaml("conf/autosubmit.yml")[[recipe$Run$filesystem]]$tmp_dir + if (recipe$Run$autosubmit && !(is.null(tmp_dir))) { + tmp_dir <- gsub("\\$.+?\\$", recipe$Run$auto_conf$hpc_user, tmp_dir) + recipe$Run$tmp_dir <- file.path(tmp_dir, folder_name) + } # Create a copy of the recipe and remove the logger recipe_copy <- recipe recipe_copy$Run$logger <- NULL + # Copy recipe to output folder write_yaml(recipe_copy, file = file.path(output_dir, folder_name, 'logs', 'recipes', diff --git a/tools/read_atomic_recipe.R b/tools/read_atomic_recipe.R index 3cb11e7ed6b1d3cb91a765debeaeebd2076ddedc..99f001554d030cf2ccee80a185f8cea71f9bc0ae 100644 --- a/tools/read_atomic_recipe.R +++ b/tools/read_atomic_recipe.R @@ -26,6 +26,10 @@ read_atomic_recipe <- function(recipe_file) { recipe <- read_yaml(recipe_file, eval.exp = TRUE) recipe$recipe_path <- recipe_file recipe$name <- tools::file_path_sans_ext(basename(recipe_file)) + # Handle temporary directory for autosubmit + if (!is.null(recipe$Run$tmp_dir)) { + recipe$Run$output_dir <- recipe$Run$tmp_dir + } # Create log file for atomic recipe logfile <- file.path(recipe$Run$output_dir, 'logs', paste0(recipe$name, '.log')) @@ -51,6 +55,7 @@ read_atomic_recipe <- function(recipe_file) { appenders = list(file_appender(logfile, append = TRUE, layout = default_log_layout()))) } + recipe$Run$logger <- logger recipe$Run$logfile <- logfile # Restructure recipe to flatten redundant lists diff --git a/tools/write_autosubmit_conf.R b/tools/write_autosubmit_conf.R index 5f3981974f85d74ae880928f918f419a2ea65462..a6bc5aa04690b5a196c31fd694883bfe5ebb7fcd 100644 --- a/tools/write_autosubmit_conf.R +++ b/tools/write_autosubmit_conf.R @@ -23,6 +23,10 @@ write_autosubmit_conf <- function(recipe, nchunks, # Output directory dest_dir <- paste0(auto_specs$experiment_dir, expid, "/conf/") proj_dir <- paste0(auto_specs$experiment_dir, expid, "/proj/auto-s2s/") + tmp_dir <- auto_specs$tmp_dir + # HPC details + platform <- recipe$Run$auto_conf$platform + hpc_user <- recipe$Run$auto_conf$hpc_user # Create project directory if it does not exist yet so that chunk_to_recipe # and split_to_recipe files can be created if (!dir.exists(proj_dir)) { @@ -58,28 +62,41 @@ write_autosubmit_conf <- function(recipe, nchunks, ## wallclock, notify_on, platform?, processors, # Create bash file to associate chunk number to recipe name chunk_file <- paste0(proj_dir, "chunk_to_recipe") + ## TODO: Specify dir in chunk_to_recipe? .create_bash_file(fileout = chunk_file, dictionary = chunk_to_recipe, variable = "CHUNK") # Define job parameters - conf$JOBS$verification$WALLCLOCK <- recipe$Run$auto_conf$wallclock + notify_on <- "" if (recipe$Run$auto_conf$notify_completed) { - conf$JOBS$verification$NOTIFY_ON <- paste(conf$JOBS$verification$NOTIFY_ON, - "COMPLETED") + notify_on <- "COMPLETED" } if (recipe$Run$auto_conf$notify_failed) { - conf$JOBS$verification$NOTIFY_ON <- paste(conf$JOBS$verification$NOTIFY_ON, - "FAILED") + notify_on <- paste(notify_on, "FAILED") } + # Define last job before file transfer + last_job <- "verification" + # Verification job + conf$JOBS$verification$WALLCLOCK <- recipe$Run$auto_conf$wallclock + conf$JOBS$verification$NOTIFY_ON <- notify_on conf$JOBS$verification$PROCESSORS <- recipe$Run$auto_conf$processors_per_job # ncores? conf$JOBS$verification$CUSTOM_DIRECTIVES <- recipe$Run$auto_conf$custom_directives - # Only include Multimodel job if sections exists in the recipe + conf$JOBS$verification$PLATFORM <- platform + # Recipe transfer job: only include if a tmp_dir is defined + if (!is.null(tmp_dir)) { + conf$JOBS$verification$DEPENDENCIES <- "transfer_recipes" + conf$JOBS$transfer_recipes$NOTIFY_ON <- notify_on + } else { + conf$JOBS$transfer_recipes <- NULL + } + # Multimodel job: Only include if section exists in the recipe # is set to execute = 'True' or 'both' if (!is.null(recipe$Analysis$Datasets$Multimodel) && tolower(recipe$Analysis$Datasets$Multimodel$execute) == "false") { conf$JOBS$multimodel <- NULL conf$JOBS$scorecards$DEPENDENCIES <- "verification" } else { + conf$JOBS$multimodel$PLATFORM <- platform conf$JOBS$scorecards$DEPENDENCIES <- "multimodel" # Create bash file to associate split number to recipe name split_file <- paste0(proj_dir, "split_to_recipe") @@ -97,45 +114,52 @@ write_autosubmit_conf <- function(recipe, nchunks, # 'Splits' parameter should be the number of mulimodel jobs conf$JOBS$multimodel$SPLITS <- length(mm_dependencies) # Define the rest of the parameters - if (recipe$Run$auto_conf$notify_completed) { - conf$JOBS$multimodel$NOTIFY_ON <- paste(conf$JOBS$multimodel$NOTIFY_ON, - "COMPLETED") - } - if (recipe$Run$auto_conf$notify_failed) { - conf$JOBS$multimodel$NOTIFY_ON <- paste(conf$JOBS$multimodel$NOTIFY_ON, - "FAILED") - } - + conf$JOBS$multimodel$NOTIFY_ON <- notify_on conf$JOBS$multimodel$PROCESSORS <- recipe$Run$auto_conf$processors_multimodel conf$JOBS$multimodel$CUSTOM_DIRECTIVES <- recipe$Run$auto_conf$custom_directives_multimodel conf$JOBS$multimodel$WALLCLOCK <- recipe$Run$auto_conf$wallclock_multimodel + last_job <- "multimodel" } - # Only include Scorecards job if section exists in the recipe and + # Scorecards job: include only if section exists in the recipe and # is set to 'execute: True' if (!("Scorecards" %in% names(recipe$Analysis$Workflow)) || (!recipe$Analysis$Workflow$Scorecards$execute)) { conf$JOBS$scorecards <- NULL } else { - if (recipe$Run$auto_conf$notify_completed) { - conf$JOBS$scorecards$NOTIFY_ON <- paste(conf$JOBS$scorecards$NOTIFY_ON, - "COMPLETED") - } - if (recipe$Run$auto_conf$notify_failed) { - conf$JOBS$scorecards$NOTIFY_ON <- paste(conf$JOBS$scorecards$NOTIFY_ON, - "FAILED") - } + conf$JOBS$scorecards$NOTIFY_ON <- notify_on + conf$JOBS$scorecards$PLATFORM <- platform + last_job <- "scorecards" + } + # Results transfer job: only include if a tmp_dir is defined + if (!is.null(tmp_dir)) { + conf$JOBS$transfer_results$DEPENDENCIES <- last_job + conf$JOBS$transfer_results$NOTIFY_ON <- notify_on + } else { + conf$JOBS$transfer_results <- NULL } } else if (conf_type == "platforms") { + ## TODO: Allow user to choose platform # Section 4: platform configuration ## nord3v2 configuration... platform name? user, processors_per_node - conf$Platforms[[auto_specs$platform]]$USER <- - recipe$Run$auto_conf$hpc_user + conf$Platforms[[platform]]$USER <- hpc_user + unused_machines <- names(conf$Platforms)[!names(conf$Platforms) %in% c("transfer", platform)] + conf$Platforms[unused_machines] <- NULL + if (!is.null(tmp_dir)) { + conf$Platforms$transfer$USER <- hpc_user + } else { + conf$Platforms$transfer <- NULL + } } else if (conf_type == "proj") { # Section 5: proj ## modules? Info that goes on script, e.g. output directory conf$common$OUTDIR <- recipe$Run$output_dir conf$common$SCRIPT <- recipe$Run$auto_conf$script conf$common$RECIPE <- paste0(recipe$name, ".yml") + if (!is.null(tmp_dir)) { + conf$common$TMPDIR <- gsub("\\$.+?\\$", hpc_user, tmp_dir) + } else { + conf$common$TMPDIR <- recipe$Run$output_dir + } } # Write config file inside autosubmit dir write.config(conf, paste0(dest_dir, dest_file),