diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8c72037204e6310847da6daed5bc7fb7b336f59d..997870c821fec43485f1923b6802f853991c5633 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,6 +7,9 @@ unit-test-seasonal: # This job runs in the test stage. - echo "Loading modules..." - module load R/4.1.2-foss-2015a-bare - module load CDO/1.9.8-foss-2015a + - module load GEOS/3.7.2-foss-2015a-Python-3.7.3 + - module load GDAL/2.2.1-foss-2015a + - module load PROJ/4.8.0-foss-2015a - module list - echo "Running seasonal unit tests..." - Rscript ./tests/test_seasonal.R @@ -17,6 +20,9 @@ unit-test-decadal: # This job runs in the test stage. - echo "Loading modules..." - module load R/4.1.2-foss-2015a-bare - module load CDO/1.9.8-foss-2015a + - module load GEOS/3.7.2-foss-2015a-Python-3.7.3 + - module load GDAL/2.2.1-foss-2015a + - module load PROJ/4.8.0-foss-2015a - module list - echo "Running decadal unit tests..." - Rscript ./tests/test_decadal.R diff --git a/autosubmit/auto-verification.sh b/autosubmit/auto-verification.sh index bbd065563c5b88c6b49fe4339c578534a883eacf..0089e322d63e89a5578c98a6a64a1369b5b9b108 100644 --- a/autosubmit/auto-verification.sh +++ b/autosubmit/auto-verification.sh @@ -2,8 +2,8 @@ ############ AUTOSUBMIT INPUTS ############ proj_dir=%PROJDIR% -outdir=%OUTDIR% -script=%SCRIPT% +outdir=%common.OUTDIR% +script=%common.SCRIPT% CHUNK=%CHUNK% ############################### diff --git a/autosubmit/conf_esarchive/autosubmit.conf b/autosubmit/conf_esarchive/autosubmit.conf deleted file mode 100644 index 685876a1ea82079a39eaa7ff1468f2549d8dc52f..0000000000000000000000000000000000000000 --- a/autosubmit/conf_esarchive/autosubmit.conf +++ /dev/null @@ -1,42 +0,0 @@ -[config] -# Experiment identifier -# No need to change -EXPID = -# No need to change. -# Autosubmit version identifier -AUTOSUBMIT_VERSION = 3.14.0 -# Default maximum number of jobs to be waiting in any platform -# Default = 3 -MAXWAITINGJOBS = 16 -# Default maximum number of jobs to be running at the same time at any platform -# Default = 6 -TOTALJOBS = 16 -# Time (seconds) between connections to the HPC queue scheduler to poll already submitted jobs status -# Default = 10 -SAFETYSLEEPTIME = 10 -# Number of retrials if a job fails. Can be override at job level -# Default = 0 -RETRIALS = 0 - -[mail] -# Enable mail notifications -# Default = False -NOTIFICATIONS = -# Mail address where notifications will be received -TO = - -[communications] -# Communications library used to connect with platforms: paramiko or saga. -# Default = paramiko -API = paramiko - -[storage] -# Defines the way of storing the progress of the experiment. The available options are: -# A PICKLE file (pkl) or an SQLite database (db). Default = pkl -TYPE = pkl -# Defines if the remote logs will be copied to the local platform. Default = True. -COPY_REMOTE_LOGS = True - -[migrate] -# Changes experiment files owner. -TO_USER = diff --git a/autosubmit/conf_esarchive/autosubmit.yml b/autosubmit/conf_esarchive/autosubmit.yml new file mode 100644 index 0000000000000000000000000000000000000000..0fd5d5c6aaf61945d131da77cda08d8d1fdd86cd --- /dev/null +++ b/autosubmit/conf_esarchive/autosubmit.yml @@ -0,0 +1,22 @@ +config: + EXPID: + AUTOSUBMIT_VERSION: 4.0.0b0 + MAXWAITINGJOBS: 16 + # Default maximum number of jobs to be running at the same time at any platform + # Default: 6 + TOTALJOBS: 16 + SAFETYSLEEPTIME: 10 + RETRIALS: 0 +mail: + NOTIFICATIONS: + TO: +communications: + # Communications library used to connect with platforms: paramiko or saga. + # Default: paramiko + API: paramiko +storage: + # Defines the way of storing the progress of the experiment. The available options are: + # A PICKLE file (pkl) or an SQLite database (db). Default: pkl + TYPE: pkl + # Defines if the remote logs will be copied to the local platform. Default: True. + COPY_REMOTE_LOGS: True diff --git a/autosubmit/conf_esarchive/expdef.conf b/autosubmit/conf_esarchive/expdef.conf deleted file mode 100644 index d3a3370ad0a0d8ec62b7bfbea3a34b50e41504d4..0000000000000000000000000000000000000000 --- a/autosubmit/conf_esarchive/expdef.conf +++ /dev/null @@ -1,76 +0,0 @@ -[DEFAULT] -# Experiment identifier -# No need to change -EXPID = -# HPC name. -# No need to change -HPCARCH = nord3v2 - -[experiment] -# Supply the list of start dates. Available formats: YYYYMMDD YYYYMMDDhh YYYYMMDDhhmm -# You can also use an abbreviated syntax for multiple dates with common parts: 200001[01 15] <=> 20000101 20000115 -# 200001[01-04] <=> 20000101 20000102 20000103 20000104 -# DATELIST = 19600101 19650101 19700101 -# DATELIST = 1960[0101 0201 0301] -# DATELIST = 19[60-65] -DATELIST = -# Supply the list of members. Format fcX -# You can also use an abreviated syntax for multiple members: fc[0 1 2] <=> fc0 fc1 fc2 -# fc[0-2] <=> fc0 fc1 fc2 -# MEMBERS = fc0 fc1 fc2 fc3 fc4 -#MEMBERS = fc4 -MEMBERS = fc0 -# Chunk size unit. STRING = hour, day, month, year -CHUNKSIZEUNIT = month -# Chunk size. NUMERIC = 4, 6, 12 -CHUNKSIZE = 1 -# Total number of chunks in experiment. NUMERIC = 30, 15, 10 -NUMCHUNKS = -# Initial chunk of the experiment. Optional. DEFAULT = 1 -CHUNKINI = -# Calendar used. LIST: standard, noleap -CALENDAR = standard - -[project] -# Select project type. STRING = git, svn, local, none -# If PROJECT_TYPE is set to none, Autosubmit self-contained dummy templates will be used -PROJECT_TYPE = local -# Destination folder name for project. type = STRING, default = leave empty, -PROJECT_DESTINATION = auto-s2s - -# If PROJECT_TYPE is not git, no need to change -[git] -# Repository URL STRING = 'https://github.com/torvalds/linux.git' -PROJECT_ORIGIN = https://earth.bsc.es/gitlab/es/auto-s2s.git -# Select branch or tag, STRING, default = 'master', help = {'master' (default), 'develop', 'v3.1b', ...} -PROJECT_BRANCH = master -# type = STRING, default = leave empty, help = if model branch is a TAG leave empty -PROJECT_COMMIT = - -# If PROJECT_TYPE is not svn, no need to change -[svn] -# type = STRING, help = 'https://svn.ec-earth.org/ecearth3' -PROJECT_URL = -# Select revision number. NUMERIC = 1778 -PROJECT_REVISION = - -# If PROJECT_TYPE is not local, no need to change -[local] -# type = STRING, help = /foo/bar/ecearth -PROJECT_PATH = /esarchive/scratch/vagudets/repos/auto-s2s/ - -# If PROJECT_TYPE is none, no need to change -[project_files] -# Where is PROJECT CONFIGURATION file location relative to project root path -FILE_PROJECT_CONF = -# Where is JOBS CONFIGURATION file location relative to project root path -FILE_JOBS_CONF = -# Default job scripts type in the project. type = STRING, default = bash, supported = 'bash', 'python' or 'r' -JOB_SCRIPTS_TYPE = - -[rerun] -# Is a rerun or not? [Default: Do set FALSE]. BOOLEAN = TRUE, FALSE -RERUN = FALSE -# If RERUN = TRUE then supply the list of chunks to rerun -# LIST = [ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ] -CHUNKLIST = diff --git a/autosubmit/conf_esarchive/expdef.yml b/autosubmit/conf_esarchive/expdef.yml new file mode 100644 index 0000000000000000000000000000000000000000..8dc29b27843729afa89be242a0d0de96bad1b3ec --- /dev/null +++ b/autosubmit/conf_esarchive/expdef.yml @@ -0,0 +1,44 @@ +DEFAULT: + EXPID: + HPCARCH: nord3v2 +experiment: + DATELIST: + MEMBERS: fc0 + CHUNKSIZEUNIT: month + CHUNKSIZE: 1 + NUMCHUNKS: + CHUNKINI: 1 + CALENDAR: standard +project: + PROJECT_TYPE: local + # Destination folder name for project. type: STRING, default: leave empty, + PROJECT_DESTINATION: auto-s2s +# If PROJECT_TYPE is not git, no need to change +git: + # Repository URL STRING: 'https://github.com/torvalds/linux.git' + PROJECT_ORIGIN: https://earth.bsc.es/gitlab/es/auto-s2s.git + # Select branch or tag, STRING, default: 'master', help: {'master' (default), 'develop', 'v3.1b', ...} + PROJECT_BRANCH: master + # type: STRING, default: leave empty, help: if model branch is a TAG leave empty + PROJECT_COMMIT: '' +svn: + PROJECT_URL: '' + PROJECT_REVISION: '' +# If PROJECT_TYPE is not local, no need to change +local: + # type: STRING, help: /foo/bar/ecearth + PROJECT_PATH: /esarchive/scratch/vagudets/repos/auto-s2s/ +# If PROJECT_TYPE is none, no need to change +project_files: + # Where is PROJECT CONFIGURATION file location relative to project root path + FILE_PROJECT_CONF: '' + # Where is JOBS CONFIGURATION file location relative to project root path + FILE_JOBS_CONF: '' + # Default job scripts type in the project. type: STRING, default: bash, supported: 'bash', 'python' or 'r' + JOB_SCRIPTS_TYPE: '' +rerun: + # Is a rerun or not? [Default: Do set FALSE]. BOOLEAN: TRUE, FALSE + RERUN: FALSE + # If RERUN: TRUE then supply the list of chunks to rerun + # LIST: [ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ] + CHUNKLIST: '' diff --git a/autosubmit/conf_esarchive/jobs.conf b/autosubmit/conf_esarchive/jobs.conf deleted file mode 100644 index f9d2663486a5bc3655d0c7b554ba7cb63c83cb23..0000000000000000000000000000000000000000 --- a/autosubmit/conf_esarchive/jobs.conf +++ /dev/null @@ -1,14 +0,0 @@ -[verification] -FILE = autosubmit/auto-verification.sh -RUNNING = chunk -WALLCLOCK = -NOTIFY_ON = -PLATFORM = nord3v2 -PROCESSORS = - -[scorecards] -FILE = autosubmit/auto-scorecards.sh -WALLCLOCK = 00:10 -PLATFORM = nord3v2 -PROCESSORS = 1 -DEPENDENCIES = verification diff --git a/autosubmit/conf_esarchive/jobs.yml b/autosubmit/conf_esarchive/jobs.yml new file mode 100644 index 0000000000000000000000000000000000000000..a3c8934bf70f92d15a8e644a6f34947afcc28847 --- /dev/null +++ b/autosubmit/conf_esarchive/jobs.yml @@ -0,0 +1,16 @@ +JOBS: + verification: + FILE: autosubmit/auto-verification.sh + RUNNING: chunk + WALLCLOCK: + NOTIFY_ON: + PLATFORM: nord3v2 + PROCESSORS: + scorecards: + FILE: autosubmit/auto-scorecards.sh + WALLCLOCK: 00:10 + PLATFORM: nord3v2 + NOTIFY_ON: + PROCESSORS: 1 + DEPENDENCIES: verification + diff --git a/autosubmit/conf_esarchive/platforms.conf b/autosubmit/conf_esarchive/platforms.conf deleted file mode 100644 index 0f6819d005b95014642909607f4a787d3d77a495..0000000000000000000000000000000000000000 --- a/autosubmit/conf_esarchive/platforms.conf +++ /dev/null @@ -1,11 +0,0 @@ -[nord3v2] -TYPE = slurm -HOST = nord4.bsc.es -PROJECT = bsc32 -ADD_PROJECT_TO_HOST = False -USER = -SCRATCH_DIR = /gpfs/scratch -PROCESSORS_PER_NODE = 16 -SERIAL_QUEUE = debug -QUEUE = bsc_es -CUSTOM_DIRECTIVES = ["#SBATCH --exclusive"] diff --git a/autosubmit/conf_esarchive/platforms.yml b/autosubmit/conf_esarchive/platforms.yml new file mode 100644 index 0000000000000000000000000000000000000000..78056d62973552432a9e7b55194b8c5e0ecac09a --- /dev/null +++ b/autosubmit/conf_esarchive/platforms.yml @@ -0,0 +1,11 @@ +## TODO: Change platform +Platforms: + nord3v2: + TYPE: slurm + HOST: nord4.bsc.es + USER: + PROJECT: bsc32 ## TO BE CHANGED + SCRATCH_DIR: /gpfs/scratch/ ## TO BE CHANGED + PROCESSORS_PER_NODE: 16 + SERIAL_QUEUE: debug + QUEUE: bsc_es diff --git a/autosubmit/conf_esarchive/proj.conf b/autosubmit/conf_esarchive/proj.conf deleted file mode 100644 index 7322a971d784429e7d0b1a7e69ff4f38470e8842..0000000000000000000000000000000000000000 --- a/autosubmit/conf_esarchive/proj.conf +++ /dev/null @@ -1,6 +0,0 @@ -[common] - -MODULES = "MODULES" -OUTDIR = -SCRIPT = -RECIPE = diff --git a/autosubmit/conf_esarchive/proj.yml b/autosubmit/conf_esarchive/proj.yml new file mode 100644 index 0000000000000000000000000000000000000000..679cf63b1ced38fd833d28ea9acfa145a1e9bc4f --- /dev/null +++ b/autosubmit/conf_esarchive/proj.yml @@ -0,0 +1,4 @@ +common: + MODULES: "MODULES" + OUTDIR: + SCRIPT: diff --git a/autosubmit/conf_mars/jobs.yml b/autosubmit/conf_mars/jobs.yml index aba048bcf6e4c6995971b83b4e9ffbece5e8c7c6..273d3d6f45c7811062fb07a4ac5620daefd82f72 100644 --- a/autosubmit/conf_mars/jobs.yml +++ b/autosubmit/conf_mars/jobs.yml @@ -10,6 +10,7 @@ JOBS: FILE: autosubmit/auto-scorecards.sh WALLCLOCK: 00:10 PLATFORM: NORD3 + NOTIFY_ON: PROCESSORS: 1 DEPENDENCIES: verification diff --git a/conf/autosubmit.yml b/conf/autosubmit.yml index 8b653a650b8d054b73819af2ec2cb2bebeb3c6ad..4ff15ffd24e2bac63543cc792e7004f22953a6ab 100644 --- a/conf/autosubmit.yml +++ b/conf/autosubmit.yml @@ -1,8 +1,8 @@ esarchive: platform: nord3v2 - module_version: autosubmit/3.14.0-foss-2015a-Python-2.7.9 - auto_version: 3.14.0 - conf_format: ini + module_version: autosubmit/4.0.0b-foss-2015a-Python-3.7.3 + auto_version: 4.0.0 + conf_format: yaml experiment_dir: /esarchive/autosubmit/ userID: bsc32 mars: diff --git a/recipes/recipe_scorecards_vic.yml b/recipes/recipe_scorecards_vic.yml index ba62072ebf385dcc612577595ac5983986e0e038..fbd2cb90e2cd162f90e517bcbd82fe634783b8e0 100644 --- a/recipes/recipe_scorecards_vic.yml +++ b/recipes/recipe_scorecards_vic.yml @@ -14,7 +14,6 @@ Analysis: Horizon: Seasonal Variables: # ECVs and Indicators? - {name: tas, freq: monthly_mean} - - {name: prlr, freq: monthly_mean} Datasets: System: # multiple systems for single model, split if Multimodel = F - {name: ECMWF-SEAS5} @@ -99,6 +98,7 @@ Run: wallclock: 03:00 # hh:mm processors_per_job: 14 platform: nord3v2 + custom_directives: ['#SBATCH --exclusive'] email_notifications: yes # enable/disable email notifications. Change it if you want to. email_address: victoria.agudetse@bsc.es # replace with your email address notify_completed: yes # notify me by email when a job finishes diff --git a/recipes/tests/recipe_seasonal_example.yml b/recipes/tests/recipe_seasonal_example.yml index 33662383725551d94d49a8e98b2b3e1763039206..4ba873c1fa034c03c554ddc18d78b957853eef81 100644 --- a/recipes/tests/recipe_seasonal_example.yml +++ b/recipes/tests/recipe_seasonal_example.yml @@ -69,14 +69,15 @@ Run: filesystem: esarchive output_dir: /esarchive/scratch/vagudets/auto-s2s-outputs/ code_dir: /esarchive/scratch/vagudets/repos/auto-s2s/ - autosubmit: no + autosubmit: yes # fill only if using autosubmit auto_conf: script: /esarchive/scratch/vagudets/repos/auto-s2s/example_scripts/test_parallel_workflow.R - expid: a5no # autosubmit experiment ID + expid: a6jr # autosubmit experiment ID hpc_user: bsc32762 # your hpc username - wallclock: 04:00 # hh:mm + wallclock: 01:00 # hh:mm processors_per_job: 8 # use ncores parameter? + custom_directives: ['#SBATCH --constraint=medmem', '#SBATCH --exclusive'] platform: nord3v2 # make this constant? email_notifications: yes # enable/disable email notifications email_address: victoria.agudetse@bsc.es # email address for notifications diff --git a/tools/write_autosubmit_conf.R b/tools/write_autosubmit_conf.R index bfc708bcb2da9c9484e8067ee0a40e1b8769da18..95ca93f0f09f10f8b8b67a5076f24918c3d6db7e 100644 --- a/tools/write_autosubmit_conf.R +++ b/tools/write_autosubmit_conf.R @@ -34,44 +34,38 @@ write_autosubmit_conf <- function(recipe, nchunks) { conf$local$PROJECT_PATH <- recipe$Run$code_dir } else if (conf_type == "jobs") { # Section 3: jobs - ## wallclock, notify_on, platform?, processors - # Different file structure depending on autosubmit version - if (auto_specs$auto_version == "4.0.0") { - jobs <- conf$JOBS - } else { - jobs <- conf - } - jobs$verification$WALLCLOCK <- recipe$Run$auto_conf$wallclock + ## wallclock, notify_on, platform?, processors, + conf$JOBS$verification$WALLCLOCK <- recipe$Run$auto_conf$wallclock if (recipe$Run$auto_conf$notify_completed) { - jobs$verification$NOTIFY_ON <- paste(jobs$verification$NOTIFY_ON, - "COMPLETED") + conf$JOBS$verification$NOTIFY_ON <- paste(conf$JOBS$verification$NOTIFY_ON, + "COMPLETED") } if (recipe$Run$auto_conf$notify_failed) { - jobs$verification$NOTIFY_ON <- paste(jobs$verification$NOTIFY_ON, - "FAILED") + conf$JOBS$verification$NOTIFY_ON <- paste(conf$JOBS$verification$NOTIFY_ON, + "FAILED") } - jobs$verification$PROCESSORS <- recipe$Run$auto_conf$processors_per_job # ncores? + conf$JOBS$verification$PROCESSORS <- recipe$Run$auto_conf$processors_per_job # ncores? + conf$JOBS$verification$CUSTOM_DIRECTIVES <- recipe$Run$auto_conf$custom_directives # Only include Scorecards job if section exists in the recipe and # is set to 'execute: True' if (!("Scorecards" %in% names(recipe$Analysis$Workflow)) || (!recipe$Analysis$Workflow$Scorecards$execute)) { - jobs$scorecards <- NULL - } - # Return to original list - if (auto_specs$auto_version == "4.0.0") { - conf$JOBS <- jobs + conf$JOBS$scorecards <- NULL } else { - conf <- jobs + if (recipe$Run$auto_conf$notify_completed) { + conf$JOBS$scorecards$NOTIFY_ON <- paste(conf$JOBS$scorecards$NOTIFY_ON, + "COMPLETED") + } + if (recipe$Run$auto_conf$notify_failed) { + conf$JOBS$scorecards$NOTIFY_ON <- paste(conf$JOBS$scorecards$NOTIFY_ON, + "FAILED") + } } } else if (conf_type == "platforms") { # Section 4: platform configuration ## nord3v2 configuration... platform name? user, processors_per_node - if (auto_specs$auto_version == "4.0.0") { - conf$Platforms[[auto_specs$platform]]$USER <- - recipe$Run$auto_conf$hpc_user - } else { - conf[[auto_specs$platform]]$USER <- recipe$Run$auto_conf$hpc_user - } + conf$Platforms[[auto_specs$platform]]$USER <- + recipe$Run$auto_conf$hpc_user } else if (conf_type == "proj") { # Section 5: proj ## modules? Info that goes on script, e.g. output directory