From fb15452d8ba53f731bce7407ce37aa2885c49395 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Tue, 10 Oct 2023 10:11:36 +0200 Subject: [PATCH 1/5] Add notifications to Scorecards jobs --- autosubmit/conf_esarchive/jobs.conf | 1 + autosubmit/conf_mars/jobs.yml | 1 + tools/write_autosubmit_conf.R | 8 ++++++++ 3 files changed, 10 insertions(+) diff --git a/autosubmit/conf_esarchive/jobs.conf b/autosubmit/conf_esarchive/jobs.conf index f9d26634..b80739a6 100644 --- a/autosubmit/conf_esarchive/jobs.conf +++ b/autosubmit/conf_esarchive/jobs.conf @@ -9,6 +9,7 @@ PROCESSORS = [scorecards] FILE = autosubmit/auto-scorecards.sh WALLCLOCK = 00:10 +NOTIFY_ON = PLATFORM = nord3v2 PROCESSORS = 1 DEPENDENCIES = verification diff --git a/autosubmit/conf_mars/jobs.yml b/autosubmit/conf_mars/jobs.yml index aba048bc..273d3d6f 100644 --- a/autosubmit/conf_mars/jobs.yml +++ b/autosubmit/conf_mars/jobs.yml @@ -10,6 +10,7 @@ JOBS: FILE: autosubmit/auto-scorecards.sh WALLCLOCK: 00:10 PLATFORM: NORD3 + NOTIFY_ON: PROCESSORS: 1 DEPENDENCIES: verification diff --git a/tools/write_autosubmit_conf.R b/tools/write_autosubmit_conf.R index bfc708bc..306664c7 100644 --- a/tools/write_autosubmit_conf.R +++ b/tools/write_autosubmit_conf.R @@ -56,6 +56,14 @@ write_autosubmit_conf <- function(recipe, nchunks) { if (!("Scorecards" %in% names(recipe$Analysis$Workflow)) || (!recipe$Analysis$Workflow$Scorecards$execute)) { jobs$scorecards <- NULL + if (recipe$Run$auto_conf$notify_completed) { + obs$scorecards$NOTIFY_ON <- paste(jobs$scorecards$NOTIFY_ON, + "COMPLETED") + } + if (recipe$Run$auto_conf$notify_failed) { + obs$scorecards$NOTIFY_ON <- paste(jobs$scorecards$NOTIFY_ON, + "FAILED") + } } # Return to original list if (auto_specs$auto_version == "4.0.0") { -- GitLab From 6f7842c5ad479fd9cc1b730d078d6229506c27b6 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Tue, 10 Oct 2023 14:46:16 +0200 Subject: [PATCH 2/5] Fix Scorecards notifications and WIP to add custom directives to AS (working only for v4) --- autosubmit/conf_esarchive/jobs.conf | 3 +- autosubmit/conf_esarchive2/autosubmit.yml | 22 ++++++++++++ autosubmit/conf_esarchive2/expdef.yml | 44 +++++++++++++++++++++++ autosubmit/conf_esarchive2/jobs.yml | 16 +++++++++ autosubmit/conf_esarchive2/platforms.yml | 11 ++++++ autosubmit/conf_esarchive2/proj.yml | 4 +++ conf/archive.yml | 2 +- conf/autosubmit.yml | 7 ++++ recipes/tests/recipe_seasonal_example.yml | 7 ++-- tools/write_autosubmit_conf.R | 10 +++--- 10 files changed, 117 insertions(+), 9 deletions(-) create mode 100644 autosubmit/conf_esarchive2/autosubmit.yml create mode 100644 autosubmit/conf_esarchive2/expdef.yml create mode 100644 autosubmit/conf_esarchive2/jobs.yml create mode 100644 autosubmit/conf_esarchive2/platforms.yml create mode 100644 autosubmit/conf_esarchive2/proj.yml diff --git a/autosubmit/conf_esarchive/jobs.conf b/autosubmit/conf_esarchive/jobs.conf index b80739a6..82898347 100644 --- a/autosubmit/conf_esarchive/jobs.conf +++ b/autosubmit/conf_esarchive/jobs.conf @@ -4,7 +4,8 @@ RUNNING = chunk WALLCLOCK = NOTIFY_ON = PLATFORM = nord3v2 -PROCESSORS = +PROCESSORS = +CUSTOM_DIRECTIVES = [scorecards] FILE = autosubmit/auto-scorecards.sh diff --git a/autosubmit/conf_esarchive2/autosubmit.yml b/autosubmit/conf_esarchive2/autosubmit.yml new file mode 100644 index 00000000..03008116 --- /dev/null +++ b/autosubmit/conf_esarchive2/autosubmit.yml @@ -0,0 +1,22 @@ +config: + EXPID: + AUTOSUBMIT_VERSION: 4.0.73 + MAXWAITINGJOBS: 16 + # Default maximum number of jobs to be running at the same time at any platform + # Default: 6 + TOTALJOBS: 16 + SAFETYSLEEPTIME: 10 + RETRIALS: 0 +mail: + NOTIFICATIONS: + TO: +communications: + # Communications library used to connect with platforms: paramiko or saga. + # Default: paramiko + API: paramiko +storage: + # Defines the way of storing the progress of the experiment. The available options are: + # A PICKLE file (pkl) or an SQLite database (db). Default: pkl + TYPE: pkl + # Defines if the remote logs will be copied to the local platform. Default: True. + COPY_REMOTE_LOGS: True diff --git a/autosubmit/conf_esarchive2/expdef.yml b/autosubmit/conf_esarchive2/expdef.yml new file mode 100644 index 00000000..b4327f65 --- /dev/null +++ b/autosubmit/conf_esarchive2/expdef.yml @@ -0,0 +1,44 @@ +DEFAULT: + EXPID: + HPCARCH: NORD3 +experiment: + DATELIST: + MEMBERS: fc0 + CHUNKSIZEUNIT: month + CHUNKSIZE: 1 + NUMCHUNKS: + CHUNKINI: 1 + CALENDAR: standard +project: + PROJECT_TYPE: local + # Destination folder name for project. type: STRING, default: leave empty, + PROJECT_DESTINATION: auto-s2s +# If PROJECT_TYPE is not git, no need to change +git: + # Repository URL STRING: 'https://github.com/torvalds/linux.git' + PROJECT_ORIGIN: https://earth.bsc.es/gitlab/es/auto-s2s.git + # Select branch or tag, STRING, default: 'master', help: {'master' (default), 'develop', 'v3.1b', ...} + PROJECT_BRANCH: master + # type: STRING, default: leave empty, help: if model branch is a TAG leave empty + PROJECT_COMMIT: '' +svn: + PROJECT_URL: '' + PROJECT_REVISION: '' +# If PROJECT_TYPE is not local, no need to change +local: + # type: STRING, help: /foo/bar/ecearth + PROJECT_PATH: /esarchive/scratch/vagudets/repos/auto-s2s/ +# If PROJECT_TYPE is none, no need to change +project_files: + # Where is PROJECT CONFIGURATION file location relative to project root path + FILE_PROJECT_CONF: '' + # Where is JOBS CONFIGURATION file location relative to project root path + FILE_JOBS_CONF: '' + # Default job scripts type in the project. type: STRING, default: bash, supported: 'bash', 'python' or 'r' + JOB_SCRIPTS_TYPE: '' +rerun: + # Is a rerun or not? [Default: Do set FALSE]. BOOLEAN: TRUE, FALSE + RERUN: FALSE + # If RERUN: TRUE then supply the list of chunks to rerun + # LIST: [ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ] + CHUNKLIST: '' diff --git a/autosubmit/conf_esarchive2/jobs.yml b/autosubmit/conf_esarchive2/jobs.yml new file mode 100644 index 00000000..273d3d6f --- /dev/null +++ b/autosubmit/conf_esarchive2/jobs.yml @@ -0,0 +1,16 @@ +JOBS: + verification: + FILE: autosubmit/auto-verification-CERISE.sh + RUNNING: chunk + WALLCLOCK: + NOTIFY_ON: + PLATFORM: NORD3 + PROCESSORS: + scorecards: + FILE: autosubmit/auto-scorecards.sh + WALLCLOCK: 00:10 + PLATFORM: NORD3 + NOTIFY_ON: + PROCESSORS: 1 + DEPENDENCIES: verification + diff --git a/autosubmit/conf_esarchive2/platforms.yml b/autosubmit/conf_esarchive2/platforms.yml new file mode 100644 index 00000000..78056d62 --- /dev/null +++ b/autosubmit/conf_esarchive2/platforms.yml @@ -0,0 +1,11 @@ +## TODO: Change platform +Platforms: + nord3v2: + TYPE: slurm + HOST: nord4.bsc.es + USER: + PROJECT: bsc32 ## TO BE CHANGED + SCRATCH_DIR: /gpfs/scratch/ ## TO BE CHANGED + PROCESSORS_PER_NODE: 16 + SERIAL_QUEUE: debug + QUEUE: bsc_es diff --git a/autosubmit/conf_esarchive2/proj.yml b/autosubmit/conf_esarchive2/proj.yml new file mode 100644 index 00000000..679cf63b --- /dev/null +++ b/autosubmit/conf_esarchive2/proj.yml @@ -0,0 +1,4 @@ +common: + MODULES: "MODULES" + OUTDIR: + SCRIPT: diff --git a/conf/archive.yml b/conf/archive.yml index efb57c9e..7d5d8bf9 100644 --- a/conf/archive.yml +++ b/conf/archive.yml @@ -1,4 +1,4 @@ -esarchive: +esarchive2: src: "/esarchive/" System: ECMWF-SEAS5: diff --git a/conf/autosubmit.yml b/conf/autosubmit.yml index 8b653a65..3a56e7c3 100644 --- a/conf/autosubmit.yml +++ b/conf/autosubmit.yml @@ -12,3 +12,10 @@ mars: conf_format: yaml experiment_dir: /esarchive/autosubmit/ ## TO BE CHANGED userID: bsc32 ## TO BE CHANGED +esarchive2: + platform: nord3v2 + module_version: autosubmit/4.0.0b-foss-2015a-Python-3.7.3 + auto_version: 4.0.0 + conf_format: yaml + experiment_dir: /esarchive/autosubmit/ ## TO BE CHANGED + userID: bsc32 ## TO BE CHANGED diff --git a/recipes/tests/recipe_seasonal_example.yml b/recipes/tests/recipe_seasonal_example.yml index 33662383..b4933b59 100644 --- a/recipes/tests/recipe_seasonal_example.yml +++ b/recipes/tests/recipe_seasonal_example.yml @@ -66,17 +66,18 @@ Analysis: Run: Loglevel: INFO Terminal: yes - filesystem: esarchive + filesystem: esarchive2 output_dir: /esarchive/scratch/vagudets/auto-s2s-outputs/ code_dir: /esarchive/scratch/vagudets/repos/auto-s2s/ - autosubmit: no + autosubmit: yes # fill only if using autosubmit auto_conf: script: /esarchive/scratch/vagudets/repos/auto-s2s/example_scripts/test_parallel_workflow.R expid: a5no # autosubmit experiment ID hpc_user: bsc32762 # your hpc username - wallclock: 04:00 # hh:mm + wallclock: 01:00 # hh:mm processors_per_job: 8 # use ncores parameter? + custom_directives: ['#SBATCH --constraint=medmem', '#SBATCH --exclusive'] platform: nord3v2 # make this constant? email_notifications: yes # enable/disable email notifications email_address: victoria.agudetse@bsc.es # email address for notifications diff --git a/tools/write_autosubmit_conf.R b/tools/write_autosubmit_conf.R index 306664c7..925cbd77 100644 --- a/tools/write_autosubmit_conf.R +++ b/tools/write_autosubmit_conf.R @@ -51,18 +51,20 @@ write_autosubmit_conf <- function(recipe, nchunks) { "FAILED") } jobs$verification$PROCESSORS <- recipe$Run$auto_conf$processors_per_job # ncores? + jobs$verification$CUSTOM_DIRECTIVES <- recipe$Run$auto_conf$custom_directives # Only include Scorecards job if section exists in the recipe and # is set to 'execute: True' if (!("Scorecards" %in% names(recipe$Analysis$Workflow)) || (!recipe$Analysis$Workflow$Scorecards$execute)) { jobs$scorecards <- NULL + } else { if (recipe$Run$auto_conf$notify_completed) { - obs$scorecards$NOTIFY_ON <- paste(jobs$scorecards$NOTIFY_ON, - "COMPLETED") + jobs$scorecards$NOTIFY_ON <- paste(jobs$scorecards$NOTIFY_ON, + "COMPLETED") } if (recipe$Run$auto_conf$notify_failed) { - obs$scorecards$NOTIFY_ON <- paste(jobs$scorecards$NOTIFY_ON, - "FAILED") + jobs$scorecards$NOTIFY_ON <- paste(jobs$scorecards$NOTIFY_ON, + "FAILED") } } # Return to original list -- GitLab From ed3e15c3e3022eb0aa081fbd1642f0a0073926e3 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Tue, 17 Oct 2023 10:47:03 +0200 Subject: [PATCH 3/5] Change autosubmit esarchive configuration to AS version 4 --- autosubmit/auto-verification.sh | 4 +- autosubmit/conf_esarchive/autosubmit.conf | 42 ---------- .../autosubmit.yml | 2 +- autosubmit/conf_esarchive/expdef.conf | 76 ------------------- .../expdef.yml | 2 +- autosubmit/conf_esarchive/jobs.conf | 16 ---- .../jobs.yml | 6 +- autosubmit/conf_esarchive/platforms.conf | 11 --- .../platforms.yml | 0 autosubmit/conf_esarchive/proj.conf | 6 -- .../proj.yml | 0 conf/autosubmit.yml | 13 +--- recipes/recipe_scorecards_vic.yml | 2 +- recipes/tests/recipe_seasonal_example.yml | 4 +- tools/write_autosubmit_conf.R | 46 ++++------- 15 files changed, 28 insertions(+), 202 deletions(-) delete mode 100644 autosubmit/conf_esarchive/autosubmit.conf rename autosubmit/{conf_esarchive2 => conf_esarchive}/autosubmit.yml (95%) delete mode 100644 autosubmit/conf_esarchive/expdef.conf rename autosubmit/{conf_esarchive2 => conf_esarchive}/expdef.yml (98%) delete mode 100644 autosubmit/conf_esarchive/jobs.conf rename autosubmit/{conf_esarchive2 => conf_esarchive}/jobs.yml (72%) delete mode 100644 autosubmit/conf_esarchive/platforms.conf rename autosubmit/{conf_esarchive2 => conf_esarchive}/platforms.yml (100%) delete mode 100644 autosubmit/conf_esarchive/proj.conf rename autosubmit/{conf_esarchive2 => conf_esarchive}/proj.yml (100%) diff --git a/autosubmit/auto-verification.sh b/autosubmit/auto-verification.sh index bbd06556..0089e322 100644 --- a/autosubmit/auto-verification.sh +++ b/autosubmit/auto-verification.sh @@ -2,8 +2,8 @@ ############ AUTOSUBMIT INPUTS ############ proj_dir=%PROJDIR% -outdir=%OUTDIR% -script=%SCRIPT% +outdir=%common.OUTDIR% +script=%common.SCRIPT% CHUNK=%CHUNK% ############################### diff --git a/autosubmit/conf_esarchive/autosubmit.conf b/autosubmit/conf_esarchive/autosubmit.conf deleted file mode 100644 index 685876a1..00000000 --- a/autosubmit/conf_esarchive/autosubmit.conf +++ /dev/null @@ -1,42 +0,0 @@ -[config] -# Experiment identifier -# No need to change -EXPID = -# No need to change. -# Autosubmit version identifier -AUTOSUBMIT_VERSION = 3.14.0 -# Default maximum number of jobs to be waiting in any platform -# Default = 3 -MAXWAITINGJOBS = 16 -# Default maximum number of jobs to be running at the same time at any platform -# Default = 6 -TOTALJOBS = 16 -# Time (seconds) between connections to the HPC queue scheduler to poll already submitted jobs status -# Default = 10 -SAFETYSLEEPTIME = 10 -# Number of retrials if a job fails. Can be override at job level -# Default = 0 -RETRIALS = 0 - -[mail] -# Enable mail notifications -# Default = False -NOTIFICATIONS = -# Mail address where notifications will be received -TO = - -[communications] -# Communications library used to connect with platforms: paramiko or saga. -# Default = paramiko -API = paramiko - -[storage] -# Defines the way of storing the progress of the experiment. The available options are: -# A PICKLE file (pkl) or an SQLite database (db). Default = pkl -TYPE = pkl -# Defines if the remote logs will be copied to the local platform. Default = True. -COPY_REMOTE_LOGS = True - -[migrate] -# Changes experiment files owner. -TO_USER = diff --git a/autosubmit/conf_esarchive2/autosubmit.yml b/autosubmit/conf_esarchive/autosubmit.yml similarity index 95% rename from autosubmit/conf_esarchive2/autosubmit.yml rename to autosubmit/conf_esarchive/autosubmit.yml index 03008116..0fd5d5c6 100644 --- a/autosubmit/conf_esarchive2/autosubmit.yml +++ b/autosubmit/conf_esarchive/autosubmit.yml @@ -1,6 +1,6 @@ config: EXPID: - AUTOSUBMIT_VERSION: 4.0.73 + AUTOSUBMIT_VERSION: 4.0.0b0 MAXWAITINGJOBS: 16 # Default maximum number of jobs to be running at the same time at any platform # Default: 6 diff --git a/autosubmit/conf_esarchive/expdef.conf b/autosubmit/conf_esarchive/expdef.conf deleted file mode 100644 index d3a3370a..00000000 --- a/autosubmit/conf_esarchive/expdef.conf +++ /dev/null @@ -1,76 +0,0 @@ -[DEFAULT] -# Experiment identifier -# No need to change -EXPID = -# HPC name. -# No need to change -HPCARCH = nord3v2 - -[experiment] -# Supply the list of start dates. Available formats: YYYYMMDD YYYYMMDDhh YYYYMMDDhhmm -# You can also use an abbreviated syntax for multiple dates with common parts: 200001[01 15] <=> 20000101 20000115 -# 200001[01-04] <=> 20000101 20000102 20000103 20000104 -# DATELIST = 19600101 19650101 19700101 -# DATELIST = 1960[0101 0201 0301] -# DATELIST = 19[60-65] -DATELIST = -# Supply the list of members. Format fcX -# You can also use an abreviated syntax for multiple members: fc[0 1 2] <=> fc0 fc1 fc2 -# fc[0-2] <=> fc0 fc1 fc2 -# MEMBERS = fc0 fc1 fc2 fc3 fc4 -#MEMBERS = fc4 -MEMBERS = fc0 -# Chunk size unit. STRING = hour, day, month, year -CHUNKSIZEUNIT = month -# Chunk size. NUMERIC = 4, 6, 12 -CHUNKSIZE = 1 -# Total number of chunks in experiment. NUMERIC = 30, 15, 10 -NUMCHUNKS = -# Initial chunk of the experiment. Optional. DEFAULT = 1 -CHUNKINI = -# Calendar used. LIST: standard, noleap -CALENDAR = standard - -[project] -# Select project type. STRING = git, svn, local, none -# If PROJECT_TYPE is set to none, Autosubmit self-contained dummy templates will be used -PROJECT_TYPE = local -# Destination folder name for project. type = STRING, default = leave empty, -PROJECT_DESTINATION = auto-s2s - -# If PROJECT_TYPE is not git, no need to change -[git] -# Repository URL STRING = 'https://github.com/torvalds/linux.git' -PROJECT_ORIGIN = https://earth.bsc.es/gitlab/es/auto-s2s.git -# Select branch or tag, STRING, default = 'master', help = {'master' (default), 'develop', 'v3.1b', ...} -PROJECT_BRANCH = master -# type = STRING, default = leave empty, help = if model branch is a TAG leave empty -PROJECT_COMMIT = - -# If PROJECT_TYPE is not svn, no need to change -[svn] -# type = STRING, help = 'https://svn.ec-earth.org/ecearth3' -PROJECT_URL = -# Select revision number. NUMERIC = 1778 -PROJECT_REVISION = - -# If PROJECT_TYPE is not local, no need to change -[local] -# type = STRING, help = /foo/bar/ecearth -PROJECT_PATH = /esarchive/scratch/vagudets/repos/auto-s2s/ - -# If PROJECT_TYPE is none, no need to change -[project_files] -# Where is PROJECT CONFIGURATION file location relative to project root path -FILE_PROJECT_CONF = -# Where is JOBS CONFIGURATION file location relative to project root path -FILE_JOBS_CONF = -# Default job scripts type in the project. type = STRING, default = bash, supported = 'bash', 'python' or 'r' -JOB_SCRIPTS_TYPE = - -[rerun] -# Is a rerun or not? [Default: Do set FALSE]. BOOLEAN = TRUE, FALSE -RERUN = FALSE -# If RERUN = TRUE then supply the list of chunks to rerun -# LIST = [ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ] -CHUNKLIST = diff --git a/autosubmit/conf_esarchive2/expdef.yml b/autosubmit/conf_esarchive/expdef.yml similarity index 98% rename from autosubmit/conf_esarchive2/expdef.yml rename to autosubmit/conf_esarchive/expdef.yml index b4327f65..8dc29b27 100644 --- a/autosubmit/conf_esarchive2/expdef.yml +++ b/autosubmit/conf_esarchive/expdef.yml @@ -1,6 +1,6 @@ DEFAULT: EXPID: - HPCARCH: NORD3 + HPCARCH: nord3v2 experiment: DATELIST: MEMBERS: fc0 diff --git a/autosubmit/conf_esarchive/jobs.conf b/autosubmit/conf_esarchive/jobs.conf deleted file mode 100644 index 82898347..00000000 --- a/autosubmit/conf_esarchive/jobs.conf +++ /dev/null @@ -1,16 +0,0 @@ -[verification] -FILE = autosubmit/auto-verification.sh -RUNNING = chunk -WALLCLOCK = -NOTIFY_ON = -PLATFORM = nord3v2 -PROCESSORS = -CUSTOM_DIRECTIVES = - -[scorecards] -FILE = autosubmit/auto-scorecards.sh -WALLCLOCK = 00:10 -NOTIFY_ON = -PLATFORM = nord3v2 -PROCESSORS = 1 -DEPENDENCIES = verification diff --git a/autosubmit/conf_esarchive2/jobs.yml b/autosubmit/conf_esarchive/jobs.yml similarity index 72% rename from autosubmit/conf_esarchive2/jobs.yml rename to autosubmit/conf_esarchive/jobs.yml index 273d3d6f..a3c8934b 100644 --- a/autosubmit/conf_esarchive2/jobs.yml +++ b/autosubmit/conf_esarchive/jobs.yml @@ -1,15 +1,15 @@ JOBS: verification: - FILE: autosubmit/auto-verification-CERISE.sh + FILE: autosubmit/auto-verification.sh RUNNING: chunk WALLCLOCK: NOTIFY_ON: - PLATFORM: NORD3 + PLATFORM: nord3v2 PROCESSORS: scorecards: FILE: autosubmit/auto-scorecards.sh WALLCLOCK: 00:10 - PLATFORM: NORD3 + PLATFORM: nord3v2 NOTIFY_ON: PROCESSORS: 1 DEPENDENCIES: verification diff --git a/autosubmit/conf_esarchive/platforms.conf b/autosubmit/conf_esarchive/platforms.conf deleted file mode 100644 index 0f6819d0..00000000 --- a/autosubmit/conf_esarchive/platforms.conf +++ /dev/null @@ -1,11 +0,0 @@ -[nord3v2] -TYPE = slurm -HOST = nord4.bsc.es -PROJECT = bsc32 -ADD_PROJECT_TO_HOST = False -USER = -SCRATCH_DIR = /gpfs/scratch -PROCESSORS_PER_NODE = 16 -SERIAL_QUEUE = debug -QUEUE = bsc_es -CUSTOM_DIRECTIVES = ["#SBATCH --exclusive"] diff --git a/autosubmit/conf_esarchive2/platforms.yml b/autosubmit/conf_esarchive/platforms.yml similarity index 100% rename from autosubmit/conf_esarchive2/platforms.yml rename to autosubmit/conf_esarchive/platforms.yml diff --git a/autosubmit/conf_esarchive/proj.conf b/autosubmit/conf_esarchive/proj.conf deleted file mode 100644 index 7322a971..00000000 --- a/autosubmit/conf_esarchive/proj.conf +++ /dev/null @@ -1,6 +0,0 @@ -[common] - -MODULES = "MODULES" -OUTDIR = -SCRIPT = -RECIPE = diff --git a/autosubmit/conf_esarchive2/proj.yml b/autosubmit/conf_esarchive/proj.yml similarity index 100% rename from autosubmit/conf_esarchive2/proj.yml rename to autosubmit/conf_esarchive/proj.yml diff --git a/conf/autosubmit.yml b/conf/autosubmit.yml index 3a56e7c3..4ff15ffd 100644 --- a/conf/autosubmit.yml +++ b/conf/autosubmit.yml @@ -1,8 +1,8 @@ esarchive: platform: nord3v2 - module_version: autosubmit/3.14.0-foss-2015a-Python-2.7.9 - auto_version: 3.14.0 - conf_format: ini + module_version: autosubmit/4.0.0b-foss-2015a-Python-3.7.3 + auto_version: 4.0.0 + conf_format: yaml experiment_dir: /esarchive/autosubmit/ userID: bsc32 mars: @@ -12,10 +12,3 @@ mars: conf_format: yaml experiment_dir: /esarchive/autosubmit/ ## TO BE CHANGED userID: bsc32 ## TO BE CHANGED -esarchive2: - platform: nord3v2 - module_version: autosubmit/4.0.0b-foss-2015a-Python-3.7.3 - auto_version: 4.0.0 - conf_format: yaml - experiment_dir: /esarchive/autosubmit/ ## TO BE CHANGED - userID: bsc32 ## TO BE CHANGED diff --git a/recipes/recipe_scorecards_vic.yml b/recipes/recipe_scorecards_vic.yml index ba62072e..fbd2cb90 100644 --- a/recipes/recipe_scorecards_vic.yml +++ b/recipes/recipe_scorecards_vic.yml @@ -14,7 +14,6 @@ Analysis: Horizon: Seasonal Variables: # ECVs and Indicators? - {name: tas, freq: monthly_mean} - - {name: prlr, freq: monthly_mean} Datasets: System: # multiple systems for single model, split if Multimodel = F - {name: ECMWF-SEAS5} @@ -99,6 +98,7 @@ Run: wallclock: 03:00 # hh:mm processors_per_job: 14 platform: nord3v2 + custom_directives: ['#SBATCH --exclusive'] email_notifications: yes # enable/disable email notifications. Change it if you want to. email_address: victoria.agudetse@bsc.es # replace with your email address notify_completed: yes # notify me by email when a job finishes diff --git a/recipes/tests/recipe_seasonal_example.yml b/recipes/tests/recipe_seasonal_example.yml index b4933b59..4ba873c1 100644 --- a/recipes/tests/recipe_seasonal_example.yml +++ b/recipes/tests/recipe_seasonal_example.yml @@ -66,14 +66,14 @@ Analysis: Run: Loglevel: INFO Terminal: yes - filesystem: esarchive2 + filesystem: esarchive output_dir: /esarchive/scratch/vagudets/auto-s2s-outputs/ code_dir: /esarchive/scratch/vagudets/repos/auto-s2s/ autosubmit: yes # fill only if using autosubmit auto_conf: script: /esarchive/scratch/vagudets/repos/auto-s2s/example_scripts/test_parallel_workflow.R - expid: a5no # autosubmit experiment ID + expid: a6jr # autosubmit experiment ID hpc_user: bsc32762 # your hpc username wallclock: 01:00 # hh:mm processors_per_job: 8 # use ncores parameter? diff --git a/tools/write_autosubmit_conf.R b/tools/write_autosubmit_conf.R index 925cbd77..95ca93f0 100644 --- a/tools/write_autosubmit_conf.R +++ b/tools/write_autosubmit_conf.R @@ -34,54 +34,38 @@ write_autosubmit_conf <- function(recipe, nchunks) { conf$local$PROJECT_PATH <- recipe$Run$code_dir } else if (conf_type == "jobs") { # Section 3: jobs - ## wallclock, notify_on, platform?, processors - # Different file structure depending on autosubmit version - if (auto_specs$auto_version == "4.0.0") { - jobs <- conf$JOBS - } else { - jobs <- conf - } - jobs$verification$WALLCLOCK <- recipe$Run$auto_conf$wallclock + ## wallclock, notify_on, platform?, processors, + conf$JOBS$verification$WALLCLOCK <- recipe$Run$auto_conf$wallclock if (recipe$Run$auto_conf$notify_completed) { - jobs$verification$NOTIFY_ON <- paste(jobs$verification$NOTIFY_ON, - "COMPLETED") + conf$JOBS$verification$NOTIFY_ON <- paste(conf$JOBS$verification$NOTIFY_ON, + "COMPLETED") } if (recipe$Run$auto_conf$notify_failed) { - jobs$verification$NOTIFY_ON <- paste(jobs$verification$NOTIFY_ON, - "FAILED") + conf$JOBS$verification$NOTIFY_ON <- paste(conf$JOBS$verification$NOTIFY_ON, + "FAILED") } - jobs$verification$PROCESSORS <- recipe$Run$auto_conf$processors_per_job # ncores? - jobs$verification$CUSTOM_DIRECTIVES <- recipe$Run$auto_conf$custom_directives + conf$JOBS$verification$PROCESSORS <- recipe$Run$auto_conf$processors_per_job # ncores? + conf$JOBS$verification$CUSTOM_DIRECTIVES <- recipe$Run$auto_conf$custom_directives # Only include Scorecards job if section exists in the recipe and # is set to 'execute: True' if (!("Scorecards" %in% names(recipe$Analysis$Workflow)) || (!recipe$Analysis$Workflow$Scorecards$execute)) { - jobs$scorecards <- NULL + conf$JOBS$scorecards <- NULL } else { if (recipe$Run$auto_conf$notify_completed) { - jobs$scorecards$NOTIFY_ON <- paste(jobs$scorecards$NOTIFY_ON, - "COMPLETED") + conf$JOBS$scorecards$NOTIFY_ON <- paste(conf$JOBS$scorecards$NOTIFY_ON, + "COMPLETED") } if (recipe$Run$auto_conf$notify_failed) { - jobs$scorecards$NOTIFY_ON <- paste(jobs$scorecards$NOTIFY_ON, - "FAILED") + conf$JOBS$scorecards$NOTIFY_ON <- paste(conf$JOBS$scorecards$NOTIFY_ON, + "FAILED") } } - # Return to original list - if (auto_specs$auto_version == "4.0.0") { - conf$JOBS <- jobs - } else { - conf <- jobs - } } else if (conf_type == "platforms") { # Section 4: platform configuration ## nord3v2 configuration... platform name? user, processors_per_node - if (auto_specs$auto_version == "4.0.0") { - conf$Platforms[[auto_specs$platform]]$USER <- - recipe$Run$auto_conf$hpc_user - } else { - conf[[auto_specs$platform]]$USER <- recipe$Run$auto_conf$hpc_user - } + conf$Platforms[[auto_specs$platform]]$USER <- + recipe$Run$auto_conf$hpc_user } else if (conf_type == "proj") { # Section 5: proj ## modules? Info that goes on script, e.g. output directory -- GitLab From 3cdf5ba4893128c7ecf0568219e62047d5b033a8 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Tue, 17 Oct 2023 11:32:45 +0200 Subject: [PATCH 4/5] Restore esarchive name --- conf/archive.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/archive.yml b/conf/archive.yml index 7d5d8bf9..efb57c9e 100644 --- a/conf/archive.yml +++ b/conf/archive.yml @@ -1,4 +1,4 @@ -esarchive2: +esarchive: src: "/esarchive/" System: ECMWF-SEAS5: -- GitLab From 127d318c162133077ec73858e4a80b7d9d839653 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Wed, 11 Oct 2023 16:06:58 +0200 Subject: [PATCH 5/5] Update CI/CD packages --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8c720372..997870c8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,6 +7,9 @@ unit-test-seasonal: # This job runs in the test stage. - echo "Loading modules..." - module load R/4.1.2-foss-2015a-bare - module load CDO/1.9.8-foss-2015a + - module load GEOS/3.7.2-foss-2015a-Python-3.7.3 + - module load GDAL/2.2.1-foss-2015a + - module load PROJ/4.8.0-foss-2015a - module list - echo "Running seasonal unit tests..." - Rscript ./tests/test_seasonal.R @@ -17,6 +20,9 @@ unit-test-decadal: # This job runs in the test stage. - echo "Loading modules..." - module load R/4.1.2-foss-2015a-bare - module load CDO/1.9.8-foss-2015a + - module load GEOS/3.7.2-foss-2015a-Python-3.7.3 + - module load GDAL/2.2.1-foss-2015a + - module load PROJ/4.8.0-foss-2015a - module list - echo "Running decadal unit tests..." - Rscript ./tests/test_decadal.R -- GitLab