diff --git a/MODULES b/MODULES index 0a01a9796c2753a74a5e946f00024e33777105ac..ff2a2a3441302fbac39213fadb212a7cfda964de 100644 --- a/MODULES +++ b/MODULES @@ -3,7 +3,7 @@ # WARNING: CDO HAS TO BE ON VERSION 1.9.4 # (If not, conflicts with weekly means computation could appear) -if [ $BSC_MACHINE == "power" ]; then +if [[ $BSC_MACHINE == "power" ]]; then module unuse /apps/modules/modulefiles/applications module use /gpfs/projects/bsc32/software/rhel/7.4/ppc64le/POWER9/modules/all/ @@ -11,12 +11,11 @@ if [ $BSC_MACHINE == "power" ]; then module load CDO/1.9.4-foss-2018b module load R/3.6.1-foss-2018b -elif [ $BSC_MACHINE == "nord3v2" ]; then +elif [[ $BSC_MACHINE == "nord3v2" ]]; then module use /gpfs/projects/bsc32/software/suselinux/11/modules/all module unuse /apps/modules/modulefiles/applications /apps/modules/modulefiles/compilers /apps/modules/modulefiles/tools /apps/modules/modulefiles/libraries /apps/modules/modulefiles/environment - module load CDO/1.9.8-foss-2019b module load R/4.1.2-foss-2019b module load OpenMPI/4.0.5-GCC-8.3.0-nord3-v2 diff --git a/autosubmit/auto-verification-v4.sh b/autosubmit/auto-verification-v4.sh new file mode 100644 index 0000000000000000000000000000000000000000..e21bef2dd1b4df17883a140943ff4f1f3bedb61c --- /dev/null +++ b/autosubmit/auto-verification-v4.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +############ AUTOSUBMIT INPUTS ############ +proj_dir=%PROJDIR% +outdir=%common.OUTDIR% +script=%common.SCRIPT% +CHUNK=%CHUNK% +############################### + +## TODO: How to define the script +cd $proj_dir + +# script=modules/test_parallel_workflow.R +atomic_recipe_number=$(printf "%02d" $CHUNK) +atomic_recipe=${outdir}/logs/recipes/atomic_recipe_${atomic_recipe_number}.yml + +source MODULES + +Rscript ${script} ${atomic_recipe} diff --git a/autosubmit/auto-verification.sh b/autosubmit/auto-verification.sh new file mode 100644 index 0000000000000000000000000000000000000000..bbd065563c5b88c6b49fe4339c578534a883eacf --- /dev/null +++ b/autosubmit/auto-verification.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +############ AUTOSUBMIT INPUTS ############ +proj_dir=%PROJDIR% +outdir=%OUTDIR% +script=%SCRIPT% +CHUNK=%CHUNK% +############################### + +cd $proj_dir + +atomic_recipe_number=$(printf "%02d" $CHUNK) +atomic_recipe=${outdir}/logs/recipes/atomic_recipe_${atomic_recipe_number}.yml + +source MODULES + +Rscript ${script} ${atomic_recipe} diff --git a/autosubmit/conf_esarchive/autosubmit.conf b/autosubmit/conf_esarchive/autosubmit.conf new file mode 100644 index 0000000000000000000000000000000000000000..685876a1ea82079a39eaa7ff1468f2549d8dc52f --- /dev/null +++ b/autosubmit/conf_esarchive/autosubmit.conf @@ -0,0 +1,42 @@ +[config] +# Experiment identifier +# No need to change +EXPID = +# No need to change. +# Autosubmit version identifier +AUTOSUBMIT_VERSION = 3.14.0 +# Default maximum number of jobs to be waiting in any platform +# Default = 3 +MAXWAITINGJOBS = 16 +# Default maximum number of jobs to be running at the same time at any platform +# Default = 6 +TOTALJOBS = 16 +# Time (seconds) between connections to the HPC queue scheduler to poll already submitted jobs status +# Default = 10 +SAFETYSLEEPTIME = 10 +# Number of retrials if a job fails. Can be override at job level +# Default = 0 +RETRIALS = 0 + +[mail] +# Enable mail notifications +# Default = False +NOTIFICATIONS = +# Mail address where notifications will be received +TO = + +[communications] +# Communications library used to connect with platforms: paramiko or saga. +# Default = paramiko +API = paramiko + +[storage] +# Defines the way of storing the progress of the experiment. The available options are: +# A PICKLE file (pkl) or an SQLite database (db). Default = pkl +TYPE = pkl +# Defines if the remote logs will be copied to the local platform. Default = True. +COPY_REMOTE_LOGS = True + +[migrate] +# Changes experiment files owner. +TO_USER = diff --git a/autosubmit/conf_esarchive/expdef.conf b/autosubmit/conf_esarchive/expdef.conf new file mode 100644 index 0000000000000000000000000000000000000000..d3a3370ad0a0d8ec62b7bfbea3a34b50e41504d4 --- /dev/null +++ b/autosubmit/conf_esarchive/expdef.conf @@ -0,0 +1,76 @@ +[DEFAULT] +# Experiment identifier +# No need to change +EXPID = +# HPC name. +# No need to change +HPCARCH = nord3v2 + +[experiment] +# Supply the list of start dates. Available formats: YYYYMMDD YYYYMMDDhh YYYYMMDDhhmm +# You can also use an abbreviated syntax for multiple dates with common parts: 200001[01 15] <=> 20000101 20000115 +# 200001[01-04] <=> 20000101 20000102 20000103 20000104 +# DATELIST = 19600101 19650101 19700101 +# DATELIST = 1960[0101 0201 0301] +# DATELIST = 19[60-65] +DATELIST = +# Supply the list of members. Format fcX +# You can also use an abreviated syntax for multiple members: fc[0 1 2] <=> fc0 fc1 fc2 +# fc[0-2] <=> fc0 fc1 fc2 +# MEMBERS = fc0 fc1 fc2 fc3 fc4 +#MEMBERS = fc4 +MEMBERS = fc0 +# Chunk size unit. STRING = hour, day, month, year +CHUNKSIZEUNIT = month +# Chunk size. NUMERIC = 4, 6, 12 +CHUNKSIZE = 1 +# Total number of chunks in experiment. NUMERIC = 30, 15, 10 +NUMCHUNKS = +# Initial chunk of the experiment. Optional. DEFAULT = 1 +CHUNKINI = +# Calendar used. LIST: standard, noleap +CALENDAR = standard + +[project] +# Select project type. STRING = git, svn, local, none +# If PROJECT_TYPE is set to none, Autosubmit self-contained dummy templates will be used +PROJECT_TYPE = local +# Destination folder name for project. type = STRING, default = leave empty, +PROJECT_DESTINATION = auto-s2s + +# If PROJECT_TYPE is not git, no need to change +[git] +# Repository URL STRING = 'https://github.com/torvalds/linux.git' +PROJECT_ORIGIN = https://earth.bsc.es/gitlab/es/auto-s2s.git +# Select branch or tag, STRING, default = 'master', help = {'master' (default), 'develop', 'v3.1b', ...} +PROJECT_BRANCH = master +# type = STRING, default = leave empty, help = if model branch is a TAG leave empty +PROJECT_COMMIT = + +# If PROJECT_TYPE is not svn, no need to change +[svn] +# type = STRING, help = 'https://svn.ec-earth.org/ecearth3' +PROJECT_URL = +# Select revision number. NUMERIC = 1778 +PROJECT_REVISION = + +# If PROJECT_TYPE is not local, no need to change +[local] +# type = STRING, help = /foo/bar/ecearth +PROJECT_PATH = /esarchive/scratch/vagudets/repos/auto-s2s/ + +# If PROJECT_TYPE is none, no need to change +[project_files] +# Where is PROJECT CONFIGURATION file location relative to project root path +FILE_PROJECT_CONF = +# Where is JOBS CONFIGURATION file location relative to project root path +FILE_JOBS_CONF = +# Default job scripts type in the project. type = STRING, default = bash, supported = 'bash', 'python' or 'r' +JOB_SCRIPTS_TYPE = + +[rerun] +# Is a rerun or not? [Default: Do set FALSE]. BOOLEAN = TRUE, FALSE +RERUN = FALSE +# If RERUN = TRUE then supply the list of chunks to rerun +# LIST = [ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ] +CHUNKLIST = diff --git a/autosubmit/conf_esarchive/jobs.conf b/autosubmit/conf_esarchive/jobs.conf new file mode 100644 index 0000000000000000000000000000000000000000..88f3565c40e841b1062556a70c8b0b4bb808f918 --- /dev/null +++ b/autosubmit/conf_esarchive/jobs.conf @@ -0,0 +1,8 @@ +[verification] +FILE = autosubmit/auto-verification.sh +RUNNING = chunk +WALLCLOCK = +NOTIFY_ON = +PLATFORM = nord3v2 +PROCESSORS = + diff --git a/autosubmit/conf_esarchive/platforms.conf b/autosubmit/conf_esarchive/platforms.conf new file mode 100644 index 0000000000000000000000000000000000000000..0f6819d005b95014642909607f4a787d3d77a495 --- /dev/null +++ b/autosubmit/conf_esarchive/platforms.conf @@ -0,0 +1,11 @@ +[nord3v2] +TYPE = slurm +HOST = nord4.bsc.es +PROJECT = bsc32 +ADD_PROJECT_TO_HOST = False +USER = +SCRATCH_DIR = /gpfs/scratch +PROCESSORS_PER_NODE = 16 +SERIAL_QUEUE = debug +QUEUE = bsc_es +CUSTOM_DIRECTIVES = ["#SBATCH --exclusive"] diff --git a/autosubmit/conf_esarchive/proj.conf b/autosubmit/conf_esarchive/proj.conf new file mode 100644 index 0000000000000000000000000000000000000000..c57e37eb502678627d99f005c5fba4f3b107c42a --- /dev/null +++ b/autosubmit/conf_esarchive/proj.conf @@ -0,0 +1,5 @@ +[common] + +MODULES = "MODULES" +OUTDIR = +SCRIPT = diff --git a/autosubmit/conf_mars/autosubmit.yml b/autosubmit/conf_mars/autosubmit.yml new file mode 100644 index 0000000000000000000000000000000000000000..0fd5d5c6aaf61945d131da77cda08d8d1fdd86cd --- /dev/null +++ b/autosubmit/conf_mars/autosubmit.yml @@ -0,0 +1,22 @@ +config: + EXPID: + AUTOSUBMIT_VERSION: 4.0.0b0 + MAXWAITINGJOBS: 16 + # Default maximum number of jobs to be running at the same time at any platform + # Default: 6 + TOTALJOBS: 16 + SAFETYSLEEPTIME: 10 + RETRIALS: 0 +mail: + NOTIFICATIONS: + TO: +communications: + # Communications library used to connect with platforms: paramiko or saga. + # Default: paramiko + API: paramiko +storage: + # Defines the way of storing the progress of the experiment. The available options are: + # A PICKLE file (pkl) or an SQLite database (db). Default: pkl + TYPE: pkl + # Defines if the remote logs will be copied to the local platform. Default: True. + COPY_REMOTE_LOGS: True diff --git a/autosubmit/conf_mars/expdef.yml b/autosubmit/conf_mars/expdef.yml new file mode 100644 index 0000000000000000000000000000000000000000..b4327f6556ceefc29336db9697b101b2ddc47134 --- /dev/null +++ b/autosubmit/conf_mars/expdef.yml @@ -0,0 +1,44 @@ +DEFAULT: + EXPID: + HPCARCH: NORD3 +experiment: + DATELIST: + MEMBERS: fc0 + CHUNKSIZEUNIT: month + CHUNKSIZE: 1 + NUMCHUNKS: + CHUNKINI: 1 + CALENDAR: standard +project: + PROJECT_TYPE: local + # Destination folder name for project. type: STRING, default: leave empty, + PROJECT_DESTINATION: auto-s2s +# If PROJECT_TYPE is not git, no need to change +git: + # Repository URL STRING: 'https://github.com/torvalds/linux.git' + PROJECT_ORIGIN: https://earth.bsc.es/gitlab/es/auto-s2s.git + # Select branch or tag, STRING, default: 'master', help: {'master' (default), 'develop', 'v3.1b', ...} + PROJECT_BRANCH: master + # type: STRING, default: leave empty, help: if model branch is a TAG leave empty + PROJECT_COMMIT: '' +svn: + PROJECT_URL: '' + PROJECT_REVISION: '' +# If PROJECT_TYPE is not local, no need to change +local: + # type: STRING, help: /foo/bar/ecearth + PROJECT_PATH: /esarchive/scratch/vagudets/repos/auto-s2s/ +# If PROJECT_TYPE is none, no need to change +project_files: + # Where is PROJECT CONFIGURATION file location relative to project root path + FILE_PROJECT_CONF: '' + # Where is JOBS CONFIGURATION file location relative to project root path + FILE_JOBS_CONF: '' + # Default job scripts type in the project. type: STRING, default: bash, supported: 'bash', 'python' or 'r' + JOB_SCRIPTS_TYPE: '' +rerun: + # Is a rerun or not? [Default: Do set FALSE]. BOOLEAN: TRUE, FALSE + RERUN: FALSE + # If RERUN: TRUE then supply the list of chunks to rerun + # LIST: [ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ] + CHUNKLIST: '' diff --git a/autosubmit/conf_mars/jobs.yml b/autosubmit/conf_mars/jobs.yml new file mode 100644 index 0000000000000000000000000000000000000000..2d8e32affa207d3fe72bac27978ce44d100853df --- /dev/null +++ b/autosubmit/conf_mars/jobs.yml @@ -0,0 +1,8 @@ +JOBS: + verification: + FILE: autosubmit/auto-verification-CERISE.sh + RUNNING: chunk + WALLCLOCK: + NOTIFY_ON: + PLATFORM: NORD3 + PROCESSORS: diff --git a/autosubmit/conf_mars/platforms.yml b/autosubmit/conf_mars/platforms.yml new file mode 100644 index 0000000000000000000000000000000000000000..5f76557fb9ab7d5a6c2621858d1d5349cce05464 --- /dev/null +++ b/autosubmit/conf_mars/platforms.yml @@ -0,0 +1,12 @@ +## TODO: Change platform +Platforms: + NORD3: + TYPE: slurm + HOST: nord4.bsc.es + USER: + PROJECT: bsc32 ## TO BE CHANGED + SCRATCH_DIR: /gpfs/scratch/ ## TO BE CHANGED + PROCESSORS_PER_NODE: 16 + SERIAL_QUEUE: debug + QUEUE: bsc_es + CUSTOM_DIRECTIVES: ["#SBATCH --exclusive"] diff --git a/autosubmit/conf_mars/proj.yml b/autosubmit/conf_mars/proj.yml new file mode 100644 index 0000000000000000000000000000000000000000..679cf63b1ced38fd833d28ea9acfa145a1e9bc4f --- /dev/null +++ b/autosubmit/conf_mars/proj.yml @@ -0,0 +1,4 @@ +common: + MODULES: "MODULES" + OUTDIR: + SCRIPT: diff --git a/conf/archive.yml b/conf/archive.yml index eb8e86a574e680085c53ac4c115b426b75e3a559..a982b84f821ed5a92ca765f36384f2384d188a5a 100644 --- a/conf/archive.yml +++ b/conf/archive.yml @@ -1,4 +1,4 @@ -archive: +esarchive: src: "/esarchive/" System: ECMWF-SEAS5: @@ -200,3 +200,184 @@ archive: +mars: + src: "/esarchive/" + System: + ECMWF-SEAS5: + name: "ECMWF SEAS5" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "exp/ecmwf/system5c3s/" + daily_mean: {"tas":"_f6h/", "rsds":"_s0-24h/", + "prlr":"_s0-24h/", "sfcWind":"_f6h/", + "tasmin":"_f24h/", "tasmax":"_f24h/", + "ta300":"_f12h/", "ta500":"_f12h/", "ta850":"_f12h/", + "g300":"_f12h/", "g500":"_f12h/", "g850":"_f12h/", + "tdps":"_f6h/", "hurs":"_f6h/"} + monthly_mean: {"tas":"_f6h/", "rsds":"_s0-24h/", + "prlr":"_s0-24h/", "sfcWind":"_f6h/", + "tasmin":"_f24h/", "tasmax":"_f24h/", + "ta300":"_f12h/", "ta500":"_f12h/", "ta850":"_f12h/", + "g300":"_f12h/", "g500":"_f12h/", "g850":"_f12h/", + "tdps":"_f6h/"} + nmember: + fcst: 51 + hcst: 25 + calendar: "proleptic_gregorian" + time_stamp_lag: "0" + reference_grid: "/esarchive/exp/ecmwf/system5c3s/monthly_mean/tas_f6h/tas_20180501.nc" + ECMWF-SEAS5.1: + name: "ECMWF SEAS5 (v5.1)" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "exp/ecmwf/system51c3s/" + daily_mean: {"tas":"_f6h/", "prlr":"_s0-24h/", "sfcWind":"_f6h/", + "uas":"_f6h/", "vas":"_f6h/", "psl":"_f6h/", + "tdps":"_f6h/"} + monthly_mean: {"tas":"_f6h/", "rsds":"_s0-24h/", "prlr":"_s0-24h/", + "sfcWind":"_f6h/", "tasmin":"_f24h/", "tasmax":"_f24h/", + "uas":"_f6h/", "vas":"_f6h/", "psl":"_f6h/", + "tdps":"_f6h/"} + nmember: + fcst: 51 + hcst: 25 + calendar: "proleptic_gregorian" + time_stamp_lag: "0" + reference_grid: "conf/grid_description/griddes_system51c3s.txt" + Meteo-France-System7: + name: "Meteo-France System 7" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "exp/meteofrance/system7c3s/" + monthly_mean: {"tas":"_f6h/", "g500":"_f12h/", + "prlr":"_f24h/", "sfcWind": "_f6h/", + "tasmax":"_f6h/", "tasmin": "_f6h/"} + nmember: + fcst: 51 + hcst: 25 + time_stamp_lag: "+1" + calendar: "proleptic_gregorian" + reference_grid: "conf/grid_description/griddes_system7c3s.txt" + DWD-GCFS2.1: + name: "DWD GCFS 2.1" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "exp/dwd/system21_m1/" + monthly_mean: {"tas":"_f6h/", "prlr":"_f24h/", + "g500":"_f12h/", "sfcWind":"_f6h/", + "tasmin":"_f24h/", "tasmax":"_f24h/"} + nmember: + fcst: 50 + hcst: 30 + calendar: "proleptic_gregorian" + time_stamp_lag: "+1" + reference_grid: "conf/grid_description/griddes_system21_m1.txt" + CMCC-SPS3.5: + name: "CMCC-SPS3.5" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "exp/cmcc/system35c3s/" + monthly_mean: {"tas":"_f6h/", "g500":"_f12h/", + "prlr":"_f24h/", "sfcWind": "_f6h/", + "tasmax":"_f24h/", "tasmin":"_f24h"} + nmember: + fcst: 50 + hcst: 40 + calendar: "proleptic_gregorian" + time_stamp_lag: "+1" + reference_grid: "conf/grid_description/griddes_system35c3s.txt" + JMA-CPS2: + name: "JMA System 2" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "exp/jma/system2c3s/" + monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/", + "tasmax":"_f6h/", "tasmin":"_f6h/"} + nmember: + fcst: 10 + hcst: 10 + calendar: "proleptic_gregorian" + time_stamp_lag: "+1" + reference_grid: "conf/grid_description/griddes_system2c3s.txt" + ECCC-CanCM4i: + name: "ECCC CanCM4i" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "exp/eccc/eccc1/" + monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/", + "tasmax":"_f6h/", "tasmin":"_f6h/"} + nmember: + fcst: 10 + hcst: 10 + calendar: "proleptic_gregorian" + time_stamp_lag: "+1" + reference_grid: "conf/grid_description/griddes_eccc1.txt" + UK-MetOffice-Glosea600: + name: "UK MetOffice GloSea 6 (v6.0)" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "exp/ukmo/glosea6_system600-c3s/" + monthly_mean: {"tas":"_f6h/", "tasmin":"_f24h/", + "tasmax":"_f24h/", "prlr":"_f24h/"} + nmember: + fcst: 62 + hcst: 28 + calendar: "proleptic_gregorian" + time_stamp_lag: "+1" + reference_grid: "conf/grid_description/griddes_ukmo600.txt" + NCEP-CFSv2: + name: "NCEP CFSv2" + institution: "NOAA NCEP" #? + src: "exp/ncep/cfs-v2/" + monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/", + "tasmax":"_f6h/", "tasmin":"_f6h/"} + nmember: + fcst: 20 + hcst: 20 + calendar: "gregorian" + time_stamp_lag: "0" + reference_grid: "conf/grid_description/griddes_ncep-cfsv2.txt" + Reference: + ERA5: + name: "ERA5" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "recon/ecmwf/era5/" + daily_mean: {"tas":"_f1h-r1440x721cds/", + "rsds":"_f1h-r1440x721cds/", + "prlr":"_f1h-r1440x721cds/", + "g300":"_f1h-r1440x721cds/", + "g500":"_f1h-r1440x721cds/", + "g850":"_f1h-r1440x721cds/", + "sfcWind":"_f1h-r1440x721cds/", + "tasmax":"_f1h-r1440x721cds/", + "tasmin":"_f1h-r1440x721cds/", + "ta300":"_f1h-r1440x721cds/", + "ta500":"_f1h-r1440x721cds/", + "ta850":"_f1h-r1440x721cds/", + "hurs":"_f1h-r1440x721cds/"} + monthly_mean: {"tas":"_f1h-r1440x721cds/", + "prlr":"_f1h-r1440x721cds/", + "rsds":"_f1h-r1440x721cds/", + "g300":"_f1h-r1440x721cds/", + "g500":"_f1h-r1440x721cds/", + "g850":"_f1h-r1440x721cds/", + "sfcWind":"_f1h-r1440x721cds/", + "tasmax":"_f1h-r1440x721cds/", + "tasmin":"_f1h-r1440x721cds/", + "ta300":"_f1h-r1440x721cds/", + "ta500":"_f1h-r1440x721cds/", + "ta850":"_f1h-r1440x721cds/"} + calendar: "standard" + reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" + ERA5-Land: + name: "ERA5-Land" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "recon/ecmwf/era5land/" + daily_mean: {"tas":"_f1h/", "rsds":"_f1h/", + "prlr":"_f1h/", "sfcWind":"_f1h/"} + monthly_mean: {"tas":"_f1h/","tasmin":"_f24h/", + "tasmax":"_f24h/", "prlr":"_f1h/", + "sfcWind":"_f1h/", "rsds":"_f1h/", + "tdps":"_f1h/"} + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/recon/ecmwf/era5land/daily_mean/tas_f1h/tas_201805.nc" + UERRA: + name: "ECMWF UERRA" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "recon/ecmwf/uerra_mescan/" + daily_mean: {"tas":"_f6h/"} + monthly_mean: {"tas":"_f6h/"} + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/recon/ecmwf/uerra_mescan/daily_mean/tas_f6h/tas_201805.nc" diff --git a/conf/archive_decadal.yml b/conf/archive_decadal.yml index 2b74bff89ab2c27db06ba8e46a55b125fee21151..91637024e099c9141898a17651d07cc7a0c61c24 100644 --- a/conf/archive_decadal.yml +++ b/conf/archive_decadal.yml @@ -1,4 +1,4 @@ -archive: +esarchive: src: "/esarchive/" System: # ---- diff --git a/conf/autosubmit.yml b/conf/autosubmit.yml new file mode 100644 index 0000000000000000000000000000000000000000..8b653a650b8d054b73819af2ec2cb2bebeb3c6ad --- /dev/null +++ b/conf/autosubmit.yml @@ -0,0 +1,14 @@ +esarchive: + platform: nord3v2 + module_version: autosubmit/3.14.0-foss-2015a-Python-2.7.9 + auto_version: 3.14.0 + conf_format: ini + experiment_dir: /esarchive/autosubmit/ + userID: bsc32 +mars: + platform: NORD3 ## TO BE CHANGED + module_version: autosubmit/4.0.0b-foss-2015a-Python-3.7.3 ## TO BE CHANGED + auto_version: 4.0.0 + conf_format: yaml + experiment_dir: /esarchive/autosubmit/ ## TO BE CHANGED + userID: bsc32 ## TO BE CHANGED diff --git a/conf/vars-dict.yml-OLD b/conf/vars-dict.yml-OLD deleted file mode 100644 index 04549d36001c848521f53fd704b752878b2eb862..0000000000000000000000000000000000000000 --- a/conf/vars-dict.yml-OLD +++ /dev/null @@ -1,114 +0,0 @@ - -vars: -# ECVs - tas: - units: "°C" - longname: "Daily mean temperature at surface" - outname: ~ - tasmin: - units: "°C" - longname: "Minimum daily temperature at surface" - outname: ~ - tasmax: - units: "°C" - longname: "Maximum daily temperature at surface" - outname: ~ - sfcwind: - units: "m/s" - longname: "Surface wind speed module" - outname: ~ - rsds: - units: "W/m2" - longname: "Surface solar radiation downwards" - outname: ~ - psl: - units: "hPa" - longname: "Mean sea level pressure" - outname: ~ - prlr: - units: "mm" - longname: "Total precipitation" - outname: ~ -# CFs - cfwnd1: - units: "%" - longname: "Wind Capacity factor IEC1" - outname: ~ - cfwnd2: - units: "%" - longname: "Wind Capacity factor IEC2" - outname: ~ - cfwnd3: - units: "%" - longname: "Wind Capacity factor IEC3" - outname: ~ - cfslr: - units: "%" - longname: "Solar Capacity factor" - outname: ~ -# Energy - edmnd: - units: "GW" - longname: "Electricity Demmand" - outname: ~ - wndpwo: - units: "GW" - longname: "Wind Power" - outname: ~ - dmndnetwnd: - units: "GW" - longname: "Demmand-net-Wind" - outname: ~ -# Indices - Spr32: - units: "days" - longname: > - Total count of days when daily maximum temp exceeded 32°C - from April 21st to June 21st - outname: ~ - SU35: - units: "days" - longname: > - Total count of days when daily maximum temp exceeded 35°C - from June 21st to September 21st - outname: ~ - SU36: - units: "days" - longname: > - Total count of days when daily maximum temp exceeded 36°C - from June 21st to September 21st - outname: ~ - SU40: - units: "days" - longname: > - Total count of days when daily maximum temp exceeded 40°C - from June 21st to September 21st - outname: ~ - GDD: - units: "days" - longname: > - The sum of the daily differences between daily mean - temperature and 10°C from April 1st to October 31st - outname: ~ - GST: - units: "°C" - longname: "The average temperature from April 1st to October 31st" - outname: ~ - SprTX: - units: "°C" - longname: "The average daily maximum temperature from April 1st to October 31st" - outname: ~ - WSDI: - units: "" - longname: > - The total count of days with at least 6 consecutives days - when the daily temperature maximum exceeds its 90th percentile - outname: ~ - SprR: - units: "mm" - longname: 'Total precipitation from April 21st to June 21st' - outname: ~ - HarR: - units: "mm" - longname: 'Total precipitation from August 21st to September 21st' - outname: ~ diff --git a/modules/Loading/Loading.R b/modules/Loading/Loading.R index d78b2749c3387d0b17b97310ce75e96b7c19b35b..c4191f2495a9ac3014721b6542e82815a687c248 100644 --- a/modules/Loading/Loading.R +++ b/modules/Loading/Loading.R @@ -1,8 +1,9 @@ ## TODO: remove paths to personal scratchs source("/esarchive/scratch/vagudets/repos/csoperational/R/get_regrid_params.R") # Load required libraries/funs -source("modules/Loading/dates2load.R") -source("modules/Loading/check_latlon.R") +source("modules/Loading/R/dates2load.R") +source("modules/Loading/R/get_timeidx.R") +source("modules/Loading/R/check_latlon.R") ## TODO: Move to prepare_outputs.R source("tools/libs.R") @@ -47,7 +48,8 @@ load_datasets <- function(recipe) { ##fcst.name <- recipe$Analysis$Datasets$System[[sys]]$name # get esarchive datasets dict: - archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive + ## TODO: Adapt to 'filesystem' option in recipe + archive <- read_yaml("conf/archive.yml")$esarchive exp_descrip <- archive$System[[exp.name]] freq.hcst <- unlist(exp_descrip[[store.freq]][variable]) @@ -332,6 +334,7 @@ load_datasets <- function(recipe) { } # Convert prlr from m/s to mm/day + ## TODO: Revise this ## TODO: Make a unit conversion function? if (variable == "prlr") { # Verify that the units are m/s and the same in obs and hcst diff --git a/modules/Loading/Loading_decadal.R b/modules/Loading/Loading_decadal.R index 4258fd18d8e7c9fbe32d89a010eb013c381d6ab8..b9a145e3029f176b5f2f51f067444f76591567a8 100644 --- a/modules/Loading/Loading_decadal.R +++ b/modules/Loading/Loading_decadal.R @@ -8,8 +8,9 @@ source("/esarchive/scratch/vagudets/repos/csoperational/R/get_regrid_params.R") # Load required libraries/funs source("modules/Loading/helper_loading_decadal.R") -source("modules/Loading/dates2load.R") -source("modules/Loading/check_latlon.R") +source("modules/Loading/R/dates2load.R") +source("modules/Loading/R/check_latlon.R") +source("modules/Loading/R/get_timeidx.R") source("tools/libs.R") #==================================================================== @@ -19,7 +20,8 @@ source("tools/libs.R") load_datasets <- function(recipe) { - archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive_decadal.yml"))$archive + ## + archive <- read_yaml(paste0("conf/archive_decadal.yml"))$esarchive # Print Start() info or not DEBUG <- FALSE diff --git a/modules/Loading/check_latlon.R b/modules/Loading/R/check_latlon.R similarity index 100% rename from modules/Loading/check_latlon.R rename to modules/Loading/R/check_latlon.R diff --git a/modules/Loading/dates2load.R b/modules/Loading/R/dates2load.R similarity index 51% rename from modules/Loading/dates2load.R rename to modules/Loading/R/dates2load.R index 0e3613f3a3a7e6b09d8317cdadb8bcb850b2bccc..f084ce62fb1e4798e5dc3948fe0edd3b2c3bfdd6 100644 --- a/modules/Loading/dates2load.R +++ b/modules/Loading/R/dates2load.R @@ -49,56 +49,3 @@ dates2load <- function(recipe, logger) { dim(data) <- default_dims return(data) } - -# Gets the corresponding dates or indices according -# to the sdate/leadtimes requested in the recipe -# -# The leadtimes are defined by months -# Ex. 20201101 with leadtimes 1-4 corresponds to -# the forecasting times covering December to March - -get_timeidx <- function(sdates, ltmin, ltmax, - time_freq="monthly_mean") { - - if (time_freq == "daily_mean") { - - sdates <- ymd(sdates) - idx_min <- sdates + months(ltmin - 1) - idx_max <- sdates + months(ltmax) - days(1) - - day_seq <- seq(idx_min[1], idx_max[1], by = 'days') - if (any("0229" %in% (format(day_seq, "%m%d")))) { - time_length <- as.integer(idx_max[1]-idx_min[1]) - } else { - time_length <- as.integer(idx_max[1]-idx_min[1]+1) - } - indxs <- array(numeric(), c(file_date = length(sdates), - time = time_length)) - #syear = length(sdates), - #sday = 1, sweek = 1, - - for (sdate in 1:length(sdates)) { - day_seq <- seq(idx_min[sdate], idx_max[sdate], by='days') - indxs[sdate,] <- day_seq[!(format(day_seq, "%m%d") == "0229")] - } - indxs <- as.POSIXct(indxs*86400, - tz = 'UTC', origin = '1970-01-01') - lubridate::hour(indxs) <- 12 - lubridate::minute(indxs) <- 00 - dim(indxs) <- c(file_date = length(sdates), - time = time_length) - - } else if (time_freq == "monthly_mean") { - - idx_min <- ltmin - idx_max <- ltmax - indxs <- indices(idx_min:idx_max) - - } - - # TODO: 6 hourly case - #idx1 <- (sdates + months(ltmin-1) - sdates)*4 - #idx2 <- idx1 + ndays*4 - 1 - - return(indxs) -} diff --git a/modules/Loading/R/get_timeidx.R b/modules/Loading/R/get_timeidx.R new file mode 100644 index 0000000000000000000000000000000000000000..e4f61a1d49dbe8a07927e2ecd76198074459e7c1 --- /dev/null +++ b/modules/Loading/R/get_timeidx.R @@ -0,0 +1,57 @@ +#'Gets the corresponding dates or indices according +#'to the start dates, min. and max. leadtimes and +#'time frequency. +# +#'The leadtimes are defined by months +#'Ex. 20201101 with leadtimes 1-4 corresponds to +#'the forecasting times covering November to february +#' +#'@param sdates vector containind the start dates +#'@param ltmin first leadtime +#'@param ltmax last leadtime +#'@param time_freq time frequency ("monthly_mean" or "daily_mean") + +get_timeidx <- function(sdates, ltmin, ltmax, + time_freq="monthly_mean") { + + if (time_freq == "daily_mean") { + + sdates <- ymd(sdates) + idx_min <- sdates + months(ltmin - 1) + idx_max <- sdates + months(ltmax) - days(1) + + day_seq <- seq(idx_min[1], idx_max[1], by = 'days') + if (any("0229" %in% (format(day_seq, "%m%d")))) { + time_length <- as.integer(idx_max[1]-idx_min[1]) + } else { + time_length <- as.integer(idx_max[1]-idx_min[1]+1) + } + indxs <- array(numeric(), c(file_date = length(sdates), + time = time_length)) + #syear = length(sdates), + #sday = 1, sweek = 1, + + for (sdate in 1:length(sdates)) { + day_seq <- seq(idx_min[sdate], idx_max[sdate], by='days') + indxs[sdate,] <- day_seq[!(format(day_seq, "%m%d") == "0229")] + } + indxs <- as.POSIXct(indxs*86400, + tz = 'UTC', origin = '1970-01-01') + lubridate::hour(indxs) <- 12 + lubridate::minute(indxs) <- 00 + dim(indxs) <- c(file_date = length(sdates), + time = time_length) + + } else if (time_freq == "monthly_mean") { + + idx_min <- ltmin + idx_max <- ltmax + indxs <- indices(idx_min:idx_max) + } + + # TODO: 6 hourly case + #idx1 <- (sdates + months(ltmin-1) - sdates)*4 + #idx2 <- idx1 + ndays*4 - 1 + + return(indxs) +} diff --git a/modules/Saving/Saving.R b/modules/Saving/Saving.R index 9d190ed81c51a34cf6d26ca6aa8ca5559e46210c..85c3a3a043bc9dffca0aaad3219d9cba410f8a58 100644 --- a/modules/Saving/Saving.R +++ b/modules/Saving/Saving.R @@ -4,8 +4,7 @@ source("modules/Saving/paths2save.R") save_data <- function(recipe, data, skill_metrics = NULL, - probabilities = NULL, - archive = NULL) { + probabilities = NULL) { # Wrapper for the saving functions. # recipe: The auto-s2s recipe # archive: The auto-s2s archive @@ -15,6 +14,7 @@ save_data <- function(recipe, data, # probabilities: output of compute_probabilities() # mean_bias: output of compute_mean_bias() + # Sanity checks if (is.null(recipe)) { error(recipe$Run$logger, "The 'recipe' parameter is mandatory.") stop() @@ -26,29 +26,21 @@ save_data <- function(recipe, data, "of at least two s2dv_cubes containing the hcst and obs.")) stop() } - if (is.null(archive)) { - if (tolower(recipe$Analysis$Horizon) == "seasonal") { - archive <- read_yaml(paste0(recipe$Run$code_dir, - "conf/archive.yml"))$archive - } else if (tolower(recipe$Analysis$Horizon) == "decadal") { - archive <- read_yaml(paste0(recipe$Run$code_dir, - "conf/archive_decadal.yml"))$archive - } - } - dict <- read_yaml("conf/variable-dictionary.yml") - # Create output directory outdir <- get_dir(recipe) dir.create(outdir, showWarnings = FALSE, recursive = TRUE) # Export hindcast, forecast and observations onto outfile - save_forecast(data$hcst, recipe, dict, outdir, archive = archive, - type = 'hcst') + save_forecast(recipe = recipe, data_cube = data$hcst, + type = 'hcst', + outdir = outdir) if (!is.null(data$fcst)) { - save_forecast(data$fcst, recipe, dict, outdir, - archive = archive, type = 'fcst') + save_forecast(recipe = recipe, data_cube = data$fcst, + type = 'fcst', + outdir = outdir) } - save_observations(data$obs, recipe, dict, outdir, archive = archive) + save_observations(recipe = recipe, data_cube = data$obs, + outdir = outdir) # Separate ensemble correlation from the rest of the metrics, as it has one # extra dimension "ensemble" and must be saved to a different file @@ -65,23 +57,30 @@ save_data <- function(recipe, data, # Export skill metrics onto outfile if (!is.null(skill_metrics)) { - save_metrics(skill_metrics, recipe, dict, data$hcst, outdir, - archive = archive) + save_metrics(recipe = recipe, skill = skill_metrics, + data_cube = data$hcst, + outdir = outdir) } if (!is.null(corr_metrics)) { - save_corr(corr_metrics, recipe, dict, data$hcst, outdir, - archive = archive) + save_corr(recipe = recipe, skill = corr_metrics, + data_cube = data$hcst, + outdir = outdir) } # Export probabilities onto outfile if (!is.null(probabilities)) { - save_percentiles(probabilities$percentiles, recipe, data$hcst, outdir, - archive = archive) - save_probabilities(probabilities$probs, recipe, data$hcst, outdir, - archive = archive, type = "hcst") + save_percentiles(recipe = recipe, percentiles = probabilities$percentiles, + data_cube = data$hcst, + outdir = outdir) + save_probabilities(recipe = recipe, probs = probabilities$probs, + data_cube = data$hcst, + type = "hcst", + outdir = outdir) if (!is.null(probabilities$probs_fcst)) { - save_probabilities(probabilities$probs_fcst, recipe, data$fcst, outdir, - archive = archive, type = "fcst") + save_probabilities(recipe = recipe, probs = probabilities$probs_fcst, + data_cube = data$fcst, + type = "fcst", + outdir = outdir) } } } @@ -156,13 +155,11 @@ get_latlon <- function(latitude, longitude) { } -save_forecast <- function(data_cube, - recipe, - dictionary, - outdir, - agg = "global", - archive = NULL, - type = NULL) { +save_forecast <- function(recipe, + data_cube, + type = "hcst", + agg = "global", + outdir = NULL) { # Loops over the years in the s2dv_cube containing a hindcast or forecast # and exports each year to a netCDF file. # data_cube: s2dv_cube containing the data and metadata @@ -171,7 +168,8 @@ save_forecast <- function(data_cube, # agg: aggregation, "global" or "country" lalo <- c('longitude', 'latitude') - + archive <- get_archive(recipe) + dictionary <- read_yaml("conf/variable-dictionary.yml") variable <- data_cube$attrs$Variable$varName var.longname <- data_cube$attrs$Variable$metadata[[variable]]$long_name global_attributes <- get_global_attributes(recipe, archive) @@ -179,23 +177,20 @@ save_forecast <- function(data_cube, store.freq <- recipe$Analysis$Variables$freq calendar <- archive$System[[global_attributes$system]]$calendar -# if (fcst.horizon == "seasonal") { -# calendar <- attr(data_cube$Variable, "variable")$dim$time$calendar -# } else { -# calendar <- attr(data_cube$Variable, "variable")$dim[[3]]$calendar -# } - + if (is.null(outdir)) { + outdir <- get_dir(recipe) + } + # Generate vector containing leadtimes dates <- as.PCICt(ClimProjDiags::Subset(data_cube$attrs$Dates, 'syear', 1), cal = calendar) if (fcst.horizon == 'decadal') { ## Method 1: Use the first date as init_date. But it may be better to use ## the real initialized date (ask users) -# init_date <- as.Date(data_cube$Dates$start[1], format = '%Y%m%d') + # init_date <- as.Date(data_cube$Dates$start[1], format = '%Y%m%d') ## Method 2: use initial month init_month <- archive$System[[recipe$Analysis$Datasets$System$name]]$initial_month if (type == 'hcst') { - ## PROBLEM for fcst!!!!!!!!!!!! init_date <- as.PCICt(paste0(recipe$Analysis$Time$hcst_start, '-', sprintf('%02d', init_month), '-01'), cal = calendar) @@ -299,12 +294,10 @@ save_forecast <- function(data_cube, } -save_observations <- function(data_cube, - recipe, - dictionary, - outdir, +save_observations <- function(recipe, + data_cube, agg = "global", - archive = NULL) { + outdir = NULL) { # Loops over the years in the s2dv_cube containing the observations and # exports each year to a netCDF file. # data_cube: s2dv_cube containing the data and metadata @@ -313,13 +306,18 @@ save_observations <- function(data_cube, # agg: aggregation, "global" or "country" lalo <- c('longitude', 'latitude') - + archive <- get_archive(recipe) + dictionary <- read_yaml("conf/variable-dictionary.yml") variable <- data_cube$attrs$Variable$varName var.longname <- data_cube$attrs$Variable$metadata[[variable]]$long_name global_attributes <- get_global_attributes(recipe, archive) fcst.horizon <- tolower(recipe$Analysis$Horizon) store.freq <- recipe$Analysis$Variables$freq calendar <- archive$Reference[[global_attributes$reference]]$calendar + + if (is.null(outdir)) { + outdir <- get_dir(recipe) + } # Generate vector containing leadtimes ## TODO: Move to a separate function? @@ -443,24 +441,25 @@ save_observations <- function(data_cube, # lat=attr(var.obs, 'Variables')$dat1$latitude) # } -save_metrics <- function(skill, - recipe, - dictionary, +save_metrics <- function(recipe, + skill, data_cube, - outdir, agg = "global", - archive = NULL) { + outdir = NULL) { # This function adds metadata to the skill metrics in 'skill' # and exports them to a netCDF file inside 'outdir'. # Define grid dimensions and names lalo <- c('longitude', 'latitude') + archive <- get_archive(recipe) + dictionary <- read_yaml("conf/variable-dictionary.yml") + + # Remove singleton dimensions and rearrange lon, lat and time dims if (tolower(agg) == "global") { skill <- lapply(skill, function(x) { Reorder(x, c(lalo, 'time'))}) } - # Add global and variable attributes global_attributes <- get_global_attributes(recipe, archive) ## TODO: Sort out the logic once default behavior is decided @@ -540,6 +539,9 @@ save_metrics <- function(skill, time <- times$time # Generate name of output file + if (is.null(outdir)) { + outdir <- get_dir(recipe) + } outfile <- get_filename(outdir, recipe, data_cube$attrs$Variable$varName, fcst.sdate, agg, "skill") @@ -559,16 +561,16 @@ save_metrics <- function(skill, info(recipe$Run$logger, "##### SKILL METRICS SAVED TO NETCDF FILE #####") } -save_corr <- function(skill, - recipe, - dictionary, +save_corr <- function(recipe, + skill, data_cube, - outdir, agg = "global", - archive = NULL) { + outdir = NULL) { # This function adds metadata to the ensemble correlation in 'skill' # and exports it to a netCDF file inside 'outdir'. + archive <- get_archive(recipe) + dictionary <- read_yaml("conf/variable-dictionary.yml") # Define grid dimensions and names lalo <- c('longitude', 'latitude') # Remove singleton dimensions and rearrange lon, lat and time dims @@ -576,7 +578,6 @@ save_corr <- function(skill, skill <- lapply(skill, function(x) { Reorder(x, c(lalo, 'ensemble', 'time'))}) } - # Add global and variable attributes global_attributes <- get_global_attributes(recipe, archive) ## TODO: Sort out the logic once default behavior is decided @@ -655,6 +656,9 @@ save_corr <- function(skill, time <- times$time # Generate name of output file + if (is.null(outdir)) { + outdir <- get_dir(recipe) + } outfile <- get_filename(outdir, recipe, data_cube$attrs$Variable$varName, fcst.sdate, agg, "corr") @@ -675,14 +679,14 @@ save_corr <- function(skill, "##### ENSEMBLE CORRELATION SAVED TO NETCDF FILE #####") } -save_percentiles <- function(percentiles, - recipe, +save_percentiles <- function(recipe, + percentiles, data_cube, - outdir, agg = "global", - archive = NULL) { + outdir = NULL) { # This function adds metadata to the percentiles # and exports them to a netCDF file inside 'outdir'. + archive <- get_archive(recipe) # Define grid dimensions and names lalo <- c('longitude', 'latitude') @@ -758,14 +762,15 @@ save_percentiles <- function(percentiles, fcst.sdate <- paste0("1970", recipe$Analysis$Time$sdate) } } - times <- get_times(store.freq, fcst.horizon, leadtimes, fcst.sdate, calendar) time <- times$time # Generate name of output file + if (is.null(outdir)) { + outdir <- get_dir(recipe) + } outfile <- get_filename(outdir, recipe, data_cube$attrs$Variable$varName, fcst.sdate, agg, "percentiles") - # Get grid data and metadata and export to netCDF if (tolower(agg) == "country") { country <- get_countries(grid) @@ -782,13 +787,12 @@ save_percentiles <- function(percentiles, info(recipe$Run$logger, "##### PERCENTILES SAVED TO NETCDF FILE #####") } -save_probabilities <- function(probs, - recipe, +save_probabilities <- function(recipe, + probs, data_cube, - outdir, agg = "global", - archive = NULL, - type = "hcst") { + type = "hcst", + outdir = NULL) { # Loops over the years in the s2dv_cube containing a hindcast or forecast # and exports the corresponding category probabilities to a netCDF file. # probs: array containing the probability data @@ -800,10 +804,13 @@ save_probabilities <- function(probs, # type: 'hcst' or 'fcst' lalo <- c('longitude', 'latitude') - + archive <- get_archive(recipe) variable <- data_cube$attrs$Variable$varName var.longname <- data_cube$attrs$Variable$metadata[[variable]]$long_name global_attributes <- get_global_attributes(recipe, archive) + if (is.null(outdir)) { + outdir <- get_dir(recipe) + } # Add anomaly computation to global attributes ## TODO: Sort out the logic once default behavior is decided if ((!is.null(recipe$Analysis$Workflow$Anomalies$compute)) && diff --git a/modules/Saving/paths2save.R b/modules/Saving/paths2save.R index 93196b86d194fa2dbed8913d6c675f84d038ec9b..d22d9856d290894fc7194218dc6f4bb8561f5da0 100644 --- a/modules/Saving/paths2save.R +++ b/modules/Saving/paths2save.R @@ -48,7 +48,7 @@ get_filename <- function(dir, recipe, var, date, agg, file.type) { "exp" = {file <- paste0(var, gg, "_", date)}, "obs" = {file <- paste0(var, gg, "-obs_", date)}, "percentiles" = {file <- paste0(var, gg, "-percentiles_", dd, - shortdate)}, + shortdate)}, "probs" = {file <- paste0(var, gg, "-probs_", date)}, "bias" = {file <- paste0(var, gg, "-bias_", date)}) } @@ -96,6 +96,9 @@ get_dir <- function(recipe, agg = "global") { calib.method <- tolower(recipe$Analysis$Workflow$Calibration$method) store.freq <- recipe$Analysis$Variables$freq ## TODO: Change "_country" + if (!is.null(recipe$Analysis$Region$name)) { + outdir <- paste0(outdir, "/", recipe$Analysis$Region$name) + } switch(tolower(agg), "country" = {dir <- paste0(outdir, "/", system, "/", calib.method, "-", store.freq, "/", variable, diff --git a/modules/Skill/compute_probs.R b/modules/Skill/R/compute_probs.R similarity index 100% rename from modules/Skill/compute_probs.R rename to modules/Skill/R/compute_probs.R diff --git a/modules/Skill/compute_quants.R b/modules/Skill/R/compute_quants.R similarity index 100% rename from modules/Skill/compute_quants.R rename to modules/Skill/R/compute_quants.R diff --git a/modules/Skill/s2s.metrics.R b/modules/Skill/R/s2s.metrics.R similarity index 100% rename from modules/Skill/s2s.metrics.R rename to modules/Skill/R/s2s.metrics.R diff --git a/modules/Skill/Skill.R b/modules/Skill/Skill.R index afe697ac60677e89c15b4275ef6ecf2f5721f92e..d83d92c16599bf372db9e92fc6d6475b67df3bcb 100644 --- a/modules/Skill/Skill.R +++ b/modules/Skill/Skill.R @@ -7,9 +7,9 @@ # - reliability diagram # - ask Carlos which decadal metrics he is currently using -source("modules/Skill/compute_quants.R") -source("modules/Skill/compute_probs.R") -source("modules/Skill/s2s.metrics.R") +source("modules/Skill/R/compute_quants.R") +source("modules/Skill/R/compute_probs.R") +source("modules/Skill/R/s2s.metrics.R") ## TODO: Implement this in the future ## Which parameter are required? diff --git a/modules/Visualization/Visualization.R b/modules/Visualization/Visualization.R index 6aa6b3134d8ece2e176d3f303253c90ae63d7d3b..3d065e6c740148fe417857a66ce928edb75a693f 100644 --- a/modules/Visualization/Visualization.R +++ b/modules/Visualization/Visualization.R @@ -1,8 +1,3 @@ -#G# TODO: Remove once released in s2dv/CSTools -source("modules/Visualization/tmp/PlotMostLikelyQuantileMap.R") -source("modules/Visualization/tmp/PlotCombinedMap.R") -source("modules/Visualization/tmp/clim.palette.R") - ## TODO: Add the possibility to read the data directly from netCDF ## TODO: Adapt to multi-model case ## TODO: Add param 'raw'? @@ -34,11 +29,11 @@ plot_data <- function(recipe, if (is.null(archive)) { if (tolower(recipe$Analysis$Horizon) == "seasonal") { - archive <- read_yaml(paste0(recipe$Run$code_dir, - "conf/archive.yml"))$archive + archive <- + read_yaml(paste0("conf/archive.yml"))[[recipe$Run$filesystem]] } else if (tolower(recipe$Analysis$Horizon) == "decadal") { - archive <- read_yaml(paste0(recipe$Run$code_dir, - "conf/archive_decadal.yml"))$archive + archive <- + read_yaml(paste0("conf/archive_decadal.yml"))[[recipe$Run$filesystem]] } } diff --git a/modules/Visualization/tmp/PlotCombinedMap.R b/modules/Visualization/tmp/PlotCombinedMap.R deleted file mode 100644 index a7b5fc9701765a9969a29ff27373405a9a89198e..0000000000000000000000000000000000000000 --- a/modules/Visualization/tmp/PlotCombinedMap.R +++ /dev/null @@ -1,608 +0,0 @@ -#'Plot Multiple Lon-Lat Variables In a Single Map According to a Decision Function -#'@description Plot a number a two dimensional matrices with (longitude, latitude) dimensions on a single map with the cylindrical equidistant latitude and longitude projection. -#'@author Nicolau Manubens, \email{nicolau.manubens@bsc.es} -#'@author Veronica Torralba, \email{veronica.torralba@bsc.es} -#' -#'@param maps List of matrices to plot, each with (longitude, latitude) dimensions, or 3-dimensional array with the dimensions (longitude, latitude, map). Dimension names are required. -#'@param lon Vector of longitudes. Must match the length of the corresponding dimension in 'maps'. -#'@param lat Vector of latitudes. Must match the length of the corresponding dimension in 'maps'. -#'@param map_select_fun Function that selects, for each grid point, which value to take among all the provided maps. This function receives as input a vector of values for a same grid point for all the provided maps, and must return a single selected value (not its index!) or NA. For example, the \code{min} and \code{max} functions are accepted. -#'@param display_range Range of values to be displayed for all the maps. This must be a numeric vector c(range min, range max). The values in the parameter 'maps' can go beyond the limits specified in this range. If the selected value for a given grid point (according to 'map_select_fun') falls outside the range, it will be coloured with 'col_unknown_map'. -#'@param map_dim Optional name for the dimension of 'maps' along which the multiple maps are arranged. Only applies when 'maps' is provided as a 3-dimensional array. Takes the value 'map' by default. -#'@param brks Colour levels to be sent to PlotEquiMap. This parameter is optional and adjusted automatically by the function. -#'@param cols List of vectors of colours to be sent to PlotEquiMap for the colour bar of each map. This parameter is optional and adjusted automatically by the function (up to 5 maps). The colours provided for each colour bar will be automatically interpolated to match the number of breaks. Each item in this list can be named, and the name will be used as title for the corresponding colour bar (equivalent to the parameter 'bar_titles'). -#'@param col_unknown_map Colour to use to paint the grid cells for which a map is not possible to be chosen according to 'map_select_fun' or for those values that go beyond 'display_range'. Takes the value 'white' by default. -#'@param mask Optional numeric array with dimensions (latitude, longitude), with values in the range [0, 1], indicating the opacity of the mask over each grid point. Cells with a 0 will result in no mask, whereas cells with a 1 will result in a totally opaque superimposed pixel coloured in 'col_mask'. -#'@param col_mask Colour to be used for the superimposed mask (if specified in 'mask'). Takes the value 'grey' by default. -#'@param dots Array of same dimensions as 'var' or with dimensions -#' c(n, dim(var)), where n is the number of dot/symbol layers to add to the -#' plot. A value of TRUE at a grid cell will draw a dot/symbol on the -#' corresponding square of the plot. By default all layers provided in 'dots' -#' are plotted with dots, but a symbol can be specified for each of the -#' layers via the parameter 'dot_symbol'. -#'@param bar_titles Optional vector of character strings providing the titles to be shown on top of each of the colour bars. -#'@param legend_scale Scale factor for the size of the colour bar labels. Takes 1 by default. -#'@param cex_bar_titles Scale factor for the sizes of the bar titles. Takes 1.5 by default. -#'@param fileout File where to save the plot. If not specified (default) a graphics device will pop up. Extensions allowed: eps/ps, jpeg, png, pdf, bmp and tiff -#'@param width File width, in the units specified in the parameter size_units (inches by default). Takes 8 by default. -#'@param height File height, in the units specified in the parameter size_units (inches by default). Takes 5 by default. -#'@param size_units Units of the size of the device (file or window) to plot in. Inches ('in') by default. See ?Devices and the creator function of the corresponding device. -#'@param res Resolution of the device (file or window) to plot in. See ?Devices and the creator function of the corresponding device. -#'@param drawleg Where to draw the common colour bar. Can take values TRUE, -#' FALSE or:\cr -#' 'up', 'u', 'U', 'top', 't', 'T', 'north', 'n', 'N'\cr -#' 'down', 'd', 'D', 'bottom', 'b', 'B', 'south', 's', 'S' (default)\cr -#' 'right', 'r', 'R', 'east', 'e', 'E'\cr -#' 'left', 'l', 'L', 'west', 'w', 'W' -#'@param ... Additional parameters to be passed on to \code{PlotEquiMap}. - -#'@seealso \code{PlotCombinedMap} and \code{PlotEquiMap} -#' -#'@importFrom s2dv PlotEquiMap ColorBar -#'@importFrom maps map -#'@importFrom graphics box image layout mtext par plot.new -#'@importFrom grDevices adjustcolor bmp colorRampPalette dev.cur dev.new dev.off hcl jpeg pdf png postscript svg tiff -#'@examples -#'# Simple example -#'x <- array(1:(20 * 10), dim = c(lat = 10, lon = 20)) / 200 -#'a <- x * 0.6 -#'b <- (1 - x) * 0.6 -#'c <- 1 - (a + b) -#'lons <- seq(0, 359.5, length = 20) -#'lats <- seq(-89.5, 89.5, length = 10) -#'PlotCombinedMap(list(a, b, c), lons, lats, -#' toptitle = 'Maximum map', -#' map_select_fun = max, -#' display_range = c(0, 1), -#' bar_titles = paste('% of belonging to', c('a', 'b', 'c')), -#' brks = 20, width = 10, height = 8) -#' -#'Lon <- c(0:40, 350:359) -#'Lat <- 51:26 -#'data <- rnorm(51 * 26 * 3) -#'dim(data) <- c(map = 3, lon = 51, lat = 26) -#'mask <- sample(c(0,1), replace = TRUE, size = 51 * 26) -#'dim(mask) <- c(lat = 26, lon = 51) -#'PlotCombinedMap(data, lon = Lon, lat = Lat, map_select_fun = max, -#' display_range = range(data), mask = mask, -#' width = 12, height = 8) -#' -#'@export -PlotCombinedMap <- function(maps, lon, lat, - map_select_fun, display_range, - map_dim = 'map', - brks = NULL, cols = NULL, - col_unknown_map = 'white', - mask = NULL, col_mask = 'grey', - dots = NULL, - bar_titles = NULL, legend_scale = 1, - cex_bar_titles = 1.5, - plot_margin = NULL, bar_margin = rep(0, 4), - fileout = NULL, width = 8, height = 5, - size_units = 'in', res = 100, drawleg = T, - ...) { - args <- list(...) - - # If there is any filenames to store the graphics, process them - # to select the right device - if (!is.null(fileout)) { - deviceInfo <- .SelectDevice(fileout = fileout, width = width, height = height, - units = size_units, res = res) - saveToFile <- deviceInfo$fun - fileout <- deviceInfo$files - } - - # Check probs - error <- FALSE - if (is.list(maps)) { - if (length(maps) < 1) { - stop("Parameter 'maps' must be of length >= 1 if provided as a list.") - } - check_fun <- function(x) { - is.numeric(x) && (length(dim(x)) == 2) - } - if (!all(sapply(maps, check_fun))) { - error <- TRUE - } - ref_dims <- dim(maps[[1]]) - equal_dims <- all(sapply(maps, function(x) identical(dim(x), ref_dims))) - if (!equal_dims) { - stop("All arrays in parameter 'maps' must have the same dimension ", - "sizes and names when 'maps' is provided as a list of arrays.") - } - num_maps <- length(maps) - maps <- unlist(maps) - dim(maps) <- c(ref_dims, map = num_maps) - map_dim <- 'map' - } - if (!is.numeric(maps)) { - error <- TRUE - } - if (is.null(dim(maps))) { - error <- TRUE - } - if (length(dim(maps)) != 3) { - error <- TRUE - } - if (error) { - stop("Parameter 'maps' must be either a numeric array with 3 dimensions ", - " or a list of numeric arrays of the same size with the 'lon' and ", - "'lat' dimensions.") - } - dimnames <- names(dim(maps)) - - # Check map_dim - if (is.character(map_dim)) { - if (is.null(dimnames)) { - stop("Specified a dimension name in 'map_dim' but no dimension names provided ", - "in 'maps'.") - } - map_dim <- which(dimnames == map_dim) - if (length(map_dim) < 1) { - stop("Dimension 'map_dim' not found in 'maps'.") - } else { - map_dim <- map_dim[1] - } - } else if (!is.numeric(map_dim)) { - stop("Parameter 'map_dim' must be either a numeric value or a ", - "dimension name.") - } - if (length(map_dim) != 1) { - stop("Parameter 'map_dim' must be of length 1.") - } - map_dim <- round(map_dim) - - # Work out lon_dim and lat_dim - lon_dim <- NULL - if (!is.null(dimnames)) { - lon_dim <- which(dimnames %in% c('lon', 'longitude'))[1] - } - if (length(lon_dim) < 1) { - lon_dim <- (1:3)[-map_dim][1] - } - lon_dim <- round(lon_dim) - - lat_dim <- NULL - if (!is.null(dimnames)) { - lat_dim <- which(dimnames %in% c('lat', 'latitude'))[1] - } - if (length(lat_dim) < 1) { - lat_dim <- (1:3)[-map_dim][2] - } - lat_dim <- round(lat_dim) - - # Check lon - if (!is.numeric(lon)) { - stop("Parameter 'lon' must be a numeric vector.") - } - if (length(lon) != dim(maps)[lon_dim]) { - stop("Parameter 'lon' does not match the longitude dimension in 'maps'.") - } - - # Check lat - if (!is.numeric(lat)) { - stop("Parameter 'lat' must be a numeric vector.") - } - if (length(lat) != dim(maps)[lat_dim]) { - stop("Parameter 'lat' does not match the longitude dimension in 'maps'.") - } - - # Check map_select_fun - if (is.numeric(map_select_fun)) { - if (length(dim(map_select_fun)) != 2) { - stop("Parameter 'map_select_fun' must be an array with dimensions ", - "'lon' and 'lat' if provided as an array.") - } - if (!identical(dim(map_select_fun), dim(maps)[-map_dim])) { - stop("The dimensions 'lon' and 'lat' in the 'map_select_fun' array must ", - "have the same size, name and order as in the 'maps' parameter.") - } - } - if (!is.function(map_select_fun)) { - stop("The parameter 'map_select_fun' must be a function or a numeric array.") - } - - # Check display_range - if (!is.numeric(display_range) || length(display_range) != 2) { - stop("Parameter 'display_range' must be a numeric vector of length 2.") - } - - # Check brks - if (is.null(brks) || (is.numeric(brks) && length(brks) == 1)) { - num_brks <- 5 - if (is.numeric(brks)) { - num_brks <- brks - } - brks <- seq(from = display_range[1], to = display_range[2], length.out = num_brks) - } - if (!is.numeric(brks)) { - stop("Parameter 'brks' must be a numeric vector.") - } - - # Check cols - col_sets <- list(c("#A1D99B", "#74C476", "#41AB5D", "#238B45"), - c("#6BAED6FF", "#4292C6FF", "#2171B5FF", "#08519CFF"), - c("#FFEDA0FF", "#FED976FF", "#FEB24CFF", "#FD8D3CFF"), - c("#FC4E2AFF", "#E31A1CFF", "#BD0026FF", "#800026FF"), - c("#FCC5C0", "#FA9FB5", "#F768A1", "#DD3497")) - if (is.null(cols)) { - if (length(col_sets) >= dim(maps)[map_dim]) { - chosen_sets <- 1:(dim(maps)[map_dim]) - chosen_sets <- chosen_sets + floor((length(col_sets) - length(chosen_sets)) / 2) - } else { - chosen_sets <- array(1:length(col_sets), dim(maps)[map_dim]) - } - cols <- col_sets[chosen_sets] - } else { - if (!is.list(cols)) { - stop("Parameter 'cols' must be a list of character vectors.") - } - if (!all(sapply(cols, is.character))) { - stop("Parameter 'cols' must be a list of character vectors.") - } - if (length(cols) != dim(maps)[map_dim]) { - stop("Parameter 'cols' must be a list of the same length as the number of ", - "maps in 'maps'.") - } - } - for (i in 1:length(cols)) { - if (length(cols[[i]]) != (length(brks) - 1)) { - cols[[i]] <- colorRampPalette(cols[[i]])(length(brks) - 1) - } - } - - # Check bar_titles - if (is.null(bar_titles)) { - if (!is.null(names(cols))) { - bar_titles <- names(cols) - } else { - bar_titles <- paste0("Map ", 1:length(cols)) - } - } else { - if (!is.character(bar_titles)) { - stop("Parameter 'bar_titles' must be a character vector.") - } - if (length(bar_titles) != length(cols)) { - stop("Parameter 'bar_titles' must be of the same length as the number of ", - "maps in 'maps'.") - } - } - - # Check legend_scale - if (!is.numeric(legend_scale)) { - stop("Parameter 'legend_scale' must be numeric.") - } - - # Check col_unknown_map - if (!is.character(col_unknown_map)) { - stop("Parameter 'col_unknown_map' must be a character string.") - } - - # Check col_mask - if (!is.character(col_mask)) { - stop("Parameter 'col_mask' must be a character string.") - } - - # Check mask - if (!is.null(mask)) { - if (!is.numeric(mask)) { - stop("Parameter 'mask' must be numeric.") - } - if (length(dim(mask)) != 2) { - stop("Parameter 'mask' must have two dimensions.") - } - if ((dim(mask)[1] != dim(maps)[lat_dim]) || - (dim(mask)[2] != dim(maps)[lon_dim])) { - stop("Parameter 'mask' must have dimensions c(lat, lon).") - } - } - # Check dots - if (!is.null(dots)) { - if (length(dim(dots)) != 2) { - stop("Parameter 'mask' must have two dimensions.") - } - if ((dim(dots)[1] != dim(maps)[lat_dim]) || - (dim(dots)[2] != dim(maps)[lon_dim])) { - stop("Parameter 'mask' must have dimensions c(lat, lon).") - } - } - - #---------------------- - # Identify the most likely map - #---------------------- - brks_norm <- seq(0, 1, length.out = length(brks)) - if (is.function(map_select_fun)) { - range_width <- display_range[2] - display_range[1] - ml_map <- apply(maps, c(lat_dim, lon_dim), function(x) { - if (any(is.na(x))) { - res <- NA - } else { - res <- which(x == map_select_fun(x)) - if (length(res) > 0) { - res <- res[1] - if (map_select_fun(x) < display_range[1] || - map_select_fun(x) > display_range[2]) { - res <- -0.5 - } else { - res <- res + (map_select_fun(x) - display_range[1]) / range_width - if (map_select_fun(x) == display_range[1]) { - res <- res + brks_norm[2] / (num_brks * 2) - } - } - } else { - res <- -0.5 - } - } - res - }) - } else { - stop("Providing 'map_select_fun' as array not implemented yet.") - ml_map <- map_select_fun - } - nmap <- dim(maps)[map_dim] - nlat <- length(lat) - nlon <- length(lon) - - #---------------------- - # Set latitudes from minimum to maximum - #---------------------- - if (lat[1] > lat[nlat]){ - lat <- lat[nlat:1] - indices <- list(nlat:1, TRUE) - ml_map <- do.call("[", c(list(x = ml_map), indices)) - if (!is.null(mask)){ - mask <- mask[nlat:1, ] - } - if (!is.null(dots)){ - dots <- dots[nlat:1,] - } - } - - #---------------------- - # Set layout and parameters - #---------------------- - # Open connection to graphical device - if (!is.null(fileout)) { - saveToFile(fileout) - } else if (names(dev.cur()) == 'null device') { - dev.new(units = size_units, res = res, width = width, height = height) - } - #NOTE: I think plot.new() is not necessary in any case. -# plot.new() - par(font.main = 1) - # If colorbars need to be plotted, re-define layout. - if (drawleg) { - layout(matrix(c(rep(1, nmap),2:(nmap + 1)), 2, nmap, byrow = TRUE), heights = c(6, 1.5)) - } - - #---------------------- - # Set colors and breaks and then PlotEquiMap - #---------------------- - tcols <- c(col_unknown_map, cols[[1]]) - for (k in 2:nmap) { - tcols <- append(tcols, c(col_unknown_map, cols[[k]])) - } - - tbrks <- c(-1, brks_norm + rep(1:nmap, each = length(brks))) - - if (is.null(plot_margin)) { - plot_margin <- c(5, 4, 4, 2) + 0.1 # default of par()$mar - } - - PlotEquiMap(var = ml_map, lon = lon, lat = lat, - brks = tbrks, cols = tcols, drawleg = FALSE, - filled.continents = FALSE, dots = dots, mar = plot_margin, ...) - - #---------------------- - # Add overplot on top - #---------------------- - if (!is.null(mask)) { - dims_mask <- dim(mask) - latb <- sort(lat, index.return = TRUE) - dlon <- lon[2:dims_mask[2]] - lon[1:(dims_mask[2] - 1)] - wher <- which(dlon > (mean(dlon) + 1)) - if (length(wher) > 0) { - lon[(wher + 1):dims_mask[2]] <- lon[(wher + 1):dims_mask[2]] - 360 - } - lonb <- sort(lon, index.return = TRUE) - - cols_mask <- sapply(seq(from = 0, to = 1, length.out = 10), - function(x) adjustcolor(col_mask, alpha.f = x)) - image(lonb$x, latb$x, t(mask)[lonb$ix, latb$ix], - axes = FALSE, col = cols_mask, - breaks = seq(from = 0, to = 1, by = 0.1), - xlab='', ylab='', add = TRUE, xpd = TRUE) - if (!exists('coast_color')) { - coast_color <- 'black' - } - if (min(lon) < 0) { - map('world', interior = FALSE, add = TRUE, lwd = 1, col = coast_color) # Low resolution world map (lon -180 to 180). - } else { - map('world2', interior = FALSE, add = TRUE, lwd = 1, col = coast_color) # Low resolution world map (lon 0 to 360). - } - box() - } - - #---------------------- - # Add colorbars - #---------------------- - if ('toptitle' %in% names(args)) { - size_title <- 1 - if ('title_scale' %in% names(args)) { - size_title <- args[['title_scale']] - } - old_mar <- par('mar') - old_mar[3] <- old_mar[3] - (2 * size_title + 1) - par(mar = old_mar) - } - - if (drawleg) { - for (k in 1:nmap) { - ColorBar(brks = brks, cols = cols[[k]], vertical = FALSE, - draw_separators = TRUE, extra_margin = c(2, 0, 2, 0), - label_scale = legend_scale * 1.5) - if (!is.null(bar_titles)) { - mtext(bar_titles[[k]], 3, line = -3, cex = cex_bar_titles) - } - #TODO: Change to below code. Plot title together. extra_margin needs to be adjusted. -# ColorBar(brks = brks, cols = cols[[k]], vertical = FALSE, -# draw_separators = TRUE, extra_margin = c(1, 0, 1, 0), -# label_scale = legend_scale * 1.5, title = bar_titles[[k]], title_scale = cex_bar_titles) - } - } - - # If the graphic was saved to file, close the connection with the device - if (!is.null(fileout)) dev.off() -} - -# Color bar for PlotMostLikelyQuantileMap -multi_ColorBar <- function(nmap, brks = NULL, cols = NULL, vertical = TRUE, subsampleg = NULL, - bar_limits = NULL, var_limits = NULL, - triangle_ends = NULL, plot = TRUE, - draw_separators = FALSE, - bar_titles = NULL, title_scale = 1, label_scale = 1, extra_margin = rep(0, 4), - ...) { - - minimum_value <- ceiling(1 / nmap * 10 * 1.1) * 10 - display_range = c(minimum_value, 100) - - # Check brks - if (is.null(brks) || (is.numeric(brks) && length(brks) == 1)) { - num_brks <- 5 - if (is.numeric(brks)) { - num_brks <- brks - } - brks <- seq(from = display_range[1], to = display_range[2], length.out = num_brks) - } - if (!is.numeric(brks)) { - stop("Parameter 'brks' must be a numeric vector.") - } - # Check cols - col_sets <- list(c("#A1D99B", "#74C476", "#41AB5D", "#238B45"), - c("#6BAED6FF", "#4292C6FF", "#2171B5FF", "#08519CFF"), - c("#FFEDA0FF", "#FED976FF", "#FEB24CFF", "#FD8D3CFF"), - c("#FC4E2AFF", "#E31A1CFF", "#BD0026FF", "#800026FF"), - c("#FCC5C0", "#FA9FB5", "#F768A1", "#DD3497")) - if (is.null(cols)) { - if (length(col_sets) >= nmap) { - chosen_sets <- 1:nmap - chosen_sets <- chosen_sets + floor((length(col_sets) - length(chosen_sets)) / 2) - } else { - chosen_sets <- array(1:length(col_sets), nmap) - } - cols <- col_sets[chosen_sets] - } else { - if (!is.list(cols)) { - stop("Parameter 'cols' must be a list of character vectors.") - } - if (!all(sapply(cols, is.character))) { - stop("Parameter 'cols' must be a list of character vectors.") - } - if (length(cols) != dim(maps)[map_dim]) { - stop("Parameter 'cols' must be a list of the same length as the number of ", - "maps in 'maps'.") - } - } - for (i in 1:length(cols)) { - if (length(cols[[i]]) != (length(brks) - 1)) { - cols[[i]] <- grDevices::colorRampPalette(cols[[i]])(length(brks) - 1) - } - } - - # Check bar_titles - if (is.null(bar_titles)) { - if (nmap == 3) { - bar_titles <- c("Below normal (%)", "Normal (%)", "Above normal (%)") - } else if (nmap == 5) { - bar_titles <- c("Low (%)", "Below normal (%)", - "Normal (%)", "Above normal (%)", "High (%)") - } else { - bar_titles <- paste0("Cat. ", 1:nmap, " (%)") - } - } - - if (plot) { - for (k in 1:nmap) { - s2dv::ColorBar(brks = brks, cols = cols[[k]], vertical = FALSE, subsampleg = subsampleg, - bar_limits = bar_limits, var_limits = var_limits, - triangle_ends = triangle_ends, plot = TRUE, - draw_separators = draw_separators, - title = bar_titles[[k]], title_scale = title_scale, - label_scale = label_scale, extra_margin = extra_margin) - } - } else { - #TODO: col_inf and col_sup - return(list(brks = brks, cols = cols)) - } - -} - -#TODO: use s2dv:::.SelectDevice and remove this function here? -.SelectDevice <- function(fileout, width, height, units, res) { - # This function is used in the plot functions to check the extension of the - # files where the graphics will be stored and select the right R device to - # save them. - # If the vector of filenames ('fileout') has files with different - # extensions, then it will only accept the first one, changing all the rest - # of the filenames to use that extension. - - # We extract the extension of the filenames: '.png', '.pdf', ... - ext <- regmatches(fileout, regexpr("\\.[a-zA-Z0-9]*$", fileout)) - - if (length(ext) != 0) { - # If there is an extension specified, select the correct device - ## units of width and height set to accept inches - if (ext[1] == ".png") { - saveToFile <- function(fileout) { - png(filename = fileout, width = width, height = height, res = res, units = units) - } - } else if (ext[1] == ".jpeg") { - saveToFile <- function(fileout) { - jpeg(filename = fileout, width = width, height = height, res = res, units = units) - } - } else if (ext[1] %in% c(".eps", ".ps")) { - saveToFile <- function(fileout) { - postscript(file = fileout, width = width, height = height) - } - } else if (ext[1] == ".pdf") { - saveToFile <- function(fileout) { - pdf(file = fileout, width = width, height = height) - } - } else if (ext[1] == ".svg") { - saveToFile <- function(fileout) { - svg(filename = fileout, width = width, height = height) - } - } else if (ext[1] == ".bmp") { - saveToFile <- function(fileout) { - bmp(filename = fileout, width = width, height = height, res = res, units = units) - } - } else if (ext[1] == ".tiff") { - saveToFile <- function(fileout) { - tiff(filename = fileout, width = width, height = height, res = res, units = units) - } - } else { - warning("file extension not supported, it will be used '.eps' by default.") - ## In case there is only one filename - fileout[1] <- sub("\\.[a-zA-Z0-9]*$", ".eps", fileout[1]) - ext[1] <- ".eps" - saveToFile <- function(fileout) { - postscript(file = fileout, width = width, height = height) - } - } - # Change filenames when necessary - if (any(ext != ext[1])) { - warning(paste0("some extensions of the filenames provided in 'fileout' are not ", ext[1],". The extensions are being converted to ", ext[1], ".")) - fileout <- sub("\\.[a-zA-Z0-9]*$", ext[1], fileout) - } - } else { - # Default filenames when there is no specification - warning("there are no extensions specified in the filenames, default to '.eps'") - fileout <- paste0(fileout, ".eps") - saveToFile <- postscript - } - - # return the correct function with the graphical device, and the correct - # filenames - list(fun = saveToFile, files = fileout) -} - diff --git a/modules/Visualization/tmp/PlotMostLikelyQuantileMap.R b/modules/Visualization/tmp/PlotMostLikelyQuantileMap.R deleted file mode 100644 index 9f9f1914d8de30f26adc1b285a3c60a41f264b6a..0000000000000000000000000000000000000000 --- a/modules/Visualization/tmp/PlotMostLikelyQuantileMap.R +++ /dev/null @@ -1,196 +0,0 @@ -#'Plot Maps of Most Likely Quantiles -#' -#'@author Veronica Torralba, \email{veronica.torralba@bsc.es}, Nicolau Manubens, \email{nicolau.manubens@bsc.es} -#'@description This function receives as main input (via the parameter \code{probs}) a collection of longitude-latitude maps, each containing the probabilities (from 0 to 1) of the different grid cells of belonging to a category. As many categories as maps provided as inputs are understood to exist. The maps of probabilities must be provided on a common rectangular regular grid, and a vector with the longitudes and a vector with the latitudes of the grid must be provided. The input maps can be provided in two forms, either as a list of multiple two-dimensional arrays (one for each category) or as a three-dimensional array, where one of the dimensions corresponds to the different categories. -#' -#'@param probs a list of bi-dimensional arrays with the named dimensions 'latitude' (or 'lat') and 'longitude' (or 'lon'), with equal size and in the same order, or a single tri-dimensional array with an additional dimension (e.g. 'bin') for the different categories. The arrays must contain probability values between 0 and 1, and the probabilities for all categories of a grid cell should not exceed 1 when added. -#'@param lon a numeric vector with the longitudes of the map grid, in the same order as the values along the corresponding dimension in \code{probs}. -#'@param lat a numeric vector with the latitudes of the map grid, in the same order as the values along the corresponding dimension in \code{probs}. -#'@param cat_dim the name of the dimension along which the different categories are stored in \code{probs}. This only applies if \code{probs} is provided in the form of 3-dimensional array. The default expected name is 'bin'. -#'@param bar_titles vector of character strings with the names to be drawn on top of the color bar for each of the categories. As many titles as categories provided in \code{probs} must be provided. -#'@param col_unknown_cat character string with a colour representation of the colour to be used to paint the cells for which no category can be clearly assigned. Takes the value 'white' by default. -#'@param drawleg Where to draw the common colour bar. Can take values TRUE, -#' FALSE or:\cr -#' 'up', 'u', 'U', 'top', 't', 'T', 'north', 'n', 'N'\cr -#' 'down', 'd', 'D', 'bottom', 'b', 'B', 'south', 's', 'S' (default)\cr -#' 'right', 'r', 'R', 'east', 'e', 'E'\cr -#' 'left', 'l', 'L', 'west', 'w', 'W' -#'@param ... additional parameters to be sent to \code{PlotCombinedMap} and \code{PlotEquiMap}. -#'@seealso \code{PlotCombinedMap} and \code{PlotEquiMap} -#' -#'@importFrom maps map -#'@importFrom graphics box image layout mtext par plot.new -#'@importFrom grDevices adjustcolor bmp colorRampPalette dev.cur dev.new dev.off hcl jpeg pdf png postscript svg tiff -#'@examples -#'# Simple example -#'x <- array(1:(20 * 10), dim = c(lat = 10, lon = 20)) / 200 -#'a <- x * 0.6 -#'b <- (1 - x) * 0.6 -#'c <- 1 - (a + b) -#'lons <- seq(0, 359.5, length = 20) -#'lats <- seq(-89.5, 89.5, length = 10) -#'PlotMostLikelyQuantileMap(list(a, b, c), lons, lats, -#' toptitle = 'Most likely tercile map', -#' bar_titles = paste('% of belonging to', c('a', 'b', 'c')), -#' brks = 20, width = 10, height = 8) -#' -#'# More complex example -#'n_lons <- 40 -#'n_lats <- 20 -#'n_timesteps <- 100 -#'n_bins <- 4 -#' -#'# 1. Generation of sample data -#'lons <- seq(0, 359.5, length = n_lons) -#'lats <- seq(-89.5, 89.5, length = n_lats) -#' -#'# This function builds a 3-D gaussian at a specified point in the map. -#'make_gaussian <- function(lon, sd_lon, lat, sd_lat) { -#' w <- outer(lons, lats, function(x, y) dnorm(x, lon, sd_lon) * dnorm(y, lat, sd_lat)) -#' min_w <- min(w) -#' w <- w - min_w -#' w <- w / max(w) -#' w <- t(w) -#' names(dim(w)) <- c('lat', 'lon') -#' w -#'} -#' -#'# This function generates random time series (with values ranging 1 to 5) -#'# according to 2 input weights. -#'gen_data <- function(w1, w2, n) { -#' r <- sample(1:5, n, -#' prob = c(.05, .9 * w1, .05, .05, .9 * w2), -#' replace = TRUE) -#' r <- r + runif(n, -0.5, 0.5) -#' dim(r) <- c(time = n) -#' r -#'} -#' -#'# We build two 3-D gaussians. -#'w1 <- make_gaussian(120, 80, 20, 30) -#'w2 <- make_gaussian(260, 60, -10, 40) -#' -#'# We generate sample data (with dimensions time, lat, lon) according -#'# to the generated gaussians -#'sample_data <- multiApply::Apply(list(w1, w2), NULL, -#' gen_data, n = n_timesteps)$output1 -#' -#'# 2. Binning sample data -#'prob_thresholds <- 1:n_bins / n_bins -#'prob_thresholds <- prob_thresholds[1:(n_bins - 1)] -#'thresholds <- quantile(sample_data, prob_thresholds) -#' -#'binning <- function(x, thresholds) { -#' n_samples <- length(x) -#' n_bins <- length(thresholds) + 1 -#' -#' thresholds <- c(thresholds, max(x)) -#' result <- 1:n_bins -#' lower_threshold <- min(x) - 1 -#' for (i in 1:n_bins) { -#' result[i] <- sum(x > lower_threshold & x <= thresholds[i]) / n_samples -#' lower_threshold <- thresholds[i] -#' } -#' -#' dim(result) <- c(bin = n_bins) -#' result -#'} -#' -#'bins <- multiApply::Apply(sample_data, 'time', binning, thresholds)$output1 -#' -#'# 3. Plotting most likely quantile/bin -#'PlotMostLikelyQuantileMap(bins, lons, lats, -#' toptitle = 'Most likely quantile map', -#' bar_titles = paste('% of belonging to', letters[1:n_bins]), -#' mask = 1 - (w1 + w2 / max(c(w1, w2))), -#' brks = 20, width = 10, height = 8) -#' -#'@export -PlotMostLikelyQuantileMap <- function(probs, lon, lat, cat_dim = 'bin', - bar_titles = NULL, - col_unknown_cat = 'white', drawleg = T, - ...) { - # Check probs - error <- FALSE - if (is.list(probs)) { - if (length(probs) < 1) { - stop("Parameter 'probs' must be of length >= 1 if provided as a list.") - } - check_fun <- function(x) { - is.numeric(x) && (length(dim(x)) == 2) - } - if (!all(sapply(probs, check_fun))) { - error <- TRUE - } - ref_dims <- dim(probs[[1]]) - equal_dims <- all(sapply(probs, function(x) identical(dim(x), ref_dims))) - if (!equal_dims) { - stop("All arrays in parameter 'probs' must have the same dimension ", - "sizes and names when 'probs' is provided as a list of arrays.") - } - num_probs <- length(probs) - probs <- unlist(probs) - dim(probs) <- c(ref_dims, map = num_probs) - cat_dim <- 'map' - } - if (!is.numeric(probs)) { - error <- TRUE - } - if (is.null(dim(probs))) { - error <- TRUE - } - if (length(dim(probs)) != 3) { - error <- TRUE - } - if (error) { - stop("Parameter 'probs' must be either a numeric array with 3 dimensions ", - " or a list of numeric arrays of the same size with the 'lon' and ", - "'lat' dimensions.") - } - dimnames <- names(dim(probs)) - - # Check cat_dim - if (is.character(cat_dim)) { - if (is.null(dimnames)) { - stop("Specified a dimension name in 'cat_dim' but no dimension names provided ", - "in 'probs'.") - } - cat_dim <- which(dimnames == cat_dim) - if (length(cat_dim) < 1) { - stop("Dimension 'cat_dim' not found in 'probs'.") - } - cat_dim <- cat_dim[1] - } else if (!is.numeric(cat_dim)) { - stop("Parameter 'cat_dim' must be either a numeric value or a ", - "dimension name.") - } - if (length(cat_dim) != 1) { - stop("Parameter 'cat_dim' must be of length 1.") - } - cat_dim <- round(cat_dim) - nprobs <- dim(probs)[cat_dim] - - # Check bar_titles - if (is.null(bar_titles)) { - if (nprobs == 3) { - bar_titles <- c("Below normal (%)", "Normal (%)", "Above normal (%)") - } else if (nprobs == 5) { - bar_titles <- c("Low (%)", "Below normal (%)", - "Normal (%)", "Above normal (%)", "High (%)") - } else { - bar_titles <- paste0("Cat. ", 1:nprobs, " (%)") - } - } - - minimum_value <- ceiling(1 / nprobs * 10 * 1.1) * 10 - - # By now, the PlotCombinedMap function is included below in this file. - # In the future, PlotCombinedMap will be part of s2dverification and will - # be properly imported. - PlotCombinedMap(probs * 100, lon, lat, map_select_fun = max, - display_range = c(minimum_value, 100), - map_dim = cat_dim, - bar_titles = bar_titles, - col_unknown_map = col_unknown_cat, - drawleg = drawleg, ...) -} diff --git a/modules/Visualization/tmp/clim.palette.R b/modules/Visualization/tmp/clim.palette.R deleted file mode 100644 index b23ff8428f201dbe4cb129e5f202ab2f1924532f..0000000000000000000000000000000000000000 --- a/modules/Visualization/tmp/clim.palette.R +++ /dev/null @@ -1,69 +0,0 @@ -#'Generate Climate Color Palettes -#' -#'Generates a colorblind friendly color palette with color ranges useful in -#'climate temperature variable plotting. -#' -#'@param palette Which type of palette to generate: from blue through white -#' to red ('bluered'), from red through white to blue ('redblue'), from -#' yellow through orange to red ('yellowred'), from red through orange to -#' red ('redyellow'), from purple through white to orange ('purpleorange'), -#' and from orange through white to purple ('orangepurple'). -#'@param n Number of colors to generate. -#' -#'@examples -#'lims <- seq(-1, 1, length.out = 21) -#' -#'ColorBar(lims, color_fun = clim.palette('redyellow')) -#' -#'cols <- clim.colors(20) -#'ColorBar(lims, cols) -#' -#'@rdname clim.palette -#'@importFrom grDevices colorRampPalette -#'@export -clim.palette <- function(palette = "bluered") { - if (palette == "bluered") { - colorbar <- colorRampPalette(rev(c("#67001f", "#b2182b", "#d6604d", - "#f4a582", "#fddbc7", "#f7f7f7", - "#d1e5f0", "#92c5de", "#4393c3", - "#2166ac", "#053061"))) - attr(colorbar, 'na_color') <- 'pink' - } else if (palette == "redblue") { - colorbar <- colorRampPalette(c("#67001f", "#b2182b", "#d6604d", - "#f4a582", "#fddbc7", "#f7f7f7", - "#d1e5f0", "#92c5de", "#4393c3", - "#2166ac", "#053061")) - attr(colorbar, 'na_color') <- 'pink' - } else if (palette == "yellowred") { - colorbar <- colorRampPalette(c("#ffffcc", "#ffeda0", "#fed976", - "#feb24c", "#fd8d3c", "#fc4e2a", - "#e31a1c", "#bd0026", "#800026")) - attr(colorbar, 'na_color') <- 'pink' - } else if (palette == "redyellow") { - colorbar <- colorRampPalette(rev(c("#ffffcc", "#ffeda0", "#fed976", - "#feb24c", "#fd8d3c", "#fc4e2a", - "#e31a1c", "#bd0026", "#800026"))) - attr(colorbar, 'na_color') <- 'pink' - } else if (palette == "purpleorange") { - colorbar <- colorRampPalette(c("#2d004b", "#542789", "#8073ac", - "#b2abd2", "#d8daeb", "#f7f7f7", - "#fee0b6", "#fdb863", "#e08214", - "#b35806", "#7f3b08")) - attr(colorbar, 'na_color') <- 'pink' - } else if (palette == "orangepurple") { - colorbar <- colorRampPalette(rev(c("#2d004b", "#542789", "#8073ac", - "#b2abd2", "#d8daeb", "#f7f7f7", - "#fee0b6", "#fdb863", "#e08214", - "#b35806", "#7f3b08"))) - attr(colorbar, 'na_color') <- 'pink' - } else { - stop("Parameter 'palette' must be one of 'bluered', 'redblue', 'yellowred'", - "'redyellow', 'purpleorange' or 'orangepurple'.") - } - colorbar -} -#'@rdname clim.palette -#'@export -clim.colors <- function(n, palette = "bluered") { - clim.palette(palette)(n) -} diff --git a/modules/test_decadal.R b/modules/test_decadal.R index 8998cfbe202e2d0a52cb69fe4e0ab59b9730d95f..f9f9521d6ce3092f6cd87cffd311a446eac228ac 100644 --- a/modules/test_decadal.R +++ b/modules/test_decadal.R @@ -5,7 +5,7 @@ source("modules/Skill/Skill.R") source("modules/Saving/Saving.R") source("modules/Visualization/Visualization.R") -recipe_file <- "modules/Loading/testing_recipes/recipe_decadal.yml" +recipe_file <- "recipes/atomic_recipes/recipe_decadal.yml" recipe <- prepare_outputs(recipe_file) # archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive_decadal.yml"))$archive diff --git a/modules/test_parallel_workflow.R b/modules/test_parallel_workflow.R new file mode 100644 index 0000000000000000000000000000000000000000..30c9cb91d85f7a8ddfbafbed6590a5788cb1493b --- /dev/null +++ b/modules/test_parallel_workflow.R @@ -0,0 +1,25 @@ +source("modules/Loading/Loading.R") +source("modules/Calibration/Calibration.R") +source("modules/Anomalies/Anomalies.R") +source("modules/Skill/Skill.R") +source("modules/Saving/Saving.R") +source("modules/Visualization/Visualization.R") + +args = commandArgs(trailingOnly = TRUE) +recipe_file <- args[1] +recipe <- read_atomic_recipe(recipe_file) +# Load datasets +data <- load_datasets(recipe) +# Calibrate datasets +data <- calibrate_datasets(recipe, data) +# Compute anomalies +data <- compute_anomalies(recipe, data) +# Compute skill metrics +skill_metrics <- compute_skill_metrics(recipe, data) +# Compute percentiles and probability bins +probabilities <- compute_probabilities(recipe, data) +# Export all data to netCDF +save_data(recipe, data, skill_metrics, probabilities) +# Plot data +plot_data(recipe, data, skill_metrics, probabilities, + significance = T) diff --git a/modules/test_seasonal.R b/modules/test_seasonal.R index b8541488c8540e61c694c42a0f36be60595699a9..4dd34b6187a35fe9157f6d27fb478a8178be8a25 100644 --- a/modules/test_seasonal.R +++ b/modules/test_seasonal.R @@ -5,21 +5,21 @@ source("modules/Skill/Skill.R") source("modules/Saving/Saving.R") source("modules/Visualization/Visualization.R") -recipe_file <- "modules/Loading/testing_recipes/recipe_seasonal-tests.yml" +recipe_file <- "recipes/atomic_recipes/recipe_system7c3s-tas.yml" recipe <- prepare_outputs(recipe_file) # Load datasets data <- load_datasets(recipe) # Calibrate datasets -calibrated_data <- calibrate_datasets(recipe, data) +data <- calibrate_datasets(recipe, data) # Compute anomalies -calibrated_data <- compute_anomalies(recipe, calibrated_data) +data <- compute_anomalies(recipe, data) # Compute skill metrics -skill_metrics <- compute_skill_metrics(recipe, calibrated_data) +skill_metrics <- compute_skill_metrics(recipe, data) # Compute percentiles and probability bins -probabilities <- compute_probabilities(recipe, calibrated_data) +probabilities <- compute_probabilities(recipe, data) # Export all data to netCDF -save_data(recipe, calibrated_data, skill_metrics, probabilities) +save_data(recipe, data, skill_metrics, probabilities) # Plot data -plot_data(recipe, calibrated_data, skill_metrics, probabilities, +plot_data(recipe, data, skill_metrics, probabilities, significance = T) diff --git a/modules/Loading/testing_recipes/recipe_decadal.yml b/recipes/atomic_recipes/recipe_decadal.yml similarity index 100% rename from modules/Loading/testing_recipes/recipe_decadal.yml rename to recipes/atomic_recipes/recipe_decadal.yml diff --git a/modules/Loading/testing_recipes/recipe_decadal_daily.yml b/recipes/atomic_recipes/recipe_decadal_daily.yml similarity index 100% rename from modules/Loading/testing_recipes/recipe_decadal_daily.yml rename to recipes/atomic_recipes/recipe_decadal_daily.yml diff --git a/modules/Loading/testing_recipes/recipe_decadal_monthly_2.yml b/recipes/atomic_recipes/recipe_decadal_monthly_2.yml similarity index 100% rename from modules/Loading/testing_recipes/recipe_decadal_monthly_2.yml rename to recipes/atomic_recipes/recipe_decadal_monthly_2.yml diff --git a/modules/Loading/testing_recipes/recipe_seasonal-tests.yml b/recipes/atomic_recipes/recipe_seasonal-tests.yml similarity index 100% rename from modules/Loading/testing_recipes/recipe_seasonal-tests.yml rename to recipes/atomic_recipes/recipe_seasonal-tests.yml diff --git a/modules/Loading/testing_recipes/recipe_system5c3s-rsds.yml b/recipes/atomic_recipes/recipe_system5c3s-rsds.yml similarity index 100% rename from modules/Loading/testing_recipes/recipe_system5c3s-rsds.yml rename to recipes/atomic_recipes/recipe_system5c3s-rsds.yml diff --git a/modules/Loading/testing_recipes/recipe_system5c3s-tas.yml b/recipes/atomic_recipes/recipe_system5c3s-tas.yml similarity index 100% rename from modules/Loading/testing_recipes/recipe_system5c3s-tas.yml rename to recipes/atomic_recipes/recipe_system5c3s-tas.yml diff --git a/modules/Loading/testing_recipes/recipe_system7c3s-prlr.yml b/recipes/atomic_recipes/recipe_system7c3s-prlr.yml similarity index 100% rename from modules/Loading/testing_recipes/recipe_system7c3s-prlr.yml rename to recipes/atomic_recipes/recipe_system7c3s-prlr.yml diff --git a/modules/Loading/testing_recipes/recipe_system7c3s-tas.yml b/recipes/atomic_recipes/recipe_system7c3s-tas.yml similarity index 92% rename from modules/Loading/testing_recipes/recipe_system7c3s-tas.yml rename to recipes/atomic_recipes/recipe_system7c3s-tas.yml index c8d3b5e891de09b5cc1236e1b9c85297fc27f1e3..a2a87c23dd2ce092b77f42c2f9984e5c3463d47d 100644 --- a/modules/Loading/testing_recipes/recipe_system7c3s-tas.yml +++ b/recipes/atomic_recipes/recipe_system7c3s-tas.yml @@ -39,11 +39,11 @@ Analysis: percentiles: [[1/3, 2/3], [1/10, 9/10], [1/4, 2/4, 3/4]] Indicators: index: no - ncores: 1 + ncores: 10 remove_NAs: yes Output_format: S2S4E Run: Loglevel: INFO Terminal: yes - output_dir: /esarchive/scratch/vagudets/repos/auto-s2s/out-logs/ + output_dir: /esarchive/scratch/vagudets/auto-s2s-outputs/ code_dir: /esarchive/scratch/vagudets/repos/auto-s2s/ diff --git a/modules/Loading/testing_recipes/recipe_tas-daily-regrid-to-reference.yml b/recipes/atomic_recipes/recipe_tas-daily-regrid-to-reference.yml similarity index 100% rename from modules/Loading/testing_recipes/recipe_tas-daily-regrid-to-reference.yml rename to recipes/atomic_recipes/recipe_tas-daily-regrid-to-reference.yml diff --git a/modules/Loading/testing_recipes/recipe_tas-daily-regrid-to-system.yml b/recipes/atomic_recipes/recipe_tas-daily-regrid-to-system.yml similarity index 100% rename from modules/Loading/testing_recipes/recipe_tas-daily-regrid-to-system.yml rename to recipes/atomic_recipes/recipe_tas-daily-regrid-to-system.yml diff --git a/modules/Loading/testing_recipes/recipe_test-logging.yml b/recipes/atomic_recipes/recipe_test-logging.yml similarity index 100% rename from modules/Loading/testing_recipes/recipe_test-logging.yml rename to recipes/atomic_recipes/recipe_test-logging.yml diff --git a/modules/Loading/testing_recipes/recipe_test-new-metrics.yml b/recipes/atomic_recipes/recipe_test-new-metrics.yml similarity index 100% rename from modules/Loading/testing_recipes/recipe_test-new-metrics.yml rename to recipes/atomic_recipes/recipe_test-new-metrics.yml diff --git a/modules/Loading/testing_recipes/recipe_test_anomalies.yml b/recipes/atomic_recipes/recipe_test_anomalies.yml similarity index 100% rename from modules/Loading/testing_recipes/recipe_test_anomalies.yml rename to recipes/atomic_recipes/recipe_test_anomalies.yml diff --git a/modules/Loading/testing_recipes/recipe_testing_nadia.yml b/recipes/atomic_recipes/recipe_testing_nadia.yml similarity index 100% rename from modules/Loading/testing_recipes/recipe_testing_nadia.yml rename to recipes/atomic_recipes/recipe_testing_nadia.yml diff --git a/modules/Loading/testing_recipes/wrong_recipe_example.yml b/recipes/atomic_recipes/wrong_recipe_example.yml similarity index 100% rename from modules/Loading/testing_recipes/wrong_recipe_example.yml rename to recipes/atomic_recipes/wrong_recipe_example.yml diff --git a/recipes/recipe_decadal_split.yml b/recipes/recipe_decadal_split.yml new file mode 100644 index 0000000000000000000000000000000000000000..708037f75dea5828e3574afb29e7482815ab0d83 --- /dev/null +++ b/recipes/recipe_decadal_split.yml @@ -0,0 +1,61 @@ +Description: + Author: Carlos Delgado Torres + Info: Test for spliting a decadal recipe (two variables) +Analysis: + Horizon: Decadal + Variables: + - {name: tas, freq: monthly_mean} + - {name: pr, freq: monthly_mean} + Datasets: + System: + name: EC-Earth3-i4 + member: r1i4p1f1,r2i4p1f1 + Multimodel: no + Reference: + name: JRA-55 + Time: + fcst_year: + hcst_start: 2015 # 2015-2016 in dcppA, 2017-2018 in dcppB + hcst_end: 2018 +# season: 'Annual' + ftime_min: 6 # Apr + ftime_max: 8 + Region: + - {latmin: -5, latmax: 5, lonmin: -5, lonmax: 5} + Regrid: + method: bilinear + type: to_system + Workflow: + Anomalies: + compute: no + cross_validation: + Calibration: + method: 'bias' + Skill: + metric: EnsCorr RPSS + Probabilities: + percentiles: [[1/3, 2/3]] + Indicators: + index: FALSE + ncores: 8 # Optional, int: number of cores, defaults to 1 + remove_NAs: FALSE # Optional, bool: Whether NAs are removed, defaults to FALSE + Output_format: S2S4E +Run: + Loglevel: INFO + Terminal: yes + filesystem: esarchive + output_dir: /esarchive/scratch/cdelgado/auto-s2s_logs/ + code_dir: /esarchive/scratch/cdelgado/gitlab/auto-s2s/ + autosubmit: yes + # fill only if using autosubmit + auto_conf: + script: /esarchive/scratch/cdelgado/gitlab/cdelgado_copernicus/ESS_evaluation_tool/main_decadal.R + expid: a5tx ## if left empty, create new exp? + hpc_user: bsc32924 # your hpc username + wallclock: 01:00 # hh:mm + processors_per_job: 8 # use ncores parameter? + platform: nord3v2 # make this constant? + email_notifications: yes # enable/disable email notifications + email_address: carlos.delgado@bsc.es # email address for notifications + notify_completed: yes # notify me by email when a job finishes + notify_failed: yes # notify me by email when a job fails diff --git a/recipes/recipe_splitting_example.yml b/recipes/recipe_splitting_example.yml index 78a4d18c566bd492ba66f483ae1e80f96b0db1ec..94a944682b26e7533188a917be0fc32c2479ff0c 100644 --- a/recipes/recipe_splitting_example.yml +++ b/recipes/recipe_splitting_example.yml @@ -48,7 +48,7 @@ Analysis: index: no # ? ncores: 7 remove_NAs: yes # bool, don't split - Output_format: S2S4E # string, don't split + Output_format: Scorecards # string, don't split ################################################################################ ## Run CONFIGURATION diff --git a/recipes/tests/recipe_autosubmit_marstest.yml b/recipes/tests/recipe_autosubmit_marstest.yml new file mode 100644 index 0000000000000000000000000000000000000000..dfd2159f9f734fb1a0aa2f63aa49d736495f596d --- /dev/null +++ b/recipes/tests/recipe_autosubmit_marstest.yml @@ -0,0 +1,76 @@ +################################################################################ +## RECIPE DESCRIPTION +################################################################################ + +Description: + Author: V. Agudetse + Info: Test Independent verification of two variables, two sdates, two systems + +################################################################################ +## ANALYSIS CONFIGURATION +################################################################################ + +Analysis: + Horizon: Seasonal + Variables: # ECVs and Indicators? + - {name: tas, freq: monthly_mean} + - {name: prlr, freq: monthly_mean} + Datasets: + System: # multiple systems for single model, split if Multimodel = F + - {name: ECMWF-SEAS5} + - {name: Meteo-France-System7} + Multimodel: False # single option + Reference: + - {name: ERA5} # multiple references for single model? + Time: + sdate: # list, split + - '0101' + - '0201' + fcst_year: # list, don't split, handled internally + hcst_start: '2000' # single option + hcst_end: '2016' # single option + ftime_min: 1 # single option + ftime_max: 6 # single option + Region: # multiple lists, Add region name if there is more than 1 region + - {latmin: -10, latmax: 10, lonmin: -10, lonmax: 10} + Regrid: + method: conservative + type: to_system + Workflow: + Anomalies: + compute: yes + cross_validation: yes + Calibration: + method: raw ## TODO: list, split? + Skill: + metric: RPS, RPSS, CRPS, CRPSS, FRPSS, BSS10, BSS90, mean_bias, mean_bias_SS # list, don't split + Probabilities: + percentiles: [[1/3, 2/3], [1/10, 9/10], [1/4, 2/4, 3/4]] # list, don't split + Indicators: + index: no # ? + ncores: 8 + remove_NAs: yes # bool, don't split + Output_format: S2S4E # string, don't split + +################################################################################ +## Run CONFIGURATION +################################################################################ +Run: + Loglevel: INFO + Terminal: yes + filesystem: mars + output_dir: /esarchive/scratch/vagudets/repos/auto-s2s/out-logs/ + code_dir: /esarchive/scratch/vagudets/repos/auto-s2s/ + autosubmit: yes + # fill only if using autosubmit + auto_conf: + script: /esarchive/scratch/vagudets/repos/auto-s2s/modules/test_parallel_workflow.R # path to the script to run + expid: a5ta ## if left empty, create new exp? + hpc_user: bsc32762 # your hpc username + wallclock: 04:00 # hh:mm + processors_per_job: 8 # use ncores parameter? + platform: nord3v2 # make this constant? + email_notifications: yes # enable/disable email notifications + email_address: victoria.agudetse@bsc.es # email address for notifications + notify_completed: no # notify me by email when a job finishes + notify_failed: yes # notify me by email when a job fails diff --git a/recipes/tests/recipe_multiregion.yml b/recipes/tests/recipe_multiregion.yml new file mode 100644 index 0000000000000000000000000000000000000000..bcb4d126b96410d183c338e8d700c400073cd762 --- /dev/null +++ b/recipes/tests/recipe_multiregion.yml @@ -0,0 +1,76 @@ +################################################################################ +## RECIPE DESCRIPTION +################################################################################ + +Description: + Author: V. Agudetse + Info: Test Independent verification of two variables, two sdates, two systems + +################################################################################ +## ANALYSIS CONFIGURATION +################################################################################ + +Analysis: + Horizon: Seasonal + Variables: + - {name: tas, freq: monthly_mean} + Datasets: + System: + - {name: ECMWF-SEAS5} + - {name: Meteo-France-System7} + Multimodel: no + Reference: + - {name: ERA5} + Time: + sdate: + - '0101' + - '0601' + fcst_year: + hcst_start: '2000' + hcst_end: '2016' + ftime_min: 1 + ftime_max: 6 + Region: + - {name: "tropics", latmin: -5, latmax: 5, lonmin: -10, lonmax: 10} + - {name: "nino34", latmin: -5, latmax: 5, lonmin: -10, lonmax: 60} + Regrid: + method: conservative + type: to_system + Workflow: + Anomalies: + compute: yes + cross_validation: yes + Calibration: + method: raw + Skill: + metric: RPS, RPSS, CRPS, CRPSS, FRPSS, BSS10, BSS90, mean_bias, mean_bias_SS + Probabilities: + percentiles: [[1/3, 2/3], [1/10, 9/10], [1/4, 2/4, 3/4]] + Indicators: + index: no + ncores: 8 + remove_NAs: yes + Output_format: S2S4E + +################################################################################ +## Run CONFIGURATION +################################################################################ +Run: + Loglevel: INFO + Terminal: yes + filesystem: esarchive + output_dir: /esarchive/scratch/vagudets/repos/auto-s2s/out-logs/ + code_dir: /esarchive/scratch/vagudets/repos/auto-s2s/ + autosubmit: yes + # fill only if using autosubmit + auto_conf: + script: /esarchive/scratch/vagudets/repos/auto-s2s/modules/test_parallel_workflow.R + expid: a5no # autosubmit experiment ID + hpc_user: bsc32762 # your hpc username + wallclock: 04:00 # hh:mm + processors_per_job: 8 # use ncores parameter? + platform: nord3v2 # make this constant? + email_notifications: yes # enable/disable email notifications + email_address: victoria.agudetse@bsc.es # email address for notifications + notify_completed: no # notify me by email when a job finishes + notify_failed: yes # notify me by email when a job fails diff --git a/recipes/tests/recipe_seasonal_example.yml b/recipes/tests/recipe_seasonal_example.yml new file mode 100644 index 0000000000000000000000000000000000000000..cb941f8422bbccddd9ca32e4f1f7cff07a95467e --- /dev/null +++ b/recipes/tests/recipe_seasonal_example.yml @@ -0,0 +1,76 @@ +################################################################################ +## RECIPE DESCRIPTION +################################################################################ + +Description: + Author: V. Agudetse + Info: Test Independent verification of two variables, two sdates, two systems + +################################################################################ +## ANALYSIS CONFIGURATION +################################################################################ + +Analysis: + Horizon: Seasonal + Variables: + - {name: tas, freq: monthly_mean} + - {name: prlr, freq: monthly_mean} + Datasets: + System: + - {name: ECMWF-SEAS5} + - {name: Meteo-France-System7} + Multimodel: no + Reference: + - {name: ERA5} + Time: + sdate: + - '0101' + - '0601' + fcst_year: + hcst_start: '2000' + hcst_end: '2016' + ftime_min: 1 + ftime_max: 6 + Region: + - {latmin: -10, latmax: 10, lonmin: -10, lonmax: 10} + Regrid: + method: conservative + type: to_system + Workflow: + Anomalies: + compute: yes + cross_validation: yes + Calibration: + method: raw + Skill: + metric: RPS, RPSS, CRPS, CRPSS, FRPSS, BSS10, BSS90, mean_bias, mean_bias_SS + Probabilities: + percentiles: [[1/3, 2/3], [1/10, 9/10], [1/4, 2/4, 3/4]] + Indicators: + index: no + ncores: 8 + remove_NAs: yes + Output_format: S2S4E + +################################################################################ +## Run CONFIGURATION +################################################################################ +Run: + Loglevel: INFO + Terminal: yes + filesystem: esarchive + output_dir: /esarchive/scratch/vagudets/repos/auto-s2s/out-logs/ + code_dir: /esarchive/scratch/vagudets/repos/auto-s2s/ + autosubmit: yes + # fill only if using autosubmit + auto_conf: + script: /esarchive/scratch/vagudets/repos/auto-s2s/modules/test_parallel_workflow.R + expid: a5no # autosubmit experiment ID + hpc_user: bsc32762 # your hpc username + wallclock: 04:00 # hh:mm + processors_per_job: 8 # use ncores parameter? + platform: nord3v2 # make this constant? + email_notifications: yes # enable/disable email notifications + email_address: victoria.agudetse@bsc.es # email address for notifications + notify_completed: no # notify me by email when a job finishes + notify_failed: yes # notify me by email when a job fails diff --git a/recipes/tests/recipe_seasonal_two-variables.yml b/recipes/tests/recipe_seasonal_two-variables.yml index 0cef06b312dde288b04512b00e93f4a09724a8c9..5dbd892f2bc783276e02e18de3f0e518f5f40d5d 100644 --- a/recipes/tests/recipe_seasonal_two-variables.yml +++ b/recipes/tests/recipe_seasonal_two-variables.yml @@ -30,7 +30,7 @@ Analysis: ftime_min: 1 # single option ftime_max: 3 # single option Region: # multiple lists, split? Add region name if length(Region) > 1 - - {name: "nino34", latmin: -5, latmax: 5, lonmin: -10, lonmax: 60} + - {name: "tropics", latmin: -5, latmax: 5, lonmin: -10, lonmax: 10} Regrid: method: bilinear ## TODO: allow multiple methods? type: to_system @@ -39,7 +39,7 @@ Analysis: compute: yes cross_validation: yes Calibration: - method: mse_min ## TODO: list, split? + method: raw ## TODO: list, split? Skill: metric: RPS, RPSS, CRPS, CRPSS, FRPSS, BSS10, BSS90, mean_bias, mean_bias_SS # list, don't split Probabilities: diff --git a/split.R b/split.R new file mode 100644 index 0000000000000000000000000000000000000000..869b47e3b68029f6b75c2fced19d9b24068cf0c4 --- /dev/null +++ b/split.R @@ -0,0 +1,17 @@ +suppressMessages(source("tools/libs.R")) + +# Retrieve recipe file path +args = commandArgs(trailingOnly = TRUE) +recipe_file <- args[1] + +# Check recipe and prepare output directories +recipe <- prepare_outputs(recipe_file) +# Split recipe into atomic recipes + +## TODO: Add autosubmit yes/no to the parameters? +run_parameters <- divide_recipe(recipe) +if (recipe$Run$autosubmit) { + write_autosubmit_conf(recipe, run_parameters$n_atomic_recipes) +} else { + run_parameters$outdir +} diff --git a/tests/testthat/test-decadal_monthly_1.R b/tests/testthat/test-decadal_monthly_1.R index c25d953ba30773c63f7038ce8d314830d36cee0c..dc01d549bbbce3a870eddb3e1b7c20f88015f7c9 100644 --- a/tests/testthat/test-decadal_monthly_1.R +++ b/tests/testthat/test-decadal_monthly_1.R @@ -33,7 +33,7 @@ probs <- compute_probabilities(recipe, calibrated_data) # Saving suppressWarnings({invisible(capture.output( save_data(recipe = recipe, data = calibrated_data, - skill_metrics = skill_metrics, probabilities = probs, archive = archive) + skill_metrics = skill_metrics, probabilities = probs) ))}) # Plotting diff --git a/tests/testthat/test-seasonal_daily.R b/tests/testthat/test-seasonal_daily.R index ae80d522c758f5711f0268b6359665ac276637ce..10cba33e9df2ef6cd18a4659127b4098581a0916 100644 --- a/tests/testthat/test-seasonal_daily.R +++ b/tests/testthat/test-seasonal_daily.R @@ -6,7 +6,7 @@ source("modules/Skill/Skill.R") source("modules/Saving/Saving.R") recipe_file <- "tests/recipes/recipe-seasonal_daily_1.yml" -recipe <- prepare_outputs(recipe_file) +recipe <- prepare_outputs(recipe_file, disable_checks = F) # Load datasets suppressWarnings({invisible(capture.output( data <- load_datasets(recipe) diff --git a/tests/testthat/test-seasonal_monthly.R b/tests/testthat/test-seasonal_monthly.R index f05efc9422d9b3237488d8ee248ec692093efa12..fa0af95816a462ad26b9e00a051b17c9ff55d69d 100644 --- a/tests/testthat/test-seasonal_monthly.R +++ b/tests/testthat/test-seasonal_monthly.R @@ -7,7 +7,7 @@ source("modules/Saving/Saving.R") source("modules/Visualization/Visualization.R") recipe_file <- "tests/recipes/recipe-seasonal_monthly_1.yml" -recipe <- prepare_outputs(recipe_file) +recipe <- prepare_outputs(recipe_file, disable_checks = F) archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive # Load datasets diff --git a/tools/check_recipe.R b/tools/check_recipe.R index 1853a8be985d453051de57469478184d460de676..eaec8f9a393868a47247ba5aea6ab5f7c308be9c 100644 --- a/tools/check_recipe.R +++ b/tools/check_recipe.R @@ -44,7 +44,7 @@ check_recipe <- function(recipe) { # Check time settings if (tolower(recipe$Analysis$Horizon) == "seasonal") { ## TODO: Specify filesystem - archive <- read_yaml(ARCHIVE_SEASONAL)$archive + archive <- read_yaml(ARCHIVE_SEASONAL)[[recipe$Run$filesystem]] if (!all(TIME_SETTINGS_SEASONAL %in% names(recipe$Analysis$Time))) { error(recipe$Run$logger, paste0("The element 'Time' in the recipe must contain all of the ", @@ -53,7 +53,7 @@ check_recipe <- function(recipe) { error_status <- T } } else if (tolower(recipe$Analysis$Horizon) == "decadal") { - archive <- read_yaml(ARCHIVE_DECADAL)$archive + archive <- read_yaml(ARCHIVE_DECADAL)[[recipe$Run$filesystem]] if (!all(TIME_SETTINGS_DECADAL %in% names(recipe$Analysis$Time))) { error(recipe$Run$logger, paste0("The element 'Time' in the recipe must contain all of the ", @@ -92,13 +92,10 @@ check_recipe <- function(recipe) { "The element 'ftime_max' must be an integer larger than 0.") error_status <- T } - if ((is.numeric(recipe$Analysis$Time$ftime_max)) && - (is.numeric(recipe$Analysis$Time$ftime_min))) { - if (recipe$Analysis$Time$ftime_max < recipe$Analysis$Time$ftime_min) { - error(recipe$Run$logger, - "'ftime_max' cannot be smaller than 'ftime_min'.") - error_status <- T - } + if (recipe$Analysis$Time$ftime_max < recipe$Analysis$Time$ftime_min) { + error(recipe$Run$logger, + "'ftime_max' cannot be smaller than 'ftime_min'.") + error_status <- T } # Check consistency of hindcast years if (!(as.numeric(recipe$Analysis$Time$hcst_start) %% 1 == 0) || @@ -194,9 +191,29 @@ check_recipe <- function(recipe) { # Region checks: LIMITS <- c('latmin', 'latmax', 'lonmin', 'lonmax') - if (!all(LIMITS %in% names(recipe$Analysis$Region))) { + # Ordinary recipe + if (is.null(names(recipe$Analysis$Region))) { + for (region in recipe$Analysis$Region) { + if (!all(LIMITS %in% names(region))) { + error(recipe$Run$logger, + paste0("There must be 4 elements in 'Region': ", + paste(LIMITS, collapse = ", "), ".")) + error_status <- T + } + } + if (length(recipe$Analysis$Region) > 1) { + for (region in recipe$Analysis$Region) { + if (!("name" %in% names(region)) || (is.null(region$name))) { + error(recipe$Run$logger, + paste("If more than one region has been defined, every region", + "must have a unique name.")) + } + } + } + # Atomic recipe + } else if (!all(LIMITS %in% names(recipe$Analysis$Region))) { error(recipe$Run$logger, - paste0("There must be 4 elements in 'Region': ", + paste0("There must be 4 elements in 'Region': ", paste(LIMITS, collapse = ", "), ".")) error_status <- T } @@ -325,6 +342,74 @@ check_recipe <- function(recipe) { error_status <- T } + # --------------------------------------------------------------------- + # AUTOSUBMIT CHECKS + # --------------------------------------------------------------------- + + AUTO_PARAMS <- c("script", "expid", "hpc_user", "wallclock", + "processors_per_job", "platform", "email_notifications", + "email_address", "notify_completed", "notify_failed") + # Autosubmit false by default + if (is.null(recipe$Run$autosubmit)) { + recipe$Run$autosubmit <- F + } + # Autosubmit configuration checks + if (recipe$Run$autosubmit) { + # Read autosubmit info for the specific filesystem (e.g. esarchive) + auto_specs <- read_yaml("conf/autosubmit.yml")[[recipe$Run$filesystem]] + # Check that the autosubmit configuration parameters are present + if (!("auto_conf" %in% names(recipe$Run))) { + error(recipe$Run$logger, + "The 'auto_conf' is missing from the 'Run' section of the recipe.") + error_status <- T + } else if (!all(AUTO_PARAMS %in% names(recipe$Run$auto_conf))) { + error(recipe$Run$logger, + paste0("The element 'Run:auto_conf' must contain all of the ", + "following: ", paste(AUTO_PARAMS, collapse = ", "), ".")) + error_status <- T + } + # Check that the script is not NULL and exists + if (is.null(recipe$Run$auto_conf$script)) { + error(recipe$Run$logger, + "A script must be provided to run the recipe with autosubmit.") + error_status <- T + } else if (!file.exists(recipe$Run$auto_conf$script)) { + error(recipe$Run$logger, + "Could not find the file for the script in 'auto_conf'.") + error_status <- T + } + # Check that the experiment ID exists + if (is.null(recipe$Run$auto_conf$expid)) { + error(recipe$Run$logger, + paste("The Autosubmit EXPID is missing. You can create one by", + "running the following commands on the autosubmit machine:")) + error(recipe$Run$logger, + paste("module load", auto_specs$module_version)) + error(recipe$Run$logger, + paste("autosubmit expid -H", auto_specs$platform, + "-d ")) + } else if (!dir.exists(paste0(auto_specs$experiment_dir, + recipe$Run$auto_conf$expid))) { + error(recipe$Run$logger, + paste0("No folder in ", auto_specs$experiment_dir, + " for the EXPID", recipe$Run$auto_conf$expid, + ". Please make sure it is correct.")) + } + if ((recipe$Run$auto_conf$email_notifications) && + (is.null(recipe$Run$auto_conf$email_address))) { + error(recipe$Run$logger, + "Autosubmit notifications are enabled but email address is empty!") + } + if (is.null(recipe$Run$auto_conf$hpc_user)) { + error(recipe$Run$logger, + "The 'Run:auto_conf:hpc_user' field can not be empty.") + } else if ((recipe$Run$filesystem == "esarchive") && + (!substr(recipe$Run$auto_conf$hpc_user, 1, 5) == "bsc32")) { + error(recipe$Run$logger, + "Please check your hpc_user ID. It should look like: 'bsc32xxx'") + } + } + # --------------------------------------------------------------------- # WORKFLOW CHECKS # --------------------------------------------------------------------- @@ -338,7 +423,8 @@ check_recipe <- function(recipe) { # Return error if any check has failed if (error_status) { error(recipe$Run$logger, "RECIPE CHECK FAILED.") - stop("The recipe contains some errors. The full list is in the logs.") + stop("The recipe contains some errors. Find the full list in the", + "startup.log file.") } else { info(recipe$Run$logger, "##### RECIPE CHECK SUCCESSFULL #####") # return(append(nverifications, fcst.sdate)) diff --git a/tools/divide_recipe.R b/tools/divide_recipe.R index 18962f93010bae7fe6693beb48a5f485075954f5..8bbe48facf11b2206d717a76036770594c8bb1a2 100644 --- a/tools/divide_recipe.R +++ b/tools/divide_recipe.R @@ -2,7 +2,7 @@ divide_recipe <- function(recipe) { ## TODO: Implement dependent vs independent verifications? - info(recipe$Run$logger, "Spliting recipe in single verifications.") + info(recipe$Run$logger, "Splitting recipe in single verifications.") beta_recipe <- list(Description = append(recipe$Description, list(Origin = paste("Atomic recipe,", "split from:", @@ -14,11 +14,13 @@ divide_recipe <- function(recipe) { Region = NULL, Regrid = recipe$Analysis$Regrid, Workflow = recipe$Analysis$Workflow, + ncores = recipe$Analysis$ncores, + remove_NAs = recipe$Analysis$remove_NAs, Output_format = recipe$Analysis$Output_format), Run = recipe$Run[c("Loglevel", "output_dir", "Terminal", - "code_dir", "logfile")]) - + "code_dir", "logfile", "filesystem")]) + # duplicate recipe by independent variables: all_recipes <- rep(list(beta_recipe), length(recipe$Analysis$Variables)) for (var in 1:length(recipe$Analysis$Variables)) { @@ -33,35 +35,52 @@ divide_recipe <- function(recipe) { # duplicate recipe by Datasets: # check Systems - if (recipe$Analysis$Datasets$Multimodel) { + if (recipe$Analysis$Datasets$Multimodel %in% c(TRUE,'both')) { for (reci in 1:length(all_recipes)) { all_recipes[[reci]]$Analysis$Datasets <- - list(System = recipe$Analysis$Datasets$System, - Multimodel = recipe$Analysis$Datasets$Multimodel, - Reference = NULL) + list(System = recipe$Analysis$Datasets$System, + Multimodel = recipe$Analysis$Datasets$Multimodel, + Reference = NULL) } } else { - for (sys in 1:length(recipe$Analysis$Datasets$System)) { - for (reci in 1:length(all_recipes)) { - all_recipes[[reci]]$Analysis$Datasets <- - list(System = recipe$Analysis$Datasets$System[[sys]], - Multimodel = recipe$Analysis$Datasets$Multimodel, - Reference = NULL) + if (tolower(recipe$Analysis$Horizon) == 'seasonal') { + for (sys in 1:length(recipe$Analysis$Datasets$System)) { + for (reci in 1:length(all_recipes)) { + all_recipes[[reci]]$Analysis$Datasets <- + list(System = recipe$Analysis$Datasets$System[[sys]], + Multimodel = recipe$Analysis$Datasets$Multimodel, + Reference = NULL) + } + if (sys == 1) { + recipes <- all_recipes + } else { + recipes <- append(recipes, all_recipes) + } } - if (sys == 1) { - recipes <- all_recipes - } else { - recipes <- append(recipes, all_recipes) + } else if (tolower(recipe$Analysis$Horizon) == 'decadal') { + for (sys in 1:length(recipe$Analysis$Datasets$System$name)) { + for (reci in 1:length(all_recipes)) { + all_recipes[[reci]]$Analysis$Datasets <- + list(System = list(name = recipe$Analysis$Datasets$System$name[[sys]], + member = recipe$Analysis$Datasets$System$member[[sys]]), + Multimodel = recipe$Analysis$Datasets$Multimodel, + Reference = NULL) + } + if (sys == 1) { + recipes <- all_recipes + } else { + recipes <- append(recipes, all_recipes) + } } - } + } # Rest of horizons all_recipes <- recipes rm(list = 'recipes') } # check References - for (ref in 1:length(recipe$Analysis$Datasets$Reference)) { + for (ref in 1:length(recipe$Analysis$Datasets$Reference$name)) { for (reci in 1:length(all_recipes)) { - all_recipes[[reci]]$Analysis$Datasets$Reference <- - recipe$Analysis$Datasets$Reference[[ref]] + all_recipes[[reci]]$Analysis$Datasets$Reference$name <- + recipe$Analysis$Datasets$Reference$name[[ref]] } if (ref == 1) { recipes <- all_recipes @@ -73,12 +92,10 @@ divide_recipe <- function(recipe) { # Duplicate recipe by Region recipes <- list() for (reg in 1:length(recipe$Analysis$Region)) { - # if (length(recipe$Analysis$Region[[reg]]) == 4) { ##TODO: THIS SHOULD BE ONLY CHECK IN THE RECIPE CHECKER? - for (reci in 1:length(all_recipes)) { - all_recipes[[reci]]$Analysis$Region <- recipe$Analysis$Region[[reg]] - } - recipes <- append(recipes, all_recipes) - # } + for (reci in 1:length(all_recipes)) { + all_recipes[[reci]]$Analysis$Region <- recipe$Analysis$Region[[reg]] + } + recipes <- append(recipes, all_recipes) } all_recipes <- recipes rm(list = 'recipes') @@ -102,9 +119,17 @@ divide_recipe <- function(recipe) { } all_recipes <- recipes rm(list = 'recipes') + } else if (tolower(recipe$Analysis$Horizon) == 'decadal') { + for (reci in 1:length(all_recipes)) { + all_recipes[[reci]]$Analysis$Time <- + list(fcst_year = recipe$Analysis$Time$fcst_year, + hcst_start = recipe$Analysis$Time$hcst_start, + hcst_end = recipe$Analysis$Time$hcst_end, + ftime_min = recipe$Analysis$Time$ftime_min, + ftime_max = recipe$Analysis$Time$ftime_max) + } } # Rest of horizons # Save all recipes in separate YAML files - ## TODO: Re-add recipe$Run$logger for (reci in 1:length(all_recipes)) { if (reci < 10) { recipe_number <- paste0("0", reci) @@ -112,15 +137,16 @@ divide_recipe <- function(recipe) { recipe_number <- reci } write_yaml(all_recipes[[reci]], - paste0(recipe$Run$output_dir, "/logs/recipes/recipe_", + paste0(recipe$Run$output_dir, "/logs/recipes/atomic_recipe_", recipe_number, ".yml")) - all_recipes[[reci]]$Run$logger <- recipe$Run$logger } info(recipe$Run$logger, paste("The main recipe has been divided into", length(all_recipes), "atomic recipes.")) - text <- paste0("See output directory ", recipe$Run$output_dir, + text <- paste0("Check output directory ", recipe$Run$output_dir, "/logs/recipes/ to see all the individual atomic recipes.") info(recipe$Run$logger, text) - return(all_recipes) + ## TODO: Change returns? + return(list(n_atomic_recipes = length(all_recipes), + outdir = recipe$Run$output_dir)) } diff --git a/tools/get_archive.R b/tools/get_archive.R new file mode 100644 index 0000000000000000000000000000000000000000..11602698b6a07ad4d82acde880291906263fc201 --- /dev/null +++ b/tools/get_archive.R @@ -0,0 +1,12 @@ +get_archive <- function(recipe) { + if (tolower(recipe$Analysis$Horizon) == "seasonal") { + archive <- + read_yaml(paste0("conf/archive.yml"))[[recipe$Run$filesystem]] + } else if (tolower(recipe$Analysis$Horizon) == "decadal") { + archive <- + read_yaml(paste0("conf/archive_decadal.yml"))[[recipe$Run$filesystem]] + } + ## TODO: Add dictionary filesystem dependency? + # dict <- read_yaml("conf/variable-dictionary.yml") + return(archive) +} diff --git a/tools/libs.R b/tools/libs.R index 387d9b904643745db44f9f40dcfcb0258ad5c68e..a13082df156a011b879345a973fff20d15b3861f 100644 --- a/tools/libs.R +++ b/tools/libs.R @@ -13,9 +13,10 @@ library(CSTools) library(lubridate) library(PCICt) library(RColorBrewer) +library(configr) # # library(parallel) -# library(pryr) # To check mem usage. +library(pryr) # To check mem usage. #setwd("/esarchive/scratch/nperez/git/S2S4E-backend-BSC/data-analysis/") # source('export_2_nc.R') # source('S2S/s2s.filefmt.R') @@ -31,4 +32,7 @@ source("tools/check_recipe.R") source("tools/prepare_outputs.R") source("tools/divide_recipe.R") source("tools/data_summary.R") +source("tools/read_atomic_recipe.R") +source("tools/write_autosubmit_conf.R") +source("tools/get_archive.R") # source("tools/add_dims.R") # Not sure if necessary yet diff --git a/tools/prepare_outputs.R b/tools/prepare_outputs.R index 1972aef0cb38f7e845ebd0fdfc1ca19a28950939..61825738fa417ea3000ce4473d64d1dfeaa22e17 100644 --- a/tools/prepare_outputs.R +++ b/tools/prepare_outputs.R @@ -7,6 +7,7 @@ #' #'@param recipe_file path to a YAML file with Auto-S2S configuration recipe #'@param disable_checks whether to disable the recipe checks +#'@param unique_ID whether to add a unique ID to the output directory #'@return list contaning recipe with logger, log file name and log dir name #' #'@import log4r @@ -21,30 +22,36 @@ #'@export prepare_outputs <- function(recipe_file, - disable_checks = FALSE) { - -# recipe_file: path to recipe YAML file -# disable_checks: If TRUE, does not perform checks on recipe - + disable_checks = FALSE, + uniqueID = TRUE) { + # recipe_file: path to recipe YAML file + # disable_checks: If TRUE, does not perform checks on recipe + # disable_uniqueID: If TRUE, does not add a unique ID to output dir recipe <- read_yaml(recipe_file) recipe$recipe_path <- recipe_file recipe$name <- tools::file_path_sans_ext(basename(recipe_file)) + output_dir = recipe$Run$output_dir - # Create output folders: - folder_name <- paste0(gsub(".yml", "", gsub("/", "_", recipe$name)), "_", - gsub(" ", "", gsub(":", "", gsub("-", "", Sys.time())))) + # Create output folders + if (!uniqueID) { + folder_name <- paste0(gsub(".yml", "", gsub("/", "_", recipe$name))) + } else { + folder_name <- paste0(gsub(".yml", "", gsub("/", "_", recipe$name)), "_", + gsub(" ", "", gsub(":", "", gsub("-", "", + Sys.time())))) + } print("Saving all outputs to:") - print(output_dir) - print(folder_name) + print(paste0(output_dir, folder_name)) dir.create(file.path(output_dir, folder_name, 'outputs'), recursive = TRUE) dir.create(file.path(output_dir, folder_name, 'logs')) dir.create(file.path(output_dir, folder_name, 'logs', 'recipes')) + ## TODO: Move this part to main recipe # Copy recipe to output folder file.copy(recipe$recipe_path, file.path(output_dir, folder_name, 'logs', 'recipes')) # Create log output file - logfile <- file.path(output_dir, folder_name, 'logs', 'log.txt') + logfile <- file.path(output_dir, folder_name, 'logs', 'main.log') file.create(logfile) # Set default behaviour of logger @@ -68,9 +75,17 @@ prepare_outputs <- function(recipe_file, appenders = list(file_appender(logfile, append = TRUE, layout = default_log_layout()))) } + ## TODO: Move output_dir assignation recipe$Run$output_dir <- file.path(output_dir, folder_name) recipe$Run$logger <- logger recipe$Run$logfile <- logfile + + # Set up default filesystem + if (is.null(recipe$Run$filesystem)) { + recipe$Run$filesystem <- "esarchive" + warn(recipe$Run$logger, + "Filesystem not specified in the recipe. Setting it to 'esarchive'.") + } # Run recipe checker if (disable_checks) { warn(recipe$Run$logger, diff --git a/tools/read_atomic_recipe.R b/tools/read_atomic_recipe.R new file mode 100644 index 0000000000000000000000000000000000000000..1eadb707b3c3293c12a2450d171471ef5070c1e9 --- /dev/null +++ b/tools/read_atomic_recipe.R @@ -0,0 +1,57 @@ +#'Read recipe YAML file and create and store logfile info +#' +#'The purpose of this function is to read the atomic recipe generated by +#'divide_recipe() and create its individual logfile in the output directory +#'specified in the recipe. It returns the recipe as a list with an object of +#'class logger added to it, that stores information on the workflow execution +#'and errors. +#' +#'@param recipe_file path to a YAML file with Auto-S2S configuration recipe +#' +#'@return list contaning recipe with logger, log file name and log dir name +#' +#'@import log4r +#'@import yaml +#' +#'@examples +#'setwd("/esarchive/scratch/vagudets/repos/auto-s2s/") +#'library(yaml) +#'recipe <- prepare_outputs("modules/data_load/recipe_1.yml") +#'info(recipe$Run$logger, "This is an info message") +#' +#'@export + +read_atomic_recipe <- function(recipe_file) { + # recipe_file: path to recipe YAML file + recipe <- read_yaml(recipe_file) + recipe$recipe_path <- recipe_file + recipe$name <- tools::file_path_sans_ext(basename(recipe_file)) + # Create log file for atomic recipe + logfile <- file.path(recipe$Run$output_dir, 'logs', + paste0(recipe$name, '.log')) + file.create(logfile) + # Set default behaviour of logger + if (is.null(recipe$Run)) { + recipe$Run <- list(Loglevel = 'INFO', Terminal = TRUE) + } + if (is.null(recipe$Run$Loglevel)) { + recipe$Run$Loglevel <- 'INFO' + } + if (!is.logical(recipe$Run$Terminal)) { + recipe$Run$Terminal <- TRUE + } + # logger set-up + if (recipe$Run$Terminal) { + logger <- log4r::logger(threshold = recipe$Run$Loglevel, + appenders = list(console_appender(layout = default_log_layout()), + file_appender(logfile, append = TRUE, + layout = default_log_layout()))) + } else { + logger <- log4r::logger(threshold = recipe$Run$Loglevel, + appenders = list(file_appender(logfile, append = TRUE, + layout = default_log_layout()))) + } + recipe$Run$logger <- logger + recipe$Run$logfile <- logfile + return(recipe) +} diff --git a/tools/write_autosubmit_conf.R b/tools/write_autosubmit_conf.R new file mode 100644 index 0000000000000000000000000000000000000000..a0208a9e12d1b0bd5640f57482d5366b93a0b5cc --- /dev/null +++ b/tools/write_autosubmit_conf.R @@ -0,0 +1,108 @@ +# Function to write autosubmit configuration from an Auto-S2S recipe +write_autosubmit_conf <- function(recipe, nchunks) { + # Experiment ID + expid <- recipe$Run$auto_conf$expid + # Directory with the experiment templates + template_dir <- paste0("autosubmit/conf_", recipe$Run$filesystem, "/") + # Read autosubmit info for the specific filesystem (e.g. esarchive) + auto_specs <- read_yaml("conf/autosubmit.yml")[[recipe$Run$filesystem]] + # Output directory + dest_dir <- paste0(auto_specs$experiment_dir, expid, "/conf/") + # Modify the configuration files according to the info in the recipe + for (file in list.files(template_dir)) { + conf_type <- strsplit(file, split = "[.]")[[1]][1] + extension <- strsplit(file, split = "[.]")[[1]][2] + dest_file <- paste0(conf_type, "_", expid, ".", extension) + # Read configuration file + conf <- read.config(file = paste0(template_dir, file)) + if (conf_type == "autosubmit") { + # Section 1: autosubmit.conf + ## expid, email notifications and address + conf$config$EXPID <- expid + if (recipe$Run$auto_conf$email_notifications) { + conf$mail$NOTIFICATIONS <- "True" + } else { + conf$mail$NOTIFICATIONS <- "False" + } + conf$mail$TO <- recipe$Run$auto_conf$email_address + } else if (conf_type == "expdef") { + # Section 2: expdef + ## expid, numchunks, project_type?, project_destination = auto-s2s + conf$DEFAULT$EXPID <- expid + conf$experiment$DATELIST <- format(Sys.Date(), "%Y%m%d") + conf$experiment$NUMCHUNKS <- nchunks + conf$local$PROJECT_PATH <- recipe$Run$code_dir + } else if (conf_type == "jobs") { + # Section 3: jobs + ## wallclock, notify_on, platform?, processors + # Different file structure depending on autosubmit version + if (auto_specs$auto_version == "4.0.0") { + jobs <- conf$JOBS + } else { + jobs <- conf + } + jobs$verification$WALLCLOCK <- recipe$Run$auto_conf$wallclock + if (recipe$Run$auto_conf$notify_completed) { + jobs$verification$NOTIFY_ON <- paste(jobs$verification$NOTIFY_ON, + "COMPLETED") + } + if (recipe$Run$auto_conf$notify_failed) { + jobs$verification$NOTIFY_ON <- paste(jobs$verification$NOTIFY_ON, + "FAILED") + } + jobs$verification$PROCESSORS <- recipe$Run$auto_conf$processors_per_job # ncores? + # Return to original list + if (auto_specs$auto_version == "4.0.0") { + conf$JOBS <- jobs + } else { + conf <- jobs + } + } else if (conf_type == "platforms") { + # Section 4: platform configuration + ## nord3v2 configuration... platform name? user, processors_per_node + if (auto_specs$auto_version == "4.0.0") { + conf$Platforms[[auto_specs$platform]]$USER <- + recipe$Run$auto_conf$hpc_user + } else { + conf[[auto_specs$platform]]$USER <- recipe$Run$auto_conf$hpc_user + } + } else if (conf_type == "proj") { + # Section 5: proj + ## modules? Info that goes on script, e.g. output directory + conf$common$OUTDIR <- recipe$Run$output_dir + conf$common$SCRIPT <- recipe$Run$auto_conf$script + } + # Write config file inside autosubmit dir + ## TODO: Change write.type depending on autosubmit version + write.config(conf, paste0(dest_dir, dest_file), + write.type = auto_specs$conf_format) + Sys.chmod(paste0(dest_dir, dest_file), mode = "755", use_umask = F) + } + info(recipe$Run$logger, + paste("##### AUTOSUBMIT CONFIGURATION WRITTEN FOR", expid, "#####")) + info(recipe$Run$logger, + paste0("You can check your experiment configuration at: ", + "/esarchive/autosubmit/", expid, "/conf/")) + # Print instructions/commands for user + if (recipe$Run$Terminal) { + ## TODO: Change SSH message for other environments (outside BSC) + info(recipe$Run$logger, + paste("Please SSH into bscesautosubmit01 or bscesautosubmit02 and run", + "the following commands:")) + info(recipe$Run$logger, + paste("module load", auto_specs$module_version)) + info(recipe$Run$logger, + paste("autosubmit create", expid)) + info(recipe$Run$logger, + paste("autosubmit refresh", expid)) + info(recipe$Run$logger, + paste("nohup autosubmit run", expid, "& disown")) + } else { + print(paste("Please SSH into bscesautosubmit01 or bscesautosubmit02 and run", + "the following commands:")) + print(paste("module load", auto_specs$module_version)) + print(paste("autosubmit create", expid)) + print(paste("autosubmit refresh", expid)) + print(paste("nohup autosubmit run", expid, "& disown")) + } +}