diff --git a/MODULES b/MODULES index 20d77dcf0ad3e61255357634944cde81aa339ba0..e2709d66a4f84df02c03d753c44c801c44e8c60e 100644 --- a/MODULES +++ b/MODULES @@ -7,21 +7,25 @@ if [[ $BSC_MACHINE == "nord3v2" ]]; then module use /gpfs/projects/bsc32/software/suselinux/11/modules/all module unuse /apps/modules/modulefiles/applications /apps/modules/modulefiles/compilers /apps/modules/modulefiles/tools /apps/modules/modulefiles/libraries /apps/modules/modulefiles/environment + module purge module load CDO/1.9.8-foss-2019b module load R/4.1.2-foss-2019b module load OpenMPI/4.0.5-GCC-8.3.0-nord3-v2 - module load GEOS - module load GDAL - module load PROJ + module load GEOS/3.7.2-foss-2019b-Python-3.7.4 + module load GDAL/3.5.0-foss-2019b-Python-3.7.4 + module load PROJ/9.0.0-GCCcore-8.3.0 + module load Phantomjs/2.1.1 else + module purge module load CDO/1.9.8-foss-2015a module load R/4.1.2-foss-2015a-bare - module load GEOS - module load GDAL - module load PROJ + module load GEOS/3.7.2-foss-2015a-Python-3.7.3 + module load GDAL/2.2.1-foss-2015a + module load PROJ/4.8.0-foss-2015a + module load Phantomjs/2.1.1 fi diff --git a/OperationalCS.R b/OperationalCS.R deleted file mode 100644 index ec01a30ec482167ee7dbc07d31389086f7548721..0000000000000000000000000000000000000000 --- a/OperationalCS.R +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env Rscript -args = commandArgs(trailingOnly = TRUE) -# To test: -# args <- NULL; args[1] <- "recipes/seasonal_oper.yml" - - -# execution: Rscript OperationalCS.R recipe.yml -# This code checks the recipe and builds and executes the workflow -print(args) -library(yaml) - -recipe <- read_yaml(args[1]) -recipe$filename <- args[1] - -# Load required libraries -source("tools/libs.R") - -# Create output folder and log: -logger <- prepare_outputs(recipe = recipe) -folder <- logger$foldername -log_file <- logger$logname -logger <- logger$logger - -# Checks: -verifications <- check_recipe(recipe, file = args[2], conf, logger) -# Divide recipe into single verifications recipes: -total_recipes <- divide_recipe(recipe, verifications, folder, logger) - -# Divide recipe into single verifications recipes: -total_recipes <- divide_recipe(recipe, verifications, folder, logger) -# Go to verification code: -capture.output(source("modules/verifications.R"), - file = log_file, type ='message', - append = TRUE) - - diff --git a/autosubmit/auto-scorecards.sh b/autosubmit/auto-scorecards.sh new file mode 100644 index 0000000000000000000000000000000000000000..4b5273725bed84811e1267048d035a0e2f712a28 --- /dev/null +++ b/autosubmit/auto-scorecards.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +############ AUTOSUBMIT INPUTS ############ +proj_dir=%PROJDIR% +outdir=%OUTDIR% +recipe=%RECIPE% +############################### + +cd $proj_dir + +recipe=${outdir}/logs/recipes/${recipe} + +source MODULES + +Rscript modules/Scorecards/execute_scorecards.R ${recipe} ${outdir} diff --git a/autosubmit/auto-verification-v4.sh b/autosubmit/auto-verification-CERISE.sh similarity index 55% rename from autosubmit/auto-verification-v4.sh rename to autosubmit/auto-verification-CERISE.sh index e21bef2dd1b4df17883a140943ff4f1f3bedb61c..caf2dd0ec8194b2868187eb26697a197003b5d1a 100644 --- a/autosubmit/auto-verification-v4.sh +++ b/autosubmit/auto-verification-CERISE.sh @@ -7,13 +7,16 @@ script=%common.SCRIPT% CHUNK=%CHUNK% ############################### -## TODO: How to define the script cd $proj_dir -# script=modules/test_parallel_workflow.R atomic_recipe_number=$(printf "%02d" $CHUNK) atomic_recipe=${outdir}/logs/recipes/atomic_recipe_${atomic_recipe_number}.yml -source MODULES +## Workaround to avoid bug in conda activate/source activate when running +## inside bash script +set +eu +source /gpfs/projects/bsc32/software/suselinux/11/software/Miniconda3/4.7.10/etc/profile.d/conda.sh +conda activate /esarchive/scratch/vagudets/conda-cerise/conda/envs +set -eu Rscript ${script} ${atomic_recipe} diff --git a/autosubmit/auto-verification.sh b/autosubmit/auto-verification.sh index bbd065563c5b88c6b49fe4339c578534a883eacf..0089e322d63e89a5578c98a6a64a1369b5b9b108 100644 --- a/autosubmit/auto-verification.sh +++ b/autosubmit/auto-verification.sh @@ -2,8 +2,8 @@ ############ AUTOSUBMIT INPUTS ############ proj_dir=%PROJDIR% -outdir=%OUTDIR% -script=%SCRIPT% +outdir=%common.OUTDIR% +script=%common.SCRIPT% CHUNK=%CHUNK% ############################### diff --git a/autosubmit/conf_esarchive/autosubmit.conf b/autosubmit/conf_esarchive/autosubmit.conf deleted file mode 100644 index 685876a1ea82079a39eaa7ff1468f2549d8dc52f..0000000000000000000000000000000000000000 --- a/autosubmit/conf_esarchive/autosubmit.conf +++ /dev/null @@ -1,42 +0,0 @@ -[config] -# Experiment identifier -# No need to change -EXPID = -# No need to change. -# Autosubmit version identifier -AUTOSUBMIT_VERSION = 3.14.0 -# Default maximum number of jobs to be waiting in any platform -# Default = 3 -MAXWAITINGJOBS = 16 -# Default maximum number of jobs to be running at the same time at any platform -# Default = 6 -TOTALJOBS = 16 -# Time (seconds) between connections to the HPC queue scheduler to poll already submitted jobs status -# Default = 10 -SAFETYSLEEPTIME = 10 -# Number of retrials if a job fails. Can be override at job level -# Default = 0 -RETRIALS = 0 - -[mail] -# Enable mail notifications -# Default = False -NOTIFICATIONS = -# Mail address where notifications will be received -TO = - -[communications] -# Communications library used to connect with platforms: paramiko or saga. -# Default = paramiko -API = paramiko - -[storage] -# Defines the way of storing the progress of the experiment. The available options are: -# A PICKLE file (pkl) or an SQLite database (db). Default = pkl -TYPE = pkl -# Defines if the remote logs will be copied to the local platform. Default = True. -COPY_REMOTE_LOGS = True - -[migrate] -# Changes experiment files owner. -TO_USER = diff --git a/autosubmit/conf_esarchive/autosubmit.yml b/autosubmit/conf_esarchive/autosubmit.yml new file mode 100644 index 0000000000000000000000000000000000000000..0fd5d5c6aaf61945d131da77cda08d8d1fdd86cd --- /dev/null +++ b/autosubmit/conf_esarchive/autosubmit.yml @@ -0,0 +1,22 @@ +config: + EXPID: + AUTOSUBMIT_VERSION: 4.0.0b0 + MAXWAITINGJOBS: 16 + # Default maximum number of jobs to be running at the same time at any platform + # Default: 6 + TOTALJOBS: 16 + SAFETYSLEEPTIME: 10 + RETRIALS: 0 +mail: + NOTIFICATIONS: + TO: +communications: + # Communications library used to connect with platforms: paramiko or saga. + # Default: paramiko + API: paramiko +storage: + # Defines the way of storing the progress of the experiment. The available options are: + # A PICKLE file (pkl) or an SQLite database (db). Default: pkl + TYPE: pkl + # Defines if the remote logs will be copied to the local platform. Default: True. + COPY_REMOTE_LOGS: True diff --git a/autosubmit/conf_esarchive/expdef.conf b/autosubmit/conf_esarchive/expdef.conf deleted file mode 100644 index d3a3370ad0a0d8ec62b7bfbea3a34b50e41504d4..0000000000000000000000000000000000000000 --- a/autosubmit/conf_esarchive/expdef.conf +++ /dev/null @@ -1,76 +0,0 @@ -[DEFAULT] -# Experiment identifier -# No need to change -EXPID = -# HPC name. -# No need to change -HPCARCH = nord3v2 - -[experiment] -# Supply the list of start dates. Available formats: YYYYMMDD YYYYMMDDhh YYYYMMDDhhmm -# You can also use an abbreviated syntax for multiple dates with common parts: 200001[01 15] <=> 20000101 20000115 -# 200001[01-04] <=> 20000101 20000102 20000103 20000104 -# DATELIST = 19600101 19650101 19700101 -# DATELIST = 1960[0101 0201 0301] -# DATELIST = 19[60-65] -DATELIST = -# Supply the list of members. Format fcX -# You can also use an abreviated syntax for multiple members: fc[0 1 2] <=> fc0 fc1 fc2 -# fc[0-2] <=> fc0 fc1 fc2 -# MEMBERS = fc0 fc1 fc2 fc3 fc4 -#MEMBERS = fc4 -MEMBERS = fc0 -# Chunk size unit. STRING = hour, day, month, year -CHUNKSIZEUNIT = month -# Chunk size. NUMERIC = 4, 6, 12 -CHUNKSIZE = 1 -# Total number of chunks in experiment. NUMERIC = 30, 15, 10 -NUMCHUNKS = -# Initial chunk of the experiment. Optional. DEFAULT = 1 -CHUNKINI = -# Calendar used. LIST: standard, noleap -CALENDAR = standard - -[project] -# Select project type. STRING = git, svn, local, none -# If PROJECT_TYPE is set to none, Autosubmit self-contained dummy templates will be used -PROJECT_TYPE = local -# Destination folder name for project. type = STRING, default = leave empty, -PROJECT_DESTINATION = auto-s2s - -# If PROJECT_TYPE is not git, no need to change -[git] -# Repository URL STRING = 'https://github.com/torvalds/linux.git' -PROJECT_ORIGIN = https://earth.bsc.es/gitlab/es/auto-s2s.git -# Select branch or tag, STRING, default = 'master', help = {'master' (default), 'develop', 'v3.1b', ...} -PROJECT_BRANCH = master -# type = STRING, default = leave empty, help = if model branch is a TAG leave empty -PROJECT_COMMIT = - -# If PROJECT_TYPE is not svn, no need to change -[svn] -# type = STRING, help = 'https://svn.ec-earth.org/ecearth3' -PROJECT_URL = -# Select revision number. NUMERIC = 1778 -PROJECT_REVISION = - -# If PROJECT_TYPE is not local, no need to change -[local] -# type = STRING, help = /foo/bar/ecearth -PROJECT_PATH = /esarchive/scratch/vagudets/repos/auto-s2s/ - -# If PROJECT_TYPE is none, no need to change -[project_files] -# Where is PROJECT CONFIGURATION file location relative to project root path -FILE_PROJECT_CONF = -# Where is JOBS CONFIGURATION file location relative to project root path -FILE_JOBS_CONF = -# Default job scripts type in the project. type = STRING, default = bash, supported = 'bash', 'python' or 'r' -JOB_SCRIPTS_TYPE = - -[rerun] -# Is a rerun or not? [Default: Do set FALSE]. BOOLEAN = TRUE, FALSE -RERUN = FALSE -# If RERUN = TRUE then supply the list of chunks to rerun -# LIST = [ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ] -CHUNKLIST = diff --git a/autosubmit/conf_esarchive/expdef.yml b/autosubmit/conf_esarchive/expdef.yml new file mode 100644 index 0000000000000000000000000000000000000000..8dc29b27843729afa89be242a0d0de96bad1b3ec --- /dev/null +++ b/autosubmit/conf_esarchive/expdef.yml @@ -0,0 +1,44 @@ +DEFAULT: + EXPID: + HPCARCH: nord3v2 +experiment: + DATELIST: + MEMBERS: fc0 + CHUNKSIZEUNIT: month + CHUNKSIZE: 1 + NUMCHUNKS: + CHUNKINI: 1 + CALENDAR: standard +project: + PROJECT_TYPE: local + # Destination folder name for project. type: STRING, default: leave empty, + PROJECT_DESTINATION: auto-s2s +# If PROJECT_TYPE is not git, no need to change +git: + # Repository URL STRING: 'https://github.com/torvalds/linux.git' + PROJECT_ORIGIN: https://earth.bsc.es/gitlab/es/auto-s2s.git + # Select branch or tag, STRING, default: 'master', help: {'master' (default), 'develop', 'v3.1b', ...} + PROJECT_BRANCH: master + # type: STRING, default: leave empty, help: if model branch is a TAG leave empty + PROJECT_COMMIT: '' +svn: + PROJECT_URL: '' + PROJECT_REVISION: '' +# If PROJECT_TYPE is not local, no need to change +local: + # type: STRING, help: /foo/bar/ecearth + PROJECT_PATH: /esarchive/scratch/vagudets/repos/auto-s2s/ +# If PROJECT_TYPE is none, no need to change +project_files: + # Where is PROJECT CONFIGURATION file location relative to project root path + FILE_PROJECT_CONF: '' + # Where is JOBS CONFIGURATION file location relative to project root path + FILE_JOBS_CONF: '' + # Default job scripts type in the project. type: STRING, default: bash, supported: 'bash', 'python' or 'r' + JOB_SCRIPTS_TYPE: '' +rerun: + # Is a rerun or not? [Default: Do set FALSE]. BOOLEAN: TRUE, FALSE + RERUN: FALSE + # If RERUN: TRUE then supply the list of chunks to rerun + # LIST: [ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ] + CHUNKLIST: '' diff --git a/autosubmit/conf_esarchive/jobs.conf b/autosubmit/conf_esarchive/jobs.conf deleted file mode 100644 index 88f3565c40e841b1062556a70c8b0b4bb808f918..0000000000000000000000000000000000000000 --- a/autosubmit/conf_esarchive/jobs.conf +++ /dev/null @@ -1,8 +0,0 @@ -[verification] -FILE = autosubmit/auto-verification.sh -RUNNING = chunk -WALLCLOCK = -NOTIFY_ON = -PLATFORM = nord3v2 -PROCESSORS = - diff --git a/autosubmit/conf_esarchive/jobs.yml b/autosubmit/conf_esarchive/jobs.yml new file mode 100644 index 0000000000000000000000000000000000000000..a3c8934bf70f92d15a8e644a6f34947afcc28847 --- /dev/null +++ b/autosubmit/conf_esarchive/jobs.yml @@ -0,0 +1,16 @@ +JOBS: + verification: + FILE: autosubmit/auto-verification.sh + RUNNING: chunk + WALLCLOCK: + NOTIFY_ON: + PLATFORM: nord3v2 + PROCESSORS: + scorecards: + FILE: autosubmit/auto-scorecards.sh + WALLCLOCK: 00:10 + PLATFORM: nord3v2 + NOTIFY_ON: + PROCESSORS: 1 + DEPENDENCIES: verification + diff --git a/autosubmit/conf_esarchive/platforms.conf b/autosubmit/conf_esarchive/platforms.conf deleted file mode 100644 index 0f6819d005b95014642909607f4a787d3d77a495..0000000000000000000000000000000000000000 --- a/autosubmit/conf_esarchive/platforms.conf +++ /dev/null @@ -1,11 +0,0 @@ -[nord3v2] -TYPE = slurm -HOST = nord4.bsc.es -PROJECT = bsc32 -ADD_PROJECT_TO_HOST = False -USER = -SCRATCH_DIR = /gpfs/scratch -PROCESSORS_PER_NODE = 16 -SERIAL_QUEUE = debug -QUEUE = bsc_es -CUSTOM_DIRECTIVES = ["#SBATCH --exclusive"] diff --git a/autosubmit/conf_esarchive/platforms.yml b/autosubmit/conf_esarchive/platforms.yml new file mode 100644 index 0000000000000000000000000000000000000000..78056d62973552432a9e7b55194b8c5e0ecac09a --- /dev/null +++ b/autosubmit/conf_esarchive/platforms.yml @@ -0,0 +1,11 @@ +## TODO: Change platform +Platforms: + nord3v2: + TYPE: slurm + HOST: nord4.bsc.es + USER: + PROJECT: bsc32 ## TO BE CHANGED + SCRATCH_DIR: /gpfs/scratch/ ## TO BE CHANGED + PROCESSORS_PER_NODE: 16 + SERIAL_QUEUE: debug + QUEUE: bsc_es diff --git a/autosubmit/conf_esarchive/proj.conf b/autosubmit/conf_esarchive/proj.conf deleted file mode 100644 index c57e37eb502678627d99f005c5fba4f3b107c42a..0000000000000000000000000000000000000000 --- a/autosubmit/conf_esarchive/proj.conf +++ /dev/null @@ -1,5 +0,0 @@ -[common] - -MODULES = "MODULES" -OUTDIR = -SCRIPT = diff --git a/autosubmit/conf_esarchive/proj.yml b/autosubmit/conf_esarchive/proj.yml new file mode 100644 index 0000000000000000000000000000000000000000..679cf63b1ced38fd833d28ea9acfa145a1e9bc4f --- /dev/null +++ b/autosubmit/conf_esarchive/proj.yml @@ -0,0 +1,4 @@ +common: + MODULES: "MODULES" + OUTDIR: + SCRIPT: diff --git a/autosubmit/conf_mars/autosubmit.yml b/autosubmit/conf_mars/autosubmit.yml index 0fd5d5c6aaf61945d131da77cda08d8d1fdd86cd..030081165150f1ac62dd897e11437ff893195f74 100644 --- a/autosubmit/conf_mars/autosubmit.yml +++ b/autosubmit/conf_mars/autosubmit.yml @@ -1,6 +1,6 @@ config: EXPID: - AUTOSUBMIT_VERSION: 4.0.0b0 + AUTOSUBMIT_VERSION: 4.0.73 MAXWAITINGJOBS: 16 # Default maximum number of jobs to be running at the same time at any platform # Default: 6 diff --git a/autosubmit/conf_mars/jobs.yml b/autosubmit/conf_mars/jobs.yml index 2d8e32affa207d3fe72bac27978ce44d100853df..273d3d6f45c7811062fb07a4ac5620daefd82f72 100644 --- a/autosubmit/conf_mars/jobs.yml +++ b/autosubmit/conf_mars/jobs.yml @@ -6,3 +6,11 @@ JOBS: NOTIFY_ON: PLATFORM: NORD3 PROCESSORS: + scorecards: + FILE: autosubmit/auto-scorecards.sh + WALLCLOCK: 00:10 + PLATFORM: NORD3 + NOTIFY_ON: + PROCESSORS: 1 + DEPENDENCIES: verification + diff --git a/conda_installation/environment-cerise-localgribR-ecmwf.yml b/conda_installation/environment-cerise-localgribR-ecmwf.yml new file mode 100644 index 0000000000000000000000000000000000000000..983347f0d2d91190e5165e86c66c3694cf28fcfb --- /dev/null +++ b/conda_installation/environment-cerise-localgribR-ecmwf.yml @@ -0,0 +1,331 @@ +name: condaCerise +channels: + - r + - conda-forge + - https://nexus.ecmwf.int/repository/conda-ecmwf + - https://nexus.ecmwf.int/repository/conda-forge +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - _r-mutex=1.0.1=anacondar_1 + - atk-1.0=2.38.0=hd4edc92_1 + - binutils_impl_linux-64=2.40=hf600244_0 + - blosc=1.21.5=h0f2a231_0 + - boost-cpp=1.78.0=h5adbc97_2 + - bwidget=1.9.14=ha770c72_1 + - bzip2=1.0.8=h7f98852_4 + - c-ares=1.19.1=hd590300_0 + - ca-certificates=2023.7.22=hbcca054_0 + - cairo=1.16.0=ha61ee94_1014 + - cfitsio=4.1.0=hd9d235c_0 + - curl=8.1.2=h409715c_0 + - eccodes=2.30.2=hc4464db_0 + - expat=2.5.0=hcb278e6_1 + - fftw=3.3.10=nompi_hc118613_108 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=hab24e00_0 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - freeglut=3.2.2=h9c3ff4c_1 + - freetype=2.12.1=h267a509_2 + - freexl=1.0.6=h166bdaf_1 + - fribidi=1.0.10=h36c2ea0_0 + - gcc_impl_linux-64=13.2.0=h338b0a0_2 + - gdal=3.5.2=py311hd39052d_7 + - gdk-pixbuf=2.42.8=hff1cb4f_1 + - geos=3.11.0=h27087fc_0 + - geotiff=1.7.1=ha76d385_4 + - gettext=0.21.1=h27087fc_0 + - gfortran_impl_linux-64=13.2.0=h76e1118_2 + - ghostscript=10.02.0=h59595ed_0 + - giflib=5.2.1=h0b41bf4_3 + - gmp=6.2.1=h58526e2_0 + - graphite2=1.3.13=h58526e2_1001 + - graphviz=6.0.2=h99bc08f_0 + - gsl=2.7=he838d99_0 + - gtk2=2.24.33=h90689f9_2 + - gts=0.7.6=h977cf35_4 + - gxx_impl_linux-64=13.2.0=h338b0a0_2 + - harfbuzz=6.0.0=h8e241bc_0 + - hdf4=4.2.15=h9772cbc_5 + - hdf5=1.12.2=nompi_h4df4325_101 + - icu=70.1=h27087fc_0 + - imagemagick=7.1.0_55=pl5321h0d24a18_0 + - jasper=2.0.33=h0ff4b12_1 + - jbig=2.1=h7f98852_2003 + - jpeg=9e=h0b41bf4_3 + - json-c=0.16=hc379101_0 + - kealib=1.4.15=ha7026e8_1 + - kernel-headers_linux-64=2.6.32=he073ed8_16 + - keyutils=1.6.1=h166bdaf_0 + - krb5=1.20.1=h81ceb04_0 + - lcms2=2.14=h6ed2654_0 + - ld_impl_linux-64=2.40=h41732ed_0 + - lerc=4.0.0=h27087fc_0 + - libaec=1.0.6=hcb278e6_1 + - libblas=3.9.0=18_linux64_openblas + - libcblas=3.9.0=18_linux64_openblas + - libcurl=8.1.2=h409715c_0 + - libdap4=3.20.6=hd7c4107_2 + - libdeflate=1.14=h166bdaf_0 + - libedit=3.1.20191231=he28a2e2_2 + - libev=4.33=h516909a_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.2=h7f98852_5 + - libgcc-devel_linux-64=13.2.0=ha9c7c90_2 + - libgcc-ng=13.2.0=h807b86a_2 + - libgd=2.3.3=h18fbbfe_3 + - libgdal=3.5.2=h27ae5c1_7 + - libgfortran-ng=13.2.0=h69a702a_2 + - libgfortran5=13.2.0=ha4646dd_2 + - libglib=2.78.0=hebfc3b9_0 + - libglu=9.0.0=he1b5a44_1001 + - libgomp=13.2.0=h807b86a_2 + - libiconv=1.17=h166bdaf_0 + - libkml=1.3.0=h37653c0_1015 + - liblapack=3.9.0=18_linux64_openblas + - libnetcdf=4.8.1=nompi_h261ec11_106 + - libnghttp2=1.52.0=h61bc06f_0 + - libnsl=2.0.0=hd590300_1 + - libopenblas=0.3.24=pthreads_h413a1c8_0 + - libpng=1.6.39=h753d276_0 + - libpq=14.5=hb675445_5 + - librsvg=2.54.4=h7abd40a_0 + - librttopo=1.1.0=hf730bdb_11 + - libsanitizer=13.2.0=h7e041cc_2 + - libspatialite=5.0.1=hfbd986c_21 + - libsqlite=3.43.0=h2797004_0 + - libssh2=1.11.0=h0841786_0 + - libstdcxx-devel_linux-64=13.2.0=ha9c7c90_2 + - libstdcxx-ng=13.2.0=h7e041cc_2 + - libtiff=4.4.0=h82bc61c_5 + - libtool=2.4.7=h27087fc_0 + - libuuid=2.38.1=h0b41bf4_0 + - libwebp=1.2.4=h522a892_0 + - libwebp-base=1.2.4=h166bdaf_0 + - libxcb=1.13=h7f98852_1004 + - libxml2=2.10.3=hca2bb57_4 + - libzip=1.10.1=h2629f0a_3 + - libzlib=1.2.13=hd590300_5 + - lz4-c=1.9.4=hcb278e6_0 + - make=4.3=hd18ef5c_1 + - ncurses=6.4=hcb278e6_0 + - nspr=4.35=h27087fc_0 + - nss=3.92=h1d7d5a4_0 + - numpy=1.26.0=py311h64a7726_0 + - openjpeg=2.5.0=h7d73246_1 + - openssl=3.1.3=hd590300_0 + - pandoc=3.1.3=h32600fe_0 + - pango=1.50.14=hd33c08f_0 + - pcre=8.45=h9c3ff4c_0 + - pcre2=10.40=hc3806b6_0 + - perl=5.32.1=4_hd590300_perl5 + - phantomjs=2.1.1=ha770c72_1 + - pip=23.2.1=pyhd8ed1ab_0 + - pixman=0.42.2=h59595ed_0 + - pkg-config=0.29.2=h36c2ea0_1008 + - poppler=22.10.0=h92391eb_0 + - poppler-data=0.4.12=hd8ed1ab_0 + - postgresql=14.5=h3248436_5 + - proj=9.1.0=h93bde94_0 + - pthread-stubs=0.4=h36c2ea0_1001 + - python=3.11.5=hab00c5b_0_cpython + - python_abi=3.11=4_cp311 + - r-abind=1.4_5=r42hc72bb7e_1005 + - r-askpass=1.2.0=r42h57805ef_0 + - r-assertthat=0.2.1=r42hc72bb7e_4 + - r-backports=1.4.1=r42h57805ef_2 + - r-base=4.2.2=h6b4767f_2 + - r-base64enc=0.1_3=r42h57805ef_1006 + - r-bigmemory=4.6.1=r42ha503ecb_2 + - r-bigmemory.sri=0.1.6=r42hc72bb7e_1 + - r-brio=1.1.3=r42h57805ef_2 + - r-bslib=0.5.1=r42hc72bb7e_0 + - r-cachem=1.0.8=r42h57805ef_1 + - r-callr=3.7.3=r42hc72bb7e_1 + - r-class=7.3_22=r42h57805ef_1 + - r-classint=0.4_10=r42h61816a4_0 + - r-cli=3.6.1=r42ha503ecb_1 + - r-climdex.pcic=1.1_11=r42ha503ecb_2 + - r-climprojdiags=0.3.2=r42hc72bb7e_1 + - r-clock=0.7.0=r42ha503ecb_1 + - r-codetools=0.2_19=r42hc72bb7e_1 + - r-colorspace=2.1_0=r42h57805ef_1 + - r-commonmark=1.9.0=r42h57805ef_1 + - r-configr=0.3.5=r42hc72bb7e_1 + - r-cowplot=1.1.1=r42hc72bb7e_2 + - r-cpp11=0.4.6=r42hc72bb7e_0 + - r-crayon=1.5.2=r42hc72bb7e_2 + - r-curl=5.0.1=r42hf9611b0_0 + - r-dbi=1.1.3=r42hc72bb7e_2 + - r-desc=1.4.2=r42hc72bb7e_2 + - r-diffobj=0.3.5=r42h57805ef_2 + - r-digest=0.6.33=r42ha503ecb_0 + - r-docopt=0.7.1=r42hc72bb7e_3 + - r-doparallel=1.0.17=r42hc72bb7e_2 + - r-dotcall64=1.0_2=r42h61816a4_2 + - r-e1071=1.7_13=r42ha503ecb_1 + - r-easyncdf=0.1.2=r42hc72bb7e_1 + - r-easyverification=0.4.5=r42ha503ecb_0 + - r-ellipsis=0.3.2=r42h57805ef_2 + - r-evaluate=0.21=r42hc72bb7e_1 + - r-fansi=1.0.4=r42h57805ef_1 + - r-farver=2.1.1=r42ha503ecb_2 + - r-fastmap=1.1.1=r42ha503ecb_1 + - r-fields=15.2=r42h61816a4_0 + - r-fnn=1.1.3.2=r42ha503ecb_1 + - r-fontawesome=0.5.2=r42hc72bb7e_0 + - r-foreach=1.5.2=r42hc72bb7e_2 + - r-formattable=0.2.1=r42ha770c72_2 + - r-fs=1.6.3=r42ha503ecb_0 + - r-future=1.33.0=r42hc72bb7e_0 + - r-generics=0.1.3=r42hc72bb7e_2 + - r-geomap=2.5_0=r42h57805ef_2 + - r-geomapdata=2.0_2=r42hc72bb7e_0 + - r-ggplot2=3.4.3=r42hc72bb7e_0 + - r-globals=0.16.2=r42hc72bb7e_1 + - r-glue=1.6.2=r42h57805ef_2 + - r-gridextra=2.3=r42hc72bb7e_1005 + - r-gtable=0.3.4=r42hc72bb7e_0 + - r-highr=0.10=r42hc72bb7e_1 + - r-htmltools=0.5.6=r42ha503ecb_0 + - r-htmlwidgets=1.6.2=r42hc72bb7e_1 + - r-httpuv=1.6.11=r42ha503ecb_1 + - r-httr=1.4.7=r42hc72bb7e_0 + - r-ini=0.3.1=r42hc72bb7e_1005 + - r-isoband=0.2.7=r42ha503ecb_2 + - r-iterators=1.0.14=r42hc72bb7e_2 + - r-jquerylib=0.1.4=r42hc72bb7e_2 + - r-jsonlite=1.8.7=r42h57805ef_0 + - r-kableextra=1.3.4=r42hc72bb7e_2 + - r-kernsmooth=2.23_22=r42h13b3f57_0 + - r-knitr=1.44=r42hc72bb7e_0 + - r-labeling=0.4.3=r42hc72bb7e_0 + - r-later=1.3.1=r42ha503ecb_1 + - r-lattice=0.21_8=r42h57805ef_1 + - r-lifecycle=1.0.3=r42hc72bb7e_2 + - r-listenv=0.9.0=r42hc72bb7e_1 + - r-log4r=0.4.3=r42h57805ef_1 + - r-lubridate=1.9.2=r42h57805ef_2 + - r-magick=2.7.3=r42h7525677_1 + - r-magrittr=2.0.3=r42h57805ef_2 + - r-mapproj=1.2.11=r42h57805ef_1 + - r-maps=3.4.1=r42h57805ef_2 + - r-mass=7.3_60=r42h57805ef_1 + - r-matrix=1.6_1.1=r42h316c678_0 + - r-mba=0.1_0=r42ha503ecb_1 + - r-memoise=2.0.1=r42hc72bb7e_2 + - r-memuse=4.2_3=r42h57805ef_1 + - r-mgcv=1.9_0=r42h316c678_0 + - r-mime=0.12=r42h57805ef_2 + - r-multiapply=2.1.4=r42hc72bb7e_1 + - r-munsell=0.5.0=r42hc72bb7e_1006 + - r-nbclust=3.0.1=r42hc72bb7e_2 + - r-ncdf4=1.21=r42h15f2bca_0 + - r-nlme=3.1_163=r42h61816a4_0 + - r-nnet=7.3_19=r42h57805ef_1 + - r-openssl=2.1.1=r42hb353fa6_0 + - r-parallelly=1.36.0=r42hc72bb7e_1 + - r-pbapply=1.7_2=r42hc72bb7e_0 + - r-pcict=0.5_4.4=r42h57805ef_1 + - r-pillar=1.9.0=r42hc72bb7e_1 + - r-pkgbuild=1.4.2=r42hc72bb7e_0 + - r-pkgconfig=2.0.3=r42hc72bb7e_3 + - r-pkgload=1.3.3=r42hc72bb7e_0 + - r-plyr=1.8.8=r42ha503ecb_1 + - r-praise=1.0.0=r42hc72bb7e_1007 + - r-prettyunits=1.2.0=r42hc72bb7e_0 + - r-processx=3.8.2=r42h57805ef_0 + - r-proj4=1.0_12=r42h4db2be8_0 + - r-promises=1.2.1=r42ha503ecb_0 + - r-proxy=0.4_27=r42h57805ef_2 + - r-ps=1.7.5=r42h57805ef_1 + - r-r6=2.5.1=r42hc72bb7e_2 + - r-rappdirs=0.3.3=r42h57805ef_2 + - r-rcolorbrewer=1.1_3=r42h6115d3f_0 + - r-rcpp=1.0.11=r42h7df8631_0 + - r-rcpparmadillo=0.12.6.4.0=r42h08d816e_0 + - r-rcpptoml=0.2.2=r42ha503ecb_1 + - r-rematch2=2.1.2=r42hc72bb7e_3 + - r-rlang=1.1.1=r42ha503ecb_1 + - r-rmarkdown=2.25=r42hc72bb7e_0 + - r-rnaturalearth=0.1.0=r42hc72bb7e_1 + - r-rpmg=2.2_7=r42hc72bb7e_0 + - r-rprojroot=2.0.3=r42hc72bb7e_1 + - r-rstudioapi=0.15.0=r42hc72bb7e_0 + - r-rvest=1.0.3=r42hc72bb7e_2 + - r-s2=1.1.4=r42h5eac2b3_1 + - r-s2dv=1.4.1=r42hc72bb7e_1 + - r-s2dverification=2.10.3=r42hc72bb7e_2 + - r-sass=0.4.7=r42ha503ecb_0 + - r-scales=1.2.1=r42hc72bb7e_2 + - r-selectr=0.4_2=r42hc72bb7e_3 + - r-sf=1.0_7=r42h25da31b_5 + - r-shiny=1.7.5=r42h785f33e_0 + - r-sourcetools=0.1.7_1=r42ha503ecb_1 + - r-sp=2.0_0=r42h57805ef_0 + - r-spam=2.9_1=r42hd9ac46e_2 + - r-specsverification=0.5_3=r42h7525677_2 + - r-splancs=2.01_44=r42h61816a4_0 + - r-startr=2.3.0=r42hc72bb7e_0 + - r-stringi=1.7.12=r42h1ae9187_0 + - r-stringr=1.5.0=r42h785f33e_1 + - r-svglite=2.1.1=r42h329214f_1 + - r-sys=3.4.2=r42h57805ef_1 + - r-systemfonts=1.0.4=r42haf97adc_2 + - r-testthat=3.1.10=r42ha503ecb_0 + - r-tibble=3.2.1=r42h57805ef_2 + - r-timechange=0.2.0=r42ha503ecb_1 + - r-tinytex=0.46=r42hc72bb7e_0 + - r-tzdb=0.4.0=r42ha503ecb_1 + - r-units=0.8_4=r42ha503ecb_0 + - r-utf8=1.2.3=r42h57805ef_1 + - r-uuid=1.1_1=r42h57805ef_0 + - r-vctrs=0.6.3=r42ha503ecb_0 + - r-viridis=0.6.4=r42hc72bb7e_0 + - r-viridislite=0.4.2=r42hc72bb7e_1 + - r-waldo=0.5.1=r42hc72bb7e_1 + - r-webshot=0.5.5=r42hc72bb7e_0 + - r-withr=2.5.1=r42hc72bb7e_0 + - r-wk=0.8.0=r42ha503ecb_0 + - r-xfun=0.40=r42ha503ecb_0 + - r-xml2=1.3.3=r42h044e5c7_2 + - r-xtable=1.8_4=r42hc72bb7e_5 + - r-yaml=2.3.5=r42h06615bd_1 + - readline=8.2=h8228510_1 + - sed=4.8=he412f7d_0 + - setuptools=68.2.2=pyhd8ed1ab_0 + - snappy=1.1.10=h9fff704_0 + - sqlite=3.43.0=h2c6b66d_0 + - sysroot_linux-64=2.12=he073ed8_16 + - tiledb=2.11.3=h3f4058f_1 + - tk=8.6.13=h2797004_0 + - tktable=2.10=h0c5db8f_5 + - tzcode=2023c=h0b41bf4_0 + - tzdata=2023c=h71feb2d_0 + - udunits2=2.2.28=h40f5838_2 + - wheel=0.41.2=pyhd8ed1ab_0 + - xerces-c=3.2.4=h55805fa_1 + - xorg-fixesproto=5.0=h7f98852_1002 + - xorg-inputproto=2.3.2=h7f98852_1002 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.0.10=h7f98852_0 + - xorg-libsm=1.2.3=hd9c2040_1000 + - xorg-libx11=1.8.4=h0b41bf4_0 + - xorg-libxau=1.0.11=hd590300_0 + - xorg-libxdmcp=1.1.3=h7f98852_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxfixes=5.0.3=h7f98852_1004 + - xorg-libxi=1.7.10=h7f98852_0 + - xorg-libxrender=0.9.10=h7f98852_1003 + - xorg-libxt=1.3.0=hd590300_0 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xz=5.2.6=h166bdaf_0 + - zlib=1.2.13=hd590300_5 + - zstd=1.5.5=hfc55251_0 +prefix: /perm/cyce/conda/envs/condaCerise diff --git a/conda_installation/environment-cerise-localgribr.yml b/conda_installation/environment-cerise-localgribr.yml new file mode 100644 index 0000000000000000000000000000000000000000..08c74f3d06e9e63c1914f21d7fdd53732dc6df01 --- /dev/null +++ b/conda_installation/environment-cerise-localgribr.yml @@ -0,0 +1,346 @@ +name: ceriseNoAS +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_kmp_llvm + - _r-mutex=1.0.0=anacondar_1 + - _sysroot_linux-64_curr_repodata_hack=3=haa98f57_10 + - atk-1.0=2.38.0=hd4edc92_1 + - aws-c-auth=0.7.0=hf8751d9_2 + - aws-c-cal=0.6.0=h93469e0_0 + - aws-c-common=0.8.23=hd590300_0 + - aws-c-compression=0.2.17=h862ab75_1 + - aws-c-event-stream=0.3.1=h9599702_1 + - aws-c-http=0.7.11=hbe98c3e_0 + - aws-c-io=0.13.28=h3870b5a_0 + - aws-c-mqtt=0.8.14=h2e270ba_2 + - aws-c-s3=0.3.13=heb0bb06_2 + - aws-c-sdkutils=0.1.11=h862ab75_1 + - aws-checksums=0.1.16=h862ab75_1 + - aws-crt-cpp=0.20.3=he9c0e7f_4 + - aws-sdk-cpp=1.10.57=hbc2ea52_17 + - binutils_impl_linux-64=2.40=hf600244_0 + - blas=2.116=openblas + - blas-devel=3.9.0=16_linux64_openblas + - blosc=1.21.4=h0f2a231_0 + - boost-cpp=1.78.0=h6582d0a_3 + - bwidget=1.9.11=1 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.0=h5eee18b_0 + - ca-certificates=2023.7.22=hbcca054_0 + - cairo=1.16.0=hb05425b_5 + - cfitsio=4.2.0=hd9d235c_0 + - curl=8.2.0=hca28451_0 + - eccodes=2.31.0=h73bf81c_0 + - expat=2.5.0=hcb278e6_1 + - fftw=3.3.10=nompi_hf0379b8_106 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=hab24e00_0 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - freeglut=3.2.2=hac7e632_2 + - freetype=2.12.1=h4a9f257_0 + - freexl=1.0.6=h166bdaf_1 + - fribidi=1.0.10=h7b6447c_0 + - gcc_impl_linux-64=13.1.0=hc4be1a9_0 + - gdk-pixbuf=2.42.10=h6b639ba_2 + - geos=3.11.2=hcb278e6_0 + - geotiff=1.7.1=h22adcc9_11 + - gettext=0.21.1=h27087fc_0 + - gfortran_impl_linux-64=13.1.0=hd511a9b_0 + - ghostscript=9.54.0=h27087fc_2 + - giflib=5.2.1=h0b41bf4_3 + - glib=2.76.4=hfc55251_0 + - glib-tools=2.76.4=hfc55251_0 + - graphite2=1.3.14=h295c915_1 + - graphviz=8.1.0=h28d9a01_0 + - gsl=2.7=he838d99_0 + - gtk2=2.24.33=h90689f9_2 + - gts=0.7.6=h977cf35_4 + - gxx_impl_linux-64=13.1.0=hc4be1a9_0 + - harfbuzz=7.3.0=hdb3a94d_0 + - hdf4=4.2.15=h501b40f_6 + - hdf5=1.14.1=nompi_h4f84152_100 + - icu=72.1=hcb278e6_0 + - imagemagick=7.1.1_14=pl5321hf48ede7_0 + - jasper=4.0.0=h32699f2_1 + - jbig=2.1=h7f98852_2003 + - json-c=0.16=hc379101_0 + - kealib=1.5.1=h3e6883b_4 + - kernel-headers_linux-64=2.6.32=he073ed8_16 + - keyutils=1.6.1=h166bdaf_0 + - krb5=1.21.1=h659d440_0 + - lcms2=2.15=haa2dc70_1 + - ld_impl_linux-64=2.40=h41732ed_0 + - lerc=4.0.0=h27087fc_0 + - libaec=1.0.6=hcb278e6_1 + - libblas=3.9.0=16_linux64_openblas + - libcblas=3.9.0=16_linux64_openblas + - libcurl=8.2.0=hca28451_0 + - libdeflate=1.18=h0b41bf4_0 + - libedit=3.1.20221030=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-devel_linux-64=13.1.0=he3cc6c4_0 + - libgcc-ng=13.1.0=he5830b7_0 + - libgd=2.3.3=hfa28ad5_6 + - libgdal=3.6.4=hd54c316_5 + - libgfortran-ng=13.1.0=h69a702a_0 + - libgfortran5=13.1.0=h15d22d2_0 + - libglib=2.76.4=hebfc3b9_0 + - libglu=9.0.0=hac7e632_1002 + - libgomp=13.1.0=he5830b7_0 + - libiconv=1.17=h166bdaf_0 + - libjpeg-turbo=2.1.5.1=h0b41bf4_0 + - libkml=1.3.0=h37653c0_1015 + - liblapack=3.9.0=16_linux64_openblas + - liblapacke=3.9.0=16_linux64_openblas + - libnetcdf=4.9.2=nompi_h7e745eb_109 + - libnghttp2=1.52.0=h2d74bed_1 + - libnsl=2.0.0=h7f98852_0 + - libopenblas=0.3.21=pthreads_h78a6416_3 + - libpng=1.6.39=h5eee18b_0 + - libpq=15.3=hfc447b1_2 + - librsvg=2.56.1=h98fae49_0 + - librttopo=1.1.0=h0d5128d_13 + - libsanitizer=13.1.0=hfd8a6a1_0 + - libspatialite=5.0.1=hca56755_27 + - libsqlite=3.42.0=h2797004_0 + - libssh2=1.11.0=h0841786_0 + - libstdcxx-devel_linux-64=13.1.0=he3cc6c4_0 + - libstdcxx-ng=13.1.0=hfd8a6a1_0 + - libtiff=4.5.1=h8b53f26_0 + - libtool=2.4.7=h27087fc_0 + - libuuid=2.38.1=h0b41bf4_0 + - libwebp=1.3.1=hbf2b3c1_0 + - libwebp-base=1.3.1=hd590300_0 + - libxcb=1.15=h7f8727e_0 + - libxml2=2.11.4=h0d562d8_0 + - libzip=1.9.2=hc929e4a_1 + - libzlib=1.2.13=hd590300_5 + - llvm-openmp=16.0.6=h4dfa4b3_0 + - lz4-c=1.9.4=h6a678d5_0 + - make=4.2.1=h1bed415_1 + - ncurses=6.4=h6a678d5_0 + - nspr=4.35=h27087fc_0 + - nss=3.89=he45b914_0 + - openblas=0.3.21=pthreads_h320a7e8_3 + - openjpeg=2.5.0=hfec8fc6_2 + - openssl=3.1.1=hd590300_1 + - pandoc=2.19.2=h32600fe_2 + - pango=1.50.14=heaa33ce_1 + - pcre=8.45=h295c915_0 + - pcre2=10.40=hc3806b6_0 + - perl=5.32.1=4_hd590300_perl5 + - phantomjs=2.1.1=1 + - pip=23.1.2=py311h06a4308_0 + - pixman=0.40.0=h7f8727e_1 + - pkg-config=0.29.2=h36c2ea0_1008 + - poppler=23.05.0=hd18248d_1 + - poppler-data=0.4.12=hd8ed1ab_0 + - postgresql=15.3=h8972f4a_2 + - proj=9.2.1=ha643af7_0 + - python=3.11.4=hab00c5b_0_cpython + - r-abind=1.4_5=r42hc72bb7e_1005 + - r-askpass=1.1=r42h57805ef_4 + - r-assertthat=0.2.1=r42hc72bb7e_4 + - r-backports=1.4.1=r42h57805ef_2 + - r-base=4.2.3=hb1b9b86_6 + - r-base64enc=0.1_3=r42h57805ef_1006 + - r-bigmemory=4.6.1=r42ha503ecb_2 + - r-bigmemory.sri=0.1.6=r42hc72bb7e_1 + - r-brio=1.1.3=r42h57805ef_2 + - r-bslib=0.3.1=r42h6115d3f_0 + - r-cachem=1.0.8=r42h57805ef_1 + - r-callr=3.7.3=r42hc72bb7e_1 + - r-class=7.3_22=r42h57805ef_1 + - r-classint=0.4_9=r42h61816a4_1 + - r-cli=3.6.1=r42ha503ecb_1 + - r-climdex.pcic=1.1_11=r42ha503ecb_2 + - r-climprojdiags=0.3.2=r42hc72bb7e_1 + - r-clock=0.7.0=r42ha503ecb_1 + - r-codetools=0.2_19=r42hc72bb7e_1 + - r-colorspace=2.1_0=r42h57805ef_1 + - r-commonmark=1.9.0=r42h57805ef_1 + - r-configr=0.3.5=r42hc72bb7e_1 + - r-cowplot=1.1.1=r42hc72bb7e_2 + - r-cpp11=0.4.5=r42hc72bb7e_0 + - r-crayon=1.5.2=r42hc72bb7e_2 + - r-curl=5.0.1=r42hf9611b0_0 + - r-dbi=1.1.3=r42hc72bb7e_2 + - r-desc=1.4.2=r42hc72bb7e_2 + - r-diffobj=0.3.5=r42h57805ef_2 + - r-digest=0.6.33=r42ha503ecb_0 + - r-docopt=0.7.1=r42hc72bb7e_3 + - r-doparallel=1.0.17=r42hc72bb7e_2 + - r-dotcall64=1.0_2=r42h61816a4_2 + - r-e1071=1.7_13=r42ha503ecb_1 + - r-easyncdf=0.1.2=r42hc72bb7e_1 + - r-easyverification=0.4.4=r42ha503ecb_3 + - r-ellipsis=0.3.2=r42h57805ef_2 + - r-evaluate=0.21=r42hc72bb7e_1 + - r-fansi=1.0.4=r42h57805ef_1 + - r-farver=2.1.1=r42ha503ecb_2 + - r-fastmap=1.1.1=r42ha503ecb_1 + - r-fields=14.1=r42h61816a4_2 + - r-fnn=1.1.3.2=r42ha503ecb_1 + - r-fontawesome=0.5.1=r42hc72bb7e_1 + - r-foreach=1.5.2=r42hc72bb7e_2 + - r-formattable=0.2.1=r42ha770c72_2 + - r-fs=1.6.3=r42ha503ecb_0 + - r-future=1.33.0=r42hc72bb7e_0 + - r-generics=0.1.3=r42hc72bb7e_2 + - r-geomap=2.5_0=r42h57805ef_2 + - r-geomapdata=2.0_0=r42hc72bb7e_2 + - r-ggplot2=3.4.2=r42hc72bb7e_1 + - r-globals=0.16.2=r42hc72bb7e_1 + - r-glue=1.6.2=r42h57805ef_2 + - r-gribr=1.2.6=r42hd590300_0 + - r-gridextra=2.3=r42hc72bb7e_1005 + - r-gtable=0.3.3=r42hc72bb7e_1 + - r-highr=0.10=r42hc72bb7e_1 + - r-htmltools=0.5.5=r42ha503ecb_1 + - r-htmlwidgets=1.6.2=r42hc72bb7e_1 + - r-httpuv=1.6.11=r42ha503ecb_1 + - r-httr=1.4.6=r42hc72bb7e_1 + - r-ini=0.3.1=r42hc72bb7e_1005 + - r-isoband=0.2.7=r42ha503ecb_2 + - r-iterators=1.0.14=r42hc72bb7e_2 + - r-jquerylib=0.1.4=r42hc72bb7e_2 + - r-jsonlite=1.8.7=r42h57805ef_0 + - r-kableextra=1.3.4=r42hc72bb7e_2 + - r-kernsmooth=2.23_22=r42h13b3f57_0 + - r-knitr=1.43=r42hc72bb7e_1 + - r-labeling=0.4.2=r42hc72bb7e_3 + - r-later=1.3.1=r42ha503ecb_1 + - r-lattice=0.21_8=r42h57805ef_1 + - r-lifecycle=1.0.3=r42hc72bb7e_2 + - r-listenv=0.9.0=r42hc72bb7e_1 + - r-lobstr=1.1.2=r42ha503ecb_3 + - r-log4r=0.4.3=r42h57805ef_1 + - r-lubridate=1.8.0=r42h884c59f_0 + - r-magick=2.7.4=r42ha503ecb_1 + - r-magrittr=2.0.3=r42h57805ef_2 + - r-mapproj=1.2.11=r42h57805ef_1 + - r-maps=3.4.1=r42h57805ef_2 + - r-mass=7.3_60=r42h57805ef_1 + - r-matrix=1.6_0=r42h316c678_0 + - r-mba=0.1_0=r42ha503ecb_1 + - r-memuse=4.2_3=r42h57805ef_1 + - r-mgcv=1.9_0=r42h316c678_0 + - r-mime=0.12=r42h57805ef_2 + - r-multiapply=2.1.4=r42hc72bb7e_1 + - r-munsell=0.5.0=r42hc72bb7e_1006 + - r-nbclust=3.0.1=r42hc72bb7e_2 + - r-ncdf4=1.21=r42hdc3962f_5 + - r-nlme=3.1_162=r42h61816a4_1 + - r-nnet=7.3_19=r42h57805ef_1 + - r-openssl=2.0.6=r42hb353fa6_1 + - r-parallelly=1.36.0=r42hc72bb7e_1 + - r-pbapply=1.7_2=r42hc72bb7e_0 + - r-pcict=0.5_4.4=r42h57805ef_1 + - r-pillar=1.9.0=r42hc72bb7e_1 + - r-pkgconfig=2.0.3=r42hc72bb7e_3 + - r-pkgload=1.3.2.1=r42hc72bb7e_0 + - r-plyr=1.8.8=r42ha503ecb_1 + - r-praise=1.0.0=r42hc72bb7e_1007 + - r-prettyunits=1.1.1=r42hc72bb7e_3 + - r-processx=3.8.2=r42h57805ef_0 + - r-proj=0.4.0=r42h57805ef_2 + - r-proj4=1.0_12=r42he1907f6_4 + - r-promises=1.2.0.1=r42ha503ecb_2 + - r-proxy=0.4_27=r42h57805ef_2 + - r-pryr=0.1.6=r42ha503ecb_1 + - r-ps=1.7.5=r42h57805ef_1 + - r-r6=2.5.1=r42hc72bb7e_2 + - r-rappdirs=0.3.3=r42h57805ef_2 + - r-rcolorbrewer=1.1_3=r42h785f33e_2 + - r-rcpp=1.0.11=r42h7df8631_0 + - r-rcpparmadillo=0.12.4.1.0=r42h08d816e_0 + - r-rcpptoml=0.2.2=r42ha503ecb_1 + - r-rematch2=2.1.2=r42hc72bb7e_3 + - r-rlang=1.1.1=r42ha503ecb_1 + - r-rmarkdown=2.14=r42h6115d3f_0 + - r-rnaturalearth=0.3.3=r42hc72bb7e_1 + - r-rpmg=2.2_3=r42hc72bb7e_3 + - r-rprojroot=2.0.3=r42hc72bb7e_1 + - r-rstudioapi=0.15.0=r42hc72bb7e_0 + - r-rvest=1.0.3=r42hc72bb7e_2 + - r-s2=1.1.4=r42h5eac2b3_1 + - r-s2dv=1.4.1=r42hc72bb7e_1 + - r-s2dverification=2.10.3=r42hc72bb7e_2 + - r-sass=0.4.7=r42ha503ecb_0 + - r-scales=1.2.1=r42hc72bb7e_2 + - r-selectr=0.4_2=r42hc72bb7e_3 + - r-sf=1.0_7=r42h6a678d5_2 + - r-shiny=1.7.4.1=r42h785f33e_0 + - r-sourcetools=0.1.7_1=r42ha503ecb_1 + - r-sp=1.6_1=r42h57805ef_1 + - r-spam=2.9_1=r42hd9ac46e_2 + - r-specsverification=0.5_3=r42h7525677_2 + - r-splancs=2.01_43=r42h61816a4_2 + - r-startr=2.2.3=r42hc72bb7e_1 + - r-stringi=1.7.12=r42hc0c3e09_2 + - r-stringr=1.5.0=r42h785f33e_1 + - r-svglite=2.1.1=r42h329214f_1 + - r-sys=3.4.2=r42h57805ef_1 + - r-systemfonts=1.0.4=r42haf97adc_2 + - r-terra=1.5_21=r42h884c59f_2 + - r-testthat=3.1.10=r42ha503ecb_0 + - r-tibble=3.2.1=r42h57805ef_2 + - r-tinytex=0.45=r42hc72bb7e_1 + - r-tzdb=0.4.0=r42ha503ecb_1 + - r-units=0.8_2=r42ha503ecb_1 + - r-utf8=1.2.3=r42h57805ef_1 + - r-uuid=1.1_0=r42h57805ef_2 + - r-vctrs=0.6.3=r42ha503ecb_0 + - r-viridis=0.6.4=r42hc72bb7e_0 + - r-viridislite=0.4.2=r42hc72bb7e_1 + - r-waldo=0.5.1=r42hc72bb7e_1 + - r-webshot=0.5.5=r42hc72bb7e_0 + - r-withr=2.5.0=r42hc72bb7e_2 + - r-wk=0.7.3=r42ha503ecb_1 + - r-xfun=0.39=r42ha503ecb_1 + - r-xml2=1.3.5=r42h1ad5fc0_0 + - r-xtable=1.8_4=r42hc72bb7e_5 + - r-yaml=2.3.5=r42h76d94ec_0 + - readline=8.2=h5eee18b_0 + - s2n=1.3.46=h06160fa_0 + - sed=4.8=he412f7d_0 + - setuptools=67.8.0=py311h06a4308_0 + - snappy=1.1.10=h9fff704_0 + - sqlite=3.41.2=h5eee18b_0 + - sysroot_linux-64=2.12=he073ed8_16 + - tiledb=2.13.2=hd532e3d_0 + - tk=8.6.12=h1ccaba5_0 + - tktable=2.10=h14c3975_0 + - tzcode=2023c=h0b41bf4_0 + - tzdata=2023c=h04d1e81_0 + - udunits2=2.2.28=hc3e0081_0 + - wheel=0.38.4=py311h06a4308_0 + - xerces-c=3.2.4=h8d71039_2 + - xorg-fixesproto=5.0=h7f98852_1002 + - xorg-inputproto=2.3.2=h7f98852_1002 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.6=h8ee46fc_0 + - xorg-libxau=1.0.11=hd590300_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxfixes=5.0.3=h7f98852_1004 + - xorg-libxi=1.7.10=h7f98852_0 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-libxt=1.3.0=hd590300_1 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xz=5.4.2=h5eee18b_0 + - zlib=1.2.13=hd590300_5 + - zstd=1.5.5=hc292b87_0 +prefix: /home/Earth/pbretonn/miniconda3/envs/ceriseNoAS diff --git a/conda_installation/environment-cerise.yml b/conda_installation/environment-cerise.yml new file mode 100644 index 0000000000000000000000000000000000000000..9be18a853f2a0d12ed0fa18e6a99118b88086eca --- /dev/null +++ b/conda_installation/environment-cerise.yml @@ -0,0 +1,399 @@ +name: condaCerise +channels: + - r + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - _r-mutex=1.0.0=anacondar_1 + - alsa-lib=1.2.8=h166bdaf_0 + - atk-1.0=2.36.0=ha1a6a79_0 + - attr=2.5.1=h166bdaf_1 + - binutils_impl_linux-64=2.40=hf600244_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotlipy=0.7.0=py37h27cfd23_1003 + - bwidget=1.9.11=1 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.0=h5eee18b_0 + - ca-certificates=2023.5.7=hbcca054_0 + - cairo=1.16.0=ha61ee94_1014 + - cdo=2.1.1=h4f55462_2 + - certifi=2023.5.7=pyhd8ed1ab_0 + - cffi=1.15.1=py37h5eee18b_3 + - conda=4.14.0=py37h89c1867_0 + - conda-package-handling=2.0.2=py37h06a4308_0 + - conda-package-streaming=0.7.0=py37h06a4308_0 + - cov-core=1.15.0=py_1 + - curl=7.88.1=hdc1c0ab_1 + - cycler=0.11.0=pyhd3eb1b0_0 + - dbus=1.13.18=hb2f20db_0 + - eccodes=2.29.0=h7986f14_1 + - expat=2.5.0=h27087fc_0 + - fftw=3.3.10=nompi_hf0379b8_106 + - findlibs=0.0.2=pyhd8ed1ab_0 + - flit-core=3.6.0=pyhd3eb1b0_0 + - font-ttf-dejavu-sans-mono=2.37=hd3eb1b0_0 + - font-ttf-inconsolata=2.001=hcb22688_0 + - font-ttf-source-code-pro=2.030=hd3eb1b0_0 + - font-ttf-ubuntu=0.83=h8b1ccd4_0 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-anaconda=1=h8fa9717_0 + - fonts-conda-ecosystem=1=hd3eb1b0_0 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freeglut=3.2.2=h9c3ff4c_1 + - freetype=2.12.1=h4a9f257_0 + - fribidi=1.0.10=h7b6447c_0 + - gcc_impl_linux-64=12.2.0=hcc96c02_19 + - gdk-pixbuf=2.42.10=h5eee18b_0 + - gettext=0.21.1=h27087fc_0 + - gfortran_impl_linux-64=12.2.0=h55be85b_19 + - giflib=5.2.1=h5eee18b_3 + - glib=2.74.1=h6239696_1 + - glib-tools=2.74.1=h6239696_1 + - gobject-introspection=1.72.0=py37hbb6d50b_1 + - graphite2=1.3.14=h295c915_1 + - graphviz=7.1.0=h2e5815a_0 + - gsl=2.7.1=h6e86dc7_1 + - gst-plugins-base=1.22.0=h4243ec0_2 + - gstreamer=1.22.0=h25f0c4b_2 + - gstreamer-orc=0.4.33=h166bdaf_0 + - gtk2=2.24.33=h73c1081_2 + - gts=0.7.6=hb67d8dd_3 + - gxx_impl_linux-64=12.2.0=hcc96c02_19 + - harfbuzz=6.0.0=h8e241bc_0 + - hdf4=4.2.15=h9772cbc_5 + - hdf5=1.12.2=nompi_h4df4325_101 + - icu=70.1=h27087fc_0 + - idna=3.4=py37h06a4308_0 + - jack=1.9.22=h11f4161_0 + - jasper=4.0.0=h0ff4b12_0 + - jpeg=9e=h5eee18b_1 + - kernel-headers_linux-64=2.6.32=he073ed8_15 + - keyutils=1.6.1=h166bdaf_0 + - kiwisolver=1.4.4=py37h6a678d5_0 + - krb5=1.20.1=h81ceb04_0 + - lame=3.100=h7b6447c_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.40=h41732ed_0 + - lerc=3.0=h295c915_0 + - libaec=1.0.6=hcb278e6_1 + - libblas=3.9.0=16_linux64_openblas + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libcap=2.66=ha37c62d_0 + - libcblas=3.9.0=16_linux64_openblas + - libclang=15.0.7=default_had23c3d_1 + - libclang13=15.0.7=default_h3e3d535_1 + - libcups=2.3.3=h36d4200_3 + - libcurl=7.88.1=hdc1c0ab_1 + - libdb=6.2.32=h6a678d5_1 + - libdeflate=1.17=h5eee18b_0 + - libedit=3.1.20221030=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libevent=2.1.10=h28343ad_4 + - libffi=3.4.4=h6a678d5_0 + - libflac=1.4.2=h27087fc_0 + - libgcc-devel_linux-64=12.2.0=h3b97bd3_19 + - libgcc-ng=12.2.0=h65d4601_19 + - libgcrypt=1.10.1=h166bdaf_0 + - libgd=2.3.3=h695aa2c_1 + - libgfortran-ng=12.2.0=h69a702a_19 + - libgfortran5=12.2.0=h337968e_19 + - libglib=2.74.1=h606061b_1 + - libglu=9.0.0=hf484d3e_1 + - libgomp=12.2.0=h65d4601_19 + - libgpg-error=1.46=h620e276_0 + - libiconv=1.17=h166bdaf_0 + - liblapack=3.9.0=16_linux64_openblas + - libllvm15=15.0.7=hadd5161_1 + - libnetcdf=4.9.1=nompi_h34a3ff0_101 + - libnghttp2=1.52.0=h61bc06f_0 + - libnsl=2.0.0=h5eee18b_0 + - libogg=1.3.5=h27cfd23_1 + - libopenblas=0.3.21=h043d6bf_0 + - libopus=1.3.1=h7b6447c_0 + - libpng=1.6.39=h5eee18b_0 + - libpq=15.2=hb675445_0 + - librsvg=2.54.4=h7abd40a_0 + - libsanitizer=12.2.0=h46fd767_19 + - libsndfile=1.2.0=hb75c966_0 + - libsodium=1.0.18=h7b6447c_0 + - libsqlite=3.40.0=h753d276_0 + - libssh2=1.10.0=hf14f497_3 + - libstdcxx-devel_linux-64=12.2.0=h3b97bd3_19 + - libstdcxx-ng=12.2.0=h46fd767_19 + - libsystemd0=252=h2a991cd_0 + - libtiff=4.5.0=h6a678d5_2 + - libtool=2.4.7=h27087fc_0 + - libudev1=253=h0b41bf4_0 + - libuuid=2.32.1=h7f98852_1000 + - libvorbis=1.3.7=h7b6447c_0 + - libwebp=1.2.4=h11a3e52_1 + - libwebp-base=1.2.4=h5eee18b_1 + - libxcb=1.13=h1bed415_1 + - libxkbcommon=1.5.0=h79f4944_1 + - libxml2=2.10.3=hca2bb57_4 + - libzip=1.9.2=hc929e4a_1 + - libzlib=1.2.13=h166bdaf_4 + - lz4-c=1.9.4=h6a678d5_0 + - magics=4.13.0=h37abd2f_1 + - magics-python=1.5.8=pyhd8ed1ab_1 + - make=4.2.1=h1bed415_1 + - matplotlib=3.5.3=py37h89c1867_2 + - matplotlib-base=3.5.3=py37hf590b9c_0 + - mock=5.0.1=pyhd8ed1ab_0 + - mpg123=1.31.3=hcb278e6_0 + - munkres=1.1.4=py_0 + - mysql-common=8.0.32=ha901b37_0 + - mysql-libs=8.0.32=hd7da12d_0 + - ncurses=6.4=h6a678d5_0 + - ninja=1.10.2=h06a4308_5 + - ninja-base=1.10.2=hd09550d_5 + - nose=1.3.7=py_1006 + - nose-cov=1.6=py_1 + - nspr=4.35=h6a678d5_0 + - nss=3.89=he45b914_0 + - numpy=1.21.6=py37h976b520_0 + - openssl=3.1.1=hd590300_1 + - packaging=22.0=py37h06a4308_0 + - pandoc=2.12=h06a4308_3 + - pango=1.50.14=hd33c08f_0 + - paramiko=3.1.0=pyhd8ed1ab_0 + - pcre2=10.40=hc3806b6_0 + - phantomjs=2.1.1=1 + - pillow=9.4.0=py37h6a678d5_0 + - pip=23.0.1=pyhd8ed1ab_0 + - pixman=0.40.0=h7f8727e_1 + - ply=3.11=py37_0 + - proj=9.1.1=h8ffa02c_2 + - pulseaudio=16.1=ha8d29e2_1 + - pycosat=0.6.4=py37h5eee18b_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pynacl=1.5.0=py37h5eee18b_0 + - pyopenssl=23.0.0=py37h06a4308_0 + - pyparsing=3.0.9=pyhd8ed1ab_0 + - pyqt=5.15.7=py37hf30b843_1 + - pyqt5-sip=12.11.0=py37hd23a5d3_1 + - pysocks=1.7.1=py37_1 + - python=3.7.12=hf930737_100_cpython + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python_abi=3.7=3_cp37m + - qt-main=5.15.8=h5d23da1_6 + - r-abind=1.4_5=r42hc72bb7e_1004 + - r-askpass=1.1=r42h06615bd_3 + - r-assertthat=0.2.1=r42hc72bb7e_3 + - r-base=4.2.3=ha7d60f8_0 + - r-base64enc=0.1_3=r42h06615bd_1005 + - r-bigmemory=4.6.1=r42h7525677_1 + - r-bigmemory.sri=0.1.6=r42hc72bb7e_0 + - r-brio=1.1.3=r42h76d94ec_0 + - r-bslib=0.4.2=r42hc72bb7e_0 + - r-cachem=1.0.7=r42h133d619_0 + - r-callr=3.7.0=r42h6115d3f_0 + - r-cli=3.6.0=r42h38f115c_0 + - r-climdex.pcic=1.1_11=r42h7525677_1 + - r-climprojdiags=0.3.0=r42hc72bb7e_0 + - r-clock=0.6.1=r42h7525677_1 + - r-codetools=0.2_19=r42hc72bb7e_0 + - r-colorspace=2.1_0=r42h133d619_0 + - r-commonmark=1.9.0=r42h133d619_0 + - r-configr=0.3.5=r42hc72bb7e_0 + - r-cpp11=0.4.3=r42hc72bb7e_0 + - r-crayon=1.5.2=r42hc72bb7e_1 + - r-curl=4.3.3=r42h06615bd_1 + - r-desc=1.4.1=r42h6115d3f_0 + - r-diffobj=0.3.5=r42h76d94ec_0 + - r-digest=0.6.31=r42h38f115c_0 + - r-doparallel=1.0.17=r42hc72bb7e_1 + - r-dotcall64=1.0_2=r42hac0b197_1 + - r-easyncdf=0.1.1=r42hc72bb7e_1 + - r-easyverification=0.4.4=r42h7525677_2 + - r-ellipsis=0.3.2=r42h06615bd_1 + - r-evaluate=0.15=r42h6115d3f_0 + - r-fansi=1.0.4=r42h133d619_0 + - r-farver=2.1.1=r42h7525677_1 + - r-fastmap=1.1.1=r42h38f115c_0 + - r-fields=14.1=r42hac0b197_1 + - r-fontawesome=0.5.1=r42hc72bb7e_0 + - r-foreach=1.5.2=r42hc72bb7e_1 + - r-formattable=0.2.1=r42ha770c72_1 + - r-fs=1.6.1=r42h38f115c_0 + - r-future=1.32.0=r42hc72bb7e_0 + - r-generics=0.1.2=r42h142f84f_0 + - r-geomap=2.5_0=r42h06615bd_1 + - r-geomapdata=2.0_0=r42hc72bb7e_1 + - r-ggplot2=3.4.1=r42hc72bb7e_0 + - r-globals=0.16.2=r42hc72bb7e_0 + - r-glue=1.6.2=r42h06615bd_1 + - r-gribr=1.2.5=r42hd590300_3 + - r-gridextra=2.3=r42hc72bb7e_1004 + - r-gtable=0.3.3=r42hc72bb7e_0 + - r-highr=0.10=r42hc72bb7e_0 + - r-htmltools=0.5.5=r42h38f115c_0 + - r-htmlwidgets=1.6.2=r42hc72bb7e_0 + - r-httpuv=1.6.9=r42h38f115c_0 + - r-httr=1.4.6=r42hc72bb7e_0 + - r-ini=0.3.1=r42hc72bb7e_1004 + - r-isoband=0.2.7=r42h38f115c_1 + - r-iterators=1.0.14=r42hc72bb7e_1 + - r-jquerylib=0.1.4=r42hc72bb7e_1 + - r-jsonlite=1.8.0=r42h76d94ec_0 + - r-kableextra=1.3.4=r42hc72bb7e_1 + - r-knitr=1.43=r42hc72bb7e_0 + - r-labeling=0.4.2=r42hc72bb7e_2 + - r-later=1.3.0=r42h7525677_1 + - r-lattice=0.20_45=r42h06615bd_1 + - r-lifecycle=1.0.3=r42hc72bb7e_1 + - r-listenv=0.9.0=r42hc72bb7e_0 + - r-lobstr=1.1.2=r42h38f115c_2 + - r-log4r=0.4.3=r42h06615bd_0 + - r-lubridate=1.8.0=r42h884c59f_0 + - r-magrittr=2.0.3=r42h06615bd_1 + - r-mapproj=1.2.11=r42h133d619_0 + - r-maps=3.4.1=r42h06615bd_1 + - r-mass=7.3_58.3=r42h133d619_0 + - r-matrix=1.5_3=r42h5f7b363_0 + - r-mba=0.1_0=r42h7525677_0 + - r-memoise=2.0.1=r42hc72bb7e_1 + - r-mgcv=1.8_42=r42he1ae0d6_0 + - r-mime=0.12=r42h06615bd_1 + - r-multiapply=2.1.3=r42hc72bb7e_1 + - r-munsell=0.5.0=r42hc72bb7e_1005 + - r-nbclust=3.0.1=r42hc72bb7e_1 + - r-ncdf4=1.21=r42h93dc0a4_1 + - r-nlme=3.1_162=r42hac0b197_0 + - r-openssl=2.0.6=r42habfbb5e_0 + - r-parallelly=1.34.0=r42hc72bb7e_0 + - r-pbapply=1.7_0=r42hc72bb7e_0 + - r-pcict=0.5_4.4=r42h133d619_0 + - r-pillar=1.8.1=r42hc72bb7e_1 + - r-pkgconfig=2.0.3=r42hc72bb7e_2 + - r-pkgload=1.2.4=r42h142f84f_0 + - r-plyr=1.8.8=r42h7525677_0 + - r-praise=1.0.0=r42h6115d3f_4 + - r-prettyunits=1.1.1=r42hc72bb7e_2 + - r-processx=3.5.3=r42h76d94ec_0 + - r-proj4=1.0_12=r42h66e2efa_1 + - r-promises=1.2.0.1=r42h7525677_1 + - r-pryr=0.1.6=r42h38f115c_0 + - r-ps=1.7.0=r42h76d94ec_0 + - r-r6=2.5.1=r42hc72bb7e_1 + - r-rappdirs=0.3.3=r42h06615bd_1 + - r-rcolorbrewer=1.1_3=r42h785f33e_1 + - r-rcpp=1.0.10=r42h38f115c_0 + - r-rcpparmadillo=0.11.4.4.0=r42h358215d_0 + - r-rcpptoml=0.1.7=r42h7525677_2 + - r-rematch2=2.1.2=r42h142f84f_0 + - r-rlang=1.1.0=r42h38f115c_0 + - r-rmarkdown=2.21=r42hc72bb7e_0 + - r-rpmg=2.2_3=r42hc72bb7e_2 + - r-rprojroot=2.0.3=r42h6115d3f_0 + - r-rstudioapi=0.13=r42h6115d3f_0 + - r-rvest=1.0.3=r42hc72bb7e_1 + - r-s2dv=1.4.0=r42hc72bb7e_0 + - r-s2dverification=2.10.3=r42hc72bb7e_1 + - r-sass=0.4.5=r42h38f115c_0 + - r-scales=1.2.1=r42hc72bb7e_1 + - r-selectr=0.4_2=r42hc72bb7e_2 + - r-shiny=1.7.4=r42h785f33e_0 + - r-sourcetools=0.1.7_1=r42h38f115c_0 + - r-sp=1.6_0=r42h133d619_0 + - r-spam=2.9_1=r42hb20cf53_1 + - r-specsverification=0.5_3=r42h7525677_2 + - r-splancs=2.01_43=r42h8da6f51_1 + - r-startr=2.2.1=r42hc72bb7e_0 + - r-stringi=1.7.12=r42h1ae9187_0 + - r-stringr=1.5.0=r42h785f33e_0 + - r-svglite=2.1.1=r42he8f5e61_0 + - r-sys=3.4.2=r42h57805ef_0 + - r-systemfonts=1.0.4=r42h0ff29ef_1 + - r-testthat=3.1.4=r42h884c59f_0 + - r-tibble=3.2.1=r42h133d619_1 + - r-tinytex=0.45=r42hc72bb7e_0 + - r-tzdb=0.3.0=r42h7525677_1 + - r-utf8=1.2.3=r42h133d619_0 + - r-uuid=1.1_0=r42h06615bd_1 + - r-vctrs=0.6.0=r42h38f115c_0 + - r-viridis=0.6.2=r42hc72bb7e_1 + - r-viridislite=0.4.1=r42hc72bb7e_1 + - r-waldo=0.4.0=r42h6115d3f_0 + - r-webshot=0.5.4=r42hc72bb7e_1 + - r-withr=2.5.0=r42hc72bb7e_1 + - r-xfun=0.39=r42ha503ecb_0 + - r-xml2=1.3.3=r42h884c59f_0 + - r-xtable=1.8_4=r42hc72bb7e_4 + - r-yaml=2.3.7=r42h133d619_0 + - readline=8.2=h5eee18b_0 + - requests=2.28.1=py37h06a4308_0 + - ruamel_yaml=0.15.100=py37h27cfd23_0 + - sed=4.8=h7b6447c_0 + - simplejson=3.17.6=py37h7f8727e_0 + - sip=6.7.2=py37hd23a5d3_0 + - six=1.16.0=pyhd3eb1b0_1 + - sqlite=3.40.0=h4ff8645_0 + - sysroot_linux-64=2.12=he073ed8_15 + - tk=8.6.12=h1ccaba5_0 + - tktable=2.10=h14c3975_0 + - toml=0.10.2=pyhd3eb1b0_0 + - toolz=0.12.0=py37h06a4308_0 + - tornado=6.2=py37h5eee18b_0 + - typing_extensions=4.4.0=py37h06a4308_0 + - udunits2=2.2.25=hd30922c_1 + - urllib3=1.26.14=py37h06a4308_0 + - wheel=0.38.4=py37h06a4308_0 + - xcb-util=0.4.0=h166bdaf_0 + - xcb-util-image=0.4.0=h166bdaf_0 + - xcb-util-keysyms=0.4.0=h166bdaf_0 + - xcb-util-renderutil=0.3.9=h166bdaf_0 + - xcb-util-wm=0.4.1=h166bdaf_0 + - xkeyboard-config=2.38=h0b41bf4_0 + - xorg-fixesproto=5.0=h7f98852_1002 + - xorg-inputproto=2.3.2=h7f98852_1002 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.0.10=h7f98852_0 + - xorg-libsm=1.2.3=hd9c2040_1000 + - xorg-libx11=1.8.4=h0b41bf4_0 + - xorg-libxau=1.0.9=h7f98852_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxfixes=5.0.3=h7f98852_1004 + - xorg-libxi=1.7.10=h7f98852_0 + - xorg-libxrender=0.9.10=h7f98852_1003 + - xorg-libxt=1.2.1=h7f98852_2 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h27cfd23_1007 + - xz=5.2.10=h5eee18b_1 + - yaml=0.2.5=h7b6447c_0 + - zlib=1.2.13=h166bdaf_4 + - zstandard=0.19.0=py37h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - argparse==1.4.0 + - attrs==22.2.0 + - autosubmit==4.0.73 + - autosubmitconfigparser==1.0.21 + - bcrypt==4.0.1 + - bscearth-utils==0.5.2 + - charset-normalizer==3.1.0 + - configobj==5.0.8 + - coverage==7.2.2 + - cryptography==39.0.2 + - exceptiongroup==1.1.1 + - importlib-metadata==6.1.0 + - iniconfig==2.0.0 + - networkx==2.6.3 + - pluggy==1.0.0 + - portalocker==2.7.0 + - py3dotplus==1.1.0 + - pygments==2.14.0 + - pytest==7.2.2 + - pythondialog==3.5.3 + - ruamel-yaml==0.17.21 + - ruamel-yaml-clib==0.2.7 + - setuptools==67.6.0 + - xlib==0.21 + - zipp==3.15.0 +prefix: /esarchive/scratch/pbretonn/conda-cerise/conda/envs/condaCerise diff --git a/conda_installation/gribr_1.2.5.tar.gz b/conda_installation/gribr_1.2.5.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..2da172102e503c5e62b0da499626f2bc628aed4c Binary files /dev/null and b/conda_installation/gribr_1.2.5.tar.gz differ diff --git a/conda_installation/load_cerise.bash b/conda_installation/load_cerise.bash new file mode 100755 index 0000000000000000000000000000000000000000..3a10f3f6143f5afb73e65fd2b778941ac937df4b --- /dev/null +++ b/conda_installation/load_cerise.bash @@ -0,0 +1,17 @@ +#!/bin/bash + +prefix=$1 +local_gribr=$2 + + +if [[ -z $prefix ]]; then + conda env create --file environment-cerise${local_gribr}.yml +else + conda env create --file environment-cerise${local_gribr}.yml --prefix $prefix +fi +conda activate condaCerise +[[ -n ${local_gribr} ]] && module load ecCodes + +R -e "options(timeout = 600) ; install.packages('CSTools', repos='https://ftp.cixug.es/CRAN/')" +[[ -n ${local_gribr} ]] && R -e "options(timeout = 600) ; install.packages('conda_installation/gribr_1.2.5.tar.gz', repos = NULL)" +#conda env export > environment-cerise${local_gribr}.yml diff --git a/conf/archive.yml b/conf/archive.yml index 29489a971f7ca535c4e2c37a1e8f868fc5cc2fa2..d10a3f44287857100d0316ade46d7d33d3ac525a 100644 --- a/conf/archive.yml +++ b/conf/archive.yml @@ -218,186 +218,25 @@ esarchive: reference_grid: "/esarchive/obs/berkeleyearth/berkeleyearth/monthly_mean/tas/tas_201805.nc" - - mars: - src: "/esarchive/" + src: "/esarchive/scratch/aho/tmp/GRIB/" #"/mars/" System: ECMWF-SEAS5: name: "ECMWF SEAS5" institution: "European Centre for Medium-Range Weather Forecasts" - src: "exp/ecmwf/system5c3s/" - daily_mean: {"tas":"_f6h/", "rsds":"_s0-24h/", - "prlr":"_s0-24h/", "sfcWind":"_f6h/", - "tasmin":"_f24h/", "tasmax":"_f24h/", - "ta300":"_f12h/", "ta500":"_f12h/", "ta850":"_f12h/", - "g300":"_f12h/", "g500":"_f12h/", "g850":"_f12h/", - "tdps":"_f6h/", "hurs":"_f6h/"} - monthly_mean: {"tas":"_f6h/", "rsds":"_s0-24h/", - "prlr":"_s0-24h/", "sfcWind":"_f6h/", - "tasmin":"_f24h/", "tasmax":"_f24h/", - "ta300":"_f12h/", "ta500":"_f12h/", "ta850":"_f12h/", - "g300":"_f12h/", "g500":"_f12h/", "g850":"_f12h/", - "tdps":"_f6h/"} - nmember: - fcst: 51 - hcst: 25 - calendar: "proleptic_gregorian" - time_stamp_lag: "0" - reference_grid: "/esarchive/exp/ecmwf/system5c3s/monthly_mean/tas_f6h/tas_20180501.nc" - ECMWF-SEAS5.1: - name: "ECMWF SEAS5 (v5.1)" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "exp/ecmwf/system51c3s/" - daily_mean: {"tas":"_f6h/", "prlr":"_s0-24h/", "sfcWind":"_f6h/", - "uas":"_f6h/", "vas":"_f6h/", "psl":"_f6h/", - "tdps":"_f6h/"} - monthly_mean: {"tas":"_f6h/", "rsds":"_s0-24h/", "prlr":"_s0-24h/", - "sfcWind":"_f6h/", "tasmin":"_f24h/", "tasmax":"_f24h/", - "uas":"_f6h/", "vas":"_f6h/", "psl":"_f6h/", - "tdps":"_f6h/"} + src: "GRIB_system5_tas_CORRECTED/" + monthly_mean: {"tas":""} nmember: fcst: 51 - hcst: 25 - calendar: "proleptic_gregorian" - time_stamp_lag: "0" - reference_grid: "conf/grid_description/griddes_system51c3s.txt" - Meteo-France-System7: - name: "Meteo-France System 7" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "exp/meteofrance/system7c3s/" - monthly_mean: {"tas":"_f6h/", "g500":"_f12h/", - "prlr":"_f24h/", "sfcWind": "_f6h/", - "tasmax":"_f6h/", "tasmin": "_f6h/"} - nmember: - fcst: 51 - hcst: 25 - time_stamp_lag: "+1" - calendar: "proleptic_gregorian" - reference_grid: "conf/grid_description/griddes_system7c3s.txt" - DWD-GCFS2.1: - name: "DWD GCFS 2.1" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "exp/dwd/system21_m1/" - monthly_mean: {"tas":"_f6h/", "prlr":"_f24h/", - "g500":"_f12h/", "sfcWind":"_f6h/", - "tasmin":"_f24h/", "tasmax":"_f24h/"} - nmember: - fcst: 50 - hcst: 30 - calendar: "proleptic_gregorian" - time_stamp_lag: "+1" - reference_grid: "conf/grid_description/griddes_system21_m1.txt" - CMCC-SPS3.5: - name: "CMCC-SPS3.5" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "exp/cmcc/system35c3s/" - monthly_mean: {"tas":"_f6h/", "g500":"_f12h/", - "prlr":"_f24h/", "sfcWind": "_f6h/", - "tasmax":"_f24h/", "tasmin":"_f24h"} - nmember: - fcst: 50 - hcst: 40 - calendar: "proleptic_gregorian" - time_stamp_lag: "+1" - reference_grid: "conf/grid_description/griddes_system35c3s.txt" - JMA-CPS2: - name: "JMA System 2" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "exp/jma/system2c3s/" - monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/", - "tasmax":"_f6h/", "tasmin":"_f6h/"} - nmember: - fcst: 10 - hcst: 10 - calendar: "proleptic_gregorian" - time_stamp_lag: "+1" - reference_grid: "conf/grid_description/griddes_system2c3s.txt" - ECCC-CanCM4i: - name: "ECCC CanCM4i" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "exp/eccc/eccc1/" - monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/", - "tasmax":"_f6h/", "tasmin":"_f6h/"} - nmember: - fcst: 10 - hcst: 10 - calendar: "proleptic_gregorian" - time_stamp_lag: "+1" - reference_grid: "conf/grid_description/griddes_eccc1.txt" - UK-MetOffice-Glosea600: - name: "UK MetOffice GloSea 6 (v6.0)" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "exp/ukmo/glosea6_system600-c3s/" - monthly_mean: {"tas":"_f6h/", "tasmin":"_f24h/", - "tasmax":"_f24h/", "prlr":"_f24h/"} - nmember: - fcst: 62 - hcst: 28 + hcst: 51 calendar: "proleptic_gregorian" time_stamp_lag: "+1" - reference_grid: "conf/grid_description/griddes_ukmo600.txt" - NCEP-CFSv2: - name: "NCEP CFSv2" - institution: "NOAA NCEP" #? - src: "exp/ncep/cfs-v2/" - monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/", - "tasmax":"_f6h/", "tasmin":"_f6h/"} - nmember: - fcst: 20 - hcst: 20 - calendar: "gregorian" - time_stamp_lag: "0" - reference_grid: "conf/grid_description/griddes_ncep-cfsv2.txt" + reference_grid: "conf/grid_description/griddes_GRIB_system5_m1.txt" Reference: ERA5: name: "ERA5" institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/era5/" - daily_mean: {"tas":"_f1h-r1440x721cds/", - "rsds":"_f1h-r1440x721cds/", - "prlr":"_f1h-r1440x721cds/", - "g300":"_f1h-r1440x721cds/", - "g500":"_f1h-r1440x721cds/", - "g850":"_f1h-r1440x721cds/", - "sfcWind":"_f1h-r1440x721cds/", - "tasmax":"_f1h-r1440x721cds/", - "tasmin":"_f1h-r1440x721cds/", - "ta300":"_f1h-r1440x721cds/", - "ta500":"_f1h-r1440x721cds/", - "ta850":"_f1h-r1440x721cds/", - "hurs":"_f1h-r1440x721cds/"} - monthly_mean: {"tas":"_f1h-r1440x721cds/", - "prlr":"_f1h-r1440x721cds/", - "rsds":"_f1h-r1440x721cds/", - "g300":"_f1h-r1440x721cds/", - "g500":"_f1h-r1440x721cds/", - "g850":"_f1h-r1440x721cds/", - "sfcWind":"_f1h-r1440x721cds/", - "tasmax":"_f1h-r1440x721cds/", - "tasmin":"_f1h-r1440x721cds/", - "ta300":"_f1h-r1440x721cds/", - "ta500":"_f1h-r1440x721cds/", - "ta850":"_f1h-r1440x721cds/"} + src: "GRIB_era5_tas/" + monthly_mean: {"tas":""} calendar: "standard" - reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" - ERA5-Land: - name: "ERA5-Land" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/era5land/" - daily_mean: {"tas":"_f1h/", "rsds":"_f1h/", - "prlr":"_f1h/", "sfcWind":"_f1h/"} - monthly_mean: {"tas":"_f1h/","tasmin":"_f24h/", - "tasmax":"_f24h/", "prlr":"_f1h/", - "sfcWind":"_f1h/", "rsds":"_f1h/", - "tdps":"_f1h/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/ecmwf/era5land/daily_mean/tas_f1h/tas_201805.nc" - UERRA: - name: "ECMWF UERRA" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/uerra_mescan/" - daily_mean: {"tas":"_f6h/"} - monthly_mean: {"tas":"_f6h/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/ecmwf/uerra_mescan/daily_mean/tas_f6h/tas_201805.nc" + reference_grid: "conf/grid_description/griddes_GRIB_system5_m1.txt" diff --git a/conf/autosubmit.yml b/conf/autosubmit.yml index 8b653a650b8d054b73819af2ec2cb2bebeb3c6ad..4ff15ffd24e2bac63543cc792e7004f22953a6ab 100644 --- a/conf/autosubmit.yml +++ b/conf/autosubmit.yml @@ -1,8 +1,8 @@ esarchive: platform: nord3v2 - module_version: autosubmit/3.14.0-foss-2015a-Python-2.7.9 - auto_version: 3.14.0 - conf_format: ini + module_version: autosubmit/4.0.0b-foss-2015a-Python-3.7.3 + auto_version: 4.0.0 + conf_format: yaml experiment_dir: /esarchive/autosubmit/ userID: bsc32 mars: diff --git a/conf/grid_description/griddes_GRIB_system5_m1.txt b/conf/grid_description/griddes_GRIB_system5_m1.txt new file mode 100644 index 0000000000000000000000000000000000000000..7e38b73a3cc887d01693e7e966b05eb2b65fc6a1 --- /dev/null +++ b/conf/grid_description/griddes_GRIB_system5_m1.txt @@ -0,0 +1,246 @@ +# +# gridID system5_m1 +# +gridtype = lonlat +gridsize = 819200 +xname = longitude +xlongname = longitude +xunits = degrees_east +yname = latitude +ylongname = latitude +yunits = degrees_north +xsize = 1280 +ysize = 640 +xvals = 0 0.2812502 0.5625004 0.8437506 1.125001 1.406251 1.687501 1.968751 2.250002 + 2.531252 2.812502 3.093752 3.375002 3.656253 3.937503 4.218753 4.500003 +4.781253 5.062504 5.343754 5.625004 5.906254 6.187504 6.468754 6.750005 +7.031255 7.312505 7.593755 7.875005 8.156256 8.437506 8.718756 9.000006 +9.281256 9.562507 9.843757 10.12501 10.40626 10.68751 10.96876 11.25001 +11.53126 11.81251 12.09376 12.37501 12.65626 12.93751 13.21876 13.50001 +13.78126 14.06251 14.34376 14.62501 14.90626 15.18751 15.46876 15.75001 +16.03126 16.31251 16.59376 16.87501 17.15626 17.43751 17.71876 18.00001 +18.28126 18.56251 18.84376 19.12501 19.40626 19.68751 19.96876 20.25001 +20.53126 20.81251 21.09376 21.37501 21.65627 21.93752 22.21877 22.50002 +22.78127 23.06252 23.34377 23.62502 23.90627 24.18752 24.46877 24.75002 +25.03127 25.31252 25.59377 25.87502 26.15627 26.43752 26.71877 27.00002 +27.28127 27.56252 27.84377 28.12502 28.40627 28.68752 28.96877 29.25002 +29.53127 29.81252 30.09377 30.37502 30.65627 30.93752 31.21877 31.50002 +31.78127 32.06252 32.34377 32.62502 32.90627 33.18752 33.46877 33.75002 +34.03127 34.31252 34.59377 34.87502 35.15627 35.43752 35.71877 36.00003 +36.28128 36.56253 36.84378 37.12503 37.40628 37.68753 37.96878 38.25003 +38.53128 38.81253 39.09378 39.37503 39.65628 39.93753 40.21878 40.50003 +40.78128 41.06253 41.34378 41.62503 41.90628 42.18753 42.46878 42.75003 +43.03128 43.31253 43.59378 43.87503 44.15628 44.43753 44.71878 45.00003 +45.28128 45.56253 45.84378 46.12503 46.40628 46.68753 46.96878 47.25003 +47.53128 47.81253 48.09378 48.37503 48.65628 48.93753 49.21878 49.50003 +49.78128 50.06253 50.34378 50.62504 50.90629 51.18754 51.46879 51.75004 +52.03129 52.31254 52.59379 52.87504 53.15629 53.43754 53.71879 54.00004 +54.28129 54.56254 54.84379 55.12504 55.40629 55.68754 55.96879 56.25004 +56.53129 56.81254 57.09379 57.37504 57.65629 57.93754 58.21879 58.50004 +58.78129 59.06254 59.34379 59.62504 59.90629 60.18754 60.46879 60.75004 +61.03129 61.31254 61.59379 61.87504 62.15629 62.43754 62.71879 63.00004 +63.28129 63.56254 63.84379 64.12504 64.40629 64.68754 64.9688 65.25005 65.5313 +65.81255 66.0938 66.37505 66.6563 66.93755 67.2188 67.50005 67.7813 68.06255 +68.3438 68.62505 68.9063 69.18755 69.4688 69.75005 70.0313 70.31255 70.5938 +70.87505 71.1563 71.43755 71.7188 72.00005 72.2813 72.56255 72.8438 73.12505 +73.4063 73.68755 73.9688 74.25005 74.5313 74.81255 75.0938 75.37505 75.6563 +75.93755 76.2188 76.50005 76.7813 77.06255 77.3438 77.62505 77.9063 78.18755 +78.4688 78.75005 79.0313 79.31256 79.59381 79.87506 80.15631 80.43756 80.71881 +81.00006 81.28131 81.56256 81.84381 82.12506 82.40631 82.68756 82.96881 +83.25006 83.53131 83.81256 84.09381 84.37506 84.65631 84.93756 85.21881 +85.50006 85.78131 86.06256 86.34381 86.62506 86.90631 87.18756 87.46881 +87.75006 88.03131 88.31256 88.59381 88.87506 89.15631 89.43756 89.71881 +90.00006 90.28131 90.56256 90.84381 91.12506 91.40631 91.68756 91.96881 +92.25006 92.53131 92.81256 93.09381 93.37506 93.65632 93.93757 94.21882 +94.50007 94.78132 95.06257 95.34382 95.62507 95.90632 96.18757 96.46882 +96.75007 97.03132 97.31257 97.59382 97.87507 98.15632 98.43757 98.71882 +99.00007 99.28132 99.56257 99.84382 100.1251 100.4063 100.6876 100.9688 +101.2501 101.5313 101.8126 102.0938 102.3751 102.6563 102.9376 103.2188 +103.5001 103.7813 104.0626 104.3438 104.6251 104.9063 105.1876 105.4688 +105.7501 106.0313 106.3126 106.5938 106.8751 107.1563 107.4376 107.7188 +108.0001 108.2813 108.5626 108.8438 109.1251 109.4063 109.6876 109.9688 +110.2501 110.5313 110.8126 111.0938 111.3751 111.6563 111.9376 112.2188 +112.5001 112.7813 113.0626 113.3438 113.6251 113.9063 114.1876 114.4688 +114.7501 115.0313 115.3126 115.5938 115.8751 116.1563 116.4376 116.7188 +117.0001 117.2813 117.5626 117.8438 118.1251 118.4063 118.6876 118.9688 +119.2501 119.5313 119.8126 120.0938 120.3751 120.6563 120.9376 121.2188 +121.5001 121.7813 122.0626 122.3438 122.6251 122.9063 123.1876 123.4688 +123.7501 124.0313 124.3126 124.5938 124.8751 125.1563 125.4376 125.7188 +126.0001 126.2813 126.5626 126.8438 127.1251 127.4063 127.6876 127.9688 +128.2501 128.5313 128.8126 129.0938 129.3751 129.6563 129.9376 130.2188 +130.5001 130.7813 131.0626 131.3438 131.6251 131.9063 132.1876 132.4688 +132.7501 133.0313 133.3126 133.5938 133.8751 134.1563 134.4376 134.7188 +135.0001 135.2813 135.5626 135.8438 136.1251 136.4063 136.6876 136.9688 +137.2501 137.5313 137.8126 138.0938 138.3751 138.6563 138.9376 139.2188 +139.5001 139.7813 140.0626 140.3438 140.6251 140.9063 141.1876 141.4688 +141.7501 142.0313 142.3126 142.5938 142.8751 143.1563 143.4376 143.7188 +144.0001 144.2814 144.5626 144.8439 145.1251 145.4064 145.6876 145.9689 +146.2501 146.5314 146.8126 147.0939 147.3751 147.6564 147.9376 148.2189 +148.5001 148.7814 149.0626 149.3439 149.6251 149.9064 150.1876 150.4689 +150.7501 151.0314 151.3126 151.5939 151.8751 152.1564 152.4376 152.7189 +153.0001 153.2814 153.5626 153.8439 154.1251 154.4064 154.6876 154.9689 +155.2501 155.5314 155.8126 156.0939 156.3751 156.6564 156.9376 157.2189 +157.5001 157.7814 158.0626 158.3439 158.6251 158.9064 159.1876 159.4689 +159.7501 160.0314 160.3126 160.5939 160.8751 161.1564 161.4376 161.7189 +162.0001 162.2814 162.5626 162.8439 163.1251 163.4064 163.6876 163.9689 +164.2501 164.5314 164.8126 165.0939 165.3751 165.6564 165.9376 166.2189 +166.5001 166.7814 167.0626 167.3439 167.6251 167.9064 168.1876 168.4689 +168.7501 169.0314 169.3126 169.5939 169.8751 170.1564 170.4376 170.7189 +171.0001 171.2814 171.5626 171.8439 172.1251 172.4064 172.6876 172.9689 +173.2501 173.5314 173.8126 174.0939 174.3751 174.6564 174.9376 175.2189 +175.5001 175.7814 176.0626 176.3439 176.6251 176.9064 177.1876 177.4689 +177.7501 178.0314 178.3126 178.5939 178.8751 179.1564 179.4376 179.7189 +180.0001 180.2814 180.5626 180.8439 181.1251 181.4064 181.6876 181.9689 +182.2501 182.5314 182.8126 183.0939 183.3751 183.6564 183.9376 184.2189 +184.5001 184.7814 185.0626 185.3439 185.6251 185.9064 186.1876 186.4689 +186.7501 187.0314 187.3126 187.5939 187.8751 188.1564 188.4376 188.7189 +189.0001 189.2814 189.5626 189.8439 190.1251 190.4064 190.6876 190.9689 +191.2501 191.5314 191.8126 192.0939 192.3751 192.6564 192.9376 193.2189 +193.5001 193.7814 194.0626 194.3439 194.6251 194.9064 195.1876 195.4689 +195.7501 196.0314 196.3126 196.5939 196.8751 197.1564 197.4376 197.7189 +198.0001 198.2814 198.5626 198.8439 199.1251 199.4064 199.6876 199.9689 +200.2501 200.5314 200.8126 201.0939 201.3751 201.6564 201.9376 202.2189 +202.5001 202.7814 203.0626 203.3439 203.6251 203.9064 204.1876 204.4689 +204.7501 205.0314 205.3126 205.5939 205.8751 206.1564 206.4376 206.7189 +207.0001 207.2814 207.5626 207.8439 208.1251 208.4064 208.6876 208.9689 +209.2501 209.5314 209.8126 210.0939 210.3751 210.6564 210.9376 211.2189 +211.5001 211.7814 212.0626 212.3439 212.6251 212.9064 213.1876 213.4689 +213.7501 214.0314 214.3126 214.5939 214.8751 215.1564 215.4376 215.7189 +216.0002 216.2814 216.5627 216.8439 217.1252 217.4064 217.6877 217.9689 +218.2502 218.5314 218.8127 219.0939 219.3752 219.6564 219.9377 220.2189 +220.5002 220.7814 221.0627 221.3439 221.6252 221.9064 222.1877 222.4689 +222.7502 223.0314 223.3127 223.5939 223.8752 224.1564 224.4377 224.7189 +225.0002 225.2814 225.5627 225.8439 226.1252 226.4064 226.6877 226.9689 +227.2502 227.5314 227.8127 228.0939 228.3752 228.6564 228.9377 229.2189 +229.5002 229.7814 230.0627 230.3439 230.6252 230.9064 231.1877 231.4689 +231.7502 232.0314 232.3127 232.5939 232.8752 233.1564 233.4377 233.7189 +234.0002 234.2814 234.5627 234.8439 235.1252 235.4064 235.6877 235.9689 +236.2502 236.5314 236.8127 237.0939 237.3752 237.6564 237.9377 238.2189 +238.5002 238.7814 239.0627 239.3439 239.6252 239.9064 240.1877 240.4689 +240.7502 241.0314 241.3127 241.5939 241.8752 242.1564 242.4377 242.7189 +243.0002 243.2814 243.5627 243.8439 244.1252 244.4064 244.6877 244.9689 +245.2502 245.5314 245.8127 246.0939 246.3752 246.6564 246.9377 247.2189 +247.5002 247.7814 248.0627 248.3439 248.6252 248.9064 249.1877 249.4689 +249.7502 250.0314 250.3127 250.5939 250.8752 251.1564 251.4377 251.7189 +252.0002 252.2814 252.5627 252.8439 253.1252 253.4064 253.6877 253.9689 +254.2502 254.5314 254.8127 255.0939 255.3752 255.6564 255.9377 256.2189 +256.5002 256.7814 257.0627 257.3439 257.6252 257.9064 258.1877 258.4689 +258.7502 259.0314 259.3127 259.5939 259.8752 260.1564 260.4377 260.7189 +261.0002 261.2814 261.5627 261.8439 262.1252 262.4064 262.6877 262.9689 +263.2502 263.5314 263.8127 264.0939 264.3752 264.6564 264.9377 265.2189 +265.5002 265.7814 266.0627 266.3439 266.6252 266.9064 267.1877 267.4689 +267.7502 268.0314 268.3127 268.5939 268.8752 269.1564 269.4377 269.7189 +270.0002 270.2814 270.5627 270.8439 271.1252 271.4064 271.6877 271.9689 +272.2502 272.5314 272.8127 273.0939 273.3752 273.6564 273.9377 274.2189 +274.5002 274.7814 275.0627 275.3439 275.6252 275.9064 276.1877 276.4689 +276.7502 277.0314 277.3127 277.5939 277.8752 278.1564 278.4377 278.7189 +279.0002 279.2814 279.5627 279.8439 280.1252 280.4064 280.6877 280.9689 +281.2502 281.5314 281.8127 282.0939 282.3752 282.6564 282.9377 283.2189 +283.5002 283.7814 284.0627 284.3439 284.6252 284.9064 285.1877 285.4689 +285.7502 286.0314 286.3127 286.5939 286.8752 287.1564 287.4377 287.7189 +288.0002 288.2815 288.5627 288.844 289.1252 289.4065 289.6877 289.969 290.2502 +290.5315 290.8127 291.094 291.3752 291.6565 291.9377 292.219 292.5002 292.7815 +293.0627 293.344 293.6252 293.9065 294.1877 294.469 294.7502 295.0315 295.3127 +295.594 295.8752 296.1565 296.4377 296.719 297.0002 297.2815 297.5627 297.844 +298.1252 298.4065 298.6877 298.969 299.2502 299.5315 299.8127 300.094 300.3752 +300.6565 300.9377 301.219 301.5002 301.7815 302.0627 302.344 302.6252 302.9065 +303.1877 303.469 303.7502 304.0315 304.3127 304.594 304.8752 305.1565 305.4377 +305.719 306.0002 306.2815 306.5627 306.844 307.1252 307.4065 307.6877 307.969 +308.2502 308.5315 308.8127 309.094 309.3752 309.6565 309.9377 310.219 310.5002 +310.7815 311.0627 311.344 311.6252 311.9065 312.1877 312.469 312.7502 313.0315 +313.3127 313.594 313.8752 314.1565 314.4377 314.719 315.0002 315.2815 315.5627 +315.844 316.1252 316.4065 316.6877 316.969 317.2502 317.5315 317.8127 318.094 +318.3752 318.6565 318.9377 319.219 319.5002 319.7815 320.0627 320.344 320.6252 +320.9065 321.1877 321.469 321.7502 322.0315 322.3127 322.594 322.8752 323.1565 +323.4377 323.719 324.0002 324.2815 324.5627 324.844 325.1252 325.4065 325.6877 +325.969 326.2502 326.5315 326.8127 327.094 327.3752 327.6565 327.9377 328.219 +328.5002 328.7815 329.0627 329.344 329.6252 329.9065 330.1877 330.469 330.7502 +331.0315 331.3127 331.594 331.8752 332.1565 332.4377 332.719 333.0002 333.2815 +333.5627 333.844 334.1252 334.4065 334.6877 334.969 335.2502 335.5315 335.8127 +336.094 336.3752 336.6565 336.9377 337.219 337.5002 337.7815 338.0627 338.344 +338.6252 338.9065 339.1877 339.469 339.7502 340.0315 340.3127 340.594 340.8752 +341.1565 341.4377 341.719 342.0002 342.2815 342.5627 342.844 343.1252 343.4065 +343.6877 343.969 344.2502 344.5315 344.8127 345.094 345.3752 345.6565 345.9377 +346.219 346.5002 346.7815 347.0627 347.344 347.6252 347.9065 348.1877 348.469 +348.7502 349.0315 349.3127 349.594 349.8752 350.1565 350.4377 350.719 351.0002 +351.2815 351.5627 351.844 352.1252 352.4065 352.6877 352.969 353.2502 353.5315 +353.8127 354.094 354.3752 354.6565 354.9377 355.219 355.5002 355.7815 356.0627 +356.344 356.6252 356.9065 357.1877 357.469 357.7502 358.0315 358.3127 358.594 +358.8752 359.1565 359.4377 359.719 +yvals = 89.78488 89.5062 89.22588 88.94519 88.66436 88.38346 88.10252 87.82156 87.54058 +87.25959 86.97859 86.69759 86.41658 86.13557 85.85456 85.57355 85.29253 +85.01151 84.73049 84.44947 84.16845 83.88742 83.6064 83.32538 83.04435 82.76333 +82.4823 82.20128 81.92025 81.63923 81.3582 81.07717 80.79615 80.51512 80.23409 +79.95306 79.67204 79.39101 79.10998 78.82895 78.54792 78.26689 77.98587 +77.70484 77.42381 77.14278 76.86175 76.58072 76.29969 76.01867 75.73764 +75.45661 75.17558 74.89455 74.61352 74.33249 74.05146 73.77043 73.4894 73.20837 +72.92734 72.64631 72.36528 72.08426 71.80323 71.5222 71.24117 70.96014 70.67911 +70.39808 70.11705 69.83602 69.55499 69.27396 68.99293 68.7119 68.43087 68.14984 +67.86881 67.58778 67.30675 67.02572 66.74469 66.46366 66.18263 65.9016 65.62057 +65.33954 65.05851 64.77748 64.49645 64.21542 63.93439 63.65336 63.37233 63.0913 +62.81027 62.52924 62.24821 61.96718 61.68615 61.40512 61.12409 60.84306 +60.56203 60.281 59.99997 59.71894 59.43791 59.15688 58.87585 58.59482 58.31379 +58.03276 57.75173 57.4707 57.18967 56.90864 56.62761 56.34658 56.06555 55.78452 +55.50349 55.22246 54.94143 54.6604 54.37937 54.09834 53.81731 53.53628 53.25525 +52.97422 52.69319 52.41216 52.13113 51.85009 51.56906 51.28803 51.007 50.72597 +50.44494 50.16391 49.88288 49.60185 49.32082 49.03979 48.75876 48.47773 48.1967 +47.91567 47.63464 47.35361 47.07258 46.79155 46.51052 46.22949 45.94846 +45.66743 45.3864 45.10537 44.82434 44.54331 44.26228 43.98125 43.70022 43.41919 +43.13816 42.85713 42.5761 42.29507 42.01404 41.73301 41.45198 41.17094 40.88991 +40.60888 40.32785 40.04682 39.76579 39.48476 39.20373 38.9227 38.64167 38.36064 +38.07961 37.79858 37.51755 37.23652 36.95549 36.67446 36.39343 36.1124 35.83137 +35.55034 35.26931 34.98828 34.70725 34.42622 34.14519 33.86416 33.58313 33.3021 +33.02107 32.74004 32.45901 32.17797 31.89694 31.61591 31.33488 31.05385 +30.77282 30.49179 30.21076 29.92973 29.6487 29.36767 29.08664 28.80561 28.52458 +28.24355 27.96252 27.68149 27.40046 27.11943 26.8384 26.55737 26.27634 25.99531 +25.71428 25.43325 25.15222 24.87119 24.59016 24.30913 24.0281 23.74706 23.46603 +23.185 22.90397 22.62294 22.34191 22.06088 21.77985 21.49882 21.21779 20.93676 +20.65573 20.3747 20.09367 19.81264 19.53161 19.25058 18.96955 18.68852 18.40749 +18.12646 17.84543 17.5644 17.28337 17.00234 16.72131 16.44028 16.15925 15.87822 +15.59718 15.31615 15.03512 14.75409 14.47306 14.19203 13.911 13.62997 13.34894 +13.06791 12.78688 12.50585 12.22482 11.94379 11.66276 11.38173 11.1007 10.81967 +10.53864 10.25761 9.976578 9.695547 9.414517 9.133487 8.852456 8.571426 +8.290396 8.009365 7.728335 7.447305 7.166274 6.885244 6.604213 6.323183 +6.042153 5.761122 5.480092 5.199062 4.918031 4.637001 4.355971 4.07494 3.79391 +3.512879 3.231849 2.950819 2.669788 2.388758 2.107728 1.826697 1.545667 +1.264637 0.9836063 0.7025759 0.4215455 0.1405152 -0.1405152 -0.4215455 +-0.7025759 -0.9836063 -1.264637 -1.545667 -1.826697 -2.107728 -2.388758 +-2.669788 -2.950819 -3.231849 -3.512879 -3.79391 -4.07494 -4.355971 -4.637001 +-4.918031 -5.199062 -5.480092 -5.761122 -6.042153 -6.323183 -6.604213 -6.885244 +-7.166274 -7.447305 -7.728335 -8.009365 -8.290396 -8.571426 -8.852456 -9.133487 +-9.414517 -9.695547 -9.976578 -10.25761 -10.53864 -10.81967 -11.1007 -11.38173 +-11.66276 -11.94379 -12.22482 -12.50585 -12.78688 -13.06791 -13.34894 -13.62997 +-13.911 -14.19203 -14.47306 -14.75409 -15.03512 -15.31615 -15.59718 -15.87822 +-16.15925 -16.44028 -16.72131 -17.00234 -17.28337 -17.5644 -17.84543 -18.12646 +-18.40749 -18.68852 -18.96955 -19.25058 -19.53161 -19.81264 -20.09367 -20.3747 +-20.65573 -20.93676 -21.21779 -21.49882 -21.77985 -22.06088 -22.34191 -22.62294 +-22.90397 -23.185 -23.46603 -23.74706 -24.0281 -24.30913 -24.59016 -24.87119 +-25.15222 -25.43325 -25.71428 -25.99531 -26.27634 -26.55737 -26.8384 -27.11943 +-27.40046 -27.68149 -27.96252 -28.24355 -28.52458 -28.80561 -29.08664 -29.36767 +-29.6487 -29.92973 -30.21076 -30.49179 -30.77282 -31.05385 -31.33488 -31.61591 +-31.89694 -32.17797 -32.45901 -32.74004 -33.02107 -33.3021 -33.58313 -33.86416 +-34.14519 -34.42622 -34.70725 -34.98828 -35.26931 -35.55034 -35.83137 -36.1124 +-36.39343 -36.67446 -36.95549 -37.23652 -37.51755 -37.79858 -38.07961 -38.36064 +-38.64167 -38.9227 -39.20373 -39.48476 -39.76579 -40.04682 -40.32785 -40.60888 +-40.88991 -41.17094 -41.45198 -41.73301 -42.01404 -42.29507 -42.5761 -42.85713 +-43.13816 -43.41919 -43.70022 -43.98125 -44.26228 -44.54331 -44.82434 -45.10537 +-45.3864 -45.66743 -45.94846 -46.22949 -46.51052 -46.79155 -47.07258 -47.35361 +-47.63464 -47.91567 -48.1967 -48.47773 -48.75876 -49.03979 -49.32082 -49.60185 +-49.88288 -50.16391 -50.44494 -50.72597 -51.007 -51.28803 -51.56906 -51.85009 +-52.13113 -52.41216 -52.69319 -52.97422 -53.25525 -53.53628 -53.81731 -54.09834 +-54.37937 -54.6604 -54.94143 -55.22246 -55.50349 -55.78452 -56.06555 -56.34658 +-56.62761 -56.90864 -57.18967 -57.4707 -57.75173 -58.03276 -58.31379 -58.59482 +-58.87585 -59.15688 -59.43791 -59.71894 -59.99997 -60.281 -60.56203 -60.84306 +-61.12409 -61.40512 -61.68615 -61.96718 -62.24821 -62.52924 -62.81027 -63.0913 +-63.37233 -63.65336 -63.93439 -64.21542 -64.49645 -64.77748 -65.05851 -65.33954 +-65.62057 -65.9016 -66.18263 -66.46366 -66.74469 -67.02572 -67.30675 -67.58778 +-67.86881 -68.14984 -68.43087 -68.7119 -68.99293 -69.27396 -69.55499 -69.83602 +-70.11705 -70.39808 -70.67911 -70.96014 -71.24117 -71.5222 -71.80323 -72.08426 +-72.36528 -72.64631 -72.92734 -73.20837 -73.4894 -73.77043 -74.05146 -74.33249 +-74.61352 -74.89455 -75.17558 -75.45661 -75.73764 -76.01867 -76.29969 -76.58072 +-76.86175 -77.14278 -77.42381 -77.70484 -77.98587 -78.26689 -78.54792 -78.82895 +-79.10998 -79.39101 -79.67204 -79.95306 -80.23409 -80.51512 -80.79615 -81.07717 +-81.3582 -81.63923 -81.92025 -82.20128 -82.4823 -82.76333 -83.04435 -83.32538 +-83.6064 -83.88742 -84.16845 -84.44947 -84.73049 -85.01151 -85.29253 -85.57355 +-85.85456 -86.13557 -86.41658 -86.69759 -86.97859 -87.25959 -87.54058 -87.82156 +-88.10252 -88.38346 -88.66436 -88.94519 -89.22588 -89.5062 -89.78488 diff --git a/conf/slurm_templates/run_parallel_workflow.sh b/conf/slurm_templates/run_parallel_workflow.sh new file mode 100644 index 0000000000000000000000000000000000000000..461ee7e2335f4e830e4e72d95319d88415d3d98c --- /dev/null +++ b/conf/slurm_templates/run_parallel_workflow.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +#SBATCH -J SUNSET_verification + +# Slurm directive description: +# -J: job name + +set -vx + +script=$1 +atomic_recipe=$2 + +source MODULES + +Rscript ${script} ${atomic_recipe} diff --git a/conf/slurm_templates/run_scorecards.sh b/conf/slurm_templates/run_scorecards.sh new file mode 100644 index 0000000000000000000000000000000000000000..5ebf65281985d4ddac408609497eb76408f2eb32 --- /dev/null +++ b/conf/slurm_templates/run_scorecards.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +#SBATCH -J SUNSET_scorecards +#SBATCH --kill-on-invalid-dep=yes + +# Slurm directive description: +# -J: job name +# --kill-on-invalid-dep: Whether to kill the job if the dependencies are +# not satisfied. If the verification jobs fail, the scorecards job +# will be canceled. + +set -vx + +recipe=$1 +outdir=$2 + +source MODULES +# Execute scorecards +Rscript modules/Scorecards/execute_scorecards.R ${recipe} ${outdir} + diff --git a/example_scripts/test_scorecards_workflow.R b/example_scripts/test_scorecards_workflow.R new file mode 100644 index 0000000000000000000000000000000000000000..1287c8e713fecc0e28b136c224e5a7e6a0a2845b --- /dev/null +++ b/example_scripts/test_scorecards_workflow.R @@ -0,0 +1,15 @@ +source("modules/Loading/Loading.R") +source("modules/Calibration/Calibration.R") +source("modules/Anomalies/Anomalies.R") +source("modules/Skill/Skill.R") +source("modules/Saving/Saving.R") + +args = commandArgs(trailingOnly = TRUE) +recipe_file <- args[1] +recipe <- read_atomic_recipe(recipe_file) +# Load datasets +data <- load_datasets(recipe) +# Compute anomalies +data <- compute_anomalies(recipe, data) +# Compute skill metrics +skill_metrics <- compute_skill_metrics(recipe, data) diff --git a/launch_SUNSET.sh b/launch_SUNSET.sh new file mode 100644 index 0000000000000000000000000000000000000000..eb8fcf4638e3c96d422db3aad0bfd0af2740885c --- /dev/null +++ b/launch_SUNSET.sh @@ -0,0 +1,157 @@ +#!/bin/bash + +################################################################################ +## Launcher script for SUNSET +################################################################################ +## +## It reads a SUNSET recipe and splits it into single ('atomic') verifications. +## Then each atomic verification is launched as an independent job with SLURM. +## Each job can be further parallelized using the --cpus parameter and the +## 'ncores' field in the recipe. +## If requested in the recipe, the Scorecards for the results are also created +## and saved once the verifications are finished. +## +################################################################################ + +# Usage statement + +function usage +{ + echo "Usage: $0 --wallclock= --custom_directives= --disable_unique_ID" + echo " " + echo " : Path to the SUNSET recipe." + echo " " + echo " : Path to the user-defined script." + echo " " + echo " --wallclock=: Maximum execution time for the jobs." + echo " Default is 02:00:00." + echo " " + echo " --cpus=: CPUs (cores) to be requested for each job." + echo " Corresponds to the --cpus-per-task parameter" + echo " in sbatch. The default number is 1." + echo " " + echo " --custom_directives=: Custom directives for sbatch." + echo " E.g. '--constraint=medmem'." + echo " " + echo " --disable_unique_ID: Do not add a unique ID to the output folder name." +} + +if [[ ( $@ == "--help") || $@ == "-h" ]]; then + usage + exit 0 +fi + +# Assign arguments +recipe=$1 +script=$2 + +for i in "$@"; do + case $i in + --wallclock*) + export wallclock=`echo $1 | sed -e 's/^[^=]*=//g'` + shift + ;; + --cpus*) + export cpus=`echo $1 | sed -e 's/^[^=]*=//g'` + shift + ;; + --custom_directives*) + export custom_directives=`echo $1 | sed -e 's/^[^=]*=//g'` + shift + ;; + --disable_unique_ID) + export disable_unique_ID="--disable_unique_ID" + shift + ;; + -h|--help) + usage + exit 0 + shift + ;; + *) + shift + ;; + esac +done + +# Check recipe +if [ ! -f "$recipe" ]; then + echo "Could not find the recipe file: $recipe" + usage + exit 1 +fi +# Check script +if [ ! -f "$script" ]; then + echo "Could not find the script file: $script" + usage + exit 1 +fi +# Assign default wallclock time if not specified +if [ -z "$wallclock" ]; then + wallclock="02:00:00" +fi +# Assign default number of cores if not specified +if [ -z "$cpus" ]; then + cpus=1 +fi +# Assign empty custom directives if not specified +if [ -z "$custom_directives" ]; then + custom_directives="" +fi + +# Define tmp file to store necessary information +tmpfile=$(mktemp ${TMPDIR-/tmp}/SUNSET.XXXXXX) + +# Create outdir and split recipes +source MODULES +Rscript split.R ${recipe} $disable_unique_ID --tmpfile $tmpfile + +# Run with Autosubmit or directly with Slurm's sbatch? +run_method=$( head -1 $tmpfile | tail -1 ) +# If run method is 'sbatch', launch jobs with dependencies +if [ $run_method == "sbatch" ]; then + # Retrieve working directory + codedir=$( head -2 $tmpfile | tail -1 ) + # Retrieve output directory + outdir=$( head -3 $tmpfile | tail -1 ) + # Scorecards TRUE/FALSE + scorecards=$( head -4 $tmpfile | tail -1) + + # Create directory for slurm output + logdir=${codedir}/out-logs/slurm_logs/ + mkdir -p $logdir + echo "Slurm job logs will be stored in $logdir" + + # Launch one job per atomic recipe + cd $codedir + job_number=0 + # Create empty array to store all the job IDs for the verification jobs + verification_job_list=() + echo "Submitting verification jobs..." + # Loop over atomic recipes + for atomic_recipe in ${outdir}/logs/recipes/atomic_recipe_??.yml; do + job_number=$(($job_number + 1)) + job_name=$(basename $outdir)_$(printf %02d $job_number) + outfile=${logdir}/run-${job_name}.out + errfile=${logdir}/run-${job_name}.err + # Send batch job and capture job ID + job_ID=$(sbatch --parsable --output=$outfile --error=$errfile --time=$wallclock --cpus-per-task=$cpus $custom_directives conf/slurm_templates/run_parallel_workflow.sh ${script} ${atomic_recipe}) + # Add job ID to array + verification_job_list+=($job_ID) + echo "Submitted batch job $job_ID" + done + + # Submit scorecards job with dependency on verification jobs, passed as a + # comma-separated string. The scorecards job will not run until all the + # verification jobs have finished successfully. + # If any of the jobs fail, it will be canceled. + if [[ $scorecards == "TRUE" ]]; then + echo "Submitting scorecards jobs..." + outfile=${logdir}/run-scorecards.out + errfile=${logdir}/run-scorecards.err + sbatch --dependency=afterok:$(IFS=,; echo "${verification_job_list[*]}") --output=$outfile --error=$errfile --time=01:00:00 conf/slurm_templates/run_scorecards.sh ${recipe} ${outdir} + fi +fi + +# Clean temporary file +rm $tmpfile diff --git a/modules/Anomalies/Anomalies.R b/modules/Anomalies/Anomalies.R index a2f0c0053791cde810f52a87379508ee227bf0b5..c00ad3793e39839fc509e1d4e03c518d68a262b9 100644 --- a/modules/Anomalies/Anomalies.R +++ b/modules/Anomalies/Anomalies.R @@ -120,6 +120,9 @@ Anomalies <- function(recipe, data) { info(recipe$Run$logger, "##### ANOMALIES NOT COMPUTED #####") } + ## TODO: Return fcst full value? + .log_memory_usage(recipe$Run$logger, "After computing anomalies") + return(list(hcst = data$hcst, obs = data$obs, fcst = data$fcst, hcst.full_val = hcst_fullvalue, obs.full_val = obs_fullvalue)) diff --git a/modules/Calibration/Calibration.R b/modules/Calibration/Calibration.R index 80fca40875b82193c08573abf0b45172b973dc0b..f4b40afa932c0100490c008564680de00278db04 100644 --- a/modules/Calibration/Calibration.R +++ b/modules/Calibration/Calibration.R @@ -166,7 +166,7 @@ Calibration <- function(recipe, data) { } } info(recipe$Run$logger, CALIB_MSG) - + .log_memory_usage(recipe$Run$logger, "After calibration") # Saving if (recipe$Analysis$Workflow$Calibration$save != 'none') { info(recipe$Run$logger, "##### START SAVING CALIBRATED DATA #####") diff --git a/modules/Loading/Loading.R b/modules/Loading/Loading.R index 2de0406c4840398af2aead31cf37a6949b5969c9..d2919594e76afe9e8ae13e5223071e08c70b50d6 100644 --- a/modules/Loading/Loading.R +++ b/modules/Loading/Loading.R @@ -3,18 +3,23 @@ source("tools/libs.R") source("modules/Loading/load_datasets.R") Loading <- function(recipe) { - # Source correct function depending on time horizon - ## TODO: Add condition for GRIB files - time_horizon <- tolower(recipe$Analysis$Horizon) - if (time_horizon == "seasonal") { - source("modules/Loading/R/load_seasonal.R") - data <- load_seasonal(recipe) - } else if (time_horizon == "decadal") { - source("modules/Loading/R/load_decadal.R") - data <- load_decadal(recipe) + # Source correct function depending on filesystem and time horizon + # Case: CERISE (Mars) + if (tolower(recipe$Run$filesystem) == "mars") { + source("modules/Loading/R/load_GRIB.R") + data <- load_GRIB(recipe) } else { - stop("Incorrect time horizon.") + # Case: esarchive + time_horizon <- tolower(recipe$Analysis$Horizon) + if (time_horizon == "seasonal") { + source("modules/Loading/R/load_seasonal.R") + data <- load_seasonal(recipe) + } else if (time_horizon == "decadal") { + source("modules/Loading/R/load_decadal.R") + data <- load_decadal(recipe) + } else { + stop("Incorrect time horizon.") + } } - return(data) } diff --git a/modules/Loading/Loading_decadal.R b/modules/Loading/Loading_decadal.R deleted file mode 100644 index 74c97e299f7e7e232fbf2fb3e261bec9673a394e..0000000000000000000000000000000000000000 --- a/modules/Loading/Loading_decadal.R +++ /dev/null @@ -1,554 +0,0 @@ -# Loading module: -# 1. archive.yml -# 2. recipe.yml -# 3. Load_decadal.R (V) -#setwd('/esarchive/scratch/aho/git/auto-s2s/') - -## TODO: remove paths to personal scratchs -source("/esarchive/scratch/vagudets/repos/csoperational/R/get_regrid_params.R") -# Load required libraries/funs -source("modules/Loading/helper_loading_decadal.R") -source("modules/Loading/R/dates2load.R") -source("modules/Loading/R/check_latlon.R") -source("modules/Loading/R/get_timeidx.R") -source("tools/libs.R") - -#==================================================================== - -# recipe_file <- "recipes/atomic_recipes/recipe_decadal.yml" -# recipe_file <- "recipes/atomic_recipes/recipe_decadal_daily.yml" - -load_datasets <- function(recipe) { - - ## - archive <- read_yaml(paste0("conf/archive_decadal.yml"))$esarchive - - # Print Start() info or not - DEBUG <- FALSE - - ## TODO: this should come from the main script - # Create output folder and log: - - #------------------------- - # Read from recipe: - #------------------------- - exp.name <- recipe$Analysis$Datasets$System$name #'HadGEM3' - ref.name <- recipe$Analysis$Datasets$Reference$name #'era5' - member <- strsplit(recipe$Analysis$Datasets$System$member, ',')[[1]] #c("r1i1p1f2", "r2i1p1f2") -# variable <- recipe$Analysis$Variables$name #'tas' - variable <- strsplit(recipe$Analysis$Variables$name, ", | |,")[[1]] - store.freq <- recipe$Analysis$Variables$freq #monthly_mean - lats.min <- as.numeric(recipe$Analysis$Region$latmin) #0 - lats.max <- as.numeric(recipe$Analysis$Region$latmax) #10 - lons.min <- as.numeric(recipe$Analysis$Region$lonmin) #0 - lons.max <- as.numeric(recipe$Analysis$Region$lonmax) #10 - - # change to: sdates <- dates2load(recipe, logger) - sdates_hcst <- as.numeric(recipe$Analysis$Time$hcst_start):as.numeric(recipe$Analysis$Time$hcst_end) #1960:2015 - sdates_fcst <- recipe$Analysis$Time$fcst - - if (store.freq == "monthly_mean") { - time_ind <- (as.numeric(recipe$Analysis$Time$ftime_min):as.numeric(recipe$Analysis$Time$ftime_max)) - - } else if (store.freq == "daily_mean") { - time_ind <- get_daily_time_ind(ftimemin = as.numeric(recipe$Analysis$Time$ftime_min), - ftimemax = as.numeric(recipe$Analysis$Time$ftime_max), - initial_month = archive$System[[exp.name]]$initial_month, - sdates = sdates_hcst, - calendar = archive$System[[exp.name]]$calendar) - } - -#NOTE: May be used in the future -# season <- recipe$Analysis$Time$season - - #------------------------- - # Read from archive: - #------------------------- - if (store.freq == "monthly_mean") { - table <- archive$System[[exp.name]][[store.freq]]$table[variable] #list(tas = 'Amon') - } else { - table <- 'day' - } - grid <- archive$System[[exp.name]][[store.freq]]$grid[variable] #list(tas = 'gr') - version <- archive$System[[exp.name]][[store.freq]]$version[variable] #list(tas = 'v20210910') - if (identical(member, 'all')) { - member <- strsplit(archive$System[[exp.name]]$member, ',')[[1]] - } - - #------------------------- - # derived from above: - #------------------------- - # Check lat and lon and decide CircularSort - circularsort <- check_latlon(latmin = lats.min, latmax = lats.max, lonmin = lons.min, lonmax = lons.max) - - # generate transform params for system and ref - regrid_params <- get_regrid_params(recipe, archive) - - # Only if the time length in each chunk may differ that we need largest_dims_length to be TRUE. Otherwise, set FALSE to increase efficiency. - need_largest_dims_length <- ifelse(exp.name == 'EC-Earth3-i2', TRUE, FALSE) - - - #------------------------------------------- - # Step 1: Load the hcst - #------------------------------------------- - #monthly and daily - tmp <- get_dcpp_path(archive = archive, exp.name = exp.name, table = table, grid = grid, - version = version, sdates = sdates_hcst) - path_list <- tmp$path_list - multi_path <- tmp$multi_path - - #TODO: to make this case work; enhance Start() if it's possible - if (multi_path & length(variable) > 1) { - stop("The recipe requests multiple variables and start dates from both dpccA-hindcast and dcppB-forecast. This case is not available for now.") - } - - Start_default_arg_list <- list( - dat = path_list, - var = variable, - syear = paste0(sdates_hcst), - chunk = 'all', - chunk_depends = 'syear', - time = indices(time_ind), - time_across = 'chunk', - merge_across_dims = TRUE, - largest_dims_length = need_largest_dims_length, - latitude = values(list(lats.min, lats.max)), - latitude_reorder = Sort(decreasing = TRUE), - longitude = values(list(lons.min, lons.max)), - longitude_reorder = circularsort, - ensemble = member, - transform = regrid_params$fcst.transform, - transform_extra_cells = 2, - transform_params = list(grid = regrid_params$fcst.gridtype, - method = regrid_params$fcst.gridmethod), - transform_vars = c('latitude', 'longitude'), -# path_glob_permissive = 2, # for version - synonims = list(longitude = c('lon', 'longitude'), - latitude = c('lat', 'latitude')), - return_vars = list(latitude = NULL, longitude = NULL, - time = c('syear', 'chunk')), - silent = !DEBUG, - retrieve = T) - - if (length(variable) > 1) { - Start_default_arg_list <- c(Start_default_arg_list, - list(table = table, grid = grid, version = version, - table_depends = 'var', grid_depends = 'var', version_depends = 'var', - metadata_dims = 'var')) - } - - if (!multi_path) { - Start_hcst_arg_list <- Start_default_arg_list - hcst <- do.call(Start, Start_hcst_arg_list) - - } else { - Start_hcst_arg_list <- Start_default_arg_list - Start_hcst_arg_list[['syear']] <- NULL - Start_hcst_arg_list[['chunk_depends']] <- NULL - remove_ind <- which(Start_hcst_arg_list[['return_vars']][['time']] == 'syear') - Start_hcst_arg_list[['return_vars']][['time']] <- Start_hcst_arg_list[['return_vars']][['time']][-remove_ind] - - hcst <- do.call(Start, Start_hcst_arg_list) - - # Reshape and reorder dimensions - ## dat should be 1, syear should be length of dat; reorder dimensions - dim(hcst) <- c(dat = 1, syear = as.numeric(dim(hcst))[1], dim(hcst)[2:6]) - hcst <- s2dv::Reorder(hcst, c('dat', 'var', 'syear', 'time', 'latitude', 'longitude', 'ensemble')) - - # Manipulate time attr because Start() cannot read it correctly - wrong_time_attr <- attr(hcst, 'Variables')$common$time # dim: [time], the first syear only - tmp <- array(dim = c(dim(hcst)[c('syear', 'time')])) - tmp[1, ] <- wrong_time_attr - yr_diff <- (sdates_hcst - sdates_hcst[1])[-1] #diff(sdates_hcst) - for (i_syear in 1:length(yr_diff)) { - tmp[(i_syear + 1), ] <- wrong_time_attr + lubridate::years(yr_diff[i_syear]) - } - attr(hcst, 'Variables')$common$time <- as.POSIXct(tmp, origin = '1970-01-01', tz = 'UTC') - - } - - tmp_time_attr <- attr(hcst, 'Variables')$common$time - - # change syear to c(sday, sweek, syear) - # dim(hcst) should be [dat, var, sday, sweek, syear, time, latitude, longitude, ensemble] - dim(hcst) <- c(dim(hcst)[1:2], sday = 1, sweek = 1, dim(hcst)[3:7]) - if (!identical(dim(tmp_time_attr), dim(hcst)[c('syear', 'time')])) { - error(recipe$Run$logger, - "hcst has problem in matching data and time attr dimension.") - stop() - } - dim(attr(hcst, 'Variables')$common$time) <- c(sday = 1, sweek = 1, dim(tmp_time_attr)) - - #TODO: as.s2dv_cube() needs to be improved to recognize "variable" is under $dat1 - if (multi_path) { - attributes(hcst)$Variables$common[[variable]] <- attributes(hcst)$Variables$dat1[[variable]] - } - - # Change class from startR_array to s2dv_cube - suppressWarnings( - hcst <- as.s2dv_cube(hcst) - ) - -#------------------------------------------- -# Step 2: Load the fcst -#------------------------------------------- - if (!is.null(recipe$Analysis$Time$fcst)) { - - tmp <- get_dcpp_path(archive = archive, exp.name = exp.name, table = table, grid = grid, - version = version, sdates = sdates_fcst) - path_list <- tmp$path_list - multi_path <- tmp$multi_path - - #TODO: to make this case work; enhance Start() if it's possible - if (multi_path & length(variable) > 1) { - stop("The recipe requests multiple variables and start dates from both dpccA-hindcast and dcppB-forecast. This case is not available for now.") - } - - # monthly & daily - if (!multi_path) { - #NOTE: the adjustment for two cases (multiple files per sdate or not) has been made in hcst - Start_fcst_arg_list <- Start_default_arg_list - Start_fcst_arg_list[['dat']] <- path_list - Start_fcst_arg_list[['syear']] <- paste0(sdates_fcst) - fcst <- do.call(Start, Start_fcst_arg_list) - - - } else { # multi_path - - #TODO: time attribute is not correct. Improve Start(). - Start_fcst_arg_list <- Start_default_arg_list - Start_fcst_arg_list[['dat']] <- path_list - Start_fcst_arg_list[['syear']] <- NULL - Start_fcst_arg_list[['chunk_depends']] <- NULL - remove_ind <- which(Start_fcst_arg_list[['return_vars']][['time']] == 'syear') - Start_fcst_arg_list[['return_vars']][['time']] <- Start_fcst_arg_list[['return_vars']][['time']][-remove_ind] - fcst <- do.call(Start, Start_fcst_arg_list) - - # Reshape and reorder dimensions - ## dat should be 1, syear should be length of dat; reorder dimensions - ## dim(fcst) should be [dat, var, syear, time, latitude, longitude, ensemble] - dim(fcst) <- c(dat = 1, syear = as.numeric(dim(fcst))[1], dim(fcst)[2:6]) - fcst <- s2dv::Reorder(fcst, c('dat', 'var', 'syear', 'time', 'latitude', 'longitude', 'ensemble')) - - # Manipulate time attr because Start() cannot read it correctly - wrong_time_attr <- attr(fcst, 'Variables')$common$time # dim: [time], the first syear only - tmp <- array(dim = c(dim(fcst)[c('syear', 'time')])) - tmp[1, ] <- wrong_time_attr - yr_diff <- (sdates_fcst - sdates_fcst[1])[-1] #diff(sdates_fcst) - for (i_syear in 1:length(yr_diff)) { - tmp[(i_syear + 1), ] <- wrong_time_attr + lubridate::years(yr_diff[i_syear]) - } - attr(fcst, 'Variables')$common$time <- as.POSIXct(tmp, origin = '1970-01-01', tz = 'UTC') - - } - - tmp_time_attr <- attr(fcst, 'Variables')$common$time - - # change syear to c(sday, sweek, syear) - # dim(fcst) should be [dat, var, sday, sweek, syear, time, latitude, longitude, ensemble] - dim(fcst) <- c(dim(fcst)[1:2], sday = 1, sweek = 1, dim(fcst)[3:7]) - if (!identical(dim(tmp_time_attr), dim(fcst)[c('syear', 'time')])) { - error(recipe$Run$logger, - "fcst has problem in matching data and time attr dimension.") - stop() - } - dim(attr(fcst, 'Variables')$common$time) <- c(sday = 1, sweek = 1, dim(tmp_time_attr)) - - #TODO: as.s2dv_cube() needs to be improved to recognize "variable" is under $dat1 - if (multi_path) { - attributes(fcst)$Variables$common[[variable]] <- attributes(fcst)$Variables$dat1[[variable]] - } - - # Change class from startR_array to s2dv_cube - suppressWarnings( - fcst <- as.s2dv_cube(fcst) - ) - - # Only syear could be different - if (!identical(dim(hcst$data)[-5], dim(fcst$data)[-5])) { - error(recipe$Run$logger, - "hcst and fcst do not share the same dimension structure.") - stop() - } - - } else { - fcst <- NULL - } - -#------------------------------------------- -# Step 3. Load the reference -#------------------------------------------- - obs.path <- file.path(archive$src, archive$Reference[[ref.name]]$src, - store.freq, "$var$$var_dir$", "$var$_$file_date$.nc") - var_dir_obs <- archive$Reference[[ref.name]][[store.freq]][variable] # list(tas = "_f1h-r1440x721cds", tos = "_f1h-r1440x721cds") - -# obs.path <- file.path(archive$src, archive$Reference[[ref.name]]$src, store.freq, -# paste0(variable, archive$Reference[[ref.name]][[store.freq]][[variable]])) -# obs.files <- paste0('$var$_$file_date$.nc') - - # Get from startR_cube -# dates <- attr(hcst, 'Variables')$common$time - # Get from s2dv_cube - dates <- hcst$attrs$Dates - dates_file <- sapply(dates, format, '%Y%m') - dim(dates_file) <- dim(dates) - - if (store.freq == "daily_mean") { -#//////////////// -# Method 1: use hcst time attr as obs time selector -#//////////////// - - # Set hour to 12:00 to ensure correct date retrieval for daily data - lubridate::hour(dates) <- 12 - lubridate::minute(dates) <- 00 - # Restore correct dimensions - dim(dates) <- dim(dates_file) - - obs <- Start(dat = obs.path, - var = variable, - var_dir = var_dir_obs, - var_dir_depends = 'var', - file_date = unique(format(dates, '%Y%m')), - time = dates, # [sday, sweek, syear, time] - time_across = 'file_date', - merge_across_dims = TRUE, - split_multiselected_dims = TRUE, - latitude = values(list(lats.min, lats.max)), - latitude_reorder = Sort(decreasing = TRUE), - longitude = values(list(lons.min, lons.max)), - longitude_reorder = circularsort, - transform = regrid_params$obs.transform, - transform_extra_cells = 2, - transform_params = list(grid = regrid_params$obs.gridtype, #nc file - method = regrid_params$obs.gridmethod), - transform_vars = c('latitude', 'longitude'), - synonims = list(latitude = c('lat','latitude'), - longitude = c('lon','longitude')), - return_vars = list(latitude = NULL, longitude = NULL, - time = 'file_date'), - silent = !DEBUG, - retrieve = TRUE) - - } else if (store.freq == "monthly_mean") { -#//////////////// -# Method 2: reshape hcst time attr's date into an array with time dim then as obs date selector -#//////////////// - - obs <- Start(dat = obs.path, - var = variable, - var_dir = var_dir_obs, - var_dir_depends = 'var', - file_date = dates_file, #dates_arr, # [sday, sweek, syear, time] - split_multiselected_dims = TRUE, - latitude = values(list(lats.min, lats.max)), - latitude_reorder = Sort(decreasing = TRUE), - longitude = values(list(lons.min, lons.max)), - longitude_reorder = circularsort, - transform = regrid_params$obs.transform, - transform_extra_cells = 2, - transform_params = list(grid = regrid_params$obs.gridtype, #nc file - method = regrid_params$obs.gridmethod), - transform_vars = c('latitude', 'longitude'), - synonims = list(latitude = c('lat','latitude'), - longitude = c('lon','longitude')), - return_vars = list(latitude = NULL, longitude = NULL, - time = 'file_date'), - metadata_dims = 'var', - silent = !DEBUG, - retrieve = TRUE) - } - - -#dim(attr(obs, 'Variables')$common$time) -# sday sweek syear time -# 1 1 2 14 - - # Remove var_dir dimension - obs <- Subset(obs, along = "var_dir", indices = 1, drop = "selected") - - # Only ensemble dim could be different - if (!identical(dim(obs), dim(hcst$data)[-9])) { - error(recipe$Run$logger, - "obs and hcst dimensions do not match.") - stop() - } - # Add ensemble dim to obs - dim(obs) <- c(dim(obs), ensemble = 1) - - # Change class from startR_array to s2dv_cube - suppressWarnings( - obs <- as.s2dv_cube(obs) - ) - -#------------------------------------------- -# Step 4. Verify the consistance between data -#------------------------------------------- - # dimension - if (any(!names(dim(obs$data)) %in% names(dim(hcst$data)))) { - error(recipe$Run$logger, - "hcst and obs don't share the same dimension names.") - stop() - } else { - ens_ind <- which(names(dim(obs$data)) == 'ensemble') - match_ind <- match(names(dim(obs$data))[-ens_ind], names(dim(hcst$data))) - if (!all(dim(hcst$data)[match_ind] == dim(obs$data)[-ens_ind])) { - error(recipe$Run$logger, - "hcst and obs don't share the same dimension length.") - stop() - } - } - - # time attribute - if (!identical(format(hcst$attrs$Dates, '%Y%m'), - format(obs$attrs$Dates, '%Y%m'))) { - error(recipe$Run$logger, - "hcst and obs don't share the same time.") - stop() - } - - # lat and lon attributes - if (!(recipe$Analysis$Regrid$type == 'none')) { - if (!isTRUE(all.equal(as.vector(hcst$lat), as.vector(obs$lat)))) { - lat_error_msg <- paste("Latitude mismatch between hcst and obs.", - "Please check the original grids and the", - "regrid parameters in your recipe.") - error(recipe$Run$logger, lat_error_msg) - hcst_lat_msg <- paste0("First hcst lat: ", hcst$lat[1], - "; Last hcst lat: ", hcst$lat[length(hcst$lat)]) - info(recipe$Run$logger, hcst_lat_msg) - obs_lat_msg <- paste0("First obs lat: ", obs$lat[1], - "; Last obs lat: ", obs$lat[length(obs$lat)]) - info(recipe$Run$logger, obs_lat_msg) - stop("hcst and obs don't share the same latitudes.") - } - - if (!isTRUE(all.equal(as.vector(hcst$lon), as.vector(obs$lon)))) { - lon_error_msg <- paste("Longitude mismatch between hcst and obs.", - "Please check the original grids and the", - "regrid parameters in your recipe.") - error(recipe$Run$logger, lon_error_msg) - hcst_lon_msg <- paste0("First hcst lon: ", hcst$lon[1], - "; Last hcst lon: ", hcst$lon[length(hcst$lon)]) - info(recipe$Run$logger, hcst_lon_msg) - obs_lon_msg <- paste0("First obs lon: ", obs$lon[1], - "; Last obs lon: ", obs$lon[length(obs$lon)]) - info(recipe$Run$logger, obs_lon_msg) - stop("hcst and obs don't share the same longitudes.") - } - } - - # Check fcst - if (!is.null(fcst)) { - # dimension - if (any(!names(dim(fcst$data)) %in% names(dim(hcst$data)))) { - error(recipe$Run$logger, - "hcst and fcst don't share the same dimension names.") - stop() - } else { - ens_ind <- which(names(dim(fcst$data)) %in% c('ensemble', 'syear')) - match_ind <- match(names(dim(fcst$data))[-ens_ind], names(dim(hcst$data))) - if (!all(dim(hcst$data)[match_ind] == dim(fcst$data)[-ens_ind])) { - error(recipe$Run$logger, - "hcst and fcst don't share the same dimension length.") - stop() - } - } - - # lat and lon attributes - if (!(recipe$Analysis$Regrid$type == 'none')) { - if (!identical(as.vector(hcst$lat), as.vector(fcst$lat))) { - lat_error_msg <- paste("Latitude mismatch between hcst and fcst.", - "Please check the original grids and the", - "regrid parameters in your recipe.") - error(recipe$Run$logger, lat_error_msg) - hcst_lat_msg <- paste0("First hcst lat: ", hcst$lat[1], - "; Last hcst lat: ", hcst$lat[length(hcst$lat)]) - info(recipe$Run$logger, hcst_lat_msg) - fcst_lat_msg <- paste0("First fcst lat: ", fcst$lat[1], - "; Last fcst lat: ", fcst$lat[length(fcst$lat)]) - info(recipe$Run$logger, fcst_lat_msg) - stop("hcst and fcst don't share the same latitudes.") - } - - if (!identical(as.vector(hcst$lon), as.vector(fcst$lon))) { - lon_error_msg <- paste("Longitude mismatch between hcst and fcst.", - "Please check the original grids and the", - "regrid parameters in your recipe.") - error(recipe$Run$logger, lon_error_msg) - hcst_lon_msg <- paste0("First hcst lon: ", hcst$lon[1], - "; Last hcst lon: ", hcst$lon[length(hcst$lon)]) - info(recipe$Run$logger, hcst_lon_msg) - fcst_lon_msg <- paste0("First fcst lon: ", fcst$lon[1], - "; Last fcst lon: ", fcst$lon[length(fcst$lon)]) - info(recipe$Run$logger, fcst_lon_msg) - stop("hcst and fcst don't share the same longitudes.") - } - } - - } - - -#------------------------------------------- -# Step 5. Tune data -#------------------------------------------- - # Remove negative values in accumulative variables - dictionary <- read_yaml("conf/variable-dictionary.yml") - for (var_idx in 1:length(variable)) { - var_name <- variable[var_idx] - if (dictionary$vars[[var_name]]$accum) { - info(recipe$Run$logger, - paste0("Accumulated variable ", var_name, - ": setting negative values to zero.")) - # obs$data[, var_idx, , , , , , , ] <- pmax(Subset(obs$data, - # along = "var", - # indices = var_idx, F), 0) - obs$data[, var_idx, , , , , , , ][obs$data[, var_idx, , , , , , , ] < 0] <- 0 - hcst$data[, var_idx, , , , , , , ][hcst$data[, var_idx, , , , , , , ] < 0] <- 0 - if (!is.null(fcst)) { - fcst$data[, var_idx, , , , , , , ][fcst$data[, var_idx, , , , , , , ] < 0] <- 0 - } - } - - # Convert prlr from m/s to mm/day - ## TODO: Make a unit conversion function - if (variable[[var_idx]] == "prlr") { - # Verify that the units are m/s and the same in obs and hcst - if (((obs$attrs$Variable$metadata[[var_name]]$units == "m s-1") || - (obs$attrs$Variable$metadata[[var_name]]$units == "m s**-1")) && - ((hcst$attrs$Variable$metadata[[var_name]]$units == "m s-1") || - (hcst$attrs$Variable$metadata[[var_name]]$units == "m s**-1"))) { - info(recipe$Run$logger, "Converting precipitation from m/s to mm/day.") - obs$data[, var_idx, , , , , , , ] <- - obs$data[, var_idx, , , , , , , ]*86400*1000 - obs$attrs$Variable$metadata[[var_name]]$units <- "mm/day" - hcst$data[, var_idx, , , , , , , ] <- - hcst$data[, var_idx, , , , , , , ]*86400*1000 - hcst$attrs$Variable$metadata[[var_name]]$units <- "mm/day" - if (!is.null(fcst)) { - fcst$data[, var_idx, , , , , , , ] <- - fcst$data[, var_idx, , , , , , , ]*86400*1000 - fcst$attrs$Variable$metadata[[var_name]]$units <- "mm/day" - } - } - } - } - -#------------------------------------------- -# Step 6. Print summary -#------------------------------------------- - - # Print a summary of the loaded data for the user, for each object - if (recipe$Run$logger$threshold <= 2) { - data_summary(hcst, recipe) - data_summary(obs, recipe) - if (!is.null(fcst)) { - data_summary(fcst, recipe) - } - } - - info(recipe$Run$logger, - "##### DATA LOADING COMPLETED SUCCESSFULLY #####") - - - return(list(hcst = hcst, fcst = fcst, obs = obs)) -} diff --git a/modules/Loading/R/GRIB/GRIB/GrbLoad.R b/modules/Loading/R/GRIB/GRIB/GrbLoad.R new file mode 100644 index 0000000000000000000000000000000000000000..ef1df0cb13b6eb46ac4b55de3fabee4dd204213a --- /dev/null +++ b/modules/Loading/R/GRIB/GRIB/GrbLoad.R @@ -0,0 +1,248 @@ +#---------------------------------------------------------------------------------- +# Use gribr package to load GRIB files +# Assume that all the messages have the same metadata; one message is one time step +# If exp, has.memb is a number; if obs, has.memb = NULL +# syear_time_dim is the time attr dim of exp as the input for obs +#---------------------------------------------------------------------------------- +GrbLoad <- function (dat, time_step = 1, has.memb = NULL, syear_time_dim = NULL, + regrid = NULL) { + library(gribr) + + result <- vector('list', length = length(dat)) + times <- vector('list', length = length(dat)) + times <- lapply(times, '[<-', rep(NA, length(time_step))) #NOTE: length is 0 (slower in loop?) +# times <- lapply(times, '[<-', .POSIXct(rep(NA, length(time_step)), tz = 'UTC')) + + for (dat_i in 1:length(dat)) { + + file_to_load <- grib_open(dat[[dat_i]]) + + #---------------------------------------- + # HOW TO FIND THE VALUE OF EACH FTIME STEP? + #---------------------------------------- + #NOTE: ValidityTime is not considered now. So if the time frequency is less than daily, it has problem. + + # METHOD 1: Get first message to figure out the validityDate/Time of each message + #NOTE: gm1$validityDate should be "s", "m", "h", etc. according to document. But our files have "1". + gm1 <- grib_get_message(file_to_load, 1) + first_ftime <- as.character(gm1$validityDate) + first_ftime_hour <- gm1$validityTime + # For monthly data + #NOTE: may not be correct because it is calculated by the first message + cdo_time_attr <- clock::add_months(as.POSIXct(paste0(first_ftime, ' ', first_ftime_hour), + format = "%Y%m%d %H", tz = 'UTC'), time_step - 1) + cdo_time <- format(cdo_time_attr, "%Y%m%d") + +# # METHOD 2: Use cdo showtimestamp (DEPENDENCY!) +# #TODO: Change to method 1 because can't predict what cdo will produce +# cdo_time <- system(paste0("cdo showtimestamp ", dat[[dat_i]]), intern = T) +# cdo_time <- strsplit(cdo_time, " ")[[length(cdo_time)]] +# cdo_time <- cdo_time[which(cdo_time != "")] +## # Check if there is member dim or not +## has_memb <- ifelse((length(unique(cdo_time)) == length(cdo_time)), FALSE, TRUE) +# if (has.memb) memb_dim_length <- length(cdo_time)/length(unique(cdo_time)) +# cdo_time <- unique(cdo_time)[time_step] #"2000-12-01T00:00:00" +# cdo_time_attr <- as.POSIXct(gsub('T', ' ', cdo_time), tz = 'UTC') +# cdo_time <- sapply(sapply(cdo_time, strsplit, "T"), '[[', 1) +# cdo_time <- gsub('-', '', cdo_time) + + #---------------------------------------- + + # all members + ftimes: length should be memb*ftime (e.g., 51*7) + ## Method 1: use grib_select and real values to filter + memb_ftime <- grib_select(file_to_load, list(validityDate = cdo_time)) + if (inherits(memb_ftime, 'gribMessage')) memb_ftime <- list(memb_ftime) + +# ## Method 2: Calculate which messages are the desired ones +# gm <- grib_get_message(file_to_load, time_step) +# if (length(time_step) == 1) { +# gm <- list(gm) +# } + + ################################################################## + # Get data as an array [longitude, latitude, (memb*)time] + ################################################################## + if (grepl("reduced", gm1$gridType)) { + #NOTE: Need to call gribr::grib_expand_grids because I don't know how to make .Call("gribr_redtoreg") work outside that function + # https://github.com/nawendt/gribr/blob/main/src/redtoreg.c + values_l <- vector('list', length = length(memb_ftime)) + for (gm_i in 1:length(memb_ftime)) { + values_l[[gm_i]] <- grib_expand_grids(memb_ftime[[gm_i]]) + } + result[[dat_i]] <- array(unlist(values_l), dim = c(longitude = gm1$Nj * 2, latitude = gm1$Nj, time = length(values_l))) + # Save memory + rm(values_l); gc() + + } else { + result[[dat_i]] <- .grib_expand_grids(memb_ftime) + } + + ################################################################## + # Get metadata + ################################################################## + ## (1-1) Everything from the first message of first file + if (dat_i == 1) { + ## (1-1) Everything from the first message of first file +# dims <- dim(result[[dat_i]]) +# attributes(result) <- gm1 +# # turn result into array again +# dim(result[[dat_i]]) <- dims + + ## (1-2) Only save the necessary attributes + attr(result, 'edition') <- gm1$edition + attr(result, 'shortName') <- gm1$shortName + #NOTE: Tune varaible name!! + if (gm1$shortName == '2t') attr(result, 'shortName') <- 'tas' + attr(result, 'name') <- gm1$name + attr(result, 'units') <- gm1$units +# attr(result, 'validityDate') <- gm1$validityDate +# attr(result, 'validityTime') <- gm1$validityTime + + ## (2) Lat and lon + latlon <- grib_latlons(gm1, expand = TRUE) + attr(result, 'latitude') <- unique(as.vector(c(latlon$lats))) + attr(result, 'longitude') <- unique(as.vector(c(latlon$lons))) + # Save memory (though it's small) + rm(latlon); gc() + + #NOTE: Find another way to check regular grid; Ni/Nj not always exist +# if (has.key(gm1, "Nx") && has.key(gm1, "Ny")) { +# nx <- gm1$Nx +# ny <- gm1$Ny +# } else { +# nx <- gm1$Ni +# ny <- gm1$Nj +# } +# if (length(lats) != ny | length(lons) != nx) { +# stop("Latitude and Longitude seem to be non-regular grid.") +# } + + } + +#-------------------------------- +#NOTE: Just use cdo_time +# ## (3) Date and time: Need to get from each massage +# for (time_i in 1:length(time_step)) { +# gm1 <- gm[[time_i]] +# #NOTE: What's the correct time? +## dates <- gm1$validityDate #Date of validity of the forecast +## times <- gm1$validityTime +## dates <- gm1$dataDate # Reference date +# times[[dat_i]][time_i] <- as.POSIXct( +# lubridate::ymd_hms(paste0(paste(gm1$year,gm1$month,gm1$day, '-'), ' ', +# paste(gm1$hour, gm1$minute, gm1$second, ':'))) +# ) +# } + times[[dat_i]] <- cdo_time_attr +#-------------------------------- + + ################################################################## + # regrid + ################################################################## + if (!is.null(regrid)) { + # result[[dat_i]]: [longitude, latitude, time] + res_data <- s2dv::CDORemap(result[[dat_i]], lons = attr(result, 'longitude'), lats = attr(result, 'latitude'), + grid = regrid$type, method = regrid$method, force_remap = TRUE) + if (dat_i == length(dat)) { + attr(result, 'longitude') <- res_data$lons + attr(result, 'latitude') <- res_data$lats + } + result[[dat_i]] <- res_data$data_array + } + + + ################################################################## + # Save memory + rm(memb_ftime); rm(gm1); gc() + grib_close(file_to_load) # Doesn't impact memory + ################################################################## +} #for loop dat + + # Turn result list into array + attr <- attributes(result) + res_dim <- c(dim(result[[1]]), syear = length(result)) #[longitude, latitude, (memb*)time, syear] + result <- unlist(result) + dim(result) <- res_dim + + # Generate date/time attributes + times <- array(unlist(times), dim = c(time = length(time_step), syear = length(dat), + sday = 1, sweek = 1)) + times <- s2dv::Reorder(times, c('sday', 'sweek', 'syear', 'time')) + if (!is.null(syear_time_dim)) dim(times) <- syear_time_dim + times <- as.POSIXct(times, origin = '1970-01-01', tz = 'UTC') + + # Reshape and reorder array + if (is.null(has.memb)) { # obs doesn't have memb; reshape syear/time dim + result <- s2dv::Reorder(result, c("syear", "time", "latitude", "longitude")) + result <- array(result, dim = c(dat = 1, var = 1, + syear_time_dim, dim(result)[3:4], + ensemble = 1)) + } else { + result <- array(result, dim = c(dim(result)[1:2], ensemble = has.memb, + time = length(time_step), dim(result)[4])) + result <- s2dv::Reorder(result, c("syear", "time", "latitude", "longitude", "ensemble")) + dim(result) <- c(dat = 1, var = 1, sday = 1, sweek = 1, dim(result)) + } + + # Add attributes back + attr$dim <- dim(result) + attributes(result) <- attr + attr(result, 'time') <- times + + # Save memory + rm(times); rm(attr); gc() + + return(result) +} + +######################################################################### +######################################################################### + +.grib_expand_grids <- function(gribMessages, vector = FALSE) { + # gribMessages is a list of multiple messages + gribMessage <- gribMessages[[1]] + + if (gribr::has.key(gribMessage, "Nx") && gribr::has.key(gribMessage, "Ny")) { + nx <- gribMessage$Nx + ny <- gribMessage$Ny + } else { + nx <- gribMessage$Ni + ny <- gribMessage$Nj + } + + if (is.null(nx) || is.null(ny)) { + stop("Unsupported grid type: ", gribMessage$gridType) + } + + if (grepl("reduced", gribMessage$gridType)) { + #TODO: This part is not used now. + nx <- ny * 2 + values <- .Call("gribr_redtoreg", nx, gribMessage$pl, + gribMessage$values) + values <- matrix(values, nx, ny, + byrow = gribMessage$jPointsAreConsecutive) + +# values_l <- vector('list', length = length(gribMessages)) +# for (gm_i in 1:length(gribMessages)) { +# values <- .Call("gribr_redtoreg", nx, gribMessages[[gm_i]]$pl, +# gribMessages[[gm_i]]$values) +# values <- matrix(values, nx, ny, +# byrow = gribMessage$jPointsAreConsecutive) +# values_l[[gm_i]] <- values +# } + + } else { +# values <- matrix(gribMessage$values, nx, ny, +# byrow = gribMessage$jPointsAreConsecutive) + values_l <- lapply(gribMessages, '[[', 'values') + values_l <- lapply(values_l, matrix, nx, ny, byrow = gribMessage$jPointsAreConsecutive) + values <- array(unlist(values_l), dim = c(longitude = nx, latitude = ny, time = length(values_l))) + } + + if (vector) { + values <- as.numeric(values) + } + + values +} + diff --git a/modules/Loading/R/GRIB/GRIB/s2dv_cube.R b/modules/Loading/R/GRIB/GRIB/s2dv_cube.R new file mode 100644 index 0000000000000000000000000000000000000000..d44b99e2e35760e34c069815717ed803cc2f465d --- /dev/null +++ b/modules/Loading/R/GRIB/GRIB/s2dv_cube.R @@ -0,0 +1,227 @@ +#'Creation of a 's2dv_cube' object +#' +#'@description This function allows to create an 's2dv_cube' object by passing +#'information through its parameters. This function will be needed if the data +#'hasn't been loaded using CST_Load or has been transformed with other methods. +#'An 's2dv_cube' object has many different components including metadata. This +#'function will allow to create 's2dv_cube' objects even if not all elements +#'are defined and for each expected missed parameter a warning message will be +#'returned. +#' +#'@author Perez-Zanon Nuria, \email{nuria.perez@bsc.es} +#' +#'@param data A multidimensional array with named dimensions, typically with +#' dimensions: dataset, member, sdate, ftime, lat and lon. +#'@param coords A named list with elements of the coordinates corresponding to +#' the dimensions of the data parameter. The names and length of each element +#' must correspond to the names of the dimensions. If any coordinate is not +#' provided, it is set as an index vector with the values from 1 to the length +#' of the corresponding dimension. +#'@param varName A character string indicating the abbreviation of the variable +#' name. +#'@param metadata A named list where each element is a variable containing the +#' corresponding information. The information can be contained in a list of +#' lists for each variable. +#'@param Datasets Character strings indicating the names of the dataset. It +#' there are multiple datasets it can be a vector of its names or a list of +#' lists with additional information. +#'@param Dates A POSIXct array of time dimensions containing the Dates. +#'@param when A time stamp of the date when the data has been loaded. This +#' parameter is also found in Load() and Start() functions output. +#'@param source_files A vector of character strings with complete paths to all +#' the found files involved in loading the data. +#'@param \dots Additional elements to be added in the object. They will be +#' stored in the end of 'attrs' element. Multiple elements are accepted. +#' +#'@return The function returns an object of class 's2dv_cube' with the following +#' elements in the structure:\cr +#'\itemize{ +#' \item{'data', array with named dimensions.} +#' \item{'dims', named vector of the data dimensions.} +#' \item{'coords', named list with elements of the coordinates corresponding to +#' the dimensions of the data parameter. If any coordinate is not provided, it +#' is set as an index vector with the values from 1 to the length of the +#' corresponding dimension. The attribute 'indices' indicates wether the +#' coordinate is an index vector (TRUE) or not (FALSE).} +#' \item{'attrs', named list with elements: +#' \itemize{ +#' \item{'Dates', array with named temporal dimensions of class 'POSIXct' from +#' time values in the data.} +#' \item{'Variable', has the following components: +#' \itemize{ +#' \item{'varName', with the short name of the loaded variable as specified +#' in the parameter 'var'.} +#' \item{''metadata', named list of elements with variable metadata. +#' They can be from coordinates variables (e.g. longitude) or +#' main variables (e.g. 'var').} +#' } +#' } +#' \item{'Datasets', character strings indicating the names of the dataset.} +#' \item{'source_files', a vector of character strings with complete paths to +#' all the found files involved in loading the data.} +#' \item{'when', a time stamp of the date issued by the Start() or Load() call to +#' obtain the data.} +#' \item{'load_parameters', it contains the components used in the arguments to +#' load the data from Start() or Load() functions.} +#' } +#' } +#'} +#' +#'@seealso \code{\link[s2dv]{Load}} and \code{\link{CST_Load}} +#'@examples +#'exp_original <- 1:100 +#'dim(exp_original) <- c(lat = 2, time = 10, lon = 5) +#'exp1 <- s2dv_cube(data = exp_original) +#'class(exp1) +#'coords <- list(lon = seq(-10, 10, 5), lat = c(45, 50)) +#'exp2 <- s2dv_cube(data = exp_original, coords = coords) +#'class(exp2) +#'metadata <- list(tas = list(level = '2m')) +#'exp3 <- s2dv_cube(data = exp_original, coords = coords, +#' varName = 'tas', metadata = metadata) +#'class(exp3) +#'Dates = as.POSIXct(paste0(rep("01", 10), rep("01", 10), 1990:1999), format = "%d%m%Y") +#'dim(Dates) <- c(time = 10) +#'exp4 <- s2dv_cube(data = exp_original, coords = coords, +#' varName = 'tas', metadata = metadata, +#' Dates = Dates) +#'class(exp4) +#'exp5 <- s2dv_cube(data = exp_original, coords = coords, +#' varName = 'tas', metadata = metadata, +#' Dates = Dates, when = "2019-10-23 19:15:29 CET") +#'class(exp5) +#'exp6 <- s2dv_cube(data = exp_original, coords = coords, +#' varName = 'tas', metadata = metadata, +#' Dates = Dates, +#' when = "2019-10-23 19:15:29 CET", +#' source_files = c("/path/to/file1.nc", "/path/to/file2.nc")) +#'class(exp6) +#'exp7 <- s2dv_cube(data = exp_original, coords = coords, +#' varName = 'tas', metadata = metadata, +#' Dates = Dates, +#' when = "2019-10-23 19:15:29 CET", +#' source_files = c("/path/to/file1.nc", "/path/to/file2.nc"), +#' Datasets = list( +#' exp1 = list(InitializationsDates = list(Member_1 = "01011990", +#' Members = "Member_1")))) +#'class(exp7) +#'dim(exp_original) <- c(dataset = 1, member = 1, time = 10, lat = 2, lon = 5) +#'exp8 <- s2dv_cube(data = exp_original, coords = coords, +#' varName = 'tas', metadata = metadata, +#' Dates = Dates, original_dates = Dates) +#'class(exp8) +#'@export +s2dv_cube <- function(data, coords = NULL, varName = NULL, metadata = NULL, + Datasets = NULL, Dates = NULL, when = NULL, + source_files = NULL, ...) { + + # data + if (is.null(data) | !is.array(data) | is.null(names(dim(data)))) { + stop("Parameter 'data' must be an array with named dimensions.") + } + # dims + dims <- dim(data) + + ## coords + if (!is.null(coords)) { + if (!all(names(coords) %in% names(dims))) { + coords <- coords[-which(!names(coords) %in% names(dims))] + } + for (i_coord in names(dims)) { + if (i_coord %in% names(coords)) { + if (length(coords[[i_coord]]) != dims[i_coord]) { + warning(paste0("Coordinate '", i_coord, "' has different lenght as ", + "its dimension and it will not be used.")) + coords[[i_coord]] <- 1:dims[i_coord] + attr(coords[[i_coord]], 'indices') <- TRUE + } else { + attr(coords[[i_coord]], 'indices') <- FALSE + } + } else { + warning(paste0("Coordinate '", i_coord, "' is not provided ", + "and it will be set as index in element coords.")) + coords[[i_coord]] <- 1:dims[i_coord] + attr(coords[[i_coord]], 'indices') <- TRUE + } + } + } else { + coords <- sapply(names(dims), function(x) 1:dims[x]) + for (i in 1:length(coords)) { + attr(coords[[i]], "indices") <- TRUE + } + } + + ## attrs + attrs <- list() + # Dates + if (is.null(Dates)) { + warning("Parameter 'Dates' is not provided so the metadata ", + "of 's2dv_cube' object will be incomplete.") + attrs$Dates <- NULL + } else if (length(Dates) == 1 & inherits(Dates[1], "POSIXct")) { + attrs$Dates <- Dates + } else { + if (!is.array(Dates)) { + warning("Parameter 'Dates' must be an array with named time dimensions.") + } else { + if (is.null(names(dim(Dates)))) { + warning("Parameter 'Dates' must have dimension names.") + } else if (!all(names(dim(Dates)) %in% names(dims))) { + warning("Parameter 'Dates' must have the corresponding time dimension names in 'data'.") + } else { + if (inherits(Dates[1], "POSIXct")) { + attrs$Dates <- Dates + } else { + warning("Parameter 'Dates' must be of class 'POSIXct'.") + } + } + } + } + # Variable + if (is.null(varName)) { + warning("Parameter 'varName' is not provided so the metadata ", + "of 's2dv_cube' object will be incomplete.") + attrs$Variable$varName <- NULL + } else { + if (!is.character(varName)) { + warning("Parameter 'varName' must be a character.") + } else { + attrs$Variable$varName <- varName + } + } + if (is.null(metadata)) { + warning("Parameter 'metadata' is not provided so the metadata ", + "of 's2dv_cube' object will be incomplete.") + attrs$Variable$metadata <- NULL + } else { + if (!is.list(metadata)) { + metadata <- list(metadata) + } + attrs$Variable$metadata <- metadata + } + # Datasets + if (!is.null(Datasets)) { + attrs$Datasets <- Datasets + } + # when + if (!is.null(when)) { + attrs$when <- when + } + # source_files + if (!is.null(source_files)) { + attrs$source_files <- source_files + } + # dots + dots <- list(...) + if (length(dots) != 0) { + for (i_arg in 1:length(dots)) { + attrs[[names(dots)[[i_arg]]]] <- dots[[i_arg]] + } + } + + ## object + object <- list(data = data, dims = dims, coords = coords, attrs = attrs) + class(object) <- 's2dv_cube' + return(object) +} + diff --git a/modules/Loading/R/load_GRIB.R b/modules/Loading/R/load_GRIB.R new file mode 100644 index 0000000000000000000000000000000000000000..8ae2a74d2efb2c3e5f82034ee1f647b6548620d7 --- /dev/null +++ b/modules/Loading/R/load_GRIB.R @@ -0,0 +1,279 @@ +################################################# +# Load GRIB files from MARS +################################################# + +source('modules/Loading/R/GRIB/GrbLoad.R') +source('tools/libs.R') + +load_GRIB <- function(recipe) { + + # Set params + #------------------------------------------------------------------- + + # get recipe info + hcst.inityear <- recipe$Analysis$Time$hcst_start + hcst.endyear <- recipe$Analysis$Time$hcst_end + hcst.sdate <- recipe$Analysis$Time$sdate + hcst.ftime <- as.numeric(recipe$Analysis$Time$ftime_min):as.numeric(recipe$Analysis$Time$ftime_max) + fcst.year <- recipe$Analysis$Time$fcst_year + lats.min <- recipe$Analysis$Region$latmin # can only be -90 + lats.max <- recipe$Analysis$Region$latmax # can only be 90 + lons.min <- recipe$Analysis$Region$lonmin # can only be 0 + lons.max <- recipe$Analysis$Region$lonmax # can only be 360 + ref.name <- recipe$Analysis$Datasets$Reference$name + exp.name <- recipe$Analysis$Datasets$System$name + variable <- recipe$Analysis$Variables$name #'tas' + store.freq <- recipe$Analysis$Variables$freq + + regrid.method <- recipe$Analysis$Regrid$method + regrid.type <- recipe$Analysis$Regrid$type + + # get MARS datasets dict: + archive <- read_yaml("conf/archive.yml")[[recipe$Run$filesystem]] + exp_descrip <- archive$System[[exp.name]] + freq.hcst <- unlist(exp_descrip[[store.freq]][variable]) + reference_descrip <- archive$Reference[[ref.name]] + freq.obs <- unlist(reference_descrip[[store.freq]][variable]) + obs.dir <- reference_descrip$src + fcst.dir <- exp_descrip$src + hcst.dir <- exp_descrip$src + #NOTE: We can use this info in GrbLoad() to substitute param 'has.memb' + fcst.nmember <- exp_descrip$nmember$fcst + hcst.nmember <- exp_descrip$nmember$hcst + + info(recipe$Run$logger, + "========== PARAMETERS RETRIEVED. ==========") + + # Load hindcast + #------------------------------------------------------------------- + +## original file dir +#exp_path <- "/esarchive/exp/ecmwf/system5_m1/original_files/fcmean_od_sfc_msmm_ecmf/" +## soft link to original file dir +#exp_path <- "/esarchive/scratch/aho/tmp/GRIB/GRIB_system5_tas/" #files are not correct +# The correct files +#exp_path <- "/esarchive/scratch/aho/tmp/GRIB/GRIB_system5_tas_CORRECTED/" + + hcst.path <- paste0(archive$src, hcst.dir) + hcst.year <- paste0(as.numeric(hcst.inityear):as.numeric(hcst.endyear)) + hcst.files <- paste0(hcst.path, variable, '_', hcst.year, hcst.sdate, '.grb') + + if (!regrid.type %in% c('none', 'to_system')) { + if (regrid.type == 'to_reference') { + regrid_list <- c(method = regrid.method, type = reference_descrip$reference_grid) + } else { # e.g., "r360x181" + regrid_list <- list(method = regrid.method, type = regrid.type) + } + } else { + regrid_list <- NULL + } + + .log_memory_usage(recipe$Run$logger, when = "Before loading the data") + hcst <- GrbLoad(dat = as.list(hcst.files), time_step = hcst.ftime, has.memb = hcst.nmember, + syear_time_dim = NULL, regrid = regrid_list) + gc() + + info(recipe$Run$logger, + "========== HCST LOADED. ==========") + + # Load forecast + #------------------------------------------------------------------- + if (!is.null(fcst.year)) { + fcst.path <- paste0(archive$src, hcst.dir) + fcst.files <- paste0(fcst.path, variable, '_', fcst.year, hcst.sdate, '.grb') + fcst <- GrbLoad(dat = as.list(fcst.files), time_step = hcst.ftime, has.memb = fcst.nmember, + syear_time_dim = NULL, regrid = regrid_list) + gc() + } else { + fcst <- NULL + } + + info(recipe$Run$logger, + "========== FCST LOADED. ==========") + + # Load reference + #------------------------------------------------------------------- +#obs_path <- "/esarchive/scratch/aho/tmp/GRIB/GRIB_era5_tas/" + obs.path <- paste0(archive$src, obs.dir) + # Use hcst time attr to load obs + hcst_times <- attr(hcst, 'time') + hcst_times_strings <- format(hcst_times, '%Y%m') + + obs.files <- paste0(obs.path, variable, '_', hcst_times_strings, '.grb') + + if (!regrid.type %in% c('none', 'to_reference')) { + if (regrid.type == 'to_system') { + regrid_list <- c(method = regrid.method, type = exp_descrip$reference_grid) + } else { # e.g., "r360x181" + regrid_list <- list(method = regrid.method, type = regrid.type) + } + } else { + regrid_list <- NULL + } + + #NOTE: only 1 time step in each obs file + obs <- GrbLoad(dat = as.list(obs.files), time_step = 1, has.memb = NULL, + syear_time_dim = dim(hcst_times), regrid = regrid_list) + gc() + + .log_memory_usage(recipe$Run$logger, when = "After loading the data") + info(recipe$Run$logger, + "========== OBS LOADED. ==========") + + +################################################################################# + +#dim(hcst) +# syear time latitude longitude ensemble +# 4 3 640 1280 51 + +##BEFORE TRANSFER TO S2DV_CUBE +#str(hcst) +# num [1:4, 1:3, 1:640, 1:1280, 1:51] 252 252 252 252 251 ... +# - attr(*, "edition")= num 1 +# - attr(*, "shortName")= chr "2t" +# - attr(*, "longitude")= num [1:1280] 0 0.281 0.563 0.844 1.125 ... +# - attr(*, "latitude")= num [1:640] 89.8 89.5 89.2 88.9 88.7 ... +# - attr(*, "time")= POSIXct[1:12], format: "2000-12-01" "2001-12-01" ... + +#dim(attr(hcst, 'time')) +#syear time +# 4 3 + +##BEFORE TRANSFER TO S2DV_CUBE +#str(obs) +# num [1:4, 1:3, 1:640, 1:1280] 251 251 251 251 251 ... +# - attr(*, "edition")= num 1 +# - attr(*, "shortName")= chr "2t" +# - attr(*, "longitude")= num [1:1280] 0 0.281 0.562 0.844 1.125 ... +# - attr(*, "latitude")= num [1:640] 89.8 89.5 89.2 88.9 88.7 ... +# - attr(*, "time")= POSIXct[1:12], format: "2000-12-01" "2001-12-01" ... + +################################################################################# + + info(recipe$Run$logger, + "========== REGRID DONE. ==========") + + + # Turn into s2dv_cube + #------------------------------------------------------------------- + # hcst + metadata_list <- vector("list", length = 1) + names(metadata_list) <- variable + metadata_list[[variable]] <- list(long_name = attr(hcst, 'name'), + units = attr(hcst, 'units')) + load_parameters_list <- list(dat1 = list(file_date = list(paste0(hcst.year, hcst.sdate)))) + + hcst <- s2dv_cube(data = array(hcst, dim = dim(hcst)), + coords = list(dat = 'dat1', + var = variable, + sday = 1, + sweek = 1, + syear = hcst.year, + time = hcst.ftime, + latitude = attr(hcst, 'latitude'), + longitude = attr(hcst, 'longitude'), + ensemble = 1:hcst.nmember), + varName = attr(hcst, 'shortName'), + metadata = metadata_list, + Dates = attributes(hcst)$time, + source_files = hcst.files, + load_parameters = load_parameters_list, + # extra attrs + gribEdition = attr(hcst, 'edition')) + + # fcst + if (!is.null(fcst)) { + metadata_list <- vector("list", length = 1) + names(metadata_list) <- variable + metadata_list[[variable]] <- list(long_name = attr(fcst, 'name'), + units = attr(fcst, 'units')) + load_parameters_list <- list(dat1 = list(file_date = list(paste0(fcst.year, hcst.sdate)))) + + fcst <- s2dv_cube(data = array(fcst, dim = dim(fcst)), + coords = list(dat = 'dat1', + var = variable, + sday = 1, + sweek = 1, + syear = fcst.year, + time = hcst.ftime, + latitude = attr(fcst, 'latitude'), + longitude = attr(fcst, 'longitude'), + ensemble = 1:fcst.nmember), + varName = attr(fcst, 'shortName'), + metadata = metadata_list, + Dates = attributes(fcst)$time, + source_files = fcst.files, + load_parameters = load_parameters_list, + gribEdition = attr(fcst, 'edition')) + } + + # obs + metadata_list <- vector("list", length = 1) + names(metadata_list) <- variable + metadata_list[[variable]] <- list(long_name = attr(obs, 'name'), + units = attr(obs, 'units')) + load_parameters_list <- list(dat1 = list(file_date = list(hcst_times_strings))) + + obs <- s2dv_cube(data = array(obs, dim = dim(obs)), + coords = list(dat = 'dat1', + var = variable, + sday = 1, + sweek = 1, + #NOTE: Can we directly use hcst info? + syear = hcst.year, + time = hcst.ftime, + latitude = attr(obs, 'latitude'), + longitude = attr(obs, 'longitude'), + ensemble = 1), + varName = attr(obs, 'shortName'), + metadata = metadata_list, + Dates = attributes(obs)$time, + source_files = obs.files, + load_parameters = load_parameters_list, + gribEdition = attr(obs, 'edition')) + + +#str(hcst) +#List of 4 +# $ data : num [1, 1, 1, 1, 1:2, 1:2, 1:640, 1:1280, 1:51] 252 253 248 251 251 ... +# ..- attr(*, "edition")= num 1 +# ..- attr(*, "shortName")= chr "2t" +# ..- attr(*, "longitude")= num [1:1280] 0 0.281 0.563 0.844 1.125 ... +# ..- attr(*, "latitude")= num [1:640] 89.8 89.5 89.2 88.9 88.7 ... +# ..- attr(*, "time")= POSIXct[1:4], format: "2000-12-01" "2001-12-01" ... +# $ dims : Named int [1:9] 1 1 1 1 2 2 640 1280 51 +# ..- attr(*, "names")= chr [1:9] "dat" "var" "sday" "sweek" ... +# $ coords:List of 9 +# ..$ dat : chr "dat1" +# .. ..- attr(*, "indices")= logi FALSE +# ..$ var : chr "tas" +# .. ..- attr(*, "indices")= logi FALSE +# ..$ sday : num 1 +# .. ..- attr(*, "indices")= logi FALSE +# ..$ sweek : num 1 +# .. ..- attr(*, "indices")= logi FALSE +# ..$ syear : chr [1:2] "2000" "2001" +# .. ..- attr(*, "indices")= logi FALSE +# ..$ time : int [1:2] 1 2 +# .. ..- attr(*, "indices")= logi FALSE +# ..$ latitude : num [1:640] 89.8 89.5 89.2 88.9 88.7 ... +# .. ..- attr(*, "indices")= logi FALSE +# ..$ longitude: num [1:1280] 0 0.281 0.563 0.844 1.125 ... +# .. ..- attr(*, "indices")= logi FALSE +# ..$ ensemble : int [1:51] 1 2 3 4 5 6 7 8 9 10 ... +# .. ..- attr(*, "indices")= logi FALSE +# $ attrs :List of 4 +# ..$ Dates : POSIXct[1:4], format: "2000-12-01" "2001-12-01" ... +# ..$ Variable :List of 1 +# .. ..$ varName: chr "2t" +# ..$ source_files: chr [1:2] "/esarchive/scratch/aho/tmp/GRIB/GRIB_system5_tas_CORRECTED/tas_20001101.grb" "/esarchive/scratch/aho/tmp/GRIB/GRIB_system5_tas_CORRECTED/tas_20011101.grb" +# ..$ gribEdition : num 1 +# - attr(*, "class")= chr "s2dv_cube" + .log_memory_usage(recipe$Run$logger, when = "After regridding") + info(recipe$Run$logger, + "##### GRIB DATA LOADED SUCCESSFULLY #####") + + return(list(hcst = hcst, fcst = fcst, obs = obs)) + +} diff --git a/modules/Loading/R/load_decadal.R b/modules/Loading/R/load_decadal.R index ba4e1386571c83563bf7453367c8f0b5476bbce8..15e5bbc88a36ae756066b6db7b8b5d191a1233cc 100644 --- a/modules/Loading/R/load_decadal.R +++ b/modules/Loading/R/load_decadal.R @@ -547,6 +547,6 @@ load_decadal <- function(recipe) { info(recipe$Run$logger, "##### DATA LOADING COMPLETED SUCCESSFULLY #####") - + .log_memory_usage(recipe$Run$logger, when = "After loading") return(list(hcst = hcst, fcst = fcst, obs = obs)) } diff --git a/modules/Loading/R/load_seasonal.R b/modules/Loading/R/load_seasonal.R index 12a19c6ce7a2b5a77506928c2fc39a219a9aa3c1..4e0c5387c39959409cb93516375eef390b23003a 100644 --- a/modules/Loading/R/load_seasonal.R +++ b/modules/Loading/R/load_seasonal.R @@ -444,7 +444,7 @@ load_seasonal <- function(recipe) { ############################################################################ ############################################################################ - + .log_memory_usage(recipe$Run$logger, when = "After loading") return(list(hcst = hcst, fcst = fcst, obs = obs)) } diff --git a/modules/Scorecards/R/tmp/LoadMetrics.R b/modules/Scorecards/R/tmp/LoadMetrics.R new file mode 100644 index 0000000000000000000000000000000000000000..e5e154213377be7ec82100507a414d93e091d0b6 --- /dev/null +++ b/modules/Scorecards/R/tmp/LoadMetrics.R @@ -0,0 +1,215 @@ +#' Scorecards load metrics from verification suite +#' +#'@description Scorecards function to load saved data files +#' +#'@param system A vector of character strings defining the names of the +#' system names following the archive.yml format from verification suite. +#' Accepted system names: 'ECMWF-SEAS5', 'DWD-GFCS2.1', 'CMCC-SPS3.5', +#' 'ecmwfs5','Meteo-France-System 7', 'UK-MetOffice-GloSea600', 'NCEP-CFSv2'. +#'@param reference A vector of character strings defining the names of +#' the references following the archive.yml format from verification suite +#' Pending to be test with more than one. The accepted names are: 'era5'. +#'@param var A character string following the format from +#' variable-dictionary.yml from verification suite (TO DO: multiple variables). +#' The accepted names are: 'psl', 'tas', 'sfcWind', 'prlr'. +#'@param start.year A numeric indicating the start year of the reference period +#'@param end.year A numeric indicating the end year of the reference period +#'@param start.months A vector indicating the numbers of the start months +#'@param forecast.months A vector indicating the numbers of the forecast months +#'@param input.path A character string indicating the path where metrics output +#' files from verification suite are saved (or any other compatible files) +#' +#'@return A is a list by system and reference containing an array of with +#' the following dimensions: longitude, latitude, forecast months, metrics, +#' start dates. + +#'@examples +#'\dontrun{ +#'loaded_metrics <- LoadMetrics(system = c('ECMWF-SEAS5','DWD-GFCS2.1'), +#' reference. = 'ERA5', +#' var = 'tas', +#' start.year = 1993, +#' end.year = 2016, +#' metrics = c('mean_bias', 'enscorr', 'rpss', 'crpss', 'enssprerr'), +#' start.months = sprintf("%02d", 1:12), +#' forecast.months = 1:6, +#' input.path = '/esarchive/scratch/nmilders/scorecards_data/input_data') +#'} +#'@import easyNCDF +#'@import multiApply +#'@export +LoadMetrics <- function(system, reference, var, start.year, end.year, + metrics, start.months, forecast.months, + inf_to_na = FALSE, + input.path) { + + # Initial checks + ## system + if (!is.character(system)) { + stop("Parameter 'system' must be a character vector with the system names.") + } + ## reference + if (!is.character(reference)) { + stop("Parameter 'reference' must be a character vector with the reference ", + "names.") + } + ## var + if (!is.character(var)) { + stop("Parameter 'var' must be a character vector with the var ", + "names.") + } + if (length(var) > 1) { + warning("Parameter 'var' must be of length one. Only the first value ", + "will be used.") + var <- var[1] + } + ## start.year + if (!is.numeric(start.year)) { + stop("Parameter 'start.year' must be a numeric value.") + } + ## end.year + if (!is.numeric(end.year)) { + stop("Parameter 'end.year' must be a numeric value.") + } + ## metrics + if (!is.character(metrics)) { + stop("Parameter 'metrics' cannot be NULL.") + } + ## start.months + if (is.character(start.months)) { + warning("Parameter 'start.months' must be a numeric vector indicating ", + "the starting months.") + start.months <- as.numeric(start.months) + } + if (!is.numeric(start.months)) { + stop("Parameter 'start.months' must be a numeric vector indicating ", + "the starting months.") + } + start.months <- sprintf("%02d", start.months) + ## Check if sdates are continuous or discrete + if (all(diff(as.numeric(start.months)) == 1)) { + consecutive_start.months <- TRUE + } else { + consecutive_start.months <- FALSE + } + ## forecast.months + if (!is.numeric(forecast.months)) { + stop("Parameter 'forecast.months' must be a numeric vector indicating ", + "the starting months.") + } + ## input.path + if (!is.character(input.path)) { + stop("Parameter 'input.path must be a character string.") + } + if (length(input.path) > 1) { + input.path <- input.path[1] + warning("Parameter 'input.path' has length greater than 1 and only the ", + "first element will be used.") + } + + ## Remove . from names + system <- gsub('.','', system, fixed = T) + reference <- gsub('.','', reference, fixed = T) + + period <- paste0(start.year, "-", end.year) + + ## Define empty list to saved data + all_metrics <- sapply(system, function(x) NULL) + ## Load data for each system + for (sys in 1:length(system)) { + ## Define empty list to saved data + by_reference <- sapply(reference, function(x) NULL) + ## Load data for each reference + for (ref in 1:length(reference)) { + ## Call function to load metrics data + met <- .Loadmetrics(input.path = input.path, # recipe$Run$output, + system = system[sys], + reference = reference[ref], + var = var, + period = period, + start.months = start.months, + forecast.months = forecast.months, + metrics = metrics) + ## Save metric data as array in reference list + by_reference[[reference[ref]]] <- met + ## Remove -Inf from crpss data if variable is precipitation + if (inf_to_na) { + by_reference[[reference]][by_reference[[reference]]==-Inf] <- NA + } + } ## close loop on reference + ## Save reference data in list of system + all_metrics[[system[sys]]] <- by_reference + } ## close loop on system + + return(all_metrics) +} ## close function + +############################################################ + +.Loadmetrics <- function(input.path, system, reference, + var, period, start.months, + forecast.months, metrics) { + + ## Load data for each start date + allfiles <- sapply(start.months, function(m) { + paste0(input.path, "/", system, "/", var, + "/scorecards_", system, "_", reference, "_", + var, "-skill_", period, "_s", m, # mod.pressure, + ".nc")}) + allfiles_exist <- sapply(allfiles, file.exists) + + # Check dims + files_exist_by_month <- seq(1:length(allfiles))[allfiles_exist] + allfiledims <- sapply(allfiles[allfiles_exist], easyNCDF::NcReadDims) + if (length(files_exist_by_month) == 0) { + stop("No files are found.") + } + + num_dims <- numeric(dim(allfiledims)[1]) + for (i in 1:dim(allfiledims)[1]) { + if (length(unique(allfiledims[i,])) > 1) { + warning(paste0("Dimensions of system ", system," with var ", var, + " don't match.")) + } + num_dims[i] <- max(allfiledims[i,]) # We take the largest dimension + } + # dims: [metric, longitude, latitude, time, smonth] + # or [metric, region, time, smonth] + + # Loop for file + dim(allfiles) <- c(dat = 1, sdate = length(allfiles)) + + array_met_by_sdate <- Apply(data = allfiles, target_dims = 'dat', fun = function(x) { + if (file.exists(x)) { + res <- easyNCDF::NcToArray(x, vars_to_read = metrics, unlist = T, + drop_var_dim = T) + names(dim(res)) <- NULL + } else { + res <- array(dim = c(length(metrics), allfiledims[-1,1])) + names(dim(res)) <- NULL + } + res})$output1 + + dim(array_met_by_sdate) <- c(metric = length(metrics), allfiledims[-1,1], + sdate = length(allfiles)) + + # Attributes + # Read attributes from the first existing file + if ("region" %in% rownames(allfiledims)) { + file_for_att <- ncdf4::nc_open(allfiles[allfiles_exist[1]]) + region <- ncdf4::ncatt_get(file_for_att, 'region') + ncdf4::nc_close(file_for_att) + attributes(array_met_by_sdate)$region <- region + } else { + lon <- easyNCDF::NcToArray(allfiles[allfiles_exist][1], vars_to_read = 'longitude', + unlist = T, drop_var_dim = T) + lat <- easyNCDF::NcToArray(allfiles[allfiles_exist][1], vars_to_read = 'latitude', + unlist = T, drop_var_dim = T) + attributes(array_met_by_sdate)$lon <- lon + attributes(array_met_by_sdate)$lat <- lat + } + attributes(array_met_by_sdate)$metrics <- metrics + attributes(array_met_by_sdate)$start.months <- start.months + attributes(array_met_by_sdate)$forecast.months <- forecast.months + return(array_met_by_sdate) +} diff --git a/modules/Scorecards/R/tmp/SCPlotScorecard.R b/modules/Scorecards/R/tmp/SCPlotScorecard.R new file mode 100644 index 0000000000000000000000000000000000000000..4373057b6e0d3901abcb3fc27c09006f5156131d --- /dev/null +++ b/modules/Scorecards/R/tmp/SCPlotScorecard.R @@ -0,0 +1,444 @@ +#'Scorecards function create simple scorecards by region (types 1 & 3) +#' +#'@description This function creates a scorecard for a single system and +#'reference combination, showing data by region and forecast month. +#' +#'@param data A multidimensional array containing spatially aggregated metrics +#' data with dimensions: metric, region, sdate and ftime. +#'@param row.dim A character string indicating the dimension name to show in the +#' rows of the plot. +#'@param subrow.dim A character string indicating the dimension name to show in +#' the sub-rows of the plot. +#'@param col.dim A character string indicating the dimension name to show in the +#' columns of the plot. +#'@param subcol.dim A character string indicating the dimension name to show in +#' the sub-columns of the plot. +#'@param legend.dim A character string indicating the dimension name to use for +#' the legend. +#'@param row.names A vector of character strings with row display names. +#'@param subrow.names A vector of character strings with sub-row display names. +#'@param col.names A vector of character strings with column display names. +#'@param subcol.names A vector of character strings with sub-column display +#' names. +#'@param row.title A character string for the title of the row names. +#'@param subrow.title A character string for the title of the sub-row names. +#'@param table.title A character string for the title of the plot. +#'@param table.subtitle A character string for the sub-title of the plot. +#'@param legend.breaks A vector of numerics or a list of vectors of numerics, +#' containing the breaks for the legends. If a vector is given as input, then +#' these breaks will be repeated for each legend.dim. A list of vectors can be +#' given as input if the legend.dims require different breaks. This parameter +#' is required even if the legend is not plotted, to define the colors in the +#' scorecard table. +#'@param plot.legend A logical value to determine if the legend is plotted. +#'@param legend.width A numeric value to define the width of the legend bars. +#'@param legend.height A numeric value to define the height of the legend bars. +#'@param palette A vector of character strings or a list of vectors of +#' character strings containing the colors to use in the legends. If a vector +#' is given as input, then these colors will be used for each legend.dim. A +#' list of vectors can be given as input if different colors are desired for +#' the legend.dims. This parameter must be included even if the the legend is +#' not plotted, to define the colors in the scorecard table. +#'@param colorunder A character string or of vector of character strings +#' defining the colors to use for data values with are inferior to the lowest +#' breaks value. This parameter will also plot a inferior triangle in the +#' legend bar. The parameter can be set to NULL if there are no inferior values. +#' If a character string is given this color will be applied to all legend.dims. +#'@param colorsup A character string or of vector of character strings +#' defining the colors to use for data values with are superior to the highest +#' breaks value. This parameter will also plot a inferior triangle in the +#' legend bar. The parameter can be set to NULL if there are no superior values. +#' If a character string is given this color will be applied to all legend.dims. +#'@param round.decimal A numeric indicating to which decimal point the data +#' is to be displayed in the scorecard table. +#'@param font.size A numeric indicating the font size on the scorecard table. +#'@param fileout A path of the location to save the scorecard plots. +#' +#'@return An image file containing the scorecard. +#'@example +#'data <- array(rnorm(1000), dim = c('sdate' = 12, 'metric' = 4, 'region' = 3, +#' 'time' = 6)) +#'row.names <- c('Tropics', 'Extra-tropical NH', 'Extra-tropical SH') +#'col.names <- c('Mean bias (K)', 'Correlation', 'RPSS','CRPSS') +#'SCPlotScorecard(data = data, row.names = row.names, col.names = col.names, +#' subcol.names = month.abb[as.numeric(1:12)], +#' row.title = 'Region', subrow.title = 'Forecast Month', +#' col.title = 'Start date', +#' table.title = "Temperature of ECMWF System 5", +#' table.subtitle = "(Ref: ERA5 1994-2016)", +#' fileout = 'test.png') +#' +#'@import kableExtra +#'@import s2dv +#'@import ClimProjDiags +#'@export +SCPlotScorecard <- function(data, row.dim = 'region', subrow.dim = 'time', + col.dim = 'metric', subcol.dim = 'sdate', + legend.dim = 'metric', row.names = NULL, + subrow.names = NULL, col.names = NULL, + subcol.names = NULL, row.title = NULL, + subrow.title = NULL, col.title = NULL, + table.title = NULL, table.subtitle = NULL, + legend.breaks = NULL, plot.legend = TRUE, + label.scale = NULL, legend.width = NULL, + legend.height = NULL, palette = NULL, + colorunder = NULL, colorsup = NULL, + round.decimal = 2, font.size = 1.1, + legend.white.space = NULL, + col1.width = NULL, col2.width = NULL, + fileout = './scorecard.png') { + # Input parameter checks + ## Check data + if (!is.array(data)) { + stop("Parameter 'data' must be a numeric array.") + } + ## Check row.dim + if (!is.character(row.dim)) { + stop("Parameter 'row.dim' must be a character string.") + } + if (!row.dim %in% names(dim(data))) { + stop("Parameter 'row.dim' is not found in 'data' dimensions.") + } + ## Check row.names + if (!is.null(row.names)) { + if (length(row.names) != as.numeric(dim(data)[row.dim])) { + stop("Parameter 'row.names' must have the same length of dimension 'row.dims'.") + } + } else { + row.names <- as.character(1:dim(data)[row.dim]) + } + ## Check subrow.dim + if (!is.character(subrow.dim)) { + stop("Parameter 'subrow.dim' must be a character string.") + } + if (!subrow.dim %in% names(dim(data))) { + stop("Parameter 'subrow.dim' is not found in 'data' dimensions.") + } + ## Check subrow.names + if (!is.null(subrow.names)) { + if (length(subrow.names) != as.numeric(dim(data)[subrow.dim])) { + stop("Parameter 'subrow.names' must have the same length of dimension 'subrow.dims'.") + } + } else { + subrow.names <- as.character(1:dim(data)[subrow.dim]) + } + ## Check col.dim + if (!is.character(col.dim)) { + stop("Parameter 'col.dim' must be a character string.") + } + if (!col.dim %in% names(dim(data))) { + stop("Parameter 'col.dim' is not found in 'data' dimensions.") + } + ## Check col.names + if (!is.null(col.names)) { + if (length(col.names) != as.numeric(dim(data)[col.dim])) { + stop("Parameter 'col.names' must have the same length of dimension 'col.dims'.") + } + } else { + col.names <- as.character(1:dim(data)[col.dim]) + } + ## Check subcol.dim + if (!is.character(subcol.dim)) { + stop("Parameter 'subcol.dim' must be a character string.") + } + if (!subcol.dim %in% names(dim(data))) { + stop("Parameter 'subcol.dim' is not found in 'data' dimensions.") + } + ## Check subcol.names + if (!is.null(subcol.names)) { + if (length(subcol.names) != as.numeric(dim(data)[subcol.dim])) { + stop("Parameter 'subcol.names' must have the same length of dimension 'subcol.dims'.") + } + } else { + subcol.names <- as.character(1:dim(data)[subcol.dim]) + } + ## Check legend.dim + if (!is.character(legend.dim)) { + stop("Parameter 'legend.dim' must be a character string.") + } + if (!legend.dim %in% names(dim(data))) { + stop("Parameter 'legend.dim' is not found in 'data' dimensions.") + } + ## Check row.title inputs + if (!is.null(row.title)) { + if (!is.character(row.title)) { + stop("Parameter 'row.title must be a character string.") + } + } else { + row.title <- "" + } + ## Check subrow.title + if (!is.null(subrow.title)) { + if (!is.character(subrow.title)) { + stop("Parameter 'subrow.title must be a character string.") + } + } else { + subrow.title <- "" + } + ## Check col.title + if (!is.null(col.title)) { + if (!is.character(col.title)) { + stop("Parameter 'col.title must be a character string.") + } + } else { + col.title <- "" + } + ## Check table.title + if (!is.null(table.title)) { + if (!is.character(table.title)) { + stop("Parameter 'table.title' must be a character string.") + } + } else { + table.title <- "" + } + ## Check table.subtitle + if (!is.null(table.subtitle)) { + if (!is.character(table.subtitle)) { + stop("Parameter 'table.subtitle' must be a character string.") + } + } else { + table.subtitle <- "" + } + # Check legend.breaks + if (is.vector(legend.breaks) && is.numeric(legend.breaks)) { + legend.breaks <- rep(list(legend.breaks), as.numeric(dim(data)[legend.dim])) + } else if (is.null(legend.breaks)) { + legend.breaks <- rep(list(seq(-1, 1, 0.2)), as.numeric(dim(data)[legend.dim])) + } else if (inherits(legend.breaks, 'list')) { + stopifnot(length(legend.breaks) == as.numeric(dim(data)[legend.dim])) + } else { + stop("Parameter 'legend.breaks' must be a numeric vector, a list or NULL.") + } + ## Check plot.legend + if (!inherits(plot.legend, 'logical')) { + stop("Parameter 'plot.legend' must be a logical value.") + } + ## Check label.scale + if (is.null(label.scale)) { + label.scale <- 1.4 + } else { + if (!is.numeric(label.scale) | length(label.scale) != 1) { + stop("Parameter 'label.scale' must be a numeric value of length 1.") + } + } + ## Check legend.width + if (is.null(legend.width)) { + legend.width <- length(subcol.names) * 46.5 + } else { + if (!is.numeric(legend.width) | length(legend.width) != 1) { + stop("Parameter 'legend.width' must be a numeric value of length 1.") + } + } + if (is.null(legend.height)) { + legend.height <- 50 + } else { + if (!is.numeric(legend.height) | length(legend.height) != 1) { + stop("Parameter 'legend.height' must be a numeric value of length 1.") + } + } + ## Check colour palette input + if (is.vector(palette)) { + palette <- rep(list(palette), as.numeric(dim(data)[legend.dim])) + } else if (is.null(palette)) { + palette <- rep(list(c('#2D004B', '#542789', '#8073AC', '#B2ABD2', '#D8DAEB', + '#FEE0B6', '#FDB863', '#E08214', '#B35806', '#7F3B08')), + as.numeric(dim(data)[legend.dim])) + } else if (inherits(palette, 'list')) { + stopifnot(length(palette) == as.numeric(dim(data)[legend.dim])) + } else { + stop("Parameter 'palette' must be a numeric vector, a list or NULL.") + } + ## Check colorunder + if (is.null(colorunder)) { + colorunder <- rep("#04040E",as.numeric(dim(data)[legend.dim])) + } else if (is.character(colorunder) && length(colorunder) == 1) { + colorunder <- rep(colorunder, as.numeric(dim(data)[legend.dim])) + } else if (is.character(colorunder) && + length(colorunder) != as.numeric(dim(data)[legend.dim])) { + stop("Parameter 'colorunder' must be a numeric vector, a list or NULL.") + } + ## Check colorsup + if (is.null(colorsup)) { + colorsup <- rep("#730C04", as.numeric(dim(data)[legend.dim])) + } else if (is.character(colorsup) && length(colorsup) == 1) { + colorsup <- rep(colorsup,as.numeric(dim(data)[legend.dim])) + } else if (is.character(colorsup) && + length(colorsup) != as.numeric(dim(data)[legend.dim])) { + stop("Parameter 'colorsup' must be a numeric vector, a list or NULL.") + } + ## Check round.decimal + if (is.null(round.decimal)) { + round.decimal <- 2 + } else if (!is.numeric(round.decimal) | length(round.decimal) != 1) { + stop("Parameter 'round.decimal' must be a numeric value of length 1.") + } + ## Check font.size + if (is.null(font.size)) { + font.size <- 1 + } else if (!is.numeric(font.size) | length(font.size) != 1) { + stop("Parameter 'font.size' must be a numeric value of length 1.") + } + ## Check legend white space + if (is.null(legend.white.space)){ + legend.white.space <- 6 + } else { + legend.white.space <- legend.white.space + } + ## Check col1.width + if (is.null(col1.width)) { + if (max(nchar(row.names)) == 1 ) { + col1.width <- max(nchar(row.names)) + } else { + col1.width <- max(nchar(row.names))/4 + } + } else if (!is.numeric(col1.width)) { + stop("Parameter 'col1.width' must be a numeric value of length 1.") + } + ## Check col2.width + if (is.null(col2.width)) { + if (max(nchar(subrow.names)) == 1 ) { + col2.width <- max(nchar(subrow.names)) + } else { + col2.width <- max(nchar(subrow.names))/4 + } + } else if (!is.numeric(col2.width)) { + stop("Parameter 'col2.width' must be a numeric value of length 1.") + } + + + # Get dimensions of inputs + n.col.names <- length(col.names) + n.subcol.names <- length(subcol.names) + n.row.names <- length(row.names) + n.subrow.names <- length(subrow.names) + + # Define table size + n.rows <- n.row.names * n.subrow.names + n.columns <- 2 + (n.col.names * n.subcol.names) + + # Column names + row.names.table <- rep("", n.rows) + for (row in 1:n.row.names) { + row.names.table[floor(n.subrow.names/2) + (row - 1) * n.subrow.names] <- row.names[row] + } + + # Define scorecard table titles + column.titles <- c(row.title, subrow.title, rep(c(subcol.names), n.col.names)) + + # Round data + data <- round(data, round.decimal) + + # Define data inside the scorecards table + for (row in 1:n.row.names) { + table_temp <- data.frame(table_column_2 = as.character(subrow.names)) + for (col in 1:n.col.names) { + table_temp <- data.frame(table_temp, + Reorder(data = Subset(x = data, along = c(col.dim, row.dim), + indices = list(col, row), drop = 'selected'), + order = c(subrow.dim, subcol.dim))) + } + if (row == 1) { + table_data <- table_temp + } else { + table_data <- rbind(table_data, table_temp) + } + } + + # All data for plotting in table + table <- data.frame(table_column_1 = row.names.table, table_data) + table_temp <- array(unlist(table[3:n.columns]), dim = c(n.rows, n.columns - 2)) + # Define colors to show in table + table_colors <- .SCTableColors(table = table_temp, n.col = n.col.names, + n.subcol = n.subcol.names, n.row = n.row.names, + n.subrow = n.subrow.names, legend.breaks = legend.breaks, + palette = palette, colorunder = colorunder, + colorsup = colorsup) + metric.color <- table_colors$metric.color + metric.text.color <- table_colors$metric.text.color + # metric.text.bold <- table_colors$metric.text.bold + + options(stringsAsFactors = FALSE) + title <- data.frame(c1 = table.title, c2 = n.columns) + subtitle <- data.frame(c1 = table.subtitle, c2 = n.columns) + header.names <- as.data.frame(data.frame(c1 = c("", col.names), + c2 = c(2, rep(n.subcol.names, n.col.names)))) + header.names2 <- as.data.frame(data.frame(c1 = c("", paste0(rep(col.title, n.col.names))), + c2 = c(2, rep(n.subcol.names, n.col.names)))) + title.space <- data.frame(c1 = "\n", c2 = n.columns) + + # Hide NA values in table + options(knitr.kable.NA = '') + + # Create HTML table + table.html.part <- list() + table.html.part[[1]] <- kbl(table, escape = F, col.names = column.titles, align = rep("c", n.columns)) %>% + kable_paper("hover", full_width = F, font_size = 14 * font.size) %>% + add_header_above(header = header.names2, font_size = 16 * font.size) %>% + add_header_above(header = title.space, font_size = 10 * font.size) %>% + add_header_above(header = header.names, font_size = 20 * font.size) %>% + add_header_above(header = title.space, font_size = 10 * font.size) %>% + add_header_above(header = subtitle, font_size = 16 * font.size, align = "left") %>% + add_header_above(header = title.space, font_size = 10 * font.size) %>% + add_header_above(header = title, font_size = 22 * font.size, align = "left") + + for (i in 1:n.col.names) { + for (j in 1:n.subcol.names) { + my.background <- metric.color[, (i - 1) * n.subcol.names + j] + my.text.color <- metric.text.color[, (i - 1) * n.subcol.names + j] + # my.bold <- metric.text.bold[(i - 1) * n.subcol.names + j] + + table.html.part[[(i - 1) * n.subcol.names + j + 1]] <- + column_spec(table.html.part[[(i - 1) * n.subcol.names + j]], + 2 + n.subcol.names * (i - 1) + j, + background = my.background[1:n.rows], + color = my.text.color[1:n.rows], + bold = T) ## strsplit(toString(bold), ', ')[[1]] + } + } + + # Define position of table borders + column.borders <- NULL + for (i in 1:n.col.names) { + column.spacing <- (n.subcol.names * i) + 2 + column.borders <- c(column.borders, column.spacing) + } + + n.last.list <- n.col.names * n.subcol.names + 1 + + table.html <- column_spec(table.html.part[[n.last.list]], 1, bold = TRUE, width_min = paste0(col1.width, 'cm')) %>% + column_spec(2, bold = TRUE, width_min = paste0(col2.width, 'cm')) %>% + column_spec(3:n.columns, width_min = "1.2cm") %>% + column_spec(c(1, 2, column.borders), border_right = "2px solid black") %>% + column_spec(1, border_left = "2px solid black") %>% + column_spec(n.columns, border_right = "2px solid black") %>% + row_spec(seq(from = 0, to = n.subrow.names * n.row.names, by = n.subrow.names), + extra_css = "border-bottom: 2px solid black", hline_after = TRUE) + + if (plot.legend == TRUE) { + # Save the scorecard (without legend) + save_kable(table.html, file = paste0(fileout, '_tmpScorecard.png'), vheight = 1) + + # White space for legend + legend.white.space <- 37.8 * legend.white.space ## converting pixels to cm + + # Create and save color bar legend + scorecard_legend <- .SCLegend(legend.breaks = legend.breaks, + palette = palette, + colorunder = colorunder, + colorsup = colorsup, + label.scale = label.scale, + legend.width = legend.width, + legend.height = legend.height, + legend.white.space = legend.white.space, + fileout = fileout) + + # Add the legends below the scorecard table + system(paste0('convert -append ', fileout, '_tmpScorecard.png ', fileout, + '_tmpScorecardLegend.png ', fileout)) + # Remove temporary scorecard table + unlink(paste0(fileout, '_tmpScorecard*.png')) + } + if (plot.legend == FALSE) { + save_kable(table.html, file = fileout) + } +} diff --git a/modules/Scorecards/R/tmp/SCTransform.R b/modules/Scorecards/R/tmp/SCTransform.R new file mode 100644 index 0000000000000000000000000000000000000000..585e75766c5e157286c0e42c996134b9b6f8bfe5 --- /dev/null +++ b/modules/Scorecards/R/tmp/SCTransform.R @@ -0,0 +1,42 @@ +#' Scorecards spatially transform calculated means +#' +#'@description Scorecards function to spatially transform the layout of the +#'calculated metric means, to show 'Target Month' instead of 'Start Date'. +#' +#'@param data A multidimensional array of spatially aggregated data containing +#' the following dimensions; system, reference, metric, time, sdate, region. +#'@param sdate_dim A character name referring to the dimension of start date +#' in the array aggregated_metrics. +#'@param ftime_dim A character name referring to the dimension of forecast +#' time in the array aggregated_metrics. +#'@param ncores An integer indicating the number of cores to use in parallel +#' computation. It is NULL by default (1 core). +#' +#'@example +#'transformed_data <- SCTransform(data = aggregated_metrics, +#' sdate_dim = 'sdate', +#' ftime_dim = 'time') +#'@import multiApply +#'@importFrom s2dv Reorder +#'@export +SCTransform <- function(data, + sdate_dim, + ftime_dim, + ncores = NULL) { + + output <- multiApply::Apply(data = data, + target_dims = c(ftime_dim, sdate_dim), + fun = .SCTransform, + ncores = ncores)$output1 + + return(Reorder(data = output, order = names(dim(data)))) +} + +.SCTransform <- function(data) { + output <- data + n_sdates <- dim(data)[sdate_dim] + for (i in 2:dim(data)[ftime_dim]) { + output[i, ] <- data[i, c((n_sdates - i + 2):n_sdates, 1:(n_sdates - i + 1))] + } + return(output) +} diff --git a/modules/Scorecards/R/tmp/ScorecardsMulti.R b/modules/Scorecards/R/tmp/ScorecardsMulti.R new file mode 100644 index 0000000000000000000000000000000000000000..89f1df44d8302759caa6d3529e32a6fbda26dbca --- /dev/null +++ b/modules/Scorecards/R/tmp/ScorecardsMulti.R @@ -0,0 +1,360 @@ +#'Function to create all multi system/reference scorecards +#' +#'@description Scorecards function to create scorecard tables for multiple systems +#' and references (types 9 to 12). +#'@param input_data is an array of spatially aggregated metrics containing the +#' following dimensions; system, reference, metric, time, sdate, region. +#'@param system a vector of character strings defining the systems following the +#' archive.yml format from verification suite +#'@param reference a vector of character strings defining the references +#' following the archive.yml format from verification suite +#'@param var a character string following the format from +#' variable-dictionary.yml from verification suite +#'@param start.year a numeric indicating the start year of the reference period +#'@param end.year a numeric indicating the end year of the reference period +#'@param start.date a vector of character strings indicating the start months +#'@param forecast.month a vector of numeric indicating the forecast months +#'@param region.names a vector of character strings containing names of the +#' regions corresponding to the input data +#'@param metrics a vector of character strings containing the metrics. +#'@param table.label a character string containing additional information to +#' include in the scorecard title +#'@param fileout.label a character string containing additional information to +#' include in the output png file when saving the scorecard. +#'@param output.path a path of the location to save the scorecard plots. +#' +#'@return +#' This function returns 4 scorecard images for each region requested, the +#' images are saved in the directory output.path. + +#'@example +#' scorecard_multi <- ScorecardsMulti(data = aggregated_metrics, +#' system.name = c('ECMWF-SEAS5','DWD-GFCS2.1'), +#' reference.name = 'ERA5', +#' var = 'tas', +#' start.year = 1993, +#' end.year = 2016, +#' start.months = 1:12, +#' forecast.months = 1:6, +#' region.names = c('Tropics', 'Extra-tropical NH', 'Extra-tropical SH') +#' metrics = c('mean_bias', 'enscorr', 'rpss','crpss', 'enssprerr'), +#' table.label = '(Interpolation = to system, Aggregation level = skill, Cross-validation = terciles)', +#' fileout.label = '_crossval-terciles_agg-skill', +#' output.path = '/esarchive/scratch/nmilders/scorecards_images/testing' +#' ) + + +ScorecardsMulti <- function(data, + system, + reference, + var, + start.year, + end.year, + start.months, + forecast.months, + region.names, + metrics, + table.label, + fileout.label, + output.path){ + + ## Checks to apply: + # first dimension in aggregated_metrics is system and second dimension is reference + # either multi-system and one reference, or multi-reference and one system + + ## Initial checks + if (is.null(table.label)){ + table.label <- "" + } + if (is.null(fileout.label)){ + fileout.label <- "" + } + + ## Make sure input_data is in correct order for using in functions: + data_order <- c('system','reference','metric','time','sdate','region') + data <- Reorder(data, data_order) + + ## Identify metrics loaded + metrics_loaded <- attributes(data)$metrics + + ## Select only the metrics to visualize from data + input_data <- Subset(data, along = 'metric', indices = match(metrics, metrics_loaded)) + attributes(input_data)$metrics <- metrics + + ## Transform data for scorecards by forecast month (types 11 & 12) + transformed_data <- SCTransform(data = input_data, + sdate_dim = 'sdate', + ftime_dim = 'time') + + ## Load configuration files + sys_dict <- read_yaml("/esarchive/scratch/nmilders/gitlab/git_clones/s2s-suite/conf/archive.yml")$esarchive + var_dict <- read_yaml("/esarchive/scratch/nmilders/gitlab/git_clones/csscorecards/inst/config/variable-dictionary.yml")$vars + + ## Get scorecards table display names from configuration files + var.name <- var_dict[[var]]$long_name + var.units <- var_dict[[var]]$units + + system.name <- NULL + reference.name <- NULL + + for(sys in 1:length(system)){ + system.name1 <- sys_dict$System[[system[sys]]]$name + system.name <- c(system.name, system.name1) + } + for(ref in 1:length(length)){ + reference.name1 <- sys_dict$Reference[[reference[ref]]]$name + reference.name <- c(reference.name, reference.name1) + } + + ## Get metric long names + metric.names.list <- .met_names(metrics, var.units) + + ## format the metric names as character instead of list + for(met in metrics){ + if(met == metrics[1]){ + metric.names <- metric.names.list[[met]] + } else { + metric.names <- c(metric.names, metric.names.list[[met]]) + } + } + + ## Define parameters depending on Multi-system or Multi-reference + if(length(system) > 1 && length(reference) == 1){ + model <- 'system' + table.model.name <- 'System' + model.name <- system.name + eval.label <- 'Ref' + eval.name <- reference.name + eval.filename <- reference + } else if(length(system) == 1 && length(reference) > 1){ + model <- 'reference' + table.model.name <- 'Reference' + model.name <- reference.name + eval.label <- 'Sys' + eval.name <- system.name + eval.filename <- system + } else {stop('Not multi system or multi reference')} + + ## Define table colors + palette <- c('#2D004B', '#542789', '#8073AC', '#B2ABD2', '#D8DAEB', '#FEE0B6', '#FDB863', '#E08214', '#B35806', '#7F3B08') + colorunder <- "#04040E" + colorsup <- "#730C04" + + ## Legend lower limit color + legend.col.inf <- .legend_col_inf(metrics, colorunder) + legend.col.inf <- legend.col.inf[metrics] + + ## Legend upper limit color + legend.col.sup <- .legend_col_sup(metrics, colorsup) + legend.col.sup <- legend.col.sup[metrics] + + ## Legend inputs + plot.legend = TRUE + label.scale = 1.4 + legend.width = 555 + legend.height = 50 + + ## Data display inputs + round.decimal = 2 + font.size = 1.1 + + legend.white.space <- col1.width <- col2.width <- NULL ## Use default values of function + + ## Loop over region + for(reg in 1:length(region.names)){ + + breaks_bias <- NULL + + ## Find position of mean bias metric to calculate breaks + if ('mean_bias' %in% metrics) { + pos_bias <- which(metrics == 'mean_bias') + if(var == 'psl'){ + data[,,pos_bias,,,] <- data[,,pos_bias,,,]/100 ## temporary + } + breaks_bias <- .SCBiasBreaks(Subset(data, along = c('metric','region'), + indices = list(pos_bias,reg))) + } + + ## Define breaks for each metric based of metric position: + legend.breaks <- .met_breaks(metrics, breaks_bias) + + ## Define scorecard titles + table.title <- paste0(var.name, ". Region: ", region.names[reg], " ", table.label) + table.subtitle <- paste0("(", eval.label, ": ", eval.name, " ", start.year, "-", end.year, ")") + + + #### Scorecard_type 9 #### + ## (no transformation or reorder) + fileout <- .Filename(model = model, eval.name = eval.filename, var = var, + start.year = start.year, end.year = end.year, scorecard.type = 9, + region = sub(" ", "-", region.names[reg]), + fileout.label = fileout.label, output.path = output.path) + if(model == 'system'){ + data_sc_9 <- Subset(input_data, c('reference','region'), list(1, reg), drop = 'selected') + } else if(model == 'reference'){ + data_sc_9 <- Subset(input_data, c('system','region'), list(1, reg), drop = 'selected') + } + SCPlotScorecard(data = data_sc_9, + row.dim = model, + subrow.dim = 'time', + col.dim = 'metric', + subcol.dim = 'sdate', + legend.dim = 'metric', + row.names = model.name, + subrow.names = forecast.months, + col.names = metric.names, + subcol.names = month.abb[as.numeric(start.months)], + table.title = table.title, + table.subtitle = table.subtitle, + row.title = table.model.name, + subrow.title = 'Forecast Month', + col.title = 'Start date', + legend.breaks = legend.breaks, + plot.legend = plot.legend, + label.scale = label.scale, + legend.width = legend.width, + legend.height = legend.height, + palette = palette, + colorunder = legend.col.inf, + colorsup = legend.col.sup, + round.decimal = round.decimal, + font.size = font.size, + legend.white.space = legend.white.space, + col1.width = 4, + col2.width = col2.width, + fileout = fileout) + + + #### Scorecard_type 10 #### + ## (reorder only) + fileout <- .Filename(model = model, eval.name = eval.filename, var = var, + start.year = start.year, end.year = end.year, scorecard.type = 10, + region = sub(" ", "-", region.names[reg]), + fileout.label = fileout.label, output.path = output.path) + new_order <- c('system', 'reference', 'metric', 'region','sdate', 'time') + if(model == 'system'){ + data_sc_10 <- Subset(Reorder(input_data, new_order), c('reference','region'), list(1, reg), drop = 'selected') + } else if(model == 'reference'){ + data_sc_10 <- Subset(Reorder(input_data, new_order), c('system','region'), list(1, reg), drop = 'selected') + } + SCPlotScorecard(data = data_sc_10, + row.dim = 'time', + subrow.dim = model, + col.dim = 'metric', + subcol.dim = 'sdate', + legend.dim = 'metric', + row.names = forecast.months, + subrow.names = model.name, + col.names = metric.names, + subcol.names = month.abb[as.numeric(start.months)], + table.title = table.title, + table.subtitle = table.subtitle, + row.title = 'Forecast month', + subrow.title = table.model.name, + col.title = 'Start date', + legend.breaks = legend.breaks, + plot.legend = plot.legend, + label.scale = label.scale, + legend.width = legend.width, + legend.height = legend.height, + palette = palette, + colorunder = legend.col.inf, + colorsup = legend.col.sup, + round.decimal = round.decimal, + font.size = font.size, + legend.white.space = legend.white.space, + col1.width = col1.width, + col2.width = 4, + fileout = fileout) + + + #### Scorecard_type 11 #### + ## (transformation only) + fileout <- .Filename(model = model, eval.name = eval.filename, var = var, + start.year = start.year, end.year = end.year, scorecard.type = 11, + region = sub(" ", "-", region.names[reg]), + fileout.label = fileout.label, output.path = output.path) + if(model == 'system'){ + data_sc_11 <- Subset(transformed_data, c('reference','region'), list(1, reg), drop = 'selected') + } else if(model == 'reference'){ + data_sc_11 <- Subset(transformed_data, c('system','region'), list(1, reg), drop = 'selected') + } + SCPlotScorecard(data = data_sc_11, + row.dim = model, + subrow.dim = 'time', + col.dim = 'metric', + subcol.dim = 'sdate', + legend.dim = 'metric', + row.names = model.name, + subrow.names = forecast.months, + col.names = metric.names, + subcol.names = month.abb[as.numeric(start.months)], + table.title = table.title, + table.subtitle = table.subtitle, + row.title = table.model.name, + subrow.title = 'Forecast Month', + col.title = 'Target month', + legend.breaks = legend.breaks, + plot.legend = plot.legend, + label.scale = label.scale, + legend.width = legend.width, + legend.height = legend.height, + palette = palette, + colorunder = legend.col.inf, + colorsup = legend.col.sup, + round.decimal = round.decimal, + font.size = font.size, + legend.white.space = legend.white.space, + col1.width = 4, + col2.width = col2.width, + fileout = fileout) + + + #### Scorecard_type 12 #### + ## (transformation and reorder) + fileout <- .Filename(model = model, eval.name = eval.filename, var = var, + start.year = start.year, end.year = end.year, scorecard.type = 12, + region = sub(" ", "-", region.names[reg]), + fileout.label = fileout.label, output.path = output.path) + new_order <- c('system', 'reference', 'metric', 'region','sdate', 'time') + if(model == 'system'){ + data_sc_12 <- Subset(Reorder(transformed_data, new_order), c('reference','region'), list(1, reg), drop = 'selected') + } else if(model == 'reference'){ + data_sc_12 <- Subset(Reorder(transformed_data, new_order), c('system','region'), list(1, reg), drop = 'selected') + } + SCPlotScorecard(data = data_sc_12, + row.dim = 'time', + subrow.dim = model, + col.dim = 'metric', + subcol.dim = 'sdate', + legend.dim = 'metric', + row.names = forecast.months, + subrow.names = model.name, + col.names = metric.names, + subcol.names = month.abb[as.numeric(start.months)], + table.title = table.title, + table.subtitle = table.subtitle, + row.title = 'Forecast Month', + subrow.title = table.model.name, + col.title = 'Target month', + legend.breaks = legend.breaks, + plot.legend = plot.legend, + label.scale = label.scale, + legend.width = legend.width, + legend.height = legend.height, + palette = palette, + colorunder = legend.col.inf, + colorsup = legend.col.sup, + round.decimal = round.decimal, + font.size = font.size, + legend.white.space = legend.white.space, + col1.width = col1.width, + col2.width = 4, + fileout = fileout) + + } ## close loop on region + + print("All multi scorecard plots created") + +} ## close function + diff --git a/modules/Scorecards/R/tmp/ScorecardsSingle.R b/modules/Scorecards/R/tmp/ScorecardsSingle.R new file mode 100644 index 0000000000000000000000000000000000000000..56f08204ad5443b94bbc281a31f3c75cf5cb7614 --- /dev/null +++ b/modules/Scorecards/R/tmp/ScorecardsSingle.R @@ -0,0 +1,343 @@ +#'Function to create all single system/reference scorecards +#' +#'@description Scorecards function to create scorecard tables for one system and +#' one reference combination (types 1 to 4). +#'@param input_data is an array of spatially aggregated metrics containing the +#' following dimensions; system, reference, metric, time, sdate, region. +#'@param system a vector of character strings defining the systems following the +#' archive.yml format from verification suite +#'@param reference a vector of character strings defining the references +#' following the archive.yml format from verification suite +#'@param var a character string following the format from +#' variable-dictionary.yml from verification suite +#'@param start.year a numeric indicating the start year of the reference period +#'@param end.year a numeric indicating the end year of the reference period +#'@param start.date a vector of character strings indicating the start months +#'@param forecast.month a vector of numeric indicating the forecast months +#'@param region.names a vector of character strings containing names of the +#' regions corresponding to the input data +#'@param metrics a vector of character strings containing the metrics. +#'@param table.label a character string containing additional information to +#' include in the scorecard title +#'@param fileout.label a character string containing additional information to +#' include in the output png file when saving the scorecard. +#'@param output.path a path of the location to save the scorecard plots +#' +#'@return +#' This function returns 4 scorecards images, saved in the directory output.path +#'@examples +#' scorecard_single <- ScorecardsSingle(data = aggregated_metrics, +#' system.name = c('ECMWF-SEAS5','DWD-GFCS2.1'), +#' reference.name = 'ERA5', +#' var = 'tas', +#' start.year = 1993, +#' end.year = 2016, +#' start.months = 1:12, +#' forecast.months = 1:6, +#' region.names = c('Tropics', 'Extra-tropical NH', 'Extra-tropical SH') +#' metrics = c('mean_bias', 'enscorr', 'rpss','crpss', 'enssprerr'), +#' table.label = '(Interpolation = to system, Aggregation level = skill, Cross-validation = terciles)', +#' fileout.label = '_crossval-terciles_agg-skill', +#' output.path = '/esarchive/scratch/nmilders/scorecards_images/test' +#' ) +#'@export +ScorecardsSingle <- function(data, system, reference, var, start.year, end.year, + start.months, forecast.months, region.names, + metrics, legend.breaks = NULL, + table.label = NULL, fileout.label = NULL, + legend.white.space = NULL, + col1.width = NULL, col2.width = NULL, + output.path){ + + ## Checks to apply: + # First dimension in aggregated_metrics is system and second dimension is reference + # To allow 1 region - if region = 1 --> only scorecards 1 & 3 need to be plotted + # If any dimension of input dat is 1, make sure dimension is still present in array + + ## Initial checks + # data + if (!is.array(data)) { + stop("Parameter 'data' must be an array with named dimensions.") + } + if (!is.array(data)) { + stop("Parameter 'data' must be an array with named dimensions.") + } + if (is.null(names(dim(data)))) { + stop("Parameter 'data' must have dimenision names.") + } + if (!all(c('system','reference','metric','time','sdate','region') %in% + names(dim(data)))) { + stop("Dimension names of 'data' must be: 'system','reference','metric', + 'time','sdate','region'.") + } + if (is.null(table.label)){ + table.label <- "" + } + if (is.null(fileout.label)){ + fileout.label <- "" + } + + ## Make sure input_data is in correct order for using in functions: + data_order <- c('system', 'reference', 'metric', 'time', 'sdate', 'region') + data <- Reorder(data, data_order) + + ## Identify metrics loaded + metrics_loaded <- attributes(data)$metrics + + ## Select only the metrics to visualize from data + input_data <- Subset(data, along = 'metric', indices = match(metrics, metrics_loaded)) + attributes(input_data)$metrics <- metrics + + ## Transform data for scorecards by forecast month (types 3 & 4) + transformed_data <- SCTransform(data = input_data, + sdate_dim = 'sdate', + ftime_dim = 'time') + + ## Load configuration files + sys_dict <- read_yaml("/esarchive/scratch/nmilders/gitlab/git_clones/s2s-suite/conf/archive.yml")$esarchive + var_dict <- read_yaml("/esarchive/scratch/nmilders/gitlab/git_clones/csscorecards/inst/config/variable-dictionary.yml")$vars + + ## Get scorecards table display names from configuration files + var.name <- var_dict[[var]]$long_name + var.units <- var_dict[[var]]$units + + ## Get metric long names + metric.names.list <- .met_names(metrics, var.units) + + ## format the metric names as character instead of list + for(met in metrics){ + if(met == metrics[1]){ + metric.names <- metric.names.list[[met]] + } else { + metric.names <- c(metric.names, metric.names.list[[met]]) + } + } + + ## Define table colors + palette <- c('#2D004B', '#542789', '#8073AC', '#B2ABD2', '#D8DAEB', '#FEE0B6', '#FDB863', '#E08214', '#B35806', '#7F3B08') + colorunder <- "#04040E" + colorsup <- "#730C04" + + ## Legend lower limit color + legend.col.inf <- .legend_col_inf(metrics, colorunder) + legend.col.inf <- legend.col.inf[metrics] + + ## Legend upper limit color + legend.col.sup <- .legend_col_sup(metrics, colorsup) + legend.col.sup <- legend.col.sup[metrics] + + ## Legend inputs + plot.legend = TRUE + label.scale = 1.4 + legend.width = 555 + legend.height = 50 + + ## Data display inputs + round.decimal = 2 + font.size = 1.1 + + ## Loop over system and reference for each scorecard plot + for (sys in 1:dim(input_data)['system']) { + for (ref in 1:dim(input_data)['reference']) { + + ## TO DO: Apply check to each scorecard function + ## check dimension 'metric' exists: + if (!("metric" %in% names(dim(input_data)))) { + dim(input_data) <- c(metric = 1, dim(input_data)) + } + + ## Find position of mean bias metric to calculate breaks + breaks_bias <- NULL + if ('mean_bias' %in% metrics){ + stopifnot(identical(names(dim(Subset(input_data, c('system', 'reference'), list(sys, ref), drop = 'selected'))), c('metric','time','sdate','region'))) + temp_data <- Subset(input_data, c('system', 'reference'), list(sys, ref), drop = 'selected') + pos_bias <- which(metrics == 'mean_bias') + if(var == 'psl'){ + temp_data[pos_bias,,,] <- temp_data[pos_bias,,,]/100 + } + breaks_bias <- .SCBiasBreaks(Subset(temp_data, along = 'metric', + indices = pos_bias)) + } + + ## Define breaks for each metric based of metric position: + legend.breaks <- .met_breaks(metrics, breaks_bias) + + ## Put breaks in same order as metrics + legend.breaks <- legend.breaks[metrics] + + ## Get scorecards table display names from configuration files + system.name <- sys_dict$System[[system[sys]]]$name + reference.name <- sys_dict$Reference[[reference[ref]]]$name + + ## Define scorecard titles + table.title <- paste0(var.name, " of ", system.name, " ", table.label) + table.subtitle <- paste0("(Ref: ", reference.name, " ", start.year, "-", end.year, ")") + + ############################################################################# + + #### Scorecard_type 1 #### + ## (no transformation or reorder) + fileout <- .Filename(system = system[sys], reference = reference[ref], var = var, + start.year = start.year, end.year = end.year, scorecard.type = 1, + fileout.label = fileout.label, output.path = output.path) + data_sc_1 <- Subset(input_data, c('system', 'reference'), list(sys, ref), drop = 'selected') + SCPlotScorecard(data = data_sc_1, + row.dim = 'region', + subrow.dim = 'time', + col.dim = 'metric', + subcol.dim = 'sdate', + legend.dim = 'metric', + row.names = region.names, + subrow.names = forecast.months, + col.names = metric.names, + subcol.names = month.abb[as.numeric(start.months)], + table.title = table.title, + table.subtitle = table.subtitle, + row.title = 'Region', + subrow.title = 'Forecast Month', + col.title = 'Start date', + legend.breaks = legend.breaks, + plot.legend = plot.legend, + label.scale = label.scale, + legend.width = legend.width, + legend.height = legend.height, + palette = palette, + colorunder = legend.col.inf, + colorsup = legend.col.sup, + round.decimal = round.decimal, + font.size = font.size, + legend.white.space = legend.white.space, + col1.width = col1.width, + col2.width = col2.width, + fileout = fileout) + + + #### Scorecard_type 2 #### + ## (reorder only) + ## Scorecard type 2 is same as type 1 for only one region, therefore is + ## only plotted if more that one region is requested + if(dim(input_data)['region'] > 1) { + fileout <- .Filename(system = system[sys], reference = reference[ref], var = var, + start.year = start.year, end.year = end.year, scorecard.type = 2, + fileout.label = fileout.label, output.path = output.path) + new_order <- c('metric', 'region', 'sdate', 'time') + data_sc_2 <- Reorder(Subset(input_data, c('system', 'reference'), list(sys, ref), drop = 'selected'), new_order) + SCPlotScorecard(data = data_sc_2, + row.dim = 'time', + subrow.dim = 'region', + col.dim = 'metric', + subcol.dim = 'sdate', + legend.dim = 'metric', + row.names = forecast.months, + subrow.names = region.names, + col.names = metric.names, + subcol.names = month.abb[as.numeric(start.months)], + table.title = table.title, + table.subtitle = table.subtitle, + row.title = 'Forecast Month', + subrow.title = 'Region', + col.title = 'Start date', + legend.breaks = legend.breaks, + plot.legend = plot.legend, + label.scale = label.scale, + legend.width = legend.width, + legend.height = legend.height, + palette = palette, + colorunder = legend.col.inf, + colorsup = legend.col.sup, + round.decimal = round.decimal, + font.size = font.size, + legend.white.space = legend.white.space, + col1.width = col1.width, + col2.width = col2.width, + fileout = fileout) + } ## close if + + + #### Scorecard_type 3 #### + ## (transformation only) + fileout <- .Filename(system = system[sys], reference = reference[ref], var = var, + start.year = start.year, end.year = end.year, scorecard.type = 3, + fileout.label = fileout.label, output.path = output.path) + data_sc_3 <- Subset(transformed_data, c('system', 'reference'), list(sys, ref), drop = 'selected') + SCPlotScorecard(data = data_sc_3, + row.dim = 'region', + subrow.dim = 'time', + col.dim = 'metric', + subcol.dim = 'sdate', + legend.dim = 'metric', + row.names = region.names, + subrow.names = forecast.months, + col.names = metric.names, + subcol.names = month.abb[as.numeric(start.months)], + table.title = table.title, + table.subtitle = table.subtitle, + row.title = 'Region', + subrow.title = 'Forecast Month', + col.title = 'Target month', + legend.breaks = legend.breaks, + plot.legend = plot.legend, + label.scale = label.scale, + legend.width = legend.width, + legend.height = legend.height, + palette = palette, + colorunder = legend.col.inf, + colorsup = legend.col.sup, + round.decimal = round.decimal, + font.size = font.size, + legend.white.space = legend.white.space, + col1.width = col1.width, + col2.width = col2.width, + fileout = fileout) + + + #### Scorecard_type 4 #### + ## (transformation and reorder) + ## Scorecard type 4 is same as type 3 for only one region, therefore is + ## only plotted if more that one region is requested + if(dim(input_data)['region'] > 1) { + fileout <- .Filename(system = system[sys], reference = reference[ref], var = var, + start.year = start.year, end.year = end.year, scorecard.type = 4, + fileout.label = fileout.label, output.path = output.path) + new_order <- c('metric', 'region', 'sdate', 'time') + data_sc_4 <- Reorder(Subset(transformed_data, c('system', 'reference'), list(sys, ref), drop = 'selected'), new_order) + SCPlotScorecard(data = data_sc_4, + row.dim = 'time', + subrow.dim = 'region', + col.dim = 'metric', + subcol.dim = 'sdate', + legend.dim = 'metric', + row.names = forecast.months, + subrow.names = region.names, + col.names = metric.names, + subcol.names = month.abb[as.numeric(start.months)], + table.title = table.title, + table.subtitle = table.subtitle, + row.title = 'Forecast Month', + subrow.title = 'Region', + col.title = 'Target month', + legend.breaks = legend.breaks, + plot.legend = plot.legend, + label.scale = label.scale, + legend.width = legend.width, + legend.height = legend.height, + palette = palette, + colorunder = legend.col.inf, + colorsup = legend.col.sup, + round.decimal = round.decimal, + font.size = font.size, + legend.white.space = legend.white.space, + col1.width = col1.width, + col2.width = col2.width, + fileout = fileout) + } ## close if + + } ## close loop on ref + } ## close loop on sys + + print("All single system scorecard plots created") + +} ## close function + + + diff --git a/modules/Scorecards/R/tmp/ScorecardsSystemDiff.R b/modules/Scorecards/R/tmp/ScorecardsSystemDiff.R new file mode 100644 index 0000000000000000000000000000000000000000..a19b1651140a5a3bcc174cc39f81117ac937d361 --- /dev/null +++ b/modules/Scorecards/R/tmp/ScorecardsSystemDiff.R @@ -0,0 +1,350 @@ +#'Function to create all multi system/reference scorecards +#' +#'@description Scorecards function to create scorecard tables for multiple systems +#' and references (types 9 to 12). +#'@param input_data is an array of spatially aggregated metrics containing the +#' following dimensions; system, reference, metric, time, sdate, region. +#'@param system a vector of character strings defining the systems following the +#' archive.yml format from verification suite +#'@param reference a vector of character strings defining the references +#' following the archive.yml format from verification suite +#'@param var a character string following the format from +#' variable-dictionary.yml from verification suite +#'@param start.year a numeric indicating the start year of the reference period +#'@param end.year a numeric indicating the end year of the reference period +#'@param start.date a vector of character strings indicating the start months +#'@param forecast.month a vector of numeric indicating the forecast months +#'@param region.names a vector of character strings containing names of the +#' regions corresponding to the input data +#'@param metrics a vector of character strings containing the metrics. +#'@param table.label a character string containing additional information to +#' include in the scorecard title +#'@param fileout.label a character string containing additional information to +#' include in the output png file when saving the scorecard. +#'@param output.path a path of the location to save the scorecard plots. +#' +#'@return +#' This function returns 4 scorecard images for each region requested, the +#' images are saved in the directory output.path. + +#'@example +#' scorecard_diff <- ScorecardsDiff(data = aggregated_metrics, +#' system.name = c('ECMWF-SEAS5','DWD-GFCS2.1'), +#' reference.name = 'ERA5', +#' var = 'tas', +#' start.year = 1993, +#' end.year = 2016, +#' start.months = 1:12, +#' forecast.months = 1:6, +#' region.names = c('Tropics', 'Extra-tropical NH', 'Extra-tropical SH') +#' metrics = c('mean_bias', 'enscorr', 'rpss','crpss', 'enssprerr'), +#' table.label = '(Interpolation = to system, Aggregation level = skill, Cross-validation = terciles)', +#' fileout.label = '_crossval-terciles_agg-skill', +#' output.path = '/esarchive/scratch/nmilders/scorecards_images/testing' +#' ) + + +ScorecardsSystemDiff <- function(data, + system, + reference, + var, + start.year, + end.year, + start.months, + forecast.months, + region.names, + metrics, + table.label = NULL, + fileout.label = NULL, + legend.white.space = NULL, + col1.width = NULL, + col2.width = NULL, + output.path){ + + ## Checks to apply: + # first dimension in aggregated_metrics is system and second dimension is reference + # either multi-system and one reference, or multi-reference and one system + + ## Initial checks + if (is.null(table.label)){ + table.label <- "" + } + if (is.null(fileout.label)){ + fileout.label <- "" + } + + ## Make sure input_data is in correct order for using in functions: + data_order <- c('system','reference','metric','time','sdate','region') + data <- Reorder(data, data_order) + + ## Identify metrics loaded + metrics_loaded <- attributes(data)$metrics + + ## Select only the metrics to visualize from data + input_data <- Subset(data, along = 'metric', indices = match(metrics, metrics_loaded)) + attributes(input_data)$metrics <- metrics + + ## Calculate difference between two systems/references + if(length(system) == 2 && length(reference) == 1){ + dataset1 <- Subset(input_data, c('system', 'reference'), list(1, 1), drop = 'selected') + dataset2 <- Subset(input_data, c('system', 'reference'), list(2, 1), drop = 'selected') + } + else if(length(system) == 1 && length(reference) == 2){ + dataset1 <- Subset(input_data, c('system', 'reference'), list(1, 1), drop = 'selected') + dataset2 <- Subset(input_data, c('system', 'reference'), list(1, 2), drop = 'selected') + } + + ## Calculate difference of mean_bias from 0 for each dataset + if ('mean_bias' %in% metrics){ + pos_bias <- which(metrics == 'mean_bias') + dataset1[pos_bias,,,] <- abs(dataset1[pos_bias,,,]) + dataset2[pos_bias,,,] <- abs(dataset2[pos_bias,,,]) + } + + ## Calculate difference of enssprerr from 1 for each dataset + if ('enssprerr' %in% metrics){ + pos_enssprerr <- which(metrics == 'enssprerr') + dataset1[pos_enssprerr,,,] <- abs(1-dataset1[pos_enssprerr,,,]) + dataset2[pos_enssprerr,,,] <- abs(1-dataset2[pos_enssprerr,,,]) + } + + diff_data <- dataset1 - dataset2 + + # Transform data for scorecards by forecast month (types 3 & 4) + transformed_data <- SCTransform(data = diff_data, + sdate_dim = 'sdate', + ftime_dim = 'time') + + ## Load configuration files + sys_dict <- read_yaml("/esarchive/scratch/nmilders/gitlab/git_clones/s2s-suite/conf/archive.yml")$esarchive + var_dict <- read_yaml("/esarchive/scratch/nmilders/gitlab/git_clones/csscorecards/inst/config/variable-dictionary.yml")$vars + + ## Get scorecards table display names from configuration files + var.name <- var_dict[[var]]$long_name + var.units <- var_dict[[var]]$units + + system.name <- NULL + reference.name <- NULL + + for(sys in 1:length(system)){ + system.name1 <- sys_dict$System[[system[sys]]]$name + system.name <- c(system.name, system.name1) + } + for(ref in 1:length(reference)){ + reference.name1 <- sys_dict$Reference[[reference[ref]]]$name + reference.name <- c(reference.name, reference.name1) + } + + ## Get metric long names + metric.names.list <- .met_names(metrics, var.units) + + ## format the metric names as character instead of list + for(met in metrics){ + if(met == metrics[1]){ + metric.names <- metric.names.list[[met]] + } else { + metric.names <- c(metric.names, metric.names.list[[met]]) + } + } + + ## Define parameters depending on system compariosn or reference comparison + if(length(system) > 1 && length(reference) == 1){ + comparison <- system + model <- 'system' + table.model.name <- 'System' + model.name <- system.name + eval.label <- 'Ref' + eval.name <- reference.name + eval.filename <- reference + } else if(length(system) == 1 && length(reference) > 1){ + comparison <- reference + model <- 'reference' + table.model.name <- 'Reference' + model.name <- reference.name + eval.label <- 'Sys' + eval.name <- system.name + eval.filename <- system + } else {stop('Not multi system or multi reference')} + + ## Define table colors + palette <- c('#2D004B', '#542789', '#8073AC', '#B2ABD2', '#D8DAEB', '#FEE0B6', '#FDB863', '#E08214', '#B35806', '#7F3B08') + colorunder <- "#04040E" + colorsup <- "#730C04" + + ## Legend inputs + plot.legend = TRUE + label.scale = 1.4 + legend.width = 555 + legend.height = 50 + + ## Data display inputs + round.decimal = 2 + font.size = 1.1 + + ## Define breaks for each metric based of metric position: + legend.breaks <- c(-0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3, 0.4, 0.5) + + ## Define scorecard titles + table.title <- paste0(var.name, " - Difference: ", model.name[1], " - ", model.name[2], " ",table.label) + table.subtitle <- paste0("(", eval.label, ": ", eval.name, " ", start.year, "-", end.year, ")") + + #### Scorecard_type 1 #### + ## (no transformation or reorder) + + fileout <- .Filename(system = paste0("diff_",comparison[1],"_",comparison[2]), reference = eval.filename, var = var, + start.year = start.year, end.year = end.year, scorecard.type = 1, + fileout.label = fileout.label, output.path = output.path) + SCPlotScorecard(data = diff_data, + row.dim = 'region', + subrow.dim = 'time', + col.dim = 'metric', + subcol.dim = 'sdate', + legend.dim = 'metric', + row.names = region.names, + subrow.names = forecast.months, + col.names = metric.names, + subcol.names = month.abb[as.numeric(start.months)], + table.title = table.title, + table.subtitle = table.subtitle, + row.title = 'Region', + subrow.title = 'Forecast Month', + col.title = 'Start date', + legend.breaks = legend.breaks, + plot.legend = plot.legend, + label.scale = label.scale, + legend.width = legend.width, + legend.height = legend.height, + palette = palette, + colorunder = colorunder, + colorsup = colorsup, + round.decimal = round.decimal, + font.size = font.size, + legend.white.space = legend.white.space, + col1.width = col1.width, + col2.width = col2.width, + fileout = fileout) + + + #### Scorecard_type 2 #### + ## (reorder only) + ## Scorecard type 2 is same as type 1 for only one region, therefore is + ## only plotted if more that one region is requested + if(dim(input_data)['region'] > 1) { + fileout <- .Filename(system = paste0("diff_",comparison[1],"_",comparison[2]), reference = eval.filename, var = var, + start.year = start.year, end.year = end.year, scorecard.type = 2, + fileout.label = fileout.label, output.path = output.path) + new_order <- c('metric', 'region', 'sdate', 'time') + data_sc_2 <- Reorder(diff_data, new_order) + SCPlotScorecard(data = data_sc_2, + row.dim = 'time', + subrow.dim = 'region', + col.dim = 'metric', + subcol.dim = 'sdate', + legend.dim = 'metric', + row.names = forecast.months, + subrow.names = region.names, + col.names = metric.names, + subcol.names = month.abb[as.numeric(start.months)], + table.title = table.title, + table.subtitle = table.subtitle, + row.title = 'Forecast Month', + subrow.title = 'Region', + col.title = 'Start date', + legend.breaks = legend.breaks, + plot.legend = plot.legend, + label.scale = label.scale, + legend.width = legend.width, + legend.height = legend.height, + palette = palette, + colorunder = colorunder, + colorsup = colorsup, + round.decimal = round.decimal, + font.size = font.size, + legend.white.space = legend.white.space, + col1.width = col1.width, + col2.width = col2.width, + fileout = fileout) + } ## close if + + + #### Scorecard_type 3 #### + ## (transformation only) + fileout <- .Filename(system = paste0("diff_",comparison[1],"_",comparison[2]), reference = eval.filename, var = var, + start.year = start.year, end.year = end.year, scorecard.type = 3, + fileout.label = fileout.label, output.path = output.path) + SCPlotScorecard(data = transformed_data, + row.dim = 'region', + subrow.dim = 'time', + col.dim = 'metric', + subcol.dim = 'sdate', + legend.dim = 'metric', + row.names = region.names, + subrow.names = forecast.months, + col.names = metric.names, + subcol.names = month.abb[as.numeric(start.months)], + table.title = table.title, + table.subtitle = table.subtitle, + row.title = 'Region', + subrow.title = 'Forecast Month', + col.title = 'Target month', + legend.breaks = legend.breaks, + plot.legend = plot.legend, + label.scale = label.scale, + legend.width = legend.width, + legend.height = legend.height, + palette = palette, + colorunder = colorunder, + colorsup = colorsup, + round.decimal = round.decimal, + font.size = font.size, + legend.white.space = legend.white.space, + col1.width = col1.width, + col2.width = col2.width, + fileout = fileout) + + + #### Scorecard_type 4 #### + ## (transformation and reorder) + ## Scorecard type 4 is same as type 3 for only one region, therefore is + ## only plotted if more that one region is requested + if(dim(input_data)['region'] > 1) { + fileout <- .Filename(system = paste0("diff_",comparison[1],"_",comparison[2]), reference = eval.filename, var = var, + start.year = start.year, end.year = end.year, scorecard.type = 4, + fileout.label = fileout.label, output.path = output.path) + new_order <- c('metric', 'region', 'sdate', 'time') + data_sc_4 <- Reorder(transformed_data, new_order) + SCPlotScorecard(data = data_sc_4, + row.dim = 'time', + subrow.dim = 'region', + col.dim = 'metric', + subcol.dim = 'sdate', + legend.dim = 'metric', + row.names = forecast.months, + subrow.names = region.names, + col.names = metric.names, + subcol.names = month.abb[as.numeric(start.months)], + table.title = table.title, + table.subtitle = table.subtitle, + row.title = 'Forecast Month', + subrow.title = 'Region', + col.title = 'Target month', + legend.breaks = legend.breaks, + plot.legend = plot.legend, + label.scale = label.scale, + legend.width = legend.width, + legend.height = legend.height, + palette = palette, + colorunder = colorunder, + colorsup = colorsup, + round.decimal = round.decimal, + font.size = font.size, + legend.white.space = legend.white.space, + col1.width = col1.width, + col2.width = col2.width, + fileout = fileout) + } ## close if + +print("All system difference scorecard plots created") + +} ## close function + diff --git a/modules/Scorecards/R/tmp/Utils.R b/modules/Scorecards/R/tmp/Utils.R new file mode 100644 index 0000000000000000000000000000000000000000..6ba49e8c5c887c938902e30d200e49bff1af142e --- /dev/null +++ b/modules/Scorecards/R/tmp/Utils.R @@ -0,0 +1,361 @@ +############### FUNCTIONS FOR SCORECARDS ################ + + +## Define metric names + +.met_names <- function(metrics, var.units) { # metrics is a object with the names of the metrics to be displayed + result <- list() + if ('mean_bias' %in% metrics) { + result <- append(result, list('mean_bias' = paste0('Mean bias (', var.units,')'))) + } + if ('enscorr' %in% metrics) { + result <- append(result, list('enscorr' = 'Correlation')) + } + if ('rps' %in% metrics ) { + result <- append(result, list('rps' = 'RPS')) + } + if ('frps' %in% metrics ) { + result <- append(result, list('frps' = 'Fair RPS')) + } + if ('rpss' %in% metrics) { + result <- append(result, list('rpss' = 'RPSS')) + } + if ('rpss_score_aggr' %in% metrics) { + result <- append(result, list('rpss_score_aggr' = 'RPSS')) + } + if ('frpss' %in% metrics) { + result <- append(result, list('frpss' = 'Fair RPSS')) + } + if ('crps' %in% metrics) { + result <- append(result, list('crps' = 'CRPS')) + } + if ('crpss' %in% metrics) { + result <- append(result, list('crpss' = 'CRPSS')) + } + if ('crpss_score_aggr' %in% metrics) { + result <- append(result, list('crpss_score_aggr' = 'CRPSS')) + } + if ('bss10' %in% metrics) { + result <- append(result, list('bss10' = 'Brier skill score 10%')) + } + if ('bss90' %in% metrics) { + result <- append(result, list('bss90' = 'Brier skill score 90%')) + } + if ('enssprerr' %in% metrics ) { + result <- append(result, list('enssprerr' = 'Spread-to-error ratio')) + } + if ('rmsss' %in% metrics ) { + result <- append(result, list('rmsss' = 'RMSSS')) + } + return(result) +} + + + +## Define metrics breaks for each input metric + +.met_breaks <- function(metrics, breaks_bias) { # metrics is a object with the names of the metrics to be displayed + result <- list() + if ('mean_bias' %in% metrics) { + result <- append(result, list('mean_bias' = breaks_bias)) + } + if ('enscorr' %in% metrics) { + result <- append(result, list('enscorr' = c(-1, -0.8, -0.6, -0.4, -0.2, 0, 0.2, 0.4, 0.6, 0.8, 1))) + } + if ('rps' %in% metrics ) { + result <- append(result, list('rps' = c(0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.8, 1))) + } + if ('frps' %in% metrics ) { + result <- append(result, list('frps' = c(-0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3, 0.4, 0.5))) + } + if ('rpss' %in% metrics) { + result <- append(result, list('rpss' = c(-0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3, 0.4, 0.5))) + } + if ('rpss_score_aggr' %in% metrics) { + result <- append(result, list('rpss_score_aggr' = c(-0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3, 0.4, 0.5))) + } + if ('frpss' %in% metrics) { + result <- append(result, list('frpss' = c(-0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3, 0.4, 0.5))) + } + if ('crps' %in% metrics) { + result <- append(result, list('crps' = c(-0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3, 0.4, 0.5))) + } + if ('crpss' %in% metrics) { + result <- append(result, list('crpss' = c(-0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3, 0.4, 0.5))) + } + if ('crpss_score_aggr' %in% metrics) { + result <- append(result, list('crpss_score_aggr' = c(-0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3, 0.4, 0.5))) + } + if ('bss10' %in% metrics) { + result <- append(result, list('bss10' = c(-0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3, 0.4, 0.5))) + } + if ('bss90' %in% metrics) { + result <- append(result, list('bss90' = c(-0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3, 0.4, 0.5))) + } + if ('enssprerr' %in% metrics ) { + result <- append(result, list('enssprerr' = c(0, 0.6, 0.7, 0.8, 0.9, 1, 1.2, 1.4, 1.6, 1.8, 2))) + } + if ('rmsss' %in% metrics ) { + result <- append(result, list('rmsss' = c(-0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3, 0.4, 0.5))) + } + return(result) +} + +## Define legend lower limit color + +.legend_col_inf <- function(metrics, colorunder) { + result <- list() + if ('mean_bias' %in% metrics) { + result <- append(result, list('mean_bias' = colorunder)) + } + if ('enscorr' %in% metrics) { + result <- append(result, list('enscorr' = NULL)) + } + if ('rps' %in% metrics ) { + result <- append(result, list('rps' = NULL)) + } + if ('frps' %in% metrics ) { + result <- append(result, list('frps' = NULL)) + } + if ('rpss' %in% metrics) { + result <- append(result, list('rpss' = colorunder)) + } + if ('rpss_score_aggr' %in% metrics) { + result <- append(result, list('rpss_score_aggr' = colorunder)) + } + if ('frpss' %in% metrics) { + result <- append(result, list('frpss' = colorunder)) + } + if ('crps' %in% metrics) { + result <- append(result, list('crps' = NULL)) + } + if ('crpss' %in% metrics) { + result <- append(result, list('crpss' = colorunder)) + } + if ('crpss_score_aggr' %in% metrics) { + result <- append(result, list('crpss_score_aggr' = colorunder)) + } + if ('bss10' %in% metrics) { + result <- append(result, list('bss10' = colorunder)) + } + if ('bss90' %in% metrics) { + result <- append(result, list('bss90' = colorunder)) + } + if ('enssprerr' %in% metrics ) { + result <- append(result, list('enssprerr' = NULL)) + } + if ('rmsss' %in% metrics ) { + result <- append(result, list('rmsss' = colorunder)) + } + return(result) +} + + +## Define legend upper limit color + +.legend_col_sup <- function(metrics, colorsup) { + result <- list() + if ('mean_bias' %in% metrics) { + result <- append(result, list('mean_bias' = colorsup)) + } + if ('enscorr' %in% metrics) { + result <- append(result, list('enscorr' = NULL)) + } + if ('rps' %in% metrics ) { + result <- append(result, list('rps' = NULL)) + } + if ('frps' %in% metrics ) { + result <- append(result, list('frps' = NULL)) + } + if ('rpss' %in% metrics) { + result <- append(result, list('rpss' = colorsup)) + } + if ('rpss_score_aggr' %in% metrics) { + result <- append(result, list('rpss_score_aggr' = colorsup)) + } + if ('frpss' %in% metrics) { + result <- append(result, list('frpss' = colorsup)) + } + if ('crps' %in% metrics) { + result <- append(result, list('crps' = colorsup)) + } + if ('crpss' %in% metrics) { + result <- append(result, list('crpss' = colorsup)) + } + if ('crpss_score_aggr' %in% metrics) { + result <- append(result, list('crpss_score_aggr' = colorsup)) + } + if ('bss10' %in% metrics) { + result <- append(result, list('bss10' = colorsup)) + } + if ('bss90' %in% metrics) { + result <- append(result, list('bss90' = colorsup)) + } + if ('enssprerr' %in% metrics ) { + result <- append(result, list('enssprerr' = colorsup)) + } + if ('rmsss' %in% metrics ) { + result <- append(result, list('rmsss' = colorsup)) + } + return(result) +} + + + + +## Output file name to save scorecard +.Filename <- function(system = NULL, reference = NULL, model = NULL, eval.name = NULL, + var = NULL, start.year = NULL, end.year = NULL, scorecard.type = NULL, + region = NULL, fileout.label = NULL, output.path = NULL) { + + ## Remove . from names + system <- gsub('.','', system, fixed = T) + reference <- gsub('.','', reference, fixed = T) + + period <- paste0(start.year, "-", end.year) + + if (scorecard.type == 1 || scorecard.type == 2 || scorecard.type == 3 || scorecard.type == 4 ) { + scorecard_save_path <- paste0(output.path, + "/scorecard-", scorecard.type, "_", system, "_", + reference, "_", var, "_", period, + fileout.label, ".png") + } else { + scorecard_save_path <- paste0(output.path, + "/scorecard-", scorecard.type, "_multi-", + tolower(model), "_", eval.name, "_", + var, "_", period, "_", region, + fileout.label, ".png") + } + + return(scorecard_save_path) +} + +# Scorecards function to assign background color of table cells, +# color of text in table and to bold the text. +# +# It will return a list with 2 arrays: +# (1) metric.color, A 2-dimensional array with character strings containing the +# color codes for each cell background. +# (2) metric.text.color, A 2-dimensional array with character strings +# containing the color codes for each cell text. +.SCTableColors <- function(table, n.col, n.subcol, n.row, n.subrow, + legend.breaks, palette, colorunder, colorsup) { + # Define rows and columns + n.rows <- n.row * n.subrow + n.columns <- n.col * n.subcol + + ## Set table background colors + metric.color <- array(colorunder, c(n.row * n.subrow, n.columns)) + metric.text.color <- array("#2A2A2A", c(n.row * n.subrow , n.columns)) + # metric.text.bold <- array(TRUE, c(n.row * n.subrow , n.columns - 2)) ## Setting all values to bold + + ## Define cell and text colors to show in table + for (i in 1:n.col) { + metric.int <- legend.breaks[[i]] + for (rr in 1:n.rows) { + for (j in 1:n.subcol) { + for (pp in 1:(length(metric.int) - 1)) { + if (is.nan(table[rr,((i - 1) * n.subcol + j)])) { + metric.color[rr,((i - 1) * n.subcol + j)] <- "gray" + } else { + if (table[rr,((i - 1) * n.subcol + j)] >= + metric.int[pp] && table[rr,((i - 1) * n.subcol + j)] <= + metric.int[pp+1]) { + metric.color[rr,((i - 1) * n.subcol + j)] <- palette[[i]][pp] #palette[pp] + } + if (table[rr,((i - 1) * n.subcol + j)] < metric.int[1]) { + metric.color[rr,((i - 1) * n.subcol + j)] <- colorunder[i] + } + if (table[rr,((i - 1) * n.subcol + j)] >= + metric.int[length(metric.int)]) { + metric.color[rr,((i - 1) * n.subcol + j)] <- colorsup[i] + } + } + ## color text in white and bold if background is white or dark blue or dark red: + if (is.nan(table[rr,((i - 1) * n.subcol + j)]) || + (!is.nan(table[rr,((i - 1) * n.subcol + j)]) && pp == 1 && + table[rr,((i - 1) * n.subcol + j)] < metric.int[2]) || + (!is.nan(table[rr,((i - 1) * n.subcol + j)]) && pp == 2 && + table[rr,((i - 1) * n.subcol + j)] < metric.int[3]) || + (!is.nan(table[rr,((i - 1) * n.subcol + j)]) && pp == (length(metric.int) - 1) && + table[rr,((i - 1) * n.subcol + j)] >= metric.int[length(metric.int) - 1]) || + (!is.nan(table[rr,((i - 1) * n.subcol + j)]) && pp == (length(metric.int) - 2) && + table[rr,((i - 1) * n.subcol + j)] >= metric.int[length(metric.int) - 2])) { + metric.text.color[rr,((i - 1) * n.subcol + j)] <- "white" + #metric.text.bold[rr,((i - 1) * n.subcol + j)] <- TRUE + } + } + } + } + } + + return(list(metric.color = metric.color, + metric.text.color = metric.text.color)) + +} + +# Scorecards function to create the color bar legends for the required metrics +# and paste them below the scorecard table +.SCLegend <- function(legend.breaks, palette, colorunder, colorsup, + label.scale, legend.width, legend.height, + legend.white.space, fileout) { + + ## Create color bar legends for each metric + for (i in 1:length(palette)) { + png(filename = paste0(fileout, '_tmpLegend', i, '.png'), width = legend.width, + height = legend.height) + ColorBar(brks = legend.breaks[[i]], cols = palette[[i]], vertical = FALSE, + label_scale = label.scale, col_inf = colorunder[[i]], + col_sup = colorsup[[i]]) + dev.off() + if (i == 1) { + ## Add white space to the left of the first color bar legend + system(paste0('convert ', fileout, '_tmpLegend1.png -background white -splice ', + legend.white.space, 'x0 ', fileout, '_tmpScorecardLegend.png')) + } else { + system(paste0('convert +append ', fileout, '_tmpScorecardLegend.png ', + fileout, '_tmpLegend', i, '.png ', fileout, + '_tmpScorecardLegend.png')) + } + } + unlink(paste0(fileout,'_tmpLegend*.png')) +} + +# Function to calculate color bar breaks for bias metric +.SCBiasBreaks <- function(data){ + + bias.minmax_max <- quantile(data, 0.98, na.rm = TRUE) + bias.minmax_min <- quantile(data, 0.02, na.rm = TRUE) + bias.max <- max(abs(bias.minmax_min), abs(bias.minmax_max)) + + ## one point more than the colors below (the intervals) + bias.int <- bias.max * c(-1, -0.8, -0.6, -0.4, -0.2, 0, 0.2, 0.4, 0.6, 0.8, 1) + + ## round to 2 significance figures + bias.int <- signif(bias.int, digits = 2) + + return(bias.int) +} + +# Function to calculate color bar breaks for CRPS metric +.SCCrpsBreaks <- function(data){ + + crps.max <- quantile(data, 0.98, na.rm = TRUE) + + crps.int <- crps.max * c(0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1) + + ## round to 2 significance figures + crps.int <- signif(crps.int, digits = 2) + + return(crps.int) +} + +# Auxiliary function to get the names of the longitude coordinate +.KnownLonNames <- function() { + known_lon_names <- c('lon', 'lons', 'longitude', 'x', 'i', 'nav_lon') +} + +# Auxiliary function to get the names of the latitude coordinate +.KnownLatNames <- function() { + known_lat_names <- c('lat', 'lats', 'latitude', 'y', 'j', 'nav_lat') +} diff --git a/modules/Scorecards/R/tmp/WeightedMetrics.R b/modules/Scorecards/R/tmp/WeightedMetrics.R new file mode 100644 index 0000000000000000000000000000000000000000..aea23c566851f523cc8ee19ae2336861e329d0c0 --- /dev/null +++ b/modules/Scorecards/R/tmp/WeightedMetrics.R @@ -0,0 +1,134 @@ +#' Scorecards spatial aggregation of loaded metrics +#' +#'@description Scorecards function to perform the spatial aggregation of the +#' loaded metrics for the specified regions. +#' +#'@param loaded_metrics is a list of arrays containing the metrics loaded by the +#' function SC_load_metrics. +#'@param region is a named list of vectors containing the desired regions to +#' analyze. For each region the following should be specified in this order: +#' lon_min, lon_max, lat_min, lat_max. +#'@param metric.aggregation a character indicating whether the skill score RPS +#' and CRPSS are calculated from aggregated scores or aggregated skill score +#' directly, either 'score' or 'skill' respectively +#'@param ncores is the number of cores to use for the calculation. +#' +#'@return An array with the following dimensions: system, reference, metrics, +#' time, sdate, region. +#' +#'@examples +#'regions <- list('global' = c(lon.min = 0, lon.max = 360, lat.min = -90, lat.max = 90), +#' 'europe' = c(lon.min = -10, lon.max = 40, lat.min = 30, lat.max = 70)) +#'aggregated_metrics <- WeightedMetrics(loaded_metrics, +#' regions = regions, +#' metric.aggregation = 'skill', +#' ncores = 4) +#'@import multiApply +#'@importFrom ClimProjDiags WeightedMean +#'@importFrom s2dv Reorder +#'@export +WeightedMetrics <- function(loaded_metrics, regions, metric.aggregation, + ncores = NULL, na.rm = TRUE) { + ## Initial checks + # loaded_metrics + if (any(sapply(loaded_metrics, function(x) { + sapply(x, function(y) {is.null(dim(y))}) + }))) { + stop(paste0("Parameter 'loaded_metrics' must be a list of lists of arrays ", + "with named dimensions.")) + } + # regions + if (!all(sapply(regions, is.numeric))) { + stop(paste0("Parameter 'regions' must be a named list of vectors ", + "containing the desired regions to analyze.")) + } + # metric.aggregation + if (!is.character(metric.aggregation)) { + stop("Parameter 'metric.aggregation' must be a character indicating.") + } + # ncores + if (!is.numeric(ncores)) { + stop("Parameter 'ncores' must be an integer.") + } + + ## Get metric names + ## TO DO: check all metric are in the same order for all sys + metrics <- attributes(loaded_metrics[[1]][[1]])$metrics + forecast.months <- attributes(loaded_metrics[[1]][[1]])$forecast.months + start.months <- attributes(loaded_metrics[[1]][[1]])$start.months + + all_metric_means <- array(dim = c(metric = length(metrics), + time = length(forecast.months), + sdate = length(start.months), + region = length(regions), + reference = length(loaded_metrics[[1]]), + system = length(loaded_metrics))) + + ## Loop over system + for (sys in 1:length(loaded_metrics)) { + ## Loop over reference + for (ref in 1:length(loaded_metrics[[sys]])) { + dimnames <- names(dim(loaded_metrics[[sys]][[ref]])) + lon_dim_name <- dimnames[which(dimnames %in% .KnownLonNames())] + lat_dim_name <- dimnames[which(dimnames %in% .KnownLatNames())] + ## Get latitude and longitude from attributes of loaded metrics + ## Loop over region + for (reg in 1:length(regions)) { + ## Calculate weighted means for defined regions for each system and reference + weighted.mean <- WeightedMean(data = loaded_metrics[[sys]][[ref]], + lon = as.vector(attributes(loaded_metrics[[sys]][[ref]])$lon), + lat = as.vector(attributes(loaded_metrics[[sys]][[ref]])$lat), + region = regions[[reg]], + londim = lon_dim_name, + latdim = lat_dim_name, + na.rm = na.rm, + ncores = ncores) + all_metric_means[, , , reg, ref, sys] <- weighted.mean + } ## close loop on region + } ## close loop on reference + } ## close loop on system + + ## skill aggregation: + if (metric.aggregation == 'score') { + if (all(c("rps", "rps_clim") %in% metrics)) { + ## Calculate RPSS from aggregated RPS and RPS_clim + all_metric_means <- multiApply::Apply(data = all_metric_means, + target_dims = 'metric', + fun = function(x, met) { + res <- 1 - x[which(met == 'rps')] / x[which(met == 'rps_clim')] + c(x, res)}, met = metrics, + output_dims = 'metric', + ncores = ncores)$output1 + ## Define name of newly calculated RPSS metric + metrics <- c(metrics, "rpss_score_aggr") + } + if (all(c("crps", "crps_clim") %in% metrics)) { + ## Calculate CRPSS from aggragated CRPS and CRPS_clim + all_metric_means <- multiApply::Apply(data = all_metric_means, + target_dims = 'metric', + fun = function(x, met) { + res <- 1 - x[which(met == 'crps')] / x[which(met == 'crps_clim')] + c(x, res)}, + met = metrics, + output_dims = 'metric', + ncores = ncores)$output1 + ## Define name of newly calculated CRPSS metric + metrics <- c(metrics, "crpss_score_aggr") + } + ## Add warning in case metric.aggregation == 'score' but 1 of the metrics from each pair is missing + } + ## reorder dimensions in array + all_metric_means <- s2dv::Reorder(all_metric_means, c('system','reference','metric','time','sdate','region')) + + ## Add attributes + attributes(all_metric_means)$metrics <- metrics + attributes(all_metric_means)$start.months <- attributes(loaded_metrics[[1]][[1]])$start.months + attributes(all_metric_means)$forecast.months <- attributes(loaded_metrics[[1]][[1]])$forecast.months + attributes(all_metric_means)$regions <- regions + attributes(all_metric_means)$system.name <- names(loaded_metrics) + attributes(all_metric_means)$reference.name <- names(loaded_metrics[[1]]) + + return(all_metric_means) + +} ## close function + diff --git a/modules/Scorecards/Scorecards.R b/modules/Scorecards/Scorecards.R new file mode 100644 index 0000000000000000000000000000000000000000..0dbcd9210c9227200c872204526ea4e6df05adcb --- /dev/null +++ b/modules/Scorecards/Scorecards.R @@ -0,0 +1,192 @@ +############################################################################### +##################### SCORECARDS MODULE FOR SUNSET SUITE ###################### +############################################################################### + +##### Load source functions ##### +source('modules/Scorecards/R/tmp/LoadMetrics.R') +source('modules/Scorecards/R/tmp/WeightedMetrics.R') +source('modules/Scorecards/R/tmp/Utils.R') +source('modules/Scorecards/R/tmp/SCTransform.R') +source('modules/Scorecards/R/tmp/ScorecardsSingle.R') +source('modules/Scorecards/R/tmp/ScorecardsMulti.R') +source('modules/Scorecards/R/tmp/ScorecardsSystemDiff.R') +source('modules/Scorecards/R/tmp/SCPlotScorecard.R') + + +## TODO: Change function name to 'Scorecards'? +## Define function +Scorecards <- function(recipe) { + + ## set parameters + input.path <- paste0(recipe$Run$output_dir, "/outputs/Skill/") + output.path <- paste0(recipe$Run$output_dir, "/plots/Scorecards/") + dir.create(output.path, recursive = T, showWarnings = F) + + system <- recipe$Analysis$Datasets$System$name + reference <- recipe$Analysis$Datasets$Reference$name + var <- recipe$Analysis$Variables$name + start.year <- as.numeric(recipe$Analysis$Time$hcst_start) + end.year <- as.numeric(recipe$Analysis$Time$hcst_end) + forecast.months <- recipe$Analysis$Time$ftime_min : recipe$Analysis$Time$ftime_max + + if (recipe$Analysis$Workflow$Scorecards$start_months == 'all') { + start.months <- 1:12 + } else { + start.months <- as.numeric(strsplit(recipe$Analysis$Workflow$Scorecards$start_months, + split = ", | |,")[[1]]) + } + + regions <- recipe$Analysis$Workflow$Scorecards$regions + for (i in names(regions)){regions[[i]] <- unlist(regions[[i]])} + + metric.aggregation <- recipe$Analysis$Workflow$Scorecards$metric_aggregation + metrics.load <- unlist(strsplit(tolower(recipe$Analysis$Workflow$Skill$metric), ", | |,")) + + ## Define skill scores in score aggregation has been requested + + if(metric.aggregation == 'score'){ + if('rps' %in% metrics.load){ + metrics.load <- c(metrics.load, 'rps_clim') + } + if('crps' %in% metrics.load){ + metrics.load <- c(metrics.load, 'crps_clim') + } + } + + metrics.visualize <- unlist(strsplit(tolower(recipe$Analysis$Workflow$Scorecards$metric), ", | |,")) + + ## Define skill scores in score aggregation has been requested + + if(metric.aggregation == 'score'){ + if('rpss' %in% metrics.visualize){ + metrics.visualize[metrics.visualize == 'rpss'] <- 'rpss_score_aggr' + } + if('crpss' %in% metrics.visualize){ + metrics.visualize[metrics.visualize == 'crpss'] <- 'crpss_score_aggr' + } + } + + inf.to.na <- recipe$Analysis$Workflow$Scorecards$inf_to_na + table.label <- recipe$Analysis$Workflow$Scorecards$table_label + fileout.label <- recipe$Analysis$Workflow$Scorecards$fileout_label + legend.white.space <- recipe$Analysis$Workflow$Scorecards$legend_white_space + col1.width <- recipe$Analysis$Workflow$Scorecards$col1_width + col2.width <- recipe$Analysis$Workflow$Scorecards$col2_width + calculate.diff <- recipe$Analysis$Workflow$Scorecards$calculate_diff + ncores <- 1 # recipe$Analysis$ncores + + ## Load data files + loaded_metrics <- LoadMetrics(system = system, + reference = reference, + var = var, + start.year = start.year, + end.year = end.year, + metrics = metrics.load, + start.months = start.months, + forecast.months = forecast.months, + inf_to_na = inf.to.na, + input.path = input.path) + + + if('region' %in% names(dim(loaded_metrics[[1]][[1]]))){ + + ### Convert loaded metrics to array for allready aggregated data + metrics.dim <- attributes(loaded_metrics[[1]][[1]])$metrics + forecast.months.dim <- attributes(loaded_metrics[[1]][[1]])$forecast.months + start.months.dim <- attributes(loaded_metrics[[1]][[1]])$start.months + regions.dim <- regions #list('NAO' = c(lon.min = -80, lon.max = 40, lat.min = 20, lat.max = 80)) + + aggregated_metrics <- array(dim = c(system = length(loaded_metrics), + reference = length(loaded_metrics[[1]]), + metric = length(metrics.dim), + time = length(forecast.months.dim), + sdate = length(start.months.dim), + region = length(regions.dim))) + + + for (sys in 1:length(names(loaded_metrics))){ + for (ref in 1:length(names(loaded_metrics[[sys]]))){ + aggregated_metrics[sys, ref, , , , ] <- s2dv::Reorder(data = loaded_metrics[[sys]][[ref]], order = c('metric','time','sdate','region')) + } + } + + ## Add attributes + attributes(aggregated_metrics)$metrics <- metrics.load + attributes(aggregated_metrics)$start.months <- attributes(loaded_metrics[[1]][[1]])$start.months + attributes(aggregated_metrics)$forecast.months <- attributes(loaded_metrics[[1]][[1]])$forecast.months + attributes(aggregated_metrics)$regions <- regions + attributes(aggregated_metrics)$system.name <- names(loaded_metrics) + attributes(aggregated_metrics)$reference.name <- names(loaded_metrics[[1]]) + + + } else { + ## Calculate weighted mean of spatial aggregation + aggregated_metrics <- WeightedMetrics(loaded_metrics, + regions = regions, + metric.aggregation = metric.aggregation, + ncores = ncores) + }## close if + + ## Create simple scorecard tables + ## (one system only) + ## Metrics input must be in the same order as function SC_spatial_aggregation + scorecard_single <- ScorecardsSingle(data = aggregated_metrics, + system = system, + reference = reference, + var = var, + start.year = start.year, + end.year = end.year, + start.months = start.months, + forecast.months = forecast.months, + region.names = names(regions), + metrics = metrics.visualize, + table.label = table.label, + fileout.label = fileout.label, + legend.white.space = legend.white.space, + col1.width = col1.width, + col2.width = col2.width, + output.path = output.path) + + ## Create multi system/reference scorecard tables + ## (multiple systems with one reference or one system with multiple references) + ## Metrics input must be in the same order as function SC_spatial_aggregation + if(length(system) > 1 || length(reference) > 1){ + scorecard_multi <- ScorecardsMulti(data = aggregated_metrics, + system = system, + reference = reference, + var = var, + start.year = start.year, + end.year = end.year, + start.months = start.months, + forecast.months = forecast.months, + region.names = attributes(regions)$names, + metrics = metrics.visualize, + table.label = table.label, + fileout.label = fileout.label, + output.path = output.path) + } ## close if + + + if(calculate.diff == TRUE){ + if(length(system) == 2 || length(reference) == 2){ + scorecard_diff <- ScorecardsSystemDiff(data = aggregated_metrics, + system = system, + reference = reference, + var = var, + start.year = start.year, + end.year = end.year, + start.months = start.months, + forecast.months = forecast.months, + region.names = attributes(regions)$names, + metrics = metrics.visualize, + table.label = table.label, + fileout.label = fileout.label, + legend.white.space = legend.white.space, + col1.width = col1.width, + col2.width = col2.width, + output.path = output.path) + } else {stop ("Difference scorecard can only be computed with two systems or two references.")} + } ## close if on calculate.diff + +} + diff --git a/modules/Scorecards/execute_scorecards.R b/modules/Scorecards/execute_scorecards.R new file mode 100644 index 0000000000000000000000000000000000000000..2c54c48f45ca0f70ecb26370e396d624948b7c0b --- /dev/null +++ b/modules/Scorecards/execute_scorecards.R @@ -0,0 +1,31 @@ +source('tools/libs.R') +source('modules/Scorecards/Scorecards.R') + +args = commandArgs(trailingOnly = TRUE) +recipe_file <- args[1] +output_dir <- args[2] + +## TODO: Replace with function +# Read recipe and set outdir +recipe <- read_yaml(recipe_file) +recipe$Run$output_dir <- output_dir + +## Loop over variables +datasets <- recipe$Analysis$Datasets +## TODO: Improve dependency system? +for (system in 1:length(datasets$System)) { + for (reference in 1:length(datasets$Reference)) { + for (variable in 1:length(recipe$Analysis$Variables)) { + scorecard_recipe <- recipe + scorecard_recipe$Analysis$Datasets$System <- + recipe$Analysis$Datasets$System[[system]] + scorecard_recipe$Analysis$Datasets$Reference <- + recipe$Analysis$Datasets$Reference[[reference]] + scorecard_recipe$Analysis$Variables <- + recipe$Analysis$Variables[[variable]] + # Plot Scorecards + Scorecards(scorecard_recipe) + } + } +} +print("##### SCORECARDS SAVED TO THE OUTPUT DIRECTORY #####") diff --git a/modules/Skill/Skill.R b/modules/Skill/Skill.R index 7fff7de4d529b4d4e28ff4c2e9d7dc116d9fd31a..0b5f729fe982d60a673fba9e74c5b31cc9f2d5f2 100644 --- a/modules/Skill/Skill.R +++ b/modules/Skill/Skill.R @@ -306,6 +306,7 @@ Skill <- function(recipe, data, agg = 'global') { } } info(recipe$Run$logger, "##### SKILL METRIC COMPUTATION COMPLETE #####") + .log_memory_usage(recipe$Run$logger, when = "After skill metric computation") # Save outputs #NURIA: I think change the output_dir is a problem for future savings if (recipe$Analysis$Workflow$Skill$save != 'none') { @@ -432,7 +433,8 @@ Probabilities <- function(recipe, data) { } info(recipe$Run$logger, - "##### PERCENTILES AND PROBABILITY CATEGORIES COMPUTED #####") + "##### PERCENTILES AND PROBABILITY CATEGORIES COMPUTED #####") + .log_memory_usage(recipe$Run$logger, when = "After anomaly computation") # Save outputs if (recipe$Analysis$Workflow$Probabilities$save != 'none') { info(recipe$Run$logger, diff --git a/modules/test_GRIB.R b/modules/test_GRIB.R new file mode 100644 index 0000000000000000000000000000000000000000..27cf3c9aa08df54e3c44872518bd7142e376c9f5 --- /dev/null +++ b/modules/test_GRIB.R @@ -0,0 +1,52 @@ +source('modules/Loading/GRIB/GrbLoad.R') +source('modules/Loading//Loading_GRIB.R') +source("tools/libs.R") # for prepare_outputs.R + +recipe <- "modules/Loading/testing_recipes/recipe_GRIB_system5_era5.yml" +recipe <- prepare_outputs(recipe) + +# Load datasets +data <- load_datasets(recipe) + +str(data) + +#============================ +# Test the remaining modules +#============================ +source("modules/Calibration/Calibration.R") +source("modules/Skill/Skill.R") +source("modules/Saving/Saving.R") +source("modules/Visualization/Visualization.R") + +# Calibrate data +suppressWarnings({invisible(capture.output( +calibrated_data <- calibrate_datasets(recipe, data) +))}) +#pryr::mem_used() +#975 MB + +# Compute skill metrics +suppressWarnings({invisible(capture.output( +skill_metrics <- compute_skill_metrics(recipe, calibrated_data) +))}) + +suppressWarnings({invisible(capture.output( +probs <- compute_probabilities(recipe, calibrated_data) +))}) + +# Saving +suppressWarnings({invisible(capture.output( +save_data(recipe = recipe, data = calibrated_data, + skill_metrics = skill_metrics, probabilities = probs, + archive = read_yaml("conf/archive_GRIB.yml")$archive) +))}) + +# Plotting +suppressWarnings({invisible(capture.output( +plot_data(recipe = recipe, data = calibrated_data, + skill_metrics = skill_metrics, probabilities = probs, + significance = T, + archive = read_yaml("conf/archive_GRIB.yml")$archive) +))}) +outdir <- get_dir(recipe) + diff --git a/modules/test_parallel_GRIB.R b/modules/test_parallel_GRIB.R new file mode 100644 index 0000000000000000000000000000000000000000..a13f9e5911bb2bebd40c19cbb9c79b8619cd8a96 --- /dev/null +++ b/modules/test_parallel_GRIB.R @@ -0,0 +1,27 @@ +source("modules/Loading/GRIB/GrbLoad.R") +source("modules/Loading/Loading_GRIB.R") +source('modules/Loading/GRIB/s2dv_cube.R') +source("modules/Calibration/Calibration.R") +source("modules/Anomalies/Anomalies.R") +source("modules/Skill/Skill.R") +source("modules/Saving/Saving.R") +source("modules/Visualization/Visualization.R") + +args = commandArgs(trailingOnly = TRUE) +recipe_file <- args[1] +recipe <- read_atomic_recipe(recipe_file) +# Load datasets +data <- load_datasets(recipe) +# Calibrate datasets +data <- calibrate_datasets(recipe, data) +# Compute anomalies +data <- compute_anomalies(recipe, data) +# Compute skill metrics +skill_metrics <- compute_skill_metrics(recipe, data) +# Compute percentiles and probability bins +probabilities <- compute_probabilities(recipe, data) +# Export all data to netCDF +save_data(recipe, data, skill_metrics, probabilities) +# Plot data +plot_data(recipe, data, skill_metrics, probabilities, + significance = T) diff --git a/recipes/atomic_recipes/recipe_scorecards_atomic.yml b/recipes/atomic_recipes/recipe_scorecards_atomic.yml new file mode 100644 index 0000000000000000000000000000000000000000..a8adb6c12f6c388c05798734f53c023426995b02 --- /dev/null +++ b/recipes/atomic_recipes/recipe_scorecards_atomic.yml @@ -0,0 +1,70 @@ +Description: + Author: nmilders + Info: scorecards data + +Analysis: + Horizon: seasonal # Mandatory, str: either subseasonal, seasonal, or decadal + Variables: + name: tas # Mandatory, str: tas prlr psl sfcWind + freq: monthly_mean # Mandatory, str: either monthly_mean or daily_mean + Datasets: + System: + name: ECMWF-SEAS5 # Mandatory ECMWF-SEAS5, CMCC-SPS3.5, DWD-GCFS2.1 + Multimodel: no # Mandatory, bool: Either yes/true or no/false + Reference: + name: ERA5 # Mandatory, str: Reference codename. See docu. + Time: + sdate: '0101' ## MMDD + fcst_year: # Optional, int: Forecast year 'YYYY' + hcst_start: '1993' # Mandatory, int: Hindcast start year 'YYYY' + hcst_end: '2016' # Mandatory, int: Hindcast end year 'YYYY' + ftime_min: 1 # Mandatory, int: First leadtime time step in months + ftime_max: 6 # Mandatory, int: Last leadtime time step in months + Region: + latmin: -90 # Mandatory, int: minimum latitude + latmax: 90 # Mandatory, int: maximum latitude + lonmin: 0 # Mandatory, int: minimum longitude + lonmax: 359.9 # Mandatory, int: maximum longitude + Regrid: + method: bilinear # conservative for prlr, bilinear for tas, psl, sfcWind + type: to_system + Workflow: + Calibration: + method: raw # Mandatory, str: Calibration method. See docu. + save: 'none' + Anomalies: + compute: yes + cross_validation: yes + save: 'none' + Skill: + metric: mean_bias EnsCorr rps rpss crps crpss EnsSprErr # str: Skill metric or list of skill metrics. See docu. + cross_validation: yes + save: 'all' + Probabilities: + percentiles: [[1/3, 2/3]] # frac: Quantile thresholds. + save: 'none' + Indicators: + index: no + Scorecards: + execute: yes # yes/no + regions: + Extra-tropical NH: {lon.min: 0, lon.max: 360, lat.min: 30, lat.max: 90} + Tropics: {lon.min: 0, lon.max: 360, lat.min: -30, lat.max: 30} + Extra-tropical SH : {lon.min: 0, lon.max: 360, lat.min: -30, lat.max: -90} + start_months: 'all' + metric: mean_bias enscorr rpss crpss enssprerr + metric_aggregation: 'score' + inf_to_na: TRUE + table_label: + fileout_label: + col1_width: + col2_width: + calculate_diff: FALSE + ncores: 7 # Optional, int: number of cores, defaults to 1 + remove_NAs: # Optional, bool: Whether NAs are removed, defaults to FALSE + Output_format: Scorecards #S2S4E +Run: + Loglevel: INFO + Terminal: yes + output_dir: /esarchive/scratch/nmilders/scorecards_data/new/to_system/cross_validation/all_cross_val/ + code_dir: /esarchive/scratch/nmilders/gitlab/git_clones/s2s-suite/ diff --git a/recipes/examples/recipe_GRIB_system5_era5.yml b/recipes/examples/recipe_GRIB_system5_era5.yml new file mode 100644 index 0000000000000000000000000000000000000000..71615afdd2364857fadab774acc6f8f08932cbfc --- /dev/null +++ b/recipes/examples/recipe_GRIB_system5_era5.yml @@ -0,0 +1,51 @@ +Description: + Author: A. Ho + +Analysis: + Horizon: Seasonal + Variables: + name: tas + freq: monthly_mean + Datasets: + System: + name: ECMWF-SEAS5 + Multimodel: False + Reference: + name: ERA5 + Time: + sdate: '1101' + fcst_year: '2020' + hcst_start: '2000' + hcst_end: '2003' + ftime_min: 1 + ftime_max: 3 + Region: #NOT USED + latmin: -90 + latmax: 90 + lonmin: 0 + lonmax: 360 + Regrid: + method: bilinear + type: 'r360x180' #to_system + Workflow: + Calibration: + method: mse_min + Anomalies: + compute: yes + cross_validation: yes + Skill: + metric: mean_bias + Probabilities: + percentiles: [[1/3, 2/3], [1/10, 9/10]] + Indicators: + index: no + ncores: 8 + remove_NAs: yes + Output_format: S2S4E #Scorecards +Run: + Loglevel: INFO + Terminal: yes + output_dir: ./tests/out-logs/ + code_dir: /esarchive/scratch/aho/git/auto-s2s/ + filesystem: mars + diff --git a/recipes/recipe_scorecards.yml b/recipes/recipe_scorecards.yml new file mode 100644 index 0000000000000000000000000000000000000000..434426d02d499db78c6884c6f6a9322390935526 --- /dev/null +++ b/recipes/recipe_scorecards.yml @@ -0,0 +1,105 @@ +################################################################################ +## RECIPE DESCRIPTION +################################################################################ + +Description: + Author: V. Agudetse + Info: Test for recipe splitting + +################################################################################ +## ANALYSIS CONFIGURATION +################################################################################ + +Analysis: + Horizon: Seasonal + Variables: # ECVs and Indicators? + - {name: tas, freq: monthly_mean} + Datasets: + System: # multiple systems for single model, split if Multimodel = F + - {name: ECMWF-SEAS5} + Multimodel: False # single option + Reference: + - {name: ERA5} # multiple references for single model? + Time: + sdate: # list, split + - '0101' + - '0201' + - '0301' + - '0401' + - '0501' + - '0601' + - '0701' + - '0801' + - '0901' + - '1001' + - '1101' + - '1201' + #fcst_year: '2020' # list, don't split, handled internally + hcst_start: '1993' # single option + hcst_end: '2016' # single option + ftime_min: 1 # single option + ftime_max: 6 # single option + Region: # multiple lists, split? Add region name if length(Region) > 1 + - {name: "global", latmin: -90, latmax: 90, lonmin: 0, lonmax: 359.9} + Regrid: + method: bilinear ## TODO: allow multiple methods? + type: to_system + Workflow: + Anomalies: + compute: yes + cross_validation: yes + save: 'none' + Calibration: + method: raw ## TODO: list, split? + save: 'none' + Skill: + metric: mean_bias EnsCorr rps rpss crps crpss EnsSprErr # list, don't split + cross_validation: yes + save: 'all' + Probabilities: + percentiles: [[1/3, 2/3]] # list, don't split + save: 'none' + Visualization: + plots: skill_metrics + Indicators: + index: no # ? + Scorecards: + execute: yes # yes/no + regions: + Extra-tropical NH: {lon.min: 0, lon.max: 360, lat.min: 30, lat.max: 90} + Tropics: {lon.min: 0, lon.max: 360, lat.min: -30, lat.max: 30} + Extra-tropical SH : {lon.min: 0, lon.max: 360, lat.min: -30, lat.max: -90} + start_months: NULL + metric: mean_bias enscorr rpss crpss enssprerr + metric_aggregation: 'score' + table_label: NULL + fileout_label: NULL + col1_width: NULL + col2_width: NULL + calculate_diff: FALSE + ncores: 7 + remove_NAs: no # bool, don't split + Output_format: Scorecards # string, don't split + +################################################################################ +## Run CONFIGURATION +################################################################################ +Run: + Loglevel: INFO + Terminal: yes + output_dir: /esarchive/scratch/nmilders/scorecards_data/test/ + code_dir: /esarchive/scratch/nmilders/gitlab/git_clones/s2s-suite/ + autosubmit: yes + # fill only if using autosubmit + auto_conf: + script: /esarchive/scratch/nmilders/gitlab/git_clones/s2s-suite/execute_scorecards_data_loading.R # replace with the path to your script + expid: a6a3 # replace with your EXPID + hpc_user: bsc32878 # replace with your hpc username + wallclock: 03:00 # hh:mm + processors_per_job: 8 + platform: nord3v2 + email_notifications: yes # enable/disable email notifications. Change it if you want to. + email_address: nadia.milders@bsc.es # replace with your email address + notify_completed: yes # notify me by email when a job finishes + notify_failed: yes # notify me by email when a job fails + diff --git a/recipes/recipe_scorecards_vic.yml b/recipes/recipe_scorecards_vic.yml new file mode 100644 index 0000000000000000000000000000000000000000..fbd2cb90e2cd162f90e517bcbd82fe634783b8e0 --- /dev/null +++ b/recipes/recipe_scorecards_vic.yml @@ -0,0 +1,106 @@ +################################################################################ +## RECIPE DESCRIPTION +################################################################################ + +Description: + Author: V. Agudetse + Info: Test for recipe splitting + +################################################################################ +## ANALYSIS CONFIGURATION +################################################################################ + +Analysis: + Horizon: Seasonal + Variables: # ECVs and Indicators? + - {name: tas, freq: monthly_mean} + Datasets: + System: # multiple systems for single model, split if Multimodel = F + - {name: ECMWF-SEAS5} + Multimodel: False # single option + Reference: + - {name: ERA5} # multiple references for single model? + Time: + sdate: # list, split + - '0101' + - '0201' + - '0301' + - '0401' + - '0501' + - '0601' + - '0701' + - '0801' + - '0901' + - '1001' + - '1101' + - '1201' + fcst_year: + hcst_start: '1993' # single option + hcst_end: '2003' # single option + ftime_min: 1 # single option + ftime_max: 6 # single option + Region: # multiple lists, split? Add region name if length(Region) > 1 + - {name: "global", latmin: -90, latmax: 90, lonmin: 0, lonmax: 359.9} + Regrid: + method: bilinear ## TODO: allow multiple methods? + type: to_system + Workflow: + Anomalies: + compute: yes + cross_validation: no + save: 'none' + Calibration: + method: raw ## TODO: list, split? + save: 'none' + Skill: + metric: mean_bias EnsCorr rps rpss crps crpss EnsSprErr # list, don't split + cross_validation: yes + save: 'all' + Probabilities: + percentiles: [[1/3, 2/3]] # list, don't split + save: 'none' + # Visualization: + # plots: skill_metrics + Indicators: + index: no # ? + Scorecards: + execute: yes # yes/no + regions: + Extra-tropical NH: {lon.min: 0, lon.max: 360, lat.min: 30, lat.max: 90} + Tropics: {lon.min: 0, lon.max: 360, lat.min: -30, lat.max: 30} + Extra-tropical SH : {lon.min: 0, lon.max: 360, lat.min: -30, lat.max: -90} + start_months: NULL + metric: mean_bias enscorr rpss crpss enssprerr + metric_aggregation: 'score' + table_label: NULL + fileout_label: NULL + col1_width: NULL + col2_width: NULL + calculate_diff: FALSE + ncores: 14 + remove_NAs: no # bool, don't split + Output_format: Scorecards # string, don't split + +################################################################################ +## Run CONFIGURATION +################################################################################ +Run: + Loglevel: INFO + Terminal: yes + output_dir: /esarchive/scratch/vagudets/auto-s2s-outputs/ + code_dir: /esarchive/scratch/vagudets/repos/auto-s2s/ + autosubmit: yes + # fill only if using autosubmit + auto_conf: + script: /esarchive/scratch/vagudets/repos/auto-s2s/example_scripts/test_scorecards_workflow.R # replace with the path to your script + expid: a6ae # replace with your EXPID + hpc_user: bsc32762 # replace with your hpc username + wallclock: 03:00 # hh:mm + processors_per_job: 14 + platform: nord3v2 + custom_directives: ['#SBATCH --exclusive'] + email_notifications: yes # enable/disable email notifications. Change it if you want to. + email_address: victoria.agudetse@bsc.es # replace with your email address + notify_completed: yes # notify me by email when a job finishes + notify_failed: no # notify me by email when a job fails + diff --git a/recipes/tests/recipe_autosubmit_marstest.yml b/recipes/tests/recipe_autosubmit_marstest.yml index 2fe903fa61a210c7b5d51a091cc47ff6edb72af8..24b907977f3ac2fde7f0cea04bae35f91fb0d990 100644 --- a/recipes/tests/recipe_autosubmit_marstest.yml +++ b/recipes/tests/recipe_autosubmit_marstest.yml @@ -4,7 +4,7 @@ Description: Author: V. Agudetse - Info: Test Independent verification of two variables, two sdates, two systems + Info: Test Autosubmit WF for CERISE ################################################################################ ## ANALYSIS CONFIGURATION @@ -14,28 +14,25 @@ Analysis: Horizon: Seasonal Variables: # ECVs and Indicators? - {name: tas, freq: monthly_mean} - - {name: prlr, freq: monthly_mean} Datasets: System: # multiple systems for single model, split if Multimodel = F - {name: ECMWF-SEAS5} - - {name: Meteo-France-System7} Multimodel: False # single option Reference: - {name: ERA5} # multiple references for single model? Time: sdate: # list, split - - '0101' - - '0201' + - '1101' fcst_year: # list, don't split, handled internally hcst_start: '2000' # single option - hcst_end: '2016' # single option + hcst_end: '2019' # single option ftime_min: 1 # single option - ftime_max: 6 # single option + ftime_max: 4 # single option Region: # multiple lists, Add region name if there is more than 1 region - - {latmin: -10, latmax: 10, lonmin: -10, lonmax: 10} + - {latmin: -90, latmax: 90, lonmin: 0, lonmax: 360} Regrid: - method: conservative - type: to_system + method: bilinear + type: r360x180 Workflow: Anomalies: compute: yes @@ -56,7 +53,7 @@ Analysis: projection: robinson Indicators: index: no # ? - ncores: 8 + ncores: 14 remove_NAs: yes # bool, don't split Output_format: S2S4E # string, don't split @@ -64,7 +61,7 @@ Analysis: ## Run CONFIGURATION ################################################################################ Run: - Loglevel: INFO + Loglevel: DEBUG Terminal: yes filesystem: mars output_dir: /esarchive/scratch/vagudets/auto-s2s-outputs/ @@ -75,8 +72,8 @@ Run: script: /esarchive/scratch/vagudets/repos/auto-s2s/example_scripts/test_parallel_workflow.R # path to the script to run expid: a5ta ## if left empty, create new exp? hpc_user: bsc32762 # your hpc username - wallclock: 04:00 # hh:mm - processors_per_job: 8 # use ncores parameter? + wallclock: 02:30 # hh:mm + processors_per_job: 14 # use ncores parameter? platform: nord3v2 # make this constant? email_notifications: yes # enable/disable email notifications email_address: victoria.agudetse@bsc.es # email address for notifications diff --git a/recipes/tests/recipe_seasonal_example.yml b/recipes/tests/recipe_seasonal_example.yml index ccd2c7f29ec443713bf2e6f8c8ca8f2d4b9eac7b..5d283fb901d6bf8843b3a8b90183b10c91477ec6 100644 --- a/recipes/tests/recipe_seasonal_example.yml +++ b/recipes/tests/recipe_seasonal_example.yml @@ -30,7 +30,7 @@ Analysis: hcst_start: '2000' hcst_end: '2016' ftime_min: 1 - ftime_max: 6 + ftime_max: 3 Region: - {latmin: -10, latmax: 10, lonmin: -10, lonmax: 10} Regrid: @@ -56,7 +56,7 @@ Analysis: projection: cylindrical_equidistant Indicators: index: no - ncores: 8 + ncores: 10 remove_NAs: yes Output_format: S2S4E @@ -73,10 +73,11 @@ Run: # fill only if using autosubmit auto_conf: script: /esarchive/scratch/vagudets/repos/auto-s2s/example_scripts/test_parallel_workflow.R - expid: a5no # autosubmit experiment ID + expid: a6jr # autosubmit experiment ID hpc_user: bsc32762 # your hpc username - wallclock: 04:00 # hh:mm + wallclock: 01:00 # hh:mm processors_per_job: 8 # use ncores parameter? + custom_directives: ['#SBATCH --constraint=medmem', '#SBATCH --exclusive'] platform: nord3v2 # make this constant? email_notifications: yes # enable/disable email notifications email_address: victoria.agudetse@bsc.es # email address for notifications diff --git a/split.R b/split.R index 6976b8a8b0029e7d8b641cb60c0edd66eae52838..a7943734b06470321f9178e5a5c72d3158ccacd7 100755 --- a/split.R +++ b/split.R @@ -9,11 +9,12 @@ configuration is also written in the folder of the specified experiment. Instructions on how to run the experiment will appear in the terminal. Usage: - split.R [--disable_unique_ID] + split.R [--disable_unique_ID] [--tmpfile=] Options: -h --help Show usage. --disable_unique_ID Do not add a unique ID to the output folder name. + --tmpfile= Temporary file to store output and code directories. Handled by launch_SUNSET.sh. Arguments: recipe path to the recipe." @@ -31,8 +32,27 @@ recipe <- prepare_outputs(recipe_file = arguments$recipe, ## TODO: Add autosubmit yes/no to the parameters? run_parameters <- divide_recipe(recipe) + if (recipe$Run$autosubmit) { write_autosubmit_conf(recipe, run_parameters$n_atomic_recipes) -} else { - run_parameters$outdir + sink(arguments$tmpfile, append = FALSE) + # Run with... + cat("autosubmit") + sink() +} else if (!is.null(arguments$tmpfile)) { + sink(arguments$tmpfile, append = FALSE) + # Run with... + cat(paste0("sbatch", "\n")) + # Code directory + cat(paste0(recipe$Run$code_dir, "\n")) + # Output directory + cat(paste0(run_parameters$outdir, "\n")) + # Scorecards + if (!("Scorecards" %in% names(recipe$Analysis$Workflow)) || + (!recipe$Analysis$Workflow$Scorecards$execute)) { + cat("FALSE") + } else { + cat("TRUE") + } + sink() } diff --git a/tools/Utils.R b/tools/Utils.R new file mode 100644 index 0000000000000000000000000000000000000000..c0acf3740a10d1d9da97d8363b52f5050260c97d --- /dev/null +++ b/tools/Utils.R @@ -0,0 +1,13 @@ +## TODO: Write header +## TODO: Add if 'DEBUG' +.log_memory_usage <- function(logger, when) { + debug(logger, paste0(when, ":")) + mem_info <- capture.output(memuse::Sys.meminfo()) + for (i in mem_info) { + debug(recipe$Run$logger, i) + } + proc_mem <- capture.output(memuse::Sys.procmem()) + for (i in proc_mem) { + debug(recipe$Run$logger, i) + } +} diff --git a/tools/check_recipe.R b/tools/check_recipe.R index 7709e9158de07d713498ce8462e3448a0d777b4f..b7b90ad43e56e5c2bef023600827e165e286de99 100644 --- a/tools/check_recipe.R +++ b/tools/check_recipe.R @@ -442,11 +442,27 @@ check_recipe <- function(recipe) { } # Skill - if (("Skill" %in% names(recipe$Analysis$Workflow)) && - (is.null(recipe$Analysis$Workflow$Skill$metric))) { + AVAILABLE_METRICS <- c("enscorr", "corr", "rps", "rpss", "frps", "frpss", + "crps", "crpss", "bss10", "bss90", "mean_bias", + "mean_bias_ss", "enssprerr", "rps_clim", "rpss_clim", + "enscorr_specs", "frps_specs", "rpss_specs", + "frpss_specs", "bss10_specs", "bss90_specs") + if ("Skill" %in% names(recipe$Analysis$Workflow)) { + if (is.null(recipe$Analysis$Workflow$Skill$metric)) { error(recipe$Run$logger, "Parameter 'metric' must be defined under 'Skill'.") error_status <- T + } else { + requested_metrics <- strsplit(recipe$Analysis$Workflow$Skill$metric, + ", | |,")[[1]] + if (!all(tolower(requested_metrics) %in% AVAILABLE_METRICS)) { + error(recipe$Run$logger, + paste0("Some of the metrics requested under 'Skill' are not ", + "available in SUNSET. Check the documentation to see the ", + "full list of accepted skill metrics.")) + error_status <- T + } + } # Saving checks SAVING_OPTIONS_SKILL <- c("all", "none") if ((is.null(recipe$Analysis$Workflow$Skill$save)) || @@ -546,7 +562,23 @@ check_recipe <- function(recipe) { } } } - + # Scorecards + if ("Scorecards" %in% names(recipe$Analysis$Workflow)) { + if (is.null(recipe$Analysis$Workflow$Scorecards$metric)) { + error(recipe$Run$logger, + "Parameter 'metric' must be defined under 'Scorecards'.") + error_status <- T + } else { + sc_metrics <- strsplit(recipe$Analysis$Workflow$Scorecards$metric, + ", | |,")[[1]] + if (!all(tolower(sc_metrics) %in% tolower(requested_metrics))) { + error(recipe$Run$logger, + paste0("All of the metrics requested under 'Scorecards' must ", + "be requested in the 'Skill' section.")) + error_status <- T + } + } + } # --------------------------------------------------------------------- # RUN CHECKS # --------------------------------------------------------------------- diff --git a/tools/libs.R b/tools/libs.R index ec34ad2e91387998375289cbef2ffc91b1873547..3b855a77f9691a060b125826cf90af9cbbe68411 100644 --- a/tools/libs.R +++ b/tools/libs.R @@ -11,6 +11,7 @@ library(easyNCDF) library(CSTools) library(lubridate) library(PCICt) +library(clock) library(RColorBrewer) library(configr) library(sf) @@ -18,7 +19,11 @@ library(ggplot2) library(rnaturalearth) library(cowplot) library(stringr) -library(pryr) # To check mem usage. +library(pryr) +library(ncdf4) +library(formattable) ## to plot horizontal color bars - used ?? +library(kableExtra) +library(memuse) # To check mem usage. # Functions ## To be removed when new package is done by library(CSOperational) @@ -29,6 +34,7 @@ source("tools/data_summary.R") source("tools/read_atomic_recipe.R") source("tools/write_autosubmit_conf.R") source("tools/get_archive.R") +source("tools/Utils.R") source("tools/restructure_recipe.R") # source("tools/add_dims.R") # Not sure if necessary yet diff --git a/tools/prepare_outputs.R b/tools/prepare_outputs.R index 8f9968f45dd7e0d4b14770539f154a552c9571f6..6f5f7fb8bf208b850181d10a5c04d8ac0444fc3a 100644 --- a/tools/prepare_outputs.R +++ b/tools/prepare_outputs.R @@ -31,7 +31,6 @@ prepare_outputs <- function(recipe_file, recipe$recipe_path <- recipe_file recipe$name <- tools::file_path_sans_ext(basename(recipe_file)) - output_dir = recipe$Run$output_dir # Create output folders if (!uniqueID) { @@ -41,6 +40,7 @@ prepare_outputs <- function(recipe_file, gsub(" ", "", gsub(":", "", gsub("-", "", Sys.time())))) } + recipe$Run$output_dir <- file.path(output_dir, folder_name) print("Saving all outputs to:") print(paste0(output_dir, folder_name)) if (dir.exists(file.path(output_dir, folder_name))) { @@ -79,8 +79,6 @@ prepare_outputs <- function(recipe_file, appenders = list(file_appender(logfile, append = TRUE, layout = default_log_layout()))) } - ## TODO: Move output_dir assignation - recipe$Run$output_dir <- file.path(output_dir, folder_name) recipe$Run$logger <- logger recipe$Run$logfile <- logfile diff --git a/tools/write_autosubmit_conf.R b/tools/write_autosubmit_conf.R index a425566d62278e109a6e9b1d53c78eef9f7e700e..95ca93f0f09f10f8b8b67a5076f24918c3d6db7e 100644 --- a/tools/write_autosubmit_conf.R +++ b/tools/write_autosubmit_conf.R @@ -34,43 +34,44 @@ write_autosubmit_conf <- function(recipe, nchunks) { conf$local$PROJECT_PATH <- recipe$Run$code_dir } else if (conf_type == "jobs") { # Section 3: jobs - ## wallclock, notify_on, platform?, processors - # Different file structure depending on autosubmit version - if (auto_specs$auto_version == "4.0.0") { - jobs <- conf$JOBS - } else { - jobs <- conf - } - jobs$verification$WALLCLOCK <- recipe$Run$auto_conf$wallclock + ## wallclock, notify_on, platform?, processors, + conf$JOBS$verification$WALLCLOCK <- recipe$Run$auto_conf$wallclock if (recipe$Run$auto_conf$notify_completed) { - jobs$verification$NOTIFY_ON <- paste(jobs$verification$NOTIFY_ON, - "COMPLETED") + conf$JOBS$verification$NOTIFY_ON <- paste(conf$JOBS$verification$NOTIFY_ON, + "COMPLETED") } if (recipe$Run$auto_conf$notify_failed) { - jobs$verification$NOTIFY_ON <- paste(jobs$verification$NOTIFY_ON, - "FAILED") + conf$JOBS$verification$NOTIFY_ON <- paste(conf$JOBS$verification$NOTIFY_ON, + "FAILED") } - jobs$verification$PROCESSORS <- recipe$Run$auto_conf$processors_per_job # ncores? - # Return to original list - if (auto_specs$auto_version == "4.0.0") { - conf$JOBS <- jobs + conf$JOBS$verification$PROCESSORS <- recipe$Run$auto_conf$processors_per_job # ncores? + conf$JOBS$verification$CUSTOM_DIRECTIVES <- recipe$Run$auto_conf$custom_directives + # Only include Scorecards job if section exists in the recipe and + # is set to 'execute: True' + if (!("Scorecards" %in% names(recipe$Analysis$Workflow)) || + (!recipe$Analysis$Workflow$Scorecards$execute)) { + conf$JOBS$scorecards <- NULL } else { - conf <- jobs + if (recipe$Run$auto_conf$notify_completed) { + conf$JOBS$scorecards$NOTIFY_ON <- paste(conf$JOBS$scorecards$NOTIFY_ON, + "COMPLETED") + } + if (recipe$Run$auto_conf$notify_failed) { + conf$JOBS$scorecards$NOTIFY_ON <- paste(conf$JOBS$scorecards$NOTIFY_ON, + "FAILED") + } } } else if (conf_type == "platforms") { # Section 4: platform configuration ## nord3v2 configuration... platform name? user, processors_per_node - if (auto_specs$auto_version == "4.0.0") { - conf$Platforms[[auto_specs$platform]]$USER <- - recipe$Run$auto_conf$hpc_user - } else { - conf[[auto_specs$platform]]$USER <- recipe$Run$auto_conf$hpc_user - } + conf$Platforms[[auto_specs$platform]]$USER <- + recipe$Run$auto_conf$hpc_user } else if (conf_type == "proj") { # Section 5: proj ## modules? Info that goes on script, e.g. output directory conf$common$OUTDIR <- recipe$Run$output_dir conf$common$SCRIPT <- recipe$Run$auto_conf$script + conf$common$RECIPE <- paste0(recipe$name, ".yml") } # Write config file inside autosubmit dir ## TODO: Change write.type depending on autosubmit version